tonic

v0.2.2

tonic v0.2.2 Tonic

A DSL for conveniently loading binary data/files.

The DSL is designed to closely represent the structure of the actual binary data layout. So it aims to be easy to read, and easy to change.

The DSL defines functionality to represent types, endianness, groups, chunks, repeated data, branches, optional segments. Where the majority of these functions can be further extended to customize the behaviour and result.

The default behaviour of these operations is to remove the data that was read from the current binary data, and append the value to the current block. Values by default are in the form of a tagged value if a name is supplied { :name, value }, otherwise are simply value if no name is supplied. The default return value behaviour can be overridden by passing in a function.

The most common types are defined in Tonic.Types for convenience. These are common integer and floating point types, and strings. The behaviour of types can further be customized when used otherwise new types can be defined using the type/2 function.

To use the DSL, call use Tonic in your module and include any additional type modules you may require. Then you are free to write the DSL directly inside the module. Certain options may be passed to the library on use, to indicate additional behaviours. The currently supported options are:

optimize: which can be passed true to enable all optimizations, or a keyword list enabling the specific optimizations. Enabling optimizations may make debugging issues trickier, so best practice is to enable after it's been tested. Current specific optimizations include:

:reduce #Enables the code reduction optimization, so the generated code is reduced as much as possible.

Example

defmodule PNG do
    use Tonic, optimize: true

    endian :big
    repeat :magic, 8, :uint8
    repeat :chunks do
        uint32 :length
        string :type, length: 4
        chunk get(:length) do
            on get(:type) do
                "IHDR" ->
                    uint32 :width
                    uint32 :height
                    uint8 :bit_depth
                    uint8 :colour_type
                    uint8 :compression_type
                    uint8 :filter_method
                    uint8 :interlace_method
                "gAMA" ->
                    uint32 :gamma, fn { name, value } -> { name, value / 100000 } end
                "cHRM" ->
                    group :white_point do
                        uint32 :x, fn { name, value } -> { name, value / 100000 } end
                        uint32 :y, fn { name, value } -> { name, value / 100000 } end
                    end
                    group :red do
                        uint32 :x, fn { name, value } -> { name, value / 100000 } end
                        uint32 :y, fn { name, value } -> { name, value / 100000 } end
                    end
                    group :green do
                        uint32 :x, fn { name, value } -> { name, value / 100000 } end
                        uint32 :y, fn { name, value } -> { name, value / 100000 } end
                    end
                    group :blue do
                        uint32 :x, fn { name, value } -> { name, value / 100000 } end
                        uint32 :y, fn { name, value } -> { name, value / 100000 } end
                    end
                "iTXt" ->
                    string :keyword, ?\0
                    string :text
                _ -> repeat :uint8
            end
        end
        uint32 :crc
    end
end

#Example load result:
#{{:magic, [137, 80, 78, 71, 13, 10, 26, 10]},
# {:chunks,
#  [{{:length, 13}, {:type, "IHDR"}, {:width, 48}, {:height, 40},
#    {:bit_depth, 8}, {:colour_type, 6}, {:compression_type, 0},
#    {:filter_method, 0}, {:interlace_method, 0}, {:crc, 3095886193}},
#   {{:length, 4}, {:type, "gAMA"}, {:gamma, 0.45455}, {:crc, 201089285}},
#   {{:length, 32}, {:type, "cHRM"}, {:white_point, {:x, 0.3127}, {:y, 0.329}},
#    {:red, {:x, 0.64}, {:y, 0.33}}, {:green, {:x, 0.3}, {:y, 0.6}},
#    {:blue, {:x, 0.15}, {:y, 0.06}}, {:crc, 2629456188}},
#   {{:length, 345}, {:type, "iTXt"}, {:keyword, "XML:com.adobe.xmp"},
#    {:text,
#     <<0, 0, 0, 0, 60, 120, 58, 120, 109, 112, 109, 101, 116, 97, 32, 120, 109, 108, 110, 115, 58, 120, 61, 34, 97, 100, 111, 98, 101, 58, 110, 115, 58, 109, 101, 116, 97, 47, 34, ...>>},
#    {:crc, 1287792473}},
#   {{:length, 1638}, {:type, "IDAT"},
#    [88, 9, 237, 216, 73, 143, 85, 69, 24, 198, 241, 11, 125, 26, 68, 148, 25,
#     109, 4, 154, 102, 114, 192, 149, 70, 137, 137, 209, 152, 152, 24, 19, 190,
#     131, 75, 22, 234, 55, 224, 59, ...], {:crc, 2269121590}},
#   {{:length, 0}, {:type, "IEND"}, [], {:crc, 2923585666}}]}}

Link to this section Summary

Types

ast()
block(body)
callback()
endianness()
length()
signedness()

Functions

chunk(length, block)

Extract a chunk of data for processing

empty!()

Assert that all the data has been loaded from the current data. If there is still data, the Tonic.NotEmpty exception will be raised

endian(endianness)

Sets the default endianness used by types where endianness is not specified

get(name_or_fun)

Get the loaded value by using either a name to lookup the value, or a function to manually look it up

get(name, fun)

Get the loaded value with name, and pass the value into a function

group(block)

Group the load operations

group(name, block)

Group the load operations, wrapping them with the given name

group(name, fun, block)

Group the load operations, wrapping them with the given name and passing the result to a callback

load(data, module)

Loads the binary data using the spec from a given module

load_file(file, module)

Loads the file data using the spec from a given module

on(condition, list)

Executes the given load operations of a particular clause that matches the condition

optional(type)

Optionally execute the given load operations

repeat(type)

Repeat the given load operations until it reaches the end

repeat(name, type)

Repeat the given load operations until it reaches the end or for length

repeat(name, length, type)

Repeat the given load operations for length

repeat(name, length, fun, block)

Repeats the load operations for length, passing the result to a callback

skip(type)

Skip the given load operations

spec()

Execute the current spec again

spec(module)

Execute the provided spec

type(name, type)

Declare a new type as an alias of another type or of a function

type(name, type, endianness)

Declare a new type as an alias of another type with an overriding (fixed) endianness

type(name, type, size, signedness)

Declare a new type for a binary type of size with signedness (if used)

type(name, type, size, signedness, endianness)

Declare a new type for a binary type of size with signedness (if used) and a overriding (fixed) endianness

Link to this section Types

Link to this type

ast() 
ast() :: Macro.t()

Link to this type

block(body) 
block(body) :: [{:do, body}]

Link to this type

callback() 
callback() :: ({any(), any()} -> any())

Link to this type

endianness() 
endianness() :: :little | :big | :native

Link to this type

length() 
length() :: non_neg_integer() | (list() -> boolean())

Link to this type

signedness() 
signedness() :: :signed | :unsigned

Link to this section Functions

Link to this macro

chunk(length, block) (macro) 
chunk(length(), block(any())) :: ast()

Extract a chunk of data for processing.

Executes the load operations only on the given chunk.

chunk(length, block(any)) :: ast
Uses the block as the load operation on the chunk of length.

Example 

chunk 4 do
    uint8 :a
    uint8 :b
end

chunk 4 do
    repeat :uint8
end
Link to this macro

empty!() (macro)

Assert that all the data has been loaded from the current data. If there is still data, the Tonic.NotEmpty exception will be raised.

Example 

int8 :a
empty!() #check that there is no data left
Link to this macro

endian(endianness) (macro) 
endian(endianness()) :: ast()

Sets the default endianness used by types where endianness is not specified.

Examples 

endian :little
uint32 :value #little endian

endian :big
uint32 :value #big endian

endian :little
uint32 :value, :big #big endian
Link to this macro

get(name_or_fun) (macro) 
get(atom()) :: ast()
get((list() -> any())) :: ast()

Get the loaded value by using either a name to lookup the value, or a function to manually look it up.

get(atom) :: any
Using a name for the lookup will cause it to search for that matched name in the current loaded data scope and containing scopes (but not separate branched scopes). If the name is not found, an exception will be raised Tonic.MarkNotFound.

get(fun) :: any
Using a function for the lookup will cause it to pass the current state to the function. where the function can return the value you want to get.

Examples 

uint8 :length
repeat get(:length), :uint8

uint8 :length
repeat get(fn [[{ :length, length }]] -> length end), :uint8
Link to this macro

get(name, fun) (macro) 
get(atom(), (list() -> any())) :: ast()

Get the loaded value with name, and pass the value into a function.

Examples 

uint8 :length
repeat get(:length, fn length -> length - 1 end)
Link to this macro

group(block) (macro) 
group(block(any())) :: ast()

Group the load operations.

Examples 

group do
    uint8 :a
    uint8 :b
end
Link to this macro

group(name, block) (macro) 
group(atom(), block(any())) :: ast()

Group the load operations, wrapping them with the given name.

Examples 

group :values do
    uint8 :a
    uint8 :b
end
Link to this macro

group(name, fun, block) (macro) 
group(atom(), callback(), block(any())) :: ast()

Group the load operations, wrapping them with the given name and passing the result to a callback.

Examples 

group :values, fn { _, value } -> value end do
    uint8 :a
    uint8 :b
end
Link to this function

load(data, module) 
load(bitstring(), module()) :: {any(), bitstring()}

Loads the binary data using the spec from a given module.

The return value consists of the loaded values and the remaining data that wasn't read.

Link to this function

load_file(file, module) 
load_file(Path.t(), module()) :: {any(), bitstring()}

Loads the file data using the spec from a given module.

The return value consists of the loaded values and the remaining data that wasn't read.

Link to this macro

on(condition, list) (macro) 
on(term(), [{:do, [{:->, any(), any()}]}]) :: ast()

Executes the given load operations of a particular clause that matches the condition.

Examples 

uint8 :type
on get(:type) do
    1 -> uint32 :value
    2 -> float32 :value
end
Link to this macro

optional(type) (macro) 
optional(atom()) :: ast()
optional(block(any())) :: ast()

Optionally execute the given load operations.

Usually if the current data does not match what is trying to be loaded, a match error will be raised and the data will not be loaded successfully. Using optional is a way to avoid that. If there is a match error the load operations it attempted to execute will be skipped, and it will continue on with the rest of the data spec. If there isn't a match error then the load operations that were attempted will be combined with the current loaded data.

optional(atom) :: ast
Optionally load the given type.

optional(block(any)) :: ast
Optionally load the given block.

Example 

optional :uint8

optional do
    uint8 :a
    uint8 :b
end
Link to this macro

repeat(type) (macro) 
repeat(atom()) :: ast()
repeat(block(any())) :: ast()

Repeat the given load operations until it reaches the end.

repeat(atom) :: ast
Uses the type as the load operation to be repeated.

repeat(block(any)) :: ast
Uses the block as the load operation to be repeated.

Examples 

repeat :uint8

repeat do
    uint8 :a
    uint8 :b
end
Link to this macro

repeat(name, type) (macro) 
repeat(atom(), atom()) :: ast()
repeat(atom(), block(any())) :: ast()
repeat(length(), atom()) :: ast()
repeat(length(), block(any())) :: ast()

Repeat the given load operations until it reaches the end or for length.

repeat(atom, atom) :: ast
Uses the type as the load operation to be repeated. And wraps the output with the given name.

repeat(atom, block(any)) :: ast
Uses the block as the load operation to be repeated. And wraps the output with the given name.

repeat(length, atom) :: ast
Uses the type as the load operation to be repeated. And repeats for length.

repeat(length, block(any)) :: ast
Uses the block as the load operation to be repeated. And repeats for length.

Examples 

repeat :values, :uint8

repeat :values do
    uint8 :a
    uint8 :b
end

repeat 4, :uint8

repeat fn _ -> false end, :uint8

repeat 2 do
    uint8 :a
    uint8 :b
end

repeat fn _ -> false end do
    uint8 :a
    uint8 :b
end
Link to this macro

repeat(name, length, type) (macro) 
repeat(atom(), length(), atom()) :: ast()
repeat(atom(), length(), block(any())) :: ast()

Repeat the given load operations for length.

repeat(atom, length, atom) :: ast
Uses the type as the load operation to be repeated. And wraps the output with the given name. Repeats for length.

repeat(atom, length, block(any)) :: ast
Uses the block as the load operation to be repeated. And wraps the output with the given name. Repeats for length.

Examples 

repeat :values, 4, :uint8

repeat :values, fn _ -> false end, :uint8

repeat :values, 4 do
    uint8 :a
    uint8 :b
end

repeat :values, fn _ -> false end do
    uint8 :a
    uint8 :b
end
Link to this macro

repeat(name, length, fun, block) (macro) 
repeat(atom(), length(), callback(), block(any())) :: ast()

Repeats the load operations for length, passing the result to a callback.

Examples 

repeat :values, 4, fn result -> result end do
    uint8 :a
    uint8 :b
end

repeat :values, 4, fn { name, value } -> value end do
    uint8 :a
    uint8 :b
end

repeat :values, fn _ -> false end, fn result -> result end do
    uint8 :a
    uint8 :b
end
Link to this macro

skip(type) (macro) 
skip(atom()) :: ast()
skip(block(any())) :: ast()

Skip the given load operations.

Executes the load operations but doesn't return the loaded data.

skip(atom) :: ast
Skip the given type.

skip(block(any)) :: ast
Skip the given block.

Example 

skip :uint8

skip do
    uint8 :a
    uint8 :b
end
Link to this macro

spec() (macro) 
spec() :: ast()

Execute the current spec again.

Examples 

defmodule Recursive do
    use Tonic

    uint8
    optional do: spec
end

# Tonic.load <<1, 2, 3>>, Recursive
# => {{1, {2, {3}}}, ""}
Link to this macro

spec(module) (macro) 
spec(module()) :: ast()

Execute the provided spec.

Examples 

defmodule Foo do
    use Tonic

    uint8
    spec Bar
end

defmodule Bar do
    use Tonic

    uint8 :a
    uint8 :b
end

# Tonic.load <<1, 2, 3>>, Foo
# => {{1, {{:a, 2}, {:b, 3}}}, ""}
Link to this macro

type(name, type) (macro) 
type(atom(), atom()) :: ast()
type(atom(), (bitstring(), atom(), endianness() -> {any(), bitstring()})) ::
  ast()

Declare a new type as an alias of another type or of a function.

type(atom, atom) :: ast
Create the new type as an alias of another type.

type(atom, (bitstring, atom, endianness -> { any, bitstring })) :: ast
Implement the type as a function.

Examples 

type :myint8, :int8

type :myint8, fn data, name, _ ->
    <<value :: integer-size(8)-signed, data :: bitstring>> data
    { { name, value }, data }
end

type :myint16, fn
    data, name, :little ->
        <<value :: integer-size(16)-signed-little, data :: bitstring>> = data
        { { name, value }, data }
    data, name, :big ->
        <<value :: integer-size(16)-signed-big, data :: bitstring>> = data
        { { name, value }, data }
    data, name, :native ->
        <<value :: integer-size(16)-signed-native, data :: bitstring>> = data
        { { name, value }, data }
end
Link to this macro

type(name, type, endianness) (macro) 
type(atom(), atom(), endianness()) :: ast()

Declare a new type as an alias of another type with an overriding (fixed) endianness.

Examples 

type :mylittleint16, :int16, :little
Link to this macro

type(name, type, size, signedness) (macro) 
type(atom(), atom(), non_neg_integer(), signedness()) :: ast()

Declare a new type for a binary type of size with signedness (if used).

Examples 

type :myint16, :integer, 16, :signed
Link to this macro

type(name, type, size, signedness, endianness) (macro) 
type(atom(), atom(), non_neg_integer(), signedness(), endianness()) :: ast()

Declare a new type for a binary type of size with signedness (if used) and a overriding (fixed) endianness.

Examples 

type :mylittleint16, :integer, 16, :signed, :little