From 0d2d5df473c2035009530f29b49a48e57378e364 Mon Sep 17 00:00:00 2001 From: Alexandre Hamez Date: Mon, 3 Feb 2025 13:40:05 +0100 Subject: [PATCH] WIP --- .../protox/benchmark/generate/payloads.ex | 4 +- benchmark/mix/tasks/protox/benchmark/run.ex | 5 +- conformance/protox/conformance/escript.ex | 4 +- lib/protox.ex | 6 +- lib/protox/define_decoder.ex | 5 +- lib/protox/define_encoder.ex | 286 ++++++++++++------ lib/protox/define_message.ex | 4 +- lib/protox/encode.ex | 83 ++--- lib/protox/parse.ex | 3 +- lib/protox/varint.ex | 29 +- test/optional_proto3_test.exs | 28 +- test/protox/decode_test.exs | 2 + test/protox/encode_test.exs | 150 ++++----- test/protox/varint_test.exs | 47 +-- test/protox_properties_test.exs | 14 +- test/protox_test.exs | 29 +- test/support/random_init.ex | 2 +- 17 files changed, 409 insertions(+), 292 deletions(-) diff --git a/benchmark/mix/tasks/protox/benchmark/generate/payloads.ex b/benchmark/mix/tasks/protox/benchmark/generate/payloads.ex index de1b6f08..e3d423f4 100644 --- a/benchmark/mix/tasks/protox/benchmark/generate/payloads.ex +++ b/benchmark/mix/tasks/protox/benchmark/generate/payloads.ex @@ -66,7 +66,9 @@ defmodule Mix.Tasks.Protox.Benchmark.Generate.Payloads do end Stream.repeatedly(fn -> :proper_gen.pick(gen, 5) end) - |> Stream.map(fn {:ok, msg} -> {msg, msg |> Protox.encode!() |> IO.iodata_to_binary()} end) + |> Stream.map(fn {:ok, msg} -> + {msg, msg |> Protox.encode!() |> elem(0) |> IO.iodata_to_binary()} + end) |> Stream.reject(fn {_msg, bytes} -> byte_size(bytes) == 0 end) |> Stream.reject(fn {_msg, bytes} -> byte_size(bytes) > 16_384 * 16 end) |> Stream.map(fn {msg, bytes} -> {msg, byte_size(bytes), bytes} end) diff --git a/benchmark/mix/tasks/protox/benchmark/run.ex b/benchmark/mix/tasks/protox/benchmark/run.ex index c586fd4b..12df077e 100644 --- a/benchmark/mix/tasks/protox/benchmark/run.ex +++ b/benchmark/mix/tasks/protox/benchmark/run.ex @@ -29,14 +29,14 @@ defmodule Mix.Tasks.Protox.Benchmark.Run do :encode -> %{ encode: fn input -> - Enum.map(input, fn {msg, _size, _bytes} -> msg.__struct__.encode!(msg) end) + Enum.each(input, fn {msg, _size, _bytes} -> msg.__struct__.encode!(msg) end) end } :decode -> %{ decode: fn input -> - Enum.map(input, fn {msg, _size, bytes} -> msg.__struct__.decode!(bytes) end) + Enum.each(input, fn {msg, _size, bytes} -> msg.__struct__.decode!(bytes) end) end } end @@ -82,5 +82,6 @@ defmodule Mix.Tasks.Protox.Benchmark.Run do path |> File.read!() |> :erlang.binary_to_term() + |> Map.drop([ProtobufTestMessages.Proto3.TestAllTypesProto3]) end end diff --git a/conformance/protox/conformance/escript.ex b/conformance/protox/conformance/escript.ex index 14979105..1d7a2559 100644 --- a/conformance/protox/conformance/escript.ex +++ b/conformance/protox/conformance/escript.ex @@ -71,7 +71,7 @@ defmodule Protox.Conformance.Escript do IO.binwrite(log_file, "Message: #{inspect(msg, limit: :infinity)}\n") try do - encoded_payload = msg |> Protox.encode!() |> :binary.list_to_bin() + encoded_payload = msg |> Protox.encode!() |> elem(0) |> :binary.list_to_bin() IO.binwrite( log_file, @@ -155,7 +155,7 @@ defmodule Protox.Conformance.Escript do end defp make_message_bytes(%Conformance.ConformanceResponse{} = msg) do - data = msg |> Protox.encode!() |> :binary.list_to_bin() + data = msg |> Protox.encode!() |> elem(0) |> :binary.list_to_bin() <> end diff --git a/lib/protox.ex b/lib/protox.ex index e1269aa1..ad40f536 100644 --- a/lib/protox.ex +++ b/lib/protox.ex @@ -127,7 +127,7 @@ defmodule Protox do Throwing version of `encode/1`. """ @doc since: "1.6.0" - @spec encode!(struct()) :: iodata() | no_return() + @spec encode!(struct()) :: {iodata(), non_neg_integer()} | no_return() def encode!(msg) do msg.__struct__.encode!(msg) end @@ -137,7 +137,7 @@ defmodule Protox do ## Examples iex> msg = %ProtoxExample{a: 3, b: %{1 => "some string"}} - iex> {:ok, iodata} = Protox.encode(msg) + iex> {:ok, iodata, _iodata_size} = Protox.encode(msg) iex> :binary.list_to_bin(iodata) <<8, 3, 18, 15, 8, 1, 18, 11, 115, 111, 109, 101, 32, 115, 116, 114, 105, 110, 103>> @@ -148,7 +148,7 @@ defmodule Protox do """ @doc since: "1.6.0" - @spec encode(struct()) :: {:ok, iodata()} | {:error, any()} + @spec encode(struct()) :: {:ok, iodata(), non_neg_integer()} | {:error, any()} def encode(msg) do msg.__struct__.encode(msg) end diff --git a/lib/protox/define_decoder.ex b/lib/protox/define_decoder.ex index 904e77ea..0ecf35ac 100644 --- a/lib/protox/define_decoder.ex +++ b/lib/protox/define_decoder.ex @@ -661,8 +661,7 @@ defmodule Protox.DefineDecoder do end end - # Compute at compile time the varint representation of a field - # tag and wire type. + # Compute at compile time the varint representation of a field tag and wire type. defp make_key_bytes(%Field{} = field) do # We need to convert the type to something recognized # by Protox.Encode.make_key_bytes/2. @@ -673,6 +672,6 @@ defmodule Protox.DefineDecoder do _ -> field.type end - Protox.Encode.make_key_bytes(field.tag, ty) |> IO.iodata_to_binary() + Protox.Encode.make_key_bytes(field.tag, ty) |> elem(0) |> IO.iodata_to_binary() end end diff --git a/lib/protox/define_encoder.ex b/lib/protox/define_encoder.ex index 127462df..1b1b8d1c 100644 --- a/lib/protox/define_encoder.ex +++ b/lib/protox/define_encoder.ex @@ -5,6 +5,12 @@ defmodule Protox.DefineEncoder do alias Protox.{Field, OneOf, Scalar} def define(fields, required_fields, syntax, opts \\ []) do + vars = %{ + acc: Macro.var(:acc, __MODULE__), + acc_size: Macro.var(:acc_size, __MODULE__), + msg: Macro.var(:msg, __MODULE__) + } + %{oneofs: oneofs, proto3_optionals: proto3_optionals, others: fields_without_oneofs} = Protox.Defs.split_oneofs(fields) @@ -12,11 +18,11 @@ defmodule Protox.DefineEncoder do make_top_level_encode_fun(oneofs, proto3_optionals ++ fields_without_oneofs) encode_oneof_funs = make_encode_oneof_funs(oneofs) - encode_field_funs = make_encode_field_funs(fields, required_fields, syntax) - - encode_unknown_fields_fun = make_encode_unknown_fields_fun(opts) + encode_field_funs = make_encode_field_funs(fields, required_fields, syntax, vars) + encode_unknown_fields_fun = make_encode_unknown_fields_fun(vars, opts) quote do + _generator = unquote(make_generator(__ENV__)) unquote(top_level_encode_fun) unquote_splicing(encode_oneof_funs) unquote_splicing(encode_field_funs) @@ -25,7 +31,7 @@ defmodule Protox.DefineEncoder do end defp make_top_level_encode_fun(oneofs, fields) do - quote(do: []) + quote(do: {_acc = [], _acc_size = 0}) |> make_encode_oneof_fun(oneofs) |> make_encode_fun_field(fields) |> make_encode_fun_body() @@ -33,17 +39,19 @@ defmodule Protox.DefineEncoder do defp make_encode_fun_body(ast) do quote do - @spec encode(struct()) :: {:ok, iodata()} | {:error, any()} + @spec encode(struct()) :: {:ok, iodata(), non_neg_integer()} | {:error, any()} def encode(msg) do + _generator = unquote(make_generator(__ENV__)) + try do - {:ok, encode!(msg)} + msg |> encode!() |> Tuple.insert_at(0, :ok) rescue e in [Protox.EncodingError, Protox.RequiredFieldsError] -> {:error, e} end end - @spec encode!(struct()) :: iodata() | no_return() + @spec encode!(struct()) :: {iodata(), non_neg_integer()} | no_return() def encode!(msg), do: unquote(ast) end end @@ -97,19 +105,14 @@ defmodule Protox.DefineEncoder do end end - defp make_encode_field_funs(fields, required_fields, syntax) do - vars = %{ - acc: Macro.var(:acc, __MODULE__), - msg: Macro.var(:msg, __MODULE__) - } - + defp make_encode_field_funs(fields, required_fields, syntax, vars) do for %Field{name: name} = field <- fields do required = name in required_fields fun_name = make_encode_field_fun_name(name) fun_ast = make_encode_field_body(field, required, syntax, vars) quote do - defp unquote(fun_name)(unquote(vars.acc), unquote(vars.msg)) do + defp unquote(fun_name)({unquote(vars.acc), unquote(vars.acc_size)}, unquote(vars.msg)) do try do unquote(fun_ast) rescue @@ -123,24 +126,34 @@ defmodule Protox.DefineEncoder do end defp make_encode_field_body(%Field{kind: %Scalar{}} = field, required, syntax, vars) do - key = Protox.Encode.make_key_bytes(field.tag, field.type) + {key, key_size} = Protox.Encode.make_key_bytes(field.tag, field.type) var = quote do: unquote(vars.msg).unquote(field.name) encode_value_ast = get_encode_value_body(field.type, var) + encode_value_clause = + quote do + {value_bytes, value_bytes_size} = unquote(encode_value_ast) + + { + [unquote(key), value_bytes | unquote(vars.acc)], + unquote(vars.acc_size) + unquote(key_size) + value_bytes_size + } + end + case syntax do :proto2 -> if required do quote do case unquote(vars.msg).unquote(field.name) do nil -> raise Protox.RequiredFieldsError.new([unquote(field.name)]) - _ -> [unquote(vars.acc), unquote(key), unquote(encode_value_ast)] + _ -> unquote(encode_value_clause) end end else quote do case unquote(var) do - nil -> unquote(vars.acc) - _ -> [unquote(vars.acc), unquote(key), unquote(encode_value_ast)] + nil -> {unquote(vars.acc), unquote(vars.acc_size)} + _ -> unquote(encode_value_clause) end end end @@ -149,9 +162,9 @@ defmodule Protox.DefineEncoder do quote do # Use == rather than pattern match for float comparison if unquote(var) == unquote(field.kind.default_value) do - unquote(vars.acc) + {unquote(vars.acc), unquote(vars.acc_size)} else - [unquote(vars.acc), unquote(key), unquote(encode_value_ast)] + unquote(encode_value_clause) end end end @@ -164,7 +177,7 @@ defmodule Protox.DefineEncoder do _syntax, vars ) do - key = Protox.Encode.make_key_bytes(field.tag, field.type) + {key, key_size} = Protox.Encode.make_key_bytes(field.tag, field.type) var = Macro.var(:child_field_value, __MODULE__) encode_value_ast = get_encode_value_body(field.type, var) @@ -173,10 +186,15 @@ defmodule Protox.DefineEncoder do quote do case unquote(vars.msg).unquote(field.name) do nil -> - [unquote(vars.acc)] + {unquote(vars.acc), unquote(vars.acc_size)} unquote(var) -> - [unquote(vars.acc), unquote(key), unquote(encode_value_ast)] + {value_bytes, value_bytes_size} = unquote(encode_value_ast) + + { + [unquote(key), value_bytes | unquote(vars.acc)], + unquote(vars.acc_size) + unquote(key_size) + value_bytes_size + } end end @@ -185,30 +203,48 @@ defmodule Protox.DefineEncoder do # this is why we don't check if the child is set. quote do {_, unquote(var)} = unquote(vars.msg).unquote(field.kind.parent) - [unquote(vars.acc), unquote(key), unquote(encode_value_ast)] + {value_bytes, value_bytes_size} = unquote(encode_value_ast) + + { + [unquote(key), value_bytes | unquote(vars.acc)], + unquote(vars.acc_size) + unquote(key_size) + value_bytes_size + } end end end + # TODO: repeated packed are always made of scalar, exploit this? defp make_encode_field_body(%Field{kind: :packed} = field, _required, _syntax, vars) do - key = Protox.Encode.make_key_bytes(field.tag, :packed) - encode_packed_ast = make_encode_packed_body(field.type) + {key_bytes, key_size} = Protox.Encode.make_key_bytes(field.tag, :packed) + encode_packed_ast = make_encode_packed_body(field.type, vars) quote do + _generator = unquote(make_generator(__ENV__)) + case unquote(vars.msg).unquote(field.name) do - [] -> unquote(vars.acc) - values -> [unquote(vars.acc), unquote(key), unquote(encode_packed_ast)] + [] -> + {unquote(vars.acc), unquote(vars.acc_size)} + + values -> + {packed_bytes, packed_size} = unquote(encode_packed_ast) + + { + [unquote(key_bytes), packed_bytes | unquote(vars.acc)], + unquote(vars.acc_size) + unquote(key_size) + packed_size + } end end end defp make_encode_field_body(%Field{kind: :unpacked} = field, _required, _syntax, vars) do - encode_repeated_ast = make_encode_repeated_body(field.tag, field.type) + encode_repeated_ast = make_encode_repeated_body(field.tag, field.type, vars) quote do + _generator = unquote(make_generator(__ENV__)) + case unquote(vars.msg).unquote(field.name) do - [] -> unquote(vars.acc) - values -> [unquote(vars.acc), unquote(encode_repeated_ast)] + [] -> {unquote(vars.acc), unquote(vars.acc_size)} + values -> unquote(encode_repeated_ast) end end end @@ -217,7 +253,7 @@ defmodule Protox.DefineEncoder do # Each key/value entry of a map has the same layout as a message. # https://developers.google.com/protocol-buffers/docs/proto3#backwards-compatibility - key = Protox.Encode.make_key_bytes(field.tag, :map_entry) + {field_key, field_key_size} = Protox.Encode.make_key_bytes(field.tag, :map_entry) {map_key_type, map_value_type} = field.type @@ -227,89 +263,158 @@ defmodule Protox.DefineEncoder do encode_map_key_ast = get_encode_value_body(map_key_type, k_var) encode_map_value_ast = get_encode_value_body(map_value_type, v_var) - map_key_key_bytes = Protox.Encode.make_key_bytes(1, map_key_type) - map_value_key_bytes = Protox.Encode.make_key_bytes(2, map_value_type) - map_keys_len = byte_size(map_value_key_bytes) + byte_size(map_key_key_bytes) + {k_key_bytes, k_key_size} = Protox.Encode.make_key_bytes(1, map_key_type) + {v_key_bytes, v_key_size} = Protox.Encode.make_key_bytes(2, map_value_type) + keys_len = k_key_size + v_key_size quote do + _generator = unquote(make_generator(__ENV__)) + map = Map.fetch!(unquote(vars.msg), unquote(field.name)) - Enum.reduce( - map, - unquote(vars.acc), - fn {unquote(k_var), unquote(v_var)}, unquote(vars.acc) -> - map_key_value_bytes = :binary.list_to_bin([unquote(encode_map_key_ast)]) - map_key_value_len = byte_size(map_key_value_bytes) - - map_value_value_bytes = :binary.list_to_bin([unquote(encode_map_value_ast)]) - map_value_value_len = byte_size(map_value_value_bytes) - - len = - Protox.Varint.encode(unquote(map_keys_len) + map_key_value_len + map_value_value_len) - - [ - unquote(vars.acc), - unquote(key), - len, - unquote(map_key_key_bytes), - map_key_value_bytes, - unquote(map_value_key_bytes), - map_value_value_bytes - ] - end - ) + if map_size(map) == 0 do + {unquote(vars.acc), unquote(vars.acc_size)} + else + Enum.reduce( + map, + {unquote(vars.acc), unquote(vars.acc_size)}, + fn {unquote(k_var), unquote(v_var)}, {unquote(vars.acc), unquote(vars.acc_size)} -> + {k_value_bytes, k_value_len} = unquote(encode_map_key_ast) + {v_value_bytes, v_value_len} = unquote(encode_map_value_ast) + + len = unquote(keys_len) + k_value_len + v_value_len + {len_varint, len_varint_size} = Protox.Varint.encode(len) + + unquote(vars.acc) = [ + <>, + k_value_bytes, + unquote(v_key_bytes), + v_value_bytes + | unquote(vars.acc) + ] + + { + unquote(vars.acc), + unquote(vars.acc_size) + unquote(field_key_size + keys_len) + k_value_len + + v_value_len + len_varint_size + } + end + ) + end end end - defp make_encode_unknown_fields_fun(opts) do + defp make_encode_unknown_fields_fun(vars, opts) do unknown_fields_name = Keyword.fetch!(opts, :unknown_fields_name) quote do - defp encode_unknown_fields(acc, msg) do - Enum.reduce(msg.unquote(unknown_fields_name), acc, fn {tag, wire_type, bytes}, acc -> - case wire_type do - 0 -> - [acc, Protox.Encode.make_key_bytes(tag, :int32), bytes] - - 1 -> - [acc, Protox.Encode.make_key_bytes(tag, :double), bytes] - - 2 -> - len_bytes = bytes |> byte_size() |> Protox.Varint.encode() - [acc, Protox.Encode.make_key_bytes(tag, :packed), len_bytes, bytes] - - 5 -> - [acc, Protox.Encode.make_key_bytes(tag, :float), bytes] + defp encode_unknown_fields({unquote(vars.acc), unquote(vars.acc_size)}, msg) do + _generator = unquote(make_generator(__ENV__)) + + Enum.reduce( + msg.unquote(unknown_fields_name), + {unquote(vars.acc), unquote(vars.acc_size)}, + fn {tag, wire_type, bytes}, {unquote(vars.acc), unquote(vars.acc_size)} -> + case wire_type do + 0 -> + {key_bytes, key_size} = Protox.Encode.make_key_bytes(tag, :int32) + + { + [unquote(vars.acc), <>], + unquote(vars.acc_size) + key_size + byte_size(bytes) + } + + 1 -> + {key_bytes, key_size} = Protox.Encode.make_key_bytes(tag, :double) + + { + [unquote(vars.acc), <>], + unquote(vars.acc_size) + key_size + byte_size(bytes) + } + + 2 -> + {len_bytes, len_size} = bytes |> byte_size() |> Protox.Varint.encode() + {key_bytes, key_size} = Protox.Encode.make_key_bytes(tag, :packed) + + { + [unquote(vars.acc), <>], + unquote(vars.acc_size) + key_size + len_size + byte_size(bytes) + } + + 5 -> + {key_bytes, key_size} = Protox.Encode.make_key_bytes(tag, :float) + + { + [unquote(vars.acc), <>], + unquote(vars.acc_size) + key_size + byte_size(bytes) + } + end end - end) + ) end end end - defp make_encode_packed_body(type) do + # TODO, for fixed*, we know the size of the value, maybe we can exploit this? + defp make_encode_packed_body(type, vars) do value_var = Macro.var(:value, __MODULE__) encode_value_ast = get_encode_value_body(type, value_var) quote do - {bytes, len} = - Enum.reduce(values, {[], 0}, fn unquote(value_var), {acc, len} -> - value_bytes = :binary.list_to_bin([unquote(encode_value_ast)]) - {[acc, value_bytes], len + byte_size(value_bytes)} - end) + _generator = unquote(make_generator(__ENV__)) + + {value_bytes, value_size} = + Enum.reduce( + values, + {_local_acc = [], _local_acc_size = 0}, + fn unquote(value_var), {local_acc, local_acc_size} -> + {value_bytes, value_bytes_size} = unquote(encode_value_ast) + + { + [value_bytes | local_acc], + local_acc_size + value_bytes_size + } + end + ) - [Protox.Varint.encode(len), bytes] + value_bytes = Enum.reverse(value_bytes) |> :binary.list_to_bin() + {value_size_bytes, value_size_size} = Protox.Varint.encode(value_size) + + { + [<> | unquote(vars.acc)], + unquote(vars.acc_size) + value_size + value_size_size + } end end - defp make_encode_repeated_body(tag, type) do - key = Protox.Encode.make_key_bytes(tag, type) + defp make_encode_repeated_body(tag, type, vars) do + {key_bytes, key_bytes_sz} = Protox.Encode.make_key_bytes(tag, type) value_var = Macro.var(:value, __MODULE__) encode_value_ast = get_encode_value_body(type, value_var) quote do - Enum.reduce(values, [], fn unquote(value_var), acc -> - [acc, unquote(key), unquote(encode_value_ast)] - end) + _generator = unquote(make_generator(__ENV__)) + + {value_bytes, value_size} = + Enum.reduce( + values, + {_local_acc = [], _local_acc_size = 0}, + fn unquote(value_var), {local_acc, local_acc_size} -> + {value_bytes, value_bytes_size} = unquote(encode_value_ast) + + { + [value_bytes, unquote(key_bytes) | local_acc], + local_acc_size + unquote(key_bytes_sz) + value_bytes_size + } + end + ) + + value_bytes = Enum.reverse(value_bytes) + + { + [value_bytes | unquote(vars.acc)], + unquote(vars.acc_size) + value_size + } end end @@ -388,4 +493,9 @@ defmodule Protox.DefineEncoder do defp make_encode_field_fun_name(field) when is_atom(field) do String.to_atom("encode_#{field}") end + + defp make_generator(%Macro.Env{} = env) do + {fun_name, _fun_arity} = env.function + "#{fun_name}:#{env.line}" + end end diff --git a/lib/protox/define_message.ex b/lib/protox/define_message.ex index 92df2469..bb215434 100644 --- a/lib/protox/define_message.ex +++ b/lib/protox/define_message.ex @@ -5,7 +5,9 @@ defmodule Protox.DefineMessage do def define(messages, opts \\ []) do for {_msg_name, msg = %Protox.Message{}} <- messages do - sorted_fields = msg.fields |> Map.values() |> Enum.sort(&(&1.tag < &2.tag)) + # Revert the order of the fields so we iterator from last field to first. + # This enables us to construct the output iodata using [ field | acc ] + sorted_fields = msg.fields |> Map.values() |> Enum.sort(&(&1.tag >= &2.tag)) required_fields = get_required_fields(sorted_fields) unknown_fields_name = make_unknown_fields_name(:__uf__, sorted_fields) diff --git a/lib/protox/encode.ex b/lib/protox/encode.ex index 4a81ee7e..f92493ca 100644 --- a/lib/protox/encode.ex +++ b/lib/protox/encode.ex @@ -17,7 +17,7 @@ defmodule Protox.Encode do } @doc false - @spec make_key_bytes(Protox.Types.tag(), Protox.Types.type()) :: iodata() + @spec make_key_bytes(Protox.Types.tag(), Protox.Types.type()) :: {binary(), non_neg_integer()} def make_key_bytes(tag, ty) do Varint.encode(make_key(tag, ty)) end @@ -34,20 +34,20 @@ defmodule Protox.Encode do def make_key(tag, ty) when is_primitive_fixed32(ty), do: tag <<< 3 ||| @wire_32bits @doc false - @spec encode_varint_signed(integer()) :: iodata() + @spec encode_varint_signed(integer()) :: {binary(), non_neg_integer()} def encode_varint_signed(value) do value |> Zigzag.encode() |> Varint.encode() end @doc false - @spec encode_varint_64(integer()) :: iodata() + @spec encode_varint_64(integer()) :: {binary(), non_neg_integer()} def encode_varint_64(value) do <> = <> Varint.encode(res) end @doc false - @spec encode_varint_32(integer()) :: iodata() + @spec encode_varint_32(integer()) :: {binary(), non_neg_integer()} def encode_varint_32(value) when value < 0 do encode_varint_64(value) end @@ -55,78 +55,80 @@ defmodule Protox.Encode do @doc false def encode_varint_32(value) do <> = <> + Varint.encode(res) end @doc false - @spec encode_bool(boolean()) :: binary - def encode_bool(false), do: <<0>> - def encode_bool(true), do: <<1>> + @spec encode_bool(boolean()) :: {binary(), non_neg_integer()} + def encode_bool(false), do: {<<0>>, 1} + def encode_bool(true), do: {<<1>>, 1} @doc false - @spec encode_int32(integer()) :: iodata() + @spec encode_int32(integer()) :: {binary(), non_neg_integer()} def encode_int32(value), do: encode_varint_32(value) @doc false - @spec encode_int64(integer()) :: iodata() + @spec encode_int64(integer()) :: {binary(), non_neg_integer()} def encode_int64(value), do: encode_varint_64(value) @doc false - @spec encode_sint32(integer()) :: iodata() + @spec encode_sint32(integer()) :: {binary(), non_neg_integer()} def encode_sint32(value), do: encode_varint_signed(value) @doc false - @spec encode_sint64(integer()) :: iodata() + @spec encode_sint64(integer()) :: {binary(), non_neg_integer()} def encode_sint64(value), do: encode_varint_signed(value) @doc false - @spec encode_uint32(non_neg_integer()) :: iodata() + @spec encode_uint32(non_neg_integer()) :: {binary(), non_neg_integer()} def encode_uint32(value), do: encode_varint_32(value) @doc false - @spec encode_uint64(non_neg_integer()) :: iodata() + @spec encode_uint64(non_neg_integer()) :: {binary(), non_neg_integer()} def encode_uint64(value), do: encode_varint_64(value) @doc false - @spec encode_fixed64(integer()) :: binary - def encode_fixed64(value), do: <> + @spec encode_fixed64(integer()) :: {binary(), non_neg_integer()} + def encode_fixed64(value), do: {<>, 8} @doc false - @spec encode_sfixed64(integer()) :: binary - def encode_sfixed64(value), do: <> + @spec encode_sfixed64(integer()) :: {binary(), non_neg_integer()} + def encode_sfixed64(value), do: {<>, 8} @doc false - @spec encode_fixed32(integer()) :: binary - def encode_fixed32(value), do: <> + @spec encode_fixed32(integer()) :: {binary(), non_neg_integer()} + def encode_fixed32(value), do: {<>, 4} @doc false - @spec encode_sfixed32(integer()) :: binary - def encode_sfixed32(value), do: <> + @spec encode_sfixed32(integer()) :: {binary(), non_neg_integer()} + def encode_sfixed32(value), do: {<>, 4} @doc false - @spec encode_double(float() | atom()) :: binary - def encode_double(:infinity), do: @positive_infinity_64 - def encode_double(:"-infinity"), do: @negative_infinity_64 - def encode_double(:nan), do: @nan_64 - def encode_double(value), do: <> + @spec encode_double(float() | atom()) :: {binary(), non_neg_integer()} + def encode_double(:infinity), do: {@positive_infinity_64, 8} + def encode_double(:"-infinity"), do: {@negative_infinity_64, 8} + def encode_double(:nan), do: {@nan_64, 8} + def encode_double(value), do: {<>, 8} @doc false - @spec encode_float(float() | atom()) :: binary - def encode_float(:infinity), do: @positive_infinity_32 - def encode_float(:"-infinity"), do: @negative_infinity_32 - def encode_float(:nan), do: @nan_32 - def encode_float(value), do: <> + @spec encode_float(float() | atom()) :: {binary(), non_neg_integer()} + def encode_float(:infinity), do: {@positive_infinity_32, 4} + def encode_float(:"-infinity"), do: {@negative_infinity_32, 4} + def encode_float(:nan), do: {@nan_32, 4} + def encode_float(value), do: {<>, 4} @doc false - @spec encode_enum(integer()) :: iodata() + @spec encode_enum(integer()) :: {binary(), non_neg_integer()} def encode_enum(value), do: encode_varint_32(value) @doc false - @spec encode_string(String.t()) :: iodata() + @spec encode_string(String.t()) :: {iodata(), non_neg_integer()} def encode_string(value) do case Protox.String.validate(value) do :ok -> - [Varint.encode(byte_size(value)), value] + {size_varint, size} = Varint.encode(byte_size(value)) + {[size_varint, value], size + byte_size(value)} {:error, :invalid_utf8} -> raise ArgumentError, message: "String is not valid UTF-8" @@ -137,15 +139,18 @@ defmodule Protox.Encode do end @doc false - @spec encode_bytes(binary()) :: iodata() + @spec encode_bytes(binary()) :: {iodata(), non_neg_integer()} def encode_bytes(value) do - [Varint.encode(byte_size(value)), value] + {size_varint, size} = Varint.encode(byte_size(value)) + {[size_varint, value], size + byte_size(value)} end @doc false - @spec encode_message(struct()) :: iodata() + @spec encode_message(struct()) :: {iodata(), non_neg_integer()} def encode_message(value) do - encoded = value |> Protox.encode!() |> :binary.list_to_bin() - [Varint.encode(byte_size(encoded)), encoded] + {value_bytes, value_size} = value.__struct__.encode!(value) + {value_size_bytes, value_size_bytes_size} = Varint.encode(value_size) + + {[value_size_bytes, value_bytes], value_size + value_size_bytes_size} end end diff --git a/lib/protox/parse.ex b/lib/protox/parse.ex index 3c3c0e1f..d08ba64d 100644 --- a/lib/protox/parse.ex +++ b/lib/protox/parse.ex @@ -156,7 +156,8 @@ defmodule Protox.Parse do file_options = msg.file_options |> Protox.Google.Protobuf.FileOptions.encode!() - |> :binary.list_to_bin() + |> elem(_bytes_position_in_tuple = 0) + |> IO.iodata_to_binary() |> then(&apply(Google.Protobuf.FileOptions, :decode!, [&1])) |> Map.from_struct() diff --git a/lib/protox/varint.ex b/lib/protox/varint.ex index 8250dea3..cd3fce0a 100644 --- a/lib/protox/varint.ex +++ b/lib/protox/varint.ex @@ -4,38 +4,41 @@ defmodule Protox.Varint do import Bitwise - @spec encode(integer) :: binary() + @spec encode(integer) :: {binary(), non_neg_integer()} def encode(v) when v < 1 <<< 7, - do: <> + do: {<>, 1} def encode(v) when v < 1 <<< 14, - do: <<1::1, v::7, v >>> 7>> + do: {<<1::1, v::7, v >>> 7>>, 2} def encode(v) when v < 1 <<< 21, - do: <<1::1, v::7, 1::1, v >>> 7::7, v >>> 14>> + do: {<<1::1, v::7, 1::1, v >>> 7::7, v >>> 14>>, 3} def encode(v) when v < 1 <<< 28, - do: <<1::1, v::7, 1::1, v >>> 7::7, 1::1, v >>> 14::7, v >>> 21>> + do: {<<1::1, v::7, 1::1, v >>> 7::7, 1::1, v >>> 14::7, v >>> 21>>, 4} def encode(v) when v < 1 <<< 35, - do: <<1::1, v::7, 1::1, v >>> 7::7, 1::1, v >>> 14::7, 1::1, v >>> 21::7, v >>> 28>> + do: {<<1::1, v::7, 1::1, v >>> 7::7, 1::1, v >>> 14::7, 1::1, v >>> 21::7, v >>> 28>>, 5} def encode(v) when v < 1 <<< 42, do: - <<1::1, v::7, 1::1, v >>> 7::7, 1::1, v >>> 14::7, 1::1, v >>> 21::7, 1::1, v >>> 28::7, - v >>> 35>> + {<<1::1, v::7, 1::1, v >>> 7::7, 1::1, v >>> 14::7, 1::1, v >>> 21::7, 1::1, v >>> 28::7, + v >>> 35>>, 6} def encode(v) when v < 1 <<< 49, do: - <<1::1, v::7, 1::1, v >>> 7::7, 1::1, v >>> 14::7, 1::1, v >>> 21::7, 1::1, v >>> 28::7, - 1::1, v >>> 35::7, v >>> 42>> + {<<1::1, v::7, 1::1, v >>> 7::7, 1::1, v >>> 14::7, 1::1, v >>> 21::7, 1::1, v >>> 28::7, + 1::1, v >>> 35::7, v >>> 42>>, 7} def encode(v) when v < 1 <<< 56, do: - <<1::1, v::7, 1::1, v >>> 7::7, 1::1, v >>> 14::7, 1::1, v >>> 21::7, 1::1, v >>> 28::7, - 1::1, v >>> 35::7, 1::1, v >>> 42::7, v >>> 49>> + {<<1::1, v::7, 1::1, v >>> 7::7, 1::1, v >>> 14::7, 1::1, v >>> 21::7, 1::1, v >>> 28::7, + 1::1, v >>> 35::7, 1::1, v >>> 42::7, v >>> 49>>, 8} - def encode(v), do: <<1::1, v::7, encode(v >>> 7)::binary>> + def encode(v) do + {next_bytes, size} = encode(v >>> 7) + {<<1::1, v::7, next_bytes::binary>>, size + 1} + end @spec decode(binary) :: {non_neg_integer, binary} def decode(<<0::1, byte0::7, rest::binary>>), diff --git a/test/optional_proto3_test.exs b/test/optional_proto3_test.exs index 6a6054e2..80c7459f 100644 --- a/test/optional_proto3_test.exs +++ b/test/optional_proto3_test.exs @@ -5,8 +5,8 @@ defmodule OptionalProto3Test do msg1 = %OptionalMsg1{foo: 1} msg2 = %OptionalMsg2{_foo: {:foo, 1}} - encoded_msg1 = msg1 |> OptionalMsg1.encode!() |> :binary.list_to_bin() - encoded_msg2 = msg2 |> OptionalMsg2.encode!() |> :binary.list_to_bin() + encoded_msg1 = msg1 |> OptionalMsg1.encode!() |> elem(0) |> IO.iodata_to_binary() + encoded_msg2 = msg2 |> OptionalMsg2.encode!() |> elem(0) |> IO.iodata_to_binary() assert encoded_msg1 == encoded_msg2 end @@ -15,24 +15,28 @@ defmodule OptionalProto3Test do msg1 = %OptionalMsg1{foo: 1} msg2 = %OptionalMsg2{_foo: {:foo, 1}} - assert msg2 |> OptionalMsg2.encode!() |> :binary.list_to_bin() |> OptionalMsg1.decode!() == - msg1 + assert msg1 == + msg2 + |> OptionalMsg2.encode!() + |> elem(0) + |> IO.iodata_to_binary() + |> OptionalMsg1.decode!() end test "A unset proto3 optional field is not serialized" do explicit_nil = %OptionalMsg1{foo: nil} implicit_nil = %OptionalMsg1{} - assert explicit_nil |> OptionalMsg1.encode!() |> :binary.list_to_bin() == <<>> - assert implicit_nil |> OptionalMsg1.encode!() |> :binary.list_to_bin() == <<>> + assert explicit_nil |> OptionalMsg1.encode!() |> elem(0) |> IO.iodata_to_binary() == <<>> + assert implicit_nil |> OptionalMsg1.encode!() |> elem(0) |> IO.iodata_to_binary() == <<>> end test "A proto3 optional empty message field is encoded as a oneof" do msg3 = %OptionalMsg3{foo: %OptionalMsg1{}} msg4 = %OptionalMsg4{_foo: {:foo, %OptionalMsg1{}}} - encoded_msg3 = msg3 |> OptionalMsg3.encode!() |> :binary.list_to_bin() - encoded_msg4 = msg4 |> OptionalMsg4.encode!() |> :binary.list_to_bin() + encoded_msg3 = msg3 |> OptionalMsg3.encode!() |> elem(0) |> IO.iodata_to_binary() + encoded_msg4 = msg4 |> OptionalMsg4.encode!() |> elem(0) |> IO.iodata_to_binary() assert encoded_msg3 == encoded_msg4 end @@ -41,8 +45,8 @@ defmodule OptionalProto3Test do msg3 = %OptionalMsg3{foo: %OptionalMsg1{foo: -42}} msg4 = %OptionalMsg4{_foo: {:foo, %OptionalMsg1{foo: -42}}} - encoded_msg3 = msg3 |> OptionalMsg3.encode!() |> :binary.list_to_bin() - encoded_msg4 = msg4 |> OptionalMsg4.encode!() |> :binary.list_to_bin() + encoded_msg3 = msg3 |> OptionalMsg3.encode!() |> elem(0) |> IO.iodata_to_binary() + encoded_msg4 = msg4 |> OptionalMsg4.encode!() |> elem(0) |> IO.iodata_to_binary() assert encoded_msg3 == encoded_msg4 end @@ -51,7 +55,7 @@ defmodule OptionalProto3Test do explicit_nil = %OptionalMsg3{foo: nil} implicit_nil = %OptionalMsg3{} - assert explicit_nil |> OptionalMsg3.encode!() |> :binary.list_to_bin() == <<>> - assert implicit_nil |> OptionalMsg3.encode!() |> :binary.list_to_bin() == <<>> + assert explicit_nil |> OptionalMsg3.encode!() |> elem(0) |> IO.iodata_to_binary() == <<>> + assert implicit_nil |> OptionalMsg3.encode!() |> elem(0) |> IO.iodata_to_binary() == <<>> end end diff --git a/test/protox/decode_test.exs b/test/protox/decode_test.exs index 79f87abd..99954142 100644 --- a/test/protox/decode_test.exs +++ b/test/protox/decode_test.exs @@ -12,6 +12,7 @@ defmodule Protox.DecodeTest do varint_of_max_string_size = Protox.String.max_size() |> Protox.Varint.encode() + |> elem(0) |> IO.iodata_to_binary() @success_tests [ @@ -266,6 +267,7 @@ defmodule Protox.DecodeTest do varint_of_min_invalid_string_size = min_invalid_string_size |> Protox.Varint.encode() + |> elem(0) |> IO.iodata_to_binary() @failure_tests [ diff --git a/test/protox/encode_test.exs b/test/protox/encode_test.exs index d973d73a..a688f465 100644 --- a/test/protox/encode_test.exs +++ b/test/protox/encode_test.exs @@ -4,11 +4,11 @@ defmodule Protox.EncodeTest do alias ProtobufTestMessages.Proto3.{NullHypothesisProto3, TestAllTypesProto3} test "Default TestAllTypesProto3" do - assert %TestAllTypesProto3{} |> Protox.encode!() |> :binary.list_to_bin() == <<>> + assert %TestAllTypesProto3{} |> Protox.encode!() == {[], 0} end test "Default TestAllTypesProto3, with non throwing encode/1" do - assert {:ok, []} == Protox.encode(%TestAllTypesProto3{}) + assert {:ok, [], 0} == Protox.encode(%TestAllTypesProto3{}) end test "Messsage with no fields, unknown fields are encoded back" do @@ -20,108 +20,87 @@ defmodule Protox.EncodeTest do ] } - assert msg |> Protox.encode!() |> :binary.list_to_bin() == - <<8, 42, 25, 246, 40, 92, 143, 194, 53, 69, 64, 136, 241, 4, 83>> + assert encode(msg) == + {<<8, 42, 25, 246, 40, 92, 143, 194, 53, 69, 64, 136, 241, 4, 83>>, 15} end test "Scalar int64" do - assert %TestAllTypesProto3{optional_int64: -300} |> Protox.encode!() |> :binary.list_to_bin() == - <<16, 212, 253, 255, 255, 255, 255, 255, 255, 255, 1>> + assert encode(%TestAllTypesProto3{optional_int64: -300}) == + {<<16, 212, 253, 255, 255, 255, 255, 255, 255, 255, 1>>, 11} end test "Scalar uint32" do - assert %TestAllTypesProto3{optional_uint32: 42} |> Protox.encode!() |> :binary.list_to_bin() == - <<24, 42>> + assert encode(%TestAllTypesProto3{optional_uint32: 42}) == {<<24, 42>>, 2} end test "Scalar uint64" do - assert %TestAllTypesProto3{optional_uint64: 300_000} - |> Protox.encode!() - |> :binary.list_to_bin() == - <<32, 224, 167, 18>> + assert encode(%TestAllTypesProto3{optional_uint64: 300_000}) == {<<32, 224, 167, 18>>, 4} end test "Scalar sint64" do - assert %TestAllTypesProto3{optional_sint64: -1323} - |> Protox.encode!() - |> :binary.list_to_bin() == <<48, 213, 20>> + assert encode(%TestAllTypesProto3{optional_sint64: -1323}) == {<<48, 213, 20>>, 3} end test "Scalar fixed32" do - assert %TestAllTypesProto3{optional_fixed32: 352} - |> Protox.encode!() - |> :binary.list_to_bin() == <<61, 96, 1, 0, 0>> + assert encode(%TestAllTypesProto3{optional_fixed32: 352}) == {<<61, 96, 1, 0, 0>>, 5} end test "Scalar sfixed64" do - assert %TestAllTypesProto3{optional_sfixed64: -352} - |> Protox.encode!() - |> :binary.list_to_bin() == <<81, 160, 254, 255, 255, 255, 255, 255, 255>> + assert encode(%TestAllTypesProto3{optional_sfixed64: -352}) == + {<<81, 160, 254, 255, 255, 255, 255, 255, 255>>, 9} end test "Repeated int32" do - assert %TestAllTypesProto3{repeated_int32: [-1, 0, 1]} - |> Protox.encode!() - |> :binary.list_to_bin() == - <<250, 1, 12, 255, 255, 255, 255, 255, 255, 255, 255, 255, 1, 0, 1>> + assert encode(%TestAllTypesProto3{repeated_int32: [-1, 0, 1]}) == + {<<250, 1, 12, 255, 255, 255, 255, 255, 255, 255, 255, 255, 1, 0, 1>>, 15} end test "Repeated fixed64" do - assert %TestAllTypesProto3{repeated_fixed64: [0, 1]} - |> Protox.encode!() - |> :binary.list_to_bin() == - <<178, 2, 16, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0>> + assert encode(%TestAllTypesProto3{repeated_fixed64: [0, 1]}) == + {<<178, 2, 16, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0>>, 19} end test "Repeated sfixed32" do - assert %TestAllTypesProto3{repeated_sfixed32: [-1, 2]} - |> Protox.encode!() - |> :binary.list_to_bin() == - <<186, 2, 8, 255, 255, 255, 255, 2, 0, 0, 0>> + assert encode(%TestAllTypesProto3{repeated_sfixed32: [-1, 2]}) == + {<<186, 2, 8, 255, 255, 255, 255, 2, 0, 0, 0>>, 11} end test "Repeated double" do - assert %TestAllTypesProto3{repeated_double: [33.2, -44.0, :infinity, :"-infinity", :nan]} - |> Protox.encode!() - |> :binary.list_to_bin() == - <<210, 2, 40, 154, 153, 153, 153, 153, 153, 64, 64, 0, 0, 0, 0, 0, 0, 70, 192, 0, 0, - 0, 0, 0, 0, 240, 127, 0, 0, 0, 0, 0, 0, 240, 255, 0, 0, 0, 0, 0, 1, 241, 255>> + assert encode(%TestAllTypesProto3{ + repeated_double: [33.2, -44.0, :infinity, :"-infinity", :nan] + }) == + {<<210, 2, 40, 154, 153, 153, 153, 153, 153, 64, 64, 0, 0, 0, 0, 0, 0, 70, 192, 0, 0, + 0, 0, 0, 0, 240, 127, 0, 0, 0, 0, 0, 0, 240, 255, 0, 0, 0, 0, 0, 1, 241, 255>>, + 43} end test "Repeated float" do - assert %TestAllTypesProto3{repeated_float: [33.2, -44.0, :infinity, :"-infinity", :nan]} - |> Protox.encode!() - |> :binary.list_to_bin() == - <<202, 2, 20, 205, 204, 4, 66, 0, 0, 48, 194, 0, 0, 128, 127, 0, 0, 128, 255, 0, 1, - 129, 255>> + assert encode(%TestAllTypesProto3{ + repeated_float: [33.2, -44.0, :infinity, :"-infinity", :nan] + }) == + {<<202, 2, 20, 205, 204, 4, 66, 0, 0, 48, 194, 0, 0, 128, 127, 0, 0, 128, 255, 0, 1, + 129, 255>>, 23} end test "Repeated bool" do - assert %TestAllTypesProto3{repeated_bool: [true, false, true, false, false, false]} - |> Protox.encode!() - |> :binary.list_to_bin() == - <<218, 2, 6, 1, 0, 1, 0, 0, 0>> + assert encode(%TestAllTypesProto3{repeated_bool: [true, false, true, false, false, false]}) == + {<<218, 2, 6, 1, 0, 1, 0, 0, 0>>, 9} end test "Repeated enum" do - assert %TestAllTypesProto3{repeated_nested_enum: [:FOO, :BAR, :BAZ, 4]} - |> Protox.encode!() - |> :binary.list_to_bin() == - <<154, 3, 4, 0, 1, 2, 4>> + assert encode(%TestAllTypesProto3{repeated_nested_enum: [:FOO, :BAR, :BAZ, 4]}) == + {<<154, 3, 4, 0, 1, 2, 4>>, 7} end test "Unpacked Repeated int32" do - assert %TestAllTypesProto3{unpacked_int32: [-1, 2, 3]} - |> Protox.encode!() - |> :binary.list_to_bin() == - <<200, 5, 255, 255, 255, 255, 255, 255, 255, 255, 255, 1, 200, 5, 2, 200, 5, 3>> + assert encode(%TestAllTypesProto3{unpacked_int32: [-1, 2, 3]}) == + {<<200, 5, 255, 255, 255, 255, 255, 255, 255, 255, 255, 1, 200, 5, 2, 200, 5, 3>>, + 18} end test "Bytes" do - assert %TestAllTypesProto3{optional_bytes: <<1, 2, 3>>} - |> Protox.encode!() - |> :binary.list_to_bin() == - <<122, 3, 1, 2, 3>> + assert encode(%TestAllTypesProto3{optional_bytes: <<1, 2, 3>>}) == + {<<122, 3, 1, 2, 3>>, 5} end test "Unknown fields (float + varint + bytes)" do @@ -132,32 +111,21 @@ defmodule Protox.EncodeTest do {10, 2, <<104, 101, 121, 33>>} ] } - |> Protox.encode!() - |> :binary.list_to_bin() == - <<101, 236, 81, 5, 66, 88, 154, 5, 82, 4, 104, 101, 121, 33>> + |> encode() == + {<<101, 236, 81, 5, 66, 88, 154, 5, 82, 4, 104, 101, 121, 33>>, 14} end test "Empty repeated bool" do - assert %TestAllTypesProto3{repeated_bool: []} |> Protox.encode!() |> :binary.list_to_bin() == - <<>> + assert encode(%TestAllTypesProto3{repeated_bool: []}) == {"", 0} end test "Optional sub message" do - assert %OptionalUpperMsg{sub: %OptionalSubMsg{a: 42}} - |> Protox.encode!() - |> :binary.list_to_bin() == <<10, 2, 8, 42>> + assert encode(%OptionalUpperMsg{sub: %OptionalSubMsg{a: 42}}) == {<<10, 2, 8, 42>>, 4} end test "Do not output default double/float" do - assert %TestAllTypesProto3{optional_float: 0.0, optional_double: 0.0} - |> Protox.encode!() - |> :binary.list_to_bin() == - <<>> - - assert %TestAllTypesProto3{optional_float: 0, optional_double: 0} - |> Protox.encode!() - |> :binary.list_to_bin() == - <<>> + assert encode(%TestAllTypesProto3{optional_float: 0.0, optional_double: 0.0}) == {<<>>, 0} + assert encode(%TestAllTypesProto3{optional_float: 0, optional_double: 0}) == {<<>>, 0} end test "Raise when required field is missing" do @@ -171,26 +139,26 @@ defmodule Protox.EncodeTest do test "UTF-8 strings" do [ - {"", <<>>}, - {"hello, 漢字, 💻, 🏁, working fine", <<114, 39, "hello, 漢字, 💻, 🏁, working fine">>} + {"", {<<>>, 0}}, + {"hello, 漢字, 💻, 🏁, working fine", {<<114, 39, "hello, 漢字, 💻, 🏁, working fine">>, 41}} ] |> Enum.each(fn {string, expected_encoded_msg} -> - assert %TestAllTypesProto3{optional_string: string} - |> Protox.encode!() - |> IO.iodata_to_binary() == - expected_encoded_msg + assert encode(%TestAllTypesProto3{optional_string: string}) == expected_encoded_msg end) end test "Largest valid string" do string_size = Protox.String.max_size() - assert %TestAllTypesProto3{ - optional_string: <<0::integer-size(string_size)-unit(8)>> - } - |> Protox.encode!() - |> IO.iodata_to_binary() == - <<114, 128, 128, 64>> <> <<0::integer-size(string_size)-unit(8)>> + assert encode(%TestAllTypesProto3{optional_string: <<0::integer-size(string_size)-unit(8)>>}) == + {<<114, 128, 128, 64>> <> <<0::integer-size(string_size)-unit(8)>>, string_size + 4} + end + + test "Map" do + msg = %TestAllTypesProto3{map_int32_int32: %{1 => 2, 3 => 4}} + + {bytes, size} = Protox.encode!(msg) + assert size == bytes |> IO.iodata_to_binary() |> byte_size() end test "Raise when string is not valid UTF-8" do @@ -229,4 +197,10 @@ defmodule Protox.EncodeTest do |> Protox.encode!() end) end + + defp encode(msg) do + {iodata, size} = Protox.encode!(msg) + + {IO.iodata_to_binary(iodata), size} + end end diff --git a/test/protox/varint_test.exs b/test/protox/varint_test.exs index c2c43d67..c4749fac 100644 --- a/test/protox/varint_test.exs +++ b/test/protox/varint_test.exs @@ -5,48 +5,55 @@ defmodule Protox.VarintTest do property "Unrolled encoding produces the same result as the reference implementation" do check all(int <- integer(0..(1 <<< 64))) do - assert int |> Protox.Varint.encode() |> IO.iodata_to_binary() == - int |> encode_reference() |> IO.iodata_to_binary() + {unrolled, size} = Protox.Varint.encode(int) + unrolled_bytes = IO.iodata_to_binary(unrolled) + assert size == byte_size(unrolled_bytes) + + reference = encode_reference(int) + + assert unrolled_bytes == IO.iodata_to_binary(reference) end end property "Symmetric" do check all(int <- integer(0..(1 <<< 64))) do - assert int |> Protox.Varint.encode() |> IO.iodata_to_binary() |> Protox.Varint.decode() == - {int, ""} + {encoded, size} = Protox.Varint.encode(int) + assert size == byte_size(encoded) + + assert {^int, ""} = encoded |> IO.iodata_to_binary() |> Protox.Varint.decode() end end test "Encode" do - assert Protox.Varint.encode(0) == <<0>> - assert Protox.Varint.encode(1) == <<1>> + assert Protox.Varint.encode(0) == {<<0>>, 1} + assert Protox.Varint.encode(1) == {<<1>>, 1} - assert Protox.Varint.encode((1 <<< 14) - 1) == <<0xFF, 0x7F>> - assert Protox.Varint.encode(1 <<< 14) == <<0x80, 0x80, 0x1>> + assert Protox.Varint.encode((1 <<< 14) - 1) == {<<0xFF, 0x7F>>, 2} + assert Protox.Varint.encode(1 <<< 14) == {<<0x80, 0x80, 0x1>>, 3} - assert Protox.Varint.encode((1 <<< 21) - 1) == <<0xFF, 0xFF, 0x7F>> - assert Protox.Varint.encode(1 <<< 21) == <<0x80, 0x80, 0x80, 0x1>> + assert Protox.Varint.encode((1 <<< 21) - 1) == {<<0xFF, 0xFF, 0x7F>>, 3} + assert Protox.Varint.encode(1 <<< 21) == {<<0x80, 0x80, 0x80, 0x1>>, 4} - assert Protox.Varint.encode((1 <<< 28) - 1) == <<0xFF, 0xFF, 0xFF, 0x7F>> - assert Protox.Varint.encode(1 <<< 28) == <<0x80, 0x80, 0x80, 0x80, 0x1>> + assert Protox.Varint.encode((1 <<< 28) - 1) == {<<0xFF, 0xFF, 0xFF, 0x7F>>, 4} + assert Protox.Varint.encode(1 <<< 28) == {<<0x80, 0x80, 0x80, 0x80, 0x1>>, 5} - assert Protox.Varint.encode((1 <<< 35) - 1) == <<0xFF, 0xFF, 0xFF, 0xFF, 0x7F>> - assert Protox.Varint.encode(1 <<< 35) == <<0x80, 0x80, 0x80, 0x80, 0x80, 0x1>> + assert Protox.Varint.encode((1 <<< 35) - 1) == {<<0xFF, 0xFF, 0xFF, 0xFF, 0x7F>>, 5} + assert Protox.Varint.encode(1 <<< 35) == {<<0x80, 0x80, 0x80, 0x80, 0x80, 0x1>>, 6} - assert Protox.Varint.encode((1 <<< 42) - 1) == <<0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x7F>> - assert Protox.Varint.encode(1 <<< 42) == <<0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x1>> + assert Protox.Varint.encode((1 <<< 42) - 1) == {<<0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x7F>>, 6} + assert Protox.Varint.encode(1 <<< 42) == {<<0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x1>>, 7} assert Protox.Varint.encode((1 <<< 56) - 1) == - <<0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x7F>> + {<<0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x7F>>, 8} assert Protox.Varint.encode(1 <<< 56) == - <<0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x1>> + {<<0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x1>>, 9} assert Protox.Varint.encode((1 <<< 63) - 1) == - <<0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x7F>> + {<<0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x7F>>, 9} assert Protox.Varint.encode(1 <<< 63) == - <<0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x1>> + {<<0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x1>>, 10} end defp encode_reference(v) when v < 1 <<< 7, do: <> diff --git a/test/protox_properties_test.exs b/test/protox_properties_test.exs index d5b7ebc8..93c55883 100644 --- a/test/protox_properties_test.exs +++ b/test/protox_properties_test.exs @@ -2,13 +2,12 @@ defmodule Protox.PropertiesTest do use ExUnit.Case use PropCheck - @moduletag timeout: 60_000 * 5 + # @moduletag timeout: 60_000 * 5 - @tag :properties property "Binary: ProtobufTestMessages.Proto3.TestAllTypesProto3" do - forall {msg, encoded, encoded_bin, decoded} <- + forall {msg, encoded, decoded} <- generate_binary(ProtobufTestMessages.Proto3.TestAllTypesProto3) do - is_list(encoded) and is_binary(encoded_bin) and decoded == msg + is_list(encoded) and decoded == msg end end @@ -17,11 +16,10 @@ defmodule Protox.PropertiesTest do defp generate_binary(mod) do let fields <- Protox.RandomInit.generate_fields(mod) do msg = Protox.RandomInit.generate_struct(mod, fields) - encoded = Protox.encode!(msg) - encoded_bin = :binary.list_to_bin(encoded) - decoded = Protox.decode!(encoded_bin, mod) + {encoded, _} = Protox.encode!(msg) + decoded = encoded |> IO.iodata_to_binary() |> Protox.decode!(mod) - {msg, encoded, encoded_bin, decoded} + {msg, encoded, decoded} end end end diff --git a/test/protox_test.exs b/test/protox_test.exs index bd8fe442..deaf4528 100644 --- a/test/protox_test.exs +++ b/test/protox_test.exs @@ -27,7 +27,11 @@ defmodule ProtoxTest do msg = %TestAllTypesProto3{optional_double: 8.73291669056208, optional_float: 0.1} decoded = - msg |> TestAllTypesProto3.encode!() |> :binary.list_to_bin() |> TestAllTypesProto3.decode!() + msg + |> TestAllTypesProto3.encode!() + |> elem(0) + |> :binary.list_to_bin() + |> TestAllTypesProto3.decode!() assert decoded.optional_double == msg.optional_double assert Float.round(decoded.optional_float, 1) == msg.optional_float @@ -38,18 +42,20 @@ defmodule ProtoxTest do assert msg |> TestAllTypesProto3.encode!() + |> elem(0) |> :binary.list_to_bin() |> TestAllTypesProto3.decode!() == msg end - test "Symmetric encoding/decoding of protobuf2 messages" do - msg = Protox.RandomInit.generate_msg(TestAllTypesProto2) + # test "Symmetric encoding/decoding of protobuf2 messages" do + # msg = Protox.RandomInit.generate_msg(TestAllTypesProto2) - assert msg - |> TestAllTypesProto2.encode!() - |> :binary.list_to_bin() - |> TestAllTypesProto2.decode!() == msg - end + # assert msg + # |> TestAllTypesProto2.encode!() + # |> elem(0) + # |> :binary.list_to_bin() + # |> TestAllTypesProto2.decode!() == msg + # end test "Enum constants" do assert ForeignEnum.constants() == [{0, :FOREIGN_FOO}, {1, :FOREIGN_BAR}, {2, :FOREIGN_BAZ}] @@ -155,7 +161,7 @@ defmodule ProtoxTest do test "Non Camel_case" do msg = Protox.RandomInit.generate_msg(Camel) - assert msg == msg |> Camel.encode!() |> :binary.list_to_bin() |> Camel.decode!() + assert msg == msg |> Camel.encode!() |> elem(0) |> :binary.list_to_bin() |> Camel.decode!() end test "Non CamelCase enums" do @@ -164,6 +170,7 @@ defmodule ProtoxTest do assert msg == msg |> MsgWithNonCamelEnum.encode!() + |> elem(0) |> :binary.list_to_bin() |> MsgWithNonCamelEnum.decode!() @@ -178,9 +185,11 @@ defmodule ProtoxTest do # -- Helper functions - defp reencode_with_protoc(encoded, mod) do + defp reencode_with_protoc({encoded, _size}, mod) do tmp_dir = Protox.TmpFs.tmp_dir!("protoc_test") + encoded = :binary.list_to_bin(encoded) + encoded_bin_path = Path.join([tmp_dir, "protoc_test.bin"]) File.write!(encoded_bin_path, encoded) diff --git a/test/support/random_init.ex b/test/support/random_init.ex index db64c1d8..c629f952 100644 --- a/test/support/random_init.ex +++ b/test/support/random_init.ex @@ -13,7 +13,7 @@ defmodule Protox.RandomInit do generate_struct(mod, fields) end - {:ok, msg} = :proper_gen.pick(gen) + {:ok, msg} = :proper_gen.pick(gen, 1) msg end