fix: add bundle tags to ans104 and tx commitments

Author: JamesPiechota

src/ar_bundles.erl

@@ -1,11 +1,10 @@
 -module(ar_bundles).
--export([signer/1, is_signed/1]).
--export([id/1, id/2, hd/1, member/2, find/2, tagfind/3]).
+-export([signer/1]).
+-export([id/1, id/2, hd/1, member/2, find/2]).
 -export([new_item/4, sign_item/2, verify_item/1]).
 -export([encode_tags/1, decode_tags/1]).
--export([serialize/1, deserialize/1, serialize_data/1, serialize_bundle/2]).
+-export([serialize/1, deserialize/1, serialize_bundle/2]).
 -export([data_item_signature_data/1]).
--export([type/1]).
 -include("include/hb.hrl").
 -include_lib("eunit/include/eunit.hrl").
 
@@ -19,10 +18,6 @@
 signer(#tx { owner = ?DEFAULT_OWNER }) -> undefined;
 signer(Item) -> crypto:hash(sha256, Item#tx.owner).
 
-%% @doc Check if an item is signed.
-is_signed(Item) ->
-    Item#tx.signature =/= ?DEFAULT_SIG.
-
 %% @doc Return the ID of an item -- either signed or unsigned as specified.
 %% If the item is unsigned and the user requests the signed ID, we return
 %% the atom `not_signed'. In all other cases, we return the ID of the item.
@@ -76,17 +71,6 @@ find(Key, Item = #tx { data = Data }) ->
 find(_Key, _) ->
     not_found.
 
-%% @doc Case-insensitively find a tag in a list and return its value.
-tagfind(Key, Tags, Default) ->
-    LowerCaseKey = hb_util:to_lower(Key),
-    Found = lists:search(fun({TagName, _}) ->
-        hb_util:to_lower(TagName) == LowerCaseKey
-    end, Tags),
-    case Found of
-        {value, {_TagName, Value}} -> Value;
-        false -> Default
-    end.
-
 %% @doc Create a new data item. Should only be used for testing.
 new_item(Target, Anchor, Tags, Data) ->
     dev_arweave_common:reset_ids(
@@ -196,7 +180,7 @@ enforce_valid_tx(TX) ->
 %% @doc Generate the data segment to be signed for a data item.
 data_item_signature_data(RawItem) ->
     true = enforce_valid_tx(RawItem),
-    Item = serialize_data(RawItem),
+    Item = dev_arweave_common:serialize_data(RawItem),
     ar_deep_hash:hash([
         utf8_encoded("dataitem"),
         utf8_encoded("1"),
@@ -237,7 +221,7 @@ serialize(not_found) -> throw(not_found);
 serialize(TX) when is_binary(TX) -> TX;
 serialize(RawTX) when is_record(RawTX, tx) ->
     true = enforce_valid_tx(RawTX),
-    TX = serialize_data(RawTX),
+    TX = dev_arweave_common:serialize_data(RawTX),
     EncodedTags = encode_tags(TX#tx.tags),
     <<
         (encode_signature_type(TX#tx.signature_type))/binary,
@@ -252,27 +236,6 @@ serialize(RawTX) when is_record(RawTX, tx) ->
 serialize(TX) ->
     throw({cannot_serialize_tx, must_be_binary_or_tx, TX}).
 
-serialize_data(Item = #tx{data = Data}) when is_binary(Data) ->
-    Item;
-serialize_data(Item = #tx{data = Data}) ->
-    IsBundleMap = type(Item) == map,
-    ?event({serialize_data,
-        hb_util:human_id(Item#tx.unsigned_id), hb_util:human_id(Item#tx.id),
-        {is_bundle_map, IsBundleMap},
-        {is_list, is_list(Data)},
-        {is_map, is_map(Data)}}),
-    ConvertedData = 
-        case {IsBundleMap, is_list(Data), is_map(Data)} of
-            {true, true, false} ->
-                dev_arweave_common:convert_bundle_list_to_map(Data);
-            {false, false, true} ->
-                dev_arweave_common:convert_bundle_map_to_list(Data);
-            _ ->
-                Data
-        end,
-    {Manifest, SerializedData} = serialize_bundle(ConvertedData, false),
-    Item#tx{data = SerializedData, manifest = Manifest}.
-
 serialize_bundle(List, Normalize) when is_list(List) ->
     FinalizedData = finalize_bundle_data(
         lists:map(
@@ -303,7 +266,7 @@ serialize_bundle(Map, Normalize) when is_map(Map) ->
         [{NewManifestUnsignedID, NewManifestSerialized} | maps:values(BinItems)]),
     {NewManifest, FinalizedData};
 serialize_bundle(Data, _Normalize) when is_binary(Data) ->
-    Data;
+    {undefined, Data};
 serialize_bundle(Data, _Normalize) ->
     throw({cannot_serialize_tx_data, must_be_list_or_map, Data}).
 
@@ -454,21 +417,8 @@ deserialize_item(Binary) ->
         })
     ).
 
-type(Item) ->
-    Format = tagfind(<<"bundle-format">>, Item#tx.tags, <<>>),
-    Version = tagfind(<<"bundle-version">>, Item#tx.tags, <<>>),
-    MapTXID = tagfind(<<"bundle-map">>, Item#tx.tags, <<>>),
-    case {hb_util:to_lower(Format), hb_util:to_lower(Version), MapTXID} of
-        {<<"binary">>, <<"2.0.0">>, <<>>} ->
-            list;
-        {<<"binary">>, <<"2.0.0">>, _} ->
-            map;
-        _ ->
-            binary
-    end.
-
 maybe_unbundle(Item) ->
-    case type(Item) of
+    case dev_arweave_common:type(Item) of
         list -> unbundle_list(Item);
         binary -> Item;
         map -> unbundle_map(Item)
@@ -481,7 +431,7 @@ unbundle_list(Item) ->
     end.
 
 unbundle_map(Item) ->
-    MapTXID = tagfind(<<"bundle-map">>, Item#tx.tags, <<>>),
+    MapTXID = dev_arweave_common:tagfind(<<"bundle-map">>, Item#tx.tags, <<>>),
     case unbundle(Item#tx.data) of
         detached -> Item#tx{data = detached};
         Items ->
@@ -921,7 +871,7 @@ arbundles_list_bundle_roundtrip_test() ->
     ?assert(verify_item(Item2)),
     ?assert(verify_item(Item3)),
 
-    Reserialized = serialize_data(Deserialized),
+    Reserialized = dev_arweave_common:normalize(Deserialized),
     ?event(debug_test, {reserialized, Reserialized}),
     ?assert(verify_item(Reserialized)),
     ?assertEqual(Bin, Reserialized#tx.data),
@@ -952,7 +902,7 @@ arbundles_single_list_bundle_roundtrip_test() ->
     ?assertEqual([{<<"Type">>, <<"list">>}, {<<"Index">>, <<"1">>}], Item#tx.tags),
     ?assert(verify_item(Item)),
 
-    Reserialized = serialize_data(Deserialized),
+    Reserialized = dev_arweave_common:normalize(Deserialized),
     ?event(debug_test, {reserialized, Reserialized}),
     ?assert(verify_item(Reserialized)),
     ?assertEqual(Bin, Reserialized#tx.data),
@@ -984,7 +934,7 @@ arbundles_map_bundle_roundtrip_test() ->
     Manifest = Deserialized#tx.manifest,
     ?event(debug_test, {manifest, Manifest}),
     ?assertNotEqual(undefined, Manifest),
-    ?assertEqual(false, is_signed(Manifest)),
+    ?assertEqual(false, dev_arweave_common:is_signed(Manifest)),
     ?assertEqual([
         {<<"data-protocol">>, <<"bundle-map">>},
         {<<"variant">>, <<"0.0.1">>}

src/ar_format.erl

@@ -59,7 +59,7 @@ format(TX, Indent, Opts) when is_record(TX, tx) ->
             format_line("!!! CAUTION: ITEM IS SIGNED BUT INVALID !!!", Indent + 1);
         false -> []
     end ++
-    case ar_bundles:is_signed(TX) of
+    case dev_arweave_common:is_signed(TX) of
         true ->
             format_line("Signer: ~s",
                 [hb_util:safe_encode(ar_bundles:signer(TX))], 

src/ar_tx.erl

@@ -237,7 +237,6 @@ json_struct_to_tx(TXStruct) ->
             Xs ->
                 Xs
         end,
-    ?event(debug_test, {json_struct_to_tx, {tags, {explicit, Tags}}}),
     Data = hb_util:decode(hb_util:find_value(<<"data">>, TXStruct)),
     Format =
         case hb_util:find_value(<<"format">>, TXStruct) of

src/dev_arweave.erl

@@ -306,12 +306,59 @@ post_ans104_tx_test() ->
         GetRes
     ),
     ok.
-get_tx_test() ->
+get_tx_basic_data_test() ->
     Node = hb_http_server:start_node(),
-    Path = <<"/~a[email protected]/tx?tx=MH1M9gedlPsu7-57-N0S_-F1OBoYacLYLe5jL0Tvr9c">>,
-    {ok, StructuredTX} = hb_http:get(Node, Path, #{}),
-    ?event(debug_test, {structured_tx, StructuredTX}),
-    ?assert(hb_message:verify(StructuredTX, all, #{})).
+    Path = <<"/~a[email protected]/tx?tx=ptBC0UwDmrUTBQX3MqZ1lB57ex20ygwzkjjCrQjIx3o">>,
+    {ok, Structured} = hb_http:get(Node, Path, #{}),
+    ?event(debug_test, {structured_tx, Structured}),
+    ?assert(hb_message:verify(Structured, all, #{})),
+    % Hash the data to make it easier to match
+    StructuredWithHash = Structured#{
+        <<"data">> => hb_util:encode(
+            crypto:hash(sha256, (maps:get(<<"data">>, Structured)))
+        )
+    },
+    ExpectedMsg = #{
+        <<"data">> => <<"PEShWA1ER2jq7CatAPpOZ30TeLrjOSpaf_Po7_hKPo4">>,
+        <<"reward">> => <<"482143296">>,
+        <<"anchor">> => <<"XTzaU2_m_hRYDLiXkcleOC4zf5MVTXIeFWBOsJSRrtEZ8kM6Oz7EKLhZY7fTAvKq">>,
+        <<"content-type">> => <<"application/json">>
+    },
+    ?assert(hb_message:match(ExpectedMsg, StructuredWithHash, only_present)),
+    ok.
+
+get_tx_rsa_nested_bundle_test() ->
+    Node = hb_http_server:start_node(),
+    Path = <<"/~a[email protected]/tx?tx=bndIwac23-s0K11TLC1N7z472sLGAkiOdhds87ZywoE">>,
+    {ok, Structured} = hb_http:get(Node, Path, #{}),
+    TABM = hb_message:convert(
+        Structured,
+        tabm,
+        #{
+            <<"device">> => <<"[email protected]">>,
+            <<"bundle">> => true
+        },
+        #{}),
+    {ok, TX} = dev_codec_tx:to(TABM, #{}, #{}),
+    ?event(debug_test, {tabm, TABM}),
+    ?event(debug_test, {tx, TX}),
+    % ?event(debug_test, {structured_tx, Structured}),
+    ?assert(ar_tx:verify(TX)),
+    ?assert(hb_message:verify(Structured, all, #{})),
+    % % Hash the data to make it easier to match
+    % StructuredWithHash = Structured#{
+    %     <<"data">> => hb_util:encode(
+    %         crypto:hash(sha256, (maps:get(<<"data">>, Structured)))
+    %     )
+    % },
+    % ExpectedMsg = #{
+    %     <<"data">> => <<"PEShWA1ER2jq7CatAPpOZ30TeLrjOSpaf_Po7_hKPo4">>,
+    %     <<"reward">> => <<"482143296">>,
+    %     <<"anchor">> => <<"XTzaU2_m_hRYDLiXkcleOC4zf5MVTXIeFWBOsJSRrtEZ8kM6Oz7EKLhZY7fTAvKq">>,
+    %     <<"content-type">> => <<"application/json">>
+    % },
+    % ?assert(hb_message:match(ExpectedMsg, StructuredWithHash, only_present)),
+    ok.
 
 
 get_bad_tx_test() ->

src/dev_arweave_common.erl

@@ -1,11 +1,40 @@
 %%% @doc Utility module for routing functionality to ar_bundles.erl or
 %%% ar_tx.erl based off #tx.format.
 -module(dev_arweave_common).
--export([reset_ids/1, generate_id/2, normalize/1]).
+-export([is_signed/1, type/1, tagfind/3]).
+-export([reset_ids/1, generate_id/2, normalize/1, serialize_data/1]).
 -export([convert_bundle_list_to_map/1, convert_bundle_map_to_list/1]).
 -include("include/hb.hrl").
 -include_lib("eunit/include/eunit.hrl").
 
+%% @doc Check if an item is signed.
+is_signed(TX) ->
+    TX#tx.signature =/= ?DEFAULT_SIG.
+
+type(Item) ->
+    Format = tagfind(<<"bundle-format">>, Item#tx.tags, <<>>),
+    Version = tagfind(<<"bundle-version">>, Item#tx.tags, <<>>),
+    MapTXID = tagfind(<<"bundle-map">>, Item#tx.tags, <<>>),
+    case {hb_util:to_lower(Format), hb_util:to_lower(Version), MapTXID} of
+        {<<"binary">>, <<"2.0.0">>, <<>>} ->
+            list;
+        {<<"binary">>, <<"2.0.0">>, _} ->
+            map;
+        _ ->
+            binary
+    end.
+
+%% @doc Case-insensitively find a tag in a list and return its value.
+tagfind(Key, Tags, Default) ->
+    LowerCaseKey = hb_util:to_lower(Key),
+    Found = lists:search(fun({TagName, _}) ->
+        hb_util:to_lower(TagName) == LowerCaseKey
+    end, Tags),
+    case Found of
+        {value, {_TagName, Value}} -> Value;
+        false -> Default
+    end.
+
 %% @doc Re-calculate both of the IDs for a #tx. This is a wrapper
 %% function around `update_ids/1' that ensures both IDs are set from
 %% scratch.
@@ -47,33 +76,55 @@ generate_signature_data_segment(TX) ->
 %% @doc Ensure that a data item (potentially containing a map or list) has a
 %% standard, serialized form.
 normalize(not_found) -> throw(not_found);
-normalize(Item = #tx{data = Bin}) when is_binary(Bin) ->
+normalize(TX = #tx{data = Bin}) when is_binary(Bin) ->
     ?event({normalize, binary,
-        hb_util:human_id(Item#tx.unsigned_id), hb_util:human_id(Item#tx.id)}),
+        hb_util:human_id(TX#tx.unsigned_id), hb_util:human_id(TX#tx.id)}),
     reset_ids(
         normalize_data_root(
             normalize_data_size(
                 reset_owner_address(
-                    Item))));
+                    TX))));
 normalize(Bundle) when is_list(Bundle); is_map(Bundle) ->
     ?event({normalize, bundle}),
     normalize(#tx{ data = Bundle });
-normalize(Item = #tx { data = Data }) when is_list(Data) ->
-    ?event({normalize, list,
-        hb_util:human_id(Item#tx.unsigned_id), hb_util:human_id(Item#tx.id)}),
-    normalize(Item#tx{data = convert_bundle_list_to_map(Data)});
-normalize(Item = #tx{data = Data}) when is_map(Data) ->
-    {Manifest, Bin} = ar_bundles:serialize_bundle(Item#tx.data, true),
-    SerializedItem =
-        Item#tx{
-            data = Bin,
-            manifest = Manifest,
-            tags = add_manifest_tags(
-                add_bundle_tags(Item#tx.tags),
-                ar_bundles:id(Manifest, unsigned)
-            )
-        },
-    normalize(SerializedItem).
+normalize(TX = #tx{data = Data}) ->
+    SerializedTX = serialize_data(TX, true),
+    NormalizedTX = case is_signed(TX) of
+        true ->
+            SerializedTX;
+        false ->
+            add_bundle_tags(SerializedTX)
+    end,
+    normalize(NormalizedTX).
+
+serialize_data(TX) -> serialize_data(TX, false).
+serialize_data(Item = #tx{data = Data}, _) when is_binary(Data) ->
+    Item;
+serialize_data(Item = #tx{data = Data}, NormalizeChildren) ->
+    IsBundleMap = type(Item) == map,
+    ?event({serialize_data,
+        hb_util:human_id(Item#tx.unsigned_id), hb_util:human_id(Item#tx.id),
+        {normalize_children, NormalizeChildren},
+        {is_bundle_map, IsBundleMap},
+        {is_list, is_list(Data)},
+        {is_map, is_map(Data)}}),
+    ConvertedData = 
+        case {is_signed(Item), IsBundleMap, is_list(Data), is_map(Data)} of
+            {true, true, true, false} ->
+                % Signed transaction with bundle-map tag and list data
+                convert_bundle_list_to_map(Data);
+            {true, false, false, true} ->
+                % Signed transaction without bundle-map tag and map data
+                convert_bundle_map_to_list(Data);
+            {false, _, true, false} ->
+                % Unsigned transaction with list data
+                convert_bundle_list_to_map(Data);
+            _ ->
+                Data
+        end,
+    {Manifest, SerializedData} =
+        ar_bundles:serialize_bundle(ConvertedData, NormalizeChildren),
+    Item#tx{data = SerializedData, manifest = Manifest}.
 
 convert_bundle_list_to_map(Data) ->
     maps:from_list(
@@ -97,15 +148,16 @@ convert_bundle_map_to_list(Data) ->
         lists:seq(1, maps:size(Data))
     ).
 
-add_bundle_tags(Tags) -> ?BUNDLE_TAGS ++ (Tags -- ?BUNDLE_TAGS).
-
-add_manifest_tags(Tags, ManifestID) ->
-    lists:filter(
+add_bundle_tags(TX) -> 
+    ManifestID = ar_bundles:id(TX#tx.manifest, unsigned),
+    BundleTags = ?BUNDLE_TAGS ++ [{<<"bundle-map">>, hb_util:encode(ManifestID)}],
+    StrippedTags = lists:filter(
         fun({TagName, _}) ->
-            hb_util:to_lower(TagName) =/= <<"bundle-map">>
+            not lists:member(hb_util:to_lower(TagName), ?BUNDLE_KEYS)
         end,
-        Tags
-    ) ++ [{<<"bundle-map">>, hb_util:encode(ManifestID)}].
+        TX#tx.tags
+    ),
+    TX#tx{tags = BundleTags ++ StrippedTags}.
 
 %% @doc Reset the data size of a data item. Assumes that the data is already normalized.
 normalize_data_size(Item = #tx{data = Bin}) when is_binary(Bin) ->