From 9ecf29e60072a7a92b71467970b1eca89a7e6370 Mon Sep 17 00:00:00 2001 From: Tang Jiawei Date: Fri, 22 Dec 2023 07:49:31 +0800 Subject: [PATCH 01/14] add timing logs to various functions --- src/app/archive/lib/processor.ml | 72 ++++++++++++++++++++------------ 1 file changed, 45 insertions(+), 27 deletions(-) diff --git a/src/app/archive/lib/processor.ml b/src/app/archive/lib/processor.ml index cbbef5942ae..04429246d01 100644 --- a/src/app/archive/lib/processor.ml +++ b/src/app/archive/lib/processor.ml @@ -2909,6 +2909,8 @@ module Block = struct (failed_str, Some display) in let%bind _seq_no = + Metrics.time ~label:"adding_transactions" + @@ fun () -> Mina_caqti.deferred_result_list_fold transactions ~init:0 ~f:(fun sequence_no -> function | { Mina_base.With_status.status @@ -3473,9 +3475,12 @@ module Block = struct then (* a new block, allows marking some pending blocks as canonical *) let%bind subchain_blocks = - get_subchain - (module Conn) - ~start_block_id:highest_canonical_block_id ~end_block_id:block_id + Metrics.time ~label:"get_subchain (> canonical_height + k)" + (fun () -> + get_subchain + (module Conn) + ~start_block_id:highest_canonical_block_id + ~end_block_id:block_id ) in let block_height_less_k_int64 = Int64.( - ) block.height k_int64 in (* mark canonical, orphaned blocks in subchain at least k behind the new block *) @@ -3483,38 +3488,45 @@ module Block = struct List.filter subchain_blocks ~f:(fun subchain_block -> Int64.( <= ) subchain_block.height block_height_less_k_int64 ) in - Mina_caqti.deferred_result_list_fold canonical_blocks ~init:() - ~f:(fun () block -> - let%bind () = - mark_as_canonical (module Conn) ~state_hash:block.state_hash - in - mark_as_orphaned - (module Conn) - ~state_hash:block.state_hash ~height:block.height ) + Metrics.time ~label:"mark_as_canonical (> canonical_height + k)" + (fun () -> + Mina_caqti.deferred_result_list_fold canonical_blocks ~init:() + ~f:(fun () block -> + let%bind () = + mark_as_canonical (module Conn) ~state_hash:block.state_hash + in + mark_as_orphaned + (module Conn) + ~state_hash:block.state_hash ~height:block.height ) ) else if Int64.( < ) block.height greatest_canonical_height then (* a missing block added in the middle of canonical chain *) let%bind canonical_block_above_id, _above_height = - get_nearest_canonical_block_above (module Conn) block.height + Metrics.time ~label:"get_nearest_canonical_block_above" (fun () -> + get_nearest_canonical_block_above (module Conn) block.height ) in let%bind canonical_block_below_id, _below_height = - get_nearest_canonical_block_below (module Conn) block.height + Metrics.time ~label:"get_neareast_canonical_block_below" (fun () -> + get_nearest_canonical_block_below (module Conn) block.height ) in (* we can always find this chain: the genesis block should be marked as canonical, and we've found a canonical block above this one *) let%bind canonical_blocks = - get_subchain - (module Conn) - ~start_block_id:canonical_block_below_id - ~end_block_id:canonical_block_above_id + Metrics.time ~label:"get_subchain (< canonical_height)" (fun () -> + get_subchain + (module Conn) + ~start_block_id:canonical_block_below_id + ~end_block_id:canonical_block_above_id ) in - Mina_caqti.deferred_result_list_fold canonical_blocks ~init:() - ~f:(fun () block -> - let%bind () = - mark_as_canonical (module Conn) ~state_hash:block.state_hash - in - mark_as_orphaned - (module Conn) - ~state_hash:block.state_hash ~height:block.height ) + Metrics.time ~label:"mark_as_canonical (< canonical_height)" + (fun () -> + Mina_caqti.deferred_result_list_fold canonical_blocks ~init:() + ~f:(fun () block -> + let%bind () = + mark_as_canonical (module Conn) ~state_hash:block.state_hash + in + mark_as_orphaned + (module Conn) + ~state_hash:block.state_hash ~height:block.height ) ) else (* a block at or above highest canonical block, not high enough to mark any blocks as canonical *) Deferred.Result.return () @@ -3644,7 +3656,10 @@ let add_block_aux ?(retries = 3) ~logger ~pool ~add_block ~hash [%log info] "Attempting to add block data for $state_hash" ~metadata: [ ("state_hash", Mina_base.State_hash.to_yojson state_hash) ] ; - let%bind block_id = add_block (module Conn : CONNECTION) block in + let%bind block_id = + Metrics.time ~label:"add_block" (fun () -> + add_block (module Conn : CONNECTION) block ) + in (* if an existing block has a parent hash that's for the block just added, set its parent id *) @@ -3654,7 +3669,10 @@ let add_block_aux ?(retries = 3) ~logger ~pool ~add_block ~hash ~parent_hash:(hash block) ~parent_id:block_id in (* update chain status for existing blocks *) - let%bind () = Block.update_chain_status (module Conn) ~block_id in + let%bind () = + Metrics.time ~label:"update_chain_status" (fun () -> + Block.update_chain_status (module Conn) ~block_id ) + in let%bind () = match delete_older_than with | Some num_blocks -> From 79d9cda0d19f279f6d8ff33b742bec2589a1e732 Mon Sep 17 00:00:00 2001 From: Tang Jiawei Date: Fri, 22 Dec 2023 10:19:37 +0800 Subject: [PATCH 02/14] add more logs to various transactions --- src/app/archive/lib/processor.ml | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/src/app/archive/lib/processor.ml b/src/app/archive/lib/processor.ml index 04429246d01..7f238cf4268 100644 --- a/src/app/archive/lib/processor.ml +++ b/src/app/archive/lib/processor.ml @@ -2920,6 +2920,8 @@ module Block = struct { Mina_base.With_status.status; data = command } in let%bind id = + Metrics.time ~label:"user_command.add_if_doesn't_exist" + @@ fun () -> User_command.add_if_doesn't_exist (module Conn) user_command.data @@ -2927,6 +2929,9 @@ module Block = struct let%map () = match command with | Signed_command _ -> + Metrics.time + ~label:"block_and_signed_command.add_with_status" + @@ fun () -> Block_and_signed_command.add_with_status (module Conn) ~block_id ~user_command_id:id ~sequence_no @@ -2936,6 +2941,9 @@ module Block = struct let status, failure_reasons = failure_reasons user_command.status in + Metrics.time + ~label:"block_and_zkapp_command.add_if_doesn't_exist" + @@ fun () -> Block_and_zkapp_command.add_if_doesn't_exist (module Conn) ~block_id ~zkapp_command_id:id ~sequence_no ~status @@ -2944,6 +2952,8 @@ module Block = struct in sequence_no + 1 | { data = Fee_transfer fee_transfer_bundled; status } -> + Metrics.time ~label:"fee_transfer.add" + @@ fun () -> let fee_transfers = Mina_base.Fee_transfer.to_numbered_list fee_transfer_bundled in @@ -3046,6 +3056,8 @@ module Block = struct in sequence_no + 1 | { data = Coinbase coinbase; status } -> + Metrics.time ~label:"conbase.add" + @@ fun () -> let fee_transfer_via_coinbase = Mina_base.Coinbase.fee_transfer coinbase in From a9347f5e66a42474fa09eb43179722acd2199b6a Mon Sep 17 00:00:00 2001 From: Tang Jiawei Date: Fri, 22 Dec 2023 10:28:47 +0800 Subject: [PATCH 03/14] add more logs for zkapp commands --- src/app/archive/lib/processor.ml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/app/archive/lib/processor.ml b/src/app/archive/lib/processor.ml index 7f238cf4268..efd68463f13 100644 --- a/src/app/archive/lib/processor.ml +++ b/src/app/archive/lib/processor.ml @@ -1600,6 +1600,8 @@ module Zkapp_account_update = struct (account_update : Account_update.Simple.t) = let open Deferred.Result.Let_syntax in let%bind body_id = + Metrics.time ~label:"Zkapp_account_update_body.add" + @@ fun () -> Zkapp_account_update_body.add_if_doesn't_exist (module Conn) account_update.body @@ -1897,11 +1899,15 @@ module User_command = struct let open Deferred.Result.Let_syntax in let zkapp_command = Zkapp_command.to_simple ps in let%bind zkapp_fee_payer_body_id = + Metrics.time ~label:"Zkapp_fee_payer_body.add" + @@ fun () -> Zkapp_fee_payer_body.add_if_doesn't_exist (module Conn) zkapp_command.fee_payer.body in let%bind zkapp_account_updates_ids = + Metrics.time ~label:"Zkapp_account_update.add" + @@ fun () -> Mina_caqti.deferred_result_list_map zkapp_command.account_updates ~f:(Zkapp_account_update.add_if_doesn't_exist (module Conn)) >>| Array.of_list From 4be459ca601f64d3145f63c32612b4a841a2f270 Mon Sep 17 00:00:00 2001 From: Tang Jiawei Date: Fri, 22 Dec 2023 11:35:19 +0800 Subject: [PATCH 04/14] add more logs to update body --- src/app/archive/lib/processor.ml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/app/archive/lib/processor.ml b/src/app/archive/lib/processor.ml index efd68463f13..118ccf6404b 100644 --- a/src/app/archive/lib/processor.ml +++ b/src/app/archive/lib/processor.ml @@ -1491,13 +1491,16 @@ module Zkapp_account_update_body = struct Account_identifiers.add_if_doesn't_exist (module Conn) account_identifier in let%bind update_id = + Metrics.time ~label:"zkapp_updates.add" @@ fun () -> Zkapp_updates.add_if_doesn't_exist (module Conn) body.update in let increment_nonce = body.increment_nonce in let%bind events_id = + Metrics.time ~label:"Zkapp_events.add" @@ fun () -> Zkapp_events.add_if_doesn't_exist (module Conn) body.events in let%bind actions_id = + Metrics.time ~label:"Zkapp_actions.add" @@ fun () -> Zkapp_events.add_if_doesn't_exist (module Conn) body.actions in let%bind call_data_id = From d9f961c1b2cfccde6a15f198958cbd6399549332 Mon Sep 17 00:00:00 2001 From: Tang Jiawei Date: Thu, 4 Jan 2024 00:50:02 +0800 Subject: [PATCH 05/14] add more logs --- src/app/archive/lib/processor.ml | 17 +++++++++++------ src/lib/mina_caqti/dune | 1 + src/lib/mina_caqti/mina_caqti.ml | 13 ++++++++++++- 3 files changed, 24 insertions(+), 7 deletions(-) diff --git a/src/app/archive/lib/processor.ml b/src/app/archive/lib/processor.ml index 118ccf6404b..c0729abd332 100644 --- a/src/app/archive/lib/processor.ml +++ b/src/app/archive/lib/processor.ml @@ -287,10 +287,13 @@ module Zkapp_field_array = struct (fps : Pickles.Backend.Tick.Field.t array) = let open Deferred.Result.Let_syntax in let%bind (element_ids : int array) = + Metrics.time ~label:"zkapp_field.add" + @@ fun () -> Mina_caqti.deferred_result_list_map (Array.to_list fps) ~f:(Zkapp_field.add_if_doesn't_exist (module Conn)) >>| Array.of_list in + Metrics.time ~label:"select_insert_into zkapp_field_array" @@ fun () -> Mina_caqti.select_insert_into_cols ~select:("id", Caqti_type.int) ~table_name ~cols:([ "element_ids" ], Mina_caqti.array_int_typ) @@ -1421,6 +1424,8 @@ module Zkapp_events = struct (events : Account_update.Body.Events'.t) = let open Deferred.Result.Let_syntax in let%bind (element_ids : int array) = + Metrics.time ~label:"zkapp_field_array.add" + @@ fun () -> Mina_caqti.deferred_result_list_map events ~f:(Zkapp_field_array.add_if_doesn't_exist (module Conn)) >>| Array.of_list @@ -1491,17 +1496,17 @@ module Zkapp_account_update_body = struct Account_identifiers.add_if_doesn't_exist (module Conn) account_identifier in let%bind update_id = - Metrics.time ~label:"zkapp_updates.add" @@ fun () -> - Zkapp_updates.add_if_doesn't_exist (module Conn) body.update + Metrics.time ~label:"zkapp_updates.add" + @@ fun () -> Zkapp_updates.add_if_doesn't_exist (module Conn) body.update in let increment_nonce = body.increment_nonce in let%bind events_id = - Metrics.time ~label:"Zkapp_events.add" @@ fun () -> - Zkapp_events.add_if_doesn't_exist (module Conn) body.events + Metrics.time ~label:"Zkapp_events.add" + @@ fun () -> Zkapp_events.add_if_doesn't_exist (module Conn) body.events in let%bind actions_id = - Metrics.time ~label:"Zkapp_actions.add" @@ fun () -> - Zkapp_events.add_if_doesn't_exist (module Conn) body.actions + Metrics.time ~label:"Zkapp_actions.add" + @@ fun () -> Zkapp_events.add_if_doesn't_exist (module Conn) body.actions in let%bind call_data_id = Zkapp_field.add_if_doesn't_exist (module Conn) body.call_data diff --git a/src/lib/mina_caqti/dune b/src/lib/mina_caqti/dune index 5d40f466222..0b39f002674 100644 --- a/src/lib/mina_caqti/dune +++ b/src/lib/mina_caqti/dune @@ -11,6 +11,7 @@ async_kernel ;; local libraries mina_base + logger ) (preprocess (pps ppx_mina ppx_version ppx_jane ppx_custom_printf h_list.ppx)) diff --git a/src/lib/mina_caqti/mina_caqti.ml b/src/lib/mina_caqti/mina_caqti.ml index 8377968c98f..693e9ae9fe1 100644 --- a/src/lib/mina_caqti/mina_caqti.ml +++ b/src/lib/mina_caqti/mina_caqti.ml @@ -282,20 +282,31 @@ let select_insert_into_cols ~(select : string * 'select Caqti_type.t) ~(table_name : string) ?tannot ~(cols : string list * 'cols Caqti_type.t) (module Conn : CONNECTION) (value : 'cols) = let open Deferred.Result.Let_syntax in + let start = Time.now () in Conn.find_opt ( Caqti_request.find_opt (snd cols) (snd select) @@ select_cols ~select:(fst select) ~table_name ?tannot ~cols:(fst cols) () ) value - >>= function + >>= fun res -> + let search_time = Time.now () in + [%log' info (Logger.create ())] + "select from table takes %s" Time.(Span.to_string_hum (diff search_time start)) ; + match res with | Some id -> return id | None -> + let%map res = Conn.find ( Caqti_request.find (snd cols) (snd select) @@ insert_into_cols ~returning:(fst select) ~table_name ?tannot ~cols:(fst cols) () ) value + in + let insert_time = Time.now () in + [%log' info (Logger.create ())] + "insert into table takes %s" Time.(Span.to_string_hum (diff insert_time search_time)) ; + res let query ~f pool = match%bind Caqti_async.Pool.use f pool with From 55433d3fa6e799458159b4fae7f282307da3f505 Mon Sep 17 00:00:00 2001 From: Tang Jiawei Date: Thu, 4 Jan 2024 00:53:47 +0800 Subject: [PATCH 06/14] add idx for element_ids --- src/app/archive/zkapp_tables.sql | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/app/archive/zkapp_tables.sql b/src/app/archive/zkapp_tables.sql index 03445a146aa..f9d71b076b5 100644 --- a/src/app/archive/zkapp_tables.sql +++ b/src/app/archive/zkapp_tables.sql @@ -33,6 +33,8 @@ CREATE TABLE zkapp_field_array , element_ids int[] NOT NULL ); +create INDEX idx_zkapp_field_array_element_ids ON zkapp_field_array(element_ids); + /* Fixed-width arrays of algebraic fields, given as id's from zkapp_field From 153bfe4f2e54407866605d4d6cc9d27c50fe80aa Mon Sep 17 00:00:00 2001 From: Tang Jiawei Date: Thu, 4 Jan 2024 07:08:36 +0800 Subject: [PATCH 07/14] add an index for zkapp_events --- src/app/archive/zkapp_tables.sql | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/app/archive/zkapp_tables.sql b/src/app/archive/zkapp_tables.sql index f9d71b076b5..3219004ca71 100644 --- a/src/app/archive/zkapp_tables.sql +++ b/src/app/archive/zkapp_tables.sql @@ -83,6 +83,9 @@ CREATE TABLE zkapp_events , element_ids int[] NOT NULL ); +create index idx_zkapp_events_element_ids ON zkapp_events(element_ids); + + /* field elements derived from verification keys */ CREATE TABLE zkapp_verification_key_hashes ( id serial PRIMARY KEY From c64cba95ca40ba471f8fc67aac4fa6ecb6e2160d Mon Sep 17 00:00:00 2001 From: Tang Jiawei Date: Fri, 5 Jan 2024 06:14:28 +0800 Subject: [PATCH 08/14] batch insertion of events and actions --- src/app/archive/lib/processor.ml | 92 +++++++++++++++++++++++++++++--- src/app/archive/zkapp_tables.sql | 4 +- 2 files changed, 88 insertions(+), 8 deletions(-) diff --git a/src/app/archive/lib/processor.ml b/src/app/archive/lib/processor.ml index cbbef5942ae..7cf9077926e 100644 --- a/src/app/archive/lib/processor.ml +++ b/src/app/archive/lib/processor.ml @@ -1417,20 +1417,97 @@ module Zkapp_events = struct let table_name = "zkapp_events" + let sep_by_comma ?(parenthesis = false) xs = + List.map xs ~f:(if parenthesis then sprintf "('%s')" else sprintf "'%s'") + |> String.concat ~sep:", " + let add_if_doesn't_exist (module Conn : CONNECTION) (events : Account_update.Body.Events'.t) = let open Deferred.Result.Let_syntax in - let%bind (element_ids : int array) = - Mina_caqti.deferred_result_list_map events - ~f:(Zkapp_field_array.add_if_doesn't_exist (module Conn)) - >>| Array.of_list + let%bind field_array_id_list = + if not @@ List.is_empty events then + let field_list_list = + List.map events ~f:(fun field_array -> + Array.map field_array ~f:Pickles.Backend.Tick.Field.to_string + |> Array.to_list ) + in + let fields = field_list_list |> List.concat in + let%bind field_id_list_list = + if not @@ List.is_empty fields then + let field_insert = + sprintf + {sql| INSERT INTO zkapp_field (field) VALUES %s + ON CONFLICT (field) + DO NOTHING |sql} + (sep_by_comma ~parenthesis:true fields) + in + let%bind () = + Conn.exec (Caqti_request.exec Caqti_type.unit field_insert) () + in + let field_search = + sprintf + {sql| SELECT field, id FROM zkapp_field + WHERE field in (%s) |sql} + @@ sep_by_comma fields + in + let%map field_map = + Conn.collect_list + (Caqti_request.collect Caqti_type.unit + Caqti_type.(tup2 string int) + field_search ) + () + >>| String.Map.of_alist_exn + in + + List.map field_list_list ~f:(List.map ~f:(Map.find_exn field_map)) + else return @@ List.map field_list_list ~f:(fun _ -> []) + in + let field_array_list = + List.map field_id_list_list ~f:(fun id_list -> + List.map id_list ~f:Int.to_string + |> String.concat ~sep:", " |> sprintf "{%s}" ) + in + let field_array_insert = + sprintf + {sql| INSERT INTO zkapp_field_array (element_ids) VALUES %s + ON CONFLICT (element_ids) + DO NOTHING |sql} + @@ sep_by_comma ~parenthesis:true field_array_list + in + let%bind () = + Conn.exec (Caqti_request.exec Caqti_type.unit field_array_insert) () + in + let field_array_search = + sprintf + {sql| SELECT element_ids, id FROM zkapp_field_array + WHERE element_ids in (%s) |sql} + @@ sep_by_comma field_array_list + in + let module Field_array_map = Map.Make (struct + type t = int array [@@deriving sexp] + + let compare = Array.compare Int.compare + end) in + let%map field_array_map = + Conn.collect_list + (Caqti_request.collect Caqti_type.unit + Caqti_type.(tup2 Mina_caqti.array_int_typ int) + field_array_search ) + () + >>| Field_array_map.of_alist_exn + in + + List.map field_id_list_list ~f:(fun field_id_list -> + Map.find_exn field_array_map (Array.of_list field_id_list) ) + |> Array.of_list + else return @@ Array.of_list [] in Mina_caqti.select_insert_into_cols ~select:("id", Caqti_type.int) ~table_name ~cols:([ "element_ids" ], Mina_caqti.array_int_typ) ~tannot:(function "element_ids" -> Some "int[]" | _ -> None) (module Conn) - element_ids + field_array_id_list let load (module Conn : CONNECTION) id = Conn.find @@ -3644,7 +3721,10 @@ let add_block_aux ?(retries = 3) ~logger ~pool ~add_block ~hash [%log info] "Attempting to add block data for $state_hash" ~metadata: [ ("state_hash", Mina_base.State_hash.to_yojson state_hash) ] ; - let%bind block_id = add_block (module Conn : CONNECTION) block in + let%bind block_id = + Metrics.time ~label:"add_block" + @@ fun () -> add_block (module Conn : CONNECTION) block + in (* if an existing block has a parent hash that's for the block just added, set its parent id *) diff --git a/src/app/archive/zkapp_tables.sql b/src/app/archive/zkapp_tables.sql index 3219004ca71..a7b0073e940 100644 --- a/src/app/archive/zkapp_tables.sql +++ b/src/app/archive/zkapp_tables.sql @@ -30,7 +30,7 @@ CREATE TABLE zkapp_field */ CREATE TABLE zkapp_field_array ( id serial PRIMARY KEY -, element_ids int[] NOT NULL +, element_ids int[] NOT NULL UNIQUE ); create INDEX idx_zkapp_field_array_element_ids ON zkapp_field_array(element_ids); @@ -80,7 +80,7 @@ CREATE TABLE zkapp_action_states */ CREATE TABLE zkapp_events ( id serial PRIMARY KEY -, element_ids int[] NOT NULL +, element_ids int[] NOT NULL UNIQUE ); create index idx_zkapp_events_element_ids ON zkapp_events(element_ids); From 2fb21f6b725546bef00c06ab079da47db9f5443d Mon Sep 17 00:00:00 2001 From: Tang Jiawei Date: Fri, 5 Jan 2024 22:48:04 +0800 Subject: [PATCH 09/14] add comments --- src/app/archive/lib/processor.ml | 38 ++++++++++++++++++++++++++------ src/lib/mina_caqti/mina_caqti.ml | 16 +++++++------- 2 files changed, 39 insertions(+), 15 deletions(-) diff --git a/src/app/archive/lib/processor.ml b/src/app/archive/lib/processor.ml index b26481e7b26..5b48574f331 100644 --- a/src/app/archive/lib/processor.ml +++ b/src/app/archive/lib/processor.ml @@ -293,7 +293,8 @@ module Zkapp_field_array = struct ~f:(Zkapp_field.add_if_doesn't_exist (module Conn)) >>| Array.of_list in - Metrics.time ~label:"select_insert_into zkapp_field_array" @@ fun () -> + Metrics.time ~label:"select_insert_into zkapp_field_array" + @@ fun () -> Mina_caqti.select_insert_into_cols ~select:("id", Caqti_type.int) ~table_name ~cols:([ "element_ids" ], Mina_caqti.array_int_typ) @@ -1424,6 +1425,22 @@ module Zkapp_events = struct List.map xs ~f:(if parenthesis then sprintf "('%s')" else sprintf "'%s'") |> String.concat ~sep:", " + (* Account_update.Body.Events'.t is defined as `field array list`, + which is ismorphic to a list of list of fields. + + We are batching the insertion of field and field_array to optimize + the speed of archiving max-cost zkapps. + + 1. we flatten the list of list of fields to get all the field elements + 2. insert all the field elements in one query + 3. construct a map "M" from `field_id` to `field` by querying against the zkapp_field table + 4. use "M" and the list of list of fields to compute the list of list of field_ids + 5. insert all list of `list of field_ids` in one query + 6. construct a map "M'" from `field_array_id` to `field_id array` by querying against + the zkapp_field_array table + 7. use "M'" and the list of list of field_ids to compute the list of field_array_ids + 8. insert the list of field_arrays + *) let add_if_doesn't_exist (module Conn : CONNECTION) (events : Account_update.Body.Events'.t) = let open Deferred.Result.Let_syntax in @@ -1461,9 +1478,13 @@ module Zkapp_events = struct () >>| String.Map.of_alist_exn in - - List.map field_list_list ~f:(List.map ~f:(Map.find_exn field_map)) - else return @@ List.map field_list_list ~f:(fun _ -> []) + let field_id_list_list = + List.map field_list_list ~f:(List.map ~f:(Map.find_exn field_map)) + in + field_id_list_list + else + (* if there's no fields, then we must have some list of empty lists *) + return @@ List.map field_list_list ~f:(fun _ -> []) in let field_array_list = List.map field_id_list_list ~f:(fun id_list -> @@ -1499,9 +1520,12 @@ module Zkapp_events = struct () >>| Field_array_map.of_alist_exn in - List.map field_id_list_list ~f:(fun field_id_list -> - Map.find_exn field_array_map (Array.of_list field_id_list) ) - |> Array.of_list + let field_array_id_list = + List.map field_id_list_list ~f:(fun field_id_list -> + Map.find_exn field_array_map (Array.of_list field_id_list) ) + |> Array.of_list + in + field_array_id_list else return @@ Array.of_list [] in Mina_caqti.select_insert_into_cols ~select:("id", Caqti_type.int) diff --git a/src/lib/mina_caqti/mina_caqti.ml b/src/lib/mina_caqti/mina_caqti.ml index 2534123f1d0..c0831442703 100644 --- a/src/lib/mina_caqti/mina_caqti.ml +++ b/src/lib/mina_caqti/mina_caqti.ml @@ -288,18 +288,18 @@ let select_insert_into_cols ~(select : string * 'select Caqti_type.t) ) value >>= fun res -> - match res with + match res with | Some id -> return id | None -> let%map res = - Conn.find - ( Caqti_request.find (snd cols) (snd select) - @@ insert_into_cols ~returning:(fst select) ~table_name ?tannot - ~cols:(fst cols) () ) - value - in - res + Conn.find + ( Caqti_request.find (snd cols) (snd select) + @@ insert_into_cols ~returning:(fst select) ~table_name ?tannot + ~cols:(fst cols) () ) + value + in + res let query ~f pool = match%bind Caqti_async.Pool.use f pool with From 49874eafcac8fb44c61625dda4553e40e86a092c Mon Sep 17 00:00:00 2001 From: Tang Jiawei Date: Sat, 6 Jan 2024 03:01:04 +0800 Subject: [PATCH 10/14] remove unnecessary logs --- src/app/archive/lib/processor.ml | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/src/app/archive/lib/processor.ml b/src/app/archive/lib/processor.ml index 5b48574f331..5b8b2d615fc 100644 --- a/src/app/archive/lib/processor.ml +++ b/src/app/archive/lib/processor.ml @@ -287,14 +287,10 @@ module Zkapp_field_array = struct (fps : Pickles.Backend.Tick.Field.t array) = let open Deferred.Result.Let_syntax in let%bind (element_ids : int array) = - Metrics.time ~label:"zkapp_field.add" - @@ fun () -> Mina_caqti.deferred_result_list_map (Array.to_list fps) ~f:(Zkapp_field.add_if_doesn't_exist (module Conn)) >>| Array.of_list in - Metrics.time ~label:"select_insert_into zkapp_field_array" - @@ fun () -> Mina_caqti.select_insert_into_cols ~select:("id", Caqti_type.int) ~table_name ~cols:([ "element_ids" ], Mina_caqti.array_int_typ) @@ -3041,9 +3037,6 @@ module Block = struct let%map () = match command with | Signed_command _ -> - Metrics.time - ~label:"block_and_signed_command.add_with_status" - @@ fun () -> Block_and_signed_command.add_with_status (module Conn) ~block_id ~user_command_id:id ~sequence_no @@ -3064,8 +3057,6 @@ module Block = struct in sequence_no + 1 | { data = Fee_transfer fee_transfer_bundled; status } -> - Metrics.time ~label:"fee_transfer.add" - @@ fun () -> let fee_transfers = Mina_base.Fee_transfer.to_numbered_list fee_transfer_bundled in @@ -3168,8 +3159,6 @@ module Block = struct in sequence_no + 1 | { data = Coinbase coinbase; status } -> - Metrics.time ~label:"conbase.add" - @@ fun () -> let fee_transfer_via_coinbase = Mina_base.Coinbase.fee_transfer coinbase in From 6e5f053d0eb7537a242a6c30b46bc68681de2147 Mon Sep 17 00:00:00 2001 From: Tang Jiawei Date: Sat, 6 Jan 2024 03:55:46 +0800 Subject: [PATCH 11/14] refactor sql code into `mina_caqti` --- src/app/archive/lib/processor.ml | 64 +++++++------------------------- src/lib/mina_caqti/mina_caqti.ml | 46 ++++++++++++++++++----- 2 files changed, 50 insertions(+), 60 deletions(-) diff --git a/src/app/archive/lib/processor.ml b/src/app/archive/lib/processor.ml index 5b8b2d615fc..c4ae85ed7e4 100644 --- a/src/app/archive/lib/processor.ml +++ b/src/app/archive/lib/processor.ml @@ -1417,9 +1417,11 @@ module Zkapp_events = struct let table_name = "zkapp_events" - let sep_by_comma ?(parenthesis = false) xs = - List.map xs ~f:(if parenthesis then sprintf "('%s')" else sprintf "'%s'") - |> String.concat ~sep:", " + module Field_array_map = Map.Make (struct + type t = int array [@@deriving sexp] + + let compare = Array.compare Int.compare + end) (* Account_update.Body.Events'.t is defined as `field array list`, which is ismorphic to a list of list of fields. @@ -1450,28 +1452,11 @@ module Zkapp_events = struct let fields = field_list_list |> List.concat in let%bind field_id_list_list = if not @@ List.is_empty fields then - let field_insert = - sprintf - {sql| INSERT INTO zkapp_field (field) VALUES %s - ON CONFLICT (field) - DO NOTHING |sql} - (sep_by_comma ~parenthesis:true fields) - in - let%bind () = - Conn.exec (Caqti_request.exec Caqti_type.unit field_insert) () - in - let field_search = - sprintf - {sql| SELECT field, id FROM zkapp_field - WHERE field in (%s) |sql} - @@ sep_by_comma fields - in let%map field_map = - Conn.collect_list - (Caqti_request.collect Caqti_type.unit - Caqti_type.(tup2 string int) - field_search ) - () + Mina_caqti.insert_multi_into_col ~table_name:"zkapp_field" + ~col:("field", Caqti_type.string) + (module Conn) + fields >>| String.Map.of_alist_exn in let field_id_list_list = @@ -1482,38 +1467,17 @@ module Zkapp_events = struct (* if there's no fields, then we must have some list of empty lists *) return @@ List.map field_list_list ~f:(fun _ -> []) in + (* this conversion should be done by caqti using `typ`, FIX this in the future *) let field_array_list = List.map field_id_list_list ~f:(fun id_list -> List.map id_list ~f:Int.to_string |> String.concat ~sep:", " |> sprintf "{%s}" ) in - let field_array_insert = - sprintf - {sql| INSERT INTO zkapp_field_array (element_ids) VALUES %s - ON CONFLICT (element_ids) - DO NOTHING |sql} - @@ sep_by_comma ~parenthesis:true field_array_list - in - let%bind () = - Conn.exec (Caqti_request.exec Caqti_type.unit field_array_insert) () - in - let field_array_search = - sprintf - {sql| SELECT element_ids, id FROM zkapp_field_array - WHERE element_ids in (%s) |sql} - @@ sep_by_comma field_array_list - in - let module Field_array_map = Map.Make (struct - type t = int array [@@deriving sexp] - - let compare = Array.compare Int.compare - end) in let%map field_array_map = - Conn.collect_list - (Caqti_request.collect Caqti_type.unit - Caqti_type.(tup2 Mina_caqti.array_int_typ int) - field_array_search ) - () + Mina_caqti.insert_multi_into_col ~table_name:"zkapp_field_array" + ~col:("element_ids", Mina_caqti.array_int_typ) + (module Conn) + field_array_list >>| Field_array_map.of_alist_exn in let field_array_id_list = diff --git a/src/lib/mina_caqti/mina_caqti.ml b/src/lib/mina_caqti/mina_caqti.ml index c0831442703..b1e49d9bcb8 100644 --- a/src/lib/mina_caqti/mina_caqti.ml +++ b/src/lib/mina_caqti/mina_caqti.ml @@ -287,19 +287,45 @@ let select_insert_into_cols ~(select : string * 'select Caqti_type.t) @@ select_cols ~select:(fst select) ~table_name ?tannot ~cols:(fst cols) () ) value - >>= fun res -> - match res with + >>= function | Some id -> return id | None -> - let%map res = - Conn.find - ( Caqti_request.find (snd cols) (snd select) - @@ insert_into_cols ~returning:(fst select) ~table_name ?tannot - ~cols:(fst cols) () ) - value - in - res + Conn.find + ( Caqti_request.find (snd cols) (snd select) + @@ insert_into_cols ~returning:(fst select) ~table_name ?tannot + ~cols:(fst cols) () ) + value + +let sep_by_comma ?(parenthesis = false) xs = + List.map xs ~f:(if parenthesis then sprintf "('%s')" else sprintf "'%s'") + |> String.concat ~sep:", " + +let insert_multi_into_col ~(table_name : string) + ~(col : string * 'col Caqti_type.t) (module Conn : CONNECTION) + (values : string list) = + let open Deferred.Result.Let_syntax in + let insert = + sprintf + {sql| INSERT INTO %s (%s) VALUES %s + ON CONFLICT (%s) + DO NOTHING |sql} + table_name (fst col) + (sep_by_comma ~parenthesis:true values) + (fst col) + in + let%bind () = Conn.exec (Caqti_request.exec Caqti_type.unit insert) () in + let search = + sprintf + {sql| SELECT %s, id FROM %s + WHERE %s in (%s) |sql} + (fst col) table_name (fst col) (sep_by_comma values) + in + Conn.collect_list + (Caqti_request.collect Caqti_type.unit + Caqti_type.(tup2 (snd col) int) + search ) + () let query ~f pool = match%bind Caqti_async.Pool.use f pool with From 8734f96b4c0d23e925bc4df02bc3f28d8e4465d7 Mon Sep 17 00:00:00 2001 From: Tang Jiawei Date: Sat, 6 Jan 2024 03:57:38 +0800 Subject: [PATCH 12/14] clean up --- src/app/archive/zkapp_tables.sql | 5 ----- src/lib/mina_caqti/dune | 1 - 2 files changed, 6 deletions(-) diff --git a/src/app/archive/zkapp_tables.sql b/src/app/archive/zkapp_tables.sql index a7b0073e940..da225171d9f 100644 --- a/src/app/archive/zkapp_tables.sql +++ b/src/app/archive/zkapp_tables.sql @@ -33,8 +33,6 @@ CREATE TABLE zkapp_field_array , element_ids int[] NOT NULL UNIQUE ); -create INDEX idx_zkapp_field_array_element_ids ON zkapp_field_array(element_ids); - /* Fixed-width arrays of algebraic fields, given as id's from zkapp_field @@ -83,9 +81,6 @@ CREATE TABLE zkapp_events , element_ids int[] NOT NULL UNIQUE ); -create index idx_zkapp_events_element_ids ON zkapp_events(element_ids); - - /* field elements derived from verification keys */ CREATE TABLE zkapp_verification_key_hashes ( id serial PRIMARY KEY diff --git a/src/lib/mina_caqti/dune b/src/lib/mina_caqti/dune index 0b39f002674..5d40f466222 100644 --- a/src/lib/mina_caqti/dune +++ b/src/lib/mina_caqti/dune @@ -11,7 +11,6 @@ async_kernel ;; local libraries mina_base - logger ) (preprocess (pps ppx_mina ppx_version ppx_jane ppx_custom_printf h_list.ppx)) From 9fb47bb59fc988b8058d9242a9fbc4c063c2658a Mon Sep 17 00:00:00 2001 From: Tang Jiawei Date: Sat, 6 Jan 2024 04:46:07 +0800 Subject: [PATCH 13/14] add sql file to patch the database --- src/app/archive/add_unique_constraints.sql | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 src/app/archive/add_unique_constraints.sql diff --git a/src/app/archive/add_unique_constraints.sql b/src/app/archive/add_unique_constraints.sql new file mode 100644 index 00000000000..96edc1ef184 --- /dev/null +++ b/src/app/archive/add_unique_constraints.sql @@ -0,0 +1,2 @@ +ALTER TABLE zkapp_field_array ADD CONSTRAINT zkapp_field_array_element_ids_key UNIQUE (element_ids); +ALTER TABLE zkapp_events ADD CONSTRAINT zkapp_events_element_ids_key UNIQUE (element_ids); \ No newline at end of file From db927546273ca2bb738de0b9fbf32aa9b36b5af8 Mon Sep 17 00:00:00 2001 From: Tang Jiawei Date: Mon, 8 Jan 2024 21:51:41 +0800 Subject: [PATCH 14/14] add tracing of processor.add_block --- src/app/archive/lib/dune | 1 + src/app/archive/lib/processor.ml | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/app/archive/lib/dune b/src/app/archive/lib/dune index 385408aca97..b5897d8c6fa 100644 --- a/src/app/archive/lib/dune +++ b/src/app/archive/lib/dune @@ -74,6 +74,7 @@ error_json ppx_deriving_yojson.runtime ppx_version.runtime + o1trace ) (inline_tests (flags -verbose -show-counts)) (modes native) diff --git a/src/app/archive/lib/processor.ml b/src/app/archive/lib/processor.ml index c4ae85ed7e4..eb7744c84cc 100644 --- a/src/app/archive/lib/processor.ml +++ b/src/app/archive/lib/processor.ml @@ -2992,8 +2992,6 @@ module Block = struct { Mina_base.With_status.status; data = command } in let%bind id = - Metrics.time ~label:"user_command.add_if_doesn't_exist" - @@ fun () -> User_command.add_if_doesn't_exist (module Conn) user_command.data @@ -3734,6 +3732,8 @@ let add_block_aux ?(retries = 3) ~logger ~pool ~add_block ~hash ~metadata: [ ("state_hash", Mina_base.State_hash.to_yojson state_hash) ] ; let%bind block_id = + O1trace.thread "archive_processor.add_block" + @@ fun () -> Metrics.time ~label:"add_block" @@ fun () -> add_block (module Conn : CONNECTION) block in