diff --git a/src/app/cli/src/cli_entrypoint/mina_cli_entrypoint.ml b/src/app/cli/src/cli_entrypoint/mina_cli_entrypoint.ml index 6927274ad02..12b3069ca61 100644 --- a/src/app/cli/src/cli_entrypoint/mina_cli_entrypoint.ml +++ b/src/app/cli/src/cli_entrypoint/mina_cli_entrypoint.ml @@ -399,15 +399,21 @@ let setup_daemon logger = dummy proofs (none)" and plugins = plugin_flag and precomputed_blocks_path = - flag "--precomputed-blocks-file" - ~aliases:[ "precomputed-blocks-file" ] + flag "--precomputed-blocks-path" + ~aliases:[ "precomputed-blocks-path" ] (optional string) - ~doc:"PATH Path to write precomputed blocks to, for replay or archiving" + ~doc: + "PATH Path to write precomputed blocks to, for replay or archiving. If \ + PATH is a directory, precomputed blocks will be logged to individual \ + files within this directory. Otherwise, they will be appended to the \ + same file." and log_precomputed_blocks = flag "--log-precomputed-blocks" ~aliases:[ "log-precomputed-blocks" ] (optional_with_default false bool) - ~doc:"true|false Include precomputed blocks in the log (default: false)" + ~doc: + "true|false Include precomputed blocks in the log (default: false). \ + See also --precomputed-block-path for additional functionality." and block_reward_threshold = flag "--minimum-block-reward" ~aliases:[ "minimum-block-reward" ] ~doc: diff --git a/src/lib/mina_lib/mina_lib.ml b/src/lib/mina_lib/mina_lib.ml index c7dd1da4d16..5e6f5c3297f 100644 --- a/src/lib/mina_lib/mina_lib.ml +++ b/src/lib/mina_lib/mina_lib.ml @@ -114,7 +114,7 @@ type t = ; subscriptions : Mina_subscriptions.t ; sync_status : Sync_status.t Mina_incremental.Status.Observer.t ; precomputed_block_writer : - ([ `Path of string ] option * [ `Log ] option) ref + ([ `Path of string | `Path_dir of string ] option * [ `Log ] option) ref ; block_production_status : [ `Producing | `Producing_in_ms of float | `Free ] ref ; vrf_evaluation_state : Block_producer.Vrf_evaluation_state.t @@ -2105,10 +2105,16 @@ let create ?wallets (config : Config.t) = ~precomputed_values:config.precomputed_values ~frontier_broadcast_pipe:frontier_broadcast_pipe_r archive_process_port ) ; + (* To log precomputed blocks to individual files, set both + --precomputed-blocks-path DIR and --log-precomputed-blocks true *) let precomputed_block_writer = ref ( Option.map config.precomputed_blocks_path ~f:(fun path -> - `Path path ) + match Core.Unix.(stat path).st_kind with + | S_DIR -> + `Path_dir path + | _ -> + `Path path ) , if config.log_precomputed_blocks then Some `Log else None ) in let subscriptions = @@ -2118,6 +2124,7 @@ let create ?wallets (config : Config.t) = ~is_storing_all:config.is_archive_rocksdb ~upload_blocks_to_gcloud:config.upload_blocks_to_gcloud ~time_controller:config.time_controller ~precomputed_block_writer + ~log_precomputed_blocks:config.log_precomputed_blocks in let open Mina_incremental.Status in let transition_frontier_incr = diff --git a/src/lib/mina_lib/mina_subscriptions.ml b/src/lib/mina_lib/mina_subscriptions.ml index f79d65f7f1e..83a22c93062 100644 --- a/src/lib/mina_lib/mina_subscriptions.ml +++ b/src/lib/mina_lib/mina_subscriptions.ml @@ -42,7 +42,7 @@ let add_new_subscription (t : t) ~pk = let create ~logger ~constraint_constants ~wallets ~new_blocks ~transition_frontier ~is_storing_all ~time_controller - ~upload_blocks_to_gcloud ~precomputed_block_writer = + ~upload_blocks_to_gcloud ~precomputed_block_writer ~log_precomputed_blocks = let subscribed_block_users = Optional_public_key.Table.of_alist_multi @@ List.map (Secrets.Wallets.pks wallets) ~f:(fun wallet -> @@ -101,13 +101,26 @@ let create ~logger ~constraint_constants ~wallets ~new_blocks let gcloud_keyfile = match Core.Sys.getenv "GCLOUD_KEYFILE" with | Some keyfile -> + if upload_blocks_to_gcloud then + [%log info] "GCLOUD_KEYFILE environment variable set to %s" keyfile ; Some keyfile | _ -> - [%log warn] - "GCLOUD_KEYFILE environment variable not set. Must be set to use \ - upload_blocks_to_gcloud" ; + if upload_blocks_to_gcloud then + [%log warn] + "GCLOUD_KEYFILE environment variable not set. Must be set to use \ + upload_blocks_to_gcloud" ; None in + Option.iter (fst !precomputed_block_writer) ~f:(fun path -> + match path with + | `Path_dir path -> + [%log info] + ~metadata:[ ("path", `String path) ] + "Precomputed blocks will be logged to individual files in $path" + | `Path path -> + [%log info] + ~metadata:[ ("path", `String path) ] + "Precomputed blocks will be logged to the same file $path" ) ; Option.iter gcloud_keyfile ~f:(fun path -> ignore ( Core.Sys.command @@ -118,7 +131,7 @@ let create ~logger ~constraint_constants ~wallets ~new_blocks let new_block = Mina_block.Validated.forget new_block_validated in let new_block_no_hash = With_hash.data new_block in let hash = State_hash.With_state_hashes.state_hash new_block in - (let path, log = !precomputed_block_writer in + (let path, _log = !precomputed_block_writer in match Broadcast_pipe.Reader.peek transition_frontier with | None -> [%log warn] @@ -158,8 +171,8 @@ let create ~logger ~constraint_constants ~wallets ~new_blocks ] ; Mina_block.Precomputed.to_yojson precomputed_block ) in - if upload_blocks_to_gcloud then ( - [%log info] "log" ; + (* Upload precomputed blocks to gcloud *) + ( if upload_blocks_to_gcloud then let json = Yojson.Safe.to_string (Lazy.force precomputed_block) in @@ -187,6 +200,13 @@ let create ~logger ~constraint_constants ~wallets ~new_blocks match (gcloud_keyfile, network, bucket) with | Some _, Some network, Some bucket -> let hash_string = State_hash.to_base58_check hash in + [%log info] + ~metadata: + [ ("hash", `String hash_string) + ; ("bucket", `String bucket) + ] + "Uploading precomputed block with $hash to gcloud \ + $bucket" ; let height = Mina_block.blockchain_length new_block_no_hash |> Mina_numbers.Length.to_string @@ -248,25 +268,73 @@ let create ~logger ~constraint_constants ~wallets ~new_blocks .upload_to_gcloud_blocks) ) | _ -> () ) ; - Option.iter path ~f:(fun (`Path path) -> - Out_channel.with_file ~append:true path - ~f:(fun out_channel -> - Out_channel.output_lines out_channel - [ Yojson.Safe.to_string - (Lazy.force precomputed_block) - ] ) ) ; - [%log info] "Saw block with state hash $state_hash" - ~metadata: - (let state_hash_data = - [ ( "state_hash" - , `String (State_hash.to_base58_check hash) ) - ] - in - if is_some log then - state_hash_data - @ [ ("precomputed_block", Lazy.force precomputed_block) - ] - else state_hash_data ) ) ) ; + (* Log precomputed blocks locally *) + Option.iter path ~f:(fun path -> + if log_precomputed_blocks then + let json = + Yojson.Safe.to_string (Lazy.force precomputed_block) + in + match path with + | `Path path -> + (* original logging functionality, appends to single file *) + Out_channel.with_file ~append:true path + ~f:(fun out_channel -> + Out_channel.output_lines out_channel [ json ] ) + | `Path_dir path -> ( + (* log precomputed blocks to individual files in the directory *) + let network = + match Core.Sys.getenv "NETWORK_NAME" with + | Some network -> + Some network + | _ -> + [%log warn] + "NETWORK_NAME environment variable not \ + set. Default to 'mainnet'" ; + Some "mainnet" + in + match network with + | Some network -> + let hash_string = + State_hash.to_base58_check hash + in + let height = + Mina_block.blockchain_length + new_block_no_hash + |> Mina_numbers.Length.to_string + in + let name = + sprintf "%s-%s-%s.json" network height + hash_string + in + let fpath = + Core.Filename.( + parts path @ [ name ] |> of_parts) + in + Out_channel.with_file ~append:false fpath + ~f:(fun out_channel -> + Out_channel.output_lines out_channel + [ json ] ) ; + [%log info] + ~metadata: + [ ("block", `String name) + ; ("path", `String path) + ; ( "time" + , `String + Time.( + now () + |> to_string_iso8601_basic + ~zone:Zone.utc) ) + ] + "Logged precomputed $block to $path at $time" + | None -> + () ) + else + [%log info] + ~metadata: + [ ( "state_hash" + , `String (State_hash.to_base58_check hash) ) + ] + "Saw block with state hash $state_hash" ) ) ) ; match Filtered_external_transition.validate_transactions ~constraint_constants new_block_no_hash