Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

(precomputed blocks) Add local logging to individual files #1

Draft
wants to merge 1 commit into
base: develop
Choose a base branch
from
Draft

(precomputed blocks) Add local logging to individual files #1

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 10 additions & 4 deletions src/app/cli/src/cli_entrypoint/mina_cli_entrypoint.ml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -399,15 +399,21 @@ let setup_daemon logger =
dummy proofs (none)"
and plugins = plugin_flag
and precomputed_blocks_path =
flag "--precomputed-blocks-file"
~aliases:[ "precomputed-blocks-file" ]
flag "--precomputed-blocks-path"
~aliases:[ "precomputed-blocks-path" ]
(optional string)
~doc:"PATH Path to write precomputed blocks to, for replay or archiving"
~doc:
"PATH Path to write precomputed blocks to, for replay or archiving. If \
PATH is a directory, precomputed blocks will be logged to individual \
files within this directory. Otherwise, they will be appended to the \
same file."
and log_precomputed_blocks =
flag "--log-precomputed-blocks"
~aliases:[ "log-precomputed-blocks" ]
(optional_with_default false bool)
~doc:"true|false Include precomputed blocks in the log (default: false)"
~doc:
"true|false Include precomputed blocks in the log (default: false). \
See also --precomputed-block-path for additional functionality."
and block_reward_threshold =
flag "--minimum-block-reward" ~aliases:[ "minimum-block-reward" ]
~doc:
Expand Down
11 changes: 9 additions & 2 deletions src/lib/mina_lib/mina_lib.ml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -114,7 +114,7 @@ type t =
; subscriptions : Mina_subscriptions.t
; sync_status : Sync_status.t Mina_incremental.Status.Observer.t
; precomputed_block_writer :
([ `Path of string ] option * [ `Log ] option) ref
([ `Path of string | `Path_dir of string ] option * [ `Log ] option) ref
; block_production_status :
[ `Producing | `Producing_in_ms of float | `Free ] ref
; vrf_evaluation_state : Block_producer.Vrf_evaluation_state.t
Expand Down Expand Up @@ -2105,10 +2105,16 @@ let create ?wallets (config : Config.t) =
~precomputed_values:config.precomputed_values
~frontier_broadcast_pipe:frontier_broadcast_pipe_r
archive_process_port ) ;
(* To log precomputed blocks to individual files, set both
--precomputed-blocks-path DIR and --log-precomputed-blocks true *)
let precomputed_block_writer =
ref
( Option.map config.precomputed_blocks_path ~f:(fun path ->
`Path path )
match Core.Unix.(stat path).st_kind with
| S_DIR ->
`Path_dir path
| _ ->
`Path path )
, if config.log_precomputed_blocks then Some `Log else None )
in
let subscriptions =
Expand All @@ -2118,6 +2124,7 @@ let create ?wallets (config : Config.t) =
~is_storing_all:config.is_archive_rocksdb
~upload_blocks_to_gcloud:config.upload_blocks_to_gcloud
~time_controller:config.time_controller ~precomputed_block_writer
~log_precomputed_blocks:config.log_precomputed_blocks
in
let open Mina_incremental.Status in
let transition_frontier_incr =
Expand Down
120 changes: 94 additions & 26 deletions src/lib/mina_lib/mina_subscriptions.ml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,7 @@ let add_new_subscription (t : t) ~pk =

let create ~logger ~constraint_constants ~wallets ~new_blocks
~transition_frontier ~is_storing_all ~time_controller
~upload_blocks_to_gcloud ~precomputed_block_writer =
~upload_blocks_to_gcloud ~precomputed_block_writer ~log_precomputed_blocks =
let subscribed_block_users =
Optional_public_key.Table.of_alist_multi
@@ List.map (Secrets.Wallets.pks wallets) ~f:(fun wallet ->
Expand Down Expand Up @@ -101,13 +101,26 @@ let create ~logger ~constraint_constants ~wallets ~new_blocks
let gcloud_keyfile =
match Core.Sys.getenv "GCLOUD_KEYFILE" with
| Some keyfile ->
if upload_blocks_to_gcloud then
[%log info] "GCLOUD_KEYFILE environment variable set to %s" keyfile ;
Some keyfile
| _ ->
[%log warn]
"GCLOUD_KEYFILE environment variable not set. Must be set to use \
upload_blocks_to_gcloud" ;
if upload_blocks_to_gcloud then
[%log warn]
"GCLOUD_KEYFILE environment variable not set. Must be set to use \
upload_blocks_to_gcloud" ;
None
in
Option.iter (fst !precomputed_block_writer) ~f:(fun path ->
match path with
| `Path_dir path ->
[%log info]
~metadata:[ ("path", `String path) ]
"Precomputed blocks will be logged to individual files in $path"
| `Path path ->
[%log info]
~metadata:[ ("path", `String path) ]
"Precomputed blocks will be logged to the same file $path" ) ;
Option.iter gcloud_keyfile ~f:(fun path ->
ignore
( Core.Sys.command
Expand All @@ -118,7 +131,7 @@ let create ~logger ~constraint_constants ~wallets ~new_blocks
let new_block = Mina_block.Validated.forget new_block_validated in
let new_block_no_hash = With_hash.data new_block in
let hash = State_hash.With_state_hashes.state_hash new_block in
(let path, log = !precomputed_block_writer in
(let path, _log = !precomputed_block_writer in
match Broadcast_pipe.Reader.peek transition_frontier with
| None ->
[%log warn]
Expand Down Expand Up @@ -158,8 +171,8 @@ let create ~logger ~constraint_constants ~wallets ~new_blocks
] ;
Mina_block.Precomputed.to_yojson precomputed_block )
in
if upload_blocks_to_gcloud then (
[%log info] "log" ;
(* Upload precomputed blocks to gcloud *)
( if upload_blocks_to_gcloud then
let json =
Yojson.Safe.to_string (Lazy.force precomputed_block)
in
Expand Down Expand Up @@ -187,6 +200,13 @@ let create ~logger ~constraint_constants ~wallets ~new_blocks
match (gcloud_keyfile, network, bucket) with
| Some _, Some network, Some bucket ->
let hash_string = State_hash.to_base58_check hash in
[%log info]
~metadata:
[ ("hash", `String hash_string)
; ("bucket", `String bucket)
]
"Uploading precomputed block with $hash to gcloud \
$bucket" ;
let height =
Mina_block.blockchain_length new_block_no_hash
|> Mina_numbers.Length.to_string
Expand Down Expand Up @@ -248,25 +268,73 @@ let create ~logger ~constraint_constants ~wallets ~new_blocks
.upload_to_gcloud_blocks) )
| _ ->
() ) ;
Option.iter path ~f:(fun (`Path path) ->
Out_channel.with_file ~append:true path
~f:(fun out_channel ->
Out_channel.output_lines out_channel
[ Yojson.Safe.to_string
(Lazy.force precomputed_block)
] ) ) ;
[%log info] "Saw block with state hash $state_hash"
~metadata:
(let state_hash_data =
[ ( "state_hash"
, `String (State_hash.to_base58_check hash) )
]
in
if is_some log then
state_hash_data
@ [ ("precomputed_block", Lazy.force precomputed_block)
]
else state_hash_data ) ) ) ;
(* Log precomputed blocks locally *)
Option.iter path ~f:(fun path ->
if log_precomputed_blocks then
let json =
Yojson.Safe.to_string (Lazy.force precomputed_block)
in
match path with
| `Path path ->
(* original logging functionality, appends to single file *)
Out_channel.with_file ~append:true path
~f:(fun out_channel ->
Out_channel.output_lines out_channel [ json ] )
| `Path_dir path -> (
(* log precomputed blocks to individual files in the directory *)
let network =
match Core.Sys.getenv "NETWORK_NAME" with
| Some network ->
Some network
| _ ->
[%log warn]
"NETWORK_NAME environment variable not \
set. Default to 'mainnet'" ;
Some "mainnet"
in
match network with
| Some network ->
let hash_string =
State_hash.to_base58_check hash
in
let height =
Mina_block.blockchain_length
new_block_no_hash
|> Mina_numbers.Length.to_string
in
let name =
sprintf "%s-%s-%s.json" network height
hash_string
in
let fpath =
Core.Filename.(
parts path @ [ name ] |> of_parts)
in
Out_channel.with_file ~append:false fpath
~f:(fun out_channel ->
Out_channel.output_lines out_channel
[ json ] ) ;
[%log info]
~metadata:
[ ("block", `String name)
; ("path", `String path)
; ( "time"
, `String
Time.(
now ()
|> to_string_iso8601_basic
~zone:Zone.utc) )
]
"Logged precomputed $block to $path at $time"
| None ->
() )
else
[%log info]
~metadata:
[ ( "state_hash"
, `String (State_hash.to_base58_check hash) )
]
"Saw block with state hash $state_hash" ) ) ) ;
match
Filtered_external_transition.validate_transactions
~constraint_constants new_block_no_hash
Expand Down