[tools/shoestring]: add support for advanced customization of rest.json

.gitignore

@@ -23,6 +23,7 @@ _data/
 .next/
 
 # python
+.python-version
 __pycache__/
 *.egg-info/
 build/

tools/shoestring/README.md

@@ -134,17 +134,17 @@ setup \
     [--package PACKAGE] \
     [--directory DIRECTORY] \
     [--overrides OVERRIDES] \
-    [--metadata METADATA] \
+    [--rest-overrides REST_OVERRIDES] \
     [--security {default,paranoid,insecure}] \
     --ca-key-path CA_KEY_PATH
 
-  --config CONFIG           path to shoestring configuration file
-  --package PACKAGE         Network configuration package. Possible values: (name | file:///filename | http(s)://uri) (default: mainnet)
-  --directory DIRECTORY     installation directory (default: $HOME)
-  --overrides OVERRIDES     path to custom user settings
-  --metadata METADATA       custom node metadata (this is only valid for API roles)
-  --security                security mode (default: default)
-  --ca-key-path CA_KEY_PATH path to main private key PEM file
+  --config CONFIG                       path to shoestring configuration file
+  --package PACKAGE                     Network configuration package. Possible values: (name | file:///filename | http(s)://uri) (default: mainnet)
+  --directory DIRECTORY                 installation directory (default: $HOME)
+  --overrides OVERRIDES                 path to custom user settings
+  --rest-overrides REST_OVERRIDES       path to custom user REST settings (this is only valid for API roles)
+  --security                            security mode (default: default)
+  --ca-key-path CA_KEY_PATH             path to main private key PEM file
 ```
 
 Please note that only security mode "default" is supported at this time.
@@ -201,13 +201,13 @@ upgrade \
     [--package PACKAGE] \
     [--directory DIRECTORY] \
     [--overrides OVERRIDES] \
-    [--metadata METADATA]
+    [--rest-overrides REST_OVERRIDES]
 
-  --config CONFIG           path to shoestring configuration file
-  --package PACKAGE         Network configuration package. Possible values: (name | file:///filename | http(s)://uri) (default: mainnet)
-  --directory DIRECTORY     installation directory (default: $HOME)
-  --overrides OVERRIDES     path to custom user settings
-  --metadata METADATA       custom node metadata (this is only valid for API roles)
+  --config CONFIG                       path to shoestring configuration file
+  --package PACKAGE                     Network configuration package. Possible values: (name | file:///filename | http(s)://uri) (default: mainnet)
+  --directory DIRECTORY                 installation directory (default: $HOME)
+  --overrides OVERRIDES                 path to custom user settings
+  --rest-overrides REST_OVERRIDES       path to custom user REST settings (this is only valid for API roles)
 ```
 
 ### renew-certificates
@@ -371,10 +371,9 @@ maxUnlockedAccounts = 2
 Notice that these custom settings are applied *BEFORE* shoestring updates the Symbol configuration files.
 In cases of conflicts, the shoestring changes will take precedence.
 
-### Node Metadata
+### REST Overrides
 
-JSON file that is ingested and used to replace the contents of `nodeMetadata` in rest.json.
-This data is then accessible via the `node/metadata` REST endpoint.
+JSON file that is ingested and used to update the contents of rest.json.
 This file is optional and only used for deployments including API role.
 
 # Running

tools/shoestring/shoestring/commands/setup.py

@@ -112,7 +112,7 @@ async def run_main(args):
 			user_patches = load_patches_from_file(args.overrides)
 
 		preparer.configure_resources(user_patches)
-		preparer.configure_rest(args.metadata)
+		preparer.configure_rest(args.rest_overrides)
 
 		hostname = _resolve_hostname_and_configure_https(config, preparer)
 
@@ -137,7 +137,7 @@ def add_arguments(parser, is_initial_setup=True):
 	parser.add_argument('--package', help=_('argument-help-setup-package'), default='mainnet')
 	parser.add_argument('--directory', help=_('argument-help-directory').format(default_path=Path.home()), default=str(Path.home()))
 	parser.add_argument('--overrides', help=_('argument-help-setup-overrides'))
-	parser.add_argument('--metadata', help=_('argument-help-setup-metadata'))
+	parser.add_argument('--rest-overrides', help=_('argument-help-setup-rest-overrides'))
 
 	if is_initial_setup:
 		parser.add_argument('--security', help=_('argument-help-setup-security'), choices=SECURITY_MODES, default='default')

tools/shoestring/shoestring/internal/ConfigurationManager.py

@@ -111,3 +111,13 @@ def parse_time_span(str_value):
 		raise ValueError(f'time span specified in unknown units \'{unit_indicator}\'')
 
 	return value.timestamp
+
+
+def merge_json_configuration(lhs, rhs):
+	"""Merges two json configuration objects."""
+
+	for key in rhs.keys():
+		if isinstance(rhs[key], dict) and key in lhs and isinstance(lhs[key], dict):
+			merge_json_configuration(lhs[key], rhs[key])
+		else:
+			lhs[key] = rhs[key]

tools/shoestring/shoestring/internal/Preparer.py

@@ -9,7 +9,7 @@
 from symbolchain.symbol.Network import NetworkTimestamp
 
 from .CertificateFactory import CertificateFactory
-from .ConfigurationManager import ConfigurationManager
+from .ConfigurationManager import ConfigurationManager, merge_json_configuration
 from .FileTemplater import apply_template
 from .HarvesterConfigurator import HarvesterConfigurator
 from .LinkTransactionBuilder import LinkTransactionBuilder
@@ -266,7 +266,7 @@ def configure_resources(self, user_patches=None):
 		# make all resources read only
 		self._make_files_readonly(self.directories.resources)
 
-	def configure_rest(self, node_metadata_filename=None):
+	def configure_rest(self, rest_overrides_filename=None):
 		"""Copies mongo and rest files."""
 
 		if NodeFeatures.API not in self.config.node.features:
@@ -277,16 +277,16 @@ def configure_rest(self, node_metadata_filename=None):
 
 		self._copy_file(self.directories.temp / 'rest' / 'rest.json', self.directories.userconfig)
 
-		if node_metadata_filename:
+		if rest_overrides_filename:
 			rest_json_filepath = self.directories.userconfig / 'rest.json'
 
 			# load the rest config
 			with open(rest_json_filepath, 'rt', encoding='utf8') as rest_config_infile:
 				rest_config = json.load(rest_config_infile)
 
-			# replace user metadata
-			with open(node_metadata_filename, 'rt', encoding='utf8') as node_metadata_infile:
-				rest_config['nodeMetadata'] = json.load(node_metadata_infile)
+			# customize user rest config
+			with open(rest_overrides_filename, 'rt', encoding='utf8') as rest_overrides_infile:
+				merge_json_configuration(rest_config, json.load(rest_overrides_infile))
 
 			# rewrite the rest config
 			with open(rest_json_filepath, 'wt', encoding='utf8') as rest_config_outfile:

tools/shoestring/shoestring/lang/en/LC_MESSAGES/messages.po

@@ -27,7 +27,7 @@ msgstr "path to main private key PEM file"
 
 #: shoestring/commands/announce_transaction.py:35
 #: shoestring/commands/health.py:74 shoestring/commands/import_bootstrap.py:33
-#: shoestring/commands/import_harvesters.py:94 shoestring/commands/init.py:29
+#: shoestring/commands/import_harvesters.py:97 shoestring/commands/init.py:29
 #: shoestring/commands/min_cosignatures_count.py:34
 #: shoestring/commands/renew_certificates.py:29
 #: shoestring/commands/renew_voting_keys.py:112
@@ -47,19 +47,19 @@ msgstr "installation directory (default: {default_path})"
 msgid "argument-help-import-bootstrap-bootstrap"
 msgstr "path to bootstrap target directory"
 
-#: shoestring/commands/import_harvesters.py:95
+#: shoestring/commands/import_harvesters.py:98
 msgid "argument-help-import-harvesters-in-harvesters"
 msgstr "input harvesters.dat file that is encrypted with in-pem"
 
-#: shoestring/commands/import_harvesters.py:96
+#: shoestring/commands/import_harvesters.py:99
 msgid "argument-help-import-harvesters-in-pem"
 msgstr "PEM file that can be used to decrypt in-harvesters"
 
-#: shoestring/commands/import_harvesters.py:98
+#: shoestring/commands/import_harvesters.py:101
 msgid "argument-help-import-harvesters-out-harvesters"
 msgstr "output harvesters.dat file that will be encrypted with out-pem"
 
-#: shoestring/commands/import_harvesters.py:99
+#: shoestring/commands/import_harvesters.py:102
 msgid "argument-help-import-harvesters-out-pem"
 msgstr "PEM file that can be used to encrypt out-harvesters"
 
@@ -91,10 +91,6 @@ msgstr "renews CA certificate too"
 msgid "argument-help-reset-data-purge-harvesters"
 msgstr "purge harvesters.dat file"
 
-#: shoestring/commands/setup.py:140
-msgid "argument-help-setup-metadata"
-msgstr "Custom node metadata (this is only valid for API roles)"
-
 #: shoestring/commands/setup.py:139
 msgid "argument-help-setup-overrides"
 msgstr "path to custom user settings"
@@ -105,6 +101,10 @@ msgstr ""
 "Network configuration package. Possible values: (name | file:///filename "
 "| http(s)://uri) (default: mainnet)"
 
+#: shoestring/commands/setup.py:140
+msgid "argument-help-setup-rest-overrides"
+msgstr "path to custom user REST settings (this is only valid for API roles)"
+
 #: shoestring/commands/setup.py:143
 msgid "argument-help-setup-security"
 msgstr "security mode (default: default)"
@@ -260,11 +260,11 @@ msgstr ""
 "bootstrap directory provided ({directory}) does not look like bootstrap's"
 " target directory, nothing to import"
 
-#: shoestring/commands/import_harvesters.py:74
+#: shoestring/commands/import_harvesters.py:75
 msgid "import-harvesters-error-in-harvesters-is-equal-to-out-harvesters"
 msgstr "in-harvesters and out-harvesters must be different"
 
-#: shoestring/commands/import_harvesters.py:51
+#: shoestring/commands/import_harvesters.py:52
 msgid "import-harvesters-list-header"
 msgstr "listing harvesters in {filepath} using public key {public_key}"
 
@@ -826,4 +826,3 @@ msgstr "Welcome"
 #: shoestring/wizard/screens/welcome.py:36
 msgid "wizard-welcome-upgrade"
 msgstr "upgrade"
-

tools/shoestring/shoestring/lang/ja/LC_MESSAGES/messages.po

@@ -25,7 +25,7 @@ msgstr ""
 
 #: shoestring/commands/announce_transaction.py:35
 #: shoestring/commands/health.py:74 shoestring/commands/import_bootstrap.py:33
-#: shoestring/commands/import_harvesters.py:94 shoestring/commands/init.py:29
+#: shoestring/commands/import_harvesters.py:97 shoestring/commands/init.py:29
 #: shoestring/commands/min_cosignatures_count.py:34
 #: shoestring/commands/renew_certificates.py:29
 #: shoestring/commands/renew_voting_keys.py:112
@@ -45,19 +45,19 @@ msgstr ""
 msgid "argument-help-import-bootstrap-bootstrap"
 msgstr ""
 
-#: shoestring/commands/import_harvesters.py:95
+#: shoestring/commands/import_harvesters.py:98
 msgid "argument-help-import-harvesters-in-harvesters"
 msgstr ""
 
-#: shoestring/commands/import_harvesters.py:96
+#: shoestring/commands/import_harvesters.py:99
 msgid "argument-help-import-harvesters-in-pem"
 msgstr ""
 
-#: shoestring/commands/import_harvesters.py:98
+#: shoestring/commands/import_harvesters.py:101
 msgid "argument-help-import-harvesters-out-harvesters"
 msgstr ""
 
-#: shoestring/commands/import_harvesters.py:99
+#: shoestring/commands/import_harvesters.py:102
 msgid "argument-help-import-harvesters-out-pem"
 msgstr ""
 
@@ -89,10 +89,6 @@ msgstr ""
 msgid "argument-help-reset-data-purge-harvesters"
 msgstr ""
 
-#: shoestring/commands/setup.py:140
-msgid "argument-help-setup-metadata"
-msgstr ""
-
 #: shoestring/commands/setup.py:139
 msgid "argument-help-setup-overrides"
 msgstr ""
@@ -101,6 +97,10 @@ msgstr ""
 msgid "argument-help-setup-package"
 msgstr ""
 
+#: shoestring/commands/setup.py:140
+msgid "argument-help-setup-rest-overrides"
+msgstr ""
+
 #: shoestring/commands/setup.py:143
 msgid "argument-help-setup-security"
 msgstr ""
@@ -250,11 +250,11 @@ msgstr ""
 msgid "import-bootstrap-invalid-directory"
 msgstr ""
 
-#: shoestring/commands/import_harvesters.py:74
+#: shoestring/commands/import_harvesters.py:75
 msgid "import-harvesters-error-in-harvesters-is-equal-to-out-harvesters"
 msgstr ""
 
-#: shoestring/commands/import_harvesters.py:51
+#: shoestring/commands/import_harvesters.py:52
 msgid "import-harvesters-list-header"
 msgstr ""
 
@@ -803,4 +803,3 @@ msgstr ""
 #: shoestring/wizard/screens/welcome.py:36
 msgid "wizard-welcome-upgrade"
 msgstr ""
-

tools/shoestring/shoestring/lang/messages.pot

@@ -18,7 +18,7 @@ msgstr ""
 
 #: shoestring/commands/announce_transaction.py:35
 #: shoestring/commands/health.py:74 shoestring/commands/import_bootstrap.py:33
-#: shoestring/commands/import_harvesters.py:94 shoestring/commands/init.py:29
+#: shoestring/commands/import_harvesters.py:97 shoestring/commands/init.py:29
 #: shoestring/commands/min_cosignatures_count.py:34
 #: shoestring/commands/renew_certificates.py:29
 #: shoestring/commands/renew_voting_keys.py:112
@@ -38,19 +38,19 @@ msgstr ""
 msgid "argument-help-import-bootstrap-bootstrap"
 msgstr ""
 
-#: shoestring/commands/import_harvesters.py:95
+#: shoestring/commands/import_harvesters.py:98
 msgid "argument-help-import-harvesters-in-harvesters"
 msgstr ""
 
-#: shoestring/commands/import_harvesters.py:96
+#: shoestring/commands/import_harvesters.py:99
 msgid "argument-help-import-harvesters-in-pem"
 msgstr ""
 
-#: shoestring/commands/import_harvesters.py:98
+#: shoestring/commands/import_harvesters.py:101
 msgid "argument-help-import-harvesters-out-harvesters"
 msgstr ""
 
-#: shoestring/commands/import_harvesters.py:99
+#: shoestring/commands/import_harvesters.py:102
 msgid "argument-help-import-harvesters-out-pem"
 msgstr ""
 
@@ -82,10 +82,6 @@ msgstr ""
 msgid "argument-help-reset-data-purge-harvesters"
 msgstr ""
 
-#: shoestring/commands/setup.py:140
-msgid "argument-help-setup-metadata"
-msgstr ""
-
 #: shoestring/commands/setup.py:139
 msgid "argument-help-setup-overrides"
 msgstr ""
@@ -94,6 +90,10 @@ msgstr ""
 msgid "argument-help-setup-package"
 msgstr ""
 
+#: shoestring/commands/setup.py:140
+msgid "argument-help-setup-rest-overrides"
+msgstr ""
+
 #: shoestring/commands/setup.py:143
 msgid "argument-help-setup-security"
 msgstr ""
@@ -243,11 +243,11 @@ msgstr ""
 msgid "import-bootstrap-invalid-directory"
 msgstr ""
 
-#: shoestring/commands/import_harvesters.py:74
+#: shoestring/commands/import_harvesters.py:75
 msgid "import-harvesters-error-in-harvesters-is-equal-to-out-harvesters"
 msgstr ""
 
-#: shoestring/commands/import_harvesters.py:51
+#: shoestring/commands/import_harvesters.py:52
 msgid "import-harvesters-list-header"
 msgstr ""
 

tools/shoestring/shoestring/wizard/ShoestringOperation.py

@@ -24,7 +24,7 @@ def build_shoestring_command(
 	shoestring_directory,
 	ca_pem_path,
 	package,
-	has_custom_node_metadata=False
+	has_custom_rest_overrides=False
 ):  # pylint: disable=too-many-arguments
 	"""Builds shoestring command arguments."""
 
@@ -58,7 +58,7 @@ def build_shoestring_command(
 	if ShoestringOperation.SETUP == operation:
 		shoestring_args.extend(['--security', 'insecure'])
 
-		if has_custom_node_metadata:
-			shoestring_args.extend(['--metadata', str(Path(shoestring_directory) / 'node_metadata.json')])
+		if has_custom_rest_overrides:
+			shoestring_args.extend(['--rest-overrides', str(Path(shoestring_directory) / 'rest_overrides.json')])
 
 	return shoestring_args

tools/shoestring/shoestring/wizard/setup_file_generator.py

@@ -20,8 +20,8 @@ def _to_bool_string(flag):
 	return 'true' if flag else 'false'
 
 
-def try_prepare_node_metadata_file(screens, output_filename):
-	"""Prepares a node metadata file based on screens, if specified."""
+def try_prepare_rest_overrides_file(screens, output_filename):
+	"""Prepares a REST overrides file based on screens, if specified."""
 
 	node_type = screens.get('node-type').current_value
 	node_settings = screens.get('node-settings')
@@ -30,7 +30,7 @@ def try_prepare_node_metadata_file(screens, output_filename):
 		return False
 
 	with open(output_filename, 'wt', encoding='utf8') as outfile:
-		outfile.write(node_settings.metadata_info)
+		outfile.write(f'{{"nodeMetadata":{node_settings.metadata_info}}}')
 
 	return True