Extended config functionality to read password from different sources

README.md

@@ -273,15 +273,16 @@ With configuration file you can specify list of signers and sign drops with chos
 [
 	{
 		"keyfile_path": "dev.json",
-		"keyfile_password_path": "password.txt"
+		"password": "password.txt",
+		"password_type": "text_file"
 	}
 ]
 ```
 
 Config also could be generated with command:
 
 ```bash
-waggle server configure --keyfile dev.json --outfile config.json
+waggle server configure --keyfile dev.json --outfile config.json --password-type text_file
 ```
 
 Run server:

cmd.go

@@ -1,6 +1,7 @@
 package main
 
 import (
+	"bytes"
 	"encoding/csv"
 	"encoding/hex"
 	"encoding/json"
@@ -9,7 +10,6 @@ import (
 	"io"
 	"log"
 	"os"
-	"path/filepath"
 	"strings"
 
 	bugout "github.com/bugout-dev/bugout-go/pkg"
@@ -533,11 +533,7 @@ func CreateServerCommand() *cobra.Command {
 
 			availableSigners := make(map[string]AvailableSigner)
 			for _, c := range *configs {
-				passwordRaw, readErr := os.ReadFile(c.KeyfilePasswordPath)
-				if readErr != nil {
-					return readErr
-				}
-				key, keyErr := KeyFromFile(c.KeyfilePath, string(passwordRaw))
+				key, keyErr := KeyFromFile(c.KeyfilePath, c.Password)
 				if keyErr != nil {
 					return keyErr
 				}
@@ -567,24 +563,46 @@ func CreateServerCommand() *cobra.Command {
 	runSubcommand.Flags().StringVar(&config, "config", "./config.json", "Path to server configuration file")
 	runSubcommand.Flags().IntVar(&logLevel, "log-level", 1, "Log verbosity level")
 
-	var keyfile, password, outfile string
+	var keyfile, passwordFlag, passwordTypeFlag, outfile string
 
 	configureCommand := &cobra.Command{
 		Use:   "configure",
 		Short: "Prepare configuration for waggle API server.",
+		PreRunE: func(cmd *cobra.Command, args []string) error {
+			var passwordType string
+			var flagsGetStringErr error
+			passwordType, flagsGetStringErr = cmd.Flags().GetString("password-type")
+			if flagsGetStringErr != nil {
+				return flagsGetStringErr
+			}
+
+			switch passwordType {
+			case string(PlainText), string(TextFile), string(AwsSecret):
+				return nil
+			}
+			return errors.New("invalid value: allowed values are 'plaintext', 'text_file' and 'aws_secret'")
+		},
 		RunE: func(cmd *cobra.Command, args []string) error {
 			serverSignerConfigs := []ServerSignerConfig{}
-			var passwordRaw []byte
+			var password string
 			var err error
-			if password == "" {
-				fmt.Print("Enter password for keyfile (it will not be displayed on screen): ")
-				passwordRaw, err = term.ReadPassword(int(os.Stdin.Fd()))
+			if passwordFlag == "" {
+				fmt.Print("Enter password (or path to text file, or aws secret manager variable name) for keyfile (it will not be displayed on screen)\nInput: ")
+				passwordRaw, err := term.ReadPassword(int(os.Stdin.Fd()))
 				fmt.Print("\n")
 				if err != nil {
 					return fmt.Errorf("error reading password from input: %s", err.Error())
 				}
+				password = string(passwordRaw)
 			} else {
-				passwordRaw = []byte(password)
+				password = passwordFlag
+			}
+
+			var passValidErr error
+			pt := PasswordType(passwordTypeFlag)
+			password, passValidErr = pt.PasswordValidation(password)
+			if passValidErr != nil {
+				return passValidErr
 			}
 
 			keyfilePath := strings.TrimSuffix(keyfile, "/")
@@ -595,19 +613,22 @@ func CreateServerCommand() *cobra.Command {
 				}
 				return fmt.Errorf("error due checking keyfile path %s, err: %v", keyfilePath, err)
 			}
-			dir, file := filepath.Split(keyfilePath)
-			passwordFilePath := fmt.Sprintf("%spassword-%s", dir, file)
-			os.WriteFile(passwordFilePath, passwordRaw, 0640)
 
 			// TODO(kompotkot): Provide functionality to generate config with multiple keyfiles
 			serverSignerConfigs = append(serverSignerConfigs, ServerSignerConfig{
-				KeyfilePath:         keyfile,
-				KeyfilePasswordPath: passwordFilePath,
+				KeyfilePath:  keyfile,
+				Password:     password,
+				PasswordType: passwordTypeFlag,
 			})
-			resultJSON, err := json.Marshal(serverSignerConfigs)
-			if err != nil {
-				return err
+
+			// Using manual encoding to prevent HTML escaping
+			buffer := &bytes.Buffer{}
+			encoder := json.NewEncoder(buffer)
+			encoder.SetEscapeHTML(false)
+			if encodeErr := encoder.Encode(serverSignerConfigs); encodeErr != nil {
+				return encodeErr
 			}
+			resultJSON := buffer.Bytes()
 
 			if outfile != "" {
 				os.WriteFile(outfile, resultJSON, 0644)
@@ -620,7 +641,8 @@ func CreateServerCommand() *cobra.Command {
 	}
 
 	configureCommand.PersistentFlags().StringVarP(&keyfile, "keystore", "k", "", "Path to keystore file (this should be a JSON file)")
-	configureCommand.PersistentFlags().StringVarP(&password, "password", "p", "", "Password for keystore file. If not provided, you will be prompted for it when you sign with the key")
+	configureCommand.PersistentFlags().StringVarP(&passwordFlag, "password", "p", "", "Password for keystore file. If not provided, you will be prompted for it when you sign with the key.")
+	configureCommand.PersistentFlags().StringVarP(&passwordTypeFlag, "password-type", "t", "plaintext", fmt.Sprintf("Format of password, available options: %s, %s, %s", string(PlainText), string(TextFile), string(AwsSecret)))
 	configureCommand.PersistentFlags().StringVarP(&outfile, "outfile", "o", "config.json", "Config file output path")
 
 	serverCommand.AddCommand(runSubcommand, configureCommand)

deploy/deploy.bash

@@ -14,11 +14,13 @@ PREFIX_WARN="${C_YELLOW}[WARN]${C_RESET} [$(date +%d-%m\ %T)]"
 PREFIX_CRIT="${C_RED}[CRIT]${C_RESET} [$(date +%d-%m\ %T)]"
 
 # Main
-AWS_DEFAULT_REGION="${AWS_DEFAULT_REGION:-us-east-1}"
+AWS_DEFAULT_REGION="${AWS_DEFAULT_REGION:-us-west-1}"
 APP_DIR="${APP_DIR:-/home/ubuntu/waggle}"
 SECRETS_DIR="${SECRETS_DIR:-/home/ubuntu/waggle-secrets}"
+STORAGE_PATH="${STORAGE_PATH:-/mnt/disks/storage}"
 PARAMETERS_ENV_PATH="${SECRETS_DIR}/app.env"
 SCRIPT_DIR="$(realpath $(dirname $0))"
+USER_SYSTEMD_DIR="${USER_SYSTEMD_DIR:-/home/ubuntu/.config/systemd/user}"
 
 # Service file
 WAGGLE_SERVICE_FILE="waggle.service"
@@ -32,13 +34,24 @@ HOME=/home/ubuntu /usr/local/go/bin/go install github.com/bugout-dev/checkenv@la
 
 echo
 echo
-echo -e "${PREFIX_INFO} Retrieving addition deployment parameters"
+echo -e "${PREFIX_INFO} Retrieving deployment parameters"
+if [ ! -d "${SECRETS_DIR}" ]; then
+  mkdir "${SECRETS_DIR}"
+  echo -e "${PREFIX_WARN} Created new secrets directory"
+fi
 AWS_DEFAULT_REGION="${AWS_DEFAULT_REGION}" /home/ubuntu/go/bin/checkenv show aws_ssm+waggle:true >> "${PARAMETERS_ENV_PATH}"
+chmod 0640 "${PARAMETERS_ENV_PATH}"
 
 echo
 echo
-echo -e "${PREFIX_INFO} Add instance local IP to parameters"
+echo -e "${PREFIX_INFO} Add instance local IP and AWS region to parameters"
 echo "AWS_LOCAL_IPV4=$(ec2metadata --local-ipv4)" >> "${PARAMETERS_ENV_PATH}"
+echo "AWS_REGION=${AWS_DEFAULT_REGION}" >> "${PARAMETERS_ENV_PATH}"
+
+echo
+echo
+echo -e "${PREFIX_INFO} Create symlink to config.json"
+ln -sf "${STORAGE_PATH}/config.json" "${SECRETS_DIR}/config.json"
 
 echo
 echo
@@ -48,10 +61,18 @@ cd "${APP_DIR}"
 HOME=/home/ubuntu /usr/local/go/bin/go build -o "${APP_DIR}/waggle" .
 cd "${EXEC_DIR}"
 
+echo
+echo
+echo -e "${PREFIX_INFO} Prepare user systemd directory"
+if [ ! -d "${USER_SYSTEMD_DIR}" ]; then
+  mkdir -p "${USER_SYSTEMD_DIR}"
+  echo -e "${PREFIX_WARN} Created new user systemd directory"
+fi
+
 echo
 echo
 echo -e "${PREFIX_INFO} Replacing existing waggle service definition with ${WAGGLE_SERVICE_FILE}"
 chmod 644 "${SCRIPT_DIR}/${WAGGLE_SERVICE_FILE}"
-cp "${SCRIPT_DIR}/${WAGGLE_SERVICE_FILE}" "/home/ubuntu/.config/systemd/user/${WAGGLE_SERVICE_FILE}"
+cp "${SCRIPT_DIR}/${WAGGLE_SERVICE_FILE}" "${USER_SYSTEMD_DIR}/${WAGGLE_SERVICE_FILE}"
 XDG_RUNTIME_DIR="/run/user/$UID" systemctl --user daemon-reload
 XDG_RUNTIME_DIR="/run/user/$UID" systemctl --user restart "${WAGGLE_SERVICE_FILE}"

deploy/waggle.service

@@ -9,7 +9,7 @@ WorkingDirectory=/home/ubuntu/waggle
 EnvironmentFile=/home/ubuntu/waggle-secrets/app.env
 Restart=on-failure
 RestartSec=15s
-ExecStart=/home/ubuntu/waggle/waggle server run --host "${AWS_LOCAL_IPV4}" --port 7379 --config /home/ubuntu/.waggle/config.json
+ExecStart=/home/ubuntu/waggle/waggle server run --host "${AWS_LOCAL_IPV4}" --port 7379 --config /home/ubuntu/waggle-secrets/config.json
 SyslogIdentifier=waggle
 
 [Install]

go.mod

@@ -9,6 +9,22 @@ require (
 )
 
 require (
+	github.com/aws/aws-sdk-go-v2/credentials v1.13.37 // indirect
+	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.11 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.41 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.35 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/ini v1.3.42 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.35 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sso v1.13.6 // indirect
+	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.15.6 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sts v1.21.5 // indirect
+	github.com/aws/smithy-go v1.14.2 // indirect
+)
+
+require (
+	github.com/aws/aws-sdk-go-v2 v1.21.0
+	github.com/aws/aws-sdk-go-v2/config v1.18.39
+	github.com/aws/aws-sdk-go-v2/service/secretsmanager v1.21.3
 	github.com/btcsuite/btcd/btcec/v2 v2.2.0 // indirect
 	github.com/bugout-dev/bugout-go v0.4.0 // indirect
 	github.com/deckarep/golang-set/v2 v2.1.0 // indirect

go.sum

@@ -22,6 +22,32 @@ github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRF
 github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o=
 github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY=
 github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8=
+github.com/aws/aws-sdk-go-v2 v1.21.0 h1:gMT0IW+03wtYJhRqTVYn0wLzwdnK9sRMcxmtfGzRdJc=
+github.com/aws/aws-sdk-go-v2 v1.21.0/go.mod h1:/RfNgGmRxI+iFOB1OeJUyxiU+9s88k3pfHvDagGEp0M=
+github.com/aws/aws-sdk-go-v2/config v1.18.39 h1:oPVyh6fuu/u4OiW4qcuQyEtk7U7uuNBmHmJSLg1AJsQ=
+github.com/aws/aws-sdk-go-v2/config v1.18.39/go.mod h1:+NH/ZigdPckFpgB1TRcRuWCB/Kbbvkxc/iNAKTq5RhE=
+github.com/aws/aws-sdk-go-v2/credentials v1.13.37 h1:BvEdm09+ZEh2XtN+PVHPcYwKY3wIeB6pw7vPRM4M9/U=
+github.com/aws/aws-sdk-go-v2/credentials v1.13.37/go.mod h1:ACLrdkd4CLZyXOghZ8IYumQbcooAcp2jo/s2xsFH8IM=
+github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.11 h1:uDZJF1hu0EVT/4bogChk8DyjSF6fof6uL/0Y26Ma7Fg=
+github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.11/go.mod h1:TEPP4tENqBGO99KwVpV9MlOX4NSrSLP8u3KRy2CDwA8=
+github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.41 h1:22dGT7PneFMx4+b3pz7lMTRyN8ZKH7M2cW4GP9yUS2g=
+github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.41/go.mod h1:CrObHAuPneJBlfEJ5T3szXOUkLEThaGfvnhTf33buas=
+github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.35 h1:SijA0mgjV8E+8G45ltVHs0fvKpTj8xmZJ3VwhGKtUSI=
+github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.35/go.mod h1:SJC1nEVVva1g3pHAIdCp7QsRIkMmLAgoDquQ9Rr8kYw=
+github.com/aws/aws-sdk-go-v2/internal/ini v1.3.42 h1:GPUcE/Yq7Ur8YSUk6lVkoIMWnJNO0HT18GUzCWCgCI0=
+github.com/aws/aws-sdk-go-v2/internal/ini v1.3.42/go.mod h1:rzfdUlfA+jdgLDmPKjd3Chq9V7LVLYo1Nz++Wb91aRo=
+github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.35 h1:CdzPW9kKitgIiLV1+MHobfR5Xg25iYnyzWZhyQuSlDI=
+github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.35/go.mod h1:QGF2Rs33W5MaN9gYdEQOBBFPLwTZkEhRwI33f7KIG0o=
+github.com/aws/aws-sdk-go-v2/service/secretsmanager v1.21.3 h1:H6ZipEknzu7RkJW3w2PP75zd8XOdR35AEY5D57YrJtA=
+github.com/aws/aws-sdk-go-v2/service/secretsmanager v1.21.3/go.mod h1:5W2cYXDPabUmwULErlC92ffLhtTuyv4ai+5HhdbhfNo=
+github.com/aws/aws-sdk-go-v2/service/sso v1.13.6 h1:2PylFCfKCEDv6PeSN09pC/VUiRd10wi1VfHG5FrW0/g=
+github.com/aws/aws-sdk-go-v2/service/sso v1.13.6/go.mod h1:fIAwKQKBFu90pBxx07BFOMJLpRUGu8VOzLJakeY+0K4=
+github.com/aws/aws-sdk-go-v2/service/ssooidc v1.15.6 h1:pSB560BbVj9ZlJZF4WYj5zsytWHWKxg+NgyGV4B2L58=
+github.com/aws/aws-sdk-go-v2/service/ssooidc v1.15.6/go.mod h1:yygr8ACQRY2PrEcy3xsUI357stq2AxnFM6DIsR9lij4=
+github.com/aws/aws-sdk-go-v2/service/sts v1.21.5 h1:CQBFElb0LS8RojMJlxRSo/HXipvTZW2S44Lt9Mk2aYQ=
+github.com/aws/aws-sdk-go-v2/service/sts v1.21.5/go.mod h1:VC7JDqsqiwXukYEDjoHh9U0fOJtNWh04FPQz4ct4GGU=
+github.com/aws/smithy-go v1.14.2 h1:MJU9hqBGbvWZdApzpvoF2WAIJDbtjK2NDJSiJP7HblQ=
+github.com/aws/smithy-go v1.14.2/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA=
 github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
 github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
 github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
@@ -92,6 +118,7 @@ github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Z
 github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
 github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
 github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
+github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
 github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
 github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
 github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
@@ -130,6 +157,8 @@ github.com/holiman/uint256 v1.2.0/go.mod h1:y4ga/t+u+Xwd7CpDgZESaRcWy0I7XMlTMA25
 github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
 github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
 github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
+github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=
+github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=
 github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo=
 github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
 github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=

sample.env

@@ -4,4 +4,4 @@ export MOONSTREAM_API_TIMEOUT_SECONDS=30
 export BUGOUT_ACCESS_TOKEN="<user_bugout_access_token>"
 
 # Server related environment variables
-export WAGGLE_CORS_ALLOWED_ORIGINS="http://localhost:3000"
+export WAGGLE_CORS_ALLOWED_ORIGINS="http://localhost:3000,https://moonstream.to,https://portal.moonstream.to,https://www.moonstream.to"