ci: update protonet reset workflow for new infra (#2033)

* ci: update protonet reset workflow for new infra

* cleanup debugging workflow action

.github/workflows/cd-protonet.yml

@@ -1,41 +1,26 @@
 name: Continuous Deployment (Protonet)
-## run after every successful CI job of new commits to the master branch
-#on:
-#  workflow_run:
-#    workflows: [Continuous Integration (Kava Master)]
-#    types:
-#      - completed
 
-jobs:
-  # in order:
-  # enter standby (prevents autoscaling group from killing node during deploy)
-  # stop kava
-  # take ebs + zfs snapshots
-  # download updated binary and genesis
-  # reset application database state (only done on internal testnet)
-  reset-chain-to-zero-state:
-    # only start cd pipeline if last ci run was successful
-    if: ${{ github.event.workflow_run.conclusion == 'success' }}
-    uses: ./.github/workflows/cd-reset-internal-testnet.yml
-    with:
-      aws-region: us-east-1
-      chain-id: proto_2221-17000
-      ssm-document-name: kava-testnet-internal-node-update
-      playbook-name: reset-protonet-playbook.yml
-      playbook-infrastructure-branch: master
-    secrets: inherit
+## run after every successful CI job of new commits to the master branch, or on manual dispatch
+on:
+  workflow_run:
+    workflows: [Continuous Integration (Kava Master)]
+    types:
+      - completed
+  workflow_dispatch:
 
-  # start kava with new binary and genesis state on api, peer and seed nodes, place nodes in service once they start and are synched to live
-  start-chain-api:
-    uses: ./.github/workflows/cd-start-chain.yml
+jobs:
+  reset-and-restart-chain:
+    # only start cd pipeline if last ci run was successful (or it this is being manually dispatched)
+    if: |
+      github.event_name == 'workflow_dispatch' ||
+      (github.event_name == 'workflow_run' && github.event.workflow_run.conclusion == 'success')
+    uses: ./.github/workflows/cd-reset-network.yml
     with:
       aws-region: us-east-1
       chain-id: proto_2221-17000
-      ssm-document-name: kava-testnet-internal-node-update
-      playbook-name: start-chain-api-playbook.yml
+      ssm-document-name: kava-protonet-api-instance-update
       playbook-infrastructure-branch: master
     secrets: inherit
-    needs: [reset-chain-to-zero-state]
 
   # setup test and development accounts and balances, deploy contracts by calling the chain's api
   seed-chain-state:
@@ -46,9 +31,10 @@ jobs:
       seed-script-filename: seed-protonet.sh
       erc20-deployer-network-name: protonet
       genesis_validator_addresses: "kavavaloper14w4avgdvqrlpww6l5dhgj4egfn6ln7gmtp7r2m"
-      kava_version_filepath:  ./ci/env/kava-protonet/KAVA.VERSION
+      kava_version_filepath: ./ci/env/kava-protonet/KAVA.VERSION
     secrets: inherit
-    needs: [start-chain-api]
+    needs: [reset-and-restart-chain]
+
   post-pipeline-metrics:
     uses: ./.github/workflows/metric-pipeline.yml
     if: always() # always run so we metric failures and successes

.github/workflows/cd-reset-network.yml

@@ -0,0 +1,87 @@
+# for use with ephemeral testnets like internal testnet
+
+name: Reset Ephemeral Network
+on:
+  workflow_call:
+    inputs:
+      chain-id:
+        required: true
+        type: string
+      aws-region:
+        required: true
+        type: string
+      ssm-document-name:
+        required: true
+        type: string
+      playbook-infrastructure-branch:
+        required: true
+        type: string
+    secrets:
+      CI_AWS_KEY_ID:
+        required: true
+      CI_AWS_KEY_SECRET:
+        required: true
+      KAVA_PRIVATE_GITHUB_ACCESS_TOKEN:
+        required: true
+jobs:
+  reset-ephemeral-network:
+    runs-on: ubuntu-latest
+    steps:
+      - name: checkout infrastructure repo
+        uses: actions/checkout@v4
+        with:
+          repository: Kava-Labs/infrastructure
+          token: ${{ secrets.KAVA_PRIVATE_GITHUB_ACCESS_TOKEN }}
+          path: infrastructure
+          ref: master
+
+      - name: Set up Go with kava-node-updater cache
+        uses: actions/setup-go@v5
+        with:
+          go-version-file: infrastructure/cli/kava-node-updater/go.mod
+          cache-dependency-path: infrastructure/cli/kava-node-updater/go.mod
+
+      - name: build kava node updater
+        run: cd infrastructure/cli/kava-node-updater && make install && cd ../../../
+
+      - name: stop chain services & reset data on all chain nodes
+        run: |
+          kava-node-updater \
+            --debug \
+            --max-retries=2 \
+            --aws-ssm-document-name=$SSM_DOCUMENT_NAME \
+            --infrastructure-git-pointer=$PLAYBOOK_INFRASTRUCTURE_BRANCH \
+            --chain-id=$CHAIN_ID \
+            --max-upgrade-batch-size=0 \
+            --wait-for-node-sync-after-upgrade=false \
+            --update-playbook-filename=reset-chain-data.yml
+        env:
+          SSM_DOCUMENT_NAME: ${{ inputs.ssm-document-name }}
+          CHAIN_ID: ${{ inputs.chain-id }}
+          AWS_SDK_LOAD_CONFIG: 1
+          AWS_REGION: ${{ inputs.aws-region }}
+          AWS_ACCESS_KEY_ID: ${{ secrets.CI_AWS_KEY_ID }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.CI_AWS_KEY_SECRET }}
+          PLAYBOOK_INFRASTRUCTURE_BRANCH: ${{ inputs.playbook-infrastructure-branch }}
+
+      # NOTE: we wait for the reset to finish before restarting the nodes to prevent nodes from
+      # re-syncing the previous chain state.
+      - name: reconfigure & restart chain services on all chain nodes
+        run: |
+          kava-node-updater \
+            --debug \
+            --max-retries=2 \
+            --aws-ssm-document-name=$SSM_DOCUMENT_NAME \
+            --infrastructure-git-pointer=$PLAYBOOK_INFRASTRUCTURE_BRANCH \
+            --chain-id=$CHAIN_ID \
+            --max-upgrade-batch-size=0 \
+            --wait-for-node-sync-after-upgrade=false \
+            --update-playbook-filename=playbook.yml
+        env:
+          SSM_DOCUMENT_NAME: ${{ inputs.ssm-document-name }}
+          CHAIN_ID: ${{ inputs.chain-id }}
+          AWS_REGION: ${{ inputs.aws-region }}
+          AWS_ACCESS_KEY_ID: ${{ secrets.CI_AWS_KEY_ID }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.CI_AWS_KEY_SECRET }}
+          AWS_SDK_LOAD_CONFIG: 1
+          PLAYBOOK_INFRASTRUCTURE_BRANCH: ${{ inputs.playbook-infrastructure-branch }}