From e1ccd5b6c82b83f87448f4cd079d8265e9df8b3d Mon Sep 17 00:00:00 2001 From: Martin Varmuza Date: Fri, 25 Oct 2024 18:43:14 +0200 Subject: [PATCH] wip --- .../template-connect-test-params.yml | 101 +++++---- .github/workflows/test-connect.yml | 89 ++++---- scripts/ci/connect-test-matrix-generator.js | 198 ++++++++++++------ 3 files changed, 231 insertions(+), 157 deletions(-) diff --git a/.github/workflows/template-connect-test-params.yml b/.github/workflows/template-connect-test-params.yml index c95e9eec5ae..2d475456f70 100644 --- a/.github/workflows/template-connect-test-params.yml +++ b/.github/workflows/template-connect-test-params.yml @@ -2,14 +2,14 @@ name: "[Template] connect unit" on: workflow_call: inputs: - methods: - description: "List of methods to include in tests (example: applySettings,applyFlags,getFeatures)" - type: "string" - required: false testPattern: - description: "Test pattern to use (example: `init` or `methods`)" + description: "Test pattern to use to match for test files (example: `init` or `methods`)" type: "string" required: true + includeFilter: + description: "List of methods to include in tests (example: applySettings,applyFlags,getFeatures)" + type: "string" + required: false testsFirmware: description: "Firmware version for the tests (example: 2-latest, 2.2.0, 2-main)" type: "string" @@ -19,16 +19,6 @@ on: description: "Firmware model for the tests (example: T3T1)" type: "string" required: false - nodeEnvironment: - description: "Should the test run on nodejs environment, it runs by default." - type: "boolean" - required: false - default: true - webEnvironment: - description: "Should the test run on web environment, it runs by default." - type: "boolean" - required: false - default: true testDescription: description: "A description to make test title more descriptive (example: T3T1-latest)" type: "string" @@ -39,12 +29,25 @@ on: type: "boolean" required: false default: false + cache_tx: + description: "Cache transactions" + type: "string" + required: false + default: false + transport: + description: "Transport to use (example: bridge, node-bridge)" + type: "string" + required: false + default: "Bridge" + testEnv: + description: "Environment to test (example: bridge, web)" + type: "string" + required: true jobs: - node: - name: "node-${{ inputs.testDescription }}" + test: + name: "${{ inputs.testDescription }}" runs-on: ubuntu-latest - if: ${{ inputs.nodeEnvironment }} steps: - uses: actions/checkout@v4 with: @@ -54,54 +57,44 @@ jobs: with: node-version-file: ".nvmrc" cache: yarn - # todo: ideally do not install everything. possibly only devDependencies could be enough for testing (if there was not for building libs)? - - run: sed -i "/\"node\"/d" package.json - - run: yarn install - # nightly test - run without cached txs - - if: ${{ github.event_name == 'schedule' }} - run: echo "ADDITIONAL_ARGS=-c" >> "$GITHUB_ENV" - - if: ${{ inputs.testFirmwareModel }} - run: echo "ADDITIONAL_ARGS=$ADDITIONAL_ARGS -m ${{ inputs.testFirmwareModel }}" >> "$GITHUB_ENV" - - if: ${{ inputs.methods }} - run: echo "ADDITIONAL_ARGS=$ADDITIONAL_ARGS -i ${{ inputs.methods }}" >> "$GITHUB_ENV" - - if: ${{ inputs.testRandomizedOrder }} - run: echo "ADDITIONAL_ARGS=$ADDITIONAL_ARGS -r" >> "$GITHUB_ENV" - - run: './docker/docker-connect-test.sh node -p "${{ inputs.testPattern }}" -f "${{ inputs.testsFirmware }}" $ADDITIONAL_ARGS' - web: - name: "web-${{ inputs.testDescription }}" - runs-on: ubuntu-latest - if: ${{ inputs.webEnvironment }} - steps: - - uses: actions/checkout@v4 - with: - submodules: true - - name: Setup node - uses: actions/setup-node@v4 - with: - node-version-file: ".nvmrc" - cache: yarn - - run: | + - if: ${{ inputs.testEnv == 'web' }} + run: | echo -e "\nenableScripts: false" >> .yarnrc.yml # Install dependencies only for @trezor/connect package - - run: yarn workspaces focus @trezor/connect - - name: Retrieve build connect-web + - if: ${{ inputs.testEnv == 'web' }} + run: yarn workspaces focus @trezor/connect + - if: ${{ inputs.testEnv == 'web' }} + name: Retrieve build connect-web uses: actions/download-artifact@v4 with: name: build-artifact-connect-web path: packages/connect-web/build - - name: Retrieve build connect-iframe + - if: ${{ inputs.testEnv == 'web' }} + name: Retrieve build connect-iframe uses: actions/download-artifact@v4 with: name: build-artifact-connect-iframe path: packages/connect-iframe/build - - run: cd packages/connect-iframe && tree . - - name: "Echo download path" + - if: ${{ inputs.testEnv == 'web' }} + run: cd packages/connect-iframe && tree . + - if: ${{ inputs.testEnv == 'web' }} + name: "Echo download path" run: echo ${{steps.download.outputs.download-path}} - - if: ${{ github.event_name == 'schedule' }} + + # todo: ideally do not install everything. possibly only devDependencies could be enough for testing (if there was not for building libs)? + - if: ${{ inputs.testEnv == 'node' }} + run: sed -i "/\"node\"/d" package.json + - if: ${{ inputs.testEnv == 'node' }} + run: yarn install + + # nightly test - run without cached txs + - if: ${{ inputs.cache_tx == 'true' }} run: echo "ADDITIONAL_ARGS=-c" >> "$GITHUB_ENV" - if: ${{ inputs.testFirmwareModel }} run: echo "ADDITIONAL_ARGS=$ADDITIONAL_ARGS -m ${{ inputs.testFirmwareModel }}" >> "$GITHUB_ENV" - - if: ${{ inputs.methods }} - run: echo "ADDITIONAL_ARGS=$ADDITIONAL_ARGS -i ${{ inputs.methods }}" >> "$GITHUB_ENV" - - run: './docker/docker-connect-test.sh web -p "${{ inputs.testPattern }}" -f "${{ inputs.testsFirmware }}" $ADDITIONAL_ARGS' + - if: ${{ inputs.includeFilter }} + run: echo "ADDITIONAL_ARGS=$ADDITIONAL_ARGS -i ${{ inputs.includeFilter }}" >> "$GITHUB_ENV" + - if: ${{ inputs.testRandomizedOrder }} + run: echo "ADDITIONAL_ARGS=$ADDITIONAL_ARGS -r" >> "$GITHUB_ENV" + - run: './docker/docker-connect-test.sh ${{ inputs.testEnv }} -p "${{ inputs.testPattern }}" -f "${{ inputs.testsFirmware }}" $ADDITIONAL_ARGS' diff --git a/.github/workflows/test-connect.yml b/.github/workflows/test-connect.yml index 30462d810ad..ab6ed758411 100644 --- a/.github/workflows/test-connect.yml +++ b/.github/workflows/test-connect.yml @@ -77,91 +77,108 @@ jobs: - name: Set daily matrix id: set-matrix-daily - run: echo "dailyMatrix=$(node ./scripts/ci/connect-test-matrix-generator.js daily)" >> $GITHUB_OUTPUT + run: echo "dailyMatrix=$(node ./scripts/ci/connect-test-matrix-generator.js --model=T2T1 --firmware=2-latest --env=all --groups=api --cache_tx=true --transport=Bridge)" >> $GITHUB_OUTPUT - name: Set legacy devices matrix id: set-matrix-legacy-firmware - run: echo "legacyFirmwareMatrix=$(node ./scripts/ci/connect-test-matrix-generator.js legacyFirmware)" >> $GITHUB_OUTPUT + run: echo "legacyFirmwareMatrix=$(node ./scripts/ci/connect-test-matrix-generator.js --model=T2T1 --firmware=2.3.0 --env=all --groups=all --cache_tx=false --transport=Bridge)" >> $GITHUB_OUTPUT - name: Set canary devices matrix id: set-matrix-canary-firmware - run: echo "canaryFirmwareMatrix=$(node ./scripts/ci/connect-test-matrix-generator.js canaryFirmware)" >> $GITHUB_OUTPUT + run: echo "canaryFirmwareMatrix=$(node ./scripts/ci/connect-test-matrix-generator.js --model=T2T1 --firmware=2-main --env=all --groups=all --cache_tx=false --transport=Bridge)" >> $GITHUB_OUTPUT - name: Set other devices matrix id: set-matrix-other-devices - run: echo "otherDevicesMatrix=$(node ./scripts/ci/connect-test-matrix-generator.js otherDevices)" >> $GITHUB_OUTPUT + run: echo "otherDevicesMatrix=$(node ./scripts/ci/connect-test-matrix-generator.js --model=all --firmware=2-latest --env=node --groups=api --cache_tx=true --transport=Bridge)" >> $GITHUB_OUTPUT - connect-PR: + T2T1_latest-fw_api: needs: [build, set-matrix] - name: PR-${{ matrix.name }} + if: github.repository == 'trezor/trezor-suite' + name: ${{ matrix.model }}-${{ matrix.firmware }} uses: ./.github/workflows/template-connect-test-params.yml with: - testPattern: ${{ matrix.pattern }} - methods: ${{ matrix.methods }} + testPattern: ${{ matrix.groups.pattern }} + includeFilter: ${{ matrix.groups.includeFilter }} testsFirmware: ${{ matrix.firmware }} - testDescription: ${{ matrix.name }} + testDescription: ${{ matrix.env }} ${{ matrix.groups.name }} + cache_tx: ${{ matrix.cache_tx }} + transport: ${{ matrix.transport }} + testEnv: ${{ matrix.env }} + testFirmwareModel: ${{ matrix.model }} strategy: - fail-fast: false + fail-fast: true # in PRs we don't want to block CI too much so fail fast is enabled matrix: ${{ fromJson(needs.set-matrix.outputs.dailyMatrix) }} connect-randomized-order: needs: [build, set-matrix] - if: github.event_name == 'schedule' && github.repository == 'trezor/trezor-suite' + # if: github.event_name == 'schedule' && github.repository == 'trezor/trezor-suite' name: randomized-${{ matrix.name }} uses: ./.github/workflows/template-connect-test-params.yml with: - testPattern: ${{ matrix.pattern }} - methods: ${{ matrix.methods }} + testPattern: ${{ matrix.groups.pattern }} + includeFilter: ${{ matrix.groups.includeFilter }} testsFirmware: ${{ matrix.firmware }} - testDescription: ${{ matrix.name }}-${{ matrix.firmware }} + testDescription: ${{ matrix.env }} ${{ matrix.groups.name }} + cache_tx: ${{ matrix.cache_tx }} + transport: ${{ matrix.transport }} + testEnv: ${{ matrix.env }} + testFirmwareModel: ${{ matrix.model }} testRandomizedOrder: true - webEnvironment: false - nodeEnvironment: true strategy: fail-fast: false matrix: ${{ fromJson(needs.set-matrix.outputs.dailyMatrix) }} - connect-legacy-firmware: + T2T1_legacy-fw: needs: [build, set-matrix] - if: github.event_name == 'schedule' && github.repository == 'trezor/trezor-suite' - name: legacy-${{ matrix.name }} + # if: github.event_name == 'schedule' && github.repository == 'trezor/trezor-suite' + name: T2T1_legacy-fw uses: ./.github/workflows/template-connect-test-params.yml with: - testPattern: ${{ matrix.pattern }} - methods: ${{ matrix.methods }} + testPattern: ${{ matrix.groups.pattern }} + includeFilter: ${{ matrix.groups.includeFilter }} testsFirmware: ${{ matrix.firmware }} - testDescription: ${{ matrix.name }}-${{ matrix.firmware }} + testDescription: ${{ matrix.env }} ${{ matrix.groups.name }} + cache_tx: ${{ matrix.cache_tx }} + transport: ${{ matrix.transport }} + testEnv: ${{ matrix.env }} + testFirmwareModel: ${{ matrix.model }} strategy: fail-fast: false matrix: ${{ fromJson(needs.set-matrix.outputs.legacyFirmwareMatrix) }} - connect-canary-firmware: + T2T1_canary-fw: needs: [build, set-matrix] - if: github.event_name == 'schedule' && github.repository == 'trezor/trezor-suite' - name: canary-${{ matrix.name }} + # if: github.event_name == 'schedule' && github.repository == 'trezor/trezor-suite' + name: T2T1_canary-fw uses: ./.github/workflows/template-connect-test-params.yml with: - testPattern: ${{ matrix.pattern }} - methods: ${{ matrix.methods }} + testPattern: ${{ matrix.groups.pattern }} + includeFilter: ${{ matrix.groups.includeFilter }} testsFirmware: ${{ matrix.firmware }} - testDescription: ${{ matrix.name }}-${{ matrix.firmware }} + # testDescription: ${{ matrix.env }} ${{ matrix.groups.name }} + testDescription: Foo bar + cache_tx: ${{ matrix.cache_tx }} + transport: ${{ matrix.transport }} + testEnv: ${{ matrix.env }} + testFirmwareModel: ${{ matrix.model }} strategy: fail-fast: false matrix: ${{ fromJson(needs.set-matrix.outputs.canaryFirmwareMatrix) }} - connect-other-devices: + all-models_api: needs: [build, set-matrix] - if: github.event_name == 'schedule' && github.repository == 'trezor/trezor-suite' - name: other-devices-${{ matrix.name }}-${{ matrix.model }} + # if: github.event_name == 'schedule' && github.repository == 'trezor/trezor-suite' + name: all-models_api uses: ./.github/workflows/template-connect-test-params.yml with: - testPattern: ${{ matrix.pattern }} - methods: ${{ matrix.methods }} + testPattern: ${{ matrix.groups.pattern }} + includeFilter: ${{ matrix.groups.includeFilter }} testsFirmware: ${{ matrix.firmware }} + testDescription: ${{ matrix.env }} ${{ matrix.groups.name }} + cache_tx: ${{ matrix.cache_tx }} + transport: ${{ matrix.transport }} + testEnv: ${{ matrix.env }} testFirmwareModel: ${{ matrix.model }} - nodeEnvironment: true - webEnvironment: false - testDescription: ${{ matrix.name }}-${{ matrix.firmware }}-${{ matrix.model }} strategy: fail-fast: false matrix: ${{ fromJson(needs.set-matrix.outputs.otherDevicesMatrix) }} diff --git a/scripts/ci/connect-test-matrix-generator.js b/scripts/ci/connect-test-matrix-generator.js index eec3ac15ae7..51ec31946e3 100644 --- a/scripts/ci/connect-test-matrix-generator.js +++ b/scripts/ci/connect-test-matrix-generator.js @@ -5,7 +5,7 @@ const groups = { name: 'api', pattern: 'init authorizeCoinjoin cancelCoinjoinAuthorization passphrase unlockPath setBusy override checkFirmwareAuthenticity keepSession cancel.test info.test', - methods: '', + includeFilter: '', }, management: { name: 'management', @@ -15,126 +15,190 @@ const groups = { btcSign: { name: 'btc-sign', pattern: 'methods', - methods: 'signTransaction', + includeFilter: 'signTransaction', }, btcOthers: { name: 'btc-others', pattern: 'methods', - methods: + includeFilter: 'getAccountInfo,getAccountDescriptor,getAddress,getPublicKey,signMessage,verifyMessage,composeTransaction,getOwnershipId,getOwnershipProof', }, stellar: { name: 'stellar', pattern: 'methods', - methods: 'stellarGetAddress,stellarSignTransaction', + includeFilter: 'stellarGetAddress,stellarSignTransaction', }, cardano: { name: 'cardano', pattern: 'methods', - methods: + includeFilter: 'cardanoGetAddress,cardanoGetNativeScriptHash,cardanoGetPublicKey,cardanoSignTransaction', }, eos: { name: 'eos', pattern: 'methods', - methods: 'eosGetPublicKey,eosSignTransaction', + includeFilter: 'eosGetPublicKey,eosSignTransaction', }, ethereum: { name: 'ethereum', pattern: 'methods', - methods: + includeFilter: 'ethereumGetAddress,ethereumGetPublicKey,ethereumSignMessage,ethereumSignTransaction,ethereumVerifyMessage,ethereumSignTypedData', }, nem: { name: 'nem', pattern: 'methods', - methods: 'nemGetAddress,nemSignTransaction', + includeFilter: 'nemGetAddress,nemSignTransaction', }, ripple: { name: 'ripple', pattern: 'methods', - methods: 'rippleGetAddress,rippleSignTransaction', + includeFilter: 'rippleGetAddress,rippleSignTransaction', }, tezos: { name: 'tezos', pattern: 'methods', - methods: 'tezosGetAddress,tezosGetPublicKey,tezosSignTransaction', + includeFilter: 'tezosGetAddress,tezosGetPublicKey,tezosSignTransaction', }, binance: { name: 'binance', pattern: 'methods', - methods: 'binanceGetAddress,binanceGetPublicKey,binanceSignTransaction', + includeFilter: 'binanceGetAddress,binanceGetPublicKey,binanceSignTransaction', }, }; -const daily = { - firmwares: ['2-latest'], - tests: [...Object.values(groups)], -}; - -const legacyFirmware = { - firmwares: ['2.3.0'], - tests: daily.tests - // Cardano supports >=2.6.0 - .filter(test => test.name !== 'cardano'), -}; +const firmwares1 = ['1.9.0', '1-latest', '1-main']; +const firmwares2 = ['2.3.0', '2-latest', '2-main']; -const canaryFirmware = { - firmwares: ['2-main'], - tests: daily.tests, -}; +const inputs = [ + { + key: 'model', + value: ['T1B1', 'T2T1', 'T2B1', 'T3B1', 'T3T1'], + }, -const otherDevices = { - firmwares: ['2-latest'], - models: ['T2B1', 'T3T1'], - tests: [ - ...Object.values(groups).filter( - // management, btc-others are specified below - // nem, eos are not supported anymore - g => ['management', 'btc-others', 'nem', 'eos'].includes(g.name) === false, - ), - { - ...groups.management, - methods: groups.management.methods - .split(',') - // getFeatures test is not abstract enough to serve all models - .filter(m => m !== 'getFeatures') - .join(','), - }, - { - ...groups.btcOthers, - methods: groups.btcOthers.methods - .split(',') - // getAddress (decred) does not work for model R - .filter(m => m !== 'getAddress') - .join(','), + { + key: 'firmware', + value: ({ model }) => { + return model === 'T1B1' ? firmwares1 : firmwares2; }, - ], + }, + { + key: 'transport', + value: ['Bridge', 'NodeBridge'], + }, + { + key: 'groups', + value: Object.values(groups), + }, + { + key: 'env', + value: ['node', 'web'], + }, + { + key: 'cache_tx', + value: ['true', 'false'], + }, +]; + +// Get command-line arguments, excluding 'node' and the script name +const args = process.argv.slice(2); + +// Initialize an object to store the parsed arguments +const parsedArgs = {}; + +// Iterate over the arguments +args.forEach(arg => { + // Split each argument by '=' to separate the key and value + const [key, value] = arg.split('='); + + // Remove the leading '--' from the key + const argName = key.replace(/^--/, ''); + + // Check if the value contains commas to create an array + parsedArgs[argName] = value.includes(',') ? value.split(',') : value; +}); + +const validateArgs = () => { + const requiredArgs = ['model', 'firmware', 'env', 'groups', 'cache_tx', 'transport']; + + requiredArgs.forEach(arg => { + if (!parsedArgs[arg]) { + throw new Error(`Missing required argument: ${arg}`); + } + }); }; -const prepareTest = ({ firmwares, tests, models }) => { - const withFirmwares = tests.flatMap(test => firmwares.map(firmware => ({ firmware, ...test }))); +validateArgs(); + +/** + a method that takes inputs and creates all combinations of results like this: + {model: T1B1, firmware: 1.9.0, transport: 'Bridge' } + {model: T1B1, firmware: 1.9.0, transport: 'NodeBridge' } + {model: T1B1, firmware: 1-latest, transport: 'Bridge' } + {model: T1B1, firmware: 1-latest, transport: 'NodeBridge' } + {model: T1B1, firmware: 1-main, transport: 'Bridge' } + {model: T1B1, firmware: 1-main, transport: 'NodeBridge' } + {model: T2T1, firmware: 2.3.0, transport: 'Bridge' } + {model: T2T1, firmware: 2.3.0, transport: 'NodeBridge' } + */ +const createCartesian = inputs => { + const keys = inputs.map(m => m.key); + const values = inputs.map(m => m.value); + + const results = []; + const create = (index, current) => { + if (index === keys.length) { + results.push(current); + return; + } - if (models && models.length > 0) { - return withFirmwares.flatMap(test => models.map(model => ({ model, ...test }))); - } else { - return withFirmwares; - } + const key = keys[index]; + const value = typeof values[index] === 'function' ? values[index](current) : values[index]; + + for (let i = 0; i < value.length; i++) { + create(index + 1, { + ...current, + [key]: value[i], + }); + } + }; + + create(0, {}); + return results; }; -const testData = { - daily, - legacyFirmware, - canaryFirmware, - otherDevices, +const cartesian = createCartesian(inputs); + +/** + * filter cartesian by passed arguments + */ +const filterCartesianResultByArgs = () => { + const getValue = input => { + if (typeof input === 'object') { + return input.name; + } + return input; + }; + + return cartesian.filter(m => { + return Object.keys(m).every(key => { + const filterBy = parsedArgs[key]; + if (filterBy === 'all') return true; + if (Array.isArray(filterBy)) { + return filterBy.includes(getValue(m[key])); + } + return getValue(m[key]) === filterBy; + }); + }); }; -const args = process.argv.slice(2); -const [tests] = args; -const json = prepareTest(testData[tests]); +const filtered = filterCartesianResultByArgs(); + +// console.log('filtered', filtered); +// console.log('filtered.length', filtered.length); process.stdout.write( JSON.stringify({ - include: json, + include: filtered, }), );