From 34e69c9e11c057538c464953fcbfe948328e955a Mon Sep 17 00:00:00 2001 From: Simon K <6615834+simon-20@users.noreply.github.com> Date: Fri, 8 Nov 2024 11:27:26 +0000 Subject: [PATCH 1/6] fix: typo in test folder name --- .../validator-services-tests.postman_collection.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/integration-tests/validator-services-tests.postman_collection.json b/integration-tests/validator-services-tests.postman_collection.json index 2e3318b..fd079e7 100644 --- a/integration-tests/validator-services-tests.postman_collection.json +++ b/integration-tests/validator-services-tests.postman_collection.json @@ -1578,10 +1578,10 @@ ] }, { - "name": "Flagged Pulishers", + "name": "Flagged Publishers", "item": [ { - "name": "GET Black Flagged Publishers", + "name": "GET Flagged Publishers", "event": [ { "listen": "test", From 680fd3b00399d547c3c3a5e639f7fc90558a3283 Mon Sep 17 00:00:00 2001 From: Simon K <6615834+simon-20@users.noreply.github.com> Date: Fri, 8 Nov 2024 11:28:10 +0000 Subject: [PATCH 2/6] build: control test via environment variables --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 6e93e70..12cb283 100644 --- a/package.json +++ b/package.json @@ -7,7 +7,7 @@ "start": "func start --javascript --cors http://localhost:4200", "start:verbose": "func start --javascript --verbose", "test": "newman run integration-tests/validator-services-tests.postman_collection.json -e integration-tests/envs/local-function-7071.postman_environment.json --working-dir integration-tests/test-files", - "test-maintenance-mode": "newman run integration-tests/validator-services-tests.postman_collection.json -e integration-tests/envs/local-function-7071.postman_environment_maintenance_mode.json --working-dir integration-tests/test-files --folder 'Maintenance Mode'", + "test-maintenance-mode": "newman run integration-tests/validator-services-tests.postman_collection.json -e integration-tests/envs/local-function-7071.postman_environment.json --working-dir integration-tests/test-files --env-var testingMaintenanceMode=true --folder Initialisation --folder 'Publishers and Documents' --folder 'Guidance Links' --folder 'Flagged Publishers' --folder Statistics --folder 'Maintenance Mode'", "prepare": "husky install", "lint:eslint": "eslint .", "lint:prettier": "prettier . --check", From f9cb153e1925b670e834ad2dd35ee68b1c4a24e8 Mon Sep 17 00:00:00 2001 From: Simon K <6615834+simon-20@users.noreply.github.com> Date: Fri, 8 Nov 2024 11:39:27 +0000 Subject: [PATCH 3/6] build: test run dependent on maintenance mode --- .github/workflows/develop-func-deploy.yml | 13 ++++++++++++- .github/workflows/prod-func-deploy.yml | 13 ++++++++++++- 2 files changed, 24 insertions(+), 2 deletions(-) diff --git a/.github/workflows/develop-func-deploy.yml b/.github/workflows/develop-func-deploy.yml index bf4550a..ab10f83 100644 --- a/.github/workflows/develop-func-deploy.yml +++ b/.github/workflows/develop-func-deploy.yml @@ -170,13 +170,24 @@ jobs: run: | sudo npm i -g newman - - name: "Run integration tests" + - name: "Run integration tests (no maintenance mode)" run: | + [[ "$MAINTENANCE_MODE" == "DISABLED" ]] && newman run integration-tests/validator-services-tests.postman_collection.json \ -e integration-tests/envs/validator-services-direct-dev.postman_environment.json \ --env-var keyValue=$API_KEY \ --working-dir integration-tests/test-files + - name: "Run integration tests (maintenance mode)" + run: | + [[ "$MAINTENANCE_MODE" != "DISABLED" ]] && + newman run integration-tests/validator-services-tests.postman_collection.json \ + -e integration-tests/envs/validator-services-direct-dev.postman_environment.json \ + --env-var keyValue=$API_KEY \ + --env-var testingMaintenanceMode=true \ + --working-dir integration-tests/test-files \ + --folder Initialisation --folder 'Publishers and Documents' --folder 'Guidance Links' --folder 'Flagged Publishers' --folder Statistics --folder 'Maintenance Mode' + # Mark status checks success/fail on dependabot commits for scheduled deploys status_check_success: needs: [deploy, integration-test] diff --git a/.github/workflows/prod-func-deploy.yml b/.github/workflows/prod-func-deploy.yml index ac38ef3..296a5c6 100644 --- a/.github/workflows/prod-func-deploy.yml +++ b/.github/workflows/prod-func-deploy.yml @@ -156,9 +156,20 @@ jobs: run: | sudo npm i -g newman - - name: "Run integration tests" + - name: "Run integration tests (no maintenance mode)" run: | + [[ "$MAINTENANCE_MODE" == "DISABLED" ]] && newman run integration-tests/validator-services-tests.postman_collection.json \ -e integration-tests/envs/validator-services-direct-PROD.postman_environment.json \ --env-var keyValue=$API_KEY \ --working-dir integration-tests/test-files + + - name: "Run integration tests (maintenance mode)" + run: | + [[ "$MAINTENANCE_MODE" != "DISABLED" ]] && + newman run integration-tests/validator-services-tests.postman_collection.json \ + -e integration-tests/envs/validator-services-direct-PROD.postman_environment.json \ + --env-var keyValue=$API_KEY \ + --env-var testingMaintenanceMode=true \ + --working-dir integration-tests/test-files \ + --folder Initialisation --folder 'Publishers and Documents' --folder 'Guidance Links' --folder 'Flagged Publishers' --folder Statistics --folder 'Maintenance Mode' From 8f7a9487fb5dfadfb5061af0e630bc43173033ff Mon Sep 17 00:00:00 2001 From: Simon K <6615834+simon-20@users.noreply.github.com> Date: Fri, 22 Nov 2024 11:05:35 +0000 Subject: [PATCH 4/6] build: fix workflow running conditional tests --- .github/workflows/develop-func-deploy.yml | 8 ++++---- .github/workflows/prod-func-deploy.yml | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/develop-func-deploy.yml b/.github/workflows/develop-func-deploy.yml index ab10f83..2942601 100644 --- a/.github/workflows/develop-func-deploy.yml +++ b/.github/workflows/develop-func-deploy.yml @@ -170,17 +170,17 @@ jobs: run: | sudo npm i -g newman - - name: "Run integration tests (no maintenance mode)" + - name: "Run integration tests (when MAINTENANCE_MODE turned off)" + if: env.MAINTENANCE_MODE == 'DISABLED' run: | - [[ "$MAINTENANCE_MODE" == "DISABLED" ]] && newman run integration-tests/validator-services-tests.postman_collection.json \ -e integration-tests/envs/validator-services-direct-dev.postman_environment.json \ --env-var keyValue=$API_KEY \ --working-dir integration-tests/test-files - - name: "Run integration tests (maintenance mode)" + - name: "Run integration tests (when MAINTENANCE_MODE turned on)" + if: env.MAINTENANCE_MODE != 'DISABLED' run: | - [[ "$MAINTENANCE_MODE" != "DISABLED" ]] && newman run integration-tests/validator-services-tests.postman_collection.json \ -e integration-tests/envs/validator-services-direct-dev.postman_environment.json \ --env-var keyValue=$API_KEY \ diff --git a/.github/workflows/prod-func-deploy.yml b/.github/workflows/prod-func-deploy.yml index 296a5c6..a47da81 100644 --- a/.github/workflows/prod-func-deploy.yml +++ b/.github/workflows/prod-func-deploy.yml @@ -156,17 +156,17 @@ jobs: run: | sudo npm i -g newman - - name: "Run integration tests (no maintenance mode)" + - name: "Run integration tests (when MAINTENANCE_MODE turned off)" + if: env.MAINTENANCE_MODE == 'DISABLED' run: | - [[ "$MAINTENANCE_MODE" == "DISABLED" ]] && newman run integration-tests/validator-services-tests.postman_collection.json \ -e integration-tests/envs/validator-services-direct-PROD.postman_environment.json \ --env-var keyValue=$API_KEY \ --working-dir integration-tests/test-files - - name: "Run integration tests (maintenance mode)" + - name: "Run integration tests (when MAINTENANCE_MODE turned on)" + if: env.MAINTENANCE_MODE != 'DISABLED' run: | - [[ "$MAINTENANCE_MODE" != "DISABLED" ]] && newman run integration-tests/validator-services-tests.postman_collection.json \ -e integration-tests/envs/validator-services-direct-PROD.postman_environment.json \ --env-var keyValue=$API_KEY \ From e91a34e53d8abf4108d12ce0cf142a8b4d414037 Mon Sep 17 00:00:00 2001 From: Simon K <6615834+simon-20@users.noreply.github.com> Date: Fri, 22 Nov 2024 11:07:02 +0000 Subject: [PATCH 5/6] doc: expanded instructions; conditional tests note --- README.md | 83 ++++++++++++++++++++++++++++++++----------------------- 1 file changed, 49 insertions(+), 34 deletions(-) diff --git a/README.md b/README.md index 28e55c9..94c8f98 100644 --- a/README.md +++ b/README.md @@ -42,16 +42,19 @@ See OpenAPI specification `postman/schemas/index.yaml`. To view locally in Swagg 1. Follow instructions for nvm/node prerequisties above 1. Run `npm i` 1. Run `npm start` to run the function locally using the Azure Functions Core Tools + - Note: depending on which aspect of this app you are developing, or testing, running the app locally may require also running locally the validator ([js-validator-api](https://github.com/iati/js-validator-api)) and/or the database part of the unified pipeline refresher ([refresher]https://github.com/iati/refresher)). ## Environment Variables -### Set Up +### Initial set up -`cp .env.example .env` +#### `.env` app configuration file -### Description +When running locally the app is configured using the `.env` environment file. -#### .env +- Copy the example environment file + - `cp .env.example .env` +- Edit as needed, variables explained below: ```bash # DB connection @@ -68,11 +71,16 @@ ADHOC_CONTAINER= # validator API url and api key VALIDATOR_API_URL= VALIDATOR_FUNC_KEY= + +MAINTENANCE_MODE=DISABLED # DISABLED | NO_WRITE | NO_READ +MAINTENANCE_MODE_MESSAGE="The Validator website is in read-only mode for essential maintenance." ``` -#### local.settings.json +See below for explanation of `MAINTENANCE_MODE` variable. + +#### `local.settings.json` for VS Code -Required due to the storage binding used by this function +If you run the app through VS Code, you need to put the Azure details in `local.settings.json` as well as the `.env` file. This is needed due to the storage binding used by this function. ```json { @@ -98,22 +106,7 @@ the standard `DEV_` and `PROD_` prefixes. - `NO_WRITE` - those API end points which write to the Pipeline refresher DB are disabled - API end points: `adhoc/upload`, `adhoc/url`, `validation/regenerate`, `validation/regenerate/all`, `blob-trigger-adhoc-file` - These endpoints will return 503 Service Unavailable, with the message in `MAINTENANCE_MODE_MESSAGE` - -### Adding New - -Add in: - -1. .env.example -1. .env -1. `/config/config.js` - -Import - -``` -const config = require("./config"); - -let myEnvVariable = config.ENV_VAR -``` +- `NO_READ` - currently unused by this app, but the code which implements maintenance mode could be (spun out as module and) used elsewhere, so this common use-case was included. `NO_READ` implies `NO_WRITE`. ## Attached Debugging (VSCode) @@ -127,7 +120,11 @@ let myEnvVariable = config.ENV_VAR [ESlint](https://eslint.org/) is used for code quality and [Prettier](https://prettier.io/) is used for formatting rules, see [Prettier vs. Linters](https://prettier.io/docs/en/comparison). -To run linting, use `npm run lint`, and to format the project, use `npm run format`. +To run linting, use `npm run lint`. This will run `eslint` and `prettier` in checking/read-only mode, and show you if there are any errors. + +To format the code, run `npm run format`. + +There is a pre-commit hook which checks the code is formatted properly, so it should not be possible to commit without properly formatting the code. If this is bypassed, linting is also done on Github when creating a PR, and this will produce an error is the code is not properly formatted. ### VS Code Integration @@ -138,23 +135,45 @@ Recommended Plugins: - [ESLint](https://marketplace.visualstudio.com/items?itemName=dbaeumer.vscode-eslint) - [Prettier](https://marketplace.visualstudio.com/items?itemName=esbenp.prettier-vscode) -## Creating a new route +## Creating a new API route using the Azure Functions cli `func new --name --template "HTTP trigger" --authlevel "function"` ## Integration Tests -### Running +Integration tests are written in Postman v2.1 format. They should be edited in Postman, and then exported into the repository. During development the test suite can be run from Postman or locally using `newman`. During deployment, they are run on Github using `newman`. + +The tests currently run against an actual server (i.e., dev, production, or a local instance). This means the test runner does not control the instance being tested. This caused an issue with the new maintenance mode (introduced October 2024) because we expect different results according to whether maintenance mode is active or not, but the test runner can't control this. For now, the Github Actions workflow has been setup to run two different sets of tests: when maintenance mode is active, certain folders in the test suite aren't run. + +It would be better to spin up a dedicate test environment on Github using docker compose. There is a Github issue for doing this work: see #487. + +### Running the tests locally + +#### Against a local instance of `validator-services` + +If you want to test against a local instance of `validator-services` you need to have it pointed to a Unified Pipeline database and an Azure storage account. + +If you use a local database, you will need to ensure that you have run the Pipeline refresher with a reasonable number of datasets through to the validate stage, because the tests search for a dataset which has warnings, to check various things. - Install the dev dependencies, or ensure you have newman installed globally (`npm i newman -g`) - Start function `npm start` - Run Tests `npm test` -### Modifying/Adding +#### Against a remote instance -Integration tests are written in Postman v2.1 format and run with newman. +You can also run the tests locally against the dev or production instance. (This can be useful if a problem has been reported with dev or production). -#### Process for editing tests +To run against the dev instance: + +``` +npx newman run integration-tests/validator-services-tests.postman_collection.json -e integration-tests/envs/validator-services-direct-dev.postman_environment.json --working-dir integration-tests/test-files --env-var "keyValue=REPLACE_WITH_FUNCTIONS_KEY" +``` + +You need to replace `REPLACE_WITH_FUNCTIONS_KEY` with the Azure Functions App dev key. + +To run against production, use `integration-tests/envs/validator-services-direct-PROD.postman_environment.json` with the `-e` flag instead and use the production Azure key. + +### Modifying/Adding Tets 1. Check the tests in Postman are in sync with the Github repo: @@ -164,7 +183,7 @@ Integration tests are written in Postman v2.1 format and run with newman. 2. If this confirms the tests are in sync, then edit / update the tests in Postman. -3. When done, export from Postman again, format again, and commit. +3. When done editin gin Postman, export from Postman again, format again, and commit. ### Limitations @@ -174,16 +193,12 @@ The final folder of Postman tests is called `Maintenance Mode`. The app can be p As such, the Maintenance Mode tests have to be run separately from all the other tests, and they can only be run locally, not as part of the CI/CD pipeline. -To run the Maintenance Mode tests: +To run the Maintenance Mode tests locally: 1. Edit `.env` to turn on Maintenance Mode. 2. Start the application as normal (with `npm start`) 3. Run just the Maintenance Mode tests with `npm run test-maintenance-mode` -#### Testing a local instance - -If you want to test against a local instance of `validator-services` you need to have it pointed to a Unified Pipeline database. If you use a local database, you will need to ensure that you have run the Pipeline refresher with a reasonable number of datasets through to the validate stage, because the tests search for a dataset which has warnings, to check various things. - ## Release / Version Management https://github.com/IATI/IATI-Internal-Wiki#detailed-workflow-steps From 376a89181857e5cc917867100a99f60c19095008 Mon Sep 17 00:00:00 2001 From: Simon K <6615834+simon-20@users.noreply.github.com> Date: Thu, 28 Nov 2024 06:09:19 +0000 Subject: [PATCH 6/6] doc: fix README.md Co-authored-by: Tilly Woodfield <22456167+tillywoodfield@users.noreply.github.com> --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 94c8f98..01987c7 100644 --- a/README.md +++ b/README.md @@ -183,7 +183,7 @@ To run against production, use `integration-tests/envs/validator-services-direct 2. If this confirms the tests are in sync, then edit / update the tests in Postman. -3. When done editin gin Postman, export from Postman again, format again, and commit. +3. When done editing in Postman, export from Postman again, format again, and commit. ### Limitations