diff --git a/CHANGELOG.md b/CHANGELOG.md index 0b7ff1b3ef5..94244e8e86b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,7 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/). ## Unreleased +## [v13.4.0] 10/31/2022 ### Notable changes - Published new tag [`43` of `cumuluss/async-operation` to Docker Hub](https://hub.docker.com/layers/cumuluss/async-operation/43/images/sha256-5f989c7d45db3dde87c88c553182d1e4e250a1e09af691a84ff6aa683088b948?context=explore) which was built with node:14.19.3-buster. @@ -6570,7 +6571,8 @@ Note: There was an issue publishing 1.12.0. Upgrade to 1.12.1. ## [v1.0.0] - 2018-02-23 -[unreleased]: https://github.com/nasa/cumulus/compare/v13.3.2...HEAD +[unreleased]: https://github.com/nasa/cumulus/compare/v13.4.0...HEAD +[v13.4.0]: https://github.com/nasa/cumulus/compare/v13.3.2...v13.4.0 [v13.3.2]: https://github.com/nasa/cumulus/compare/v13.3.0...v13.3.2 [v13.3.0]: https://github.com/nasa/cumulus/compare/v13.2.1...v13.3.0 [v13.2.1]: https://github.com/nasa/cumulus/compare/v13.2.0...v13.2.1 diff --git a/CONTRIBUTORS.md b/CONTRIBUTORS.md index 7a7a13a5f79..a45be4a67b5 100644 --- a/CONTRIBUTORS.md +++ b/CONTRIBUTORS.md @@ -30,6 +30,7 @@ * Mark Schwab * Matt Savoie * Menno Van Diermen +* Nagavenkata Nageswaran * Nate Pauzenga * Pat Cappelaere * Patrick Quinn diff --git a/example/lambdas/asyncOperations/package.json b/example/lambdas/asyncOperations/package.json index f178bdcf53a..eeb4f8f7d38 100644 --- a/example/lambdas/asyncOperations/package.json +++ b/example/lambdas/asyncOperations/package.json @@ -1,6 +1,6 @@ { "name": "@cumulus/test-async-operations", - "version": "13.3.2", + "version": "13.4.0", "description": "AsyncOperations Test Lambda", "main": "index.js", "private": true, diff --git a/example/lambdas/ftpPopulateTestLambda/package.json b/example/lambdas/ftpPopulateTestLambda/package.json index eb7bb352ac1..15a3692d09d 100644 --- a/example/lambdas/ftpPopulateTestLambda/package.json +++ b/example/lambdas/ftpPopulateTestLambda/package.json @@ -1,6 +1,6 @@ { "name": "@cumulus/ftp-populate-test-lambda", - "version": "13.3.2", + "version": "13.4.0", "description": "FTP Population Utility Lambda", "main": "index.js", "private": true, @@ -19,12 +19,12 @@ "access": "private" }, "dependencies": { - "@cumulus/api": "13.3.2", - "@cumulus/api-client": "13.3.2", - "@cumulus/common": "13.3.2", - "@cumulus/integration-tests": "13.3.2", - "@cumulus/logger": "13.3.2", - "@cumulus/test-data": "13.3.2", + "@cumulus/api": "13.4.0", + "@cumulus/api-client": "13.4.0", + "@cumulus/common": "13.4.0", + "@cumulus/integration-tests": "13.4.0", + "@cumulus/logger": "13.4.0", + "@cumulus/test-data": "13.4.0", "aws-sdk": "^2.585.0", "fs-extra": "^9.0.0", "js-yaml": "^3.13.1", diff --git a/example/lambdas/lzardsClientTest/package.json b/example/lambdas/lzardsClientTest/package.json index 86af7920d23..a2d7f09a2c3 100644 --- a/example/lambdas/lzardsClientTest/package.json +++ b/example/lambdas/lzardsClientTest/package.json @@ -1,6 +1,6 @@ { "name": "@cumulus/test-lzards-api-lambda", - "version": "13.3.2", + "version": "13.4.0", "description": "LZARDS API Client Test Lambda", "private": true, "engines": { @@ -20,7 +20,7 @@ "author": "Cumulus Authors", "license": "Apache-2.0", "dependencies": { - "@cumulus/lzards-api-client": "13.3.2-alpha.0", - "@cumulus/logger": "13.3.2" + "@cumulus/logger": "13.4.0", + "@cumulus/lzards-api-client": "13.4.0" } } diff --git a/example/lambdas/python-processing/package.json b/example/lambdas/python-processing/package.json index 53fa19bddaf..44256bdb5a8 100644 --- a/example/lambdas/python-processing/package.json +++ b/example/lambdas/python-processing/package.json @@ -1,7 +1,7 @@ { "name": "@cumulus/python-process-activity", "private": true, - "version": "13.3.2", + "version": "13.4.0", "description": "Python reference activity", "homepage": "https://github.com/nasa/cumulus/tree/master/example/lambdas/python-reference-activity", "repository": { diff --git a/example/lambdas/python-reference-activity/package.json b/example/lambdas/python-reference-activity/package.json index b7b7bedb17c..934d93491f0 100644 --- a/example/lambdas/python-reference-activity/package.json +++ b/example/lambdas/python-reference-activity/package.json @@ -1,7 +1,7 @@ { "name": "@cumulus/python-reference-activity", "private": true, - "version": "13.3.2", + "version": "13.4.0", "description": "Python reference activity", "homepage": "https://github.com/nasa/cumulus/tree/master/example/lambdas/python-reference-activity", "repository": { diff --git a/example/lambdas/python-reference-task/package.json b/example/lambdas/python-reference-task/package.json index 46b27144ddd..0f6c1e4de5d 100644 --- a/example/lambdas/python-reference-task/package.json +++ b/example/lambdas/python-reference-task/package.json @@ -1,7 +1,7 @@ { "name": "@cumulus/python-reference-task", "private": true, - "version": "13.3.2", + "version": "13.4.0", "description": "Python reference task", "main": "index.js", "homepage": "https://github.com/nasa/cumulus/tree/master/example/lambdas/python-reference-task", diff --git a/example/lambdas/s3AccessTest/package.json b/example/lambdas/s3AccessTest/package.json index ccb66187a2b..817f669dee1 100644 --- a/example/lambdas/s3AccessTest/package.json +++ b/example/lambdas/s3AccessTest/package.json @@ -1,6 +1,6 @@ { "name": "@cumulus/test-s3-access", - "version": "13.3.2", + "version": "13.4.0", "description": "S3 Access Test Lambda", "main": "index.js", "private": true, diff --git a/example/lambdas/snsS3Test/package.json b/example/lambdas/snsS3Test/package.json index ad79eac961b..c7f24cbfd39 100644 --- a/example/lambdas/snsS3Test/package.json +++ b/example/lambdas/snsS3Test/package.json @@ -1,6 +1,6 @@ { "name": "@cumulus/test-sns-s3", - "version": "13.3.2", + "version": "13.4.0", "description": "SNS to S3 Test Lambda", "main": "index.js", "private": true, diff --git a/example/lambdas/versionUpTest/package.json b/example/lambdas/versionUpTest/package.json index 34bf634960b..ab214f51a96 100644 --- a/example/lambdas/versionUpTest/package.json +++ b/example/lambdas/versionUpTest/package.json @@ -1,6 +1,6 @@ { "name": "@cumulus/test-version-up", - "version": "13.3.2", + "version": "13.4.0", "description": "Version Up Test Lambda", "main": "index.js", "private": true, diff --git a/example/package.json b/example/package.json index a651fac971f..89e80051c10 100644 --- a/example/package.json +++ b/example/package.json @@ -1,6 +1,6 @@ { "name": "@cumulus/cumulus-integration-tests", - "version": "13.3.2", + "version": "13.4.0", "description": "Cumulus Integration Test Deployment", "private": true, "main": "index.js", @@ -45,32 +45,32 @@ ] }, "dependencies": { - "@cumulus/api": "13.3.2", - "@cumulus/api-client": "13.3.2", - "@cumulus/async-operations": "13.3.2", - "@cumulus/aws-client": "13.3.2", - "@cumulus/checksum": "13.3.2", - "@cumulus/cmr-client": "13.3.2", - "@cumulus/cmrjs": "13.3.2", - "@cumulus/common": "13.3.2", - "@cumulus/discover-granules": "13.3.2", - "@cumulus/discover-pdrs": "13.3.2", - "@cumulus/files-to-granules": "13.3.2", - "@cumulus/hello-world": "13.3.2", - "@cumulus/ingest": "13.3.2", - "@cumulus/integration-tests": "13.3.2", - "@cumulus/message": "13.3.2", - "@cumulus/move-granules": "13.3.2", - "@cumulus/parse-pdr": "13.3.2", - "@cumulus/pdr-status-check": "13.3.2", - "@cumulus/post-to-cmr": "13.3.2", - "@cumulus/queue-granules": "13.3.2", - "@cumulus/queue-pdrs": "13.3.2", - "@cumulus/sf-sqs-report": "13.3.2", - "@cumulus/sync-granule": "13.3.2", - "@cumulus/test-processing": "13.3.2" + "@cumulus/api": "13.4.0", + "@cumulus/api-client": "13.4.0", + "@cumulus/async-operations": "13.4.0", + "@cumulus/aws-client": "13.4.0", + "@cumulus/checksum": "13.4.0", + "@cumulus/cmr-client": "13.4.0", + "@cumulus/cmrjs": "13.4.0", + "@cumulus/common": "13.4.0", + "@cumulus/discover-granules": "13.4.0", + "@cumulus/discover-pdrs": "13.4.0", + "@cumulus/files-to-granules": "13.4.0", + "@cumulus/hello-world": "13.4.0", + "@cumulus/ingest": "13.4.0", + "@cumulus/integration-tests": "13.4.0", + "@cumulus/message": "13.4.0", + "@cumulus/move-granules": "13.4.0", + "@cumulus/parse-pdr": "13.4.0", + "@cumulus/pdr-status-check": "13.4.0", + "@cumulus/post-to-cmr": "13.4.0", + "@cumulus/queue-granules": "13.4.0", + "@cumulus/queue-pdrs": "13.4.0", + "@cumulus/sf-sqs-report": "13.4.0", + "@cumulus/sync-granule": "13.4.0", + "@cumulus/test-processing": "13.4.0" }, "devDependencies": { - "@cumulus/test-data": "13.3.2" + "@cumulus/test-data": "13.4.0" } } diff --git a/example/scripts/generate_ingest/package.json b/example/scripts/generate_ingest/package.json index 00badeda049..7f83e61ac40 100644 --- a/example/scripts/generate_ingest/package.json +++ b/example/scripts/generate_ingest/package.json @@ -1,7 +1,7 @@ { "name": "@cumulus/generate_ingest", "private": true, - "version": "13.3.2", + "version": "13.4.0", "description": "Script to generate test data for scaled ingest", "keywords": [ "GIBS", @@ -22,8 +22,8 @@ "directory": "packages/types" }, "dependencies": { - "@cumulus/aws-client": "13.3.2", - "@cumulus/common": "13.3.2" + "@cumulus/aws-client": "13.4.0", + "@cumulus/common": "13.4.0" }, "author": "Cumulus Authors", "license": "Apache-2.0" diff --git a/example/scripts/lib/package.json b/example/scripts/lib/package.json index dbf01295a34..9689fe8ae72 100644 --- a/example/scripts/lib/package.json +++ b/example/scripts/lib/package.json @@ -1,7 +1,7 @@ { "name": "@cumulus/example-lib", "private": true, - "version": "13.3.2", + "version": "13.4.0", "description": "example project libs", "homepage": "https://github.com/nasa/cumulus/tree/master/example/scripts/lib", "engines": { diff --git a/lambdas/data-migration1/package.json b/lambdas/data-migration1/package.json index bc6e0a29daa..7dd86af803b 100644 --- a/lambdas/data-migration1/package.json +++ b/lambdas/data-migration1/package.json @@ -1,6 +1,6 @@ { "name": "@cumulus/data-migration1", - "version": "13.3.2", + "version": "13.4.0", "description": "A Lambda function used for doing data migrations", "license": "Apache-2.0", "engines": { @@ -25,18 +25,18 @@ "timeout": "15m" }, "dependencies": { - "@cumulus/api": "13.3.2", - "@cumulus/aws-client": "13.3.2", - "@cumulus/common": "13.3.2", - "@cumulus/db": "13.3.2", - "@cumulus/errors": "13.3.2", - "@cumulus/logger": "13.3.2", - "@cumulus/types": "13.3.2", + "@cumulus/api": "13.4.0", + "@cumulus/aws-client": "13.4.0", + "@cumulus/common": "13.4.0", + "@cumulus/db": "13.4.0", + "@cumulus/errors": "13.4.0", + "@cumulus/logger": "13.4.0", + "@cumulus/types": "13.4.0", "knex": "0.95.15", "lodash": "^4.17.21", "pg": "^8.3.0" }, "devDependencies": { - "@cumulus/test-data": "13.3.2" + "@cumulus/test-data": "13.4.0" } } diff --git a/lambdas/data-migration2/package.json b/lambdas/data-migration2/package.json index 9d6d1af8966..373592a2905 100644 --- a/lambdas/data-migration2/package.json +++ b/lambdas/data-migration2/package.json @@ -1,6 +1,6 @@ { "name": "@cumulus/data-migration2", - "version": "13.3.2", + "version": "13.4.0", "description": "A Lambda function used for doing data migrations intended to be executed after data-migration1.", "license": "Apache-2.0", "engines": { @@ -28,14 +28,14 @@ "@aws-sdk/client-dynamodb": "^3.58.0", "@aws-sdk/lib-dynamodb": "^3.58.0", "@aws-sdk/util-dynamodb": "^3.58.0", - "@cumulus/api": "13.3.2", - "@cumulus/aws-client": "13.3.2", - "@cumulus/common": "13.3.2", - "@cumulus/db": "13.3.2", - "@cumulus/errors": "13.3.2", - "@cumulus/logger": "13.3.2", - "@cumulus/message": "13.3.2", - "@cumulus/types": "13.3.2", + "@cumulus/api": "13.4.0", + "@cumulus/aws-client": "13.4.0", + "@cumulus/common": "13.4.0", + "@cumulus/db": "13.4.0", + "@cumulus/errors": "13.4.0", + "@cumulus/logger": "13.4.0", + "@cumulus/message": "13.4.0", + "@cumulus/types": "13.4.0", "JSONStream": "1.3.5", "knex": "0.95.15", "lodash": "^4.17.21", @@ -43,6 +43,6 @@ "pg": "^8.3.0" }, "devDependencies": { - "@cumulus/test-data": "13.3.2" + "@cumulus/test-data": "13.4.0" } } diff --git a/lambdas/db-migration/package.json b/lambdas/db-migration/package.json index de50a1aff01..603957dd455 100644 --- a/lambdas/db-migration/package.json +++ b/lambdas/db-migration/package.json @@ -1,6 +1,6 @@ { "name": "@cumulus/db-migration-lambda", - "version": "13.3.2", + "version": "13.4.0", "description": "A Lambda function used for deploying DB migrations", "license": "Apache-2.0", "engines": { @@ -20,7 +20,7 @@ "tsc:listEmittedFiles": "../../node_modules/.bin/tsc --listEmittedFiles" }, "dependencies": { - "@cumulus/db": "13.3.2", + "@cumulus/db": "13.4.0", "knex": "0.95.15", "pg": "^8.3.0" } diff --git a/lambdas/db-provision-user-database/package.json b/lambdas/db-provision-user-database/package.json index 7792a822707..a3a1b40379d 100644 --- a/lambdas/db-provision-user-database/package.json +++ b/lambdas/db-provision-user-database/package.json @@ -1,6 +1,6 @@ { "name": "@cumulus/db-provision-user-database-lambda", - "version": "13.3.2", + "version": "13.4.0", "description": "A Lambda function used for provisioning user databases", "engines": { "node": ">=14.19.1" @@ -24,8 +24,8 @@ "timeout": "2m" }, "dependencies": { - "@cumulus/common": "13.3.2", - "@cumulus/db": "13.3.2", + "@cumulus/common": "13.4.0", + "@cumulus/db": "13.4.0", "knex": "0.95.15", "pg": "^8.3.0" }, diff --git a/lambdas/postgres-migration-async-operation/package.json b/lambdas/postgres-migration-async-operation/package.json index 7e84e038724..48a1ac63ebd 100644 --- a/lambdas/postgres-migration-async-operation/package.json +++ b/lambdas/postgres-migration-async-operation/package.json @@ -1,6 +1,6 @@ { "name": "@cumulus/postgres-migration-async-operation", - "version": "13.3.2", + "version": "13.4.0", "description": "A Lambda function used to start an ECS task to run data-migrations2 lambda", "license": "Apache-2.0", "engines": { @@ -28,12 +28,12 @@ "timeout": "15m" }, "dependencies": { - "@cumulus/api": "13.3.2", - "@cumulus/async-operations": "13.3.2", - "@cumulus/logger": "13.3.2", - "@cumulus/types": "13.3.2" + "@cumulus/api": "13.4.0", + "@cumulus/async-operations": "13.4.0", + "@cumulus/logger": "13.4.0", + "@cumulus/types": "13.4.0" }, "devDependencies": { - "@cumulus/test-data": "13.3.2" + "@cumulus/test-data": "13.4.0" } } diff --git a/lambdas/postgres-migration-count-tool/package.json b/lambdas/postgres-migration-count-tool/package.json index 7583cadf0ab..4360a1673f1 100644 --- a/lambdas/postgres-migration-count-tool/package.json +++ b/lambdas/postgres-migration-count-tool/package.json @@ -1,6 +1,6 @@ { "name": "@cumulus/postgres-reconcile", - "version": "13.3.2", + "version": "13.4.0", "description": "A Lambda function used for generating counts between Dynamo/ES and Postgres", "license": "Apache-2.0", "engines": { @@ -21,14 +21,14 @@ "tsc:listEmittedFiles": "../../node_modules/.bin/tsc --listEmittedFiles" }, "dependencies": { - "@cumulus/api": "13.3.2", - "@cumulus/api-client": "13.3.2", - "@cumulus/aws-client": "13.3.2", - "@cumulus/common": "13.3.2", - "@cumulus/db": "13.3.2", - "@cumulus/logger": "13.3.2", - "@cumulus/message": "13.3.2", - "@cumulus/types": "13.3.2", + "@cumulus/api": "13.4.0", + "@cumulus/api-client": "13.4.0", + "@cumulus/aws-client": "13.4.0", + "@cumulus/common": "13.4.0", + "@cumulus/db": "13.4.0", + "@cumulus/logger": "13.4.0", + "@cumulus/message": "13.4.0", + "@cumulus/types": "13.4.0", "knex": "0.95.15", "p-map": "^4.0.0", "pg": "^8.3.0" diff --git a/lambdas/sqs-message-remover/package.json b/lambdas/sqs-message-remover/package.json index 9768d23e003..b66dbabc1d7 100644 --- a/lambdas/sqs-message-remover/package.json +++ b/lambdas/sqs-message-remover/package.json @@ -1,6 +1,6 @@ { "name": "@cumulus/sqs-message-remover-lambda", - "version": "13.3.2", + "version": "13.4.0", "description": "Remove processed messages from SQS queues", "main": "src/index.js", "private": true, @@ -36,10 +36,10 @@ "author": "Cumulus Authors", "license": "Apache-2.0", "dependencies": { - "@cumulus/aws-client": "13.3.2", - "@cumulus/common": "13.3.2", - "@cumulus/ingest": "13.3.2", - "@cumulus/logger": "13.3.2", + "@cumulus/aws-client": "13.4.0", + "@cumulus/common": "13.4.0", + "@cumulus/ingest": "13.4.0", + "@cumulus/logger": "13.4.0", "lodash": "^4.17.21" } } diff --git a/lerna.json b/lerna.json index 633bef9dc6c..c7ddf2afb81 100644 --- a/lerna.json +++ b/lerna.json @@ -1,6 +1,6 @@ { "lerna": "3.20.2", - "version": "13.3.2", + "version": "13.4.0", "packages": [ "example", "example/lambdas/*", diff --git a/packages/api-client/package.json b/packages/api-client/package.json index 678ccee8175..7d8ec9471b3 100644 --- a/packages/api-client/package.json +++ b/packages/api-client/package.json @@ -1,6 +1,6 @@ { "name": "@cumulus/api-client", - "version": "13.3.2", + "version": "13.4.0", "description": "API client for working with the Cumulus archive API", "keywords": [ "GIBS", @@ -38,11 +38,11 @@ "author": "Cumulus Authors", "license": "Apache-2.0", "dependencies": { - "@cumulus/aws-client": "13.3.2", - "@cumulus/logger": "13.3.2", + "@cumulus/aws-client": "13.4.0", + "@cumulus/logger": "13.4.0", "p-retry": "^2.0.0" }, "devDependencies": { - "@cumulus/types": "13.3.2" + "@cumulus/types": "13.4.0" } } diff --git a/packages/api/ecs/async-operation/package.json b/packages/api/ecs/async-operation/package.json index b0dc113ca65..0dd2d444195 100644 --- a/packages/api/ecs/async-operation/package.json +++ b/packages/api/ecs/async-operation/package.json @@ -1,6 +1,6 @@ { "name": "@cumulus/ecs-async-operation", - "version": "13.3.2", + "version": "13.4.0", "description": "The docker image for running async operations", "keywords": [ "NASA", @@ -20,10 +20,10 @@ "test": "../../../../node_modules/.bin/ava" }, "dependencies": { - "@cumulus/aws-client": "13.3.2", - "@cumulus/db": "13.3.2", - "@cumulus/es-client": "13.3.2", - "@cumulus/logger": "13.3.2", + "@cumulus/aws-client": "13.4.0", + "@cumulus/db": "13.4.0", + "@cumulus/es-client": "13.4.0", + "@cumulus/logger": "13.4.0", "aws-sdk": "^2.585.0", "crypto-random-string": "^3.2.0", "got": "^11.8.5", diff --git a/packages/api/package.json b/packages/api/package.json index 60f515e8082..44fbcde40f5 100644 --- a/packages/api/package.json +++ b/packages/api/package.json @@ -1,6 +1,6 @@ { "name": "@cumulus/api", - "version": "13.3.2", + "version": "13.4.0", "description": "Lambda functions for handling all daac's API operations", "main": "index.js", "engines": { @@ -52,26 +52,26 @@ "author": "Cumulus Authors", "license": "Apache-2.0", "dependencies": { - "@cumulus/api-client": "13.3.2", - "@cumulus/async-operations": "13.3.2", - "@cumulus/aws-client": "13.3.2", - "@cumulus/cmr-client": "13.3.2", - "@cumulus/cmrjs": "13.3.2", - "@cumulus/collection-config-store": "13.3.2", - "@cumulus/common": "13.3.2", - "@cumulus/db": "13.3.2", - "@cumulus/distribution-utils": "13.3.2", - "@cumulus/errors": "13.3.2", - "@cumulus/es-client": "13.3.2", - "@cumulus/ingest": "13.3.2", - "@cumulus/launchpad-auth": "13.3.2", - "@cumulus/logger": "13.3.2", - "@cumulus/message": "13.3.2", - "@cumulus/oauth-client": "13.3.2", - "@cumulus/object-store": "13.3.2", - "@cumulus/pvl": "13.3.2", - "@cumulus/sftp-client": "13.3.2", - "@cumulus/types": "13.3.2", + "@cumulus/api-client": "13.4.0", + "@cumulus/async-operations": "13.4.0", + "@cumulus/aws-client": "13.4.0", + "@cumulus/cmr-client": "13.4.0", + "@cumulus/cmrjs": "13.4.0", + "@cumulus/collection-config-store": "13.4.0", + "@cumulus/common": "13.4.0", + "@cumulus/db": "13.4.0", + "@cumulus/distribution-utils": "13.4.0", + "@cumulus/errors": "13.4.0", + "@cumulus/es-client": "13.4.0", + "@cumulus/ingest": "13.4.0", + "@cumulus/launchpad-auth": "13.4.0", + "@cumulus/logger": "13.4.0", + "@cumulus/message": "13.4.0", + "@cumulus/oauth-client": "13.4.0", + "@cumulus/object-store": "13.4.0", + "@cumulus/pvl": "13.4.0", + "@cumulus/sftp-client": "13.4.0", + "@cumulus/types": "13.4.0", "@mapbox/dyno": "^1.4.2", "aggregate-error": "^3.1.0", "ajv": "^6.12.3", @@ -116,6 +116,6 @@ "xml2js": "^0.4.22" }, "devDependencies": { - "@cumulus/test-data": "13.3.2" + "@cumulus/test-data": "13.4.0" } } diff --git a/packages/async-operations/package.json b/packages/async-operations/package.json index 54a93ab9cd1..6030d50fea5 100644 --- a/packages/async-operations/package.json +++ b/packages/async-operations/package.json @@ -1,6 +1,6 @@ { "name": "@cumulus/async-operations", - "version": "13.3.2", + "version": "13.4.0", "description": "Cumulus Core internal async operations module", "main": "./dist/index.js", "types": "./dist/index.d.ts", @@ -28,17 +28,17 @@ "author": "Cumulus Authors", "license": "Apache-2.0", "dependencies": { - "@cumulus/aws-client": "13.3.2", - "@cumulus/db": "13.3.2", - "@cumulus/errors": "13.3.2", - "@cumulus/es-client": "13.3.2", - "@cumulus/logger": "13.3.2", - "@cumulus/types": "13.3.2", + "@cumulus/aws-client": "13.4.0", + "@cumulus/db": "13.4.0", + "@cumulus/errors": "13.4.0", + "@cumulus/es-client": "13.4.0", + "@cumulus/logger": "13.4.0", + "@cumulus/types": "13.4.0", "knex": "0.95.15", "uuid": "8.3.2" }, "devDependencies": { - "@cumulus/common": "13.3.2", + "@cumulus/common": "13.4.0", "@types/aws-sdk": "2.7.0", "@types/uuid": "^8.0.0" } diff --git a/packages/aws-client/package.json b/packages/aws-client/package.json index d23abe9ff4e..7f9086b06c8 100644 --- a/packages/aws-client/package.json +++ b/packages/aws-client/package.json @@ -1,6 +1,6 @@ { "name": "@cumulus/aws-client", - "version": "13.3.2", + "version": "13.4.0", "description": "Utilities for working with AWS", "keywords": [ "GIBS", @@ -52,9 +52,9 @@ "@aws-sdk/s3-request-presigner": "^3.58.0", "@aws-sdk/signature-v4-crt": "^3.58.0", "@aws-sdk/types": "^3.58.0", - "@cumulus/checksum": "13.3.2", - "@cumulus/errors": "13.3.2", - "@cumulus/logger": "13.3.2", + "@cumulus/checksum": "13.4.0", + "@cumulus/errors": "13.4.0", + "@cumulus/logger": "13.4.0", "aws-sdk": "^2.585.0", "jsonpath-plus": "^1.1.0", "lodash": "~4.17.21", diff --git a/packages/checksum/package.json b/packages/checksum/package.json index 036d1d80de3..73b964f699c 100644 --- a/packages/checksum/package.json +++ b/packages/checksum/package.json @@ -1,6 +1,6 @@ { "name": "@cumulus/checksum", - "version": "13.3.2", + "version": "13.4.0", "description": "Cumulus checksum utilities", "engines": { "node": ">=14.19.1" diff --git a/packages/cmr-client/package.json b/packages/cmr-client/package.json index d0c736dacca..408d8aaaa33 100644 --- a/packages/cmr-client/package.json +++ b/packages/cmr-client/package.json @@ -1,6 +1,6 @@ { "name": "@cumulus/cmr-client", - "version": "13.3.2", + "version": "13.4.0", "description": "A Node.js client to NASA's Common Metadata Repository (CMR) API.", "engines": { "node": ">=14.19.1" @@ -34,9 +34,9 @@ "author": "Cumulus Authors", "license": "Apache-2.0", "dependencies": { - "@cumulus/aws-client": "13.3.2", - "@cumulus/errors": "13.3.2", - "@cumulus/logger": "13.3.2", + "@cumulus/aws-client": "13.4.0", + "@cumulus/errors": "13.4.0", + "@cumulus/logger": "13.4.0", "got": "^11.8.5", "lodash": "^4.17.21", "public-ip": "^3.0.0", diff --git a/packages/cmrjs/package.json b/packages/cmrjs/package.json index dabab16a4af..1397f11da62 100644 --- a/packages/cmrjs/package.json +++ b/packages/cmrjs/package.json @@ -1,6 +1,6 @@ { "name": "@cumulus/cmrjs", - "version": "13.3.2", + "version": "13.4.0", "description": "A node SDK for CMR", "engines": { "node": ">=14.19.1" @@ -33,13 +33,13 @@ "author": "Cumulus Authors", "license": "Apache-2.0", "dependencies": { - "@cumulus/aws-client": "13.3.2", - "@cumulus/cmr-client": "13.3.2", - "@cumulus/common": "13.3.2", - "@cumulus/distribution-utils": "13.3.2", - "@cumulus/errors": "13.3.2", - "@cumulus/launchpad-auth": "13.3.2", - "@cumulus/logger": "13.3.2", + "@cumulus/aws-client": "13.4.0", + "@cumulus/cmr-client": "13.4.0", + "@cumulus/common": "13.4.0", + "@cumulus/distribution-utils": "13.4.0", + "@cumulus/errors": "13.4.0", + "@cumulus/launchpad-auth": "13.4.0", + "@cumulus/logger": "13.4.0", "got": "^11.8.1", "js2xmlparser": "^4.0.0", "lodash": "^4.17.21", diff --git a/packages/collection-config-store/package.json b/packages/collection-config-store/package.json index f1e9ca307ce..42da6d4822a 100644 --- a/packages/collection-config-store/package.json +++ b/packages/collection-config-store/package.json @@ -1,6 +1,6 @@ { "name": "@cumulus/collection-config-store", - "version": "13.3.2", + "version": "13.4.0", "description": "Utility for persisting collection configuration to S3 and retrieving it", "keywords": [ "CUMULUS", @@ -32,8 +32,8 @@ "author": "Cumulus Authors", "license": "Apache-2.0", "dependencies": { - "@cumulus/aws-client": "13.3.2", - "@cumulus/common": "13.3.2", - "@cumulus/message": "13.3.2" + "@cumulus/aws-client": "13.4.0", + "@cumulus/common": "13.4.0", + "@cumulus/message": "13.4.0" } } diff --git a/packages/common/package.json b/packages/common/package.json index 0cb9d33b5e5..c666314f1b8 100644 --- a/packages/common/package.json +++ b/packages/common/package.json @@ -1,6 +1,6 @@ { "name": "@cumulus/common", - "version": "13.3.2", + "version": "13.4.0", "description": "Common utilities used across tasks", "keywords": [ "GIBS", @@ -43,9 +43,9 @@ "dependencies": { "@aws-sdk/client-s3": "^3.58.0", "@aws-sdk/signature-v4-crt": "^3.58.0", - "@cumulus/aws-client": "13.3.2", - "@cumulus/errors": "13.3.2", - "@cumulus/logger": "13.3.2", + "@cumulus/aws-client": "13.4.0", + "@cumulus/errors": "13.4.0", + "@cumulus/logger": "13.4.0", "ajv": "^6.12.3", "aws-sdk": "^2.585.0", "follow-redirects": "^1.2.4", diff --git a/packages/db/package.json b/packages/db/package.json index 679d1413584..6a2f64b9ff9 100644 --- a/packages/db/package.json +++ b/packages/db/package.json @@ -1,6 +1,6 @@ { "name": "@cumulus/db", - "version": "13.3.2", + "version": "13.4.0", "description": "Utilities for working with the Cumulus DB", "license": "Apache-2.0", "main": "./dist/index.js", @@ -29,12 +29,12 @@ "node": ">=14.19.1" }, "dependencies": { - "@cumulus/aws-client": "13.3.2", - "@cumulus/common": "13.3.2", - "@cumulus/errors": "13.3.2", - "@cumulus/logger": "13.3.2", - "@cumulus/message": "13.3.2", - "@cumulus/types": "13.3.2", + "@cumulus/aws-client": "13.4.0", + "@cumulus/common": "13.4.0", + "@cumulus/errors": "13.4.0", + "@cumulus/logger": "13.4.0", + "@cumulus/message": "13.4.0", + "@cumulus/types": "13.4.0", "crypto-random-string": "^3.2.0", "is-valid-hostname": "1.0.2", "knex": "0.95.15", diff --git a/packages/distribution-utils/package.json b/packages/distribution-utils/package.json index 3acb86c3e36..c28213a9e2a 100644 --- a/packages/distribution-utils/package.json +++ b/packages/distribution-utils/package.json @@ -1,6 +1,6 @@ { "name": "@cumulus/distribution-utils", - "version": "13.3.2", + "version": "13.4.0", "description": "Cumulus Distribution utilities", "keywords": [ "CUMULUS" @@ -33,9 +33,9 @@ "author": "Cumulus Authors", "license": "Apache-2.0", "dependencies": { - "@cumulus/aws-client": "13.3.2", - "@cumulus/common": "13.3.2", - "@cumulus/errors": "13.3.2", + "@cumulus/aws-client": "13.4.0", + "@cumulus/common": "13.4.0", + "@cumulus/errors": "13.4.0", "url-join": "^1.1.0" }, "devDependencies": { diff --git a/packages/errors/package.json b/packages/errors/package.json index 42081d53bde..afcb93ef827 100644 --- a/packages/errors/package.json +++ b/packages/errors/package.json @@ -1,6 +1,6 @@ { "name": "@cumulus/errors", - "version": "13.3.2", + "version": "13.4.0", "description": "Provides error classes for Cumulus", "keywords": [ "GIBS", diff --git a/packages/es-client/package.json b/packages/es-client/package.json index dba41f66d03..55c1c4f73e3 100644 --- a/packages/es-client/package.json +++ b/packages/es-client/package.json @@ -1,6 +1,6 @@ { "name": "@cumulus/es-client", - "version": "13.3.2", + "version": "13.4.0", "description": "Utilities for working with Elasticsearch", "keywords": [ "CUMULUS", @@ -30,10 +30,10 @@ "author": "Cumulus Authors", "license": "Apache-2.0", "dependencies": { - "@cumulus/common": "13.3.2", - "@cumulus/errors": "13.3.2", - "@cumulus/logger": "13.3.2", - "@cumulus/message": "13.3.2", + "@cumulus/common": "13.4.0", + "@cumulus/errors": "13.4.0", + "@cumulus/logger": "13.4.0", + "@cumulus/message": "13.4.0", "@elastic/elasticsearch": "^5.6.20", "aws-elasticsearch-connector": "8.2.0", "aws-sdk": "^2.585.0", @@ -42,8 +42,8 @@ "p-limit": "^1.2.0" }, "devDependencies": { - "@cumulus/aws-client": "13.3.2", - "@cumulus/test-data": "13.3.2", + "@cumulus/aws-client": "13.4.0", + "@cumulus/test-data": "13.4.0", "p-each-series": "^2.1.0" } } diff --git a/packages/ingest/package.json b/packages/ingest/package.json index 5cd9e1f84fd..e4e2e92c2e0 100644 --- a/packages/ingest/package.json +++ b/packages/ingest/package.json @@ -1,6 +1,6 @@ { "name": "@cumulus/ingest", - "version": "13.3.2", + "version": "13.4.0", "description": "Ingest utilities", "engines": { "node": ">=14.19.1" @@ -38,13 +38,13 @@ "author": "Cumulus Authors", "license": "Apache-2.0", "dependencies": { - "@cumulus/aws-client": "13.3.2", - "@cumulus/common": "13.3.2", - "@cumulus/db": "13.3.2", - "@cumulus/errors": "13.3.2", - "@cumulus/logger": "13.3.2", - "@cumulus/message": "13.3.2", - "@cumulus/sftp-client": "13.3.2", + "@cumulus/aws-client": "13.4.0", + "@cumulus/common": "13.4.0", + "@cumulus/db": "13.4.0", + "@cumulus/errors": "13.4.0", + "@cumulus/logger": "13.4.0", + "@cumulus/message": "13.4.0", + "@cumulus/sftp-client": "13.4.0", "aws-sdk": "^2.585.0", "cksum": "^1.3.0", "delay": "^4.3.0", @@ -61,9 +61,9 @@ "tough-cookie": "~4.0.0" }, "devDependencies": { - "@cumulus/checksum": "13.3.2", - "@cumulus/cmrjs": "13.3.2", - "@cumulus/test-data": "13.3.2", - "@cumulus/types": "13.3.2" + "@cumulus/checksum": "13.4.0", + "@cumulus/cmrjs": "13.4.0", + "@cumulus/test-data": "13.4.0", + "@cumulus/types": "13.4.0" } } diff --git a/packages/integration-tests/package.json b/packages/integration-tests/package.json index 3db01611231..4bc19986880 100644 --- a/packages/integration-tests/package.json +++ b/packages/integration-tests/package.json @@ -1,6 +1,6 @@ { "name": "@cumulus/integration-tests", - "version": "13.3.2", + "version": "13.4.0", "description": "Integration tests", "bin": { "cumulus-test": "./bin/cli.js" @@ -28,16 +28,16 @@ "author": "Cumulus Authors", "license": "Apache-2.0", "dependencies": { - "@cumulus/api": "13.3.2", - "@cumulus/api-client": "13.3.2", - "@cumulus/aws-client": "13.3.2", - "@cumulus/cmr-client": "13.3.2", - "@cumulus/cmrjs": "13.3.2", - "@cumulus/common": "13.3.2", - "@cumulus/launchpad-auth": "13.3.2", - "@cumulus/logger": "13.3.2", - "@cumulus/message": "13.3.2", - "@cumulus/oauth-client": "13.3.2", + "@cumulus/api": "13.4.0", + "@cumulus/api-client": "13.4.0", + "@cumulus/aws-client": "13.4.0", + "@cumulus/cmr-client": "13.4.0", + "@cumulus/cmrjs": "13.4.0", + "@cumulus/common": "13.4.0", + "@cumulus/launchpad-auth": "13.4.0", + "@cumulus/logger": "13.4.0", + "@cumulus/message": "13.4.0", + "@cumulus/oauth-client": "13.4.0", "aws-sdk": "^2.585.0", "base-64": "^0.1.0", "commander": "^2.15.0", diff --git a/packages/launchpad-auth/package.json b/packages/launchpad-auth/package.json index 4786524ee65..ba6c27492b7 100644 --- a/packages/launchpad-auth/package.json +++ b/packages/launchpad-auth/package.json @@ -1,6 +1,6 @@ { "name": "@cumulus/launchpad-auth", - "version": "13.3.2", + "version": "13.4.0", "description": "Utilities for authentication with Launchpad", "keywords": [ "CUMULUS", @@ -37,8 +37,8 @@ "author": "Cumulus Authors", "license": "Apache-2.0", "dependencies": { - "@cumulus/aws-client": "13.3.2", - "@cumulus/logger": "13.3.2", + "@cumulus/aws-client": "13.4.0", + "@cumulus/logger": "13.4.0", "got": "^11.8.5", "lodash": "^4.17.21", "uuid": "^3.2.1" diff --git a/packages/logger/package.json b/packages/logger/package.json index f3f2d9bd7ad..a1f256520cb 100644 --- a/packages/logger/package.json +++ b/packages/logger/package.json @@ -1,6 +1,6 @@ { "name": "@cumulus/logger", - "version": "13.3.2", + "version": "13.4.0", "description": "A log library for use on Cumulus", "keywords": [ "GIBS", diff --git a/packages/lzards-api-client/package.json b/packages/lzards-api-client/package.json index 1dff9d486f1..76fd34c94fa 100644 --- a/packages/lzards-api-client/package.json +++ b/packages/lzards-api-client/package.json @@ -1,6 +1,6 @@ { "name": "@cumulus/lzards-api-client", - "version": "13.3.2-alpha.0", + "version": "13.4.0", "description": "A Node.js client to NASA's Level Zero and Repositories Data Store (LZARDS) API.", "engines": { "node": ">=14.19.1" @@ -33,11 +33,11 @@ "author": "Cumulus Authors", "license": "Apache-2.0", "dependencies": { - "@cumulus/aws-client": "13.3.2", - "@cumulus/common": "13.3.2", - "@cumulus/errors": "13.3.2", - "@cumulus/launchpad-auth": "13.3.2", - "@cumulus/logger": "13.3.2", + "@cumulus/aws-client": "13.4.0", + "@cumulus/common": "13.4.0", + "@cumulus/errors": "13.4.0", + "@cumulus/launchpad-auth": "13.4.0", + "@cumulus/logger": "13.4.0", "got": "11.8.5", "lodash": "^4.17.21" } diff --git a/packages/message/package.json b/packages/message/package.json index 1427f4bd167..ae647c05e5d 100644 --- a/packages/message/package.json +++ b/packages/message/package.json @@ -1,6 +1,6 @@ { "name": "@cumulus/message", - "version": "13.3.2", + "version": "13.4.0", "description": "Utilities for building and parsing Cumulus messages", "keywords": [ "GIBS", @@ -38,11 +38,11 @@ "author": "Cumulus Authors", "license": "Apache-2.0", "dependencies": { - "@cumulus/aws-client": "13.3.2", - "@cumulus/common": "13.3.2", - "@cumulus/errors": "13.3.2", - "@cumulus/logger": "13.3.2", - "@cumulus/types": "13.3.2", + "@cumulus/aws-client": "13.4.0", + "@cumulus/common": "13.4.0", + "@cumulus/errors": "13.4.0", + "@cumulus/logger": "13.4.0", + "@cumulus/types": "13.4.0", "jsonpath-plus": "^3.0.0", "lodash": "^4.17.21", "uuid": "^8.2.0" diff --git a/packages/oauth-client/package.json b/packages/oauth-client/package.json index 099f1a4615c..ebdd3025156 100644 --- a/packages/oauth-client/package.json +++ b/packages/oauth-client/package.json @@ -1,6 +1,6 @@ { "name": "@cumulus/oauth-client", - "version": "13.3.2", + "version": "13.4.0", "description": "A generic auth client", "homepage": "https://github.com/nasa/cumulus/tree/master/packages/oauth-client#readme", "repository": { diff --git a/packages/object-store/package.json b/packages/object-store/package.json index 6978e23d601..32f1d63f9b1 100644 --- a/packages/object-store/package.json +++ b/packages/object-store/package.json @@ -1,6 +1,6 @@ { "name": "@cumulus/object-store", - "version": "13.3.2", + "version": "13.4.0", "description": "Utilities for managing object stores", "keywords": [ "GIBS", @@ -39,6 +39,6 @@ "author": "Cumulus Authors", "license": "Apache-2.0", "dependencies": { - "@cumulus/aws-client": "13.3.2" + "@cumulus/aws-client": "13.4.0" } } diff --git a/packages/pvl/package.json b/packages/pvl/package.json index 4a476b22f7c..133e9fc2635 100644 --- a/packages/pvl/package.json +++ b/packages/pvl/package.json @@ -1,6 +1,6 @@ { "name": "@cumulus/pvl", - "version": "13.3.2", + "version": "13.4.0", "description": "Parse and serialize Parameter Value Language, a data markup language used by NASA", "main": "index.js", "engine": { diff --git a/packages/s3-credentials-endpoint/package.json b/packages/s3-credentials-endpoint/package.json index 21208158bb8..450f97ee199 100644 --- a/packages/s3-credentials-endpoint/package.json +++ b/packages/s3-credentials-endpoint/package.json @@ -1,6 +1,6 @@ { "name": "@cumulus/s3-credentials-endpoint", - "version": "13.3.2", + "version": "13.4.0", "description": "An API Gateway Lambda to return AWS credentials for fetching objects from S3", "license": "Apache-2.0", "engines": { @@ -17,12 +17,12 @@ "timeout": "15m" }, "dependencies": { - "@cumulus/api": "13.3.2", - "@cumulus/aws-client": "13.3.2", - "@cumulus/cmrjs": "13.3.2", - "@cumulus/errors": "13.3.2", - "@cumulus/logger": "13.3.2", - "@cumulus/oauth-client": "13.3.2", + "@cumulus/api": "13.4.0", + "@cumulus/aws-client": "13.4.0", + "@cumulus/cmrjs": "13.4.0", + "@cumulus/errors": "13.4.0", + "@cumulus/logger": "13.4.0", + "@cumulus/oauth-client": "13.4.0", "aws-serverless-express": "^3.3.6", "body-parser": "^1.19.0", "cookie-parser": "^1.4.4", diff --git a/packages/schemas/package.json b/packages/schemas/package.json index 87c12adf5e7..86dc2bdf033 100644 --- a/packages/schemas/package.json +++ b/packages/schemas/package.json @@ -1,6 +1,6 @@ { "name": "@cumulus/schemas", - "version": "13.3.2", + "version": "13.4.0", "description": "Helpers for managing Cumulus task schemas", "homepage": "https://github.com/nasa/cumulus/tree/master/packages/schemas", "repository": { diff --git a/packages/sftp-client/package.json b/packages/sftp-client/package.json index 5e366e700a3..1038f0c9318 100644 --- a/packages/sftp-client/package.json +++ b/packages/sftp-client/package.json @@ -1,6 +1,6 @@ { "name": "@cumulus/sftp-client", - "version": "13.3.2", + "version": "13.4.0", "description": "A Promise-based SFTP client", "keywords": [ "GIBS", @@ -36,16 +36,16 @@ "author": "Cumulus Authors", "license": "Apache-2.0", "dependencies": { - "@cumulus/aws-client": "13.3.2", - "@cumulus/common": "13.3.2", + "@cumulus/aws-client": "13.4.0", + "@cumulus/common": "13.4.0", "lodash": "^4.17.21", "mime-types": "^2.1.27", "ssh2": "^1.0.0", "ssh2-sftp-client": "^7.0.4" }, "devDependencies": { - "@cumulus/checksum": "13.3.2", - "@cumulus/test-data": "13.3.2", + "@cumulus/checksum": "13.4.0", + "@cumulus/test-data": "13.4.0", "@types/ssh2-sftp-client": "^7.0.0" } } diff --git a/packages/tea-map-cache/package.json b/packages/tea-map-cache/package.json index 2c2729ee91d..7ea074633af 100644 --- a/packages/tea-map-cache/package.json +++ b/packages/tea-map-cache/package.json @@ -1,6 +1,6 @@ { "name": "@cumulus/tea-map-cache", - "version": "13.3.2", + "version": "13.4.0", "description": "Tea Bucket Map Cache Writer", "main": "index.js", "engines": { @@ -27,8 +27,8 @@ "author": "Cumulus Authors", "license": "Apache-2.0", "dependencies": { - "@cumulus/aws-client": "13.3.2", - "@cumulus/logger": "13.3.2", + "@cumulus/aws-client": "13.4.0", + "@cumulus/logger": "13.4.0", "got": "^11.8.5", "p-retry": "^4.2.0" }, diff --git a/packages/test-data/package.json b/packages/test-data/package.json index 1cdefe21bf2..a6fe0a7ac8b 100644 --- a/packages/test-data/package.json +++ b/packages/test-data/package.json @@ -1,6 +1,6 @@ { "name": "@cumulus/test-data", - "version": "13.3.2", + "version": "13.4.0", "description": "Includes the test data for various packages", "keywords": [ "GIBS", diff --git a/packages/tf-inventory/package.json b/packages/tf-inventory/package.json index f85ad1323f0..4ba66267a19 100644 --- a/packages/tf-inventory/package.json +++ b/packages/tf-inventory/package.json @@ -1,6 +1,6 @@ { "name": "@cumulus/tf-inventory", - "version": "13.3.2", + "version": "13.4.0", "description": "Package to help keep track of what resources are managed by Terraform in the AWS account", "main": "index.js", "engines": { @@ -30,11 +30,11 @@ "author": "Cumulus Authors", "license": "Apache-2.0", "dependencies": { - "@cumulus/aws-client": "13.3.2", + "@cumulus/aws-client": "13.4.0", "commander": "^4.1.0", "lodash": "^4.17.21" }, "devDependencies": { - "@cumulus/common": "13.3.2" + "@cumulus/common": "13.4.0" } } diff --git a/packages/types/package.json b/packages/types/package.json index 2405f909de2..6bd5811a276 100644 --- a/packages/types/package.json +++ b/packages/types/package.json @@ -1,6 +1,6 @@ { "name": "@cumulus/types", - "version": "13.3.2", + "version": "13.4.0", "description": "TypeScript definitions for working with Cumulus data structures", "keywords": [ "GIBS", diff --git a/tasks/add-missing-file-checksums/package.json b/tasks/add-missing-file-checksums/package.json index 0d41240deba..a155b269a51 100644 --- a/tasks/add-missing-file-checksums/package.json +++ b/tasks/add-missing-file-checksums/package.json @@ -1,6 +1,6 @@ { "name": "@cumulus/add-missing-file-checksums", - "version": "13.3.2", + "version": "13.4.0", "description": "Add checksums to files in S3 which don't have one", "author": "Cumulus Authors", "license": "Apache-2.0", @@ -42,12 +42,12 @@ } }, "dependencies": { - "@cumulus/aws-client": "13.3.2", + "@cumulus/aws-client": "13.4.0", "@cumulus/cumulus-message-adapter-js": "2.0.4" }, "devDependencies": { - "@cumulus/schemas": "13.3.2", - "@cumulus/types": "13.3.2", + "@cumulus/schemas": "13.4.0", + "@cumulus/types": "13.4.0", "@types/aws-lambda": "^8.10.58" } } diff --git a/tasks/discover-granules/package.json b/tasks/discover-granules/package.json index 0a8a7416a98..a1e25ed4420 100644 --- a/tasks/discover-granules/package.json +++ b/tasks/discover-granules/package.json @@ -1,6 +1,6 @@ { "name": "@cumulus/discover-granules", - "version": "13.3.2", + "version": "13.4.0", "description": "Discover Granules in FTP/HTTP/HTTPS/SFTP/S3 endpoints", "main": "index.js", "directories": { @@ -35,16 +35,16 @@ "author": "Cumulus Authors", "license": "Apache-2.0", "dependencies": { - "@cumulus/api-client": "13.3.2", + "@cumulus/api-client": "13.4.0", "@cumulus/cumulus-message-adapter-js": "2.0.4", - "@cumulus/ingest": "13.3.2", - "@cumulus/logger": "13.3.2", + "@cumulus/ingest": "13.4.0", + "@cumulus/logger": "13.4.0", "got": "^11.8.5", "lodash": "^4.17.21", "p-map": "^4.0.0" }, "devDependencies": { - "@cumulus/aws-client": "13.3.2", - "@cumulus/common": "13.3.2" + "@cumulus/aws-client": "13.4.0", + "@cumulus/common": "13.4.0" } } diff --git a/tasks/discover-pdrs/package.json b/tasks/discover-pdrs/package.json index 9d76caf4513..096ebba76ab 100644 --- a/tasks/discover-pdrs/package.json +++ b/tasks/discover-pdrs/package.json @@ -1,6 +1,6 @@ { "name": "@cumulus/discover-pdrs", - "version": "13.3.2", + "version": "13.4.0", "description": "Discover PDRs in FTP and HTTP endpoints", "main": "index.js", "directories": { @@ -34,14 +34,14 @@ "author": "Cumulus Authors", "license": "Apache-2.0", "dependencies": { - "@cumulus/aws-client": "13.3.2", + "@cumulus/aws-client": "13.4.0", "@cumulus/cumulus-message-adapter-js": "2.0.4", - "@cumulus/ingest": "13.3.2", + "@cumulus/ingest": "13.4.0", "lodash": "^4.17.21", "p-filter": "^2.1.0" }, "devDependencies": { - "@cumulus/common": "13.3.2", - "@cumulus/errors": "13.3.2" + "@cumulus/common": "13.4.0", + "@cumulus/errors": "13.4.0" } } diff --git a/tasks/files-to-granules/package.json b/tasks/files-to-granules/package.json index 229f9147e0b..9f8ece8bb9c 100644 --- a/tasks/files-to-granules/package.json +++ b/tasks/files-to-granules/package.json @@ -1,6 +1,6 @@ { "name": "@cumulus/files-to-granules", - "version": "13.3.2", + "version": "13.4.0", "description": "Converts array-of-files input into a granules object by extracting granuleId from filename", "main": "index.js", "directories": { @@ -32,12 +32,12 @@ "author": "Cumulus Authors", "license": "Apache-2.0", "dependencies": { - "@cumulus/aws-client": "13.3.2", + "@cumulus/aws-client": "13.4.0", "@cumulus/cumulus-message-adapter-js": "2.0.4", "lodash": "^4.17.21" }, "devDependencies": { - "@cumulus/common": "13.3.2", - "@cumulus/schemas": "13.3.2" + "@cumulus/common": "13.4.0", + "@cumulus/schemas": "13.4.0" } } diff --git a/tasks/hello-world/package.json b/tasks/hello-world/package.json index ec96f1ade67..47cdc346ffe 100644 --- a/tasks/hello-world/package.json +++ b/tasks/hello-world/package.json @@ -1,6 +1,6 @@ { "name": "@cumulus/hello-world", - "version": "13.3.2", + "version": "13.4.0", "description": "Example task", "main": "index.js", "directories": { @@ -32,8 +32,8 @@ "author": "Cumulus Authors", "license": "Apache-2.0", "dependencies": { - "@cumulus/aws-client": "13.3.2", - "@cumulus/common": "13.3.2", + "@cumulus/aws-client": "13.4.0", + "@cumulus/common": "13.4.0", "@cumulus/cumulus-message-adapter-js": "2.0.4" } } diff --git a/tasks/hyrax-metadata-updates/package.json b/tasks/hyrax-metadata-updates/package.json index 45ea3bd631d..c5a6d9c7557 100644 --- a/tasks/hyrax-metadata-updates/package.json +++ b/tasks/hyrax-metadata-updates/package.json @@ -1,6 +1,6 @@ { "name": "@cumulus/hyrax-metadata-updates", - "version": "13.3.2", + "version": "13.4.0", "description": "Update granule metadata with hooks to OPeNDAP URL", "main": "index.js", "directories": { @@ -38,18 +38,18 @@ "author": "Cumulus Authors", "license": "Apache-2.0", "dependencies": { - "@cumulus/aws-client": "13.3.2", - "@cumulus/cmr-client": "13.3.2", - "@cumulus/cmrjs": "13.3.2", - "@cumulus/common": "13.3.2", + "@cumulus/aws-client": "13.4.0", + "@cumulus/cmr-client": "13.4.0", + "@cumulus/cmrjs": "13.4.0", + "@cumulus/common": "13.4.0", "@cumulus/cumulus-message-adapter-js": "2.0.4", - "@cumulus/errors": "13.3.2", + "@cumulus/errors": "13.4.0", "libxmljs": "^0.19.7", "lodash": "^4.17.21", "xml2js": "^0.4.23" }, "devDependencies": { - "@cumulus/schemas": "13.3.2", + "@cumulus/schemas": "13.4.0", "nock": "^12.0.1", "rewire": "^6.0.0" }, diff --git a/tasks/lzards-backup/package.json b/tasks/lzards-backup/package.json index 4f67d5e53b6..125ab94bc02 100644 --- a/tasks/lzards-backup/package.json +++ b/tasks/lzards-backup/package.json @@ -1,6 +1,6 @@ { "name": "@cumulus/lzards-backup", - "version": "13.3.2", + "version": "13.4.0", "description": "Run LZARDS backup", "author": "Cumulus Authors", "license": "Apache-2.0", @@ -42,20 +42,20 @@ } }, "dependencies": { - "@cumulus/api-client": "13.3.2", - "@cumulus/aws-client": "13.3.2", - "@cumulus/common": "13.3.2", + "@cumulus/api-client": "13.4.0", + "@cumulus/aws-client": "13.4.0", + "@cumulus/common": "13.4.0", "@cumulus/cumulus-message-adapter-js": "2.0.4", - "@cumulus/db": "13.3.2", - "@cumulus/distribution-utils": "13.3.2", - "@cumulus/launchpad-auth": "13.3.2", - "@cumulus/logger": "13.3.2", - "@cumulus/message": "13.3.2", - "@cumulus/lzards-api-client": "13.3.2-alpha.0", + "@cumulus/db": "13.4.0", + "@cumulus/distribution-utils": "13.4.0", + "@cumulus/launchpad-auth": "13.4.0", + "@cumulus/logger": "13.4.0", + "@cumulus/lzards-api-client": "13.4.0", + "@cumulus/message": "13.4.0", "got": "11.8.3" }, "devDependencies": { - "@cumulus/schemas": "13.3.2", - "@cumulus/types": "13.3.2" + "@cumulus/schemas": "13.4.0", + "@cumulus/types": "13.4.0" } } diff --git a/tasks/move-granules/package.json b/tasks/move-granules/package.json index 37111976f07..7b69b068e87 100644 --- a/tasks/move-granules/package.json +++ b/tasks/move-granules/package.json @@ -1,6 +1,6 @@ { "name": "@cumulus/move-granules", - "version": "13.3.2", + "version": "13.4.0", "description": "Move granule files from staging to final location", "main": "index.js", "directories": { @@ -38,17 +38,17 @@ "author": "Cumulus Authors", "license": "Apache-2.0", "dependencies": { - "@cumulus/aws-client": "13.3.2", - "@cumulus/cmrjs": "13.3.2", - "@cumulus/common": "13.3.2", + "@cumulus/aws-client": "13.4.0", + "@cumulus/cmrjs": "13.4.0", + "@cumulus/common": "13.4.0", "@cumulus/cumulus-message-adapter-js": "2.0.4", - "@cumulus/distribution-utils": "13.3.2", - "@cumulus/errors": "13.3.2", - "@cumulus/ingest": "13.3.2", - "@cumulus/message": "13.3.2", + "@cumulus/distribution-utils": "13.4.0", + "@cumulus/errors": "13.4.0", + "@cumulus/ingest": "13.4.0", + "@cumulus/message": "13.4.0", "lodash": "^4.17.21" }, "devDependencies": { - "@cumulus/schemas": "13.3.2" + "@cumulus/schemas": "13.4.0" } } diff --git a/tasks/parse-pdr/package.json b/tasks/parse-pdr/package.json index 63b171bfc2e..364e63ed7a4 100644 --- a/tasks/parse-pdr/package.json +++ b/tasks/parse-pdr/package.json @@ -1,6 +1,6 @@ { "name": "@cumulus/parse-pdr", - "version": "13.3.2", + "version": "13.4.0", "description": "Download and Parse a given PDR", "license": "Apache-2.0", "main": "index.js", @@ -30,17 +30,17 @@ "timeout": "15m" }, "dependencies": { - "@cumulus/api-client": "13.3.2", - "@cumulus/aws-client": "13.3.2", - "@cumulus/collection-config-store": "13.3.2", - "@cumulus/common": "13.3.2", + "@cumulus/api-client": "13.4.0", + "@cumulus/aws-client": "13.4.0", + "@cumulus/collection-config-store": "13.4.0", + "@cumulus/common": "13.4.0", "@cumulus/cumulus-message-adapter-js": "2.0.4", - "@cumulus/errors": "13.3.2", - "@cumulus/ingest": "13.3.2", - "@cumulus/pvl": "13.3.2", + "@cumulus/errors": "13.4.0", + "@cumulus/ingest": "13.4.0", + "@cumulus/pvl": "13.4.0", "lodash": "^4.17.21" }, "devDependencies": { - "@cumulus/test-data": "13.3.2" + "@cumulus/test-data": "13.4.0" } } diff --git a/tasks/pdr-status-check/package.json b/tasks/pdr-status-check/package.json index 4149b0c2ac2..f561214daa5 100644 --- a/tasks/pdr-status-check/package.json +++ b/tasks/pdr-status-check/package.json @@ -1,6 +1,6 @@ { "name": "@cumulus/pdr-status-check", - "version": "13.3.2", + "version": "13.4.0", "description": "Checks execution status of granules in a PDR", "main": "index.js", "directories": { @@ -32,9 +32,9 @@ "timeout": "15m" }, "dependencies": { - "@cumulus/aws-client": "13.3.2", - "@cumulus/common": "13.3.2", + "@cumulus/aws-client": "13.4.0", + "@cumulus/common": "13.4.0", "@cumulus/cumulus-message-adapter-js": "2.0.4", - "@cumulus/errors": "13.3.2" + "@cumulus/errors": "13.4.0" } } diff --git a/tasks/post-to-cmr/package.json b/tasks/post-to-cmr/package.json index e069497afb2..ab2191236ad 100644 --- a/tasks/post-to-cmr/package.json +++ b/tasks/post-to-cmr/package.json @@ -1,6 +1,6 @@ { "name": "@cumulus/post-to-cmr", - "version": "13.3.2", + "version": "13.4.0", "description": "Post a given granule to CMR", "main": "index.js", "directories": { @@ -33,16 +33,16 @@ "author": "Cumulus Authors", "license": "Apache-2.0", "dependencies": { - "@cumulus/aws-client": "13.3.2", - "@cumulus/cmrjs": "13.3.2", - "@cumulus/common": "13.3.2", + "@cumulus/aws-client": "13.4.0", + "@cumulus/cmrjs": "13.4.0", + "@cumulus/common": "13.4.0", "@cumulus/cumulus-message-adapter-js": "2.0.4", - "@cumulus/errors": "13.3.2", - "@cumulus/launchpad-auth": "13.3.2", + "@cumulus/errors": "13.4.0", + "@cumulus/launchpad-auth": "13.4.0", "lodash": "^4.17.21" }, "devDependencies": { - "@cumulus/cmr-client": "13.3.2", - "@cumulus/schemas": "13.3.2" + "@cumulus/cmr-client": "13.4.0", + "@cumulus/schemas": "13.4.0" } } diff --git a/tasks/queue-granules/package.json b/tasks/queue-granules/package.json index 48a5882e4af..612fba24385 100644 --- a/tasks/queue-granules/package.json +++ b/tasks/queue-granules/package.json @@ -1,6 +1,6 @@ { "name": "@cumulus/queue-granules", - "version": "13.3.2", + "version": "13.4.0", "description": "Add discovered granules to the queue", "main": "index.js", "directories": { @@ -31,13 +31,13 @@ "author": "Cumulus Authors", "license": "Apache-2.0", "dependencies": { - "@cumulus/api-client": "13.3.2", - "@cumulus/aws-client": "13.3.2", - "@cumulus/collection-config-store": "13.3.2", - "@cumulus/common": "13.3.2", + "@cumulus/api-client": "13.4.0", + "@cumulus/aws-client": "13.4.0", + "@cumulus/collection-config-store": "13.4.0", + "@cumulus/common": "13.4.0", "@cumulus/cumulus-message-adapter-js": "2.0.4", - "@cumulus/ingest": "13.3.2", - "@cumulus/message": "13.3.2", + "@cumulus/ingest": "13.4.0", + "@cumulus/message": "13.4.0", "lodash": "^4.17.21", "p-map": "^4.0.0" } diff --git a/tasks/queue-pdrs/package.json b/tasks/queue-pdrs/package.json index 580858efaca..4342117f43d 100644 --- a/tasks/queue-pdrs/package.json +++ b/tasks/queue-pdrs/package.json @@ -1,6 +1,6 @@ { "name": "@cumulus/queue-pdrs", - "version": "13.3.2", + "version": "13.4.0", "description": "Add discovered PDRs to a queue", "main": "index.js", "directories": { @@ -31,11 +31,11 @@ "author": "Cumulus Authors", "license": "Apache-2.0", "dependencies": { - "@cumulus/aws-client": "13.3.2", - "@cumulus/common": "13.3.2", + "@cumulus/aws-client": "13.4.0", + "@cumulus/common": "13.4.0", "@cumulus/cumulus-message-adapter-js": "2.0.4", - "@cumulus/ingest": "13.3.2", - "@cumulus/message": "13.3.2", + "@cumulus/ingest": "13.4.0", + "@cumulus/message": "13.4.0", "lodash": "^4.17.21" } } diff --git a/tasks/queue-workflow/package.json b/tasks/queue-workflow/package.json index aec0f704056..b8f951eb6fb 100644 --- a/tasks/queue-workflow/package.json +++ b/tasks/queue-workflow/package.json @@ -1,6 +1,6 @@ { "name": "@cumulus/queue-workflow", - "version": "13.3.2", + "version": "13.4.0", "description": "Add workflow to the queue", "main": "index.js", "directories": { @@ -31,11 +31,11 @@ "author": "Cumulus Authors", "license": "Apache-2.0", "dependencies": { - "@cumulus/aws-client": "13.3.2", - "@cumulus/common": "13.3.2", + "@cumulus/aws-client": "13.4.0", + "@cumulus/common": "13.4.0", "@cumulus/cumulus-message-adapter-js": "2.0.4", - "@cumulus/ingest": "13.3.2", - "@cumulus/message": "13.3.2", + "@cumulus/ingest": "13.4.0", + "@cumulus/message": "13.4.0", "lodash": "^4.17.21" } } diff --git a/tasks/sf-sqs-report/package.json b/tasks/sf-sqs-report/package.json index a70274df5f4..7904bb3810a 100644 --- a/tasks/sf-sqs-report/package.json +++ b/tasks/sf-sqs-report/package.json @@ -1,6 +1,6 @@ { "name": "@cumulus/sf-sqs-report", - "version": "13.3.2", + "version": "13.4.0", "description": "Sends an incoming Cumulus message to SQS", "main": "index.js", "directories": { @@ -32,11 +32,11 @@ "author": "Cumulus Authors", "license": "Apache-2.0", "dependencies": { - "@cumulus/aws-client": "13.3.2", + "@cumulus/aws-client": "13.4.0", "@cumulus/cumulus-message-adapter-js": "2.0.4", "lodash": "^4.17.21" }, "devDependencies": { - "@cumulus/common": "13.3.2" + "@cumulus/common": "13.4.0" } } diff --git a/tasks/sync-granule/package.json b/tasks/sync-granule/package.json index cf4913200b5..ac3587df2b4 100644 --- a/tasks/sync-granule/package.json +++ b/tasks/sync-granule/package.json @@ -1,6 +1,6 @@ { "name": "@cumulus/sync-granule", - "version": "13.3.2", + "version": "13.4.0", "description": "Download a given granule", "main": "index.js", "directories": { @@ -37,19 +37,19 @@ "timeout": "15m" }, "dependencies": { - "@cumulus/aws-client": "13.3.2", - "@cumulus/collection-config-store": "13.3.2", - "@cumulus/common": "13.3.2", + "@cumulus/aws-client": "13.4.0", + "@cumulus/collection-config-store": "13.4.0", + "@cumulus/common": "13.4.0", "@cumulus/cumulus-message-adapter-js": "2.0.4", - "@cumulus/errors": "13.3.2", - "@cumulus/ingest": "13.3.2", - "@cumulus/message": "13.3.2", + "@cumulus/errors": "13.4.0", + "@cumulus/ingest": "13.4.0", + "@cumulus/message": "13.4.0", "lodash": "^4.17.21", "p-map": "^2.1.0", "uuid": "^3.4.0" }, "devDependencies": { - "@cumulus/schemas": "13.3.2", - "@cumulus/test-data": "13.3.2" + "@cumulus/schemas": "13.4.0", + "@cumulus/test-data": "13.4.0" } } diff --git a/tasks/test-processing/package.json b/tasks/test-processing/package.json index 25fcee16f3f..4543e8550b5 100644 --- a/tasks/test-processing/package.json +++ b/tasks/test-processing/package.json @@ -1,6 +1,6 @@ { "name": "@cumulus/test-processing", - "version": "13.3.2", + "version": "13.4.0", "description": "Fake processing task used for integration tests", "main": "index.js", "homepage": "https://github.com/nasa/cumulus/tree/master/tasks/test-processing", @@ -21,8 +21,8 @@ "author": "Cumulus Authors", "license": "Apache-2.0", "dependencies": { - "@cumulus/aws-client": "13.3.2", + "@cumulus/aws-client": "13.4.0", "@cumulus/cumulus-message-adapter-js": "2.0.4", - "@cumulus/integration-tests": "13.3.2" + "@cumulus/integration-tests": "13.4.0" } } diff --git a/tasks/update-cmr-access-constraints/package.json b/tasks/update-cmr-access-constraints/package.json index 48846a22785..1dcfb44eeb1 100644 --- a/tasks/update-cmr-access-constraints/package.json +++ b/tasks/update-cmr-access-constraints/package.json @@ -1,6 +1,6 @@ { "name": "@cumulus/update-cmr-access-constraints", - "version": "13.3.2", + "version": "13.4.0", "description": "Updates CMR metadata to set access constraints", "author": "Cumulus Authors", "license": "Apache-2.0", @@ -34,13 +34,13 @@ "verbose": true }, "dependencies": { - "@cumulus/aws-client": "13.3.2", - "@cumulus/cmrjs": "13.3.2", + "@cumulus/aws-client": "13.4.0", + "@cumulus/cmrjs": "13.4.0", "@cumulus/cumulus-message-adapter-js": "2.0.4", "lodash": "^4.17.5" }, "devDependencies": { - "@cumulus/common": "13.3.2", - "@cumulus/schemas": "13.3.2" + "@cumulus/common": "13.4.0", + "@cumulus/schemas": "13.4.0" } } diff --git a/tasks/update-granules-cmr-metadata-file-links/package.json b/tasks/update-granules-cmr-metadata-file-links/package.json index 4ef5ebcc5b9..32186b4cf1d 100644 --- a/tasks/update-granules-cmr-metadata-file-links/package.json +++ b/tasks/update-granules-cmr-metadata-file-links/package.json @@ -1,6 +1,6 @@ { "name": "@cumulus/update-granules-cmr-metadata-file-links", - "version": "13.3.2", + "version": "13.4.0", "description": "Update CMR metadata files with correct online access urls and etags and transfer etag info to granules' CMR files", "main": "index.js", "directories": { @@ -38,14 +38,14 @@ "author": "Cumulus Authors", "license": "Apache-2.0", "dependencies": { - "@cumulus/cmrjs": "13.3.2", - "@cumulus/common": "13.3.2", + "@cumulus/cmrjs": "13.4.0", + "@cumulus/common": "13.4.0", "@cumulus/cumulus-message-adapter-js": "2.0.4", - "@cumulus/distribution-utils": "13.3.2", + "@cumulus/distribution-utils": "13.4.0", "lodash": "^4.17.15" }, "devDependencies": { - "@cumulus/aws-client": "13.3.2", - "@cumulus/schemas": "13.3.2" + "@cumulus/aws-client": "13.4.0", + "@cumulus/schemas": "13.4.0" } } diff --git a/tf-modules/ingest/package.json b/tf-modules/ingest/package.json index 041586dfe00..1571a011ad9 100644 --- a/tf-modules/ingest/package.json +++ b/tf-modules/ingest/package.json @@ -1,6 +1,6 @@ { "name": "@cumulus/ingest-module", - "version": "13.3.2", + "version": "13.4.0", "description": "Terraform module for data ingest related functionality", "engines": { "node": ">=14.19.1" diff --git a/tf-modules/internal/cumulus-test-cleanup/package.json b/tf-modules/internal/cumulus-test-cleanup/package.json index 47ff24144bb..1a8caa5e96c 100644 --- a/tf-modules/internal/cumulus-test-cleanup/package.json +++ b/tf-modules/internal/cumulus-test-cleanup/package.json @@ -1,6 +1,6 @@ { "name": "@cumulus/cumulus-test-cleanup", - "version": "13.3.2", + "version": "13.4.0", "description": "Nightly cron job for cleaning up integration test artifacts", "main": "index.js", "engines": { diff --git a/tf-modules/s3-replicator/package.json b/tf-modules/s3-replicator/package.json index 18cbde69cf4..a4041f24d07 100644 --- a/tf-modules/s3-replicator/package.json +++ b/tf-modules/s3-replicator/package.json @@ -1,6 +1,6 @@ { "name": "@cumulus/s3-replicator", - "version": "13.3.2", + "version": "13.4.0", "description": "Replicate S3 Events to alternate bucket. Solves same-region replication.", "main": "index.js", "engines": { diff --git a/website/versioned_docs/version-v13.4.0/configuration/task-configuration.md b/website/versioned_docs/version-v13.4.0/configuration/task-configuration.md new file mode 100644 index 00000000000..48a8a5b7c49 --- /dev/null +++ b/website/versioned_docs/version-v13.4.0/configuration/task-configuration.md @@ -0,0 +1,132 @@ +--- +id: version-v13.4.0-task-configuration +title: Configuration of Tasks +hide_title: false +original_id: task-configuration +--- + +The `cumulus` module exposes values for configuration for some of the provided archive and ingest tasks. Currently the following are available as configurable variables: + +## cmr_search_client_config + +Configuration parameters for CMR search client for cumulus archive module tasks in the form: + +```hcl +_report_cmr_limit = +_report_cmr_page_size = + type = map(string) +``` + +More information about cmr limit and cmr page_size can be found from [@cumulus/cmr-client](https://github.com/nasa/cumulus/blob/master/packages/cmr-client/src/searchConcept.ts) and [CMR Search API document](https://cmr.earthdata.nasa.gov/search/site/docs/search/api.html#query-parameters). + +Currently the following values are supported: + +- create_reconciliation_report_cmr_limit +- create_reconciliation_report_cmr_page_size + +### Example + +```tf +cmr_search_client_config = { + create_reconciliation_report_cmr_limit = 2500 + create_reconciliation_report_cmr_page_size = 250 +} +``` + +## elasticsearch_client_config + +Configuration parameters for Elasticsearch client for cumulus archive module tasks in the form: + +```hcl +_es_scroll_duration = +_es_scroll_size = + type = map(string) +``` + +Currently the following values are supported: + +- create_reconciliation_report_es_scroll_duration +- create_reconciliation_report_es_scroll_size + +### Example + +```tf +elasticsearch_client_config = { + create_reconciliation_report_es_scroll_duration = "15m" + create_reconciliation_report_es_scroll_size = 2000 +} +``` + +## lambda_timeouts + +A configurable map of timeouts (in seconds) for cumulus ingest module task lambdas in the form: + +```hcl +_timeout: + type = map(string) +``` + +Currently the following values are supported: + +- discover_granules_task_timeout +- discover_pdrs_task_timeout +- fake_processing_task_timeout +- files_to_granules_task_timeout +- hello_world_task_timeout +- hyrax_metadata_update_tasks_timeout +- lzards_backup_task_timeout +- move_granules_task_timeout +- parse_pdr_task_timeout +- pdr_status_check_task_timeout +- post_to_cmr_task_timeout +- queue_granules_task_timeout +- queue_pdrs_task_timeout +- queue_workflow_task_timeout +- sf_sqs_report_task_timeout +- sync_granule_task_timeout +- update_granules_cmr_metadata_file_links_task_timeout + +### Example + +```tf +lambda_timeouts = { + discover_granules_task_timeout = 300 +} +``` + +## lambda_memory_sizes + +A configurable map of memory sizes (in MBs) for cumulus ingest module task lambdas in the form: + +```hcl +_memory_size: + type = map(string) +``` + +Currently the following values are supported: + +- add_missing_file_checksums_task_memory_size +- discover_granules_task_memory_size +- discover_pdrs_task_memory_size +- fake_processing_task_memory_size +- hyrax_metadata_updates_task_memory_size +- lzards_backup_task_memory_size +- move_granules_task_memory_size +- parse_pdr_task_memory_size +- pdr_status_check_task_memory_size +- post_to_cmr_task_memory_size +- queue_granules_task_memory_size +- queue_pdrs_task_memory_size +- queue_workflow_task_memory_size +- sf_sqs_report_task_memory_size +- sync_granule_task_memory_size +- update_cmr_acess_constraints_task_memory_size +- update_granules_cmr_metadata_file_links_task_memory_size + +### Example + +```tf +lambda_memory_sizes = { + queue_granules_task_memory_size = 1036 +} +``` diff --git a/website/versioned_docs/version-v13.4.0/data-cookbooks/cnm-workflow.md b/website/versioned_docs/version-v13.4.0/data-cookbooks/cnm-workflow.md new file mode 100644 index 00000000000..06085d256f5 --- /dev/null +++ b/website/versioned_docs/version-v13.4.0/data-cookbooks/cnm-workflow.md @@ -0,0 +1,620 @@ +--- +id: version-v13.4.0-cnm-workflow +title: CNM Workflow +hide_title: false +original_id: cnm-workflow +--- + +This entry documents how to setup a workflow that utilizes the built-in CNM/Kinesis functionality in Cumulus. + +Prior to working through this entry you should be familiar with the [Cloud Notification Mechanism](https://github.com/podaac/cloud-notification-message-schema). + +## Sections + +- [Prerequisites](#prerequisites) +- [Configure the Workflow](#configure-the-workflow) +- [Execute the Workflow](#execute-the-workflow) +- [Verify Results](#verify-results) +- [Kinesis Record Error Handling](#kinesis-record-error-handling) + +--- + +## Prerequisites + +### Cumulus + +This entry assumes you have a deployed instance of Cumulus (version >= 1.16.0). The entry assumes you are deploying Cumulus via the [`cumulus` terraform module](https://github.com/nasa/cumulus/tree/master/tf-modules/cumulus) sourced from the [release page](https://github.com/nasa/cumulus/releases). + +### AWS CLI + +This entry assumes you have the [AWS CLI](https://aws.amazon.com/cli/) installed and configured. If you do not, please take a moment to review the documentation - particularly the [examples relevant to Kinesis](https://docs.aws.amazon.com/streams/latest/dev/fundamental-stream.html) - and install it now. + +### Kinesis + +This entry assumes you already have two [Kinesis](https://aws.amazon.com/kinesis/) data steams created for use as CNM notification and response data streams. + +If you do not have two streams setup, please take a moment to review the [Kinesis documentation](https://aws.amazon.com/documentation/kinesis/) and setup two basic single-shard streams for this example: + +Using the "Create Data Stream" button on the [Kinesis Dashboard](https://console.aws.amazon.com/kinesis/home), work through the dialogue. + +You should be able to quickly use the "Create Data Stream" button on the [Kinesis Dashboard](https://console.aws.amazon.com/kinesis/home), and setup streams that are similar to the following example: + +![Screenshot of AWS console page for creating a Kinesis stream](assets/cnm_create_kinesis_stream.jpg) + +Please bear in mind that your `{{prefix}}-lambda-processing` IAM role will need permissions to write to the response stream for this workflow to succeed if you create the Kinesis stream with a dashboard user. If you are using the `cumulus` top-level module for your deployment this should be set properly. + +If not, the most straightforward approach is to attach the `AmazonKinesisFullAccess` policy for the stream resource to whatever role your Lambda +s are using, however your environment/security policies may require an approach specific to your deployment environment. + +In operational environments it's likely science data providers would typically be responsible for providing a Kinesis stream with the appropriate permissions. + +For more information on how this process works and how to develop a process that will add records to a stream, read the [Kinesis documentation](https://aws.amazon.com/documentation/kinesis/) and the [developer guide](https://docs.aws.amazon.com/streams/latest/dev/introduction.html). + +### Source Data + +This entry will run the SyncGranule task against a single target data file. To that end it will require a single data file to be present in an S3 bucket matching the Provider configured in the next section. + +### Collection and Provider + +Cumulus will need to be configured with a Collection and Provider entry of your choosing. The provider should match the location of the source data from the `Ingest Source Data` section. + +This can be done via the [Cumulus Dashboard](https://github.com/nasa/cumulus-dashboard) if installed or the [API](../api.md). It is strongly recommended to use the dashboard if possible. + +--- + +## Configure the Workflow + +Provided the prerequisites have been fulfilled, you can begin adding the needed values to your Cumulus configuration to configure the example workflow. + +The following are steps that are required to set up your Cumulus instance to run the example workflow: + +### Example CNM Workflow + +In this example, we're going to trigger a workflow by creating a Kinesis rule and sending a record to a Kinesis stream. + +The following [workflow definition](workflows/README.md) should be added to a new `.tf` workflow resource (e.g. `cnm_workflow.tf`) in your deployment directory. For the complete CNM workflow example, see [examples/cumulus-tf/cnm_workflow.tf](https://github.com/nasa/cumulus/blob/master/example/cumulus-tf/cnm_workflow.tf). + +Add the following to the new terraform file in your deployment directory, updating the following: + +- Set the `response-endpoint` key in the `CnmResponse` task in the workflow JSON to match the name of the Kinesis response stream you configured in the prerequisites section +- Update the `source` key to the workflow module to match the Cumulus release associated with your deployment. + +```hcl +module "cnm_workflow" { + source = "https://github.com/nasa/cumulus/releases/download/{version}/terraform-aws-cumulus-workflow.zip" + + prefix = var.prefix + name = "CNMExampleWorkflow" + workflow_config = module.cumulus.workflow_config + system_bucket = var.system_bucket + +{ +state_machine_definition = < +- + +**Please note:** To utilize these tasks you need to ensure you have a compatible CMA layer. See the [deployment instructions](../deployment/README.md#deploy-cumulus-message-adapter-layer) for more details on how to deploy a CMA layer. + +- `CNMToCMA`: +- `CnmResponse`: + +Below is a description of each of these tasks: + +#### CNMToCMA + +`CNMToCMA` is meant for the beginning of a workflow: it maps CNM granule information to a payload for downstream tasks. For other CNM workflows, you would need to ensure that downstream tasks in your workflow either understand the CNM message _or_ include a translation task like this one. + +You can also manipulate the data sent to downstream tasks using `task_config` for various states in your workflow resource configuration. Read more about how to configure data on the [Workflow Input & Output](https://nasa.github.io/cumulus/docs/workflows/input_output) page. + +##### CnmResponse + +The `CnmResponse` Lambda generates a CNM response message and puts it on the `response-endpoint` Kinesis stream. + +You can read more about the expected schema of a `CnmResponse` record in the [Cloud Notification Mechanism schema repository](https://github.com/podaac/cloud-notification-message-schema#response-message-fields). + +##### Additional Tasks + +Lastly, this entry also makes use of the `SyncGranule` task from the [`cumulus` module](https://github.com/nasa/cumulus/releases). + +### Redeploy + +Once the above configuration changes have been made, redeploy your stack. + +Please refer to `Update Cumulus resources` in the [deployment documentation](../deployment/upgrade.md#update-cumulus-resources) if you are unfamiliar with redeployment. + +### Rule Configuration + +Cumulus includes a `messageConsumer` Lambda function ([message-consumer](https://github.com/nasa/cumulus/blob/master/packages/api/lambdas/message-consumer.js)). Cumulus kinesis-type rules create the [event source mappings](https://docs.aws.amazon.com/lambda/latest/dg/API_CreateEventSourceMapping.html) between Kinesis streams and the `messageConsumer` Lambda. The `messageConsumer` Lambda consumes records from one or more Kinesis streams, as defined by enabled kinesis-type rules. When new records are pushed to one of these streams, the `messageConsumer` triggers workflows associated with the enabled kinesis-type rules. + +To add a rule via the dashboard (if you'd like to use the API, see the docs [here](https://nasa.github.io/cumulus-api/#create-rule)), navigate to the `Rules` page and click `Add a rule`, then configure the new rule using the following template (substituting correct values for parameters denoted by `${}`): + +```json +{ + "collection": { + "name": "L2_HR_PIXC", + "version": "000" + }, + "name": "L2_HR_PIXC_kinesisRule", + "provider": "PODAAC_SWOT", + "rule": { + "type": "kinesis", + "value": "arn:aws:kinesis:{{awsRegion}}:{{awsAccountId}}:stream/{{streamName}}" + }, + "state": "ENABLED", + "workflow": "CNMExampleWorkflow" +} +``` + +**Please Note:** + +- The rule's `value` attribute value must match the Amazon Resource Name [ARN](https://docs.aws.amazon.com/general/latest/gr/aws-arns-and-namespaces.html) for the Kinesis data stream you've preconfigured. You should be able to obtain this ARN from the Kinesis Dashboard entry for the selected stream. +- The collection and provider should match the collection and provider you setup in the [`Prerequisites`](#prerequisites) section. + +Once you've clicked on 'submit' a new rule should appear in the dashboard's Rule Overview. + +--- + +## Execute the Workflow + +Once Cumulus has been redeployed and a rule has been added, we're ready to trigger the workflow and watch it execute. + +### How to Trigger the Workflow + +To trigger matching workflows, you will need to put a record on the Kinesis stream that the [message-consumer](https://github.com/nasa/cumulus/blob/master/packages/api/lambdas/message-consumer.js) Lambda will recognize as a matching event. Most importantly, it should include a `collection` name that matches a valid collection. + +For the purpose of this example, the easiest way to accomplish this is using the [AWS CLI](https://aws.amazon.com/cli/). + +#### Create Record JSON + +Construct a JSON file containing an object that matches the values that have been previously setup. This JSON object should be a valid [Cloud Notification Mechanism](https://github.com/podaac/cloud-notification-message-schema#cumulus-sns-schema) message. + +**Please note**: _this example is somewhat contrived, as the downstream tasks don't care about most of these fields. A 'real' data ingest workflow would._ + +The following values (denoted by `${}` in the sample below) should be replaced to match values we've previously configured: + +- `TEST_DATA_FILE_NAME`: The filename of the test data that is available in the S3 (or other) provider we created earlier. +- `TEST_DATA_URI`: The full S3 path to the test data (e.g. s3://bucket-name/path/granule) +- `COLLECTION`: The collection name defined in the prerequisites for this product + +```json +{ + "product": { + "files": [ + { + "checksumType": "md5", + "name": "${TEST_DATA_FILE_NAME}", + "checksum": "bogus_checksum_value", + "uri": "${TEST_DATA_URI}", + "type": "data", + "size": 12345678 + } + ], + "name": "${TEST_DATA_FILE_NAME}", + "dataVersion": "006" + }, + "identifier ": "testIdentifier123456", + "collection": "${COLLECTION}", + "provider": "TestProvider", + "version": "001", + "submissionTime": "2017-09-30T03:42:29.791198" +} +``` + +#### Add Record to Kinesis Data Stream + +Using the JSON file you created, push it to the Kinesis notification stream: + +```bash +aws kinesis put-record --stream-name YOUR_KINESIS_NOTIFICATION_STREAM_NAME_HERE --partition-key 1 --data file:///path/to/file.json +``` + +**Please note**: The above command uses the stream name, _not_ the ARN. + +The command should return output similar to: + +```json +{ + "ShardId": "shardId-000000000000", + "SequenceNumber": "42356659532578640215890215117033555573986830588739321858" +} +``` + +This command will put a record containing the JSON from the `--data` flag onto the Kinesis data stream. The `messageConsumer` Lambda will consume the record and construct a valid CMA payload to trigger workflows. For this example, the record will trigger the `CNMExampleWorkflow` workflow as defined by the rule previously configured. + +You can view the current running executions on the `Executions` dashboard page which presents a list of all executions, their status (running, failed, or completed), to which workflow the execution belongs, along with other information. + +### Verify Workflow Execution + +As detailed above, once the record is added to the Kinesis data stream, the `messageConsumer` Lambda will trigger the `CNMExampleWorkflow` . + +#### TranslateMessage + +`TranslateMessage` (which corresponds to the `CNMToCMA` Lambda) will take the CNM object payload and add a granules object to the CMA payload that's consistent with other Cumulus ingest tasks, and add a `meta.cnm` key (as well as the payload) to store the original message. + +_For more on the Message Adapter, please see [the Message Flow documentation](workflows/cumulus-task-message-flow.md)_. + +An example of what is happening in the `CNMToCMA` Lambda is as follows: + +Example Input Payload: + +```json +"payload": { + "identifier ": "testIdentifier123456", + "product": { + "files": [ + { + "checksumType": "md5", + "name": "MOD09GQ.A2016358.h13v04.006.2016360104606.hdf", + "checksum": "bogus_checksum_value", + "uri": "s3://some_bucket/cumulus-test-data/pdrs/MOD09GQ.A2016358.h13v04.006.2016360104606.hdf", + "type": "data", + "size": 12345678 + } + ], + "name": "TestGranuleUR", + "dataVersion": "006" + }, + "version": "123456", + "collection": "MOD09GQ", + "provider": "TestProvider", + "submissionTime": "2017-09-30T03:42:29.791198" +} +``` + +Example Output Payload: + +```json + "payload": { + "cnm": { + "identifier ": "testIdentifier123456", + "product": { + "files": [ + { + "checksumType": "md5", + "name": "MOD09GQ.A2016358.h13v04.006.2016360104606.hdf", + "checksum": "bogus_checksum_value", + "uri": "s3://some-bucket/cumulus-test-data/data/MOD09GQ.A2016358.h13v04.006.2016360104606.hdf", + "type": "data", + "size": 12345678 + } + ], + "name": "TestGranuleUR", + "dataVersion": "006" + }, + "version": "123456", + "collection": "MOD09GQ", + "provider": "TestProvider", + "submissionTime": "2017-09-30T03:42:29.791198", + "receivedTime": "2017-09-30T03:42:31.634552" + }, + "output": { + "granules": [ + { + "granuleId": "TestGranuleUR", + "files": [ + { + "path": "some-bucket/data", + "url_path": "s3://some-bucket/cumulus-test-data/data/MOD09GQ.A2016358.h13v04.006.2016360104606.hdf", + "bucket": "some-bucket", + "name": "MOD09GQ.A2016358.h13v04.006.2016360104606.hdf", + "size": 12345678 + } + ] + } + ] + } + } +``` + +#### SyncGranules + +This Lambda will take the files listed in the payload and move them to `s3://{deployment-private-bucket}/file-staging/{deployment-name}/{COLLECTION}/{file_name}`. + +#### CnmResponse + +Assuming a successful execution of the workflow, this task will recover the `meta.cnm` key from the CMA output, and add a "SUCCESS" record to the notification Kinesis stream. + +If a prior step in the workflow has failed, this will add a "FAILURE" record to the stream instead. + +The data written to the `response-endpoint` should adhere to the [Response Message Fields](https://github.com/podaac/cloud-notification-message-schema#cumulus-sns-schema) schema. + +**Example CNM Success Response**: + +```json +{ + "provider": "PODAAC_SWOT", + "collection": "SWOT_Prod_l2:1", + "processCompleteTime": "2017-09-30T03:45:29.791198", + "submissionTime": "2017-09-30T03:42:29.791198", + "receivedTime": "2017-09-30T03:42:31.634552", + "identifier": "1234-abcd-efg0-9876", + "response": { + "status": "SUCCESS" + } +} +``` + +**Example CNM Error Response**: + +```json +{ + "provider": "PODAAC_SWOT", + "collection": "SWOT_Prod_l2:1", + "processCompleteTime": "2017-09-30T03:45:29.791198", + "submissionTime": "2017-09-30T03:42:29.791198", + "receivedTime": "2017-09-30T03:42:31.634552", + "identifier": "1234-abcd-efg0-9876", + "response": { + "status": "FAILURE", + "errorCode": "PROCESSING_ERROR", + "errorMessage": "File [cumulus-dev-a4d38f59-5e57-590c-a2be-58640db02d91/prod_20170926T11:30:36/production_file.nc] did not match gve checksum value." + } +} +``` + +Note the `CnmResponse` state defined in the `.tf` workflow definition above configures `$.exception` to be passed to the `CnmResponse` Lambda keyed under `config.WorkflowException`. This is required for the `CnmResponse` code to deliver a failure response. + +To test the failure scenario, send a record missing the `product.name` key. + +--- + +## Verify results + +### Check for successful execution on the dashboard + +Following the successful execution of this workflow, you should expect to see the workflow complete successfully on the dashboard: + +![Screenshot of a successful CNM workflow appearing on the executions page of the Cumulus dashboard](assets/cnm_success_example.png) + +### Check the test granule has been delivered to S3 staging + +The test granule identified in the Kinesis record should be moved to the deployment's private staging area. + +### Check for Kinesis records + +A `SUCCESS` notification should be present on the `response-endpoint` Kinesis stream. + +You should be able to validate the notification and response streams have the expected records with the following steps (the AWS CLI Kinesis [Basic Stream Operations](https://docs.aws.amazon.com/streams/latest/dev/fundamental-stream.html) is useful to review before proceeding): + +Get a shard iterator (substituting your stream name as appropriate): + +```bash +aws kinesis get-shard-iterator \ + --shard-id shardId-000000000000 \ + --shard-iterator-type LATEST \ + --stream-name NOTIFICATION_OR_RESPONSE_STREAM_NAME +``` + +which should result in an output to: + +```json +{ + "ShardIterator": "VeryLongString==" +} +``` + +- Re-trigger the workflow by using the `put-record` command from +- As the workflow completes, use the output from the `get-shard-iterator` command to request data from the stream: + +```bash +aws kinesis get-records --shard-iterator SHARD_ITERATOR_VALUE +``` + +This should result in output similar to: + +```json +{ + "Records": [ + { + "SequenceNumber": "49586720336541656798369548102057798835250389930873978882", + "ApproximateArrivalTimestamp": 1532664689.128, + "Data": "eyJpZGVudGlmaWVyICI6InRlc3RJZGVudGlmaWVyMTIzNDU2IiwidmVyc2lvbiI6IjAwNiIsImNvbGxlY3Rpb24iOiJNT0QwOUdRIiwicHJvdmlkZXIiOiJUZXN0UHJvdmlkZXIiLCJwcm9kdWN0U2l6ZSI6MTkwODYzNS4wLCJyZXNwb25zZSI6eyJzdGF0dXMiOiJTVUNDRVNTIn0sInByb2Nlc3NDb21wbGV0ZVRpbWUiOiIyMDE4LTA3LTI3VDA0OjExOjI4LjkxOSJ9", + "PartitionKey": "1" + }, + { + "SequenceNumber": "49586720336541656798369548102059007761070005796999266306", + "ApproximateArrivalTimestamp": 1532664707.149, + "Data": "eyJpZGVudGlmaWVyICI6InRlc3RJZGVudGlmaWVyMTIzNDU2IiwidmVyc2lvbiI6IjAwNiIsImNvbGxlY3Rpb24iOiJNT0QwOUdRIiwicHJvdmlkZXIiOiJUZXN0UHJvdmlkZXIiLCJwcm9kdWN0U2l6ZSI6MTkwODYzNS4wLCJyZXNwb25zZSI6eyJzdGF0dXMiOiJTVUNDRVNTIn0sInByb2Nlc3NDb21wbGV0ZVRpbWUiOiIyMDE4LTA3LTI3VDA0OjExOjQ2Ljk1OCJ9", + "PartitionKey": "1" + } + ], + "NextShardIterator": "AAAAAAAAAAFo9SkF8RzVYIEmIsTN+1PYuyRRdlj4Gmy3dBzsLEBxLo4OU+2Xj1AFYr8DVBodtAiXbs3KD7tGkOFsilD9R5tA+5w9SkGJZ+DRRXWWCywh+yDPVE0KtzeI0andAXDh9yTvs7fLfHH6R4MN9Gutb82k3lD8ugFUCeBVo0xwJULVqFZEFh3KXWruo6KOG79cz2EF7vFApx+skanQPveIMz/80V72KQvb6XNmg6WBhdjqAA==", + "MillisBehindLatest": 0 +} +``` + +Note the data encoding is not human readable and would need to be parsed/converted to be interpretable. There are many options to build a Kineis consumer such as the [KCL](https://docs.aws.amazon.com/streams/latest/dev/developing-consumers-with-kcl.html). + +For purposes of validating the workflow, it may be simpler to locate the workflow in the [Step Function Management Console](https://console.aws.amazon.com/states/home) and assert the expected output is similar to the below examples. + +**Successful CNM Response Object Example:** + +```json +{ + "cnmResponse": { + "provider": "TestProvider", + "collection": "MOD09GQ", + "version": "123456", + "processCompleteTime": "2017-09-30T03:45:29.791198", + "submissionTime": "2017-09-30T03:42:29.791198", + "receivedTime": "2017-09-30T03:42:31.634552", + "identifier ": "testIdentifier123456", + "response": { + "status": "SUCCESS" + } + } +} +``` + +--- + +## Kinesis Record Error Handling + +### messageConsumer + +The default Kinesis stream processing in the Cumulus system is configured for record error tolerance. + +When the `messageConsumer` fails to process a record, the failure is captured and the record is published to the `kinesisFallback` SNS Topic. The `kinesisFallback` SNS topic broadcasts the record and a subscribed copy of the `messageConsumer` Lambda named `kinesisFallback` consumes these failures. + +At this point, the [normal Lambda asynchronous invocation retry behavior](https://docs.aws.amazon.com/lambda/latest/dg/retries-on-errors.html) will attempt to process the record 3 mores times. After this, if the record cannot successfully be processed, it is written to a [dead letter queue](https://docs.aws.amazon.com/AWSSimpleQueueService/latest/SQSDeveloperGuide/sqs-dead-letter-queues.html). Cumulus' dead letter queue is an SQS Queue named `kinesisFailure`. Operators can use this queue to inspect failed records. + +This system ensures when `messageConsumer` fails to process a record and trigger a workflow, the record is retried 3 times. This retry behavior improves system reliability in case of any external service failure outside of Cumulus control. + +The Kinesis error handling system - the `kinesisFallback` SNS topic, `messageConsumer` Lambda, and `kinesisFailure` SQS queue - come with the API package and do not need to be configured by the operator. + +To examine records that were unable to be processed at any step you need to go look at the dead letter queue `{{prefix}}-kinesisFailure`. +Check the [Simple Queue Service (SQS) console](https://console.aws.amazon.com/sqs/home). Select your queue, and under the `Queue Actions` tab, you can choose `View/Delete Messages`. `Start polling` for messages and you will see records that failed to process through the `messageConsumer`. + +Note, these are only records that occurred when processing records from Kinesis streams. Workflow failures are handled differently. + +### Kinesis Stream logging + +#### Notification Stream messages + +Cumulus includes two Lambdas (`KinesisInboundEventLogger` and `KinesisOutboundEventLogger`) that utilize the same code to take a Kinesis record event as input, deserialize the data field and output the modified event to the logs. + +When a `kinesis` rule is created, in addition to the `messageConsumer` event mapping, an event mapping is created to trigger `KinesisInboundEventLogger` to record a log of the inbound record, to allow for analysis in case of unexpected failure. + +#### Response Stream messages + +Cumulus also supports this feature for all outbound messages. To take advantage of this feature, you will need to set an event mapping on the `KinesisOutboundEventLogger` Lambda that targets your `response-endpoint`. You can do this in the Lambda management page for `KinesisOutboundEventLogger`. Add a Kinesis trigger, and configure it to target the cnmResponseStream for your workflow: + +![Screenshot of the AWS console showing configuration for Kinesis stream trigger on KinesisOutboundEventLogger Lambda](assets/KinesisLambdaTriggerConfiguration.png) + +Once this is done, all records sent to the `response-endpoint` will also be logged in CloudWatch. For more on configuring Lambdas to trigger on Kinesis events, please see [creating an event source mapping](https://docs.aws.amazon.com/lambda/latest/dg/with-kinesis.html#services-kinesis-eventsourcemapping). diff --git a/website/versioned_docs/version-v13.4.0/deployment/README.md b/website/versioned_docs/version-v13.4.0/deployment/README.md new file mode 100644 index 00000000000..d3b6583745d --- /dev/null +++ b/website/versioned_docs/version-v13.4.0/deployment/README.md @@ -0,0 +1,579 @@ +--- +id: version-v13.4.0-deployment-readme +title: How to Deploy Cumulus +hide_title: false +original_id: deployment-readme +--- + +## Overview + +This is a guide for deploying a new instance of Cumulus. + +This document assumes familiarity with Terraform. If you are not comfortable +working with Terraform, the following links should bring you up to speed: + +- [Introduction to Terraform](https://www.terraform.io/intro/index.html) +- [Getting Started with Terraform and AWS](https://learn.hashicorp.com/terraform/?track=getting-started#getting-started) +- [Terraform Configuration Language](https://www.terraform.io/docs/configuration/index.html) + +The process involves: + +- Creating [AWS S3 Buckets](https://docs.aws.amazon.com/AmazonS3/latest/dev/UsingBucket.html) +- Configuring a VPC, if necessary +- Configuring an Earthdata application, if necessary +- Creating/configuring a [PostgreSQL 10.2 compatible database](../deployment/postgres_database_deployment), and an AWS Secrets Manager secret to allow database access +- Creating a Lambda layer for the [Cumulus Message Adapter](./../workflows/input_output.md#cumulus-message-adapter) +- Creating resources for your Terraform backend +- Using [Terraform](https://www.terraform.io) to deploy resources to AWS + +--- + +## Requirements + +### Linux/MacOS software requirements + +- git +- zip +- AWS CLI - [AWS command line interface](https://aws.amazon.com/cli/) +- [Terraform](https://www.terraform.io) + +### Install Terraform + +It is recommended to keep a consistent version of Terraform as you deploy. Once your state files are migrated to a higher version, they are not always backwards compatible so integrators should pin their Terraform version. This is easily accomplished using the Terraform Version Manager [tfenv](https://github.com/tfutils/tfenv). If you have a CI environment (or any other machine) that you are using to deploy the same stack, **you should pin your version across those machines as well**, otherwise you will run into errors trying to re-deploy from your local machine. + +If you are using a Mac and [Homebrew](https://brew.sh), installing tfenv is +as simple as: + +```shell +brew update +brew install tfenv +``` + +For other cases, +[installation instructions](https://github.com/tfutils/tfenv#installation) +are available. + +```shell + $ tfenv install 0.13.6 +[INFO] Installing Terraform v0.13.6 +... +[INFO] Switching completed + +$ tfenv use 0.13.6 +[INFO] Switching to v0.13.6 +... +[INFO] Switching completed +``` + +It is recommended to stay on the Cumulus Core TF version which can be found [here](https://github.com/nasa/cumulus/blob/master/example/.tfversion). Any changes to that will be noted in the release notes. + +To verify your Terraform version run: + +```shell +$ terraform --version +Terraform v0.13.6 +``` + +### Credentials + +- [CMR](https://earthdata.nasa.gov/about/science-system-description/eosdis-components/common-metadata-repository) username and password. CMR credentials must be provided if you are exporting metadata to CMR with Earthdata Login authentication. +- [NASA Launchpad](https://launchpad.nasa.gov). Launchpad credentials must be provided if you are using Launchpad authentication to export metadata to CMR or to authenticate with the Cumulus API. +- [Earthdata Login](https://earthdata.nasa.gov/about/science-system-description/eosdis-components/earthdata-login) username and password. User must have the ability to administer and/or create applications in URS. It's recommended to obtain an account in the test environment (UAT). + +### Needed Git Repositories + +- [Deployment Template](https://github.com/nasa/cumulus-template-deploy) +- [Cumulus Dashboard](https://github.com/nasa/cumulus-dashboard) + +--- + +## Prepare deployment repository + + > _If you already are working with an existing repository that is configured appropriately for the version of Cumulus you intend to deploy or update, skip to [Prepare AWS configuration.](deployment-readme#prepare-aws-configuration)_ + +Clone the [`cumulus-template-deploy`](https://github.com/nasa/cumulus-template-deploy) repo and name appropriately for your organization: + +```bash + git clone https://github.com/nasa/cumulus-template-deploy +``` + +We will return to [configuring this repo and using it for deployment below](#deploying-the-cumulus-instance). + +
+ Optional: Create a new repository + + [Create a new repository](https://help.github.com/articles/creating-a-new-repository/) on Github so that you can add your workflows and other modules to source control: + +```bash + git remote set-url origin https://github.com/nasa/ + git push origin master +``` + +You can then [add/commit](https://help.github.com/articles/adding-a-file-to-a-repository-using-the-command-line/) changes as needed. + +> **Note**: If you are pushing your deployment code to a git repo, make sure to add `terraform.tf` and `terraform.tfvars` to `.gitignore`, **as these files will contain sensitive data related to your AWS account**. + +
+ +--- + +## Prepare AWS configuration + +### Set Access Keys + +You need to make some AWS information available to your environment. If you don't already have the access key and secret access key of an AWS user with IAM Create-User permissions, you must [Create Access Keys](https://docs.aws.amazon.com/general/latest/gr/managing-aws-access-keys.html) for such a user with IAM Create-User permissions, then export the access keys: + +```bash + export AWS_ACCESS_KEY_ID= + export AWS_SECRET_ACCESS_KEY= + export AWS_REGION= +``` + +If you don't want to set environment variables, [access keys can be stored locally via the AWS CLI.](http://docs.aws.amazon.com/cli/latest/userguide/cli-chap-getting-started.html) + +### Create S3 Buckets + +See [creating s3 buckets](deployment/create_bucket.md) for more information on how to create a bucket. + +The following s3 bucket should be created (replacing `` with whatever you'd like, generally your organization/DAAC's name): + +- `-internal` + +You can create additional s3 buckets based on the needs of your workflows. + +These buckets do not need any non-default permissions to function with Cumulus, however your local security requirements may vary. + +**Note**: S3 bucket object names are global and must be unique across all accounts/locations/etc. + +### VPC, Subnets and Security Group + +Cumulus supports operation within a VPC, but you will need to separately create: + +- VPC +- Subnet +- Security group +- VPC endpoints for the various services used by Cumulus if you wish to route traffic through the VPC + +These resources only need to be created once per AWS account and their ids will be used to configure your Terraform deployment. + +#### Elasticsearch in a VPC + +Amazon Elasticsearch Service does not use a VPC Endpoint. To use ES within a VPC, before deploying run: + +```shell +aws iam create-service-linked-role --aws-service-name es.amazonaws.com +``` + +This operation only needs to be done once per account, but it must be done for both NGAP and regular AWS environments. + +### Look up ECS-optimized AMI (DEPRECATED) + +> **Note:** This step is unnecessary if you using the latest changes in the [`cumulus-template-deploy` repo which will automatically determine the AMI ID for you +based on your `deploy_to_ngap` variable](https://github.com/nasa/cumulus-template-deploy/commit/8472e2f3a7185d77bb68bf9e0f21a92a91b0cba9). + +Look up the recommended machine image ID for the Linux version and AWS region of your deployment. See [Linux Amazon ECS-optimized AMIs docs](https://docs.aws.amazon.com/AmazonECS/latest/developerguide/ecs-optimized_AMI.html#ecs-optimized-ami-linux). The image ID, beginning with `ami-`, will be assigned to the `ecs_cluster_instance_image_id` variable for the [cumulus-tf module](https://github.com/nasa/cumulus/blob/master/tf-modules/cumulus/variables.tf). + +### Set up EC2 key pair (optional) + +The key pair will be used to SSH into your EC2 instance(s). It is recommended to [create or import a key pair](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-key-pairs.html) and specify it in your Cumulus deployment. + +This can also be done post-deployment by redeploying your Cumulus instance. + +--- + +## Configure EarthData application + +The Cumulus stack can authenticate with [Earthdata Login](https://urs.earthdata.nasa.gov/documentation). If you want to use this functionality, you must create and register a new Earthdata application. Use the [User Acceptance Tools (UAT) site](https://uat.urs.earthdata.nasa.gov) unless you intend use a different URS environment (which will require updating the `urs_url` value shown below). + +Follow the directions on [how to register an application](https://wiki.earthdata.nasa.gov/display/EL/How+To+Register+An+Application). Use any url for the `Redirect URL`, it will be deleted in a later step. Also note the password in step 3 and client ID in step 4 use these to replace `urs_client_id` and `urs_client_password` in the `terraform.tfvars` for the `cumulus-tf` module shown below. + +--- + +## Create resources for Terraform state + +> _If you're re-deploying an existing Cumulus configuration you should skip to [Deploy the Cumulus instance](deployment-readme#deploy-the-cumulus-instance), as these values should already be configured._ + +The state of the Terraform deployment is stored in S3. In the following examples, it will be assumed that state is being stored in a bucket called `my-tf-state`. You can also use an existing bucket, if desired. + +### Create the state bucket + +```shell +aws s3api create-bucket --bucket my-tf-state +``` + +In order to help prevent loss of state information, **it is strongly recommended that versioning be enabled on the state bucket**. + +```shell +aws s3api put-bucket-versioning \ + --bucket my-tf-state \ + --versioning-configuration Status=Enabled +``` + +⚠️ **Note:** If your state information does become lost or corrupt, then deployment (via `terraform apply`) will have unpredictable results, including possible loss of data and loss of deployed resources. In order to reduce your risk of the corruption or loss of your Terraform state file, or otherwise corrupt your Cumulus deployment, please see the [Terraform Best Practices](terraform-best-practices.md) guide. + +### Create the locks table + +Terraform uses a lock stored in DynamoDB in order to prevent multiple simultaneous updates. In the following examples, that table will be called `my-tf-locks`. + +```shell +$ aws dynamodb create-table \ + --table-name my-tf-locks \ + --attribute-definitions AttributeName=LockID,AttributeType=S \ + --key-schema AttributeName=LockID,KeyType=HASH \ + --billing-mode PAY_PER_REQUEST \ + --region us-east-1 +``` + +--- + +## Configure the PostgreSQL database + +Cumulus requires a PostgreSQL 10.2 compatible database cluster deployed to AWS. We suggest utilizing [RDS](https://docs.aws.amazon.com/rds/index.html). For further guidance about what type of RDS database to use, please [see the guide on choosing and configuring your RDS database](./choosing_configuring_rds.md). + +Cumulus provides a default [template and RDS cluster module](./postgres-database-deployment.md) utilizing Aurora Serverless. + +However, Core intentionally provides a "bring your own" approach, and any well-planned cluster setup should work, given the following: + +- Appropriate testing/evaluation is given to ensure the database capacity will scale and the database deployment will allow access to Cumulus's internal components. Core provides for security-group oriented permissions management via the `rds_security_group` configuration parameter. +- The database is configured such that its endpoint is accessible from the VPC and subnets configured for the Core deployment. +- An AWS Secrets Manager secret exists that has the following format: + +```json +{ + "database": "databaseName", + "host": "xxx", + "password": "defaultPassword", + "port": 5432, + "username": "xxx" +} +``` + +- `database` -- the PostgreSQL database used by the configured user +- `host` -- the RDS service host for the database in the form (dbClusterIdentifier)-(AWS ID string).(region).rds.amazonaws.com +- `password` -- the database password +- `port` -- The database connection port, should always be 5432 +- `username` -- the database username + +This secret should provide access to a PostgreSQL database provisioned on the cluster. + +To configure Cumulus you will need: + +- The AWS Secrets Manager ARN for the _user_ Core will write with (e.g. `arn:aws:secretsmanager:AWS-REGION:xxxxx:secret:xxxxxxxxxx20210407182709367700000002-dpmpXA` ) for use in configuring `rds_user_access_secret_arn`. +- (Optionally) The security group ID that provides access to the cluster to configure `rds_security_group`. + +--- + +## Deploy the Cumulus instance + +A typical Cumulus deployment is broken into two +[Terraform root modules](https://www.terraform.io/docs/configuration/modules.html): +[`data-persistence`](https://github.com/nasa/cumulus/tree/master/tf-modules/data-persistence) and [`cumulus`](https://github.com/nasa/cumulus/tree/master/tf-modules/cumulus). + +The `data-persistence` module should be deployed first. This module creates the Elasticsearch domain, DynamoDB tables, RDS database tables, and performs any structural updates needed on the RDS tables via migrations. During the RDS migration, duplicate tables will be deployed by the `data-persistence` module in both DynamoDB and the RDS database. The `cumulus` module deploys the rest of Cumulus: distribution, API, ingest, workflows, etc. The `cumulus` module depends on the resources created in the `data-persistence` deployment. + +Each of these modules have to be deployed independently and require their own Terraform backend, variable, and output settings. The template deploy repo that was cloned previously already contains the scaffolding of the necessary files for the deployment of each module: `data-persistence-tf` deploys the `data-persistence` module and `cumulus-tf` deploys the `cumulus` module. For reference on the files that are included, see the [documentation on adding components to a Terraform deployment](components.md#adding-components-to-your-terraform-deployment). + +### Troubleshooting + +Please see our [troubleshooting documentation for any issues with your deployment](../troubleshooting/troubleshooting-deployment) when performing the upcoming steps. + +### Configure and deploy the `data-persistence-tf` root module + +These steps should be executed in the `data-persistence-tf` directory of the template deploy repo that you previously cloned. Run the following to copy the example files. + +```shell +cd data-persistence-tf/ +cp terraform.tf.example terraform.tf +cp terraform.tfvars.example terraform.tfvars +``` + +In `terraform.tf`, configure the remote state settings by substituting the appropriate values for: + +- `bucket` +- `dynamodb_table` +- `PREFIX` (whatever prefix you've chosen for your deployment) + +Fill in the appropriate values in `terraform.tfvars`. See the [data-persistence module variable definitions](https://github.com/nasa/cumulus/blob/master/tf-modules/data-persistence/variables.tf) for more detail on each variable. + +Consider [the size of your Elasticsearch cluster](#elasticsearch) when configuring data-persistence. + +**Reminder:** _Elasticsearch is optional and can be disabled using `include_elasticsearch = false` in your `terraform.tfvars`. Your Cumulus dashboard will not work without Elasticsearch._ + +**Reminder:** _If you are including `subnet_ids` in your `terraform.tfvars`, Elasticsearch will need a service-linked role to deploy successfully. Follow the [instructions above](#elasticsearch-in-a-vpc) to create the service-linked role if you haven't already._ + +#### Initialize Terraform + +Run `terraform init`[^3] + +You should see output like: + +```shell +* provider.aws: version = "~> 2.32" + +Terraform has been successfully initialized! +``` + +#### Deploy + +Run `terraform apply` to deploy your data persistence resources. Type `yes` when prompted to confirm that you want to create the resources. Assuming the operation is successful, you should see output like: + +```shell +Apply complete! Resources: 16 added, 0 changed, 0 destroyed. + +Outputs: + +dynamo_tables = { + "access_tokens" = { + "arn" = "arn:aws:dynamodb:us-east-1:12345:table/prefix-AccessTokensTable" + "name" = "prefix-AccessTokensTable" + } + # ... more tables ... +} +elasticsearch_alarms = [ + { + "arn" = "arn:aws:cloudwatch:us-east-1:12345:alarm:prefix-es-vpc-NodesLowAlarm" + "name" = "prefix-es-vpc-NodesLowAlarm" + }, + # ... more alarms ... +] +elasticsearch_domain_arn = arn:aws:es:us-east-1:12345:domain/prefix-es-vpc +elasticsearch_hostname = vpc-prefix-es-vpc-abcdef.us-east-1.es.amazonaws.com +elasticsearch_security_group_id = sg-12345 +``` + +Your data persistence resources are now deployed. + +### Deploy the Cumulus Message Adapter layer (DEPRECATED) + +> **Note:** This step is unnecessary if you using the latest changes in the [`cumulus-template-deploy` repo which will automatically download the Cumulus Message Adapter and create the layer for you based on your `cumulus_message_adapter_version` variable](https://github.com/nasa/cumulus-template-deploy/commit/8472e2f3a7185d77bb68bf9e0f21a92a91b0cba9). + +The [Cumulus Message Adapter (CMA)](./../workflows/input_output.md#cumulus-message-adapter) is necessary for interpreting the input and output of Cumulus workflow steps. The CMA is now integrated with Cumulus workflow steps as a Lambda layer. + +To deploy a CMA layer to your account: + +1. Go to the [CMA releases page](https://github.com/nasa/cumulus-message-adapter/releases) and download the `cumulus-message-adapter.zip` for the desired release +2. Use the AWS CLI to publish your layer: + +```shell +$ aws lambda publish-layer-version \ + --layer-name prefix-CMA-layer \ + --region us-east-1 \ + --zip-file fileb:///path/to/cumulus-message-adapter.zip +{ + ... more output ... + "LayerVersionArn": "arn:aws:lambda:us-east-1:1234567890:layer:prefix-CMA-layer:1", + ... more output ... +} +``` + +Make sure to copy the `LayerVersionArn` of the deployed layer, as it will be used to configure the `cumulus-tf` deployment in the next step. + +### Configure and deploy the `cumulus-tf` root module + +These steps should be executed in the `cumulus-tf` directory of the template repo that was cloned previously. + +```shell +cd cumulus-tf/ +cp terraform.tf.example terraform.tf +cp terraform.tfvars.example terraform.tfvars +``` + +In `terraform.tf`, configure the remote state settings by substituting the appropriate values for: + +- `bucket` +- `dynamodb_table` +- `PREFIX` (whatever prefix you've chosen for your deployment) + +Fill in the appropriate values in `terraform.tfvars`. See the [Cumulus module variable definitions](https://github.com/nasa/cumulus/blob/master/tf-modules/cumulus/variables.tf) for more detail on each variable. + +Notes on specific variables: + +- **`deploy_to_ngap`**: This variable controls the provisioning of certain resources and policies that are specific to an NGAP environment. **If you are deploying to NGAP, you must set this variable to `true`.** +- **`prefix`**: The value should be the same as the `prefix` from the data-persistence deployment. +- **`data_persistence_remote_state_config`**: This object should contain the remote state values that you configured in `data-persistence-tf/terraform.tf`. These settings allow `cumulus-tf` to determine the names of the resources created in `data-persistence-tf`. +- **`rds_security_group`**: The ID of the security group used to allow access to the PostgreSQL database +- **`rds_user_access_secret_arn`**: The ARN for the Secrets Manager secret that provides database access information +- **`cumulus_message_adapter_version`**: The version number (e.g. `1.3.0`) of the [Cumulus Message Adapter](https://github.com/nasa/cumulus-message-adapter/releases) to deploy +- **`key_name` (optional)**: The name of your key pair from [setting up your key pair](#set-up-ec2-key-pair-optional). Adding your `key_name` sets the EC2 keypair +for deployment's EC2 instances and allows you to connect to them via [SSH/SSM](https://docs.aws.amazon.com/systems-manager/latest/userguide/session-manager-working-with-sessions-start.html). + +Consider [the sizing of your Cumulus instance](#cumulus-instance-sizing) when configuring your variables. + +#### Choose a distribution API + +Cumulus can be configured to use either the Thin Egress App (TEA) or the Cumulus Distribution API. The default selection is the Thin Egress App if you're using the [Deployment Template](https://github.com/nasa/cumulus-template-deploy). + +**IMPORTANT!** If you already have a deployment using the TEA distribution and want to switch to Cumulus Distribution, there will be an API Gateway change. This means that there will be downtime while you update your CloudFront endpoint to use +the new API gateway. + +#### Configure the Thin Egress App + +The Thin Egress App can be used for Cumulus distribution and is the default selection. It allows authentication using Earthdata Login. Follow the steps [in the documentation](./thin_egress_app) to configure distribution in your `cumulus-tf` deployment. + +#### Configure the Cumulus Distribution API (optional) + +If you would prefer to use the Cumulus Distribution API, which supports [AWS Cognito authentication](https://aws.amazon.com/cognito/), follow [these steps](./cumulus_distribution) to configure distribution in your `cumulus-tf` deployment. + +#### Initialize Terraform + +Follow the [above instructions to initialize Terraform](#initialize-terraform) using `terraform init`[^3]. + +#### Deploy + +Run `terraform apply` to deploy the resources. Type `yes` when prompted to confirm that you want to create the resources. Assuming the operation is successful, you should see output like this: + +```shell +Apply complete! Resources: 292 added, 0 changed, 0 destroyed. + +Outputs: + +archive_api_redirect_uri = https://abc123.execute-api.us-east-1.amazonaws.com/dev/token +archive_api_uri = https://abc123.execute-api.us-east-1.amazonaws.com/dev/ +distribution_redirect_uri = https://abc123.execute-api.us-east-1.amazonaws.com/DEV/login +distribution_url = https://abc123.execute-api.us-east-1.amazonaws.com/DEV/ +``` + +**Note:** Be sure to copy the redirect URLs, as you will use them to update your Earthdata application. + +### Update Earthdata Application + +You will need to add two redirect URLs to your EarthData login application. + +1. Login to URS. +2. Under My Applications -> Application Administration -> use the edit icon of your application. +3. Under Manage -> redirect URIs, add the Archive API url returned from the stack deployment + - e.g. `archive_api_redirect_uri = https://.execute-api.us-east-1.amazonaws.com/dev/token`. +4. Also add the Distribution url + - e.g. `distribution_redirect_uri = https://.execute-api.us-east-1.amazonaws.com/dev/login`[^1]. +5. You may delete the placeholder url you used to create the application. + +If you've lost track of the needed redirect URIs, they can be located on the [API Gateway](https://console.aws.amazon.com/apigateway). Once there, select `-archive` and/or `-thin-egress-app-EgressGateway`, `Dashboard` and utilizing the base URL at the top of the page that is accompanied by the text `Invoke this API at:`. Make sure to append `/token` for the archive URL and `/login` to the thin egress app URL. + +--- + +## Deploy Cumulus dashboard + +### Dashboard Requirements + +Please note that the requirements are similar to the [Cumulus stack deployment requirements](deployment-readme#requirements). The installation instructions below include a step that will install/use the required node version referenced in the `.nvmrc` file in the dashboard repository. + +- git +- [node 12.18](https://nodejs.org/en/) (use [nvm](https://github.com/creationix/nvm) to upgrade/downgrade) +- [npm](https://www.npmjs.com/get-npm) +- zip +- AWS CLI - [AWS command line interface](https://aws.amazon.com/cli/) +- python + +### Prepare AWS + +**Create S3 bucket for dashboard:** + +- Create it, e.g. `-dashboard`. Use the command line or console as you did when [preparing AWS configuration](deployment-readme#prepare-aws-configuration). +- Configure the bucket to host a website: + - AWS S3 console: Select `-dashboard` bucket then, "Properties" -> "Static Website Hosting", point to `index.html` + - CLI: `aws s3 website s3://-dashboard --index-document index.html` +- The bucket's url will be `http://-dashboard.s3-website-.amazonaws.com` or you can find it on the AWS console via "Properties" -> "Static website hosting" -> "Endpoint" +- Ensure the bucket's access permissions allow your deployment user access to write to the bucket + +### Install dashboard + +To install the dashboard, clone the [Cumulus dashboard](https://github.com/nasa/cumulus-dashboard) repository into the root deploy directory and install dependencies with `npm install`: + +```bash + git clone https://github.com/nasa/cumulus-dashboard + cd cumulus-dashboard + nvm use + npm install +``` + +If you do not have the correct version of node installed, replace `nvm use` with `nvm install $(cat .nvmrc)` in the above example. + +#### Dashboard versioning + +By default, the `master` branch will be used for dashboard deployments. The `master` branch of the dashboard repo contains the most recent stable release of the dashboard. + +If you want to test unreleased changes to the dashboard, use the `develop` branch. + +Each [release/version of the dashboard](https://github.com/nasa/cumulus-dashboard/releases) will have [a tag in the dashboard repo](https://github.com/nasa/cumulus-dashboard/tags). Release/version numbers will use semantic versioning (major/minor/patch). + +To checkout and install a specific version of the dashboard: + +```bash + git fetch --tags + git checkout # e.g. v1.2.0 + nvm use + npm install +``` + +If you do not have the correct version of node installed, replace `nvm use` with `nvm install $(cat .nvmrc)` in the above example. + +### Building the dashboard + +**Note**: These environment variables are available during the build: `APIROOT`, `DAAC_NAME`, `STAGE`, `HIDE_PDR`. Any of these can be set on the command line to override the values contained in `config.js` when running the build below. + +To configure your dashboard for deployment, set the `APIROOT` environment variable to your app's API root.[^2] + +Build the dashboard from the dashboard repository root directory, `cumulus-dashboard`: + +```bash + APIROOT= npm run build +``` + +### Dashboard deployment + +Deploy dashboard to s3 bucket from the `cumulus-dashboard` directory: + +Using AWS CLI: + +```bash + aws s3 sync dist s3://-dashboard --acl public-read +``` + +From the S3 Console: + +- Open the `-dashboard` bucket, click 'upload'. Add the contents of the 'dist' subdirectory to the upload. Then select 'Next'. On the permissions window allow the public to view. Select 'Upload'. + +You should be able to visit the dashboard website at `http://-dashboard.s3-website-.amazonaws.com` or find the url +`-dashboard` -> "Properties" -> "Static website hosting" -> "Endpoint" and login with a user that you configured for access in the [Configure and Deploy the Cumulus Stack](deployment-readme#configure-and-deploy-the-cumulus-stack) step. + +--- + +## Cumulus Instance Sizing + +The Cumulus deployment default sizing for Elasticsearch instances, EC2 instances, and Autoscaling Groups are small and designed for testing and cost savings. The default settings are likely not suitable for production workloads. Sizing is highly individual and dependent on expected load and archive size. + +> Please be cognizant of costs as any change in size will affect your AWS bill. AWS provides a [pricing calculator](https://calculator.aws/#/) for estimating costs. + +### Elasticsearch + +The [mappings file](https://github.com/nasa/cumulus/blob/master/packages/api/models/mappings.json) contains all of the data types that will be indexed into Elasticsearch. Elasticsearch sizing is tied to your archive size, including your collections, granules, and workflow executions that will be stored. + +AWS provides [documentation](https://docs.aws.amazon.com/elasticsearch-service/latest/developerguide/sizing-domains.html) on calculating and configuring for sizing. + +In addition to size you'll want to consider the [number of nodes](https://docs.aws.amazon.com/elasticsearch-service/latest/developerguide/es-managedomains-dedicatedmasternodes.html) which determine how the system reacts in the event of a failure. + +Configuration can be done in the [data persistence module](https://github.com/nasa/cumulus/blob/master/tf-modules/data-persistence/variables.tf#L16) in `elasticsearch_config` and the [cumulus module](https://github.com/nasa/cumulus/blob/master/tf-modules/cumulus/variables.tf#L541) in `es_index_shards`. + +> If you make changes to your Elasticsearch configuration you will need to [reindex](../troubleshooting/reindex-elasticsearch) for those changes to take effect. + +### EC2 instances and autoscaling groups + +EC2 instances are used for long-running operations (i.e. generating a reconciliation report) and long-running workflow tasks. Configuration for your ECS cluster is achieved via [Cumulus deployment variables](https://github.com/nasa/cumulus/blob/master/tf-modules/cumulus/variables.tf). + +When configuring your ECS cluster consider: + +- The [EC2 instance type](https://aws.amazon.com/ec2/instance-types/) and [EBS volume size](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/volume_constraints.html) needed to accommodate your workloads. Configured as `ecs_cluster_instance_type` and `ecs_cluster_instance_docker_volume_size`. +- The minimum and desired number of instances on hand to accommodate your workloads. Configured as `ecs_cluster_min_size` and `ecs_cluster_desired_size`. +- The maximum number of instances you will need and are willing to pay for to accommodate your heaviest workloads. Configured as `ecs_cluster_max_size`. +- Your autoscaling parameters: `ecs_cluster_scale_in_adjustment_percent`, `ecs_cluster_scale_out_adjustment_percent`, `ecs_cluster_scale_in_threshold_percent`, and `ecs_cluster_scale_out_threshold_percent`. + +--- + +## Footnotes + +[^1]: To add another redirect URIs to your application. On Earthdata home page, select "My Applications". Scroll down to "Application Administration" and use the edit icon for your application. Then Manage -> Redirect URIs. +[^2]: The API root can be found a number of ways. The easiest is to note it in the output of the app deployment step. But you can also find it from the `AWS console -> Amazon API Gateway -> APIs -> -archive -> Dashboard`, and reading the URL at the top after "Invoke this API at" +[^3]: Run `terraform init` if: + - This is the first time deploying the module + - You have added any additional child modules, including [Cumulus components](./components.md#available-cumulus-components) + - You have updated the `source` for any of the child modules diff --git a/website/versioned_docs/version-v13.4.0/features/dead_letter_archive.md b/website/versioned_docs/version-v13.4.0/features/dead_letter_archive.md new file mode 100644 index 00000000000..4274a08cd8b --- /dev/null +++ b/website/versioned_docs/version-v13.4.0/features/dead_letter_archive.md @@ -0,0 +1,22 @@ +--- +id: version-v13.4.0-dead_letter_archive +title: Cumulus Dead Letter Archive +hide_title: false +original_id: dead_letter_archive +--- + +This documentation explains the Cumulus dead letter archive and associated functionality. + +## DB Records DLQ Archive + +The Cumulus system contains a number of [dead letter queues](./lambda_dead_letter_queue). Perhaps the most important system lambda function supported by a DLQ is the `sfEventSqsToDbRecords` lambda function which parses Cumulus messages from workflow executions to generate and write database records to the Cumulus database. + +As of Cumulus v9+, the dead letter queue for this lambda (named `sfEventSqsToDbRecordsDeadLetterQueue`) has been updated with a consumer lambda that will automatically write any incoming records to the S3 system bucket, under the path `/dead-letter-archive/sqs/`. This will allow integrators and operators engaged in debugging missing records to inspect any Cumulus messages which failed to process and did not result in the successful creation of database records. + +## Dead Letter Archive recovery + +In addition to the above, as of Cumulus v9+, the Cumulus API also contains a new endpoint at `/deadLetterArchive/recoverCumulusMessages`. + +Sending a POST request to this endpoint will trigger a Cumulus AsyncOperation that will attempt to reprocess (and if successful delete) all Cumulus messages in the dead letter archive, using the same underlying logic as the existing `sfEventSqsToDbRecords`. Otherwise, all Cumulus messages that fail to be reprocessed will be moved to a new archive location under the path `/dead-letter-archive/failed-sqs/`. + +This endpoint may prove particularly useful when recovering from extended or unexpected database outage, where messages failed to process due to external outage and there is no essential malformation of each Cumulus message. diff --git a/website/versions.json b/website/versions.json index 6d3e910814f..ad9d971723d 100644 --- a/website/versions.json +++ b/website/versions.json @@ -1,4 +1,5 @@ [ + "v13.4.0", "v13.3.0", "v13.2.1", "v13.2.0",