diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index 0628cfa..1bc4ca2 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -40,4 +40,4 @@ jobs: flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics - name: Testing CLI (Runs both unit and integration tests) run: | - coverage run --source=src -m pytest -v -s . && coverage report --show-missing --fail-under=63 \ No newline at end of file + coverage run --source=src -m pytest -v -s . && coverage report --show-missing --fail-under=75 \ No newline at end of file diff --git a/README.md b/README.md index b5dbc08..0fe2c82 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,8 @@ + ## Greengrass Labs InfluxDB Component - `aws.greengrass.labs.database.InfluxDB` ## Overview -This AWS IoT Greengrass component allows you to provision and manage an [InfluxDB database](https://www.influxdata.com/) on your device. +This AWS IoT Greengrass component allows you to provision and manage an [InfluxDB database](https://www.influxdata.com/) on your device. At a high level, the component will do the following: @@ -10,12 +11,15 @@ At a high level, the component will do the following: 3. Retrieve a pre-configured secret containing a username and password from AWS Secret Manager via the `aws.greengrass.SecretManager` Greengrass component. These secrets will be used to setup InfluxDB. 3. Create a new InfluxDB container using the self-signed certificates and retrieved username/password, persisting the database by mounting it to a location of your choice on your host machine. 4. Validate the status of the InfluxDB instance. -5. Create a new InfluxDB auth token with read/write bucket privileges, and set up a local IPC pub/sub subscription to a configurable response topic to vend this auth token, along with other InfluxDB metadata. - * Other Greengrass components on your device can send a pub/sub request to a configurable request topic to retrieve this data, and use it to connect to InfluxDB on their own. If you would like to view an example, see [the `aws.greengrass.labs.telemetry.InfluxDBPublisher` component, which relays Greengrass system health telemetry to InfluxDB](https://github.com/awslabs/aws-greengrass-labs-telemetry-influxdbpublisher). - -This component works with the `aws.greengrass.labs.telemetry.InfluxDBPublisher` and `aws.greengrass.labs.dashboard.Grafana` components to persist and visualize Greengrass System Telemetry data, but can be used on its own or as a primitive for any application. +5. Create a new InfluxDB auth token with read/write bucket privileges, and set up a local IPC pub/sub subscription to a configurable response topic to vend this auth token, along with other InfluxDB metadata. + * Other Greengrass components on your device can send a pub/sub request to a configurable request topic to retrieve this data, and use it to connect to InfluxDB on their own. If you would like to view an example, see [the `aws.greengrass.labs.telemetry.InfluxDBPublisher` component, which relays Greengrass system health telemetry to InfluxDB](https://github.com/awslabs/aws-greengrass-labs-telemetry-influxdbpublisher). + +This component works with the `aws.greengrass.labs.dashboard.InfluxDBGrafana`, `aws.greengrass.labs.telemetry.InfluxDBPublisher` and `aws.greengrass.labs.dashboard.Grafana` components to persist and visualize Greengrass System Telemetry data, but can be used on its own or as a primitive for any application. +The `aws.greengrass.labs.dashboard.InfluxDBGrafana` component automates the setup of Grafana with InfluxDB to provide a "one-click" experience, but this component still needs to be configured first before creation. See the `Setup` section below for instructions. + * [aws.greengrass.labs.telemetry.InfluxDBPublisher](https://github.com/awslabs/aws-greengrass-labs-telemetry-influxdbpublisher) * [aws.greengrass.labs.dashboard.Grafana](https://github.com/awslabs/aws-greengrass-labs-dashboard-grafana) +* [aws.greengrass.labs.dashboard.InfluxDBGrafana](https://github.com/awslabs/aws-greengrass-labs-dashboard-influxdb-grafana) ![Architecture - Component](images/influxdb.png) @@ -24,12 +28,12 @@ The `aws.greengrass.labs.database.InfluxDB` component supports the following con * `AutoProvision` - Retrieves a username/password from Secret Manager in order to provision InfluxDB. If turned off, an InfluxDB instance will still be set up, but you will need to [provision the instance on your own](https://docs.influxdata.com/influxdb/v2.0/install/?t=Docker). * (`true`|`false`) - * default: `true` - + * default: `true` + * `SecretArn` - The ARN of the AWS Secret Manager secret containing your desired InfluxDB username/password. You must configure and deploy this secret with the [Secret manager component](https://docs.aws.amazon.com/greengrass/v2/developerguide/secret-manager-component.html), and you must specify this secret in the `accessControl` configuration parameter to allow this component to use it. - * (`string`) - * default: `arn:aws:secretsmanager:::secret:` + * (`string`) + * default: `arn:aws:secretsmanager:::secret:` * `InfluxDBMountPath` - Absolute path of a directory on your host machine that will be used to persist InfluxDB data and certs. @@ -52,14 +56,14 @@ The `aws.greengrass.labs.database.InfluxDB` component supports the following con * default: `greengrass-telemetry` -* `InfluxDBInterface` - The IP for the InfluxDB container to bind on. +* `InfluxDBInterface` - The IP for the InfluxDB container to bind on. * (`string`) * default: `127.0.0.1` * `InfluxDBPort` -The port for the InfluxDB Docker container to bind to. - * (`string`) - * default: `8086` + * (`string`) + * default: `8086` * `BridgeNetworkName` - The Docker bridge network to create and use for the InfluxDB Docker container. @@ -96,7 +100,7 @@ The `aws.greengrass.labs.database.InfluxDB` component supports the following con * `accessControl` - [Greengrass Access Control Policy](https://docs.aws.amazon.com/greengrass/v2/developerguide/interprocess-communication.html#ipc-authorization-policies), required for secret retrieval and pub/sub token vending. - * A default `accessControl` policy allowing subscribe access to the `greengrass/influxdb/token/request` topic and publish access to the `greengrass/influxdb/token/response` has been included, as well as an incomplete policy for retrieving a secret, which you will need to configure. + * A default `accessControl` policy allowing subscribe access to the `greengrass/influxdb/token/request` topic and publish access to the `greengrass/influxdb/token/response` has been included, as well as an incomplete policy for retrieving a secret, which you will need to configure. ## Setup @@ -116,13 +120,13 @@ The `aws.greengrass.labs.database.InfluxDB` component supports the following con python3-pip; \ python3 -m pip install awsiotsdk influxdb-client ``` -2. Install Docker on the host machine using [the instructions for Ubuntu](https://docs.docker.com/engine/install/ubuntu/): +2. Install Docker on the host machine using [the instructions for Ubuntu](https://docs.docker.com/engine/install/ubuntu/): ``` echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] https://download.docker.com/linux/ubuntu \ $(lsb_release -cs) stable" | sudo tee /etc/apt/sources.list.d/docker.list > /dev/null sudo apt-get update sudo apt-get install -y docker-ce docker-ce-cli containerd.io ``` - + 3. Setup AWS IoT Greengrass on the host machine [according to the installation instructions](https://docs.aws.amazon.com/greengrass/v2/developerguide/install-greengrass-core-v2.html): 4. Log in as superuser with `sudo su` and then allow `ggc_user:ggc_group` to use Docker, [as per the Docker documentation](https://docs.docker.com/engine/install/linux-postinstall/): ``` @@ -132,29 +136,29 @@ The `aws.greengrass.labs.database.InfluxDB` component supports the following con ### Component Setup 1. Install [the Greengrass Development Kit CLI](https://docs.aws.amazon.com/greengrass/v2/developerguide/install-greengrass-development-kit-cli.html) in your local workspace. - 1. Run `python3 -m pip install git+https://github.com/aws-greengrass/aws-greengrass-gdk-cli.git` + 1. Run `python3 -m pip install git+https://github.com/aws-greengrass/aws-greengrass-gdk-cli.git` 2. Pull down the component in a new directory using the GDK CLI. ``` mkdir aws-greengrass-labs-database-influxdb; cd aws-greengrass-labs-database-influxdb gdk component init --repository aws-greengrass-labs-database-influxdb ``` 3. Create an AWS Secrets Manager Secret to store your InfluxDB username/password. - 1. Go to [AWS Secrets Manager](https://console.aws.amazon.com/secretsmanager/home?region=us-east-1#!/listSecrets): - 2. Create new secret → Other type of Secret → Plaintext. The secret you use should be in the following format: - ``` - { - "influxdb_username": "myInfluxDBUsername", - "influxdb_password": "myInfluxDBPassword123!" - } - ``` - Note that your password **must** be at least 16 characters long and must include uppercase and lowercase letters, numbers, and special characters. - - Note down the ARN of the secrets you just made. + 1. Go to [AWS Secrets Manager](https://console.aws.amazon.com/secretsmanager/home?region=us-east-1#!/listSecrets): + 2. Create new secret → Other type of Secret → Plaintext. The secret you use should be in the following format: + ``` + { + "influxdb_username": "myInfluxDBUsername", + "influxdb_password": "myInfluxDBPassword123!" + } + ``` + Note that your password **must** be at least 16 characters long and must include uppercase and lowercase letters, numbers, and special characters(#$@%+*&!^). + + Note down the ARN of the secrets you just made. 4. Authorize Greengrass to retrieve this secret using IAM: - 1. Follow [the Greengrass documentation](:https://docs.aws.amazon.com/greengrass/v2/developerguide/device-service-role.html) to add authorization - 2. See the [`aws.greengrass.SecretManager` documentation for more information.](https://docs.aws.amazon.com/greengrass/v2/developerguide/secret-manager-component.html) - 3. Your policy should include `secretsmanager:GetSecretValue` for the secret you just created: + 1. Follow [the Greengrass documentation](:https://docs.aws.amazon.com/greengrass/v2/developerguide/device-service-role.html) to add authorization + 2. See the [`aws.greengrass.SecretManager` documentation for more information.](https://docs.aws.amazon.com/greengrass/v2/developerguide/secret-manager-component.html) + 3. Your policy should include `secretsmanager:GetSecretValue` for the secret you just created: ``` { "Version": "2012-10-17", @@ -171,21 +175,21 @@ The `aws.greengrass.labs.database.InfluxDB` component supports the following con } ``` -5. Create the component by following [the Greengrass documentation](https://docs.aws.amazon.com/greengrass/v2/developerguide/develop-greengrass-components.html) - 1. Modify the `aws.greengrass.labs.database.InfluxDB` recipe at `recipe.yaml`. - 2. Replace the two occurrences of `'arn:aws:secretsmanager:::secret:'` with your created secret ARN. - 3. (Optional) Modify the mount path. The default used will be `/home/ggc_user/dashboard`. +5. Create the component: + 1. (Optional) Modify the `aws.greengrass.labs.database.InfluxDB` recipe at `recipe.yaml`. NOTE: if you would like to specify this configuration during deployment, you can also specify this configuration during a deployment (see Step 6). + 2. Replace the two occurrences of `'arn:aws:secretsmanager:region:account:secret:name'` with your created secret ARN, including in the `accessControl` policy. + 3. (Optional) Modify the mount path. The default used will be `/home/ggc_user/dashboard`. 1. When specifying a mount path, note that this mount path will be used to store sensitive data, including secrets and certs used for InfluxDB auth. You are responsible for securing this directory on your device. Ensure that `ggc_user:ggc_group` has read/write/execute access to this directory with the following command: `namei -m `. - 2. Use the [GDK CLI](https://docs.aws.amazon.com/greengrass/v2/developerguide/greengrass-development-kit-cli.html) to build the component to prepare for publishing. + 3. Use the [GDK CLI](https://docs.aws.amazon.com/greengrass/v2/developerguide/greengrass-development-kit-cli.html) to build the component to prepare for publishing. ``` gdk component build ``` - 4. Use the [GDK CLI](https://docs.aws.amazon.com/greengrass/v2/developerguide/greengrass-development-kit-cli.html) to create a private component. + 4. Use the [GDK CLI](https://docs.aws.amazon.com/greengrass/v2/developerguide/greengrass-development-kit-cli.html) to create a private component. ``` gdk component publish ``` 6. Create deployment via the AWS CLI or AWS Console, from [Greengrass documentation](https://docs.aws.amazon.com/greengrass/v2/developerguide/create-deployments.html). The following components should be configured in your deployment: - 1. `aws.greengrass.SecretManager`: + 1. `aws.greengrass.SecretManager`: ``` "cloudSecrets": [ { @@ -193,50 +197,109 @@ The `aws.greengrass.labs.database.InfluxDB` component supports the following con } ] ``` - -7. View the component logs at `/greengrass/v2/logs/aws.greengrass.labs.database.InfluxDB.log`. If correctly set up, you will see the message `InfluxDB has been successfully set up; now listening to token requests` and see logs from InfluxDB as it runs. - 1. If you would like to forward the port from a remote machine, ssh in with the following command to forward the port: - `ssh -L 8086:localhost:8086 ubuntu@` -8. Visit `https://localhost:8086` to view InfluxDB, and login with your username and password. - 1. If using self-signed certificates (the default), you will either need to add trust for these certificates, or possibly use your browser's incognito mode. -Please see the Troubleshooting section to resolve any issues you may encounter. + 2. If you would like to specify your mount path/Secret Arn/Access Control during deployment instead, ***you must first remove the entire accessControl section from the recipe.yaml file before you create the component***. Then, make sure to merge in the following configuration to your component configuration during deployment. + + Note that specifying a non-default mount path is optional, and omitting it will result in the component using `/home/ggc_user/dashboard` instead. + ``` + { + "InfluxDBMountPath": "" (Optional) + "SecretArn": "", + "accessControl": { + "aws.greengrass.SecretManager": { + "aws.greengrass.labs.database.InfluxDB:secrets:1": { + "operations": [ + "aws.greengrass#GetSecretValue" + ], + "policyDescription": "Allows access to the secret containing InfluxDB credentials.", + "resources": [ + "" + ] + } + }, + "aws.greengrass.ipc.pubsub": { + "aws.greengrass.labs.database.InfluxDB:pubsub:1": { + "operations": [ + "aws.greengrass#SubscribeToTopic" + ], + "policyDescription": "Allows access to subscribe to the token request topic.", + "resources": [ + "greengrass/influxdb/token/request" + ] + }, + "aws.greengrass.labs.database.InfluxDB:pubsub:2": { + "operations": [ + "aws.greengrass#PublishToTopic" + ], + "policyDescription": "Allows access to publish to the token response topic.", + "resources": [ + "greengrass/influxdb/token/response" + ] + } + } + } + } + ``` + +8. View the component logs at `/greengrass/v2/logs/aws.greengrass.labs.database.InfluxDB.log`. If correctly set up, you will see the message `InfluxDB has been successfully set up; now listening to token requests` and see logs from InfluxDB as it runs. + 1. If you would like to forward the port from a remote machine, ssh in with the following command to forward the port: + `ssh -L 8086:localhost:8086 ubuntu@` +9. Visit `https://localhost:8086` to view InfluxDB, and login with your username and password. + 1. If using self-signed certificates (the default), you will either need to add trust for these certificates, or possibly use your browser's incognito mode. + Please see the Troubleshooting section to resolve any issues you may encounter. ## Component Lifecycle Management * InfluxDB data will be persisted between container restarts and removals since it is mounted to the location of your choice on your host machine. * Upon start, by default the component will look for the following and create them if they are not present: - * The docker bridge network `greengrass-telemetry-bridge` - * The directory `{configuration:/InfluxDBMountPath}/influxdb2_certs` along with a `.cert` and `.key` file for HTTPS - * By default, this directory has file permissions set to `077` for maximum compatability. [You are responsible for securing file permission on your device](https://docs.aws.amazon.com/greengrass/v2/developerguide/encryption-at-rest.html), and we would recommend scoping these permissions down to fit your use case. - * The directories `{configuration:/InfluxDBMountPath}/influxdb2/data` to store InfluxDB data and `{configuration:/InfluxDBMountPath}/influxdb2/config` for the InfluxDB config. See more information [on the Dockerhub page](https://hub.docker.com/_/influxdb). These directories are mounted into the container. + * The docker bridge network `greengrass-telemetry-bridge` + * The directory `{configuration:/InfluxDBMountPath}/influxdb2_certs` along with a `.cert` and `.key` file for HTTPS + * By default, this directory has file permissions set to `077` for maximum compatability. [You are responsible for securing file permission on your device](https://docs.aws.amazon.com/greengrass/v2/developerguide/encryption-at-rest.html), and we would recommend scoping these permissions down to fit your use case. + * The directories `{configuration:/InfluxDBMountPath}/influxdb2/data` to store InfluxDB data and `{configuration:/InfluxDBMountPath}/influxdb2/config` for the InfluxDB config. See more information [on the Dockerhub page](https://hub.docker.com/_/influxdb). These directories are mounted into the container. ## InfluxDB Token Vending -* After initialization and setup, this component will set up a local pub/sub subscription over the Greengrass IPC to vend InfluxDB credentials and metadata to other components that would like to use it to connect to InfluxDB. - * For more information, see the [Greengrass documentation on local pub/sub](https://docs.aws.amazon.com/greengrass/v2/developerguide/ipc-publish-subscribe.html). -* By default, the request topic is `greengrass/influxdb/token/request`, but can be configured. This component will listen to requests on this topic with a message of `GetInfluxDBData` and respond on the response topic. +* After initialization and setup, this component will set up a local pub/sub subscription over the Greengrass IPC to vend InfluxDB credentials and metadata to other components that would like to use it to connect to InfluxDB. + * For more information, see the [Greengrass documentation on local pub/sub](https://docs.aws.amazon.com/greengrass/v2/developerguide/ipc-publish-subscribe.html). +* By default, the request topic is `greengrass/influxdb/token/request`, but can be configured. This component will listen to requests on this topic and respond on the response topic with the requested data. + * The contents of the JSON request should consist of one of the following: + * `{"action": "RetrieveToken", "accessLevel": "RO"}` + * Retrieve an InfluxDB read-only token along with all necessary metadata. + * `{"action": "RetrieveToken", "accessLevel": "RW"}` + * Retrieve an InfluxDB read/write token along with all necessary metadata. + * `{"action": "RetrieveToken", "accessLevel": "Admin"}` + * Retrieve an InfluxDB admin token along with all necessary metadata. * By default, the response topic is `/greengrass/influxdb/token/response`, but can be configurable. Responses sent on this topic will be in the following JSON format: - * ``` + * ``` { InfluxDBContainerName : , InfluxDBOrg : , InfluxDBBucket : , InfluxDBPort : , InfluxDBInterface: , - InfluxDBRWToken : , + InfluxDBToken : , InfluxDBServerProtocol , - InfluxDBSkipTLSVerify: + InfluxDBSkipTLSVerify: , + InfluxDBTokenAccessType: } ``` - * If you would like to view an example of usage, see [the `aws.greengrass.labs.telemetry.InfluxDBPublisher` component, which relays Greengrass system health telemetry to InfluxDB](https://github.com/awslabs/aws-greengrass-labs-telemetry-influxdbpublisher). + * If you would like to view an example of usage, see + * [the `aws.greengrass.labs.telemetry.InfluxDBPublisher` component, which retrives a RW token and relays Greengrass system health telemetry to InfluxDB](https://github.com/awslabs/aws-greengrass-labs-telemetry-influxdbpublisher) + * [the `aws.greengrass.labs.dashboard.InfluxDBGrafana` component, which retrieves a RO token and uses it to automatically connect Grafana with InfluxDB](https://github.com/awslabs/aws-greengrass-labs-dashboard-influxdb-grafana) + ## Sending Telemetry to InfluxDB * The [aws.greengrass.labs.telemetry.InfluxDBPublisher](https://github.com/awslabs/aws-greengrass-labs-telemetry-influxdbpublisher) component, when deployed will forward Greengrass System Telemetry to InfluxDB. - * See the [Gather system health telemetry data from AWS IoT Greengrass core devices](https://docs.aws.amazon.com/greengrass/v2/developerguide/telemetry.html) documentation page to learn more about system health telemetry - * Telemetry is retrieved from the [Nucleus Telemetry Emitter component plugin ](https://docs.aws.amazon.com/greengrass/v2/developerguide/nucleus-emitter-component.html) and relayed to InfluxDB. + * See the [Gather system health telemetry data from AWS IoT Greengrass core devices](https://docs.aws.amazon.com/greengrass/v2/developerguide/telemetry.html) documentation page to learn more about system health telemetry + * Telemetry is retrieved from the [Nucleus Telemetry Emitter component plugin ](https://docs.aws.amazon.com/greengrass/v2/developerguide/nucleus-emitter-component.html) and relayed to InfluxDB. * To send custom telemetry to InfluxDB, you will need to either use InfluxDB APIs outside of Greengrass, or simply send messages over local pub/sub from a Greengrass component to the topic `$local/greengrass/telemetry`. The [aws.greengrass.labs.telemetry.InfluxDBPublisher](https://github.com/awslabs/aws-greengrass-labs-telemetry-influxdbpublisher) component will automatically forward all telemetry to InfluxDB. - * [See the aws.greengrass.labs.telemetry.InfluxDBPublisher README](https://github.com/awslabs/aws-greengrass-labs-telemetry-influxdbpublisher/blob/main/README.md) for more information + * [See the aws.greengrass.labs.telemetry.InfluxDBPublisher README](https://github.com/awslabs/aws-greengrass-labs-telemetry-influxdbpublisher/blob/main/README.md) for more information ## InfluxDB Token creation -If you would like to connect InfluxDB to Grafana or another application with read-only access, you can do so with the following commands to create a separate read-only token that will restrict access. Add `--skip-verify` to these commands only if using self-signed certificates with HTTPS (the default configuration). +This component by default creates and vends the following: +* An InfluxDB admin token, with full access +* An InfluxDB token with RW access to the default bucket +* An InfluxDB token with RO access to the default bucket + + +If you would like to create other tokens to connect InfluxDB to Grafana or another application, you can do so with the following sample commands that create a separate read-only token that will restrict access. Add `--skip-verify` to these commands only if using self-signed certificates with HTTPS (the default configuration). Please see the [official InfluxDB token creation documentation](https://docs.influxdata.com/influxdb/cloud/security/tokens/create-token/) for more information. 1. Retrieve the InfluxDB admin token: ``` @@ -280,8 +343,8 @@ docker exec -it greengrass_InfluxDB influx auth create \ * This component by default generates self-signed certificates to use for TLS encryption. We would recommend you sign your certificates with a Certificate Authority as described in [the InfluxDB v2 documentation for TLS encryption](https://docs.influxdata.com/influxdb/v2.0/security/enable-tls/). * If `GenerateSelfSignedCert` is set to false in the component configuration while using HTTPS, the component will look for the following two files to use, which you can provide: - * `{configuration:/InfluxDBMountPath}/influxdb2_certs/influxdb.crt` - * `{configuration:/InfluxDBMountPath}/influxdb2_certs/influxdb.key` + * `{configuration:/InfluxDBMountPath}/influxdb2_certs/influxdb.crt` + * `{configuration:/InfluxDBMountPath}/influxdb2_certs/influxdb.key` * The HTTPS certificates generated by default will expire in 365 days. If they are removed and the component redeployed or regenerated, new certificates will be created. @@ -298,45 +361,45 @@ This project also uses but does not distribute the InfluxDBv2 Docker image from ## Troubleshooting -* - ``` - Could not import awsiot - ``` - Ensure that `ggc_user` can import this Python library by running first `sudo su` and then `su - ggc_user -c "python3 -c 'import awsiot'"` - -* - ``` - mkdir: Operation not permitted - ``` - Ensure that your mount path has sufficient permission to create and mount directories into the container. If necessary, you can use `RequiresPrivilege: true` in the component recipe's lifecycle to run as root, although this is not recommended. -* - ``` - Attempt 0: Waiting until InfluxDB reports a status of OK.... - Error: Get "https://greengrass_InfluxDB:8086/api/v2/setup": dial tcp 172.18.0.2:8086: connect: connection refused.. - ``` - Not necessarily an error - InfluxDB can take a little while to start up. If after repeated retries with the same message the component fails, it means that the container did not start up correctly and exited. View the Docker logs retrieved inside the component log to debug further. -* - ``` - aws.greengrass.labs.database.InfluxDB: stderr. Error: open /etc/ssl/greengrass/influxdb.crt: operation not permitted. OR - aws.greengrass.labs.database.InfluxDB: stderr. Error: open /etc/ssl/greengrass/influxdb.crt: no such file or directory. - ``` - HTTPS cert file permissions are not permissive enough to allow Docker to mount them/have them be accessible by InfluxDB. Please review your file permissions. - Logging in as `ggc_user` may be helpful for debugging: `su - ggc_user` or `sudo -u ggc_user -i` -* - ``` - ERROR: Max retries exceeded while waiting for InfluxDB to start. Dumping InfluxDB Docker logs and exiting.... - ``` - There was an issue starting the InfluxDB Docker container. Check the log dumps for more information - this is likely due to insufficient file permissions -* - ``` - aws.greengrass.labs.database.InfluxDB: stdout. ts=2021-11-11T19:55:59.847535Z lvl=info msg=Unauthorized log_id=0XkE00UW000 error="authorization not found" - ``` - If you are attempting to connect to InfluxDB or perform an operation, this error can occur due to insufficient token privileges. -* - ``` - warn msg="Flux query failed" logger=tsdb.influx_flux err="unauthorized: unauthorized access" - ``` - Check that the InfluxDB token used for your request is up to date and replace if necessary. +* + ``` + Could not import awsiot + ``` +Ensure that `ggc_user` can import this Python library by running first `sudo su` and then `su - ggc_user -c "python3 -c 'import awsiot'"` + +* + ``` + mkdir: Operation not permitted + ``` +Ensure that your mount path has sufficient permission to create and mount directories into the container. If necessary, you can use `RequiresPrivilege: true` in the component recipe's lifecycle to run as root, although this is not recommended. +* + ``` + Attempt 0: Waiting until InfluxDB reports a status of OK.... + Error: Get "https://greengrass_InfluxDB:8086/api/v2/setup": dial tcp 172.18.0.2:8086: connect: connection refused.. + ``` +Not necessarily an error - InfluxDB can take a little while to start up. If after repeated retries with the same message the component fails, it means that the container did not start up correctly and exited. View the Docker logs retrieved inside the component log to debug further. +* + ``` + aws.greengrass.labs.database.InfluxDB: stderr. Error: open /etc/ssl/greengrass/influxdb.crt: operation not permitted. OR + aws.greengrass.labs.database.InfluxDB: stderr. Error: open /etc/ssl/greengrass/influxdb.crt: no such file or directory. + ``` +HTTPS cert file permissions are not permissive enough to allow Docker to mount them/have them be accessible by InfluxDB. Please review your file permissions. +Logging in as `ggc_user` may be helpful for debugging: `su - ggc_user` or `sudo -u ggc_user -i` +* + ``` + ERROR: Max retries exceeded while waiting for InfluxDB to start. Dumping InfluxDB Docker logs and exiting.... + ``` +There was an issue starting the InfluxDB Docker container. Check the log dumps for more information - this is likely due to insufficient file permissions +* + ``` + aws.greengrass.labs.database.InfluxDB: stdout. ts=2021-11-11T19:55:59.847535Z lvl=info msg=Unauthorized log_id=0XkE00UW000 error="authorization not found" + ``` +If you are attempting to connect to InfluxDB or perform an operation, this error can occur due to insufficient token privileges. +* + ``` + warn msg="Flux query failed" logger=tsdb.influx_flux err="unauthorized: unauthorized access" + ``` +Check that the InfluxDB token used for your request is up to date and replace if necessary. diff --git a/gdk-config.json b/gdk-config.json index 96383b5..076b5d7 100644 --- a/gdk-config.json +++ b/gdk-config.json @@ -2,7 +2,7 @@ "component" :{ "aws.greengrass.labs.database.InfluxDB": { "author": "AWS IoT Greengrass", - "version": "NEXT_PATCH", + "version": "2.0.0", "build": { "build_system": "zip" }, diff --git a/recipe.yaml b/recipe.yaml index 03d73e3..50ad018 100644 --- a/recipe.yaml +++ b/recipe.yaml @@ -1,7 +1,7 @@ --- RecipeFormatVersion: '2020-01-25' ComponentName: aws.greengrass.labs.database.InfluxDB -ComponentVersion: '1.0.0' +ComponentVersion: '2.0.0' ComponentDescription: 'A component that provisions and manages an InfluxDB instance.' ComponentPublisher: Amazon ComponentDependencies: @@ -15,7 +15,7 @@ ComponentConfiguration: DefaultConfiguration: AutoProvision: 'true' InfluxDBMountPath: '/home/ggc_user/dashboard' - SecretArn: 'arn:aws:secretsmanager:::secret:' + SecretArn: 'arn:aws:secretsmanager:region:account:secret:name' InfluxDBContainerName: greengrass_InfluxDB InfluxDBOrg: 'greengrass' InfluxDBBucket: 'greengrass-telemetry' @@ -48,7 +48,7 @@ ComponentConfiguration: operations: - aws.greengrass#GetSecretValue resources: - - 'arn:aws:secretsmanager:::secret:' + - 'arn:aws:secretsmanager:region:account:secret:name' Manifests: - Platform: os: /darwin|linux/ diff --git a/src/influxDBTokenPublisher.py b/src/influxDBTokenPublisher.py index d8437cd..551277d 100644 --- a/src/influxDBTokenPublisher.py +++ b/src/influxDBTokenPublisher.py @@ -19,6 +19,9 @@ logging.basicConfig(level=logging.INFO) TIMEOUT = 10 +# Influx commands need to be given the port of InfluxDB inside the container, which is always 8086 unless +# overridden inside the InfluxDB config +INFLUX_CONTAINER_PORT = 8086 def parse_arguments() -> Namespace: @@ -47,7 +50,7 @@ def parse_arguments() -> Namespace: return parser.parse_args() -def retrieve_influxDB_token(args) -> str: +def retrieve_influxDB_token_json(args) -> str: """ Retrieve the created RW token from InfluxDB. @@ -61,11 +64,10 @@ def retrieve_influxDB_token(args) -> str: """ token_json = "" - dockerExecProcess = "" authListCommand = ['docker', 'exec', '-t', args.influxdb_container_name, 'influx', 'auth', 'list', '--json'] if args.server_protocol == "https": authListCommand.append('--host') - authListCommand.append('https://{}:{}'.format(args.influxdb_container_name, args.influxdb_port)) + authListCommand.append('https://{}:{}'.format(args.influxdb_container_name, INFLUX_CONTAINER_PORT)) if bool(strtobool(args.skip_tls_verify)): authListCommand.append('--skip-verify') @@ -78,24 +80,24 @@ def retrieve_influxDB_token(args) -> str: if dockerExecProcess.stderr: logging.error(dockerExecProcess.stderr) if(len(token_json) == 0): - logging.error('Failed to retrieve InfluxDB RW token data from Docker! Retrieved token was: {}'.format(token_json)) + logging.error('Failed to retrieve InfluxDB RW token data from Docker! Retrieved data was: {}'.format(token_json)) exit(1) - influxdb_rw_token = next(d for d in json.loads(token_json) if d['description'] == 'greengrass_readwrite')['token'] - if(len(influxdb_rw_token) == 0): - logging.error('Failed to parse InfluxDB RW token! Retrieved token was: {}'.format(influxdb_rw_token)) + influxdb_token = json.loads(token_json)[0]['token'] + if(len(influxdb_token) == 0): + logging.error('Retrieved InfluxDB tokens was empty!') exit(1) - return influxdb_rw_token + return token_json -def listen_to_token_requests(args, influxdb_rw_token) -> None: +def listen_to_token_requests(args, influxdb_token_json) -> None: """ Setup a new IPC subscription over local pub/sub to listen to token requests and vend tokens. Parameters ---------- args(Namespace): Parsed arguments - influxdb_rw_token(str): InfluxDB RW token + influxdb_token_json(str): InfluxDB token JSON string Returns ------- @@ -103,23 +105,22 @@ def listen_to_token_requests(args, influxdb_rw_token) -> None: """ try: - influxDB_data = {} - influxDB_data['InfluxDBContainerName'] = args.influxdb_container_name - influxDB_data['InfluxDBOrg'] = args.influxdb_org - influxDB_data['InfluxDBBucket'] = args.influxdb_bucket - influxDB_data['InfluxDBPort'] = args.influxdb_port - influxDB_data['InfluxDBInterface'] = args.influxdb_interface - influxDB_data['InfluxDBRWToken'] = influxdb_rw_token - influxDB_data['InfluxDBServerProtocol'] = args.server_protocol - influxDB_data['InfluxDBSkipTLSVerify'] = args.skip_tls_verify - influxDB_json = json.dumps(influxDB_data) + influxdb_metadata = {} + influxdb_metadata['InfluxDBContainerName'] = args.influxdb_container_name + influxdb_metadata['InfluxDBOrg'] = args.influxdb_org + influxdb_metadata['InfluxDBBucket'] = args.influxdb_bucket + influxdb_metadata['InfluxDBPort'] = args.influxdb_port + influxdb_metadata['InfluxDBInterface'] = args.influxdb_interface + influxdb_metadata['InfluxDBServerProtocol'] = args.server_protocol + influxdb_metadata['InfluxDBSkipTLSVerify'] = args.skip_tls_verify + influxdb_metadata_json = json.dumps(influxdb_metadata) logging.info('Successfully retrieved InfluxDB parameters!') ipc_client = awsiot.greengrasscoreipc.connect() request = SubscribeToTopicRequest() request.topic = args.subscribe_topic - handler = InfluxDBTokenStreamHandler(influxDB_json, args.publish_topic) + handler = InfluxDBTokenStreamHandler(influxdb_metadata_json, influxdb_token_json, args.publish_topic) operation = ipc_client.new_subscribe_to_topic(handler) operation.activate(request) logging.info('Successfully subscribed to topic: {}'.format(args.subscribe_topic)) @@ -138,8 +139,8 @@ def listen_to_token_requests(args, influxdb_rw_token) -> None: if __name__ == "__main__": try: args = parse_arguments() - influxdb_rw_token = retrieve_influxDB_token(args) - listen_to_token_requests(args, influxdb_rw_token) + influxdb_token_json = retrieve_influxDB_token_json(args) + listen_to_token_requests(args, influxdb_token_json) # Keep the main thread alive, or the process will exit. while True: time.sleep(10) diff --git a/src/influxDBTokenStreamHandler.py b/src/influxDBTokenStreamHandler.py index 8205d13..d574ffd 100644 --- a/src/influxDBTokenStreamHandler.py +++ b/src/influxDBTokenStreamHandler.py @@ -3,25 +3,28 @@ import concurrent.futures import logging - +import json import awsiot.greengrasscoreipc import awsiot.greengrasscoreipc.client as client from awsiot.greengrasscoreipc.model import ( PublishToTopicRequest, PublishMessage, - BinaryMessage, + JsonMessage, SubscriptionResponseMessage, UnauthorizedError ) TIMEOUT = 10 +# Admin token description is in the format "USERNAME's Token" +ADMIN_TOKEN_IDENTIFIER = "'s Token" class InfluxDBTokenStreamHandler(client.SubscribeToTopicStreamHandler): - def __init__(self, influxDB_json, publish_topic): + def __init__(self, influxdb_metadata_json, influxdb_token_json, publish_topic): super().__init__() # We need a separate IPC client for publishing - self.influxDB_json = influxDB_json + self.influxDB_metadata_json = influxdb_metadata_json + self.influxDB_token_json = influxdb_token_json self.publish_topic = publish_topic self.publish_client = awsiot.greengrasscoreipc.connect() logging.info("Initialized InfluxDBTokenStreamHandler") @@ -39,12 +42,12 @@ def handle_stream_event(self, event: SubscriptionResponseMessage) -> None: None """ try: - message = str(event.binary_message.message, "utf-8") - if message == 'GetInfluxDBData': - logging.info('Sending InfluxDB RW Token on the response topic') - self.publish_response() - else: - logging.warning('Unknown request type received over pub/sub') + message = event.json_message.message + publish_json = self.get_publish_json(message) + if not publish_json: + logging.error("Failed to construct requested response for access") + return + self.publish_response(publish_json) except Exception: logging.error('Received an error', exc_info=True) @@ -80,13 +83,49 @@ def on_stream_closed(self) -> None: """ logging.info('Subscribe to topic stream closed.') - def publish_response(self) -> None: + def get_publish_json(self, message): + """ + Parse the correct token based on the IPC message received, and construct the final JSON to publish. + + :param message: the received IPC messsage + :return: the complete JSON, including token, to publish + """ + + loaded_token_json = json.loads(self.influxDB_token_json) + publish_json = json.loads(self.influxDB_metadata_json) + + if not message['action'] == 'RetrieveToken': + logging.warning('Unknown request type received over pub/sub') + return None + + token = '' + if message['accessLevel'] == 'RW': + token = next(d for d in loaded_token_json if d['description'] == 'greengrass_readwrite')['token'] + elif message['accessLevel'] == 'RO': + token = next(d for d in loaded_token_json if d['description'] == 'greengrass_read')['token'] + elif message['accessLevel'] == 'Admin': + if not ADMIN_TOKEN_IDENTIFIER in loaded_token_json[0]['description']: + logging.warning("InfluxDB admin token is missing or in an incorrect format") + return None + token = loaded_token_json[0]['token'] + else: + logging.warning('Unknown token request type specified over pub/sub') + return None + + if len(token) == 0: + raise ValueError('Failed to parse InfluxDB {} token!'.format(message['accessLevel'])) + publish_json['InfluxDBTokenAccessType'] = message['accessLevel'] + publish_json['InfluxDBToken'] = token + logging.info('Sending InfluxDB {} Token on the response topic'.format(message['accessLevel'])) + return publish_json + + def publish_response(self, publishMessage) -> None: """ Publish the InfluxDB token on the token response topic. Parameters ---------- - None + publishMessage(str): the message to send including InfluxDB metadata and token Returns ------- @@ -96,8 +135,8 @@ def publish_response(self) -> None: request = PublishToTopicRequest() request.topic = self.publish_topic publish_message = PublishMessage() - publish_message.binary_message = BinaryMessage() - publish_message.binary_message.message = bytes(self.influxDB_json, "utf-8") + publish_message.json_message = JsonMessage() + publish_message.json_message.message = publishMessage request.publish_message = publish_message operation = self.publish_client.new_publish_to_topic() operation.activate(request) diff --git a/src/influxdb_utils.sh b/src/influxdb_utils.sh index 44c5d0d..355c715 100644 --- a/src/influxdb_utils.sh +++ b/src/influxdb_utils.sh @@ -6,24 +6,31 @@ wait_for_influxdb_start(){ # InfluxDB can take some time to start # Retry `influx ping` until we receive confirmation that it is up and running + # Influx commands need to be given the port of InfluxDB inside the container, which is always 8086 unless overridden inside the InfluxDB config CONTAINER_NAME=$1 INFLUXDB_PORT=$2 SERVER_PROTOCOL=$3 + SKIP_TLS_VERIFY=$4 - if [[ -z $CONTAINER_NAME || -z $INFLUXDB_PORT || -z $SERVER_PROTOCOL ]]; then - echo 'Container name, InfluxDB port, or server protocol was not provided when waiting for InfluxDB to start!' + if [[ -z $CONTAINER_NAME || -z $INFLUXDB_PORT || -z $SERVER_PROTOCOL || -z $SKIP_TLS_VERIFY ]]; then + echo 'Container name, InfluxDB port, server protocol, or skip TLS verify was not provided when waiting for InfluxDB to start!' exit 1 fi + SKIP_TLS_VERIFY_ARG="" + if [ "$SKIP_TLS_VERIFY" == "true" ]; then + SKIP_TLS_VERIFY_ARG="--skip-verify" + fi + CONTAINER_SETUP_STATUS="" RETRIES=0 until [ "$CONTAINER_SETUP_STATUS" == "OK" ] || [ "$RETRIES" -eq 4 ]; do sleep 10 echo "Attempt $RETRIES: Waiting until InfluxDB reports a status of OK..." - if [ $SERVER_PROTOCOL == "http" ]; then - CONTAINER_SETUP_STATUS=$(docker exec $CONTAINER_NAME influx ping) || true - elif [ $SERVER_PROTOCOL == "https" ]; then - CONTAINER_SETUP_STATUS=$(docker exec $CONTAINER_NAME influx ping --host "https://$CONTAINER_NAME:$INFLUXDB_PORT" --skip-verify) || true + if [ "$SERVER_PROTOCOL" == "http" ]; then + CONTAINER_SETUP_STATUS=$(docker exec "$CONTAINER_NAME" influx ping --host "http://$CONTAINER_NAME:8086") || true + elif [ "$SERVER_PROTOCOL" == "https" ]; then + CONTAINER_SETUP_STATUS=$(docker exec "$CONTAINER_NAME" influx ping --host "https://$CONTAINER_NAME:8086" "${SKIP_TLS_VERIFY_ARG:+$SKIP_TLS_VERIFY_ARG}") || true fi echo "Container status: $CONTAINER_SETUP_STATUS" RETRIES="$((RETRIES+1))" @@ -32,35 +39,64 @@ wait_for_influxdb_start(){ if [ "$CONTAINER_SETUP_STATUS" != "OK" ]; then echo "ERROR: Max retries exceeded while waiting for InfluxDB to start. Dumping InfluxDB Docker logs and exiting..." # Dump Docker logs before the container is removed - docker logs $CONTAINER_NAME + docker logs "$CONTAINER_NAME" exit 1 fi echo "Successfully waited for InfluxDB to start up!" } -create_readwrite_token(){ +create_token(){ CONTAINER_NAME=$1 INFLUXDB_PORT=$2 BUCKET_NAME=$3 ORG_NAME=$4 SERVER_PROTOCOL=$5 + SKIP_TLS_VERIFY=$6 + ACCESS=$7 - if [[ -z $CONTAINER_NAME || -z $BUCKET_NAME || -z $ORG_NAME || -z $SERVER_PROTOCOL ]]; then - echo 'Missing one or more arguments when trying to create the RW token!' + if [[ -z $CONTAINER_NAME || -z $BUCKET_NAME || -z $ORG_NAME || -z $SERVER_PROTOCOL || -z $SKIP_TLS_VERIFY || -z $ACCESS ]]; then + echo 'Missing one or more arguments when trying to create the token!' exit 1 fi + SKIP_TLS_VERIFY_ARG="" + if [ "$SKIP_TLS_VERIFY" == "true" ]; then + SKIP_TLS_VERIFY_ARG="--skip-verify" + fi + BUCKET_ID="" - if [ $SERVER_PROTOCOL == "http" ]; then - BUCKET_ID=$(docker exec -t $CONTAINER_NAME influx bucket list --json --name $BUCKET_NAME | python3 -c "import sys, json; print(json.load(sys.stdin)[0]['id'])") + if [ "$SERVER_PROTOCOL" == "http" ]; then + BUCKET_ID=$(docker exec -t "$CONTAINER_NAME" influx bucket list --json --name "$BUCKET_NAME" --host "http://$CONTAINER_NAME:8086" | python3 -c "import sys, json; print(json.load(sys.stdin)[0]['id'])") echo "Retrieved bucket ID: $BUCKET_ID" - INFLUXDB_RW_TOKEN_METADATA=$(docker exec -t $CONTAINER_NAME influx auth create --read-bucket $BUCKET_ID --write-bucket $BUCKET_ID --org $ORG_NAME --description "greengrass_readwrite" --hide-headers) - elif [ $SERVER_PROTOCOL == "https" ]; then - BUCKET_ID=$(docker exec -t $CONTAINER_NAME influx bucket list --json --name $BUCKET_NAME --host "https://$CONTAINER_NAME:$INFLUXDB_PORT" --skip-verify | python3 -c "import sys, json; print(json.load(sys.stdin)[0]['id'])") + elif [ "$SERVER_PROTOCOL" == "https" ]; then + BUCKET_ID=$(docker exec -t "$CONTAINER_NAME" influx bucket list --json --name "$BUCKET_NAME" --host "https://$CONTAINER_NAME:8086" "${SKIP_TLS_VERIFY_ARG:+$SKIP_TLS_VERIFY_ARG}" | python3 -c "import sys, json; print(json.load(sys.stdin)[0]['id'])") echo "Retrieved bucket ID: $BUCKET_ID" - INFLUXDB_RW_TOKEN_METADATA=$(docker exec -t $CONTAINER_NAME influx auth create --host "https://$CONTAINER_NAME:$INFLUXDB_PORT" --skip-verify --read-bucket $BUCKET_ID --write-bucket $BUCKET_ID --org $ORG_NAME --description "greengrass_readwrite" --hide-headers) fi + + ACCESS_POLICY_ARGS=() + DESCRIPTION="" + if [ "$ACCESS" == "readonly" ]; then + ACCESS_POLICY_ARGS=("--read-bucket" "${BUCKET_ID}") + DESCRIPTION="greengrass_read" + elif [ "$ACCESS" == "readwrite" ]; then + ACCESS_POLICY_ARGS=("--read-bucket" "${BUCKET_ID}" "--write-bucket" "${BUCKET_ID}") + DESCRIPTION="greengrass_readwrite" + fi + + INFLUXDB_RW_TOKEN_METADATA="" + if [ "$SERVER_PROTOCOL" == "http" ]; then + INFLUXDB_RW_TOKEN_METADATA=$(docker exec -t "$CONTAINER_NAME" influx auth create --host "http://$CONTAINER_NAME:8086" "${ACCESS_POLICY_ARGS[@]}" --org "$ORG_NAME" --description "$DESCRIPTION" --hide-headers) + elif [ "$SERVER_PROTOCOL" == "https" ]; then + INFLUXDB_RW_TOKEN_METADATA=$(docker exec -t "$CONTAINER_NAME" influx auth create --host "https://$CONTAINER_NAME:8086" "${SKIP_TLS_VERIFY_ARG:+$SKIP_TLS_VERIFY_ARG}" "${ACCESS_POLICY_ARGS[@]}" --org "$ORG_NAME" --description "$DESCRIPTION" --hide-headers) + fi + + if [ -z "$INFLUXDB_RW_TOKEN_METADATA" ]; then + echo "Failed to create InfluxDB Token $ACCESS" + exit 1 + fi + + echo "Successfully created InfluxDB token ${DESCRIPTION}" } validate_password(){ @@ -88,12 +124,12 @@ setup_blank_influxdb_with_http() { echo "Setting up a blank InfluxDB instance with HTTP..." docker run -d \ - -p $INFLUXDB_INTERFACE:$INFLUXDB_PORT:8086 \ - --network=$BRIDGE_NETWORK_NAME \ - --name $CONTAINER_NAME \ + -p "$INFLUXDB_INTERFACE":"$INFLUXDB_PORT":8086 \ + --network="$BRIDGE_NETWORK_NAME" \ + --name "$CONTAINER_NAME" \ --read-only \ - -v $INFLUXDB_MOUNT_PATH/influxdb2/data:/var/lib/influxdb2 \ - -v $INFLUXDB_MOUNT_PATH/influxdb2/config:/etc/influxdb2 \ + -v "$INFLUXDB_MOUNT_PATH"/influxdb2/data:/var/lib/influxdb2 \ + -v "$INFLUXDB_MOUNT_PATH"/influxdb2/config:/etc/influxdb2 \ influxdb:2.0.9 } @@ -108,6 +144,7 @@ provision_influxdb(){ BRIDGE_NETWORK_NAME=$8 INFLUXDB_MOUNT_PATH=$9 INFLUXDB_INTERFACE=${10} + SKIP_TLS_VERIFY=${11} if [[ -z $CONTAINER_NAME \ || -z $BUCKET_NAME \ @@ -118,56 +155,65 @@ provision_influxdb(){ || -z $SERVER_PROTOCOL \ || -z $BRIDGE_NETWORK_NAME \ || -z $INFLUXDB_MOUNT_PATH \ - || -z $INFLUXDB_INTERFACE ]]; then + || -z $INFLUXDB_INTERFACE + || -z $SKIP_TLS_VERIFY ]]; then echo 'Missing one or more arguments when trying to provision InfluxDB!' exit 1 fi - if [ $SERVER_PROTOCOL == "https" ]; then + if [ "$SERVER_PROTOCOL" == "https" ]; then echo "Setting up a blank InfluxDB instance with HTTPS..." docker run -d \ - -p $INFLUXDB_INTERFACE:$INFLUXDB_PORT:8086 \ - --network=$BRIDGE_NETWORK_NAME \ - --name $CONTAINER_NAME \ + -p "$INFLUXDB_INTERFACE":"$INFLUXDB_PORT":8086 \ + --network="$BRIDGE_NETWORK_NAME" \ + --name "$CONTAINER_NAME" \ --read-only \ - -v $INFLUXDB_MOUNT_PATH/influxdb2/data:/var/lib/influxdb2 \ - -v $INFLUXDB_MOUNT_PATH/influxdb2/config:/etc/influxdb2 \ - -v $INFLUXDB_MOUNT_PATH/influxdb2_certs/:/etc/ssl/greengrass:ro \ + -v "$INFLUXDB_MOUNT_PATH"/influxdb2/data:/var/lib/influxdb2 \ + -v "$INFLUXDB_MOUNT_PATH"/influxdb2/config:/etc/influxdb2 \ + -v "$INFLUXDB_MOUNT_PATH"/influxdb2_certs/:/etc/ssl/greengrass:ro \ -e INFLUXD_TLS_CERT=/etc/ssl/greengrass/influxdb.crt \ -e INFLUXD_TLS_KEY=/etc/ssl/greengrass/influxdb.key \ influxdb:2.0.9 - wait_for_influxdb_start $CONTAINER_NAME $INFLUXDB_PORT $SERVER_PROTOCOL + wait_for_influxdb_start "$CONTAINER_NAME" "$INFLUXDB_PORT" "$SERVER_PROTOCOL" "$SKIP_TLS_VERIFY" else - setup_blank_influxdb_with_http $CONTAINER_NAME $INFLUXDB_PORT $BRIDGE_NETWORK_NAME $INFLUXDB_MOUNT_PATH $INFLUXDB_INTERFACE - wait_for_influxdb_start $CONTAINER_NAME $INFLUXDB_PORT $SERVER_PROTOCOL + setup_blank_influxdb_with_http "$CONTAINER_NAME" "$INFLUXDB_PORT" "$BRIDGE_NETWORK_NAME" "$INFLUXDB_MOUNT_PATH" "$INFLUXDB_INTERFACE" + wait_for_influxdb_start "$CONTAINER_NAME" "$INFLUXDB_PORT" "$SERVER_PROTOCOL" "$SKIP_TLS_VERIFY" + fi + + SKIP_TLS_VERIFY_ARG="" + if [ "$SKIP_TLS_VERIFY" == "true" ]; then + SKIP_TLS_VERIFY_ARG="--skip-verify" fi # Check if auth tokens already exists SETUP_EXIT_CODE=0 - + echo "Checking if auth tokens already exist..." if [ $SERVER_PROTOCOL == "http" ]; then - docker exec $CONTAINER_NAME influx auth list > /dev/null 2>&1 || SETUP_EXIT_CODE=$? + docker exec $CONTAINER_NAME influx auth list --host "http://$CONTAINER_NAME:8086" > /dev/null 2>&1 || SETUP_EXIT_CODE=$? elif [ $SERVER_PROTOCOL == "https" ]; then - docker exec $CONTAINER_NAME influx auth list --host "https://$CONTAINER_NAME:$INFLUXDB_PORT" --skip-verify > /dev/null 2>&1 || SETUP_EXIT_CODE=$? + docker exec $CONTAINER_NAME influx auth list --host "https://$CONTAINER_NAME:8086" "${SKIP_TLS_VERIFY_ARG:+$SKIP_TLS_VERIFY_ARG}" > /dev/null 2>&1 || SETUP_EXIT_CODE=$? fi if [ "$SETUP_EXIT_CODE" -eq 1 ]; then # Setup auth echo "Setting up InfluxDB with provided credentials..." - INFLUXDB_CREDENTIALS=$(python3 $ARTIFACT_PATH/retrieveInfluxDBSecrets.py --secret_arn $SECRET_ARN ) + INFLUXDB_CREDENTIALS=$(python3 "$ARTIFACT_PATH"/retrieveInfluxDBSecrets.py --secret_arn "$SECRET_ARN" ) INFLUX_CREDENTIALS_ARRAY=($INFLUXDB_CREDENTIALS) INFLUXDB_USERNAME=${INFLUX_CREDENTIALS_ARRAY[0]} INFLUXDB_PASSWORD=${INFLUX_CREDENTIALS_ARRAY[1]} echo "Validating password..." - validate_password $INFLUXDB_PASSWORD + validate_password "$INFLUXDB_PASSWORD" + if [ $SERVER_PROTOCOL == "http" ]; then - docker exec -t $CONTAINER_NAME influx setup --host "http://$CONTAINER_NAME:$INFLUXDB_PORT" --force --username $INFLUXDB_USERNAME --password $INFLUXDB_PASSWORD --org $ORG_NAME --bucket $BUCKET_NAME + docker exec -t $CONTAINER_NAME influx setup --host "http://$CONTAINER_NAME:8086" --force --username $INFLUXDB_USERNAME --password $INFLUXDB_PASSWORD --org $ORG_NAME --bucket $BUCKET_NAME elif [ $SERVER_PROTOCOL == "https" ]; then - docker exec -t $CONTAINER_NAME influx setup --host "https://$CONTAINER_NAME:$INFLUXDB_PORT" --skip-verify --force --username $INFLUXDB_USERNAME --password $INFLUXDB_PASSWORD --org $ORG_NAME --bucket $BUCKET_NAME + docker exec -t $CONTAINER_NAME influx setup --host "https://$CONTAINER_NAME:8086" "${SKIP_TLS_VERIFY_ARG:+$SKIP_TLS_VERIFY_ARG}" --force --username $INFLUXDB_USERNAME --password $INFLUXDB_PASSWORD --org $ORG_NAME --bucket $BUCKET_NAME fi - create_readwrite_token $CONTAINER_NAME $INFLUXDB_PORT $BUCKET_NAME $ORG_NAME $SERVER_PROTOCOL + + create_token "$CONTAINER_NAME" "$INFLUXDB_PORT" "$BUCKET_NAME" "$ORG_NAME" "$SERVER_PROTOCOL" "$SKIP_TLS_VERIFY" "readonly" + create_token "$CONTAINER_NAME" "$INFLUXDB_PORT" "$BUCKET_NAME" "$ORG_NAME" "$SERVER_PROTOCOL" "$SKIP_TLS_VERIFY" "readwrite" else # Reuse auth echo "Reusing existing InfluxDB setup..." diff --git a/src/retrieveInfluxDBSecrets.py b/src/retrieveInfluxDBSecrets.py index 2c2ba95..1051fba 100644 --- a/src/retrieveInfluxDBSecrets.py +++ b/src/retrieveInfluxDBSecrets.py @@ -50,21 +50,17 @@ def get_secret_over_ipc(secret_arn) -> str: operation = ipc_client.new_get_secret_value() operation.activate(request) futureResponse = operation.get_response() - try: - response = futureResponse.result(TIMEOUT) - return response.secret_value.secret_string - except concurrent.futures.TimeoutError as e: - logging.error("Timeout occurred while getting secret: {}".format(secret_arn), exc_info=True) - raise e - except UnauthorizedError as e: - logging.error("Unauthorized error while getting secret: {}".format(secret_arn), exc_info=True) - raise e - except Exception as e: - logging.error("Exception while getting secret: {}".format(secret_arn), exc_info=True) - raise e - except Exception: - logging.error("Exception occurred when using IPC.", exc_info=True) - exit(1) + response = futureResponse.result(TIMEOUT) + return response.secret_value.secret_string + except concurrent.futures.TimeoutError as e: + logging.error("Timeout occurred while getting secret: {}".format(secret_arn), exc_info=True) + raise e + except UnauthorizedError as e: + logging.error("Unauthorized error while getting secret: {}".format(secret_arn), exc_info=True) + raise e + except Exception as e: + logging.error("Exception while getting secret: {}".format(secret_arn), exc_info=True) + raise e def retrieve_secret(secret_arn): diff --git a/src/run_influxdb.sh b/src/run_influxdb.sh index 1e03be3..3e5ab3d 100644 --- a/src/run_influxdb.sh +++ b/src/run_influxdb.sh @@ -44,8 +44,8 @@ fi # If auto-provisioning, provision the container and begin vending the token child_pid="" if [ "$AUTO_PROVISION" == "true" ]; then - echo "Auto-provisioning InfluxDB with provided secrets..." - provision_influxdb $CONTAINER_NAME $BUCKET_NAME $ORG_NAME $ARTIFACT_PATH $SECRET_ARN $INFLUXDB_PORT $SERVER_PROTOCOL $BRIDGE_NETWORK_NAME $INFLUXDB_MOUNT_PATH $INFLUXDB_INTERFACE + echo "Using InfluxDB in auto-provisioning mode..." + provision_influxdb $CONTAINER_NAME $BUCKET_NAME $ORG_NAME $ARTIFACT_PATH $SECRET_ARN $INFLUXDB_PORT $SERVER_PROTOCOL $BRIDGE_NETWORK_NAME $INFLUXDB_MOUNT_PATH $INFLUXDB_INTERFACE $SKIP_TLS_VERIFY python3 -u "$ARTIFACT_PATH/influxDBTokenPublisher.py" \ --subscribe_topic $TOKEN_REQUEST_TOPIC \ @@ -62,7 +62,7 @@ if [ "$AUTO_PROVISION" == "true" ]; then else echo "Auto-provisioning is disabled, skippping..." setup_blank_influxdb_with_http $CONTAINER_NAME $INFLUXDB_PORT $BRIDGE_NETWORK_NAME $INFLUXDB_MOUNT_PATH $INFLUXDB_INTERFACE - wait_for_influxdb_start $CONTAINER_NAME $INFLUXDB_PORT $SERVER_PROTOCOL + wait_for_influxdb_start $CONTAINER_NAME $INFLUXDB_PORT $SERVER_PROTOCOL $SKIP_TLS_VERIFY fi echo "InfluxDB is running..." diff --git a/test/test_influxDBTokenPublisher.py b/test/test_influxDBTokenPublisher.py index 9864ade..7725c50 100644 --- a/test/test_influxDBTokenPublisher.py +++ b/test/test_influxDBTokenPublisher.py @@ -79,8 +79,9 @@ def test_retrieve_secret_valid_response(mocker): import src.influxDBTokenPublisher as publisher - influxdb_rw_token = publisher.retrieve_influxDB_token(testArgs) - assert influxdb_rw_token == "testToken" + json_output = json.loads(publisher.retrieve_influxDB_token_json(testArgs))[0] + assert json_output['description'] == "greengrass_readwrite" + assert json_output['token'] == "testToken" assert mock_subprocess_call.call_count == 1 @@ -113,8 +114,8 @@ def test_retrieve_secret_invalid_response(mocker): import src.influxDBTokenPublisher as publisher with pytest.raises(SystemExit) as pytest_wrapped_e: - publisher.retrieve_influxDB_token(testArgs) - assert pytest_wrapped_e.type == SystemExit + publisher.retrieve_influxDB_token_json(testArgs) + assert pytest_wrapped_e.type == SystemExit def test_retrieve_secret_failed_response(mocker): @@ -137,7 +138,7 @@ def test_retrieve_secret_failed_response(mocker): import src.influxDBTokenPublisher as publisher with pytest.raises(SystemExit) as pytest_wrapped_e: - publisher.retrieve_influxDB_token(testArgs) + publisher.retrieve_influxDB_token_json(testArgs) assert pytest_wrapped_e.type == SystemExit @@ -159,3 +160,25 @@ def test_listen_to_token_requests(mocker): import src.influxDBTokenPublisher as publisher publisher.listen_to_token_requests(testArgs, test_influxdb_rw_token) assert mock_ipc_client.call_count == 2 + + +def test_no_ipc_connection(mocker): + + testArgs = argparse.Namespace( + subscribe_topic="test/subscribe", + publish_topic="test/publish", + influxdb_container_name="test_containername", + influxdb_org="testorg", + influxdb_bucket="testbucket", + influxdb_port="testport", + influxdb_interface="testinterface", + server_protocol="https", + skip_tls_verify="true" + ) + test_influxdb_rw_token = "testToken" + mocker.patch("awsiot.greengrasscoreipc.connect", side_effect=TimeoutError("test")) + + import src.influxDBTokenPublisher as publisher + + with pytest.raises(TimeoutError, match='test'): + publisher.listen_to_token_requests(testArgs, test_influxdb_rw_token) diff --git a/test/test_influxDBTokenStreamHandler.py b/test/test_influxDBTokenStreamHandler.py index 4951fe4..b50b06f 100644 --- a/test/test_influxDBTokenStreamHandler.py +++ b/test/test_influxDBTokenStreamHandler.py @@ -3,14 +3,67 @@ import sys import json +import pytest from awsiot.greengrasscoreipc.model import ( - BinaryMessage, + JsonMessage, SubscriptionResponseMessage ) sys.path.append("src/") +testTokenJson = [ + { + "id": "0895c16b9de9e000", + "description": "test's Token", + "token": "testAdminToken", + "status": "active", + "userName": "test", + "userID": "0895c16b80a9e000", + "permissions": [ + "read:authorizations", + "write:authorizations" + ] + }, + { + "id": "0895c16bfba9e000", + "description": "greengrass_read", + "token": "testROToken", + "status": "active", + "userName": "test", + "userID": "0895c16b80a9e000", + "permissions": [ + "read:orgs/d13dcc4c7cd25bf9/buckets/2f1dc2bba2275383" + ] + }, + { + "id": "0895c16c8ee9e000", + "description": "greengrass_readwrite", + "token": "testRWToken", + "status": "active", + "userName": "test", + "userID": "0895c16b80a9e000", + "permissions": [ + "read:orgs/d13dcc4c7cd25bf9/buckets/2f1dc2bba2275383", + "write:orgs/d13dcc4c7cd25bf9/buckets/2f1dc2bba2275383" + ] + } +] + +testMetadataJson = { + 'InfluxDBContainerName': 'greengrass_InfluxDB', + 'InfluxDBOrg': 'greengrass', + 'InfluxDBBucket': 'greengrass-telemetry', + 'InfluxDBPort': '8086', + 'InfluxDBInterface': '127.0.0.1', + 'InfluxDBServerProtocol': 'https', + 'InfluxDBSkipTLSVerify': 'true', +} + +testPublishJson = testMetadataJson +testPublishJson['InfluxDBTokenAccessType'] = "RW" +testPublishJson['InfluxDBToken'] = "testRWToken" + def testHandleValidStreamEvent(mocker): mock_ipc_client = mocker.patch("awsiot.greengrasscoreipc.connect") @@ -18,10 +71,11 @@ def testHandleValidStreamEvent(mocker): import src.influxDBTokenStreamHandler as streamHandler - handler = streamHandler.InfluxDBTokenStreamHandler(json.dumps("{}"), "test") - binary_message = BinaryMessage(message=str.encode("GetInfluxDBData")) - response_message = SubscriptionResponseMessage(binary_message=binary_message) - handler.handle_stream_event(response_message) + handler = streamHandler.InfluxDBTokenStreamHandler(json.dumps(testMetadataJson), json.dumps(testTokenJson), "test/topic") + message = JsonMessage(message={"action": "RetrieveToken", "accessLevel": "RW"}) + response_message = SubscriptionResponseMessage(json_message=message) + t = handler.handle_stream_event(response_message) + mock_publish_response.assert_called_with(testPublishJson) assert mock_ipc_client.call_count == 1 assert mock_publish_response.call_count == 1 @@ -32,9 +86,37 @@ def testHandleInvalidStreamEvent(mocker): import src.influxDBTokenStreamHandler as streamHandler - handler = streamHandler.InfluxDBTokenStreamHandler(json.dumps("{}"), "test") - binary_message = BinaryMessage(message=str.encode("test")) - response_message = SubscriptionResponseMessage(binary_message=binary_message) + handler = streamHandler.InfluxDBTokenStreamHandler(json.dumps({}), json.dumps(testTokenJson), "test") + message = JsonMessage(message={}) + response_message = SubscriptionResponseMessage(json_message=message) + handler.handle_stream_event(response_message) + assert mock_ipc_client.call_count == 1 + assert not mock_publish_response.called + + +def testHandleInvalidRequestType(mocker): + mock_ipc_client = mocker.patch("awsiot.greengrasscoreipc.connect") + mock_publish_response = mocker.patch('src.influxDBTokenStreamHandler.InfluxDBTokenStreamHandler.publish_response') + + import src.influxDBTokenStreamHandler as streamHandler + + handler = streamHandler.InfluxDBTokenStreamHandler(json.dumps({}), json.dumps(testTokenJson), "test") + message = JsonMessage(message={"action": "invalid", "accessLevel": "RW"}) + response_message = SubscriptionResponseMessage(json_message=message) + handler.handle_stream_event(response_message) + assert mock_ipc_client.call_count == 1 + assert not mock_publish_response.called + + +def testHandleInvalidTokenRequestType(mocker): + mock_ipc_client = mocker.patch("awsiot.greengrasscoreipc.connect") + mock_publish_response = mocker.patch('src.influxDBTokenStreamHandler.InfluxDBTokenStreamHandler.publish_response') + + import src.influxDBTokenStreamHandler as streamHandler + + handler = streamHandler.InfluxDBTokenStreamHandler(json.dumps({}), json.dumps(testTokenJson), "test") + message = JsonMessage(message={"action": "RetrieveToken", "accessLevel": "invalid"}) + response_message = SubscriptionResponseMessage(json_message=message) handler.handle_stream_event(response_message) assert mock_ipc_client.call_count == 1 assert not mock_publish_response.called @@ -46,8 +128,62 @@ def testHandleNullStreamEvent(mocker): import src.influxDBTokenStreamHandler as streamHandler - handler = streamHandler.InfluxDBTokenStreamHandler(json.dumps("{}"), "test") + handler = streamHandler.InfluxDBTokenStreamHandler(json.dumps(testMetadataJson), json.dumps(testTokenJson), "test") response_message = None handler.handle_stream_event(response_message) assert mock_ipc_client.call_count == 1 assert not mock_publish_response.called + + +def testGetValidPublishJson(mocker): + + mocker.patch("awsiot.greengrasscoreipc.connect") + + import src.influxDBTokenStreamHandler as streamHandler + + handler = streamHandler.InfluxDBTokenStreamHandler(json.dumps(testMetadataJson), json.dumps(testTokenJson), "test/topic") + message = json.loads('{"action": "RetrieveToken", "accessLevel": "RW"}') + publish_json = handler.get_publish_json(message) + assert publish_json == testPublishJson + + message = json.loads('{"action": "RetrieveToken", "accessLevel": "RO"}') + publish_json = handler.get_publish_json(message) + testPublishJson['InfluxDBTokenAccessType'] = "RO" + testPublishJson['InfluxDBToken'] = "testROToken" + assert publish_json == testPublishJson + + message = json.loads('{"action": "RetrieveToken", "accessLevel": "Admin"}') + publish_json = handler.get_publish_json(message) + testPublishJson['InfluxDBTokenAccessType'] = "Admin" + testPublishJson['InfluxDBToken'] = "testAdminToken" + assert publish_json == testPublishJson + + +def testGetInvalidPublishJson(mocker): + + mocker.patch("awsiot.greengrasscoreipc.connect") + + import src.influxDBTokenStreamHandler as streamHandler + + testTokenJson[0]['token'] = "" + testTokenJson[1]['token'] = "" + testTokenJson[2]['token'] = "" + handler = streamHandler.InfluxDBTokenStreamHandler(json.dumps(testMetadataJson), json.dumps(testTokenJson), "test/topic") + + with pytest.raises(ValueError, match='Failed to parse InfluxDB RW token!'): + message = json.loads('{"action": "RetrieveToken", "accessLevel": "RW"}') + handler.get_publish_json(message) + + with pytest.raises(ValueError, match='Failed to parse InfluxDB RO token!'): + message = json.loads('{"action": "RetrieveToken", "accessLevel": "RO"}') + handler.get_publish_json(message) + + with pytest.raises(ValueError, match='Failed to parse InfluxDB Admin token!'): + message = json.loads('{"action": "RetrieveToken", "accessLevel": "Admin"}') + handler.get_publish_json(message) + + testTokenJson[0]['description'] = "" + handler = streamHandler.InfluxDBTokenStreamHandler(json.dumps(testMetadataJson), json.dumps(testTokenJson), "test/topic") + message = json.loads('{"action": "RetrieveToken", "accessLevel": "Admin"}') + retval = handler.get_publish_json(message) + assert retval is None diff --git a/test/test_retrieveInfluxDBSecrets.py b/test/test_retrieveInfluxDBSecrets.py index 116b321..01f66c1 100644 --- a/test/test_retrieveInfluxDBSecrets.py +++ b/test/test_retrieveInfluxDBSecrets.py @@ -5,6 +5,7 @@ import sys import pytest import json +from awsiot.greengrasscoreipc.model import UnauthorizedError sys.path.append("src/retrieveInfluxDBSecrets.py") @@ -63,3 +64,23 @@ def test_retrieve_secret_empty_response(mocker): ris.retrieve_secret("arn:test:object") assert mock_ipc_call.call_count == 1 mock_ipc_call.assert_any_call("arn:test:object") + + +def test_no_ipc_connection(mocker): + + import src.retrieveInfluxDBSecrets as ris + mock_ipc_call = mocker.patch("awsiot.greengrasscoreipc.connect", side_effect=TimeoutError("test")) + + with pytest.raises(TimeoutError, match='test'): + ris.get_secret_over_ipc("arn:test:object") + assert mock_ipc_call.call_count == 1 + + mock_ipc_call = mocker.patch("awsiot.greengrasscoreipc.connect", side_effect=UnauthorizedError()) + with pytest.raises(UnauthorizedError): + ris.get_secret_over_ipc("arn:test:object") + assert mock_ipc_call.call_count == 1 + + mock_ipc_call = mocker.patch("awsiot.greengrasscoreipc.connect", side_effect=Exception("test")) + with pytest.raises(Exception, match='test'): + ris.get_secret_over_ipc("arn:test:object") + assert mock_ipc_call.call_count == 1