From 79e373bcc6978ca532e2551c63b87a20984ba3a3 Mon Sep 17 00:00:00 2001 From: Scott Trent Date: Tue, 8 Oct 2024 13:37:58 +0900 Subject: [PATCH 01/13] Starting point for Impact Framework Signed-off-by: Scott Trent --- samples/README.md | 3 ++ samples/impact-framework/Dockerfile | 10 +++++ samples/impact-framework/README.md | 25 +++++++++++ samples/impact-framework/demo.yaml | 30 ++++++++++++++ .../impact-framework/impact-framework.yaml | 30 ++++++++++++++ samples/impact-framework/prepare.sh | 41 +++++++++++++++++++ 6 files changed, 139 insertions(+) create mode 100644 samples/impact-framework/Dockerfile create mode 100644 samples/impact-framework/README.md create mode 100644 samples/impact-framework/demo.yaml create mode 100644 samples/impact-framework/impact-framework.yaml create mode 100644 samples/impact-framework/prepare.sh diff --git a/samples/README.md b/samples/README.md index cfe62b7..d9151bb 100644 --- a/samples/README.md +++ b/samples/README.md @@ -20,6 +20,9 @@ - `training-job-2.yaml` - `rhosaij.yaml` +## directories +- `impact-framework` - sample files for using SusQL with the [Green Software Impact Framework](https://if.greensoftware.foundation). + ## License diff --git a/samples/impact-framework/Dockerfile b/samples/impact-framework/Dockerfile new file mode 100644 index 0000000..0267d63 --- /dev/null +++ b/samples/impact-framework/Dockerfile @@ -0,0 +1,10 @@ +FROM ubuntu:latest + +WORKDIR /work +COPY demo.yaml . + +RUN apt-get update -y && \ +apt-get upgrade -y && \ +apt-get install git npm vim lsb-release -y && \ +chown -R 1000:1000 /work + diff --git a/samples/impact-framework/README.md b/samples/impact-framework/README.md new file mode 100644 index 0000000..7fe8484 --- /dev/null +++ b/samples/impact-framework/README.md @@ -0,0 +1,25 @@ +# Using SusQL with the Green Software Foundation's Impact Framework + +### Using prebuilt container image: +- Ensure that you are logged into your cluster, then start the Impact Framework container: `oc apply -f impact-framework.yaml --wait` +- Log into the newly created container: `oc rsh impact-framework bash` +- Clone the Impact Framework repository, install it, and run a sample manifest: +``` +cd if +npm install +npm run if-run -- --manifest manifests/examples/builtins/sum/success.yml +``` + +### Building your own container image +For those who wish to build their own Impact Framework, log into both your +cluster and your image repository, and run the following commands: +``` +export IMG=// demo.yaml +name: basic-demo +description: +tags: +initialize: + plugins: + double-a-value: + path: 'builtin' + method: Coefficient + config: + input-parameter: "cpu-utilization" + coefficient: 2 + output-parameter: "cpu-utilization-doubled" + +tree: + children: + child-0: + defaults: + cpu/thermal-design-power: 100 + pipeline: + observe: + regroup: + compute: + - double-a-value + inputs: + - timestamp: 2023-07-06T00:00 + duration: 1 + cpu/utilization: 20 + - timestamp: 2023-07-06T00:01 + duration: 1 + cpu/utilization: 80 +EOF +git clone https://github.com/Green-Software-Foundation/if +cd if +npm install +npm run if-run -- --manifest manifests/examples/builtins/sum/success.yml From e6ce23186112d98d2576b845404a4df373b7960e Mon Sep 17 00:00:00 2001 From: Scott Trent Date: Wed, 9 Oct 2024 10:50:50 +0900 Subject: [PATCH 02/13] IF update for local use Signed-off-by: Scott Trent --- .gitignore | 1 + samples/impact-framework/Dockerfile | 17 ++++++++++++----- samples/impact-framework/ptest.yaml | 27 +++++++++++++++++++++++++++ 3 files changed, 40 insertions(+), 5 deletions(-) create mode 100644 samples/impact-framework/ptest.yaml diff --git a/.gitignore b/.gitignore index eea678d..ddef438 100644 --- a/.gitignore +++ b/.gitignore @@ -31,3 +31,4 @@ temp-prometheus.yaml .susql-deploy-info.txt .susql-deploy-info-last.txt c3-susql-config.yaml +nohup.out diff --git a/samples/impact-framework/Dockerfile b/samples/impact-framework/Dockerfile index 0267d63..c7a2dda 100644 --- a/samples/impact-framework/Dockerfile +++ b/samples/impact-framework/Dockerfile @@ -1,10 +1,17 @@ FROM ubuntu:latest WORKDIR /work -COPY demo.yaml . +COPY ptest.yaml demo.yaml . -RUN apt-get update -y && \ -apt-get upgrade -y && \ -apt-get install git npm vim lsb-release -y && \ -chown -R 1000:1000 /work +RUN apt-get update -y && \ +apt-get upgrade -y && \ +apt-get install git npm vim lsb-release -y && \ +echo "Finished apt-get install" && \ +npm install -g --save-dev husky && \ +echo "Finished husky install" && \ +npm install -g @grnsft/if && \ +echo "Finished if install" && \ +npm install -g https://github.com/Shivani-G/prometheus-importer +echo "Finsihed pi isntall" && \ +chown -R 1000:1000 /work /usr/local/lib/node_modules diff --git a/samples/impact-framework/ptest.yaml b/samples/impact-framework/ptest.yaml new file mode 100644 index 0000000..02e430d --- /dev/null +++ b/samples/impact-framework/ptest.yaml @@ -0,0 +1,27 @@ +name: prometheus importer test +description: simple susql demo invoking prometheus-importer plugin +initialize: + plugins: + prometheus-importer: + method: PrometheusImporter + path: 'https://github.com/Shivani-G/prometheus-importer' + global-config: + step: '1h' + start: '2024-08-01T23:50:30.781Z' + end: '2024-08-31T00:00:30.781Z' + query: 'rate(susql_container_joules_total)[1m])' + metricLabels: + - name + metricName: 'cpu/utilization' + defaultLabels: + duration: 3600 + cloud/vendor: 'ibm' + cloud/instance-type: medium' + outputs: + - yaml +tree: + children: + child: + pipeline: + - prometheus-importer + inputs: From 999b81dadcd80c12d6c134eb39f8e2d1ce7f9da8 Mon Sep 17 00:00:00 2001 From: Scott Trent Date: Thu, 10 Oct 2024 16:53:51 +0900 Subject: [PATCH 03/13] testing prometheus-importer Signed-off-by: Scott Trent --- samples/impact-framework/Dockerfile | 24 +++++++++++++++++------- samples/impact-framework/README.md | 18 ++++++++++++------ samples/impact-framework/ptest.yaml | 8 ++++---- 3 files changed, 33 insertions(+), 17 deletions(-) diff --git a/samples/impact-framework/Dockerfile b/samples/impact-framework/Dockerfile index c7a2dda..e5dd4c4 100644 --- a/samples/impact-framework/Dockerfile +++ b/samples/impact-framework/Dockerfile @@ -1,17 +1,27 @@ FROM ubuntu:latest +ENV NPM_CONFIG_PREFIX=/usr/local +ARG NPM_CONFIG_PREFIX=/usr/local + WORKDIR /work COPY ptest.yaml demo.yaml . RUN apt-get update -y && \ apt-get upgrade -y && \ apt-get install git npm vim lsb-release -y && \ -echo "Finished apt-get install" && \ +echo " --- Finished apt-get install" && \ npm install -g --save-dev husky && \ -echo "Finished husky install" && \ -npm install -g @grnsft/if && \ -echo "Finished if install" && \ -npm install -g https://github.com/Shivani-G/prometheus-importer -echo "Finsihed pi isntall" && \ -chown -R 1000:1000 /work /usr/local/lib/node_modules +echo " --- Finished husky install" && \ +npm install -g "@grnsft/if" && \ +echo " --- Finished IF install" && \ +git clone https://github.com/Green-Software-Foundation/if.git && cd if && npm install -g && npm install && npm link && cd - && \ +echo " --- Finished IF npm link" && \ +npm install -g "https://github.com/Shivani-G/prometheus-importer" && \ +echo " --- Finished PI install" && \ +git clone https://github.com/Shivani-G/prometheus-importer.git && cd prometheus-importer && npm install -g && npm install && npm link && cd - && \ +echo " --- Finished cloning PI" && \ +cd if && npm link prometheus-importer && cd - && \ +echo " --- Finished last link " && \ +chown -R 1000:1000 /work /usr/local/ + diff --git a/samples/impact-framework/README.md b/samples/impact-framework/README.md index 7fe8484..09382be 100644 --- a/samples/impact-framework/README.md +++ b/samples/impact-framework/README.md @@ -1,20 +1,26 @@ # Using SusQL with the Green Software Foundation's Impact Framework ### Using prebuilt container image: -- Ensure that you are logged into your cluster, then start the Impact Framework container: `oc apply -f impact-framework.yaml --wait` +- Ensure that you are logged into your cluster, then start the Impact Framework container: `oc apply -f impact-framework.yaml` - Log into the newly created container: `oc rsh impact-framework bash` -- Clone the Impact Framework repository, install it, and run a sample manifest: +- Try simple unit test: ``` cd if -npm install -npm run if-run -- --manifest manifests/examples/builtins/sum/success.yml +npm run if-run -- --manifest manifests/examples/builtins/sum/success.yml +``` +- Use SusQL data: (Edit HOST and BEARER_TOKEN as necessary.) +``` +cd if +echo "BEARER_TOKEN="$(cat /var/run/secrets/kubernetes.io/serviceaccount/token) >>.env +echo "HOST=https://thanos-querier.openshift-monitoring.svc.cluster.local:9091" >>.env +npm run if-run -- --manifest ../ptest.yaml ``` ### Building your own container image For those who wish to build their own Impact Framework, log into both your -cluster and your image repository, and run the following commands: +cluster and your container image repository, and run the following commands: ``` -export IMG=//// Date: Thu, 10 Oct 2024 20:12:04 +0900 Subject: [PATCH 04/13] remove unworking file Signed-off-by: Scott Trent --- samples/impact-framework/demo.yaml | 30 ------------------------------ 1 file changed, 30 deletions(-) delete mode 100644 samples/impact-framework/demo.yaml diff --git a/samples/impact-framework/demo.yaml b/samples/impact-framework/demo.yaml deleted file mode 100644 index b243f9d..0000000 --- a/samples/impact-framework/demo.yaml +++ /dev/null @@ -1,30 +0,0 @@ -name: basic-demo -description: -tags: -initialize: - plugins: - double-a-value: - path: 'builtin' - method: Coefficient - config: - input-parameter: "cpu-utilization" - coefficient: 2 - output-parameter: "cpu-utilization-doubled" - -tree: - children: - child-0: - defaults: - cpu/thermal-design-power: 100 - pipeline: - observe: - regroup: - compute: - - double-a-value - inputs: - - timestamp: 2023-07-06T00:00 - duration: 1 - cpu/utilization: 20 - - timestamp: 2023-07-06T00:01 - duration: 1 - cpu/utilization: 80 From 9d2c4345485d3d43274f657db7980df4cb58faa1 Mon Sep 17 00:00:00 2001 From: Scott Trent Date: Thu, 10 Oct 2024 20:13:00 +0900 Subject: [PATCH 05/13] remove unneeded file Signed-off-by: Scott Trent --- samples/impact-framework/prepare.sh | 41 ----------------------------- 1 file changed, 41 deletions(-) delete mode 100644 samples/impact-framework/prepare.sh diff --git a/samples/impact-framework/prepare.sh b/samples/impact-framework/prepare.sh deleted file mode 100644 index c03dc93..0000000 --- a/samples/impact-framework/prepare.sh +++ /dev/null @@ -1,41 +0,0 @@ -apt update -y -apt upgrade -y -apt upgrade npm vim lsb-release -y -cd /root - -cat < demo.yaml -name: basic-demo -description: -tags: -initialize: - plugins: - double-a-value: - path: 'builtin' - method: Coefficient - config: - input-parameter: "cpu-utilization" - coefficient: 2 - output-parameter: "cpu-utilization-doubled" - -tree: - children: - child-0: - defaults: - cpu/thermal-design-power: 100 - pipeline: - observe: - regroup: - compute: - - double-a-value - inputs: - - timestamp: 2023-07-06T00:00 - duration: 1 - cpu/utilization: 20 - - timestamp: 2023-07-06T00:01 - duration: 1 - cpu/utilization: 80 -EOF -git clone https://github.com/Green-Software-Foundation/if -cd if -npm install -npm run if-run -- --manifest manifests/examples/builtins/sum/success.yml From 00c44aca1a8cb8dd11426c0f848450bb3459253b Mon Sep 17 00:00:00 2001 From: Scott Trent Date: Thu, 10 Oct 2024 20:14:04 +0900 Subject: [PATCH 06/13] remove reference to deleted file Signed-off-by: Scott Trent --- samples/impact-framework/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/samples/impact-framework/Dockerfile b/samples/impact-framework/Dockerfile index e5dd4c4..af66a23 100644 --- a/samples/impact-framework/Dockerfile +++ b/samples/impact-framework/Dockerfile @@ -4,7 +4,7 @@ ENV NPM_CONFIG_PREFIX=/usr/local ARG NPM_CONFIG_PREFIX=/usr/local WORKDIR /work -COPY ptest.yaml demo.yaml . +COPY ptest.yaml . RUN apt-get update -y && \ apt-get upgrade -y && \ From 59220b6a6afddc87673f0071a468c35e25c7aeae Mon Sep 17 00:00:00 2001 From: Scott Trent Date: Fri, 11 Oct 2024 20:06:30 +0900 Subject: [PATCH 07/13] framework updates Signed-off-by: Scott Trent --- samples/impact-framework/Dockerfile | 14 +++++++++++--- samples/impact-framework/ptest.yaml | 7 ++++--- 2 files changed, 15 insertions(+), 6 deletions(-) diff --git a/samples/impact-framework/Dockerfile b/samples/impact-framework/Dockerfile index af66a23..bbc3d96 100644 --- a/samples/impact-framework/Dockerfile +++ b/samples/impact-framework/Dockerfile @@ -10,18 +10,26 @@ RUN apt-get update -y && \ apt-get upgrade -y && \ apt-get install git npm vim lsb-release -y && \ echo " --- Finished apt-get install" && \ +npm install -g typescript && \ +echo " --- Finished typescript install" && \ +npm install -g rimraf && \ +echo " --- Finished rimraf install" && \ +npm install -g @types/node && \ +echo " --- Finished @types/node install" && \ npm install -g --save-dev husky && \ echo " --- Finished husky install" && \ npm install -g "@grnsft/if" && \ echo " --- Finished IF install" && \ +# npm install -g "@grnsft/if-core" && \ +# echo " --- Finished if-core install" && \ git clone https://github.com/Green-Software-Foundation/if.git && cd if && npm install -g && npm install && npm link && cd - && \ echo " --- Finished IF npm link" && \ -npm install -g "https://github.com/Shivani-G/prometheus-importer" && \ +npm install -g "https://github.com/trent-s/prometheus-importer" && \ echo " --- Finished PI install" && \ -git clone https://github.com/Shivani-G/prometheus-importer.git && cd prometheus-importer && npm install -g && npm install && npm link && cd - && \ +git clone https://github.com/trent-s/prometheus-importer.git && cd prometheus-importer && \ +npm install --save-dev @types/node && npm run build && npm install -g && npm install && npm link && cd - && \ echo " --- Finished cloning PI" && \ cd if && npm link prometheus-importer && cd - && \ echo " --- Finished last link " && \ chown -R 1000:1000 /work /usr/local/ - diff --git a/samples/impact-framework/ptest.yaml b/samples/impact-framework/ptest.yaml index 66d4315..d84d11d 100644 --- a/samples/impact-framework/ptest.yaml +++ b/samples/impact-framework/ptest.yaml @@ -1,14 +1,15 @@ name: prometheus importer test description: simple susql demo invoking prometheus-importer plugin +tags: susql initialize: plugins: prometheus-importer: method: PrometheusImporter - path: 'https://github.com/Shivani-G/prometheus-importer' + path: 'prometheus-importer' global-config: step: '1h' - start: '2024-09-01T23:50:30.781Z' - end: '2024-09-30T00:00:30.781Z' + start: '2024-09-14T23:50:30.781Z' + end: '2024-10-14T00:00:30.781Z' query: 'rate(susql_container_joules_total)[1m])' metricLabels: - name From ddb5588e348b8e9d2bbb8104fc18f4b1267a2c34 Mon Sep 17 00:00:00 2001 From: Scott Trent Date: Fri, 11 Oct 2024 20:30:29 +0900 Subject: [PATCH 08/13] replace string with object Signed-off-by: Scott Trent --- samples/impact-framework/ptest.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/samples/impact-framework/ptest.yaml b/samples/impact-framework/ptest.yaml index d84d11d..959f8e3 100644 --- a/samples/impact-framework/ptest.yaml +++ b/samples/impact-framework/ptest.yaml @@ -1,6 +1,6 @@ name: prometheus importer test description: simple susql demo invoking prometheus-importer plugin -tags: susql +tags: null initialize: plugins: prometheus-importer: From 3207fe5b57234b7b82848af9c439950aeafe42d1 Mon Sep 17 00:00:00 2001 From: Scott Trent Date: Sun, 13 Oct 2024 19:49:10 +0900 Subject: [PATCH 09/13] experimental plugin Signed-off-by: Scott Trent --- samples/impact-framework/Dockerfile | 7 ++++++- samples/impact-framework/htest.yaml | 22 ++++++++++++++++++++++ 2 files changed, 28 insertions(+), 1 deletion(-) create mode 100644 samples/impact-framework/htest.yaml diff --git a/samples/impact-framework/Dockerfile b/samples/impact-framework/Dockerfile index bbc3d96..e42f4f9 100644 --- a/samples/impact-framework/Dockerfile +++ b/samples/impact-framework/Dockerfile @@ -4,7 +4,7 @@ ENV NPM_CONFIG_PREFIX=/usr/local ARG NPM_CONFIG_PREFIX=/usr/local WORKDIR /work -COPY ptest.yaml . +COPY ?test.yaml . RUN apt-get update -y && \ apt-get upgrade -y && \ @@ -29,6 +29,11 @@ echo " --- Finished PI install" && \ git clone https://github.com/trent-s/prometheus-importer.git && cd prometheus-importer && \ npm install --save-dev @types/node && npm run build && npm install -g && npm install && npm link && cd - && \ echo " --- Finished cloning PI" && \ +npm install -g "https://github.com/trent-s/hello-plugin" && \ +git clone https://github.com/trent-s/hello-plugin.git && cd hello-plugin && \ +npm install --save-dev @types/node && npm run build && npm install -g && npm install && npm link && cd - && \ +cd if && npm link hello-plugin && cd - && \ +echo " --- Finished cloning and initializing hello-plugin" && \ cd if && npm link prometheus-importer && cd - && \ echo " --- Finished last link " && \ chown -R 1000:1000 /work /usr/local/ diff --git a/samples/impact-framework/htest.yaml b/samples/impact-framework/htest.yaml new file mode 100644 index 0000000..d7c40e0 --- /dev/null +++ b/samples/impact-framework/htest.yaml @@ -0,0 +1,22 @@ +name: hello plugin test +description: simple verification of plugin tech +tags: null +initialize: + plugins: + hello-plugin: + method: HelloPlugin + path: "https://github.com/trent-s/hello-plugin" + outputs: + - yaml +tree: + children: + child: + pipeline: + - hello-plugin + inputs: + - timestamp: 2023-08-06T00:00 + duration: 3600 + carbon: 30 + - timestamp: 2023-09-06T00:00 + duration: 3600 + carbon: 30 From 2492e5d3319802baf54500655149fa63658a0b6c Mon Sep 17 00:00:00 2001 From: Scott Trent Date: Tue, 15 Oct 2024 10:24:42 +0900 Subject: [PATCH 10/13] more recent nodejs Signed-off-by: Scott Trent --- samples/impact-framework/Dockerfile | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/samples/impact-framework/Dockerfile b/samples/impact-framework/Dockerfile index e42f4f9..fa7acfb 100644 --- a/samples/impact-framework/Dockerfile +++ b/samples/impact-framework/Dockerfile @@ -8,8 +8,15 @@ COPY ?test.yaml . RUN apt-get update -y && \ apt-get upgrade -y && \ -apt-get install git npm vim lsb-release -y && \ +apt-get install git curl vim lsb-release -y && \ echo " --- Finished apt-get install" && \ +# npm install -g npm@latest && \ +# echo " --- Finished npm update" && \ +curl -fsSL https://deb.nodesource.com/setup_lts.x | bash && \ +apt-get upgrade -y && \ +apt-get install nodejs -y && \ +npm install -g npm@10.9.0 && \ +echo " --- Finished node update" && \ npm install -g typescript && \ echo " --- Finished typescript install" && \ npm install -g rimraf && \ From 0d13a7f22f78cb1279d5e121cac8c56759744be7 Mon Sep 17 00:00:00 2001 From: Scott Trent Date: Thu, 17 Oct 2024 16:20:56 +0900 Subject: [PATCH 11/13] two approaches Signed-off-by: Scott Trent --- samples/impact-framework/Dockerfile | 47 ------- samples/impact-framework/README.md | 33 +---- .../offClusterApproach/README.md | 29 ++++ .../{ => offClusterApproach}/htest.yaml | 6 +- .../{ => offClusterApproach}/ptest.yaml | 9 +- .../offClusterApproach/test/package-lock.json | 124 ++++++++++++++++++ .../offClusterApproach/test/package.json | 15 +++ .../onClusterApproach/Dockerfile | 38 ++++++ .../onClusterApproach/README.md | 39 ++++++ .../onClusterApproach/htest.yaml | 24 ++++ .../impact-framework.yaml | 0 .../onClusterApproach/ptest.yaml | 27 ++++ 12 files changed, 310 insertions(+), 81 deletions(-) delete mode 100644 samples/impact-framework/Dockerfile create mode 100644 samples/impact-framework/offClusterApproach/README.md rename samples/impact-framework/{ => offClusterApproach}/htest.yaml (77%) rename samples/impact-framework/{ => offClusterApproach}/ptest.yaml (74%) create mode 100644 samples/impact-framework/offClusterApproach/test/package-lock.json create mode 100644 samples/impact-framework/offClusterApproach/test/package.json create mode 100644 samples/impact-framework/onClusterApproach/Dockerfile create mode 100644 samples/impact-framework/onClusterApproach/README.md create mode 100644 samples/impact-framework/onClusterApproach/htest.yaml rename samples/impact-framework/{ => onClusterApproach}/impact-framework.yaml (100%) create mode 100644 samples/impact-framework/onClusterApproach/ptest.yaml diff --git a/samples/impact-framework/Dockerfile b/samples/impact-framework/Dockerfile deleted file mode 100644 index fa7acfb..0000000 --- a/samples/impact-framework/Dockerfile +++ /dev/null @@ -1,47 +0,0 @@ -FROM ubuntu:latest - -ENV NPM_CONFIG_PREFIX=/usr/local -ARG NPM_CONFIG_PREFIX=/usr/local - -WORKDIR /work -COPY ?test.yaml . - -RUN apt-get update -y && \ -apt-get upgrade -y && \ -apt-get install git curl vim lsb-release -y && \ -echo " --- Finished apt-get install" && \ -# npm install -g npm@latest && \ -# echo " --- Finished npm update" && \ -curl -fsSL https://deb.nodesource.com/setup_lts.x | bash && \ -apt-get upgrade -y && \ -apt-get install nodejs -y && \ -npm install -g npm@10.9.0 && \ -echo " --- Finished node update" && \ -npm install -g typescript && \ -echo " --- Finished typescript install" && \ -npm install -g rimraf && \ -echo " --- Finished rimraf install" && \ -npm install -g @types/node && \ -echo " --- Finished @types/node install" && \ -npm install -g --save-dev husky && \ -echo " --- Finished husky install" && \ -npm install -g "@grnsft/if" && \ -echo " --- Finished IF install" && \ -# npm install -g "@grnsft/if-core" && \ -# echo " --- Finished if-core install" && \ -git clone https://github.com/Green-Software-Foundation/if.git && cd if && npm install -g && npm install && npm link && cd - && \ -echo " --- Finished IF npm link" && \ -npm install -g "https://github.com/trent-s/prometheus-importer" && \ -echo " --- Finished PI install" && \ -git clone https://github.com/trent-s/prometheus-importer.git && cd prometheus-importer && \ -npm install --save-dev @types/node && npm run build && npm install -g && npm install && npm link && cd - && \ -echo " --- Finished cloning PI" && \ -npm install -g "https://github.com/trent-s/hello-plugin" && \ -git clone https://github.com/trent-s/hello-plugin.git && cd hello-plugin && \ -npm install --save-dev @types/node && npm run build && npm install -g && npm install && npm link && cd - && \ -cd if && npm link hello-plugin && cd - && \ -echo " --- Finished cloning and initializing hello-plugin" && \ -cd if && npm link prometheus-importer && cd - && \ -echo " --- Finished last link " && \ -chown -R 1000:1000 /work /usr/local/ - diff --git a/samples/impact-framework/README.md b/samples/impact-framework/README.md index 09382be..277018c 100644 --- a/samples/impact-framework/README.md +++ b/samples/impact-framework/README.md @@ -1,31 +1,10 @@ # Using SusQL with the Green Software Foundation's Impact Framework -### Using prebuilt container image: -- Ensure that you are logged into your cluster, then start the Impact Framework container: `oc apply -f impact-framework.yaml` -- Log into the newly created container: `oc rsh impact-framework bash` -- Try simple unit test: -``` -cd if -npm run if-run -- --manifest manifests/examples/builtins/sum/success.yml -``` -- Use SusQL data: (Edit HOST and BEARER_TOKEN as necessary.) -``` -cd if -echo "BEARER_TOKEN="$(cat /var/run/secrets/kubernetes.io/serviceaccount/token) >>.env -echo "HOST=https://thanos-querier.openshift-monitoring.svc.cluster.local:9091" >>.env -npm run if-run -- --manifest ../ptest.yaml -``` +A large design point in using SusQL data with the Green Software Foundation's +Impact Framework is whether to run in a container ON the cluster, or run on a machine +external to the cluster. -### Building your own container image -For those who wish to build their own Impact Framework, log into both your -cluster and your container image repository, and run the following commands: -``` -export IMG=//=18", + "npm": ">=8" + } + }, + "node_modules/dotenv": { + "version": "16.3.1", + "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-16.3.1.tgz", + "integrity": "sha512-IPzF4w4/Rd94bA9imS68tZBaYyBWSCE47V1RGuMrB94iyTOIEwRmVL2x/4An+6mETpLrKJ5hQkB8W4kFAadeIQ==", + "license": "BSD-2-Clause", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/motdotla/dotenv?sponsor=1" + } + }, + "node_modules/node-domexception": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/node-domexception/-/node-domexception-1.0.0.tgz", + "integrity": "sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/jimmywarting" + }, + { + "type": "github", + "url": "https://paypal.me/jimmywarting" + } + ], + "license": "MIT", + "engines": { + "node": ">=10.5.0" + } + }, + "node_modules/node-fetch-commonjs": { + "version": "3.3.2", + "resolved": "https://registry.npmjs.org/node-fetch-commonjs/-/node-fetch-commonjs-3.3.2.tgz", + "integrity": "sha512-VBlAiynj3VMLrotgwOS3OyECFxas5y7ltLcK4t41lMUZeaK15Ym4QRkqN0EQKAFL42q9i21EPKjzLUPfltR72A==", + "license": "MIT", + "dependencies": { + "node-domexception": "^1.0.0", + "web-streams-polyfill": "^3.0.3" + }, + "engines": { + "node": "^12.20.0 || ^14.13.1 || >=16.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/node-fetch" + } + }, + "node_modules/prometheus-importer": { + "version": "0.2.2", + "resolved": "git+ssh://git@github.com/trent-s/prometheus-importer.git#00b87d68d4d5351ba3768d9a70c9e0f6e3f199bd", + "license": "MIT", + "dependencies": { + "@grnsft/if-core": "^0.0.7", + "dotenv": "16.3.1", + "node-fetch-commonjs": "^3.3.2", + "typescript": "^5.6.3", + "zod": "^3.23.8" + }, + "engines": { + "node": ">=18", + "npm": ">=8" + } + }, + "node_modules/typescript": { + "version": "5.6.3", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.6.3.tgz", + "integrity": "sha512-hjcS1mhfuyi4WW8IWtjP7brDrG2cuDZukyrYrSauoXGNgx0S7zceP07adYkJycEr56BOUTNPzbInooiN3fn1qw==", + "license": "Apache-2.0", + "bin": { + "tsc": "bin/tsc", + "tsserver": "bin/tsserver" + }, + "engines": { + "node": ">=14.17" + } + }, + "node_modules/web-streams-polyfill": { + "version": "3.3.3", + "resolved": "https://registry.npmjs.org/web-streams-polyfill/-/web-streams-polyfill-3.3.3.tgz", + "integrity": "sha512-d2JWLCivmZYTSIoge9MsgFCZrt571BikcWGYkjC1khllbTeDlGqZ2D8vD8E/lJa8WGWbb7Plm8/XJYV7IJHZZw==", + "license": "MIT", + "engines": { + "node": ">= 8" + } + }, + "node_modules/zod": { + "version": "3.23.8", + "resolved": "https://registry.npmjs.org/zod/-/zod-3.23.8.tgz", + "integrity": "sha512-XBx9AXhXktjUqnepgTiE5flcKIYWi/rme0Eaj+5Y0lftuGBq+jyRu/md4WnuxqgP1ubdpNCsYEYPxrzVHD8d6g==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/colinhacks" + } + } + } +} diff --git a/samples/impact-framework/offClusterApproach/test/package.json b/samples/impact-framework/offClusterApproach/test/package.json new file mode 100644 index 0000000..1ed8e6f --- /dev/null +++ b/samples/impact-framework/offClusterApproach/test/package.json @@ -0,0 +1,15 @@ +{ + "name": "test", + "version": "1.0.0", + "main": "index.js", + "scripts": { + "test": "echo \"Error: no test specified\" && exit 1" + }, + "keywords": [], + "author": "", + "license": "ISC", + "description": "", + "dependencies": { + "prometheus-importer": "github:trent-s/prometheus-importer" + } +} diff --git a/samples/impact-framework/onClusterApproach/Dockerfile b/samples/impact-framework/onClusterApproach/Dockerfile new file mode 100644 index 0000000..08b9b11 --- /dev/null +++ b/samples/impact-framework/onClusterApproach/Dockerfile @@ -0,0 +1,38 @@ +FROM ubuntu:latest + +ENV NPM_CONFIG_PREFIX=/usr/local +ARG NPM_CONFIG_PREFIX=/usr/local + +WORKDIR /work +COPY ?test.yaml . + +RUN apt-get update -y && \ +apt-get upgrade -y && \ +apt-get install git curl vim lsb-release -y && \ +echo " --- Finished apt-get install" && \ +# npm install -g npm@latest && \ +# echo " --- Finished npm update" && \ +curl -fsSL https://deb.nodesource.com/setup_lts.x | bash && \ +apt-get upgrade -y && \ +apt-get install nodejs -y && \ +npm install -g npm@10.9.0 && \ +echo " --- Finished node update" && \ +npm install -g typescript rimraf husky @grnsft/if && \ +echo " --- Finished if install" && \ +# npm install -g @types/node && \ +# echo " --- Finished @types/node install" && \ +# git clone https://github.com/Green-Software-Foundation/if.git && cd if && npm install -g && npm install && npm link && cd - && \ +# echo " --- Finished IF npm link" && \ +# npm install -g "https://github.com/trent-s/prometheus-importer" && \ +# echo " --- Finished PI install" && \ +# git clone https://github.com/trent-s/prometheus-importer.git && cd prometheus-importer && \ +# npm install --save-dev @types/node && npm run build && npm install -g && npm install && npm link && cd - && \ +# echo " --- Finished cloning PI" && \ +# npm install -g "https://github.com/trent-s/hello-plugin" && \ +# git clone https://github.com/trent-s/hello-plugin.git && cd hello-plugin && \ +# npm install --save-dev @types/node && npm run build && npm install -g && npm install && npm link && cd - && \ +# cd if && npm link hello-plugin && cd - && \ +# echo " --- Finished cloning and initializing hello-plugin" && \ +# cd if && npm link prometheus-importer && cd - && \ +# echo " --- Finished last link " && \ +chown -R 1000:1000 /work /usr/local/ diff --git a/samples/impact-framework/onClusterApproach/README.md b/samples/impact-framework/onClusterApproach/README.md new file mode 100644 index 0000000..f72c9c2 --- /dev/null +++ b/samples/impact-framework/onClusterApproach/README.md @@ -0,0 +1,39 @@ +# Using SusQL with the Green Software Foundation's Impact Framework + +The following README explains an approach to utilize SusQL data in the +Green Software Foundation's Impact Framework using a slightly modified version +of the PrometheusImporter plugin. The fundamental focus of this particular +approach is to run entirely on Cluster, and is based on OpenShift. + +However, this is a work in progress related to an apparent limitation +in the use of certain required nodejs functionality within a container. + +### Using prebuilt container image: +- Ensure that you are logged into your cluster, then start the Impact Framework container: `oc apply -f impact-framework.yaml` +- Log into the newly created container: `oc rsh impact-framework bash` +- Try simple unit test: +``` +cd if +npm run if-run -- --manifest manifests/examples/builtins/sum/success.yml +``` +- Use SusQL data: (Edit HOST and BEARER_TOKEN as necessary.) +``` +cd if +echo "BEARER_TOKEN="$(cat /var/run/secrets/kubernetes.io/serviceaccount/token) >>.env +echo "HOST=https://thanos-querier.openshift-monitoring.svc.cluster.local:9091" >>.env +npm run if-run -- --manifest ../ptest.yaml +``` + +### Building your own container image +For those who wish to build their own Impact Framework, log into both your +cluster and your container image repository, and run the following commands: +``` +export IMG=// Date: Mon, 21 Oct 2024 14:32:12 +0900 Subject: [PATCH 12/13] ignore .env file Signed-off-by: Scott Trent --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index ddef438..0fa175c 100644 --- a/.gitignore +++ b/.gitignore @@ -32,3 +32,4 @@ temp-prometheus.yaml .susql-deploy-info-last.txt c3-susql-config.yaml nohup.out +.env From 3a98ae13c092023fc71f9a8016aaaa0ea408c6b0 Mon Sep 17 00:00:00 2001 From: Scott Trent Date: Mon, 28 Oct 2024 09:54:17 +0900 Subject: [PATCH 13/13] update to latest Signed-off-by: Scott Trent --- .gitignore | 1 + samples/impact-framework/README.md | 4 +-- .../offClusterApproach/.trent/README.md | 1 + .../{ => .trent}/htest.yaml | 0 .../.trent/pi1/pi1test.yaml | 28 +++++++++++++++++++ .../offClusterApproach/.trent/pi1/steps.txt | 20 +++++++++++++ .../offClusterApproach/README.md | 15 +++++++--- .../offClusterApproach/ptest.yaml | 2 +- .../offClusterApproach/test/package-lock.json | 8 +++--- .../offClusterApproach/test/package.json | 2 +- .../onClusterApproach/{ => .trent}/htest.yaml | 0 .../onClusterApproach/Dockerfile | 18 ++---------- .../onClusterApproach/README.md | 9 +++++- .../onClusterApproach/ptest.yaml | 2 +- 14 files changed, 80 insertions(+), 30 deletions(-) create mode 100644 samples/impact-framework/offClusterApproach/.trent/README.md rename samples/impact-framework/offClusterApproach/{ => .trent}/htest.yaml (100%) create mode 100644 samples/impact-framework/offClusterApproach/.trent/pi1/pi1test.yaml create mode 100644 samples/impact-framework/offClusterApproach/.trent/pi1/steps.txt rename samples/impact-framework/onClusterApproach/{ => .trent}/htest.yaml (100%) diff --git a/.gitignore b/.gitignore index 0fa175c..31533fb 100644 --- a/.gitignore +++ b/.gitignore @@ -33,3 +33,4 @@ temp-prometheus.yaml c3-susql-config.yaml nohup.out .env +node_modules diff --git a/samples/impact-framework/README.md b/samples/impact-framework/README.md index 277018c..7edd94e 100644 --- a/samples/impact-framework/README.md +++ b/samples/impact-framework/README.md @@ -1,8 +1,8 @@ # Using SusQL with the Green Software Foundation's Impact Framework A large design point in using SusQL data with the Green Software Foundation's -Impact Framework is whether to run in a container ON the cluster, or run on a machine -external to the cluster. +Impact Framework is whether to run in a container ON the cluster with SusQL, or run +on a machine external to the cluster. This directory contains two directories with experiments into both approaches: diff --git a/samples/impact-framework/offClusterApproach/.trent/README.md b/samples/impact-framework/offClusterApproach/.trent/README.md new file mode 100644 index 0000000..86478a9 --- /dev/null +++ b/samples/impact-framework/offClusterApproach/.trent/README.md @@ -0,0 +1 @@ +experimentation... diff --git a/samples/impact-framework/offClusterApproach/htest.yaml b/samples/impact-framework/offClusterApproach/.trent/htest.yaml similarity index 100% rename from samples/impact-framework/offClusterApproach/htest.yaml rename to samples/impact-framework/offClusterApproach/.trent/htest.yaml diff --git a/samples/impact-framework/offClusterApproach/.trent/pi1/pi1test.yaml b/samples/impact-framework/offClusterApproach/.trent/pi1/pi1test.yaml new file mode 100644 index 0000000..7b78b86 --- /dev/null +++ b/samples/impact-framework/offClusterApproach/.trent/pi1/pi1test.yaml @@ -0,0 +1,28 @@ +name: prometheus importer test +description: simple susql demo invoking prometheus-importer plugin +tags: null +initialize: + plugins: + prometheus-importer: + method: PrometheusImporter + path: 'https://github.com/Shivani-G/prometheus-importer' + global-config: + step: '1h' + start: '2024-09-16T23:50:30.000Z' + end: '2024-10-16T00:00:30.000Z' + query: 'rate(susql_container_joules_total)[1m])' + metricLabels: + - name + metricName: 'energy' + defaultLabels: + duration: 3600 + cloud/vendor: 'ibm' + cloud/instance-type: 'medium' + outputs: + - yaml +tree: + children: + child: + pipeline: + - prometheus-importer + inputs: diff --git a/samples/impact-framework/offClusterApproach/.trent/pi1/steps.txt b/samples/impact-framework/offClusterApproach/.trent/pi1/steps.txt new file mode 100644 index 0000000..28ed6cd --- /dev/null +++ b/samples/impact-framework/offClusterApproach/.trent/pi1/steps.txt @@ -0,0 +1,20 @@ +# in this directory +npm init -y +npm install -g husky +npm install -g @grnsft/if +# npm remove -g "https://github.com/trent-s/prometheus-importer" +npm remove -g prometheus-importer +npm remove prometheus-importer +npm install -g "https://github.com/Shivani-G/prometheus-importer" +npm update +echo "BEARER_TOKEN="$(oc whoami -t) >.env +echo HOST=https://$(oc get routes -n openshift-monitoring thanos-querier -o jsonpath='{.status.ingress[0].host}') >>.env +if-run --manifest pi1test.yaml --debug + + + + +## troubleshootign +# npm cache verify +# npm install npm@latest -g + diff --git a/samples/impact-framework/offClusterApproach/README.md b/samples/impact-framework/offClusterApproach/README.md index b54f065..2748b4d 100644 --- a/samples/impact-framework/offClusterApproach/README.md +++ b/samples/impact-framework/offClusterApproach/README.md @@ -1,20 +1,27 @@ # Using SusQL with the Green Software Foundation's Impact Framework This particular experiment was performed using a recent RHEL 9 x86 machine, -however, any OS that supports a sufficiently recent version of nodejs should +however, any OS that supports a sufficiently recent version of `nodejs` should work in principle. +The key to importing data from SusQL is a Prometheus Importer Plugin for the GSF Impact Framework. +Currently there are two such plugins described in the (IF Explorer)[https://explorer.if.greensoftware.foundation]: +- (`Prometheus Importer`)[https://github.com/andreic94/if-prometheus-importer/blob/main/README.md] by `andreic94`, et al. +- (`prometheus-importer`)[https://github.com/Shivani-G/prometheus-importer/blob/main/README.md] by `Shibani-G`. + +The following is an approach using the later plugin: + ### steps - Ensure that you are logged into your cluster and can use the `oc` (or `kubectl` command). - Make sure that a recent version of `node` is installed. (This test used v22.9.0) - Install Impact Framework - `npm install -g @grnsft/if` - Install Prometheus Importer - - `npm install -g "https://github.com/trent-s/prometheus-importer"` -- Update, just to be sure: + - `npm install -g "https://github.com/Shivani-G/prometheus-importer"` +- Update, just to be sure: (Starting from the directory that contains this README...) - `cd test; npm update; cd -` -- Create required credential file, and attempt to use prometheus-importer: +- Create required credential file, and attempt to use prometheus-importer: (Starting from the directory that contains this README...) ``` cd test echo "BEARER_TOKEN="$(oc whoami -t) >.env diff --git a/samples/impact-framework/offClusterApproach/ptest.yaml b/samples/impact-framework/offClusterApproach/ptest.yaml index 3af2d41..ef96335 100644 --- a/samples/impact-framework/offClusterApproach/ptest.yaml +++ b/samples/impact-framework/offClusterApproach/ptest.yaml @@ -5,7 +5,7 @@ initialize: plugins: prometheus-importer: method: PrometheusImporter - path: 'https://github.com/trent-s/prometheus-importer' + path: 'https://github.com/Shivani-G/prometheus-importer' config: step: '1h' start: '2024-09-16T23:50:30.000Z' diff --git a/samples/impact-framework/offClusterApproach/test/package-lock.json b/samples/impact-framework/offClusterApproach/test/package-lock.json index 98ee414..1ff56e9 100644 --- a/samples/impact-framework/offClusterApproach/test/package-lock.json +++ b/samples/impact-framework/offClusterApproach/test/package-lock.json @@ -9,7 +9,7 @@ "version": "1.0.0", "license": "ISC", "dependencies": { - "prometheus-importer": "github:trent-s/prometheus-importer" + "prometheus-importer": "github:Shivani-G/prometheus-importer" } }, "node_modules/@grnsft/if-core": { @@ -74,14 +74,14 @@ } }, "node_modules/prometheus-importer": { - "version": "0.2.2", - "resolved": "git+ssh://git@github.com/trent-s/prometheus-importer.git#00b87d68d4d5351ba3768d9a70c9e0f6e3f199bd", + "version": "v0.1.1", + "resolved": "git+ssh://git@github.com/Shivani-G/prometheus-importer.git#861a0a7ef3e5f81ec2904838f0b6905a94686a0d", "license": "MIT", "dependencies": { "@grnsft/if-core": "^0.0.7", "dotenv": "16.3.1", "node-fetch-commonjs": "^3.3.2", - "typescript": "^5.6.3", + "typescript": "^5.5.2", "zod": "^3.23.8" }, "engines": { diff --git a/samples/impact-framework/offClusterApproach/test/package.json b/samples/impact-framework/offClusterApproach/test/package.json index 1ed8e6f..78e39f8 100644 --- a/samples/impact-framework/offClusterApproach/test/package.json +++ b/samples/impact-framework/offClusterApproach/test/package.json @@ -10,6 +10,6 @@ "license": "ISC", "description": "", "dependencies": { - "prometheus-importer": "github:trent-s/prometheus-importer" + "prometheus-importer": "github:Shivani-G/prometheus-importer" } } diff --git a/samples/impact-framework/onClusterApproach/htest.yaml b/samples/impact-framework/onClusterApproach/.trent/htest.yaml similarity index 100% rename from samples/impact-framework/onClusterApproach/htest.yaml rename to samples/impact-framework/onClusterApproach/.trent/htest.yaml diff --git a/samples/impact-framework/onClusterApproach/Dockerfile b/samples/impact-framework/onClusterApproach/Dockerfile index 08b9b11..aa18a52 100644 --- a/samples/impact-framework/onClusterApproach/Dockerfile +++ b/samples/impact-framework/onClusterApproach/Dockerfile @@ -19,20 +19,6 @@ npm install -g npm@10.9.0 && \ echo " --- Finished node update" && \ npm install -g typescript rimraf husky @grnsft/if && \ echo " --- Finished if install" && \ -# npm install -g @types/node && \ -# echo " --- Finished @types/node install" && \ -# git clone https://github.com/Green-Software-Foundation/if.git && cd if && npm install -g && npm install && npm link && cd - && \ -# echo " --- Finished IF npm link" && \ -# npm install -g "https://github.com/trent-s/prometheus-importer" && \ -# echo " --- Finished PI install" && \ -# git clone https://github.com/trent-s/prometheus-importer.git && cd prometheus-importer && \ -# npm install --save-dev @types/node && npm run build && npm install -g && npm install && npm link && cd - && \ -# echo " --- Finished cloning PI" && \ -# npm install -g "https://github.com/trent-s/hello-plugin" && \ -# git clone https://github.com/trent-s/hello-plugin.git && cd hello-plugin && \ -# npm install --save-dev @types/node && npm run build && npm install -g && npm install && npm link && cd - && \ -# cd if && npm link hello-plugin && cd - && \ -# echo " --- Finished cloning and initializing hello-plugin" && \ -# cd if && npm link prometheus-importer && cd - && \ -# echo " --- Finished last link " && \ +npm install -g "https://github.com/Shivani-G/prometheus-importer" && \ +echo " --- Finished PI install" && \ chown -R 1000:1000 /work /usr/local/ diff --git a/samples/impact-framework/onClusterApproach/README.md b/samples/impact-framework/onClusterApproach/README.md index f72c9c2..e85bc87 100644 --- a/samples/impact-framework/onClusterApproach/README.md +++ b/samples/impact-framework/onClusterApproach/README.md @@ -5,8 +5,15 @@ Green Software Foundation's Impact Framework using a slightly modified version of the PrometheusImporter plugin. The fundamental focus of this particular approach is to run entirely on Cluster, and is based on OpenShift. +The key to importing data from SusQL is a Prometheus Importer Plugin for the GSF Impact Framework. +Currently there are two such plugins described in the (IF Explorer)[https://explorer.if.greensoftware.foundation]: +- (`Prometheus Importer`)[https://github.com/andreic94/if-prometheus-importer/blob/main/README.md] by `andreic94`, et al. +- (`prometheus-importer`)[https://github.com/Shivani-G/prometheus-importer/blob/main/README.md] by `Shibani-G`. + +This approach uses the later plugin. + However, this is a work in progress related to an apparent limitation -in the use of certain required nodejs functionality within a container. +in the use of certain required global`nodejs` functionality within a container. ### Using prebuilt container image: - Ensure that you are logged into your cluster, then start the Impact Framework container: `oc apply -f impact-framework.yaml` diff --git a/samples/impact-framework/onClusterApproach/ptest.yaml b/samples/impact-framework/onClusterApproach/ptest.yaml index 3af2d41..ef96335 100644 --- a/samples/impact-framework/onClusterApproach/ptest.yaml +++ b/samples/impact-framework/onClusterApproach/ptest.yaml @@ -5,7 +5,7 @@ initialize: plugins: prometheus-importer: method: PrometheusImporter - path: 'https://github.com/trent-s/prometheus-importer' + path: 'https://github.com/Shivani-G/prometheus-importer' config: step: '1h' start: '2024-09-16T23:50:30.000Z'