diff --git a/README.md b/README.md index 8309a8b..5c17ace 100644 --- a/README.md +++ b/README.md @@ -10,11 +10,22 @@ SusQL is an operator that can be deployed in a Kubernetes/OpenShift cluster. You ## Carbon Dioxide Emission Calculation -CO2 emission calculation is currently a work in progress. At the moment we use -a static conversion factor based on +There are two currently CO2 emission calculation methods available, and an additional method is under development. +- The default CO2 emission calculation method is called "static", and simply uses a grams of CO2 per Joule +of electricity consumed coefficient to calculate grams of CO2 emitted. This value is user tunable by +modifying the `CARBON-INTENSITY` configmap value. The default value is based on [US EPA](https://www.epa.gov/energy/greenhouse-gases-equivalencies-calculator-calculations-and-references) -data. Although users can configure this value to match their runtime environment, we are working on providing -automatic up to date conversion functionality. +data. +- The "simpledynamic" method periodically queries the carbon intensity value for a user specified location to provide a more accurate estimation of CO2 emission. +The configmap user configurable items are: + - `CARBON-METHOD` - The "simpledynamic" method is enabled when set to `simpledynamic`. + - `CARBON-INTENSITY` - This value is set automatically. User specified values will be overwritten. + - `CARBON-INTENSITY-URL` - Specifies a web API that returns carbon intensity. The default value works as of the date of this writing. + - `CARBON-LOCATION` - The location identifiers are defined by the API provider, but can be selected by he user. + - `CARBON-QUERY-RATE` - Interval in seconds at which the carbon intensity is queried. The data available from the source is updated less than hourly, so an interval of greater than 3600 seconds is recommended. + - `CARBON-QUERY-FILTER` - When the return value is embedded in a JSON object, this specification enables the extraction of the data. The default value matches the default provider. + - `CARBON-QUERY-CONV-2J` - If the carbon data provider does not provide data in the standard "grams of CO2 per Joule" then, this factor can be specified to normalize the units displayed. The default value ensures tha the default provider data is in the correct unit. +- The third method "sdk" is still under development, and will offer an integration with the Green Software Foundation's [Carbon Aware SDK](https://github.com/Green-Software-Foundation/carbon-aware-sdk). ### Prerequisites @@ -27,7 +38,7 @@ Follow these instructions to install the SusQL Operator with Helm. Follow these instructions to install the SusQL Operator from [OperatorHub](https://operatorhub.io). - [Installation with OperatorHub](doc/operatorhub-installation.md) - + ## Using SusQL @@ -74,7 +85,7 @@ Energy of the group of pods is exposed in 2 ways: ## License -Copyright 2024. +Copyright 2023, 2024. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/VERSION b/VERSION index b056f41..d34586a 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.0.24 +0.0.25 \ No newline at end of file diff --git a/bundle/manifests/susql-operator.clusterserviceversion.yaml b/bundle/manifests/susql-operator.clusterserviceversion.yaml index f730f2d..d049ac3 100644 --- a/bundle/manifests/susql-operator.clusterserviceversion.yaml +++ b/bundle/manifests/susql-operator.clusterserviceversion.yaml @@ -22,15 +22,15 @@ metadata: ] capabilities: Basic Install categories: Monitoring - containerImage: quay.io/sustainable_computing_io/susql_operator:0.0.24 - createdAt: "2024-08-26T05:08:49Z" + containerImage: quay.io/sustainable_computing_io/susql_operator:0.0.25 + createdAt: "2024-08-28T09:06:10Z" description: 'Aggregates energy and CO2 emission data for pods tagged with SusQL labels ' operators.operatorframework.io/builder: operator-sdk-v1.36.1 operators.operatorframework.io/project_layout: go.kubebuilder.io/v4 repository: https://github.com/sustainable-computing-io/susql-operator support: https://github.com/sustainable-computing-io/susql-operator/issues - name: susql-operator.v0.0.24 + name: susql-operator.v0.0.25 namespace: placeholder spec: apiservicedefinitions: {} @@ -248,13 +248,79 @@ spec: key: SUSQL-LOG-LEVEL name: susql-config optional: true - - name: STATIC-CARBON-INTENSITY + - name: CARBON-METHOD valueFrom: configMapKeyRef: - key: STATIC-CARBON-INTENSITY + key: CARBON-METHOD name: susql-config optional: true - image: quay.io/sustainable_computing_io/susql_operator:0.0.24 + - name: CARBON-INTENSITY + valueFrom: + configMapKeyRef: + key: CARBON-INTENSITY + name: susql-config + optional: true + - name: CARBON-INTENSITY-URL + valueFrom: + configMapKeyRef: + key: CARBON-INTENSITY-URL + name: susql-config + optional: true + - name: CARBON-LOCATION + valueFrom: + configMapKeyRef: + key: CARBON-LOCATION + name: susql-config + optional: true + - name: CARBON-QUERY-RATE + valueFrom: + configMapKeyRef: + key: CARBON-QUERY-RATE + name: susql-config + optional: true + - name: CARBON-QUERY-FILTER + valueFrom: + configMapKeyRef: + key: CARBON-QUERY-FILTER + name: susql-config + optional: true + - name: CARBON-INTENSITY + valueFrom: + configMapKeyRef: + key: CARBON-INTENSITY + name: susql-config + optional: true + - name: CARBON-INTENSITY-URL + valueFrom: + configMapKeyRef: + key: CARBON-INTENSITY-URL + name: susql-config + optional: true + - name: CARBON-LOCATION + valueFrom: + configMapKeyRef: + key: CARBON-LOCATION + name: susql-config + optional: true + - name: CARBON-QUERY-RATE + valueFrom: + configMapKeyRef: + key: CARBON-QUERY-RATE + name: susql-config + optional: true + - name: CARBON-QUERY-FILTER + valueFrom: + configMapKeyRef: + key: CARBON-QUERY-FILTER + name: susql-config + optional: true + - name: CARBON-QUERY-CONV-2J + valueFrom: + configMapKeyRef: + key: CARBON-QUERY-CONV-2J + name: susql-config + optional: true + image: quay.io/sustainable_computing_io/susql_operator:0.0.25 imagePullPolicy: IfNotPresent livenessProbe: httpGet: @@ -340,6 +406,8 @@ spec: - energy - kepler - susql + - carbon dioxide (CO2) emissions + - green computing links: - name: SusQL Operator url: https://github.com/sustainable-computing-io/susql-operator @@ -355,4 +423,4 @@ spec: provider: name: SusQL Operator Contributors url: https://github.com/sustainable-computing-io/susql-operator - version: 0.0.24 + version: 0.0.25 \ No newline at end of file diff --git a/cmd/main.go b/cmd/main.go index 1aaf56c..670623b 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -69,8 +69,14 @@ func main() { var susqlPrometheusDatabaseUrl string = "https://thanos-querier.openshift-monitoring.svc.cluster.local:9091" var samplingRate string = "2" var susqlLogLevel string = "-5" - // Static Carbon Intensity Factor in grams CO2 / Joule - var staticCarbonIntensity string = "0.00000000011583333" + // Carbon Intensity Factor in grams CO2 / Joule + var carbonMethod string = "static" // options: static, simpledynamic, sdk + var carbonIntensity string = "0.0001158333333333" + var carbonIntensityUrl string = "https://api.electricitymap.org/v3/carbon-intensity/latest?zone=%s" + var carbonLocation string = "JP-TK" + var carbonQueryRate string = "7200" + var carbonQueryFilter string = "carbonIntensity" + var carbonQueryConv2J string = "0.0000002777777778" // NOTE: these can be set as env or flag, flag takes precedence over env keplerPrometheusUrlEnv := getEnv("KEPLER-PROMETHEUS-URL", keplerPrometheusUrl) @@ -80,7 +86,13 @@ func main() { samplingRateEnv := getEnv("SAMPLING-RATE", samplingRate) probeAddrEnv := getEnv("HEALTH-PROBE-BIND-ADDRESS", probeAddr) susqlLogLevelEnv := getEnv("SUSQL-LOG-LEVEL", susqlLogLevel) - staticCarbonIntensityEnv := getEnv("STATIC-CARBON-INTENSITY", staticCarbonIntensity) + carbonMethodEnv := getEnv("CARBON-METHOD", carbonMethod) + carbonIntensityEnv := getEnv("CARBON-INTENSITY", carbonIntensity) + carbonIntensityUrlEnv := getEnv("CARBON-INTENSITY-URL", carbonIntensityUrl) + carbonLocationEnv := getEnv("CARBON-LOCATION", carbonLocation) + carbonQueryRateEnv := getEnv("CARBON-QUERY-RATE", carbonQueryRate) + carbonQueryFilterEnv := getEnv("CARBON-QUERY-FILTER", carbonQueryFilter) + carbonQueryConv2JEnv := getEnv("CARBON-QUERY-CONV-2J", carbonQueryConv2J) enableLeaderElectionEnv, err := strconv.ParseBool(getEnv("LEADER-ELECT", strconv.FormatBool(enableLeaderElection))) if err != nil { enableLeaderElectionEnv = false @@ -93,7 +105,13 @@ func main() { flag.StringVar(&samplingRate, "sampling-rate", samplingRateEnv, "Sampling rate in seconds") flag.StringVar(&probeAddr, "health-probe-bind-address", probeAddrEnv, "The address the probe endpoint binds to.") flag.StringVar(&susqlLogLevel, "susql-log-level", susqlLogLevelEnv, "SusQL log level") - flag.StringVar(&staticCarbonIntensity, "static-carbon-intensity", staticCarbonIntensityEnv, "Static Carbon Intensity Factor in grams CO2 / Joule") + flag.StringVar(&carbonMethod, "carbon-method", carbonMethodEnv, "Method used to calculate CO2 emissions") + flag.StringVar(&carbonIntensity, "carbon-intensity", carbonIntensityEnv, "Carbon Intensity Factor in grams CO2 / Joule") + flag.StringVar(&carbonIntensityUrl, "carbon-intensity-url", carbonIntensityUrlEnv, "URL used to query calculate carbon intensity") + flag.StringVar(&carbonLocation, "carbon-location", carbonLocationEnv, "Location identfier used in carbon intensity query") + flag.StringVar(&carbonQueryRate, "carbon-query-rate", carbonQueryRateEnv, "How often to query carbon intensity query (seconds)") + flag.StringVar(&carbonQueryFilter, "carbon-query-filter", carbonQueryFilterEnv, "Parameter to extract carbon intensity from JSON returned by query") + flag.StringVar(&carbonQueryConv2J, "carbon-query-conv-2j", carbonQueryConv2JEnv, "Factor to convert carbon intensity returned by query to grams CO2 / Joule") flag.BoolVar(&enableLeaderElection, "leader-elect", enableLeaderElectionEnv, "Enable leader election for controller manager. "+ "Enabling this will ensure there is only one active controller manager.") @@ -121,7 +139,13 @@ func main() { susqlLog.Info("susqlPrometheusDatabaseUrl=" + susqlPrometheusDatabaseUrl) susqlLog.Info("samplingRate=" + samplingRate) susqlLog.Info("susqlLogLevel=" + susqlLogLevel) - susqlLog.Info("staticCarbonIntensity=" + staticCarbonIntensity) + susqlLog.Info("carbonMethod=" + carbonMethod) + susqlLog.Info("carbonIntensity=" + carbonIntensity) + susqlLog.Info("carbonIntensityUrl=" + carbonIntensityUrl) + susqlLog.Info("carbonLocation=" + carbonLocation) + susqlLog.Info("carbonQueryRate=" + carbonQueryRate) + susqlLog.Info("carbonQueryFilter=" + carbonQueryFilter) + susqlLog.Info("carbonQueryConv2J=" + carbonQueryConv2J) // if the enable-http2 flag is false (the default), http/2 should be disabled // due to its vulnerabilities. More specifically, disabling http/2 will @@ -164,15 +188,30 @@ func main() { os.Exit(1) } + // TODO: verify that carbonMethod is an expected value. If not log warning and set to default value. + // (static, simpledynamic, sdk) + // Note: assume that carbonIntensityUrl, carbonLocation, and carbonQueryFilter are OK. If not, we will log errors at runtime. + samplingRateInteger, err := strconv.Atoi(samplingRate) if err != nil { samplingRateInteger = 2 } - staticCarbonIntensityFloat, err := strconv.ParseFloat(staticCarbonIntensity, 64) + carbonQueryRateInteger, err := strconv.ParseInt(carbonQueryRate, 10, 64) + if err != nil { + carbonQueryRateInteger = 7200 + } + + carbonIntensityFloat, err := strconv.ParseFloat(carbonIntensity, 64) + if err != nil { + susqlLog.Error(err, "Unable to obtain initial carbon intensity value. Using 0.0.") + carbonIntensityFloat = 0.0 + } + + carbonQueryConv2JFloat, err := strconv.ParseFloat(carbonQueryConv2J, 64) if err != nil { - susqlLog.Error(err, "Unable to obtain static carbon intensity value. Using 0.0.") - staticCarbonIntensityFloat = 0.0 + susqlLog.Error(err, "Unable to obtain carbon query Conv 2J value. Using 0.0.") + carbonQueryConv2JFloat = 0.0 } susqlLog.Info("Setting up labelGroupReconciler.") @@ -185,7 +224,14 @@ func main() { SusQLPrometheusDatabaseUrl: susqlPrometheusDatabaseUrl, SusQLPrometheusMetricsUrl: susqlPrometheusMetricsUrl, SamplingRate: time.Duration(samplingRateInteger) * time.Second, - StaticCarbonIntensity: staticCarbonIntensityFloat, + CarbonMethod: carbonMethod, + CarbonIntensity: carbonIntensityFloat, + CarbonIntensityUrl: carbonIntensityUrl, + CarbonIntensityTimeStamp: 0, + CarbonLocation: carbonLocation, + CarbonQueryRate: carbonQueryRateInteger, + CarbonQueryFilter: carbonQueryFilter, + CarbonQueryConv2J: carbonQueryConv2JFloat, Logger: susqlLog, }).SetupWithManager(mgr); err != nil { susqlLog.Error(err, "unable to create controller", "controller", "LabelGroup") diff --git a/config/manager/kustomization.yaml b/config/manager/kustomization.yaml index f5a0faa..fc8a50c 100644 --- a/config/manager/kustomization.yaml +++ b/config/manager/kustomization.yaml @@ -5,4 +5,4 @@ kind: Kustomization images: - name: controller newName: quay.io/sustainable_computing_io/susql_operator - newTag: 0.0.24 + newTag: 0.0.25 \ No newline at end of file diff --git a/config/manager/manager.yaml b/config/manager/manager.yaml index e4609e8..8a40dd8 100644 --- a/config/manager/manager.yaml +++ b/config/manager/manager.yaml @@ -117,11 +117,47 @@ spec: name: susql-config key: SUSQL-LOG-LEVEL optional: true - - name: STATIC-CARBON-INTENSITY + - name: CARBON-METHOD valueFrom: configMapKeyRef: name: susql-config - key: STATIC-CARBON-INTENSITY + key: CARBON-METHOD + optional: true + - name: CARBON-INTENSITY + valueFrom: + configMapKeyRef: + name: susql-config + key: CARBON-INTENSITY + optional: true + - name: CARBON-INTENSITY-URL + valueFrom: + configMapKeyRef: + name: susql-config + key: CARBON-INTENSITY-URL + optional: true + - name: CARBON-LOCATION + valueFrom: + configMapKeyRef: + name: susql-config + key: CARBON-LOCATION + optional: true + - name: CARBON-QUERY-RATE + valueFrom: + configMapKeyRef: + name: susql-config + key: CARBON-QUERY-RATE + optional: true + - name: CARBON-QUERY-FILTER + valueFrom: + configMapKeyRef: + name: susql-config + key: CARBON-QUERY-FILTER + optional: true + - name: CARBON-QUERY-CONV-2J + valueFrom: + configMapKeyRef: + name: susql-config + key: CARBON-QUERY-CONV-2J optional: true image: '' imagePullPolicy: IfNotPresent diff --git a/config/manifests/bases/susql-operator.clusterserviceversion.yaml b/config/manifests/bases/susql-operator.clusterserviceversion.yaml index 3eea457..c1c54a9 100644 --- a/config/manifests/bases/susql-operator.clusterserviceversion.yaml +++ b/config/manifests/bases/susql-operator.clusterserviceversion.yaml @@ -97,6 +97,8 @@ spec: - energy - kepler - susql + - carbon dioxide (CO2) emissions + - green computing links: - name: SusQL Operator url: https://github.com/sustainable-computing-io/susql-operator diff --git a/deployment/deploy.sh b/deployment/deploy.sh index 17e4907..074e801 100644 --- a/deployment/deploy.sh +++ b/deployment/deploy.sh @@ -79,10 +79,33 @@ if [[ -z ${SUSQL_LOG_LEVEL} ]]; then SUSQL_LOG_LEVEL="-5" fi -if [[ -z ${STATIC_CARBON_INTENSITY} ]]; then - STATIC_CARBON_INTENSITY="0.00000000011583333" +if [[ -z ${CARBON_METHOD} ]]; then + CARBON_METHOD="static" fi +if [[ -z ${CARBON_INTENSITY} ]]; then + CARBON_INTENSITY="0.0001158333333333" +fi + +if [[ -z ${CARBON_INTENSITY_URL} ]]; then + CARBON_INTENSITY_URL="https://api.electricitymap.org/v3/carbon-intensity/latest?zone=%s" +fi + +if [[ -z ${CARBON_LOCATION} ]]; then + CARBON_LOCATION="JP-TK" +fi + +if [[ -z ${CARBON_QUERY_RATE} ]]; then + CARBON_QUERY_RATE="7200" +fi + +if [[ -z ${CARBON_QUERY_FILTER} ]]; then + CARBON_QUERY_FILTER="carbonIntensity" +fi + +if [[ -z ${CARBON_QAUERY_CONV_2J} ]]; then + CARBON_QAUERY_CONV_2J="0.0000002777777778" +fi # Check if namespace exists if [[ -z $(kubectl get namespaces --no-headers -o custom-columns=':{.metadata.name}' | grep ${SUSQL_NAMESPACE}) ]]; then @@ -115,7 +138,13 @@ echo "PROMETHEUS_DOMAIN - '${PROMETHEUS_DOMAIN}'" echo "PROMETHEUS_PORT - '${PROMETHEUS_PORT}'" echo "KEPLER_PROMETHEUS_URL - '${KEPLER_PROMETHEUS_URL}'" echo "KEPLER_METRIC_NAME - '${KEPLER_METRIC_NAME}'" -echo "STATIC_CARBON_INTENSITY - '${STATIC_CARBON_INTENSITY}'" +echo "CARBON_METHOD - '${CARBON_METHOD}'" +echo "CARBON_INTENSITY - '${CARBON_INTENSITY}'" +echo "CARBON_INTENSITY_URL - '${CARBON_INTENSITY_URL}'" +echo "CARBON_LOCATION - '${CARBON_LOCATION}'" +echo "CARBON_QUERY_RATE - '${CARBON_QUERY_RATE}'" +echo "CARBON_QUERY_FILTER - '${CARBON_QUERY_FILTER}'" +echo "CARBON_QUERY_CONV_2J - '${CARBON_QUERY_CONV_2J}'" echo "SUSQL_PROMETHEUS_URL - '${SUSQL_PROMETHEUS_URL}'" echo "SUSQL_SAMPLING_RATE - '${SUSQL_SAMPLING_RATE}'" echo "SUSQL_ENHANCED - '${SUSQL_ENHANCED}'" @@ -146,7 +175,13 @@ echo "export PROMETHEUS_DOMAIN=${PROMETHEUS_DOMAIN}" >> ${LOGFILE} echo "export PROMETHEUS_PORT=${PROMETHEUS_PORT}" >> ${LOGFILE} echo "export KEPLER_PROMETHEUS_URL=${KEPLER_PROMETHEUS_URL}" >> ${LOGFILE} echo "export KEPLER_METRIC_NAME=${KEPLER_METRIC_NAME}" >> ${LOGFILE} -echo "export STATIC_CARBON_INTENSITY=${STATIC_CARBON_INTENSITY}" >> ${LOGFILE} +echo "export CARBON_METHOD=${CARBON_METHOD}" >> ${LOGFILE} +echo "export CARBON_INTENSITY=${CARBON_INTENSITY}" >> ${LOGFILE} +echo "export CARBON_INTENSITY_URL=${CARBON_INTENSITY_URL}" >> ${LOGFILE} +echo "export CARBON_LOCATION=${CARBON_LOCATION}" >> ${LOGFILE} +echo "export CARBON_QUERY_RATE=${CARBON_QUERY_RATE}" >> ${LOGFILE} +echo "export CARBON_QUERY_FILTER=${CARBON_QUERY_FILTER}" >> ${LOGFILE} +echo "export CARBON_QUERY_CONV_2J=${CARBON_QUERY_CONV_2J}" >> ${LOGFILE} echo "export SUSQL_PROMETHEUS_URL=${SUSQL_PROMETHEUS_URL}" >> ${LOGFILE} echo "export SUSQL_SAMPLING_RATE=${SUSQL_SAMPLING_RATE}" >> ${LOGFILE} echo "export SUSQL_ENHANCED=${SUSQL_ENHANCED}" >> ${LOGFILE} @@ -240,7 +275,13 @@ do --set keplerMetricName="${KEPLER_METRIC_NAME}" \ --set susqlPrometheusDatabaseUrl="${SUSQL_PROMETHEUS_URL}" \ --set susqlPrometheusMetricsUrl="http://0.0.0.0:8082" \ - --set staticCarbonIntensity="${STATIC_CARBON_INTENSITY}" \ + --set CarbonMethod="${CARBON_METHOD}" \ + --set CarbonIntensity="${CARBON_INTENSITY}" \ + --set CarbonIntensityUrl="${CARBON_INTENSITY_URL}" \ + --set CarbonLocation="${CARBON_LOCATION}" \ + --set CarbonQueryRate="${CARBON_QUERY_RATE}" \ + --set CarbonQueryFilter="${CARBON_QUERY_FILTER}" \ + --set CarbonQueryConv2J="${CARBON_QUERY_CONV_2J}" \ --set samplingRate="${SUSQL_SAMPLING_RATE}" \ --set susqlLogLevel="${SUSQL_LOG_LEVEL}" \ --set imagePullPolicy="Always" \ diff --git a/deployment/susql-controller/templates/deployment.yaml b/deployment/susql-controller/templates/deployment.yaml index 5a81862..241ae19 100644 --- a/deployment/susql-controller/templates/deployment.yaml +++ b/deployment/susql-controller/templates/deployment.yaml @@ -30,7 +30,13 @@ spec: - "--susql-prometheus-metrics-url={{ .Values.susqlPrometheusMetricsUrl }}" - "--susql-log-level={{ .Values.susqlLogLevel }}" - "--sampling-rate={{ .Values.samplingRate }}" - - "--static-carbon-intensity={{ .Values.staticCarbonIntensity }}" + - "--carbon-method={{ .Values.carbonMethod }}" + - "--carbon-intensity={{ .Values.carbonIntensity }}" + - "--carbon-intensity-url={{ .Values.carbonIntensityUrl }}" + - "--carbon-location={{ .Values.carbonLocation }}" + - "--carbon-query-rate={{ .Values.carbonQueryRate }}" + - "--carbon-query-filter={{ .Values.carbonQueryFilter }}" + - "--carbon-query-conv-2j={{ .Values.carbonQueryConv2J }}" - "--health-prove-bind-address={{ .Values.healthProbeAddr }}" - "--leader-elect={{ .Values.leaderElect }}" ports: diff --git a/deployment/susql-controller/values.yaml b/deployment/susql-controller/values.yaml index be307c4..fa66bc9 100644 --- a/deployment/susql-controller/values.yaml +++ b/deployment/susql-controller/values.yaml @@ -27,4 +27,10 @@ samplingRate: "2" healthProbeAddr: ":8081" leaderElect: "true" susqlLogLevel: "-5" -staticCarbonIntensity: "0.00000000011583333" +carbonMethod: "static" +carbonIntensity: "0.0001158333333333" +carbonIntensityUrl: "https://api.electricitymap.org/v3/carbon-intensity/latest?zone=%s" +carbonLocation: "JP-TK" +carbonQueryRate: "3600" +carbonQueryFilter: "carbonIntensity" +carbonQueryConv2J: "0.0000002777777778" \ No newline at end of file diff --git a/doc/helm-installation.md b/doc/helm-installation.md index 9b92ac8..21198a7 100644 --- a/doc/helm-installation.md +++ b/doc/helm-installation.md @@ -65,12 +65,12 @@ The following environment variables will influence the way that the SusQL Operat | SUSQL_REGISTRY | quay.io/sustainable_computing_io | Container registry that SusQL is stored in | | SUSQL_IMAGE_NAME | susql_operator | Image name used on SusQL container registry | | SUSQL_IMAGE_TAG | latest | Tag for SusQL container | -| STATIC_CARBON_INTENSITY | "0.00000000011583333" | Static carbon intensity in grams CO2 / Joule | +| CARBON_INTENSITY | "0.00011583333" | Carbon intensity in grams CO2 / Joule | ## License -Copyright 2024. +Copyright 2023, 2024. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/doc/operatorhub-installation.md b/doc/operatorhub-installation.md index cd017ef..3d810b5 100644 --- a/doc/operatorhub-installation.md +++ b/doc/operatorhub-installation.md @@ -52,7 +52,7 @@ enable the changes. (e.g., Delete the pod, and allow it to be recreated automati ## License -Copyright 2024. +Copyright 2023, 2024. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/doc/simplefaq.md b/doc/simplefaq.md index 19b9c41..85451b8 100644 --- a/doc/simplefaq.md +++ b/doc/simplefaq.md @@ -1,7 +1,7 @@ # Simple SusQL FAQ - What is SusQL? - - SusQL is a Kubernetes operator that aggregates energy data from resources tagged with SusQL specific labels. + - SusQL is a Kubernetes operator that aggregates energy and CO2 emission data from resources tagged with SusQL specific labels. - What does SusQL require? @@ -29,7 +29,7 @@ https://github.com/sustainable-computing-io/susql-operator/blob/main/doc/helm-in - Who uses SusQL? - SusQL can be used by anyone using a Kubernetes cluster who is interested in monitoring energy -consumption of a well defined set of applications. +consumption or carbon dioxide emission of a well defined set of applications. - Who works on SusQL? @@ -61,11 +61,11 @@ spec: - Why is SusQL important? - - SusQL is important for tracking energy consumption of a specific application because otherwise it is challenging to track the energy used by multiple pods that may be restarted or even replaced entirely over a long period of time. + - SusQL is important for tracking both energy consumption and carbon dioxide emission of a specific application because otherwise it is challenging to track the energy used by multiple pods that may be restarted or even replaced entirely over a long period of time. - What can I do with SusQL? - - Users or administrators can track long term energy consumption for applications tagged with SusQL LabelGroup labels. + - Users or administrators can track long term energy consumption and carbon dioxide emission for applications tagged with SusQL LabelGroup labels. - How can I visualize SusQL data? diff --git a/doc/susql-config.yaml b/doc/susql-config.yaml index 7a5cbbc..e921316 100644 --- a/doc/susql-config.yaml +++ b/doc/susql-config.yaml @@ -11,4 +11,10 @@ data: LEADER-ELECT: "true" HEALTH-PROBE-BIND-ADDRESS: ":8081" SUSQL-LOG-LEVEL: "-5" - STATIC-CARBON-INTENSITY: "0.00000000011583333" + CARBON-METHOD: "static" + CARBON-INTENSITY: "0.0001158333333333" + CARBON-INTENSITY-URL: "https://api.electricitymap.org/v3/carbon-intensity/latest?zone=%s" + CARBON-LOCATION: "JP-TK" + CARBON-QUERY-RATE: "7200" + CARBON-QUERY-FILTER: "carbonIntensity" + CARBON-QUERY-CONV-2J: "0.0000002777777778" \ No newline at end of file diff --git a/go.mod b/go.mod index d27f964..2d640a4 100644 --- a/go.mod +++ b/go.mod @@ -8,6 +8,7 @@ require ( github.com/onsi/gomega v1.31.0 github.com/prometheus/client_golang v1.18.0 github.com/prometheus/common v0.45.0 + github.com/tidwall/gjson v1.17.3 go.uber.org/zap v1.26.0 k8s.io/api v0.29.4 k8s.io/apimachinery v0.29.4 @@ -49,6 +50,8 @@ require ( github.com/prometheus/client_model v0.5.0 // indirect github.com/prometheus/procfs v0.12.0 // indirect github.com/spf13/pflag v1.0.5 // indirect + github.com/tidwall/match v1.1.1 // indirect + github.com/tidwall/pretty v1.2.0 // indirect go.uber.org/multierr v1.11.0 // indirect golang.org/x/exp v0.0.0-20220722155223-a9213eeb770e // indirect golang.org/x/net v0.23.0 // indirect diff --git a/go.sum b/go.sum index bf2b9db..6e86e39 100644 --- a/go.sum +++ b/go.sum @@ -109,6 +109,12 @@ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/tidwall/gjson v1.17.3 h1:bwWLZU7icoKRG+C+0PNwIKC6FCJO/Q3p2pZvuP0jN94= +github.com/tidwall/gjson v1.17.3/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= +github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA= +github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM= +github.com/tidwall/pretty v1.2.0 h1:RWIZEg2iJ8/g6fDDYzMpobmaoGh5OLl4AXtGUGPcqCs= +github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= diff --git a/internal/controller/labelgroup_controller.go b/internal/controller/labelgroup_controller.go index 3e56ca2..bdc0f03 100644 --- a/internal/controller/labelgroup_controller.go +++ b/internal/controller/labelgroup_controller.go @@ -1,5 +1,5 @@ /* -Copyright 2024. +Copyright 2023, 2024. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -42,7 +42,14 @@ type LabelGroupReconciler struct { SusQLPrometheusDatabaseUrl string SusQLPrometheusMetricsUrl string SamplingRate time.Duration // Sampling rate for all label groups - StaticCarbonIntensity float64 + CarbonMethod string + CarbonIntensity float64 + CarbonIntensityUrl string + CarbonIntensityTimeStamp int64 + CarbonLocation string + CarbonQueryRate int64 + CarbonQueryFilter string + CarbonQueryConv2J float64 Logger logr.Logger } @@ -70,7 +77,7 @@ var ( // the user. // // For more details, check Reconcile and its Result here: -// - https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.15.0/pkg/reconcile +// - https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.17.3/pkg/reconcile func (r *LabelGroupReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { _ = log.FromContext(ctx) @@ -103,6 +110,23 @@ func (r *LabelGroupReconciler) Reconcile(ctx context.Context, req ctrl.Request) return ctrl.Result{}, nil } + // Is it time to update the Carbon Intensity value? + // TODO: put this code only in Reloading and Aggregating cases + if r.CarbonMethod == "simpledynamic" { + currentEpoch := time.Now().Unix() + if (currentEpoch - r.CarbonIntensityTimeStamp) > r.CarbonQueryRate { + newCarbonIntensity, err := queryCarbonIntensity(r.CarbonIntensityUrl, r.CarbonLocation, r.CarbonQueryFilter, r.CarbonQueryConv2J) + if err == nil { + r.CarbonIntensity = newCarbonIntensity + r.CarbonIntensityTimeStamp = currentEpoch + r.Logger.V(5).Info("[Reconcile] Entered initializing case.") + r.Logger.V(5).Info(fmt.Sprintf("[Reconcile] Obtained dynamic carbon intensity of %.10f.", newCarbonIntensity)) + } else { + r.Logger.V(0).Error(err, "[Reconcile] Unable to query carbon intensity.") + } + } + } + // Decide what action to take based on the state of the labelGroup switch labelGroup.Status.Phase { case susqlv1.Initializing: @@ -169,7 +193,7 @@ func (r *LabelGroupReconciler) Reconcile(ctx context.Context, req ctrl.Request) labelGroup.Status.TotalEnergy = fmt.Sprintf("%f", totalEnergy) - labelGroup.Status.TotalCarbon = fmt.Sprintf("%.15f", float64(totalEnergy)*r.StaticCarbonIntensity) + labelGroup.Status.TotalCarbon = fmt.Sprintf("%.10f", float64(totalEnergy)*r.CarbonIntensity) } labelGroup.Status.Phase = susqlv1.Aggregating @@ -249,7 +273,7 @@ func (r *LabelGroupReconciler) Reconcile(ctx context.Context, req ctrl.Request) // 4) Update ETCD with the values labelGroup.Status.TotalEnergy = fmt.Sprintf("%.2f", totalEnergy) - labelGroup.Status.TotalCarbon = fmt.Sprintf("%.15f", float64(totalEnergy)*r.StaticCarbonIntensity) + labelGroup.Status.TotalCarbon = fmt.Sprintf("%.10f", float64(totalEnergy)*r.CarbonIntensity) if err := r.Status().Update(ctx, labelGroup); err != nil { return ctrl.Result{}, err @@ -257,7 +281,7 @@ func (r *LabelGroupReconciler) Reconcile(ctx context.Context, req ctrl.Request) // 5) Add energy aggregation to Prometheus table r.SetAggregatedEnergyForLabels(totalEnergy, labelGroup.Status.PrometheusLabels) - r.SetAggregatedCarbonForLabels(float64(totalEnergy)*r.StaticCarbonIntensity, labelGroup.Status.PrometheusLabels) + r.SetAggregatedCarbonForLabels(float64(totalEnergy)*r.CarbonIntensity, labelGroup.Status.PrometheusLabels) // Requeue return ctrl.Result{RequeueAfter: r.SamplingRate}, nil diff --git a/internal/controller/prometheus_manager.go b/internal/controller/prometheus_manager.go index 2fbe26a..9d65163 100644 --- a/internal/controller/prometheus_manager.go +++ b/internal/controller/prometheus_manager.go @@ -1,5 +1,5 @@ /* -Copyright 2023. +Copyright 2023, 2024. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/internal/controller/resource_manager.go b/internal/controller/resource_manager.go index 6987b3b..dfe1d15 100644 --- a/internal/controller/resource_manager.go +++ b/internal/controller/resource_manager.go @@ -1,5 +1,5 @@ /* -Copyright 2023. +Copyright 2023, 2024. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/internal/controller/simple_dynamic_query.go b/internal/controller/simple_dynamic_query.go new file mode 100644 index 0000000..cea4886 --- /dev/null +++ b/internal/controller/simple_dynamic_query.go @@ -0,0 +1,48 @@ +/* +Copyright 2023, 2024. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controller + +import ( + "fmt" + "github.com/tidwall/gjson" + "io/ioutil" + "net/http" + "strconv" +) + +func queryCarbonIntensity(url string, location string, filter string, conv2J float64) (float64, error) { + + response, err := http.Get(fmt.Sprintf(url, location)) + if err != nil { + return 0.0, err + } + + responseData, err := ioutil.ReadAll(response.Body) + if err != nil { + return 0.0, err + } + + carbonIntensityString := gjson.Get(string(responseData), filter).String() + + carbonIntensityFloat, err := strconv.ParseFloat(carbonIntensityString, 64) + if err != nil { + return 0.0, err + } + + // return nil error since no error + return carbonIntensityFloat * conv2J, nil +} diff --git a/internal/controller/suite_test.go b/internal/controller/suite_test.go index 43b4929..310042c 100644 --- a/internal/controller/suite_test.go +++ b/internal/controller/suite_test.go @@ -1,5 +1,5 @@ /* -Copyright 2024. +Copyright 2023, 2024. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License.