build(deps): update opentelemetry

.lando.yml

@@ -106,7 +106,7 @@ services:
     type: compose
     scanner: false
     services:
-      image: jaegertracing/all-in-one:1.22
+      image: jaegertracing/all-in-one:1.42
       command: /go/bin/all-in-one-linux --sampling.strategies-file=/etc/jaeger/sampling_strategies.json
       healthcheck:
         test: wget -nv -t1 --spider localhost:14269/ || exit 1
@@ -117,7 +117,7 @@ services:
     type: compose
     scanner: false
     services:
-      image: prom/prometheus:v2.26.0
+      image: prom/prometheus:v2.33.4
       command: /bin/prometheus --config.file=/prometheus.yaml --web.enable-lifecycle
       volumes:
         - ./otel/prometheus.yaml:/prometheus.yaml:ro

.local.env

@@ -36,6 +36,7 @@ PIPELINE_TYPE=local # local | github | gitlab
 PIPELINE_URI=http://pipeline.cube-creator.lndo.site
 OTEL_EXPORTER_OTLP_TRACES_ENDPOINT=http://otel.cube-creator.lndo.site/v1/traces
 OTEL_EXPORTER_OTLP_METRICS_ENDPOINT=http://otel.cube-creator.lndo.site/v1/metrics
+OTEL_METRICS_EXPORTER=otlp
 
 # Shared Dimensions
 MANAGED_DIMENSIONS_GRAPH=https://lindas.admin.ch/cube/dimension

cli/lib/bufferDebug.ts

@@ -1,6 +1,5 @@
 import stream from 'readable-stream'
 import type Pipeline from 'barnard59-core/lib/Pipeline'
-import type { BoundBaseObserver } from '@opentelemetry/api-metrics'
 import { bufferObserver } from './otel/metrics'
 
 const { finished } = stream as any
@@ -54,19 +53,6 @@ function bufferDebug(pipeline: Pipeline, jobUri: string, { interval = 10 } = {})
   })
 
   const pid = process.pid.toString()
-  const boundMeters = new Map<string, BoundBaseObserver>()
-  function getMeter(step: string) {
-    const bound = boundMeters.get(step) || bufferObserver.bind({
-      job_uri: jobUri,
-      pipeline: pipeline.ptr.value,
-      step,
-      pid,
-    })
-
-    boundMeters.set(step, bound)
-
-    return bound
-  }
 
   const next = async () => {
     if (done) {
@@ -77,7 +63,12 @@ function bufferDebug(pipeline: Pipeline, jobUri: string, { interval = 10 } = {})
 
     if (data) {
       [...Object.entries(data)].forEach(([step, value]) => {
-        getMeter(step).update(value)
+        bufferObserver.record(value, {
+          job_uri: jobUri,
+          pipeline: pipeline.ptr.value,
+          step,
+          pid,
+        })
       })
     }
 

cli/lib/counters.ts

@@ -7,16 +7,15 @@ import { quadCounter } from './otel/metrics'
 
 export function quads(this: Context, name: string) {
   const jobUri = this.variables.get('jobUri')
-  const bound = quadCounter.bind({
-    name,
-    job_id: jobUri.substr(jobUri.lastIndexOf('/') + 1),
-  })
   const forwarder = new PassThrough({
     objectMode: true,
   })
 
   forwarder.on('data', () => {
-    bound.add(1)
+    quadCounter.add(1, {
+      name,
+      job_id: jobUri.substr(jobUri.lastIndexOf('/') + 1),
+    })
   })
 
   return forwarder

cli/lib/otel/index.ts

@@ -1,15 +1,12 @@
-import { CollectorTraceExporter, CollectorMetricExporter } from '@opentelemetry/exporter-collector'
 import { HttpInstrumentation } from '@opentelemetry/instrumentation-http'
 import { WinstonInstrumentation } from '@opentelemetry/instrumentation-winston'
-import { Resource, envDetector, processDetector } from '@opentelemetry/resources'
+import { Resource } from '@opentelemetry/resources'
 import { NodeSDK } from '@opentelemetry/sdk-node'
 import { SemanticResourceAttributes } from '@opentelemetry/semantic-conventions'
 
 const sdk = new NodeSDK({
   // Automatic detection is disabled, see comment below
   autoDetectResources: false,
-  traceExporter: new CollectorTraceExporter(),
-  metricExporter: new CollectorMetricExporter(),
   instrumentations: [
     new HttpInstrumentation(),
     new WinstonInstrumentation(),
@@ -35,7 +32,7 @@ const onError = async (err: Error) => {
 
 export async function opentelemetry() {
   try {
-    await sdk.detectResources({ detectors: [envDetector, processDetector] })
+    await sdk.detectResources()
 
     await sdk.start()
 

cli/lib/otel/metrics.ts

@@ -1,7 +1,19 @@
-import { metrics } from '@opentelemetry/api-metrics'
+import { MeterProvider, PeriodicExportingMetricReader } from '@opentelemetry/sdk-metrics'
+import { OTLPMetricExporter } from '@opentelemetry/exporter-metrics-otlp-http'
+
+const otlpMetricExporter = new OTLPMetricExporter()
+
+const metrics = new MeterProvider()
 
 const meter = metrics.getMeter('@cube-creator/cli')
 
 export const quadCounter = meter.createCounter('quads')
 
-export const bufferObserver = meter.createValueObserver('buffer')
+export const bufferObserver = meter.createHistogram('buffer')
+
+export const metricReader = new PeriodicExportingMetricReader({
+  exporter: otlpMetricExporter,
+  exportIntervalMillis: 1000,
+})
+
+metrics.addMetricReader(metricReader)

cli/package.json

@@ -13,15 +13,15 @@
   "dependencies": {
     "@cube-creator/core": "1.0.0",
     "@cube-creator/model": "0.1.27",
-    "@opentelemetry/api": "^1.0.2",
-    "@opentelemetry/api-metrics": "^0.24.0",
-    "@opentelemetry/exporter-collector": "^0.24.0",
-    "@opentelemetry/instrumentation-http": "^0.24.0",
-    "@opentelemetry/instrumentation-winston": "^0.24.0",
+    "@opentelemetry/api": "^1.4.0",
+    "@opentelemetry/exporter-metrics-otlp-http": "^0.35.1",
+    "@opentelemetry/instrumentation-http": "^0.35.1",
+    "@opentelemetry/instrumentation-winston": "^0.31.1",
     "@opentelemetry/metrics": "^0.24.0",
-    "@opentelemetry/resources": "^0.24.0",
-    "@opentelemetry/sdk-node": "^0.24.0",
-    "@opentelemetry/semantic-conventions": "^0.24.0",
+    "@opentelemetry/resources": "^0.26.0",
+    "@opentelemetry/sdk-metrics": "^1.9.1",
+    "@opentelemetry/sdk-node": "^0.35.1",
+    "@opentelemetry/semantic-conventions": "^0.26.0",
     "@opentelemetry/tracing": "^0.24.0",
     "@rdfine/csvw": "^0.6.3",
     "@rdfine/prov": "^0.1.1",

cli/server.ts

@@ -8,6 +8,7 @@ import cors from 'cors'
 import bodyParser from 'body-parser'
 import dotenv from 'dotenv'
 import * as command from './lib/commands'
+import { opentelemetry } from './lib/otel/index'
 
 dotenv.config({
   path: path.resolve(__dirname, '.env'),
@@ -16,7 +17,11 @@ dotenv.config({
   path: path.resolve(__dirname, '.test.env'),
 })
 
+let shutdownOtel: () => Promise<void> | undefined
+
 async function main() {
+  shutdownOtel = await opentelemetry()
+
   const log = debug('cube-creator')
 
   const app = express()
@@ -61,4 +66,6 @@ async function main() {
   http.createServer(app).listen(80, () => log('Api ready'))
 }
 
-main()
+main().finally(() => {
+  shutdownOtel()
+})

otel/.Dockerfile

@@ -1,4 +1,4 @@
-FROM otel/opentelemetry-collector:0.26.0 as otel
+FROM otel/opentelemetry-collector:0.71.0 as otel
 FROM alpine:3.13
 
 COPY --from=otel / /otel

otel/collector.yaml

@@ -7,7 +7,8 @@ receivers:
 exporters:
   jaeger:
     endpoint: jaeger:14250
-    insecure: true
+    tls:
+      insecure: true
   prometheus:
     endpoint: "0.0.0.0:8889"
     resource_to_telemetry_conversion: