build(deps): update opentelemetry

.changeset/poor-hotels-unite.md

@@ -0,0 +1,5 @@
+---
+"@cube-creator/cli": patch
+---
+
+Pipeline traces were not showing in jeager

.lando.yml

@@ -57,7 +57,7 @@ services:
     app_mount: false
     ssl: true
     services:
-      image: zazuko/trifid
+      image: ghcr.io/zazuko/trifid:v2
       user: root
       command: tini -- /app/server.js --config=/config-sparql.json --sparql-endpoint-url=http://admin:[email protected]/cube-creator
       environment:
@@ -106,7 +106,7 @@ services:
     type: compose
     scanner: false
     services:
-      image: jaegertracing/all-in-one:1.22
+      image: jaegertracing/all-in-one:1.42
       command: /go/bin/all-in-one-linux --sampling.strategies-file=/etc/jaeger/sampling_strategies.json
       healthcheck:
         test: wget -nv -t1 --spider localhost:14269/ || exit 1
@@ -117,7 +117,7 @@ services:
     type: compose
     scanner: false
     services:
-      image: prom/prometheus:v2.26.0
+      image: prom/prometheus:v2.33.4
       command: /bin/prometheus --config.file=/prometheus.yaml --web.enable-lifecycle
       volumes:
         - ./otel/prometheus.yaml:/prometheus.yaml:ro

.local.env

@@ -36,6 +36,8 @@ PIPELINE_TYPE=local # local | github | gitlab
 PIPELINE_URI=http://pipeline.cube-creator.lndo.site
 OTEL_EXPORTER_OTLP_TRACES_ENDPOINT=http://otel.cube-creator.lndo.site/v1/traces
 OTEL_EXPORTER_OTLP_METRICS_ENDPOINT=http://otel.cube-creator.lndo.site/v1/metrics
+OTEL_METRICS_EXPORTER=otlp
+OTEL_LOG_LEVEL=debug
 
 # Shared Dimensions
 MANAGED_DIMENSIONS_GRAPH=https://lindas.admin.ch/cube/dimension

cli/lib/bufferDebug.ts

@@ -1,10 +1,13 @@
 import stream from 'readable-stream'
 import type Pipeline from 'barnard59-core/lib/Pipeline'
-import type { BoundBaseObserver } from '@opentelemetry/api-metrics'
-import { bufferObserver } from './otel/metrics'
+import { metrics } from '@opentelemetry/api'
 
 const { finished } = stream as any
 
+const meter = metrics.getMeter('@cube-creator/cli')
+
+export const bufferObserver = meter.createHistogram('buffer')
+
 function bufferStatePair({ state, step }: any): { key: string; value: number } {
   const key = step.ptr.value // `[${index}] (${mode}) ${step.ptr.value} (${state.length}/${state.highWaterMark})`
   const value = state.length > 0 ? Math.round(state.length / state.highWaterMark * 100.0) : 0
@@ -54,19 +57,6 @@ function bufferDebug(pipeline: Pipeline, jobUri: string, { interval = 10 } = {})
   })
 
   const pid = process.pid.toString()
-  const boundMeters = new Map<string, BoundBaseObserver>()
-  function getMeter(step: string) {
-    const bound = boundMeters.get(step) || bufferObserver.bind({
-      job_uri: jobUri,
-      pipeline: pipeline.ptr.value,
-      step,
-      pid,
-    })
-
-    boundMeters.set(step, bound)
-
-    return bound
-  }
 
   const next = async () => {
     if (done) {
@@ -77,7 +67,12 @@ function bufferDebug(pipeline: Pipeline, jobUri: string, { interval = 10 } = {})
 
     if (data) {
       [...Object.entries(data)].forEach(([step, value]) => {
-        getMeter(step).update(value)
+        bufferObserver.record(value, {
+          job_uri: jobUri,
+          pipeline: pipeline.ptr.value,
+          step,
+          pid,
+        })
       })
     }
 

cli/lib/counters.ts

@@ -3,20 +3,23 @@ import type { Context } from 'barnard59-core/lib/Pipeline'
 import { PassThrough } from 'readable-stream'
 import through2 from 'through2'
 import { csvw } from '@tpluscode/rdf-ns-builders/strict'
-import { quadCounter } from './otel/metrics'
+import { metrics } from '@opentelemetry/api'
+
+const meter = metrics.getMeter('@cube-creator/cli')
+
+export const quadCounter = meter.createCounter('quads')
 
 export function quads(this: Context, name: string) {
   const jobUri = this.variables.get('jobUri')
-  const bound = quadCounter.bind({
-    name,
-    job_id: jobUri.substr(jobUri.lastIndexOf('/') + 1),
-  })
   const forwarder = new PassThrough({
     objectMode: true,
   })
 
   forwarder.on('data', () => {
-    bound.add(1)
+    quadCounter.add(1, {
+      name,
+      job_id: jobUri.substr(jobUri.lastIndexOf('/') + 1),
+    })
   })
 
   return forwarder

cli/lib/otel/index.ts

@@ -1,15 +1,23 @@
-import { CollectorTraceExporter, CollectorMetricExporter } from '@opentelemetry/exporter-collector'
+import { NodeSDK } from '@opentelemetry/sdk-node'
+import { Resource } from '@opentelemetry/resources'
 import { HttpInstrumentation } from '@opentelemetry/instrumentation-http'
 import { WinstonInstrumentation } from '@opentelemetry/instrumentation-winston'
-import { Resource, envDetector, processDetector } from '@opentelemetry/resources'
-import { NodeSDK } from '@opentelemetry/sdk-node'
 import { SemanticResourceAttributes } from '@opentelemetry/semantic-conventions'
+import { PeriodicExportingMetricReader } from '@opentelemetry/sdk-metrics'
+import { OTLPMetricExporter } from '@opentelemetry/exporter-metrics-otlp-http'
+import { diag, DiagConsoleLogger, DiagLogLevel } from '@opentelemetry/api'
+
+diag.setLogger(new DiagConsoleLogger(), DiagLogLevel.ERROR)
+
+export const metricReader = new PeriodicExportingMetricReader({
+  exporter: new OTLPMetricExporter(),
+  exportIntervalMillis: 1000,
+})
 
 const sdk = new NodeSDK({
   // Automatic detection is disabled, see comment below
   autoDetectResources: false,
-  traceExporter: new CollectorTraceExporter(),
-  metricExporter: new CollectorMetricExporter(),
+  metricReader,
   instrumentations: [
     new HttpInstrumentation(),
     new WinstonInstrumentation(),
@@ -35,7 +43,7 @@ const onError = async (err: Error) => {
 
 export async function opentelemetry() {
   try {
-    await sdk.detectResources({ detectors: [envDetector, processDetector] })
+    await sdk.detectResources()
 
     await sdk.start()
 

cli/package.json

@@ -13,15 +13,15 @@
   "dependencies": {
     "@cube-creator/core": "1.0.0",
     "@cube-creator/model": "0.1.27",
-    "@opentelemetry/api": "^1.0.2",
-    "@opentelemetry/api-metrics": "^0.24.0",
-    "@opentelemetry/exporter-collector": "^0.24.0",
-    "@opentelemetry/instrumentation-http": "^0.24.0",
-    "@opentelemetry/instrumentation-winston": "^0.24.0",
+    "@opentelemetry/api": "^1.4.0",
+    "@opentelemetry/exporter-metrics-otlp-http": "^0.35.1",
+    "@opentelemetry/instrumentation-http": "^0.35.1",
+    "@opentelemetry/instrumentation-winston": "^0.31.1",
     "@opentelemetry/metrics": "^0.24.0",
-    "@opentelemetry/resources": "^0.24.0",
-    "@opentelemetry/sdk-node": "^0.24.0",
-    "@opentelemetry/semantic-conventions": "^0.24.0",
+    "@opentelemetry/resources": "^0.26.0",
+    "@opentelemetry/sdk-metrics": "^1.9.1",
+    "@opentelemetry/sdk-node": "^0.35.1",
+    "@opentelemetry/semantic-conventions": "^0.26.0",
     "@opentelemetry/tracing": "^0.24.0",
     "@rdfine/csvw": "^0.6.3",
     "@rdfine/prov": "^0.1.1",

cli/server.ts

@@ -1,3 +1,6 @@
+// eslint-disable-next-line import/order
+import { opentelemetry } from './lib/otel/index'
+
 /* eslint-disable import/no-extraneous-dependencies */
 import * as http from 'http'
 import * as path from 'path'
@@ -17,6 +20,8 @@ dotenv.config({
 })
 
 async function main() {
+  await opentelemetry()
+
   const log = debug('cube-creator')
 
   const app = express()

otel/.Dockerfile

@@ -1,4 +1,4 @@
-FROM otel/opentelemetry-collector:0.26.0 as otel
+FROM otel/opentelemetry-collector:0.71.0 as otel
 FROM alpine:3.13
 
 COPY --from=otel / /otel

otel/collector.yaml

@@ -2,16 +2,21 @@ receivers:
   otlp:
     protocols:
       grpc:
+        endpoint: 0.0.0.0:4317
+        max_recv_msg_size_mib: 100
       http:
+        endpoint: 0.0.0.0:55681
 
 exporters:
   jaeger:
     endpoint: jaeger:14250
-    insecure: true
+    tls:
+      insecure: true
   prometheus:
     endpoint: "0.0.0.0:8889"
     resource_to_telemetry_conversion:
       enabled: true
+  logging:
 
 processors:
   batch:
@@ -25,4 +30,4 @@ service:
     metrics:
       receivers: [otlp]
       processors: [batch]
-      exporters: [prometheus]
+      exporters: [prometheus, logging]