chore(cli): fix export of metrics and traces

.local.env

@@ -37,6 +37,7 @@ PIPELINE_URI=http://pipeline.cube-creator.lndo.site
 OTEL_EXPORTER_OTLP_TRACES_ENDPOINT=http://otel.cube-creator.lndo.site/v1/traces
 OTEL_EXPORTER_OTLP_METRICS_ENDPOINT=http://otel.cube-creator.lndo.site/v1/metrics
 OTEL_METRICS_EXPORTER=otlp
+OTEL_LOG_LEVEL=debug
 
 # Shared Dimensions
 MANAGED_DIMENSIONS_GRAPH=https://lindas.admin.ch/cube/dimension

cli/lib/bufferDebug.ts

@@ -1,9 +1,13 @@
 import stream from 'readable-stream'
 import type Pipeline from 'barnard59-core/lib/Pipeline'
-import { bufferObserver } from './otel/metrics'
+import { metrics } from '@opentelemetry/api'
 
 const { finished } = stream as any
 
+const meter = metrics.getMeter('@cube-creator/cli')
+
+export const bufferObserver = meter.createHistogram('buffer')
+
 function bufferStatePair({ state, step }: any): { key: string; value: number } {
   const key = step.ptr.value // `[${index}] (${mode}) ${step.ptr.value} (${state.length}/${state.highWaterMark})`
   const value = state.length > 0 ? Math.round(state.length / state.highWaterMark * 100.0) : 0

cli/lib/counters.ts

@@ -3,7 +3,11 @@ import type { Context } from 'barnard59-core/lib/Pipeline'
 import { PassThrough } from 'readable-stream'
 import through2 from 'through2'
 import { csvw } from '@tpluscode/rdf-ns-builders/strict'
-import { quadCounter } from './otel/metrics'
+import { metrics } from '@opentelemetry/api'
+
+const meter = metrics.getMeter('@cube-creator/cli')
+
+export const quadCounter = meter.createCounter('quads')
 
 export function quads(this: Context, name: string) {
   const jobUri = this.variables.get('jobUri')

cli/lib/otel/index.ts

@@ -1,12 +1,23 @@
+import { NodeSDK } from '@opentelemetry/sdk-node'
+import { Resource } from '@opentelemetry/resources'
 import { HttpInstrumentation } from '@opentelemetry/instrumentation-http'
 import { WinstonInstrumentation } from '@opentelemetry/instrumentation-winston'
-import { Resource } from '@opentelemetry/resources'
-import { NodeSDK } from '@opentelemetry/sdk-node'
 import { SemanticResourceAttributes } from '@opentelemetry/semantic-conventions'
+import { PeriodicExportingMetricReader } from '@opentelemetry/sdk-metrics'
+import { OTLPMetricExporter } from '@opentelemetry/exporter-metrics-otlp-http'
+import { diag, DiagConsoleLogger, DiagLogLevel } from '@opentelemetry/api'
+
+diag.setLogger(new DiagConsoleLogger(), DiagLogLevel.ERROR)
+
+export const metricReader = new PeriodicExportingMetricReader({
+  exporter: new OTLPMetricExporter(),
+  exportIntervalMillis: 1000,
+})
 
 const sdk = new NodeSDK({
   // Automatic detection is disabled, see comment below
   autoDetectResources: false,
+  metricReader,
   instrumentations: [
     new HttpInstrumentation(),
     new WinstonInstrumentation(),

cli/server.ts

@@ -1,3 +1,6 @@
+// eslint-disable-next-line import/order
+import { opentelemetry } from './lib/otel/index'
+
 /* eslint-disable import/no-extraneous-dependencies */
 import * as http from 'http'
 import * as path from 'path'
@@ -8,7 +11,6 @@ import cors from 'cors'
 import bodyParser from 'body-parser'
 import dotenv from 'dotenv'
 import * as command from './lib/commands'
-import { opentelemetry } from './lib/otel/index'
 
 dotenv.config({
   path: path.resolve(__dirname, '.env'),
@@ -17,10 +19,8 @@ dotenv.config({
   path: path.resolve(__dirname, '.test.env'),
 })
 
-let shutdownOtel: () => Promise<void> | undefined
-
 async function main() {
-  shutdownOtel = await opentelemetry()
+  await opentelemetry()
 
   const log = debug('cube-creator')
 
@@ -66,6 +66,4 @@ async function main() {
   http.createServer(app).listen(80, () => log('Api ready'))
 }
 
-main().finally(() => {
-  shutdownOtel()
-})
+main()

otel/collector.yaml

@@ -2,7 +2,10 @@ receivers:
   otlp:
     protocols:
       grpc:
+        endpoint: 0.0.0.0:4317
+        max_recv_msg_size_mib: 100
       http:
+        endpoint: 0.0.0.0:55681
 
 exporters:
   jaeger:
@@ -13,6 +16,7 @@ exporters:
     endpoint: "0.0.0.0:8889"
     resource_to_telemetry_conversion:
       enabled: true
+  logging:
 
 processors:
   batch:
@@ -26,4 +30,4 @@ service:
     metrics:
       receivers: [otlp]
       processors: [batch]
-      exporters: [prometheus]
+      exporters: [prometheus, logging]