From e11f0f8fd96a3b0a5780b1848ec08f1cac17135b Mon Sep 17 00:00:00 2001 From: linglp Date: Tue, 2 Jul 2024 12:48:02 -0400 Subject: [PATCH 1/4] only trigger tracing when tracing export format is set --- schematic_api/api/routes.py | 83 ++++++++++++++++++++----------------- 1 file changed, 45 insertions(+), 38 deletions(-) diff --git a/schematic_api/api/routes.py b/schematic_api/api/routes.py index b2439133e..d9de09401 100644 --- a/schematic_api/api/routes.py +++ b/schematic_api/api/routes.py @@ -1,63 +1,55 @@ +import json +import logging import os -from json.decoder import JSONDecodeError +import pathlib +import pickle import shutil import tempfile -import shutil +import time import urllib.request -import logging -import pathlib -import pickle +from functools import wraps +from json.decoder import JSONDecodeError +from typing import Any, List, Optional import connexion +import pandas as pd from connexion.decorators.uri_parsing import Swagger2URIParser -from werkzeug.debug import DebuggedApplication - -from flask_cors import cross_origin -from flask import send_from_directory from flask import current_app as app -from flask import request - -import pandas as pd -import json -from typing import Optional, List, Any -from functools import wraps - +from flask import request, send_from_directory +from flask_cors import cross_origin from opentelemetry import trace -from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter from opentelemetry.sdk.resources import SERVICE_NAME, Resource +from opentelemetry.sdk.trace import TracerProvider from opentelemetry.sdk.trace.export import ( BatchSpanProcessor, ConsoleSpanExporter, + SimpleSpanProcessor, Span, ) -from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter - -from schematic.configuration.configuration import CONFIG -from schematic.visualization.attributes_explorer import AttributesExplorer -from schematic.visualization.tangled_tree import TangledTree -from schematic.manifest.generator import ManifestGenerator -from schematic.models.metadata import MetadataModel - -from schematic.schemas.data_model_parser import DataModelParser -from schematic.schemas.data_model_graph import DataModelGraph, DataModelGraphExplorer - -from schematic.store.synapse import SynapseStorage, ManifestDownload +from opentelemetry.sdk.trace.sampling import ALWAYS_OFF from synapseclient.core.exceptions import ( - SynapseHTTPError, SynapseAuthenticationError, - SynapseUnmetAccessRestrictions, + SynapseHTTPError, SynapseNoCredentialsError, SynapseTimeoutError, + SynapseUnmetAccessRestrictions, ) +from werkzeug.debug import DebuggedApplication + +from schematic.configuration.configuration import CONFIG +from schematic.manifest.generator import ManifestGenerator +from schematic.models.metadata import MetadataModel +from schematic.schemas.data_model_graph import DataModelGraph, DataModelGraphExplorer +from schematic.schemas.data_model_parser import DataModelParser +from schematic.store.synapse import ManifestDownload, SynapseStorage from schematic.utils.general import entity_type_mapping from schematic.utils.schema_utils import ( - get_property_label_from_display_name, DisplayLabelType, -) -from schematic.utils.schema_utils import ( get_property_label_from_display_name, - DisplayLabelType, ) +from schematic.visualization.attributes_explorer import AttributesExplorer +from schematic.visualization.tangled_tree import TangledTree logger = logging.getLogger(__name__) logging.basicConfig(level=logging.DEBUG) @@ -83,9 +75,24 @@ def export(self, spans: List[Span]) -> None: f.write(span_json_one_line) -trace.get_tracer_provider().add_span_processor(BatchSpanProcessor(OTLPSpanExporter())) -# processor = SimpleSpanProcessor(FileSpanExporter("otel_spans_schemati_api.json")) -# trace.get_tracer_provider().add_span_processor(processor) +def set_up_tracing(): + """Set up tracing for the API.""" + tracing_export = os.environ.get("TRACING_EXPORT", None) + if tracing_export == "otlp": + trace.get_tracer_provider().add_span_processor( + BatchSpanProcessor(OTLPSpanExporter()) + ) + elif tracing_export == "file": + timestamp_millis = int(time.time() * 1000) + file_name = f"otel_spans_integration_testing_{timestamp_millis}.ndjson" + file_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), file_name) + processor = SimpleSpanProcessor(FileSpanExporter(file_path)) + trace.get_tracer_provider().add_span_processor(processor) + else: + trace.set_tracer_provider(TracerProvider(sampler=ALWAYS_OFF)) + + +set_up_tracing() tracer = trace.get_tracer("Schematic") From cf61f008b109332b6425cc54bd52372e8e0861a7 Mon Sep 17 00:00:00 2001 From: linglp Date: Tue, 2 Jul 2024 12:49:02 -0400 Subject: [PATCH 2/4] change env variable name --- schematic_api/api/routes.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/schematic_api/api/routes.py b/schematic_api/api/routes.py index d9de09401..489813fd9 100644 --- a/schematic_api/api/routes.py +++ b/schematic_api/api/routes.py @@ -77,7 +77,7 @@ def export(self, spans: List[Span]) -> None: def set_up_tracing(): """Set up tracing for the API.""" - tracing_export = os.environ.get("TRACING_EXPORT", None) + tracing_export = os.environ.get("TRACING_EXPORT_FORMAT", None) if tracing_export == "otlp": trace.get_tracer_provider().add_span_processor( BatchSpanProcessor(OTLPSpanExporter()) From d4101a306cf3ad7d62a4517565242ad37af6e40f Mon Sep 17 00:00:00 2001 From: linglp Date: Tue, 2 Jul 2024 14:56:28 -0400 Subject: [PATCH 3/4] add return type --- schematic_api/api/routes.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/schematic_api/api/routes.py b/schematic_api/api/routes.py index 489813fd9..6c1f33d27 100644 --- a/schematic_api/api/routes.py +++ b/schematic_api/api/routes.py @@ -75,13 +75,14 @@ def export(self, spans: List[Span]) -> None: f.write(span_json_one_line) -def set_up_tracing(): +def set_up_tracing() -> None: """Set up tracing for the API.""" tracing_export = os.environ.get("TRACING_EXPORT_FORMAT", None) if tracing_export == "otlp": trace.get_tracer_provider().add_span_processor( BatchSpanProcessor(OTLPSpanExporter()) ) + print("going through here") elif tracing_export == "file": timestamp_millis = int(time.time() * 1000) file_name = f"otel_spans_integration_testing_{timestamp_millis}.ndjson" From b11af55da44cb62229854c628a383f51ce7d9990 Mon Sep 17 00:00:00 2001 From: linglp Date: Tue, 2 Jul 2024 16:20:34 -0400 Subject: [PATCH 4/4] remove print statement --- schematic_api/api/routes.py | 1 - 1 file changed, 1 deletion(-) diff --git a/schematic_api/api/routes.py b/schematic_api/api/routes.py index 6c1f33d27..ae73428eb 100644 --- a/schematic_api/api/routes.py +++ b/schematic_api/api/routes.py @@ -82,7 +82,6 @@ def set_up_tracing() -> None: trace.get_tracer_provider().add_span_processor( BatchSpanProcessor(OTLPSpanExporter()) ) - print("going through here") elif tracing_export == "file": timestamp_millis = int(time.time() * 1000) file_name = f"otel_spans_integration_testing_{timestamp_millis}.ndjson"