feat: supported otlp export for metrics and logs (epam#88)

aidial_sdk/telemetry/init.py

@@ -1,4 +1,11 @@
+import logging
+
 from fastapi import FastAPI
+from opentelemetry._logs import set_logger_provider
+from opentelemetry.exporter.otlp.proto.grpc._log_exporter import OTLPLogExporter
+from opentelemetry.exporter.otlp.proto.grpc.metric_exporter import (
+    OTLPMetricExporter,
+)
 from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import (
     OTLPSpanExporter,
 )
@@ -15,15 +22,19 @@
 )
 from opentelemetry.instrumentation.urllib import URLLibInstrumentor
 from opentelemetry.metrics import set_meter_provider
+from opentelemetry.sdk._logs import LoggerProvider, LoggingHandler
+from opentelemetry.sdk._logs.export import BatchLogRecordProcessor
 from opentelemetry.sdk.metrics import MeterProvider
+from opentelemetry.sdk.metrics._internal.export import (
+    PeriodicExportingMetricReader,
+)
 from opentelemetry.sdk.resources import SERVICE_NAME, Resource
 from opentelemetry.sdk.trace import TracerProvider
 from opentelemetry.sdk.trace.export import BatchSpanProcessor
 from opentelemetry.trace import set_tracer_provider
 from prometheus_client import start_http_server
 
 from aidial_sdk.telemetry.types import TelemetryConfig
-from aidial_sdk.utils.logging import logger
 
 
 def init_telemetry(
@@ -52,21 +63,43 @@ def init_telemetry(
         HTTPXClientInstrumentor().instrument()
 
         if config.tracing.logging:
-            LoggingInstrumentor().instrument(
-                set_logging_format=True,
-                log_level=logger.getEffectiveLevel(),
+            # Setting the root logger format in order to include
+            # tracing information: span_id, trace_id
+            LoggingInstrumentor().instrument(set_logging_format=True)
+
+    if config.logs is not None:
+        # Adding a handler to the root logger which exports the logs to OTLP
+        provider = LoggerProvider(resource=resource)
+
+        if config.logs.otlp_export:
+            provider.add_log_record_processor(
+                BatchLogRecordProcessor(OTLPLogExporter())
             )
 
+        set_logger_provider(provider)
+
+        handler = LoggingHandler(level=config.logs.level)
+        logging.getLogger().addHandler(handler)
+
     if config.metrics is not None:
-        set_meter_provider(
-            MeterProvider(
-                resource=resource, metric_readers=[PrometheusMetricReader()]
+        metric_readers = []
+
+        if config.metrics.prometheus_export:
+            metric_readers.append(PrometheusMetricReader())
+
+        if config.metrics.otlp_export:
+            metric_readers.append(
+                PeriodicExportingMetricReader(OTLPMetricExporter())
             )
+
+        set_meter_provider(
+            MeterProvider(resource=resource, metric_readers=metric_readers)
         )
 
         SystemMetricsInstrumentor().instrument()
 
-        start_http_server(port=config.metrics.port)
+        if config.metrics.prometheus_export:
+            start_http_server(port=config.metrics.port)
 
     if config.tracing is not None or config.metrics is not None:
         # FastAPI instrumentor reports both metrics and traces

aidial_sdk/telemetry/types.py

@@ -1,18 +1,50 @@
+import logging
+import os
 from typing import Optional
 
 from aidial_sdk.pydantic_v1 import BaseModel
+from aidial_sdk.utils.env import env_var_list
+
+# OpenTelemetry SDK configuration env vars:
+# https://opentelemetry.io/docs/specs/otel/configuration/sdk-environment-variables/
+
+OTEL_LOGS_EXPORTER = env_var_list("OTEL_LOGS_EXPORTER")
+OTEL_TRACES_EXPORTER = env_var_list("OTEL_TRACES_EXPORTER")
+OTEL_METRICS_EXPORTER = env_var_list("OTEL_METRICS_EXPORTER")
+OTEL_EXPORTER_PROMETHEUS_PORT = int(
+    os.getenv("OTEL_EXPORTER_PROMETHEUS_PORT", 9464)
+)
+OTEL_PYTHON_LOG_CORRELATION = (
+    os.getenv("OTEL_PYTHON_LOG_CORRELATION", "false").lower() == "true"
+)
+
+
+class LogsConfig(BaseModel):
+    otlp_export: bool = "otlp" in OTEL_LOGS_EXPORTER
+    level: int = logging.INFO
 
 
 class TracingConfig(BaseModel):
-    otlp_export: bool = False
-    logging: bool = False
+    otlp_export: bool = "otlp" in OTEL_TRACES_EXPORTER
+
+    """Configure logging to include tracing context
+    into console log messages"""
+    logging: bool = OTEL_PYTHON_LOG_CORRELATION
 
 
 class MetricsConfig(BaseModel):
-    port: int = 9464
+    otlp_export: bool = "otlp" in OTEL_METRICS_EXPORTER
+    prometheus_export: bool = "prometheus" in OTEL_METRICS_EXPORTER
+    port: int = OTEL_EXPORTER_PROMETHEUS_PORT
 
 
 class TelemetryConfig(BaseModel):
     service_name: Optional[str] = None
-    tracing: Optional[TracingConfig] = None
-    metrics: Optional[MetricsConfig] = None
+
+    logs: Optional[LogsConfig] = LogsConfig() if OTEL_LOGS_EXPORTER else None
+    tracing: Optional[TracingConfig] = (
+        TracingConfig() if OTEL_TRACES_EXPORTER else None
+    )
+    metrics: Optional[MetricsConfig] = (
+        MetricsConfig() if OTEL_METRICS_EXPORTER else None
+    )

aidial_sdk/utils/env.py

@@ -0,0 +1,9 @@
+import os
+from typing import List
+
+
+def env_var_list(name: str) -> List[str]:
+    value = os.getenv(name)
+    if value is None:
+        return []
+    return value.split(",")