diff --git a/aidial_sdk/telemetry/init.py b/aidial_sdk/telemetry/init.py index f70b09a9..f4693f21 100644 --- a/aidial_sdk/telemetry/init.py +++ b/aidial_sdk/telemetry/init.py @@ -1,4 +1,11 @@ +import logging + from fastapi import FastAPI +from opentelemetry._logs import set_logger_provider +from opentelemetry.exporter.otlp.proto.grpc._log_exporter import OTLPLogExporter +from opentelemetry.exporter.otlp.proto.grpc.metric_exporter import ( + OTLPMetricExporter, +) from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import ( OTLPSpanExporter, ) @@ -15,7 +22,12 @@ ) from opentelemetry.instrumentation.urllib import URLLibInstrumentor from opentelemetry.metrics import set_meter_provider +from opentelemetry.sdk._logs import LoggerProvider, LoggingHandler +from opentelemetry.sdk._logs.export import BatchLogRecordProcessor from opentelemetry.sdk.metrics import MeterProvider +from opentelemetry.sdk.metrics._internal.export import ( + PeriodicExportingMetricReader, +) from opentelemetry.sdk.resources import SERVICE_NAME, Resource from opentelemetry.sdk.trace import TracerProvider from opentelemetry.sdk.trace.export import BatchSpanProcessor @@ -23,7 +35,6 @@ from prometheus_client import start_http_server from aidial_sdk.telemetry.types import TelemetryConfig -from aidial_sdk.utils.logging import logger def init_telemetry( @@ -52,21 +63,43 @@ def init_telemetry( HTTPXClientInstrumentor().instrument() if config.tracing.logging: - LoggingInstrumentor().instrument( - set_logging_format=True, - log_level=logger.getEffectiveLevel(), + # Setting the root logger format in order to include + # tracing information: span_id, trace_id + LoggingInstrumentor().instrument(set_logging_format=True) + + if config.logs is not None: + # Adding a handler to the root logger which exports the logs to OTLP + provider = LoggerProvider(resource=resource) + + if config.logs.otlp_export: + provider.add_log_record_processor( + BatchLogRecordProcessor(OTLPLogExporter()) ) + set_logger_provider(provider) + + handler = LoggingHandler(level=config.logs.level) + logging.getLogger().addHandler(handler) + if config.metrics is not None: - set_meter_provider( - MeterProvider( - resource=resource, metric_readers=[PrometheusMetricReader()] + metric_readers = [] + + if config.metrics.prometheus_export: + metric_readers.append(PrometheusMetricReader()) + + if config.metrics.otlp_export: + metric_readers.append( + PeriodicExportingMetricReader(OTLPMetricExporter()) ) + + set_meter_provider( + MeterProvider(resource=resource, metric_readers=metric_readers) ) SystemMetricsInstrumentor().instrument() - start_http_server(port=config.metrics.port) + if config.metrics.prometheus_export: + start_http_server(port=config.metrics.port) if config.tracing is not None or config.metrics is not None: # FastAPI instrumentor reports both metrics and traces diff --git a/aidial_sdk/telemetry/types.py b/aidial_sdk/telemetry/types.py index 613e4ac1..e85a2894 100644 --- a/aidial_sdk/telemetry/types.py +++ b/aidial_sdk/telemetry/types.py @@ -1,18 +1,50 @@ +import logging +import os from typing import Optional from aidial_sdk.pydantic_v1 import BaseModel +from aidial_sdk.utils.env import env_var_list + +# OpenTelemetry SDK configuration env vars: +# https://opentelemetry.io/docs/specs/otel/configuration/sdk-environment-variables/ + +OTEL_LOGS_EXPORTER = env_var_list("OTEL_LOGS_EXPORTER") +OTEL_TRACES_EXPORTER = env_var_list("OTEL_TRACES_EXPORTER") +OTEL_METRICS_EXPORTER = env_var_list("OTEL_METRICS_EXPORTER") +OTEL_EXPORTER_PROMETHEUS_PORT = int( + os.getenv("OTEL_EXPORTER_PROMETHEUS_PORT", 9464) +) +OTEL_PYTHON_LOG_CORRELATION = ( + os.getenv("OTEL_PYTHON_LOG_CORRELATION", "false").lower() == "true" +) + + +class LogsConfig(BaseModel): + otlp_export: bool = "otlp" in OTEL_LOGS_EXPORTER + level: int = logging.INFO class TracingConfig(BaseModel): - otlp_export: bool = False - logging: bool = False + otlp_export: bool = "otlp" in OTEL_TRACES_EXPORTER + + """Configure logging to include tracing context + into console log messages""" + logging: bool = OTEL_PYTHON_LOG_CORRELATION class MetricsConfig(BaseModel): - port: int = 9464 + otlp_export: bool = "otlp" in OTEL_METRICS_EXPORTER + prometheus_export: bool = "prometheus" in OTEL_METRICS_EXPORTER + port: int = OTEL_EXPORTER_PROMETHEUS_PORT class TelemetryConfig(BaseModel): service_name: Optional[str] = None - tracing: Optional[TracingConfig] = None - metrics: Optional[MetricsConfig] = None + + logs: Optional[LogsConfig] = LogsConfig() if OTEL_LOGS_EXPORTER else None + tracing: Optional[TracingConfig] = ( + TracingConfig() if OTEL_TRACES_EXPORTER else None + ) + metrics: Optional[MetricsConfig] = ( + MetricsConfig() if OTEL_METRICS_EXPORTER else None + ) diff --git a/aidial_sdk/utils/env.py b/aidial_sdk/utils/env.py new file mode 100644 index 00000000..27347e1c --- /dev/null +++ b/aidial_sdk/utils/env.py @@ -0,0 +1,9 @@ +import os +from typing import List + + +def env_var_list(name: str) -> List[str]: + value = os.getenv(name) + if value is None: + return [] + return value.split(",")