From a679fb3f456dca47fa374186c78d45f34a21032d Mon Sep 17 00:00:00 2001 From: expruc Date: Sat, 2 Aug 2025 11:30:34 +0300 Subject: [PATCH] split otel metrics from general otel configuration --- backend/open_webui/env.py | 17 ++++++ backend/open_webui/utils/telemetry/metrics.py | 58 ++++++++++++++----- backend/open_webui/utils/telemetry/setup.py | 9 +-- 3 files changed, 65 insertions(+), 19 deletions(-) diff --git a/backend/open_webui/env.py b/backend/open_webui/env.py index 8cddfc6b08..b4b2a38a06 100644 --- a/backend/open_webui/env.py +++ b/backend/open_webui/env.py @@ -643,12 +643,19 @@ AUDIT_EXCLUDED_PATHS = [path.lstrip("/") for path in AUDIT_EXCLUDED_PATHS] ENABLE_OTEL = os.environ.get("ENABLE_OTEL", "False").lower() == "true" ENABLE_OTEL_METRICS = os.environ.get("ENABLE_OTEL_METRICS", "False").lower() == "true" + OTEL_EXPORTER_OTLP_ENDPOINT = os.environ.get( "OTEL_EXPORTER_OTLP_ENDPOINT", "http://localhost:4317" ) +OTEL_METRICS_EXPORTER_OTLP_ENDPOINT = os.environ.get( + "OTEL_METRICS_EXPORTER_OTLP_ENDPOINT", OTEL_EXPORTER_OTLP_ENDPOINT +) OTEL_EXPORTER_OTLP_INSECURE = ( os.environ.get("OTEL_EXPORTER_OTLP_INSECURE", "False").lower() == "true" ) +OTEL_METRICS_EXPORTER_OTLP_INSECURE = ( + os.environ.get("OTEL_METRICS_EXPORTER_OTLP_INSECURE", "False").lower() == "true" +) OTEL_SERVICE_NAME = os.environ.get("OTEL_SERVICE_NAME", "open-webui") OTEL_RESOURCE_ATTRIBUTES = os.environ.get( "OTEL_RESOURCE_ATTRIBUTES", "" @@ -659,11 +666,21 @@ OTEL_TRACES_SAMPLER = os.environ.get( OTEL_BASIC_AUTH_USERNAME = os.environ.get("OTEL_BASIC_AUTH_USERNAME", "") OTEL_BASIC_AUTH_PASSWORD = os.environ.get("OTEL_BASIC_AUTH_PASSWORD", "") +OTEL_METRICS_BASIC_AUTH_USERNAME = os.environ.get( + "OTEL_METRICS_BASIC_AUTH_USERNAME", OTEL_BASIC_AUTH_USERNAME +) +OTEL_METRICS_BASIC_AUTH_PASSWORD = os.environ.get( + "OTEL_METRICS_BASIC_AUTH_PASSWORD", OTEL_BASIC_AUTH_PASSWORD +) OTEL_OTLP_SPAN_EXPORTER = os.environ.get( "OTEL_OTLP_SPAN_EXPORTER", "grpc" ).lower() # grpc or http +OTEL_METRICS_OTLP_SPAN_EXPORTER = os.environ.get( + "OTEL_METRICS_OTLP_SPAN_EXPORTER", OTEL_OTLP_SPAN_EXPORTER +).lower() # grpc or http + #################################### # TOOLS/FUNCTIONS PIP OPTIONS diff --git a/backend/open_webui/utils/telemetry/metrics.py b/backend/open_webui/utils/telemetry/metrics.py index f3e82c7dab..75c13ccc0a 100644 --- a/backend/open_webui/utils/telemetry/metrics.py +++ b/backend/open_webui/utils/telemetry/metrics.py @@ -19,37 +19,69 @@ from __future__ import annotations import time from typing import Dict, List, Sequence, Any +from base64 import b64encode from fastapi import FastAPI, Request from opentelemetry import metrics from opentelemetry.exporter.otlp.proto.grpc.metric_exporter import ( OTLPMetricExporter, ) + +from opentelemetry.exporter.otlp.proto.http.metric_exporter import ( + OTLPMetricExporter as OTLPHttpMetricExporter, +) from opentelemetry.sdk.metrics import MeterProvider from opentelemetry.sdk.metrics.view import View from opentelemetry.sdk.metrics.export import ( PeriodicExportingMetricReader, ) -from opentelemetry.sdk.resources import SERVICE_NAME, Resource - -from open_webui.env import OTEL_SERVICE_NAME, OTEL_EXPORTER_OTLP_ENDPOINT +from opentelemetry.sdk.resources import Resource +from open_webui.env import ( + OTEL_SERVICE_NAME, + OTEL_METRICS_EXPORTER_OTLP_ENDPOINT, + OTEL_METRICS_BASIC_AUTH_USERNAME, + OTEL_METRICS_BASIC_AUTH_PASSWORD, + OTEL_METRICS_OTLP_SPAN_EXPORTER, + OTEL_METRICS_EXPORTER_OTLP_INSECURE, +) from open_webui.socket.main import get_active_user_ids from open_webui.models.users import Users _EXPORT_INTERVAL_MILLIS = 10_000 # 10 seconds -def _build_meter_provider() -> MeterProvider: +def _build_meter_provider(resource: Resource) -> MeterProvider: """Return a configured MeterProvider.""" + headers = [] + if OTEL_METRICS_BASIC_AUTH_USERNAME and OTEL_METRICS_BASIC_AUTH_PASSWORD: + auth_string = ( + f"{OTEL_METRICS_BASIC_AUTH_USERNAME}:{OTEL_METRICS_BASIC_AUTH_PASSWORD}" + ) + auth_header = b64encode(auth_string.encode()).decode() + headers = [("authorization", f"Basic {auth_header}")] # Periodic reader pushes metrics over OTLP/gRPC to collector - readers: List[PeriodicExportingMetricReader] = [ - PeriodicExportingMetricReader( - OTLPMetricExporter(endpoint=OTEL_EXPORTER_OTLP_ENDPOINT), - export_interval_millis=_EXPORT_INTERVAL_MILLIS, - ) - ] + if OTEL_METRICS_OTLP_SPAN_EXPORTER == "http": + readers: List[PeriodicExportingMetricReader] = [ + PeriodicExportingMetricReader( + OTLPHttpMetricExporter( + endpoint=OTEL_METRICS_EXPORTER_OTLP_ENDPOINT, headers=headers + ), + export_interval_millis=_EXPORT_INTERVAL_MILLIS, + ) + ] + else: + readers: List[PeriodicExportingMetricReader] = [ + PeriodicExportingMetricReader( + OTLPMetricExporter( + endpoint=OTEL_METRICS_EXPORTER_OTLP_ENDPOINT, + insecure=OTEL_METRICS_EXPORTER_OTLP_INSECURE, + headers=headers, + ), + export_interval_millis=_EXPORT_INTERVAL_MILLIS, + ) + ] # Optional view to limit cardinality: drop user-agent etc. views: List[View] = [ @@ -70,17 +102,17 @@ def _build_meter_provider() -> MeterProvider: ] provider = MeterProvider( - resource=Resource.create({SERVICE_NAME: OTEL_SERVICE_NAME}), + resource=resource, metric_readers=list(readers), views=views, ) return provider -def setup_metrics(app: FastAPI) -> None: +def setup_metrics(app: FastAPI, resource: Resource) -> None: """Attach OTel metrics middleware to *app* and initialise provider.""" - metrics.set_meter_provider(_build_meter_provider()) + metrics.set_meter_provider(_build_meter_provider(resource)) meter = metrics.get_meter(__name__) # Instruments diff --git a/backend/open_webui/utils/telemetry/setup.py b/backend/open_webui/utils/telemetry/setup.py index 80beec0bbb..cd1f45ea6a 100644 --- a/backend/open_webui/utils/telemetry/setup.py +++ b/backend/open_webui/utils/telemetry/setup.py @@ -26,11 +26,8 @@ from open_webui.env import ( def setup(app: FastAPI, db_engine: Engine): # set up trace - trace.set_tracer_provider( - TracerProvider( - resource=Resource.create(attributes={SERVICE_NAME: OTEL_SERVICE_NAME}) - ) - ) + resource = Resource.create(attributes={SERVICE_NAME: OTEL_SERVICE_NAME}) + trace.set_tracer_provider(TracerProvider(resource=resource)) # Add basic auth header only if both username and password are not empty headers = [] @@ -56,4 +53,4 @@ def setup(app: FastAPI, db_engine: Engine): # set up metrics only if enabled if ENABLE_OTEL_METRICS: - setup_metrics(app) + setup_metrics(app, resource)