feat: proof of concept integrating Open WebUI

.context/index.md

@@ -0,0 +1,38 @@
+---
+# Basic project information
+module-name: cognative
+description: |
+  Traditional approaches to observability, operations, and business intelligence treat these concerns as separate and
+  distinct. This often results in several, different technology stacks, often dramatically increasing the demand and
+  requirements of your on-call staff.
+
+  Cognative deploys a turn-key solution that centralizes log, metric, trace, and wide-event data into a common store and
+  exposes them for analysis behind a single pane of glass. Traditional approaches often require engineering effort to
+  centralize data from numerous, distinct sources into a common location. This results in toil and requires further
+  maintenance as the organization grows. Cognative removes this toil by centralizing the data FIRST.
+
+# Technical stack
+technologies:
+  - Docker
+  - Kubernetes
+  - Helm
+  - SQL
+  - Clickhouse
+  - OpenTelemetry
+
+# Development conventions
+conventions:
+  - PNPM monorepo
+  - SPDX license header
+  - Conventional Commits
+  - Prettier + ESLint
+---
+
+## Overview
+
+This document provides an overview of the ecosystem that exists within this repository. It captures important context,
+patterns, and memories that are useful for AI code agents to produce and manipulate code.
+
+## Architecture Details
+
+### Component Interactions

.context/schema/histogram_metrics.sql

@@ -0,0 +1,43 @@
+-- Default Histogram metrics table DDL
+
+CREATE TABLE IF NOT EXISTS otel_metrics_histogram (
+    ResourceAttributes Map(LowCardinality(String), String) CODEC(ZSTD(1)),
+    ResourceSchemaUrl String CODEC(ZSTD(1)),
+    ScopeName String CODEC(ZSTD(1)),
+    ScopeVersion String CODEC(ZSTD(1)),
+    ScopeAttributes Map(LowCardinality(String), String) CODEC(ZSTD(1)),
+    ScopeDroppedAttrCount UInt32 CODEC(ZSTD(1)),
+    ScopeSchemaUrl String CODEC(ZSTD(1)),
+    ServiceName LowCardinality(String) CODEC(ZSTD(1)),
+    MetricName String CODEC(ZSTD(1)),
+    MetricDescription String CODEC(ZSTD(1)),
+    MetricUnit String CODEC(ZSTD(1)),
+    Attributes Map(LowCardinality(String), String) CODEC(ZSTD(1)),
+	StartTimeUnix DateTime64(9) CODEC(Delta, ZSTD(1)),
+	TimeUnix DateTime64(9) CODEC(Delta, ZSTD(1)),
+    Count UInt64 CODEC(Delta, ZSTD(1)),
+    Sum Float64 CODEC(ZSTD(1)),
+    BucketCounts Array(UInt64) CODEC(ZSTD(1)),
+    ExplicitBounds Array(Float64) CODEC(ZSTD(1)),
+	Exemplars Nested (
+		FilteredAttributes Map(LowCardinality(String), String),
+		TimeUnix DateTime64(9),
+		Value Float64,
+		SpanId String,
+		TraceId String
+    ) CODEC(ZSTD(1)),
+    Flags UInt32 CODEC(ZSTD(1)),
+    Min Float64 CODEC(ZSTD(1)),
+    Max Float64 CODEC(ZSTD(1)),
+    AggregationTemporality Int32 CODEC(ZSTD(1)),
+	INDEX idx_res_attr_key mapKeys(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
+	INDEX idx_res_attr_value mapValues(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
+	INDEX idx_scope_attr_key mapKeys(ScopeAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
+	INDEX idx_scope_attr_value mapValues(ScopeAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
+	INDEX idx_attr_key mapKeys(Attributes) TYPE bloom_filter(0.01) GRANULARITY 1,
+	INDEX idx_attr_value mapValues(Attributes) TYPE bloom_filter(0.01) GRANULARITY 1
+) ENGINE = MergeTree()
+TTL toDateTime("TimeUnix") + toIntervalDay(180)
+PARTITION BY toDate(TimeUnix)
+ORDER BY (ServiceName, MetricName, Attributes, toUnixTimestamp64Nano(TimeUnix))
+SETTINGS index_granularity=8192, ttl_only_drop_parts = 1;

.context/schema/logs.sql

@@ -0,0 +1,34 @@
+-- Default Logs table DDL
+
+CREATE TABLE IF NOT EXISTS otel_logs (
+	Timestamp DateTime64(9) CODEC(Delta(8), ZSTD(1)),
+	TimestampTime DateTime DEFAULT toDateTime(Timestamp),
+	TraceId String CODEC(ZSTD(1)),
+	SpanId String CODEC(ZSTD(1)),
+	TraceFlags UInt8,
+	SeverityText LowCardinality(String) CODEC(ZSTD(1)),
+	SeverityNumber UInt8,
+	ServiceName LowCardinality(String) CODEC(ZSTD(1)),
+	Body String CODEC(ZSTD(1)),
+	ResourceSchemaUrl LowCardinality(String) CODEC(ZSTD(1)),
+	ResourceAttributes Map(LowCardinality(String), String) CODEC(ZSTD(1)),
+	ScopeSchemaUrl LowCardinality(String) CODEC(ZSTD(1)),
+	ScopeName String CODEC(ZSTD(1)),
+	ScopeVersion LowCardinality(String) CODEC(ZSTD(1)),
+	ScopeAttributes Map(LowCardinality(String), String) CODEC(ZSTD(1)),
+	LogAttributes Map(LowCardinality(String), String) CODEC(ZSTD(1)),
+
+	INDEX idx_trace_id TraceId TYPE bloom_filter(0.001) GRANULARITY 1,
+	INDEX idx_res_attr_key mapKeys(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
+	INDEX idx_res_attr_value mapValues(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
+	INDEX idx_scope_attr_key mapKeys(ScopeAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
+	INDEX idx_scope_attr_value mapValues(ScopeAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
+	INDEX idx_log_attr_key mapKeys(LogAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
+	INDEX idx_log_attr_value mapValues(LogAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
+	INDEX idx_body Body TYPE tokenbf_v1(32768, 3, 0) GRANULARITY 8
+) ENGINE = MergeTree()
+PARTITION BY toDate(TimestampTime)
+PRIMARY KEY (ServiceName, TimestampTime)
+ORDER BY (ServiceName, TimestampTime, Timestamp)
+TTL TimestampTime + toIntervalDay(180)
+SETTINGS index_granularity = 8192, ttl_only_drop_parts = 1;

.context/schema/sum_metrics.sql

@@ -0,0 +1,39 @@
+-- Default Sum metrics table DDL
+
+CREATE TABLE IF NOT EXISTS otel_metrics_sum (
+    ResourceAttributes Map(LowCardinality(String), String) CODEC(ZSTD(1)),
+    ResourceSchemaUrl String CODEC(ZSTD(1)),
+    ScopeName String CODEC(ZSTD(1)),
+    ScopeVersion String CODEC(ZSTD(1)),
+    ScopeAttributes Map(LowCardinality(String), String) CODEC(ZSTD(1)),
+    ScopeDroppedAttrCount UInt32 CODEC(ZSTD(1)),
+    ScopeSchemaUrl String CODEC(ZSTD(1)),
+    ServiceName LowCardinality(String) CODEC(ZSTD(1)),
+    MetricName String CODEC(ZSTD(1)),
+    MetricDescription String CODEC(ZSTD(1)),
+    MetricUnit String CODEC(ZSTD(1)),
+    Attributes Map(LowCardinality(String), String) CODEC(ZSTD(1)),
+	StartTimeUnix DateTime64(9) CODEC(Delta, ZSTD(1)),
+	TimeUnix DateTime64(9) CODEC(Delta, ZSTD(1)),
+	Value Float64 CODEC(ZSTD(1)),
+	Flags UInt32  CODEC(ZSTD(1)),
+    Exemplars Nested (
+		FilteredAttributes Map(LowCardinality(String), String),
+		TimeUnix DateTime64(9),
+		Value Float64,
+		SpanId String,
+		TraceId String
+    ) CODEC(ZSTD(1)),
+    AggregationTemporality Int32 CODEC(ZSTD(1)),
+	IsMonotonic Boolean CODEC(Delta, ZSTD(1)),
+	INDEX idx_res_attr_key mapKeys(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
+	INDEX idx_res_attr_value mapValues(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
+	INDEX idx_scope_attr_key mapKeys(ScopeAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
+	INDEX idx_scope_attr_value mapValues(ScopeAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
+	INDEX idx_attr_key mapKeys(Attributes) TYPE bloom_filter(0.01) GRANULARITY 1,
+	INDEX idx_attr_value mapValues(Attributes) TYPE bloom_filter(0.01) GRANULARITY 1
+) ENGINE = MergeTree()
+TTL toDateTime("TimeUnix") + toIntervalDay(180)
+PARTITION BY toDate(TimeUnix)
+ORDER BY (ServiceName, MetricName, Attributes, toUnixTimestamp64Nano(TimeUnix))
+SETTINGS index_granularity=8192, ttl_only_drop_parts = 1;

.context/schema/summary_metrics.sql

@@ -0,0 +1,35 @@
+-- Default Summary metrics DDL
+
+CREATE TABLE IF NOT EXISTS otel_metrics_summary (
+    ResourceAttributes Map(LowCardinality(String), String) CODEC(ZSTD(1)),
+    ResourceSchemaUrl String CODEC(ZSTD(1)),
+    ScopeName String CODEC(ZSTD(1)),
+    ScopeVersion String CODEC(ZSTD(1)),
+    ScopeAttributes Map(LowCardinality(String), String) CODEC(ZSTD(1)),
+    ScopeDroppedAttrCount UInt32 CODEC(ZSTD(1)),
+    ScopeSchemaUrl String CODEC(ZSTD(1)),
+    ServiceName LowCardinality(String) CODEC(ZSTD(1)),
+    MetricName String CODEC(ZSTD(1)),
+    MetricDescription String CODEC(ZSTD(1)),
+    MetricUnit String CODEC(ZSTD(1)),
+    Attributes Map(LowCardinality(String), String) CODEC(ZSTD(1)),
+	StartTimeUnix DateTime64(9) CODEC(Delta, ZSTD(1)),
+	TimeUnix DateTime64(9) CODEC(Delta, ZSTD(1)),
+    Count UInt64 CODEC(Delta, ZSTD(1)),
+    Sum Float64 CODEC(ZSTD(1)),
+    ValueAtQuantiles Nested(
+		Quantile Float64,
+		Value Float64
+	) CODEC(ZSTD(1)),
+    Flags UInt32  CODEC(ZSTD(1)),
+	INDEX idx_res_attr_key mapKeys(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
+	INDEX idx_res_attr_value mapValues(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
+	INDEX idx_scope_attr_key mapKeys(ScopeAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
+	INDEX idx_scope_attr_value mapValues(ScopeAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
+	INDEX idx_attr_key mapKeys(Attributes) TYPE bloom_filter(0.01) GRANULARITY 1,
+	INDEX idx_attr_value mapValues(Attributes) TYPE bloom_filter(0.01) GRANULARITY 1
+) ENGINE = MergeTree()
+TTL toDateTime("TimeUnix") + toIntervalDay(180)
+PARTITION BY toDate(TimeUnix)
+ORDER BY (ServiceName, MetricName, Attributes, toUnixTimestamp64Nano(TimeUnix))
+SETTINGS index_granularity=8192, ttl_only_drop_parts = 1;

.context/schema/traces.sql

@@ -0,0 +1,63 @@
+-- Default Trace table DDL
+
+CREATE TABLE IF NOT EXISTS otel_traces (
+    Timestamp DateTime64(9) CODEC(Delta, ZSTD(1)),
+    TraceId String CODEC(ZSTD(1)),
+    SpanId String CODEC(ZSTD(1)),
+    ParentSpanId String CODEC(ZSTD(1)),
+    TraceState String CODEC(ZSTD(1)),
+    SpanName LowCardinality(String) CODEC(ZSTD(1)),
+    SpanKind LowCardinality(String) CODEC(ZSTD(1)),
+    ServiceName LowCardinality(String) CODEC(ZSTD(1)),
+    ResourceAttributes Map(LowCardinality(String), String) CODEC(ZSTD(1)),
+    ScopeName String CODEC(ZSTD(1)),
+    ScopeVersion String CODEC(ZSTD(1)),
+    SpanAttributes Map(LowCardinality(String), String) CODEC(ZSTD(1)),
+    Duration UInt64 CODEC(ZSTD(1)),
+    StatusCode LowCardinality(String) CODEC(ZSTD(1)),
+    StatusMessage String CODEC(ZSTD(1)),
+    Events Nested (
+        Timestamp DateTime64(9),
+        Name LowCardinality(String),
+        Attributes Map(LowCardinality(String), String)
+    ) CODEC(ZSTD(1)),
+    Links Nested (
+        TraceId String,
+        SpanId String,
+        TraceState String,
+        Attributes Map(LowCardinality(String), String)
+    ) CODEC(ZSTD(1)),
+    INDEX idx_trace_id TraceId TYPE bloom_filter(0.001) GRANULARITY 1,
+    INDEX idx_res_attr_key mapKeys(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
+    INDEX idx_res_attr_value mapValues(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
+    INDEX idx_span_attr_key mapKeys(SpanAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
+    INDEX idx_span_attr_value mapValues(SpanAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
+    INDEX idx_duration Duration TYPE minmax GRANULARITY 1
+) ENGINE = MergeTree()
+PARTITION BY toDate(Timestamp)
+ORDER BY (ServiceName, SpanName, toDateTime(Timestamp))
+TTL toDate(Timestamp) + toIntervalDay(180)
+SETTINGS index_granularity=8192, ttl_only_drop_parts = 1;
+
+
+CREATE TABLE IF NOT EXISTS otel_traces_trace_id_ts (
+     TraceId String CODEC(ZSTD(1)),
+     Start DateTime CODEC(Delta, ZSTD(1)),
+     End DateTime CODEC(Delta, ZSTD(1)),
+     INDEX idx_trace_id TraceId TYPE bloom_filter(0.01) GRANULARITY 1
+) ENGINE = MergeTree()
+PARTITION BY toDate(Start)
+ORDER BY (TraceId, Start)
+TTL toDate(Start) + toIntervalDay(180)
+SETTINGS index_granularity=8192, ttl_only_drop_parts = 1;
+
+
+CREATE MATERIALIZED VIEW IF NOT EXISTS otel_traces_trace_id_ts_mv
+TO otel_traces_trace_id_ts
+AS SELECT
+    TraceId,
+    min(Timestamp) as Start,
+    max(Timestamp) as End
+FROM otel_traces
+WHERE TraceId != ''
+GROUP BY TraceId;

infra/docker/_scripts/run.sh

@@ -0,0 +1,9 @@
+#!/usr/bin/env bash
+
+readonly iface="${INTERFACE:-en0}"
+
+readonly host_ip=$(ifconfig "${iface}" | sed -En 's/127.0.0.1//;s/.*inet (addr:)?(([0-9]*\.){3}[0-9]*).*/\2/p')
+
+export OLLAMA_HOST="${host_ip}:11434"
+
+"$@"

infra/docker/collector/config.yaml

@@ -14,13 +14,11 @@ processors:
 exporters:
   clickhouse:
     endpoint: tcp://clickhouse:9000?dial_timeout=10s&compress=lz4
-    database: otel
     username: clickhouse
     password: clickhouse
+    database: otel
+    create_schema: true
     ttl: 72h
-    logs_table_name: otel_logs
-    traces_table_name: otel_traces
-    metrics_table_name: otel_metrics
     timeout: 5s
     retry_on_failure:
       enabled: true

infra/docker/docker-compose.yaml

@@ -3,13 +3,13 @@ name: cognative
 volumes:
   minio_data: {}
   clickhouse_data: {}
+  open_webui_data: {}
   grafana_data: {}
 
 services:
   #==
   # Supporting Services
   #==
-
   traefik:
     image: traefik:v3.3
     restart: always
@@ -98,17 +98,40 @@ services:
   # cluster-agent: {}
   # host-agent: {}
 
-  grafana:
+  mcpo:
+    build:
+      dockerfile_inline: |
+        FROM ghcr.io/astral-sh/uv:debian
+        RUN apt-get update && apt-get install -y curl
+        RUN curl -fsSL https://deb.nodesource.com/setup_current.x | bash -
+        RUN apt-get install -y nodejs
+        RUN npm install -g npm@latest
+    restart: always
+    command: uvx mcpo --config /etc/mcpo/config.json
+    volumes:
+      - ./mcpo:/etc/mcpo
+    labels:
+      - "traefik.http.routers.mcpo.rule=Host(`mcp.local.cognative.work`)"
+      - "traefik.http.routers.mcpo.service=mcpo"
+      - "traefik.http.services.mcpo.loadbalancer.server.port=8000"
+
+  open-webui:
+    image: ghcr.io/open-webui/open-webui:v0.6.0
+    restart: always
     depends_on:
-      clickhouse:
-        condition: service_healthy
-    image: grafana/grafana-oss:11.5.1 # amd64 + arm64
+      mcpo:
+        condition: service_started
     environment:
-      GF_INSTALL_PLUGINS: "grafana-clickhouse-datasource"
+      - WEBUI_AUTH=False
+      - ENABLE_OPENAI_API=flase
+      - OLLAMA_BASE_URL=http://${OLLAMA_HOST:?error}
+      - WEBUI_URL=https://chat.local.cognative.work
+      - SCARF_NO_ANALYTICS=true
+      - DO_NOT_TRACK=true
+      - ANONYMIZED_TELEMETRY=false
     volumes:
-      - ./grafana:/etc/grafana
-      - grafana_data:/var/lib/grafana
+      - open_webui_data:/app/backend/data
     labels:
-      - "traefik.http.routers.grafana.rule=Host(`grafana.local.cognative.work`)"
-      - "traefik.http.routers.grafana.service=grafana"
-      - "traefik.http.services.grafana.loadbalancer.server.port=3000"
+      - "traefik.http.routers.open-webui.rule=Host(`chat.local.cognative.work`)"
+      - "traefik.http.routers.open-webui.service=open-webui"
+      - "traefik.http.services.open-webui.loadbalancer.server.port=8080"