opensource/mattermost
3
0
Fork 0
mirror of https://github.com/mattermost/mattermost.git synced 2026-08-28 18:19:38 +08:00
Code Issues Packages Projects Releases Wiki Activity

MM-67668: Replace Promtail with OpenTelemetry collector (#35381)

Browse Source 
* Add container name to Docker logs

This will allow for querying Loki by container's name:

  {job="docker",container_name="mattermost-postgres"}

* Configue Loki to prepare for OTLP ingestion

- Add a volume to Loki container to get the config
- Configure Loki with the expected labels so that we can query by job,
  app, container.name...

* Add OpenTelemetry collector configuration

There are three pipelines:
1. logs/mattermost scrapes the logs from mattermost.log, parsing the
   timestamp and severity, and pushes them to Loki.
2. logs/docker scrapes the Docker logs from *-json.log, parsing the
   timestamp, the log itself and the container name, and pushes them to Loki.
3. metrics/docker scrapes the Docker socket to retrieve the containers'
   uptime values and pushes them to Prometheus.

* Replace Promtail with OpenTelemetry collector

* Update build tooling for OpenTelemetry collector

1. Make sure that the logs directory is created
2. Swap Promtail with OpenTelemetry collector

* Scrape collector to get Docker stats

Prometheus needs to scrape the OpenTelemetry collector in the exposed
port to get the Docker stats, so that we can query the uptime with
metric container_uptime_seconds, which has a container_name label to
filter by container.

* Update Grafana dashboard for Docker health checks

1. Use Prometheus as the datasource in all queries
2. Simplify the mappings to either 0 (offline, red) or 1 (online,
   green).
3. Unify all queries on container_uptime_seconds, filtering by
   container_name, and making sure that the latest value we got is at most
   15 seconds old, so that it does not show stale data.
4. Add Redis health check, that was missing
5. Update the dashboard title to Docker containers

* Tune Loki and OTel collector configs for local dev

- Switch filelog receivers to start_at: beginning so existing logs are
  ingested on collector startup, not just new entries.
- Fix Docker log timestamp layout to use 9s (variable-length nanos)
  instead of 0s (fixed-width), matching actual Docker JSON log format.
- Add ingester max_chunk_age to keep chunks open longer in the
  single-instance dev setup, so that we can ingest older logs (the
  window is max_chunk_age/2).
- Relax Loki limits for local development: allow unordered writes,
  disable old-sample rejection, and raise ingestion rate/burst to 64 MB
  to avoid throttling during bulk ingest.
This commit is contained in:
Alejandro García Montoro
2026-02-27 16:48:17 +01:00
committed by GitHub
parent 96226a9761
commit fa7668ae0d
12 changed files with 237 additions and 134 deletions
Download Patch File Download Diff File Expand all files Collapse all files
server/Makefile
+1
View File
@@ -252,6 +252,7 @@ ifeq ($(MM_USE_PGVECTOR),true)
@echo Using pgvector PostgreSQL image
endif
mkdir -p logs
docker compose rm start_dependencies
$(GO) run ./build/docker-compose-generator/main.go $(ENABLED_DOCKER_SERVICES) | docker compose -f docker-compose.makefile.yml -f /dev/stdin $(DOCKER_COMPOSE_OVERRIDE) run -T --rm start_dependencies
ifneq (,$(findstring openldap,$(ENABLED_DOCKER_SERVICES)))
server/build/docker-compose-generator/main.go
+13 -13
View File
@@ -24,19 +24,19 @@ type Container struct {
func main() {
validServices := map[string]int{
"postgres": 5432,
"minio": 9000,
"inbucket": 9001,
"openldap": 389,
"elasticsearch": 9200,
"opensearch": 9201,
"redis": 6379,
"dejavu": 1358,
"keycloak": 8080,
"prometheus": 9090,
"grafana": 3000,
"loki": 3100,
"promtail": 3180,
"postgres": 5432,
"minio": 9000,
"inbucket": 9001,
"openldap": 389,
"elasticsearch": 9200,
"opensearch": 9201,
"redis": 6379,
"dejavu": 1358,
"keycloak": 8080,
"prometheus": 9090,
"grafana": 3000,
"loki": 3100,
"otel-collector": 13133,
}
command := []string{}
for _, arg := range os.Args[1:] {
server/build/docker-compose.common.yml
+28 -9
View File
@@ -1,6 +1,12 @@
x-logging: &default-logging
driver: "json-file"
options:
tag: "{{.Name}}"
services:
postgres:
image: "postgres:14"
logging: *default-logging
restart: always
networks:
- mm-test
@@ -20,6 +26,7 @@ services:
retries: 3
minio:
image: "minio/minio:RELEASE.2024-06-22T05-26-45Z"
logging: *default-logging
command: "server /data --console-address :9002"
networks:
- mm-test
@@ -29,6 +36,7 @@ services:
MINIO_KMS_SECRET_KEY: my-minio-key:OSMM+vkKUTCvQs9YL/CVMIMt43HFhkUpqJxTmGl6rYw=
inbucket:
image: "inbucket/inbucket:stable"
logging: *default-logging
restart: always
environment:
INBUCKET_WEB_ADDR: "0.0.0.0:9001"
@@ -38,6 +46,7 @@ services:
- mm-test
openldap:
image: "osixia/openldap:1.4.0"
logging: *default-logging
restart: always
networks:
- mm-test
@@ -82,14 +91,17 @@ services:
ES_JAVA_OPTS: "-Xms512m -Xmx512m"
redis:
image: "redis:7.4.0"
logging: *default-logging
networks:
- mm-test
dejavu:
image: "appbaseio/dejavu:3.4.2"
logging: *default-logging
networks:
- mm-test
keycloak:
image: "quay.io/keycloak/keycloak:23.0.7"
logging: *default-logging
restart: always
entrypoint: /opt/keycloak/bin/kc.sh start --import-realm
networks:
@@ -104,6 +116,7 @@ services:
- "./docker/keycloak/realm-export.json:/opt/keycloak/data/import/realm-export.json:Z"
prometheus:
image: "prom/prometheus:v2.46.0"
logging: *default-logging
user: root
volumes:
- "./docker/prometheus.yml:/etc/prometheus/prometheus.yml:Z"
@@ -114,6 +127,7 @@ services:
- "host.docker.internal:host-gateway"
grafana:
image: "grafana/grafana:10.4.2"
logging: *default-logging
volumes:
- "./docker/grafana/grafana.ini:/etc/grafana/grafana.ini:Z"
- "./docker/grafana/provisioning:/etc/grafana/provisioning:Z"
@@ -122,15 +136,20 @@ services:
- mm-test
loki:
image: "grafana/loki:3.0.0"
networks:
- mm-test
promtail:
image: "grafana/promtail:3.0.0"
logging: *default-logging
volumes:
- "./docker/promtail/promtail-local-config.yaml:/etc/promtail/docker-config.yaml:Z"
- "/var/lib/docker/containers:/var/lib/docker/containers:ro"
- "/var/run/docker.sock:/var/run/docker.sock"
- "../logs:/logs:Z"
command: -config.file=/etc/promtail/docker-config.yaml
- "./docker/loki/loki-config.yaml:/etc/loki/local-config.yaml:Z"
networks:
- mm-test
otel-collector:
image: "otel/opentelemetry-collector-contrib:0.145.0"
logging: *default-logging
user: "0:0"
volumes:
- "./docker/otel-collector/otel-collector-config.yaml:/etc/otelcol-contrib/config.yaml:Z"
- "/var/lib/docker/containers:/var/lib/docker/containers:ro"
- "/var/run/docker.sock:/var/run/docker.sock:ro"
- "../logs:/logs:ro,Z"
command: ["--config=/etc/otelcol-contrib/config.yaml"]
networks:
- mm-test
server/build/docker-compose.yml
+2 -2
View File
@@ -48,10 +48,10 @@ services:
extends:
file: docker-compose.common.yml
service: loki
promtail:
otel-collector:
extends:
file: docker-compose.common.yml
service: promtail
service: otel-collector
start_dependencies:
image: mattermost/mattermost-wait-for-dep:latest
server/build/docker/grafana/dashboards/home.json
+51 -51
View File
@@ -83,8 +83,8 @@
},
{
"datasource": {
"type": "datasource",
"uid": "-- Mixed --"
"type": "prometheus",
"uid": "PBFA97CFB590B2093"
},
"fieldConfig": {
"defaults": {
@@ -98,21 +98,14 @@
"color": "red",
"index": 0,
"text": "Offline"
}
},
"type": "value"
},
{
"options": {
"from": 0,
"result": {
},
"1": {
"color": "green",
"index": 1,
"text": "Online"
},
"to": 99999999999999
}
},
"type": "range"
"type": "value"
}
],
"thresholds": {
@@ -154,79 +147,79 @@
"targets": [
{
"datasource": {
"type": "loki",
"uid": "P8E80F9AEF21F6940"
"type": "prometheus",
"uid": "PBFA97CFB590B2093"
},
"editorMode": "code",
"expr": "sum(count_over_time({container=\"mattermost-postgres\"}[5m])) or vector(0)",
"expr": "max(container_uptime_seconds{container_name=\"mattermost-postgres\"}) >bool 0 and on() (time() - max(timestamp(container_uptime_seconds{container_name=\"mattermost-postgres\"})) < 15) or vector(0)",
"legendFormat": "Postgres",
"queryType": "range",
"range": true,
"refId": "A"
},
{
"datasource": {
"type": "loki",
"uid": "P8E80F9AEF21F6940"
"type": "prometheus",
"uid": "PBFA97CFB590B2093"
},
"editorMode": "code",
"expr": "sum(count_over_time({container=\"mattermost-inbucket\"}[5m])) or vector(0)",
"expr": "max(container_uptime_seconds{container_name=\"mattermost-inbucket\"}) >bool 0 and on() (time() - max(timestamp(container_uptime_seconds{container_name=\"mattermost-inbucket\"})) < 15) or vector(0)",
"legendFormat": "Inbucket",
"queryType": "range",
"range": true,
"refId": "C"
},
{
"datasource": {
"type": "loki",
"uid": "P8E80F9AEF21F6940"
"type": "prometheus",
"uid": "PBFA97CFB590B2093"
},
"editorMode": "code",
"expr": "sum(count_over_time({container=\"mattermost-minio\"}[5m])) or vector(0)",
"expr": "max(container_uptime_seconds{container_name=\"mattermost-minio\"}) >bool 0 and on() (time() - max(timestamp(container_uptime_seconds{container_name=\"mattermost-minio\"})) < 15) or vector(0)",
"legendFormat": "MinIO",
"queryType": "range",
"range": true,
"refId": "D"
},
{
"datasource": {
"type": "loki",
"uid": "P8E80F9AEF21F6940"
"type": "prometheus",
"uid": "PBFA97CFB590B2093"
},
"editorMode": "code",
"expr": "sum(count_over_time({container=\"mattermost-openldap\"}[5m])) or vector(0)",
"expr": "max(container_uptime_seconds{container_name=\"mattermost-openldap\"}) >bool 0 and on() (time() - max(timestamp(container_uptime_seconds{container_name=\"mattermost-openldap\"})) < 15) or vector(0)",
"legendFormat": "OpenLDAP",
"queryType": "range",
"range": true,
"refId": "E"
},
{
"datasource": {
"type": "loki",
"uid": "P8E80F9AEF21F6940"
"type": "prometheus",
"uid": "PBFA97CFB590B2093"
},
"editorMode": "code",
"expr": "sum(count_over_time({container=\"mattermost-dejavu\"}[5m])) or vector(0)",
"expr": "max(container_uptime_seconds{container_name=\"mattermost-dejavu\"}) >bool 0 and on() (time() - max(timestamp(container_uptime_seconds{container_name=\"mattermost-dejavu\"})) < 15) or vector(0)",
"legendFormat": "DejaVu",
"queryType": "range",
"range": true,
"refId": "F"
},
{
"datasource": {
"type": "loki",
"uid": "P8E80F9AEF21F6940"
"type": "prometheus",
"uid": "PBFA97CFB590B2093"
},
"editorMode": "code",
"expr": "sum(count_over_time({container=\"mattermost-keycloak\"}[5m])) or vector(0)",
"expr": "max(container_uptime_seconds{container_name=\"mattermost-keycloak\"}) >bool 0 and on() (time() - max(timestamp(container_uptime_seconds{container_name=\"mattermost-keycloak\"})) < 15) or vector(0)",
"legendFormat": "Keycloak",
"queryType": "range",
"range": true,
"refId": "G"
},
{
"datasource": {
"type": "loki",
"uid": "P8E80F9AEF21F6940"
"type": "prometheus",
"uid": "PBFA97CFB590B2093"
},
"editorMode": "code",
"expr": "sum(count_over_time({container=\"mattermost-elasticsearch\"}[5m])) or vector(0)",
"expr": "max(container_uptime_seconds{container_name=\"mattermost-elasticsearch\"}) >bool 0 and on() (time() - max(timestamp(container_uptime_seconds{container_name=\"mattermost-elasticsearch\"})) < 15) or vector(0)",
"legendFormat": "Elasticsearch",
"queryType": "range",
"range": true,
"refId": "H"
},
{
@@ -235,8 +228,7 @@
"uid": "PBFA97CFB590B2093"
},
"editorMode": "code",
"expr": "sum(up{container=\"mattermost-prometheus\"}) or vector(0)",
"hide": false,
"expr": "max(container_uptime_seconds{container_name=\"mattermost-prometheus\"}) >bool 0 and on() (time() - max(timestamp(container_uptime_seconds{container_name=\"mattermost-prometheus\"})) < 15) or vector(0)",
"legendFormat": "Prometheus",
"range": true,
"refId": "I"
@@ -247,8 +239,7 @@
"uid": "PBFA97CFB590B2093"
},
"editorMode": "code",
"expr": "sum(up{container=\"mattermost-grafana\"}) or vector(0)",
"hide": false,
"expr": "max(container_uptime_seconds{container_name=\"mattermost-grafana\"}) >bool 0 and on() (time() - max(timestamp(container_uptime_seconds{container_name=\"mattermost-grafana\"})) < 15) or vector(0)",
"legendFormat": "Grafana",
"range": true,
"refId": "J"
@@ -259,8 +250,7 @@
"uid": "PBFA97CFB590B2093"
},
"editorMode": "code",
"expr": "sum(up{container=\"mattermost-loki\"}) or vector(0)",
"hide": false,
"expr": "max(container_uptime_seconds{container_name=\"mattermost-loki\"}) >bool 0 and on() (time() - max(timestamp(container_uptime_seconds{container_name=\"mattermost-loki\"})) < 15) or vector(0)",
"legendFormat": "Loki",
"range": true,
"refId": "K"
@@ -271,14 +261,24 @@
"uid": "PBFA97CFB590B2093"
},
"editorMode": "code",
"expr": "sum(up{container=\"mattermost-promtail\"}) or vector(0)",
"hide": false,
"legendFormat": "Promtail",
"expr": "max(container_uptime_seconds{container_name=\"mattermost-otel-collector\"}) >bool 0 and on() (time() - max(timestamp(container_uptime_seconds{container_name=\"mattermost-otel-collector\"})) < 15) or vector(0)",
"legendFormat": "OTel Collector",
"range": true,
"refId": "L"
},
{
"datasource": {
"type": "prometheus",
"uid": "PBFA97CFB590B2093"
},
"editorMode": "code",
"expr": "max(container_uptime_seconds{container_name=\"mattermost-redis\"}) >bool 0 and on() (time() - max(timestamp(container_uptime_seconds{container_name=\"mattermost-redis\"})) < 15) or vector(0)",
"legendFormat": "Redis",
"range": true,
"refId": "M"
}
],
"title": "Panel Title",
"title": "Docker containers",
"type": "stat"
},
{
server/build/docker/loki/loki-config.yaml
+45
View File
@@ -0,0 +1,45 @@
auth_enabled: false
server:
http_listen_port: 3100
common:
ring:
instance_addr: 127.0.0.1
kvstore:
store: inmemory
replication_factor: 1
path_prefix: /loki
schema_config:
configs:
- from: "2026-01-01"
store: tsdb
object_store: filesystem
schema: v13
index:
prefix: loki_index_
period: 24h
storage_config:
filesystem:
directory: /loki/chunks
distributor:
otlp_config:
default_resource_attributes_as_index_labels:
- service.name
- service.instance.id
- job
- app
- container.name
ingester:
max_chunk_age: 480h
limits_config:
allow_structured_metadata: true
unordered_writes: true
reject_old_samples: false
ingestion_rate_mb: 64
ingestion_burst_size_mb: 64
server/build/docker/otel-collector/otel-collector-config.yaml
+83
View File
@@ -0,0 +1,83 @@
receivers:
filelog/mattermost:
include:
- /logs/*.log
start_at: beginning
operators:
- type: json_parser
timestamp:
parse_from: attributes.timestamp
layout_type: gotime
layout: '2006-01-02 15:04:05.000 -07:00'
severity:
parse_from: attributes.level
filelog/docker:
include:
- /var/lib/docker/containers/*/*-json.log
start_at: beginning
operators:
- type: json_parser
timestamp:
parse_from: attributes.time
layout_type: gotime
layout: '2006-01-02T15:04:05.999999999Z'
- type: move
from: attributes.log
to: body
- type: move
from: attributes.attrs.tag
to: resource["container.name"]
docker_stats:
endpoint: unix:///var/run/docker.sock
collection_interval: 10s
metrics:
container.uptime:
enabled: true
processors:
resource/mattermost:
attributes:
- key: service.name
value: mattermost
action: upsert
- key: job
value: mattermost
action: upsert
- key: app
value: mattermost
action: upsert
resource/docker:
attributes:
- key: job
value: docker
action: upsert
exporters:
otlp_http/loki:
endpoint: http://loki:3100/otlp
prometheus:
endpoint: 0.0.0.0:8889
resource_to_telemetry_conversion:
enabled: true
extensions:
health_check:
endpoint: 0.0.0.0:13133
service:
extensions: [health_check]
pipelines:
logs/mattermost:
receivers: [filelog/mattermost]
processors: [resource/mattermost]
exporters: [otlp_http/loki]
logs/docker:
receivers: [filelog/docker]
processors: [resource/docker]
exporters: [otlp_http/loki]
metrics/docker:
receivers: [docker_stats]
exporters: [prometheus]
server/build/docker/prometheus.yml
+3
View File
@@ -6,6 +6,9 @@ scrape_configs:
- job_name: 'mattermost'
static_configs:
- targets: ['host.docker.internal:8067']
- job_name: 'otel-collector'
static_configs:
- targets: ['otel-collector:8889']
- job_name: fmetrics_scrape
docker_sd_configs:
- host: unix:///var/run/docker.sock
server/build/docker/promtail/promtail-local-config.yaml
-46
View File
@@ -1,46 +0,0 @@
server:
http_listen_port: 3180
grpc_listen_port: 0
positions:
filename: /tmp/positions.yaml
clients:
- url: http://loki:3100/loki/api/v1/push
scrape_configs:
- job_name: mattermost
pipeline_stages:
- match:
selector: '{job="mattermost"}'
stages:
- json:
expressions:
timestamp: timestamp
level: level
- labels:
level:
- timestamp:
format: '2006-01-02 15:04:05.999 -07:00'
source: timestamp
static_configs:
- targets:
- localhost
labels:
job: mattermost
app: mattermost
__path__: /logs/*.log
# https://ruanbekker.medium.com/logging-with-docker-promtail-and-grafana-loki-d920fd790ca8
- job_name: docker
docker_sd_configs:
- host: unix:///var/run/docker.sock
refresh_interval: 5s
relabel_configs:
- source_labels: ['__meta_docker_container_name']
regex: '/(.*)'
target_label: 'container'
- source_labels: ['__meta_docker_container_log_stream']
target_label: 'logstream'
pipeline_stages:
- static_labels:
job: docker
server/config.mk
+2 -2
View File
@@ -6,12 +6,12 @@
#
# Possible options: postgres, minio, inbucket, openldap, dejavu,
# keycloak, elasticsearch, opensearch, redis, prometheus,
# grafana, loki and promtail.
# grafana, loki and otel-collector.
#
# Must be space separated names.
#
# Example: postgres elasticsearch
ENABLED_DOCKER_SERVICES ?= postgres inbucket redis prometheus grafana loki promtail
ENABLED_DOCKER_SERVICES ?= postgres inbucket redis prometheus grafana loki otel-collector
# Disable entirely the use of docker
MM_NO_DOCKER ?= false
server/docker-compose.makefile.yml
+4 -4
View File
@@ -99,13 +99,13 @@ services:
extends:
file: build/docker-compose.common.yml
service: loki
promtail:
container_name: mattermost-promtail
otel-collector:
container_name: mattermost-otel-collector
ports:
- "3180:3180"
- "13133:13133"
extends:
file: build/docker-compose.common.yml
service: promtail
service: otel-collector
networks:
mm-test:
server/docker-compose.yaml
+5 -7
View File
@@ -87,13 +87,11 @@ services:
extends:
file: build/docker-compose.common.yml
service: loki
promtail:
container_name: mattermost-promtail
ports:
- "3180:3180"
otel-collector:
container_name: mattermost-otel-collector
extends:
file: build/docker-compose.common.yml
service: promtail
service: otel-collector
start_dependencies:
image: mattermost/mattermost-wait-for-dep:latest
@@ -109,8 +107,8 @@ services:
- prometheus
- grafana
- loki
- promtail
command: postgres:5432 minio:9000 inbucket:9001 openldap:389 elasticsearch:9200 opensearch:9201 prometheus:9090 grafana:3000 loki:3100 promtail:3180
- otel-collector
command: postgres:5432 minio:9000 inbucket:9001 openldap:389 elasticsearch:9200 opensearch:9201 prometheus:9090 grafana:3000 loki:3100 otel-collector:13133
leader:
build: