Lab 18: Ship container logs to Loki

Objective

Centralise container logs in Loki and verify they are queryable.

Tasks

Task 1: Compose file

# compose.yml
services:
  loki:
    image: grafana/loki:3.3.0
    command: -config.file=/etc/loki/local-config.yaml
    ports:
      - "3100:3100"
    networks: [obs]

  promtail:
    image: grafana/promtail:3.3.0
    command: -config.file=/etc/promtail/config.yml
    volumes:
      - /var/lib/docker/containers:/var/lib/docker/containers:ro
      - /var/run/docker.sock:/var/run/docker.sock:ro
      - ./promtail-config.yml:/etc/promtail/config.yml:ro
    networks: [obs]
    depends_on:
      - loki

networks:
  obs:

Task 2: Promtail config

# promtail-config.yml
server:
  http_listen_port: 9080

positions:
  filename: /tmp/positions.yaml

clients:
  - url: http://loki:3100/loki/api/v1/push

scrape_configs:
  - job_name: containers
    docker_sd_configs:
      - host: unix:///var/run/docker.sock
        refresh_interval: 5s
    relabel_configs:
      - source_labels: ['__meta_docker_container_name']
        target_label: 'container'
      - source_labels: ['__meta_docker_container_log_stream']
        target_label: 'stream'
      - source_labels: ['__meta_docker_compose_service']
        target_label: 'service'

Task 3: Bring up

docker compose up -d
sleep 15

Task 4: Generate logs

docker run -d --name log-source --network obs \
  busybox sh -c 'while true; do echo "hello at $(date)"; sleep 2; done'

Task 5: Query

curl -s 'http://localhost:3100/loki/api/v1/query?query={container="log-source"}' | jq .

You should see the log lines with their timestamps.

Cleanup

docker compose down -v
docker rm -f log-source