풀스택 개발자를 위한 인프라 7강 — 관측성

# 사람만 읽기 좋은 로그
2026-09-17 10:12:03 ERROR 주문 처리 실패 user=42
 
# 구조화 로그 (JSON)
{"ts":"2026-09-17T10:12:03.120Z","level":"error","msg":"order failed",
 "userId":42,"orderId":"o-981","requestId":"b7f1c2","durationMs":812,"err":"timeout"}

import pino from "pino";
export const logger = pino({
  level: process.env.LOG_LEVEL ?? "info",
  redact: ["req.headers.authorization", "*.password", "*.token"],
});
 
// 요청마다 requestId를 붙인 자식 로거
app.use((req, res, next) => {
  req.id = req.headers["x-request-id"] ?? crypto.randomUUID();
  res.setHeader("x-request-id", req.id);
  req.log = logger.child({ requestId: req.id });
  next();
});

앱(stdout) → Docker/journald → 수집 에이전트 → 저장소 → 조회 UI
                                  │
                   Grafana Alloy / Fluent Bit / Vector
                                              → Loki → Grafana
                                              → Elasticsearch/OpenSearch → Kibana

{service="api"} |= "error"
{service="api"} | json | durationMs > 500
sum by (level) (count_over_time({service="api"} | json [5m]))

[앱 /metrics] [node_exporter :9100] [cAdvisor] [postgres_exporter]
        ▲             ▲                ▲              ▲
        └──────── Prometheus가 주기적으로 긁어감 (pull) ────────┘
                                │
                     ┌──────────┴──────────┐
                  Grafana (시각화)    Alertmanager (알림)

global:
  scrape_interval: 15s
scrape_configs:
  - job_name: api
    static_configs:
      - targets: ["api:3000"]
  - job_name: node
    static_configs:
      - targets: ["node-exporter:9100"]
rule_files:
  - /etc/prometheus/alerts.yml
alerting:
  alertmanagers:
    - static_configs:
        - targets: ["alertmanager:9093"]

import client from "prom-client";
 
client.collectDefaultMetrics();   // 이벤트 루프 지연, 힙, GC 등
 
const httpDuration = new client.Histogram({
  name: "http_request_duration_seconds",
  help: "HTTP request duration",
  labelNames: ["method", "route", "status"],
  buckets: [0.01, 0.05, 0.1, 0.3, 0.5, 1, 2, 5],
});
 
app.use((req, res, next) => {
  const end = httpDuration.startTimer();
  res.on("finish", () =>
    end({ method: req.method, route: req.route?.path ?? "unknown", status: res.statusCode })
  );
  next();
});
 
app.get("/metrics", async (_req, res) => {
  res.set("Content-Type", client.register.contentType);
  res.end(await client.register.metrics());
});

# 초당 요청 수 (5분 기준)
sum(rate(http_request_duration_seconds_count[5m]))
 
# 에러율
sum(rate(http_request_duration_seconds_count{status=~"5.."}[5m]))
  / sum(rate(http_request_duration_seconds_count[5m]))
 
# p95 응답 시간 (라우트별)
histogram_quantile(0.95,
  sum by (le, route) (rate(http_request_duration_seconds_bucket[5m])))
 
# 디스크 사용률
1 - node_filesystem_avail_bytes{mountpoint="/"} / node_filesystem_size_bytes{mountpoint="/"}

groups:
  - name: api
    rules:
      - alert: HighErrorRate
        expr: |
          sum(rate(http_request_duration_seconds_count{status=~"5.."}[5m]))
            / sum(rate(http_request_duration_seconds_count[5m])) > 0.05
        for: 5m
        labels:
          severity: critical
        annotations:
          summary: "API 에러율 5% 초과"
          runbook: "https://wiki.example.com/runbooks/api-errors"
 
      - alert: DiskWillFillIn24h
        expr: predict_linear(node_filesystem_avail_bytes{mountpoint="/"}[6h], 24*3600) < 0
        for: 30m
        labels:
          severity: warning
 
      - alert: TargetDown
        expr: up == 0
        for: 2m
        labels:
          severity: critical

Trace a1b2 (총 820ms)
└─ GET /orders/9            [api]         820ms
   ├─ SELECT orders         [postgres]     40ms
   ├─ GET /stock            [inventory]   710ms   ← 병목
   │   └─ SELECT stock      [postgres]    690ms   ← 인덱스 없음
   └─ SET cache             [redis]         2ms

// instrumentation.ts — 앱보다 먼저 로드
import { NodeSDK } from "@opentelemetry/sdk-node";
import { getNodeAutoInstrumentations } from "@opentelemetry/auto-instrumentations-node";
import { OTLPTraceExporter } from "@opentelemetry/exporter-trace-otlp-http";
 
new NodeSDK({
  serviceName: "api",
  traceExporter: new OTLPTraceExporter({ url: "http://otel-collector:4318/v1/traces" }),
  instrumentations: [getNodeAutoInstrumentations()],  // http, express, pg, redis 등 자동
}).start();

app.get("/healthz", (_req, res) => res.send("ok"));            // liveness
 
app.get("/readyz", async (_req, res) => {                        // readiness
  try {
    await db.query("SELECT 1");
    res.json({ status: "ready" });
  } catch {
    res.status(503).json({ status: "not ready" });
  }
});

services:
  prometheus:
    image: prom/prometheus
    volumes:
      - ./prometheus.yml:/etc/prometheus/prometheus.yml:ro
      - ./alerts.yml:/etc/prometheus/alerts.yml:ro
      - promdata:/prometheus
    ports: ["127.0.0.1:9090:9090"]
 
  alertmanager:
    image: prom/alertmanager
    volumes:
      - ./alertmanager.yml:/etc/alertmanager/alertmanager.yml:ro
 
  grafana:
    image: grafana/grafana
    ports: ["127.0.0.1:3001:3000"]
    volumes:
      - grafanadata:/var/lib/grafana
 
  loki:
    image: grafana/loki
    command: -config.file=/etc/loki/local-config.yaml
 
  alloy:
    image: grafana/alloy
    volumes:
      - ./config.alloy:/etc/alloy/config.alloy:ro
      - /var/run/docker.sock:/var/run/docker.sock:ro
    command: run /etc/alloy/config.alloy
 
  node-exporter:
    image: prom/node-exporter
    pid: host
    volumes:
      - /:/host:ro,rslave
    command: --path.rootfs=/host
 
  cadvisor:
    image: gcr.io/cadvisor/cadvisor
    volumes:
      - /:/rootfs:ro
      - /var/run:/var/run:ro
      - /sys:/sys:ro
      - /var/lib/docker/:/var/lib/docker:ro
 
volumes:
  promdata:
  grafanadata: