Nginx & Caddy 완전 정복 7편 — 실전: 같은 아키텍처를 Nginx와 Caddy로 각각 구성하기

                                 ┌──▶ web     (Next.js, :3000)
                                 │
인터넷 ──▶ 프록시 (80/443) ───────┼──▶ api-1, api-2 (REST, :8080)  ← /api/*  (접두사 제거)
         myservice.com           │
         www → 리다이렉트          ├──▶ api-1, api-2 (WebSocket)     ← /ws/*
                                 │
                                 └──▶ /admin/*  (사내 IP만 허용, Basic 인증)

# compose.yaml
services:
  web:
    image: myservice/web:latest      # Next.js (output: 'standalone')
    expose: ["3000"]
    restart: unless-stopped
 
  api-1:
    image: myservice/api:latest
    expose: ["8080"]
    restart: unless-stopped
 
  api-2:
    image: myservice/api:latest
    expose: ["8080"]
    restart: unless-stopped
 
  # ↓ 아래 둘 중 하나만 사용
  proxy:
    # (A) Nginx
    image: nginx:stable-alpine
    volumes:
      - ./nginx/nginx.conf:/etc/nginx/nginx.conf:ro
      - ./nginx/conf.d:/etc/nginx/conf.d:ro
      - ./nginx/snippets:/etc/nginx/snippets:ro
      - ./nginx/.htpasswd:/etc/nginx/.htpasswd:ro
      - ./certbot/conf:/etc/letsencrypt:ro
      - ./certbot/www:/var/www/certbot:ro
    ports: ["80:80", "443:443"]
    depends_on: [web, api-1, api-2]
    restart: unless-stopped
 
  certbot:                            # Nginx 사용 시에만 필요
    image: certbot/certbot
    volumes:
      - ./certbot/conf:/etc/letsencrypt
      - ./certbot/www:/var/www/certbot
    entrypoint: >
      sh -c "trap exit TERM; while :; do certbot renew --webroot -w /var/www/certbot;
      sleep 12h & wait $${!}; done"

  proxy:
    # (B) Caddy (Rate Limit 플러그인 포함 커스텀 빌드)
    build: ./caddy                   # Dockerfile은 6편의 xcaddy 멀티 스테이지 빌드
    volumes:
      - ./caddy/Caddyfile:/etc/caddy/Caddyfile:ro
      - caddy_data:/data
      - caddy_config:/config
    ports: ["80:80", "443:443", "443:443/udp"]
    environment:
      - ADMIN_PASSWORD_HASH=${ADMIN_PASSWORD_HASH}
    depends_on: [web, api-1, api-2]
    restart: unless-stopped
 
volumes:
  caddy_data:
  caddy_config:

user  nginx;
worker_processes  auto;
worker_rlimit_nofile 65535;
error_log  /var/log/nginx/error.log warn;
pid        /var/run/nginx.pid;
 
events {
    worker_connections 4096;
}
 
http {
    include       /etc/nginx/mime.types;
    default_type  application/octet-stream;
 
    server_tokens off;
    sendfile      on;
    tcp_nopush    on;
    keepalive_timeout 65;
    client_max_body_size 10m;
 
    # Docker 내장 DNS (컨테이너 재생성 시 IP 변경 대응)
    resolver 127.0.0.11 valid=10s ipv6=off;
 
    log_format json escape=json
      '{"time":"$time_iso8601","ip":"$remote_addr","method":"$request_method",'
      '"uri":"$request_uri","status":$status,"bytes":$body_bytes_sent,'
      '"rt":$request_time,"urt":"$upstream_response_time","upstream":"$upstream_addr",'
      '"ua":"$http_user_agent"}';
    access_log /var/log/nginx/access.log json;
 
    gzip on;
    gzip_comp_level 5;
    gzip_min_length 1024;
    gzip_proxied any;
    gzip_vary on;
    gzip_types text/plain text/css application/json application/javascript image/svg+xml;
 
    map $http_upgrade $connection_upgrade {
        default upgrade;
        ''      close;
    }
 
    limit_req_zone $binary_remote_addr zone=api_rl:10m rate=20r/s;
    limit_req_status 429;
 
    upstream web_upstream {
        server web:3000;
        keepalive 16;
    }
 
    upstream api_upstream {
        least_conn;
        server api-1:8080 max_fails=3 fail_timeout=15s;
        server api-2:8080 max_fails=3 fail_timeout=15s;
        keepalive 32;
    }
 
    include /etc/nginx/conf.d/*.conf;
}

# snippets/proxy.conf
proxy_http_version 1.1;
proxy_set_header Connection        "";
proxy_set_header Host              $host;
proxy_set_header X-Real-IP         $remote_addr;
proxy_set_header X-Forwarded-For   $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $scheme;

# snippets/security.conf
add_header Strict-Transport-Security "max-age=63072000; includeSubDomains" always;
add_header X-Content-Type-Options "nosniff" always;
add_header X-Frame-Options "SAMEORIGIN" always;
add_header Referrer-Policy "strict-origin-when-cross-origin" always;

# HTTP: ACME 챌린지 + HTTPS 리다이렉트
server {
    listen 80;
    server_name myservice.com www.myservice.com;
 
    location ^~ /.well-known/acme-challenge/ {
        root /var/www/certbot;
    }
    location / {
        return 301 https://myservice.com$request_uri;
    }
}
 
# www → apex
server {
    listen 443 ssl;
    http2 on;
    server_name www.myservice.com;
 
    ssl_certificate     /etc/letsencrypt/live/myservice.com/fullchain.pem;
    ssl_certificate_key /etc/letsencrypt/live/myservice.com/privkey.pem;
 
    return 301 https://myservice.com$request_uri;
}
 
# 메인
server {
    listen 443 ssl;
    http2 on;
    server_name myservice.com;
 
    ssl_certificate     /etc/letsencrypt/live/myservice.com/fullchain.pem;
    ssl_certificate_key /etc/letsencrypt/live/myservice.com/privkey.pem;
    ssl_protocols TLSv1.2 TLSv1.3;
    ssl_session_cache shared:SSL:10m;
    ssl_session_timeout 1d;
 
    include snippets/security.conf;
 
    # 3) REST API: 접두사 제거 + 로드 밸런싱 + Rate Limit
    location /api/ {
        limit_req zone=api_rl burst=40 nodelay;
        proxy_pass http://api_upstream/;
        include snippets/proxy.conf;
        proxy_next_upstream error timeout http_502 http_503;
    }
 
    # 4) WebSocket
    location /ws/ {
        proxy_pass http://api_upstream;
        proxy_http_version 1.1;
        proxy_set_header Upgrade    $http_upgrade;
        proxy_set_header Connection $connection_upgrade;
        proxy_set_header Host       $host;
        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
        proxy_read_timeout 1h;
    }
 
    # 5) Next.js 정적 자산
    location ^~ /_next/static/ {
        proxy_pass http://web_upstream;
        include snippets/proxy.conf;
        include snippets/security.conf;          # add_header 상속 함정 회피!
        add_header Cache-Control "public, max-age=31536000, immutable";
    }
 
    # 6) 관리자
    location /admin/ {
        allow 10.0.0.0/8;
        deny  all;
        auth_basic           "Admin";
        auth_basic_user_file /etc/nginx/.htpasswd;
 
        proxy_pass http://web_upstream;
        include snippets/proxy.conf;
    }
 
    # 나머지 → Next.js
    location / {
        proxy_pass http://web_upstream;
        include snippets/proxy.conf;
    }
}

upstream api_upstream {
    zone api_upstream 64k;          # resolve 사용 시 공유 메모리 zone 필수
    least_conn;
    server api-1:8080 resolve max_fails=3 fail_timeout=15s;
    server api-2:8080 resolve max_fails=3 fail_timeout=15s;
    keepalive 32;
}

{
	email ops@myservice.com
	order rate_limit before basic_auth
}
 
(security) {
	header {
		Strict-Transport-Security "max-age=63072000; includeSubDomains"
		X-Content-Type-Options "nosniff"
		X-Frame-Options "SAMEORIGIN"
		Referrer-Policy "strict-origin-when-cross-origin"
		-Server
	}
}
 
# 1) www → apex
www.myservice.com {
	redir https://myservice.com{uri} permanent
}
 
# 2) 메인 (HTTPS, HTTP→HTTPS, HTTP/2·3 자동)
myservice.com {
	import security
	encode zstd gzip
 
	log {
		output file /data/logs/access.log {
			roll_size 100MiB
			roll_keep 10
		}
		format json
	}
 
	# 3) REST API
	handle_path /api/* {
		rate_limit {
			zone api {
				key    {remote_host}
				events 20
				window 1s
			}
		}
		reverse_proxy api-1:8080 api-2:8080 {
			lb_policy least_conn
			lb_try_duration 5s
			health_uri /health
			health_interval 10s
			fail_duration 15s
		}
	}
 
	# 4) WebSocket (Upgrade 헤더 자동 처리)
	handle /ws/* {
		reverse_proxy api-1:8080 api-2:8080 {
			lb_policy least_conn
			health_uri /health
		}
	}
 
	# 6) 관리자: IP 검사 → 인증 → 프록시 순서를 보장하려고 route 사용
	@outside_office not remote_ip 10.0.0.0/8
 
	handle /admin/* {
		route {
			respond @outside_office 403
			basic_auth {
				admin {$ADMIN_PASSWORD_HASH}
			}
			reverse_proxy web:3000
		}
	}
 
	# 5) + 나머지 → Next.js
	handle {
		header /_next/static/* Cache-Control "public, max-age=31536000, immutable"
		reverse_proxy web:3000
	}
}

# Nginx
docker compose exec proxy nginx -t
docker compose exec proxy nginx -s reload
 
# Caddy
docker compose exec -w /etc/caddy proxy caddy validate --config Caddyfile
docker compose exec -w /etc/caddy proxy caddy reload --config Caddyfile

# 예: k6로 간단한 부하 테스트
k6 run --vus 200 --duration 60s script.js
# 또는
wrk -t4 -c400 -d60s https://myservice.com/api/health