# Example Prometheus alerting rules for Descrybe v2.
# Load via rule_files / Prometheus Operator PrometheusRule.
# Tune thresholds to your baseline before paging. No secrets.
#
# Docs: docs/production-readiness.md#ops-prometheus-scrape
# Worker triage (not PromQL): GET /readyz → checks.worker + worker_last_seen_age_s
# (stale after 60s = jobs.DefaultHeartbeatStaleAfter).

groups:
  - name: descrybe-red
    rules:
      - alert: DescrybeAPIHigh5xxRate
        expr: |
          (
            sum(rate(http_requests_total{job="descrybe-api",code=~"5.."}[5m]))
            /
            clamp_min(sum(rate(http_requests_total{job="descrybe-api"}[5m])), 1e-9)
          ) > 0.05
        for: 10m
        labels:
          severity: warning
          service: descrybe-api
        annotations:
          summary: "Descrybe API 5xx rate >5% (10m)"
          description: "RED Errors — investigate recent deploys, DB, and upstreams."

      - alert: DescrybeAPIHighP99Latency
        expr: |
          histogram_quantile(
            0.99,
            sum by (le) (rate(http_request_duration_seconds_bucket{job="descrybe-api"}[5m]))
          ) > 2
        for: 15m
        labels:
          severity: warning
          service: descrybe-api
        annotations:
          summary: "Descrybe API p99 latency >2s (15m)"
          description: "RED Duration — check slow routes and DB pool saturation."

  - name: descrybe-sync
    rules:
      - alert: DescrybeSyncFailures
        expr: |
          sum by (kind) (increase(sync_failures_total{job="descrybe-worker"}[15m])) > 0
        for: 15m
        labels:
          severity: warning
          service: descrybe-worker
        annotations:
          summary: "Descrybe sync failures for kind {{ $labels.kind }}"
          description: "Sustained sync_failures_total increase — check feed/Woo/Shopify connectors and worker logs."

      - alert: DescrybeSyncSlowP95
        expr: |
          histogram_quantile(
            0.95,
            sum by (le, kind) (rate(sync_duration_seconds_bucket{job="descrybe-worker"}[15m]))
          ) > 120
        for: 30m
        labels:
          severity: warning
          service: descrybe-worker
        annotations:
          summary: "Descrybe sync p95 >120s for kind {{ $labels.kind }}"
          description: "Sync stall / slow — check source feed size and worker claim loops."

  - name: descrybe-availability
    rules:
      - alert: DescrybeMetricsScrapeDown
        expr: up{job=~"descrybe-api|descrybe-worker"} == 0
        for: 5m
        labels:
          severity: critical
          service: descrybe
        annotations:
          summary: "Prometheus cannot scrape {{ $labels.job }}"
          description: "Check private scrape path, Gate (loopback / METRICS_PUBLIC), and process health. Do not open /metrics on the public VIP."

      # /readyz returns 503 when worker heartbeat is missing or stale (>60s).
      # Requires job descrybe-readyz from scrape.example.yml (blackbox http_2xx).
      - alert: DescrybeWorkerHeartbeatStale
        expr: probe_success{job="descrybe-readyz",probe="readyz"} == 0
        for: 2m
        labels:
          severity: critical
          service: descrybe-worker
        annotations:
          summary: "Descrybe /readyz probe failing (worker missing or stale >60s)"
          description: >-
            On-call triage (not a Prom series): curl private GET /readyz and read
            checks.worker (ok|missing|stale) and worker_last_seen_age_s when present.
            Stale after 60s. Do not start a second worker while age is still fresh.
            See docs/ops-runtime.md and scripts/cutover-deploy-check.mjs.