greeneclipse
8580c996c3
Drop one-shot tmp/axe scripts and agent i18n scratch so the Gitea tree is deployable.
95 lines
3.6 KiB
YAML
95 lines
3.6 KiB
YAML
# Example Prometheus alerting rules for Descrybe v2.
|
|
# Load via rule_files / Prometheus Operator PrometheusRule.
|
|
# Tune thresholds to your baseline before paging. No secrets.
|
|
#
|
|
# Docs: docs/production-readiness.md#ops-prometheus-scrape
|
|
# Worker triage (not PromQL): GET /readyz → checks.worker + worker_last_seen_age_s
|
|
# (stale after 60s = jobs.DefaultHeartbeatStaleAfter).
|
|
|
|
groups:
|
|
- name: descrybe-red
|
|
rules:
|
|
- alert: DescrybeAPIHigh5xxRate
|
|
expr: |
|
|
(
|
|
sum(rate(http_requests_total{job="descrybe-api",code=~"5.."}[5m]))
|
|
/
|
|
clamp_min(sum(rate(http_requests_total{job="descrybe-api"}[5m])), 1e-9)
|
|
) > 0.05
|
|
for: 10m
|
|
labels:
|
|
severity: warning
|
|
service: descrybe-api
|
|
annotations:
|
|
summary: "Descrybe API 5xx rate >5% (10m)"
|
|
description: "RED Errors — investigate recent deploys, DB, and upstreams."
|
|
|
|
- alert: DescrybeAPIHighP99Latency
|
|
expr: |
|
|
histogram_quantile(
|
|
0.99,
|
|
sum by (le) (rate(http_request_duration_seconds_bucket{job="descrybe-api"}[5m]))
|
|
) > 2
|
|
for: 15m
|
|
labels:
|
|
severity: warning
|
|
service: descrybe-api
|
|
annotations:
|
|
summary: "Descrybe API p99 latency >2s (15m)"
|
|
description: "RED Duration — check slow routes and DB pool saturation."
|
|
|
|
- name: descrybe-sync
|
|
rules:
|
|
- alert: DescrybeSyncFailures
|
|
expr: |
|
|
sum by (kind) (increase(sync_failures_total{job="descrybe-worker"}[15m])) > 0
|
|
for: 15m
|
|
labels:
|
|
severity: warning
|
|
service: descrybe-worker
|
|
annotations:
|
|
summary: "Descrybe sync failures for kind {{ $labels.kind }}"
|
|
description: "Sustained sync_failures_total increase — check feed/Woo/Shopify connectors and worker logs."
|
|
|
|
- alert: DescrybeSyncSlowP95
|
|
expr: |
|
|
histogram_quantile(
|
|
0.95,
|
|
sum by (le, kind) (rate(sync_duration_seconds_bucket{job="descrybe-worker"}[15m]))
|
|
) > 120
|
|
for: 30m
|
|
labels:
|
|
severity: warning
|
|
service: descrybe-worker
|
|
annotations:
|
|
summary: "Descrybe sync p95 >120s for kind {{ $labels.kind }}"
|
|
description: "Sync stall / slow — check source feed size and worker claim loops."
|
|
|
|
- name: descrybe-availability
|
|
rules:
|
|
- alert: DescrybeMetricsScrapeDown
|
|
expr: up{job=~"descrybe-api|descrybe-worker"} == 0
|
|
for: 5m
|
|
labels:
|
|
severity: critical
|
|
service: descrybe
|
|
annotations:
|
|
summary: "Prometheus cannot scrape {{ $labels.job }}"
|
|
description: "Check private scrape path, Gate (loopback / METRICS_PUBLIC), and process health. Do not open /metrics on the public VIP."
|
|
|
|
# /readyz returns 503 when worker heartbeat is missing or stale (>60s).
|
|
# Requires job descrybe-readyz from scrape.example.yml (blackbox http_2xx).
|
|
- alert: DescrybeWorkerHeartbeatStale
|
|
expr: probe_success{job="descrybe-readyz",probe="readyz"} == 0
|
|
for: 2m
|
|
labels:
|
|
severity: critical
|
|
service: descrybe-worker
|
|
annotations:
|
|
summary: "Descrybe /readyz probe failing (worker missing or stale >60s)"
|
|
description: >-
|
|
On-call triage (not a Prom series): curl private GET /readyz and read
|
|
checks.worker (ok|missing|stale) and worker_last_seen_age_s when present.
|
|
Stale after 60s. Do not start a second worker while age is still fresh.
|
|
See docs/ops-runtime.md and scripts/cutover-deploy-check.mjs.
|