461 lines
14 KiB
Go
461 lines
14 KiB
Go
package httpapi
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"net/http"
|
|
"net/http/httptest"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/descrybe/descrybe-v2/apps/api/internal/config"
|
|
"github.com/descrybe/descrybe-v2/apps/api/internal/jobs"
|
|
"github.com/jackc/pgx/v5"
|
|
)
|
|
|
|
type stubPinger struct{ err error }
|
|
|
|
func (p stubPinger) Ping(context.Context) error { return p.err }
|
|
|
|
type stubHBRow struct {
|
|
scan func(dest ...any) error
|
|
}
|
|
|
|
func (r stubHBRow) Scan(dest ...any) error {
|
|
if r.scan == nil {
|
|
return pgx.ErrNoRows
|
|
}
|
|
return r.scan(dest...)
|
|
}
|
|
|
|
type stubHeartbeat struct {
|
|
pending int64
|
|
lastSeen time.Time
|
|
seenErr error
|
|
calls int
|
|
}
|
|
|
|
func (q *stubHeartbeat) QueryRow(ctx context.Context, sql string, args ...any) pgx.Row {
|
|
q.calls++
|
|
if q.calls == 1 {
|
|
return stubHBRow{scan: func(dest ...any) error {
|
|
*(dest[0].(*int64)) = q.pending
|
|
return nil
|
|
}}
|
|
}
|
|
return stubHBRow{scan: func(dest ...any) error {
|
|
if q.seenErr != nil {
|
|
return q.seenErr
|
|
}
|
|
*(dest[0].(*time.Time)) = q.lastSeen
|
|
return nil
|
|
}}
|
|
}
|
|
|
|
func liveWorkerProbe() jobs.HeartbeatQuerier {
|
|
return &stubHeartbeat{pending: 2, lastSeen: time.Now()}
|
|
}
|
|
|
|
func TestHandleHealthzOK(t *testing.T) {
|
|
t.Parallel()
|
|
s := &Server{Config: config.Config{MaintenanceMode: true, ReadOnlyMode: true, HypercareMode: true}}
|
|
rec := httptest.NewRecorder()
|
|
s.handleHealthz(rec, httptest.NewRequest(http.MethodGet, "/healthz", nil))
|
|
if rec.Code != http.StatusOK {
|
|
t.Fatalf("status = %d", rec.Code)
|
|
}
|
|
var body map[string]any
|
|
if err := json.Unmarshal(rec.Body.Bytes(), &body); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if body["status"] != "ok" {
|
|
t.Fatalf("body = %#v", body)
|
|
}
|
|
if body["service"] != healthServiceName {
|
|
t.Fatalf("service = %#v", body["service"])
|
|
}
|
|
if body["maintenance"] != true || body["read_only"] != true || body["hypercare"] != true {
|
|
t.Fatalf("expected maintenance/read_only/hypercare flags, got %#v", body)
|
|
}
|
|
}
|
|
|
|
func TestHandleReadyzNilPool(t *testing.T) {
|
|
t.Parallel()
|
|
s := &Server{Config: config.Config{MaintenanceMode: true, ReadOnlyMode: true}, Pool: nil}
|
|
rec := httptest.NewRecorder()
|
|
s.handleReadyz(rec, httptest.NewRequest(http.MethodGet, "/readyz", nil))
|
|
if rec.Code != http.StatusServiceUnavailable {
|
|
t.Fatalf("status = %d, want 503", rec.Code)
|
|
}
|
|
var body map[string]any
|
|
if err := json.Unmarshal(rec.Body.Bytes(), &body); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if body["status"] != "not_ready" {
|
|
t.Fatalf("body = %#v", body)
|
|
}
|
|
if body["service"] != healthServiceName {
|
|
t.Fatalf("service = %#v", body["service"])
|
|
}
|
|
if body["maintenance"] != true || body["read_only"] != true {
|
|
t.Fatalf("expected flags on 503, got %#v", body)
|
|
}
|
|
checks, _ := body["checks"].(map[string]any)
|
|
if checks["database"] != "unavailable" {
|
|
t.Fatalf("checks = %#v", body["checks"])
|
|
}
|
|
if body["error"] != "database pool unavailable" {
|
|
t.Fatalf("error = %#v", body["error"])
|
|
}
|
|
}
|
|
|
|
func TestWriteReadyzPingOK(t *testing.T) {
|
|
t.Parallel()
|
|
s := &Server{Config: config.Config{ReadOnlyMode: true}}
|
|
rec := httptest.NewRecorder()
|
|
s.writeReadyz(rec, httptest.NewRequest(http.MethodGet, "/readyz", nil), stubPinger{}, liveWorkerProbe())
|
|
if rec.Code != http.StatusOK {
|
|
t.Fatalf("status = %d body=%s", rec.Code, rec.Body.String())
|
|
}
|
|
var body map[string]any
|
|
if err := json.Unmarshal(rec.Body.Bytes(), &body); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if body["status"] != "ready" || body["service"] != healthServiceName {
|
|
t.Fatalf("body = %#v", body)
|
|
}
|
|
if body["read_only"] != true {
|
|
t.Fatalf("read_only = %#v", body["read_only"])
|
|
}
|
|
checks, _ := body["checks"].(map[string]any)
|
|
if checks["database"] != "ok" || checks["worker"] != "ok" || checks["queue"] != "ok" {
|
|
t.Fatalf("checks = %#v", body["checks"])
|
|
}
|
|
if body["queue_pending"] != float64(2) {
|
|
t.Fatalf("queue_pending = %#v", body["queue_pending"])
|
|
}
|
|
}
|
|
|
|
func TestWriteReadyzWorkerStale(t *testing.T) {
|
|
t.Parallel()
|
|
s := &Server{Config: config.Config{}}
|
|
rec := httptest.NewRecorder()
|
|
stale := &stubHeartbeat{pending: 5, lastSeen: time.Now().Add(-2 * time.Minute)}
|
|
s.writeReadyz(rec, httptest.NewRequest(http.MethodGet, "/readyz", nil), stubPinger{}, stale)
|
|
if rec.Code != http.StatusServiceUnavailable {
|
|
t.Fatalf("status = %d, want 503 body=%s", rec.Code, rec.Body.String())
|
|
}
|
|
var body map[string]any
|
|
if err := json.Unmarshal(rec.Body.Bytes(), &body); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
checks, _ := body["checks"].(map[string]any)
|
|
if body["status"] != "not_ready" || checks["worker"] != "stale" || checks["database"] != "ok" {
|
|
t.Fatalf("body = %#v", body)
|
|
}
|
|
if body["queue_pending"] != float64(5) {
|
|
t.Fatalf("queue_pending = %#v", body["queue_pending"])
|
|
}
|
|
if body["error"] != "worker heartbeat stale" {
|
|
t.Fatalf("error = %#v", body["error"])
|
|
}
|
|
reason, _ := body["reason"].(string)
|
|
if reason == "" || !strings.Contains(reason, "npm run dev") {
|
|
t.Fatalf("reason = %#v", body["reason"])
|
|
}
|
|
}
|
|
|
|
func TestWriteReadyzPingFail(t *testing.T) {
|
|
t.Parallel()
|
|
s := &Server{Config: config.Config{}}
|
|
rec := httptest.NewRecorder()
|
|
s.writeReadyz(rec, httptest.NewRequest(http.MethodGet, "/readyz", nil), stubPinger{err: errors.New("boom")}, liveWorkerProbe())
|
|
if rec.Code != http.StatusServiceUnavailable {
|
|
t.Fatalf("status = %d, want 503", rec.Code)
|
|
}
|
|
var body map[string]any
|
|
if err := json.Unmarshal(rec.Body.Bytes(), &body); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
checks, _ := body["checks"].(map[string]any)
|
|
if body["status"] != "not_ready" || checks["database"] != "fail" {
|
|
t.Fatalf("body = %#v", body)
|
|
}
|
|
if body["error"] != "database ping failed" {
|
|
t.Fatalf("error leaked detail: %#v", body["error"])
|
|
}
|
|
}
|
|
|
|
func TestDatabaseReady(t *testing.T) {
|
|
t.Parallel()
|
|
ctx := context.Background()
|
|
|
|
ok, check, msg := databaseReady(ctx, nil)
|
|
if ok || check != "unavailable" || msg == "" {
|
|
t.Fatalf("nil pinger: ok=%v check=%s msg=%q", ok, check, msg)
|
|
}
|
|
ok, check, msg = databaseReady(ctx, stubPinger{err: errors.New("x")})
|
|
if ok || check != "fail" || msg != "database ping failed" {
|
|
t.Fatalf("fail pinger: ok=%v check=%s msg=%q", ok, check, msg)
|
|
}
|
|
ok, check, msg = databaseReady(ctx, stubPinger{})
|
|
if !ok || check != "ok" || msg != "" {
|
|
t.Fatalf("ok pinger: ok=%v check=%s msg=%q", ok, check, msg)
|
|
}
|
|
}
|
|
|
|
func TestMaintenanceGateBlocksNonHealth(t *testing.T) {
|
|
t.Parallel()
|
|
s := &Server{Config: config.Config{MaintenanceMode: true}}
|
|
h := s.MaintenanceGate(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
|
w.WriteHeader(http.StatusOK)
|
|
}))
|
|
|
|
blocked := httptest.NewRecorder()
|
|
h.ServeHTTP(blocked, httptest.NewRequest(http.MethodGet, "/api/auth/me", nil))
|
|
if blocked.Code != http.StatusServiceUnavailable {
|
|
t.Fatalf("blocked status = %d", blocked.Code)
|
|
}
|
|
assertGateBody(t, blocked, "maintenance", true, false)
|
|
|
|
ok := httptest.NewRecorder()
|
|
h.ServeHTTP(ok, httptest.NewRequest(http.MethodGet, "/healthz", nil))
|
|
if ok.Code != http.StatusOK {
|
|
t.Fatalf("health status = %d", ok.Code)
|
|
}
|
|
|
|
ready := httptest.NewRecorder()
|
|
h.ServeHTTP(ready, httptest.NewRequest(http.MethodGet, "/readyz", nil))
|
|
if ready.Code != http.StatusOK {
|
|
t.Fatalf("readyz status = %d", ready.Code)
|
|
}
|
|
|
|
// Query string must not defeat the probe exemption (Path is still /healthz).
|
|
probeQ := httptest.NewRecorder()
|
|
h.ServeHTTP(probeQ, httptest.NewRequest(http.MethodGet, "/healthz?ping=1", nil))
|
|
if probeQ.Code != http.StatusOK {
|
|
t.Fatalf("healthz?query status = %d", probeQ.Code)
|
|
}
|
|
}
|
|
|
|
func TestReadOnlyGateBlocksMutations(t *testing.T) {
|
|
t.Parallel()
|
|
s := &Server{Config: config.Config{ReadOnlyMode: true}}
|
|
h := s.MaintenanceGate(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
|
w.WriteHeader(http.StatusOK)
|
|
}))
|
|
|
|
for _, method := range []string{http.MethodGet, http.MethodHead, http.MethodOptions} {
|
|
rec := httptest.NewRecorder()
|
|
h.ServeHTTP(rec, httptest.NewRequest(method, "/api/products", nil))
|
|
if rec.Code != http.StatusOK {
|
|
t.Fatalf("%s status = %d, want 200", method, rec.Code)
|
|
}
|
|
}
|
|
|
|
for _, method := range []string{http.MethodPost, http.MethodPut, http.MethodPatch, http.MethodDelete} {
|
|
rec := httptest.NewRecorder()
|
|
h.ServeHTTP(rec, httptest.NewRequest(method, "/api/products", nil))
|
|
if rec.Code != http.StatusServiceUnavailable {
|
|
t.Fatalf("%s status = %d, want 503", method, rec.Code)
|
|
}
|
|
assertGateBody(t, rec, "read_only", false, true)
|
|
}
|
|
}
|
|
|
|
func TestMaintenanceGatePrecedenceOverReadOnly(t *testing.T) {
|
|
t.Parallel()
|
|
s := &Server{Config: config.Config{MaintenanceMode: true, ReadOnlyMode: true}}
|
|
h := s.MaintenanceGate(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
|
w.WriteHeader(http.StatusOK)
|
|
}))
|
|
|
|
getRec := httptest.NewRecorder()
|
|
h.ServeHTTP(getRec, httptest.NewRequest(http.MethodGet, "/api/products", nil))
|
|
if getRec.Code != http.StatusServiceUnavailable {
|
|
t.Fatalf("GET status = %d, want 503", getRec.Code)
|
|
}
|
|
assertGateBody(t, getRec, "maintenance", true, true)
|
|
|
|
postRec := httptest.NewRecorder()
|
|
h.ServeHTTP(postRec, httptest.NewRequest(http.MethodPost, "/api/products", nil))
|
|
if postRec.Code != http.StatusServiceUnavailable {
|
|
t.Fatalf("POST status = %d, want 503", postRec.Code)
|
|
}
|
|
assertGateBody(t, postRec, "maintenance", true, true)
|
|
|
|
ok := httptest.NewRecorder()
|
|
h.ServeHTTP(ok, httptest.NewRequest(http.MethodGet, "/readyz", nil))
|
|
if ok.Code != http.StatusOK {
|
|
t.Fatalf("readyz status = %d", ok.Code)
|
|
}
|
|
}
|
|
|
|
func TestRouterMaintenanceAndReadOnlyBeforeCSRF(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
maint := testAPIServer()
|
|
maint.Config.MaintenanceMode = true
|
|
maintH := maint.Router()
|
|
|
|
maintPOST := httptest.NewRecorder()
|
|
maintH.ServeHTTP(maintPOST, httptest.NewRequest(http.MethodPost, "/api/auth/login", nil))
|
|
if maintPOST.Code != http.StatusServiceUnavailable {
|
|
t.Fatalf("maintenance POST without CSRF status = %d, want 503 (not csrf 403); body=%s", maintPOST.Code, maintPOST.Body.String())
|
|
}
|
|
assertGateBody(t, maintPOST, "maintenance", true, false)
|
|
|
|
maintGET := httptest.NewRecorder()
|
|
maintH.ServeHTTP(maintGET, httptest.NewRequest(http.MethodGet, "/api/auth/me", nil))
|
|
if maintGET.Code != http.StatusServiceUnavailable {
|
|
t.Fatalf("maintenance GET status = %d, want 503", maintGET.Code)
|
|
}
|
|
assertGateBody(t, maintGET, "maintenance", true, false)
|
|
|
|
health := httptest.NewRecorder()
|
|
maintH.ServeHTTP(health, httptest.NewRequest(http.MethodGet, "/healthz", nil))
|
|
if health.Code != http.StatusOK {
|
|
t.Fatalf("healthz under maintenance status = %d", health.Code)
|
|
}
|
|
var healthBody map[string]any
|
|
if err := json.Unmarshal(health.Body.Bytes(), &healthBody); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if healthBody["maintenance"] != true {
|
|
t.Fatalf("healthz flags = %#v", healthBody)
|
|
}
|
|
|
|
ro := testAPIServer()
|
|
ro.Config.ReadOnlyMode = true
|
|
roH := ro.Router()
|
|
|
|
roPOST := httptest.NewRecorder()
|
|
roH.ServeHTTP(roPOST, httptest.NewRequest(http.MethodPost, "/api/auth/login", nil))
|
|
if roPOST.Code != http.StatusServiceUnavailable {
|
|
t.Fatalf("read-only POST without CSRF status = %d, want 503 (not csrf 403); body=%s", roPOST.Code, roPOST.Body.String())
|
|
}
|
|
assertGateBody(t, roPOST, "read_only", false, true)
|
|
|
|
roGET := httptest.NewRecorder()
|
|
roH.ServeHTTP(roGET, httptest.NewRequest(http.MethodGet, "/api/auth/me", nil))
|
|
if roGET.Code == http.StatusServiceUnavailable {
|
|
t.Fatalf("read-only GET must pass the gate; got 503 body=%s", roGET.Body.String())
|
|
}
|
|
if roGET.Code != http.StatusUnauthorized {
|
|
t.Fatalf("read-only GET /api/auth/me status = %d, want 401", roGET.Code)
|
|
}
|
|
}
|
|
|
|
func TestRouterMetricsMounted(t *testing.T) {
|
|
t.Parallel()
|
|
h := testAPIServer().Router()
|
|
|
|
// Drive one request so RED counters are non-empty.
|
|
health := httptest.NewRecorder()
|
|
h.ServeHTTP(health, httptest.NewRequest(http.MethodGet, "/healthz", nil))
|
|
if health.Code != http.StatusOK {
|
|
t.Fatalf("healthz status=%d", health.Code)
|
|
}
|
|
|
|
rec := httptest.NewRecorder()
|
|
h.ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/metrics", nil))
|
|
if rec.Code != http.StatusOK {
|
|
t.Fatalf("metrics status=%d body=%s", rec.Code, rec.Body.String())
|
|
}
|
|
ct := rec.Header().Get("Content-Type")
|
|
if !strings.Contains(ct, "text/plain") {
|
|
t.Fatalf("Content-Type=%q", ct)
|
|
}
|
|
body := rec.Body.String()
|
|
for _, want := range []string{
|
|
"http_requests_total{",
|
|
`path="/healthz"`,
|
|
"# TYPE sync_failures_total counter",
|
|
} {
|
|
if !strings.Contains(body, want) {
|
|
t.Fatalf("missing %q in metrics:\n%s", want, body)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestRouterMetricsHiddenInProductionForRemote(t *testing.T) {
|
|
t.Parallel()
|
|
s := testAPIServer()
|
|
s.Config.AppEnv = "production"
|
|
s.Config.MetricsPublic = false
|
|
h := s.Router()
|
|
req := httptest.NewRequest(http.MethodGet, "/metrics", nil)
|
|
req.RemoteAddr = "203.0.113.9:9999"
|
|
rec := httptest.NewRecorder()
|
|
h.ServeHTTP(rec, req)
|
|
if rec.Code != http.StatusNotFound {
|
|
t.Fatalf("prod remote metrics status=%d want 404", rec.Code)
|
|
}
|
|
|
|
loop := httptest.NewRequest(http.MethodGet, "/metrics", nil)
|
|
loop.RemoteAddr = "127.0.0.1:4242"
|
|
recLoop := httptest.NewRecorder()
|
|
h.ServeHTTP(recLoop, loop)
|
|
if recLoop.Code != http.StatusOK {
|
|
t.Fatalf("prod loopback metrics status=%d", recLoop.Code)
|
|
}
|
|
|
|
s.Config.MetricsPublic = true
|
|
hPub := s.Router()
|
|
reqPub := httptest.NewRequest(http.MethodGet, "/metrics", nil)
|
|
reqPub.RemoteAddr = "203.0.113.9:9999"
|
|
recPub := httptest.NewRecorder()
|
|
hPub.ServeHTTP(recPub, reqPub)
|
|
if recPub.Code != http.StatusOK {
|
|
t.Fatalf("METRICS_PUBLIC remote status=%d", recPub.Code)
|
|
}
|
|
}
|
|
|
|
func TestRouterMetricsHiddenInProductionForSpoofedLoopbackXFF(t *testing.T) {
|
|
t.Parallel()
|
|
s := testAPIServer()
|
|
s.Config.AppEnv = "production"
|
|
s.Config.MetricsPublic = false
|
|
s.Config.TrustedProxies = []string{"10.0.0.0/8"}
|
|
h := s.Router()
|
|
|
|
req := httptest.NewRequest(http.MethodGet, "/metrics", nil)
|
|
req.RemoteAddr = "10.0.0.5:443"
|
|
req.Header.Set("X-Forwarded-For", "127.0.0.1")
|
|
rec := httptest.NewRecorder()
|
|
h.ServeHTTP(rec, req)
|
|
if rec.Code != http.StatusNotFound {
|
|
t.Fatalf("trusted proxy spoofed loopback XFF status=%d want 404", rec.Code)
|
|
}
|
|
|
|
s.Config.MetricsPublic = true
|
|
hPub := s.Router()
|
|
reqPub := httptest.NewRequest(http.MethodGet, "/metrics", nil)
|
|
reqPub.RemoteAddr = "10.0.0.5:443"
|
|
reqPub.Header.Set("X-Forwarded-For", "127.0.0.1")
|
|
recPub := httptest.NewRecorder()
|
|
hPub.ServeHTTP(recPub, reqPub)
|
|
if recPub.Code != http.StatusOK {
|
|
t.Fatalf("METRICS_PUBLIC trusted proxy XFF status=%d", recPub.Code)
|
|
}
|
|
}
|
|
|
|
func assertGateBody(t *testing.T, rec *httptest.ResponseRecorder, errorCode string, maintenance, readOnly bool) {
|
|
t.Helper()
|
|
var body map[string]any
|
|
if err := json.Unmarshal(rec.Body.Bytes(), &body); err != nil {
|
|
t.Fatalf("json: %v body=%s", err, rec.Body.String())
|
|
}
|
|
if body["error"] != errorCode {
|
|
t.Fatalf("error = %#v, want %q", body["error"], errorCode)
|
|
}
|
|
if body["maintenance"] != maintenance {
|
|
t.Fatalf("maintenance = %#v, want %v", body["maintenance"], maintenance)
|
|
}
|
|
if body["read_only"] != readOnly {
|
|
t.Fatalf("read_only = %#v, want %v", body["read_only"], readOnly)
|
|
}
|
|
}
|