package httpapi import ( "context" "errors" "net/http" "os" "path/filepath" "strconv" "strings" "time" "github.com/descrybe/descrybe-v2/apps/api/internal/billing" "github.com/descrybe/descrybe-v2/apps/api/internal/jobs" "github.com/descrybe/descrybe-v2/apps/api/internal/metrics" "github.com/descrybe/descrybe-v2/apps/api/internal/platformsettings" "github.com/descrybe/descrybe-v2/apps/api/internal/processing" "github.com/google/uuid" ) const ( adminDiagnosticsTimeout = 3 * time.Second adminDiagnosticsDefaultFails = 25 adminDiagnosticsMaxFails = 50 adminDiagnosticsStuckAfter = 2 * time.Hour adminDiagnosticsAIFailDefault = 15 adminDiagnosticsAIFailMax = 30 // Schema head expected by cutover-deploy-check (goose 039–042). adminDiagnosticsGooseExpectedMin = int64(42) ) // Required goose versions for cutover readiness (match scripts/cutover-deploy-check.mjs). var adminDiagnosticsGooseRequired = []struct { ID int64 Name string }{ {39, "039_worker_heartbeats"}, {40, "040_job_hotpath_indexes"}, {41, "041_password_reset_tokens"}, {42, "042_user_session_version"}, } // handleAdminDiagnostics returns operational health for platform admins. // GET /api/admin/diagnostics?failures_limit=25&status=failed // Never exposes secrets, DSNs, API keys, or passwords. func (s *Server) handleAdminDiagnostics(w http.ResponseWriter, r *http.Request) { ctx, cancel := context.WithTimeout(r.Context(), adminDiagnosticsTimeout) defer cancel() failLimit := adminDiagnosticsDefaultFails if raw := strings.TrimSpace(r.URL.Query().Get("failures_limit")); raw != "" { if n, err := strconv.Atoi(raw); err == nil && n > 0 { failLimit = n } } if failLimit > adminDiagnosticsMaxFails { failLimit = adminDiagnosticsMaxFails } statusFilter := strings.ToLower(strings.TrimSpace(r.URL.Query().Get("status"))) switch statusFilter { case "", "all", "failed", "running", "pending", "completed", "cancelled": default: Error(w, http.StatusBadRequest, "invalid status filter") return } if statusFilter == "all" { statusFilter = "" } checks := make([]map[string]any, 0, 7) overall := "ok" stripeCfg := s.resolveStripeDiagCfg(ctx) dbCheck, dbOK := s.diagDatabase(ctx) checks = append(checks, dbCheck) if !dbOK { overall = "fail" } queueCheck, queueSummary, queueOK := s.diagQueue(ctx) checks = append(checks, queueCheck) if !queueOK && overall != "fail" { overall = "degraded" } cacheCheck := s.diagCache() checks = append(checks, cacheCheck) storageCheck, storageOK := s.diagStorage() checks = append(checks, storageCheck) if !storageOK && overall != "fail" { overall = "degraded" } mailCheck := s.diagMail() checks = append(checks, mailCheck) stripeCheck, stripeOK := diagStripeReadiness(s.Config.IsProduction(), stripeCfg) checks = append(checks, stripeCheck) if !stripeOK && overall == "ok" { overall = "degraded" } if stripeCheck["status"] == "fail" { overall = "fail" } cutover := s.diagCutoverReadiness(ctx) cutoverCheck := map[string]any{ "name": "cutover", "status": cutover["status"], } if detail, ok := cutover["detail"].(string); ok && detail != "" { cutoverCheck["detail"] = detail } checks = append(checks, cutoverCheck) if st, _ := cutover["status"].(string); st == "warn" && overall == "ok" { overall = "degraded" } if st, _ := cutover["status"].(string); st == "fail" { overall = "fail" } failures, failErr := s.diagRecentJobFailures(ctx, failLimit, statusFilter) if failErr != nil && overall == "ok" { overall = "degraded" } if statusFilter == "" || statusFilter == "failed" { if n, ok := queueSummary["failed"].(int64); ok && n > 0 && overall == "ok" { overall = "degraded" } if n, ok := queueSummary["stuck_running"].(int64); ok && n > 0 && overall == "ok" { overall = "degraded" } } aiFails, _ := s.diagRecentAIFailures(ctx, adminDiagnosticsAIFailDefault) migrationInventory := s.diagMigrationInventory(ctx) JSON(w, http.StatusOK, map[string]any{ "status": overall, "generated_at": time.Now().UTC().Format(time.RFC3339), "checks": checks, "queue": queueSummary, "cutover": cutover, "migration_inventory": migrationInventory, "config": s.diagConfigSanity(stripeCfg), "runtime_metrics": metrics.Snapshot(), "recent_failures": failures, "recent_ai_failures": aiFails, "filters": map[string]any{ "status": statusFilter, "failures_limit": failLimit, }, "links": map[string]string{ "stuck_products": "/admin/stuck-products", "orphan_processed": "/admin/orphan-processed", "tasks_cleanup": "/admin/tasks-cleanup", "logs": "/admin/logs", "bootstrap": "/admin/bootstrap", "analytics": "/admin/analytics", "metrics": "/metrics", "readiness": "/api/admin/readiness", }, "notes": []string{ "Diagnostics is for troubleshooting, not marketing analytics.", "Secrets, passwords, and API keys are never included.", "/admin/logs redirects here; stuck cleanup lives under Stuck products.", "Orphan processed: /admin/orphan-processed (dry-run report → confirm delete). API: GET/POST /api/admin/jobs/orphan-processed(-cleanup); POST needs confirm=true.", "Prometheus scrape: GET /metrics (HTTP RED). In production: loopback only unless METRICS_PUBLIC=1. Worker sync series need METRICS_ADDR on the worker process.", "Cutover block: goose version hints + worker age + companies_without_plan + companies_without_api_keys (reissue inventory; presence/counts only; no live Stripe/SMTP; no fake key migration).", "migration_inventory: read-only COUNT of metadata-only files + jobs/history tags — not an import path; blob bytes and default job history stay unmigrated unless ops ran optional domain jobs.", }, }) } func (s *Server) diagDatabase(ctx context.Context) (check map[string]any, ok bool) { start := time.Now() var pinger dbPinger if s.Pool != nil { pinger = s.Pool } ready, status, errMsg := databaseReady(ctx, pinger) check = map[string]any{ "name": "database", "status": status, "latency_ms": time.Since(start).Milliseconds(), } if !ready { check["status"] = "fail" if errMsg != "" { check["detail"] = errMsg } return check, false } check["status"] = "ok" check["detail"] = "ping ok" return check, true } func (s *Server) diagQueue(ctx context.Context) (check map[string]any, summary map[string]any, ok bool) { summary = map[string]any{ "by_status": map[string]int64{}, "stuck_running": int64(0), "driver": "postgres_processing_jobs", } check = map[string]any{ "name": "queue", "status": "ok", "detail": "processing_jobs poller (SKIP LOCKED)", } if s.Pool == nil { check["status"] = "fail" check["detail"] = "database pool unavailable" return check, summary, false } start := time.Now() rows, err := s.Pool.Query(ctx, ` SELECT status, COUNT(*)::bigint FROM processing_jobs GROUP BY status`) if err != nil { check["status"] = "fail" check["detail"] = "queue status query failed" check["latency_ms"] = time.Since(start).Milliseconds() return check, summary, false } defer rows.Close() byStatus := map[string]int64{} var total int64 for rows.Next() { var st string var n int64 if err := rows.Scan(&st, &n); err != nil { check["status"] = "fail" check["detail"] = "queue status scan failed" check["latency_ms"] = time.Since(start).Milliseconds() return check, summary, false } byStatus[st] = n total += n summary[st] = n } if err := rows.Err(); err != nil { check["status"] = "fail" check["detail"] = "queue status rows failed" check["latency_ms"] = time.Since(start).Milliseconds() return check, summary, false } summary["by_status"] = byStatus summary["total"] = total var stuck int64 _ = s.Pool.QueryRow(ctx, ` SELECT COUNT(*)::bigint FROM processing_jobs WHERE status = 'running' AND updated_at < now() - make_interval(secs => $1)`, adminDiagnosticsStuckAfter.Seconds(), ).Scan(&stuck) summary["stuck_running"] = stuck check["latency_ms"] = time.Since(start).Milliseconds() if stuck > 0 { check["status"] = "warn" check["detail"] = "stuck running jobs detected" return check, summary, false } return check, summary, true } func (s *Server) diagCache() map[string]any { // No Redis/memcached in this stack — support KB uses process-local cache only. return map[string]any{ "name": "cache", "status": "ok", "detail": "in-process only (no external cache)", } } func (s *Server) diagStorage() (check map[string]any, ok bool) { check = map[string]any{ "name": "storage", "status": "ok", } dir := strings.TrimSpace(s.Config.UploadDir) if dir == "" { check["status"] = "warn" check["detail"] = "upload dir not configured" return check, false } abs, err := filepath.Abs(dir) if err != nil { check["status"] = "fail" check["detail"] = "upload dir path invalid" return check, false } info, err := os.Stat(abs) if err != nil { check["status"] = "fail" if os.IsNotExist(err) { check["detail"] = "upload dir missing" } else { check["detail"] = "upload dir unavailable" } return check, false } if !info.IsDir() { check["status"] = "fail" check["detail"] = "upload path is not a directory" return check, false } probe := filepath.Join(abs, ".diag_write_probe") if err := os.WriteFile(probe, []byte("ok"), 0o600); err != nil { check["status"] = "fail" check["detail"] = "upload dir not writable" return check, false } _ = os.Remove(probe) // Never return absolute path (may leak host layout); only configured relative name. check["detail"] = "upload dir writable" check["configured"] = true return check, true } func (s *Server) diagMail() map[string]any { enabled := s.Config.SMTPEnabled if s.Mail != nil { enabled = s.Mail.Enabled() } dryRun := s.Config.EmailDryRun hostSet := strings.TrimSpace(s.Config.SMTPHost) != "" status := "ok" detail := "smtp disabled (noop)" switch { case !enabled: detail = "smtp disabled (noop)" case dryRun && hostSet: detail = "smtp enabled; dry-run; host set" case dryRun && !hostSet: detail = "smtp enabled; dry-run; host not set" status = "warn" case hostSet: detail = "smtp enabled; host set" default: detail = "smtp enabled; host not set" status = "warn" } // Presence flags only — never host hostname or credentials. return map[string]any{ "name": "mail", "status": status, "detail": detail, "enabled": enabled, "dry_run": dryRun, "host_set": hostSet, } } // diagCutoverReadiness reports deploy/cutover presence signals for platform admins. // Goose version hints + worker heartbeat age + cheap companies_without_plan / // companies_without_api_keys counts (reissue inventory; no fake key migration). // Never runs live Stripe charges or SMTP sends; never returns secrets/DSNs. func (s *Server) diagCutoverReadiness(ctx context.Context) map[string]any { goose := s.diagGooseVersionHints(ctx) worker := s.diagWorkerAge(ctx) out := map[string]any{ "status": "ok", "detail": "cutover presence ok", "goose": goose, "worker": worker, } if n, ok := s.diagCompaniesWithoutPlan(ctx); ok { out["companies_without_plan"] = n } if n, ok := s.diagCompaniesWithoutAPIKeys(ctx); ok { out["companies_without_api_keys"] = n } status := "ok" detail := "cutover presence ok" gooseStatus, _ := goose["status"].(string) workerStatus, _ := worker["status"].(string) switch { case gooseStatus == "fail" || workerStatus == "fail": status = "fail" detail = "cutover probe failed" case gooseStatus == "warn" || gooseStatus == "skip": status = "warn" if d, ok := goose["detail"].(string); ok && d != "" { detail = d } else { detail = "goose version hints incomplete" } case workerStatus == "missing" || workerStatus == "stale" || workerStatus == "unavailable": status = "warn" if d, ok := worker["detail"].(string); ok && d != "" { detail = d } else { detail = "worker heartbeat not fresh" } } out["status"] = status out["detail"] = detail return out } func (s *Server) diagGooseVersionHints(ctx context.Context) map[string]any { required := make(map[string]bool, len(adminDiagnosticsGooseRequired)) ids := make([]int64, 0, len(adminDiagnosticsGooseRequired)) idToName := make(map[int64]string, len(adminDiagnosticsGooseRequired)) for _, m := range adminDiagnosticsGooseRequired { required[m.Name] = false ids = append(ids, m.ID) idToName[m.ID] = m.Name } out := map[string]any{ "status": "skip", "detail": "database unavailable", "required": required, "expected_min": adminDiagnosticsGooseExpectedMin, } if s.Pool == nil { return out } var versionMax int64 if err := s.Pool.QueryRow(ctx, ` SELECT COALESCE(MAX(version_id), 0)::bigint FROM goose_db_version WHERE is_applied = true`).Scan(&versionMax); err != nil { out["status"] = "warn" out["detail"] = "goose version query unavailable" return out } out["version_max"] = versionMax rows, err := s.Pool.Query(ctx, ` SELECT version_id::bigint FROM goose_db_version WHERE is_applied = true AND version_id = ANY($1)`, ids) if err != nil { out["status"] = "warn" out["detail"] = "goose required migration query unavailable" return out } defer rows.Close() for rows.Next() { var id int64 if err := rows.Scan(&id); err != nil { out["status"] = "warn" out["detail"] = "goose required migration scan failed" return out } if name, ok := idToName[id]; ok { required[name] = true } } if err := rows.Err(); err != nil { out["status"] = "warn" out["detail"] = "goose required migration rows failed" return out } out["required"] = required allApplied := true for _, m := range adminDiagnosticsGooseRequired { if !required[m.Name] { allApplied = false break } } if !allApplied { out["status"] = "warn" out["detail"] = "required cutover migrations missing" return out } if versionMax < adminDiagnosticsGooseExpectedMin { out["status"] = "warn" out["detail"] = "schema behind expected head" return out } out["status"] = "ok" out["detail"] = "required migrations applied" return out } func (s *Server) diagWorkerAge(ctx context.Context) map[string]any { staleAfterS := int64(jobs.DefaultHeartbeatStaleAfter / time.Second) out := map[string]any{ "status": "unavailable", "detail": "worker probe unavailable", "stale_after_s": staleAfterS, } var prober jobs.HeartbeatQuerier if s.Pool != nil { prober = s.Pool } probe := jobs.ProbeWorkerReadiness(ctx, prober, jobs.DefaultHeartbeatStaleAfter) out["status"] = probe.WorkerCheck if probe.LastSeenAgeS >= 0 { out["last_seen_age_s"] = probe.LastSeenAgeS } if probe.Reason != "" { out["reason"] = probe.Reason } switch probe.WorkerCheck { case "ok": out["detail"] = "worker heartbeat fresh" case "missing": out["detail"] = "worker heartbeat missing" case "stale": out["detail"] = "worker heartbeat stale" case "fail": out["detail"] = "worker heartbeat query failed" default: out["detail"] = "worker probe unavailable" } return out } // diagCompaniesWithoutPlan is the cheap cutover hypercare count (same shape as /api/admin/readiness). func (s *Server) diagCompaniesWithoutPlan(ctx context.Context) (count int64, ok bool) { if s.Pool == nil { return 0, false } err := s.Pool.QueryRow(ctx, ` SELECT COUNT(*)::bigint FROM companies c WHERE c.id <> $1 AND NOT EXISTS ( SELECT 1 FROM company_plans cp WHERE cp.company_id = c.id AND cp.is_active = true )`, platformsettings.SystemCompanyID).Scan(&count) if err != nil { return 0, false } return count, true } // diagCompaniesWithoutAPIKeys counts tenants with no non-revoked api_keys. // Legacy secrets were not migrated — inventory for reissue only (no key invent/import). func (s *Server) diagCompaniesWithoutAPIKeys(ctx context.Context) (count int64, ok bool) { if s.Pool == nil { return 0, false } err := s.Pool.QueryRow(ctx, ` SELECT COUNT(*)::bigint FROM companies c WHERE c.id <> $1 AND NOT EXISTS ( SELECT 1 FROM api_keys k WHERE k.company_id = c.id AND k.revoked_at IS NULL )`, platformsettings.SystemCompanyID).Scan(&count) if err != nil { return 0, false } return count, true } // diagMigrationInventory returns cheap read-only COUNTs for accepted ETL gaps // (metadata-only file blobs, optional jobs-domain backfill, tasks history). // Never imports or invents data; never exposes paths/secrets. func (s *Server) diagMigrationInventory(ctx context.Context) map[string]any { notes := []string{ "File blob bytes were never ETL'd — files_metadata_only tags metadata_only_resync_paths or _legacy_file_id.", "Cutover default skips domain jobs; processing_jobs_migrated>0 means optional jobs backfill ran (ai_provider_mode=migrated).", "tasks_total is present history only — no migrated tag on tasks; empty Processing UI after cutover is expected unless jobs ran.", "Read-only inventory — no fake blob/job import from this endpoint.", } out := map[string]any{ "status": "skip", "detail": "database unavailable", "files_total": int64(0), "files_metadata_only": int64(0), "processing_jobs_total": int64(0), "processing_jobs_migrated": int64(0), "tasks_total": int64(0), "jobs_domain_ran": false, "notes": notes, } if s.Pool == nil { return out } var filesTotal, filesMeta, jobsTotal, jobsMigrated, tasksTotal int64 err := s.Pool.QueryRow(ctx, ` SELECT (SELECT COUNT(*)::bigint FROM files), (SELECT COUNT(*)::bigint FROM files WHERE COALESCE(metadata->>'_blob_strategy', '') = 'metadata_only_resync_paths' OR metadata ? '_legacy_file_id'), (SELECT COUNT(*)::bigint FROM processing_jobs), (SELECT COUNT(*)::bigint FROM processing_jobs WHERE COALESCE(ai_provider_mode, '') = 'migrated'), (SELECT COUNT(*)::bigint FROM tasks)`).Scan( &filesTotal, &filesMeta, &jobsTotal, &jobsMigrated, &tasksTotal, ) if err != nil { out["status"] = "warn" out["detail"] = "migration inventory query failed" return out } out["files_total"] = filesTotal out["files_metadata_only"] = filesMeta out["processing_jobs_total"] = jobsTotal out["processing_jobs_migrated"] = jobsMigrated out["tasks_total"] = tasksTotal out["jobs_domain_ran"] = jobsMigrated > 0 out["status"] = "ok" out["detail"] = "read-only ETL gap inventory" return out } // resolveStripeDiagCfg merges env bootstrap with platform_settings when available. // Presence flags only — never returns secret values to callers that stringify cfg. func (s *Server) resolveStripeDiagCfg(ctx context.Context) billing.StripeConfig { if s.Stripe != nil { base := s.Stripe.Cfg if s.Stripe.ResolveCfg != nil { if cfg, err := s.Stripe.ResolveCfg(ctx, base); err == nil { return cfg } } return base } return billing.StripeConfig{ SecretKey: s.Config.StripeSecretKey, WebhookSecret: s.Config.StripeWebhookSecret, ForceMock: s.Config.StripeMock, } } // diagStripeReadiness reports Stripe ops readiness without leaking secret values. // Production: mock must be off; missing secret/webhook keys degrade (fail-closed at use). func diagStripeReadiness(prod bool, cfg billing.StripeConfig) (check map[string]any, ok bool) { secretSet := strings.TrimSpace(cfg.SecretKey) != "" webhookSet := strings.TrimSpace(cfg.WebhookSecret) != "" mock := cfg.ForceMock mockRejectedInProd := !prod || !mock check = map[string]any{ "name": "stripe", "status": "ok", "secret_key_set": secretSet, "webhook_secret_set": webhookSet, "mock": mock, "mock_rejected_in_prod": mockRejectedInProd, } if prod && mock { check["status"] = "fail" check["detail"] = "STRIPE_MOCK must be false in production" return check, false } if prod && (!secretSet || !webhookSet) { check["status"] = "warn" parts := make([]string, 0, 2) if !secretSet { parts = append(parts, "secret key") } if !webhookSet { parts = append(parts, "webhook secret") } check["detail"] = "missing " + strings.Join(parts, " and ") + " (checkout/webhooks fail closed)" return check, false } if mock { check["detail"] = "mock mode enabled" return check, true } if !secretSet { check["status"] = "warn" check["detail"] = "secret key not set (mock purchases require STRIPE_MOCK)" return check, false } if !webhookSet { check["status"] = "warn" check["detail"] = "webhook secret not set" return check, false } check["detail"] = "keys present" return check, true } func (s *Server) diagConfigSanity(stripe billing.StripeConfig) map[string]any { smtpEnabled := s.Config.SMTPEnabled if s.Mail != nil { smtpEnabled = s.Mail.Enabled() } secretSet := strings.TrimSpace(stripe.SecretKey) != "" webhookSet := strings.TrimSpace(stripe.WebhookSecret) != "" return map[string]any{ "app_env": s.Config.AppEnv, "maintenance_mode": s.Config.MaintenanceMode, "read_only_mode": s.Config.ReadOnlyMode, "session_secure": s.Config.SessionSecure, "smtp_enabled": smtpEnabled, "email_dry_run": s.Config.EmailDryRun, "smtp_host_set": strings.TrimSpace(s.Config.SMTPHost) != "", "stripe_mock": stripe.ForceMock, "eprel_enabled": s.Config.EPRELEnabled, "processing_rpm": s.Config.ProcessingRPM, "processing_batch_size": s.Config.ProcessingBatchSize, "processing_max_retries": s.Config.ProcessingMaxRetries, "upload_dir_configured": strings.TrimSpace(s.Config.UploadDir) != "", "trusted_proxies_configured": len(s.Config.TrustedProxies) > 0, "web_origin_set": strings.TrimSpace(s.Config.WebOrigin) != "", "public_api_url_set": strings.TrimSpace(s.Config.PublicAPIURL) != "", // Presence flags only — never the secret values. "token_signing_secret_set": strings.TrimSpace(s.Config.TokenSigningSecret) != "", "openai_key_set": strings.TrimSpace(s.Config.OpenAIAPIKey) != "", "pinecone_key_set": strings.TrimSpace(s.Config.PineconeAPIKey) != "", "stripe_secret_set": secretSet, "stripe_webhook_secret_set": webhookSet, "stripe_mock_rejected_in_prod": !s.Config.IsProduction() || !stripe.ForceMock, "credentials_encryption_key_set": strings.TrimSpace(s.Config.CredentialsEncryptionKey) != "", } } func (s *Server) diagRecentJobFailures(ctx context.Context, limit int, statusFilter string) ([]map[string]any, error) { out := make([]map[string]any, 0) if s.Pool == nil { return out, errors.New("database unavailable") } status := statusFilter if status == "" { status = "failed" } rows, err := s.Pool.Query(ctx, ` SELECT id, company_id, status, total_products, processed_products, error, created_at, updated_at FROM processing_jobs WHERE status = $1 ORDER BY updated_at DESC LIMIT $2`, status, limit) if err != nil { return out, err } defer rows.Close() for rows.Next() { var ( id, companyID uuid.UUID st string total, processed int errMsg *string createdAt, updatedAt time.Time ) if err := rows.Scan(&id, &companyID, &st, &total, &processed, &errMsg, &createdAt, &updatedAt); err != nil { return out, err } safeErr := "" if errMsg != nil && *errMsg != "" { safeErr = processing.TruncateError(errors.New(*errMsg)) } out = append(out, map[string]any{ "id": id, "company_id": companyID, "status": st, "total_products": total, "processed_products": processed, "error": safeErr, "created_at": createdAt.UTC().Format(time.RFC3339), "updated_at": updatedAt.UTC().Format(time.RFC3339), }) } return out, rows.Err() } func (s *Server) diagRecentAIFailures(ctx context.Context, limit int) ([]map[string]any, error) { out := make([]map[string]any, 0) if s.Pool == nil { return out, nil } if limit <= 0 { limit = adminDiagnosticsAIFailDefault } if limit > adminDiagnosticsAIFailMax { limit = adminDiagnosticsAIFailMax } rows, err := s.Pool.Query(ctx, ` SELECT id, ticket_id, company_id, kind, created_at FROM support_ticket_activity WHERE kind = 'ai_failed' ORDER BY created_at DESC LIMIT $1`, limit) if err != nil { // Table may be absent on older DBs — soft-skip. return out, nil } defer rows.Close() for rows.Next() { var ( id, ticketID, companyID uuid.UUID kind string createdAt time.Time ) if err := rows.Scan(&id, &ticketID, &companyID, &kind, &createdAt); err != nil { return out, nil } out = append(out, map[string]any{ "id": id, "ticket_id": ticketID, "company_id": companyID, "kind": kind, "created_at": createdAt.UTC().Format(time.RFC3339), }) } return out, nil }