818 lines
25 KiB
Go
818 lines
25 KiB
Go
|
package httpapi
|
|||
|
|
|
|||
|
|
import (
|
|||
|
|
"context"
|
|||
|
|
"errors"
|
|||
|
|
"net/http"
|
|||
|
|
"os"
|
|||
|
|
"path/filepath"
|
|||
|
|
"strconv"
|
|||
|
|
"strings"
|
|||
|
|
"time"
|
|||
|
|
|
|||
|
|
"github.com/descrybe/descrybe-v2/apps/api/internal/billing"
|
|||
|
|
"github.com/descrybe/descrybe-v2/apps/api/internal/jobs"
|
|||
|
|
"github.com/descrybe/descrybe-v2/apps/api/internal/metrics"
|
|||
|
|
"github.com/descrybe/descrybe-v2/apps/api/internal/platformsettings"
|
|||
|
|
"github.com/descrybe/descrybe-v2/apps/api/internal/processing"
|
|||
|
|
"github.com/google/uuid"
|
|||
|
|
)
|
|||
|
|
|
|||
|
|
const (
|
|||
|
|
adminDiagnosticsTimeout = 3 * time.Second
|
|||
|
|
adminDiagnosticsDefaultFails = 25
|
|||
|
|
adminDiagnosticsMaxFails = 50
|
|||
|
|
adminDiagnosticsStuckAfter = 2 * time.Hour
|
|||
|
|
adminDiagnosticsAIFailDefault = 15
|
|||
|
|
adminDiagnosticsAIFailMax = 30
|
|||
|
|
// Schema head expected by cutover-deploy-check (goose 039–042).
|
|||
|
|
adminDiagnosticsGooseExpectedMin = int64(42)
|
|||
|
|
)
|
|||
|
|
|
|||
|
|
// Required goose versions for cutover readiness (match scripts/cutover-deploy-check.mjs).
|
|||
|
|
var adminDiagnosticsGooseRequired = []struct {
|
|||
|
|
ID int64
|
|||
|
|
Name string
|
|||
|
|
}{
|
|||
|
|
{39, "039_worker_heartbeats"},
|
|||
|
|
{40, "040_job_hotpath_indexes"},
|
|||
|
|
{41, "041_password_reset_tokens"},
|
|||
|
|
{42, "042_user_session_version"},
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
// handleAdminDiagnostics returns operational health for platform admins.
|
|||
|
|
// GET /api/admin/diagnostics?failures_limit=25&status=failed
|
|||
|
|
// Never exposes secrets, DSNs, API keys, or passwords.
|
|||
|
|
func (s *Server) handleAdminDiagnostics(w http.ResponseWriter, r *http.Request) {
|
|||
|
|
ctx, cancel := context.WithTimeout(r.Context(), adminDiagnosticsTimeout)
|
|||
|
|
defer cancel()
|
|||
|
|
|
|||
|
|
failLimit := adminDiagnosticsDefaultFails
|
|||
|
|
if raw := strings.TrimSpace(r.URL.Query().Get("failures_limit")); raw != "" {
|
|||
|
|
if n, err := strconv.Atoi(raw); err == nil && n > 0 {
|
|||
|
|
failLimit = n
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
if failLimit > adminDiagnosticsMaxFails {
|
|||
|
|
failLimit = adminDiagnosticsMaxFails
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
statusFilter := strings.ToLower(strings.TrimSpace(r.URL.Query().Get("status")))
|
|||
|
|
switch statusFilter {
|
|||
|
|
case "", "all", "failed", "running", "pending", "completed", "cancelled":
|
|||
|
|
default:
|
|||
|
|
Error(w, http.StatusBadRequest, "invalid status filter")
|
|||
|
|
return
|
|||
|
|
}
|
|||
|
|
if statusFilter == "all" {
|
|||
|
|
statusFilter = ""
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
checks := make([]map[string]any, 0, 7)
|
|||
|
|
overall := "ok"
|
|||
|
|
stripeCfg := s.resolveStripeDiagCfg(ctx)
|
|||
|
|
|
|||
|
|
dbCheck, dbOK := s.diagDatabase(ctx)
|
|||
|
|
checks = append(checks, dbCheck)
|
|||
|
|
if !dbOK {
|
|||
|
|
overall = "fail"
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
queueCheck, queueSummary, queueOK := s.diagQueue(ctx)
|
|||
|
|
checks = append(checks, queueCheck)
|
|||
|
|
if !queueOK && overall != "fail" {
|
|||
|
|
overall = "degraded"
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
cacheCheck := s.diagCache()
|
|||
|
|
checks = append(checks, cacheCheck)
|
|||
|
|
|
|||
|
|
storageCheck, storageOK := s.diagStorage()
|
|||
|
|
checks = append(checks, storageCheck)
|
|||
|
|
if !storageOK && overall != "fail" {
|
|||
|
|
overall = "degraded"
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
mailCheck := s.diagMail()
|
|||
|
|
checks = append(checks, mailCheck)
|
|||
|
|
|
|||
|
|
stripeCheck, stripeOK := diagStripeReadiness(s.Config.IsProduction(), stripeCfg)
|
|||
|
|
checks = append(checks, stripeCheck)
|
|||
|
|
if !stripeOK && overall == "ok" {
|
|||
|
|
overall = "degraded"
|
|||
|
|
}
|
|||
|
|
if stripeCheck["status"] == "fail" {
|
|||
|
|
overall = "fail"
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
cutover := s.diagCutoverReadiness(ctx)
|
|||
|
|
cutoverCheck := map[string]any{
|
|||
|
|
"name": "cutover",
|
|||
|
|
"status": cutover["status"],
|
|||
|
|
}
|
|||
|
|
if detail, ok := cutover["detail"].(string); ok && detail != "" {
|
|||
|
|
cutoverCheck["detail"] = detail
|
|||
|
|
}
|
|||
|
|
checks = append(checks, cutoverCheck)
|
|||
|
|
if st, _ := cutover["status"].(string); st == "warn" && overall == "ok" {
|
|||
|
|
overall = "degraded"
|
|||
|
|
}
|
|||
|
|
if st, _ := cutover["status"].(string); st == "fail" {
|
|||
|
|
overall = "fail"
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
failures, failErr := s.diagRecentJobFailures(ctx, failLimit, statusFilter)
|
|||
|
|
if failErr != nil && overall == "ok" {
|
|||
|
|
overall = "degraded"
|
|||
|
|
}
|
|||
|
|
if statusFilter == "" || statusFilter == "failed" {
|
|||
|
|
if n, ok := queueSummary["failed"].(int64); ok && n > 0 && overall == "ok" {
|
|||
|
|
overall = "degraded"
|
|||
|
|
}
|
|||
|
|
if n, ok := queueSummary["stuck_running"].(int64); ok && n > 0 && overall == "ok" {
|
|||
|
|
overall = "degraded"
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
aiFails, _ := s.diagRecentAIFailures(ctx, adminDiagnosticsAIFailDefault)
|
|||
|
|
migrationInventory := s.diagMigrationInventory(ctx)
|
|||
|
|
|
|||
|
|
JSON(w, http.StatusOK, map[string]any{
|
|||
|
|
"status": overall,
|
|||
|
|
"generated_at": time.Now().UTC().Format(time.RFC3339),
|
|||
|
|
"checks": checks,
|
|||
|
|
"queue": queueSummary,
|
|||
|
|
"cutover": cutover,
|
|||
|
|
"migration_inventory": migrationInventory,
|
|||
|
|
"config": s.diagConfigSanity(stripeCfg),
|
|||
|
|
"runtime_metrics": metrics.Snapshot(),
|
|||
|
|
"recent_failures": failures,
|
|||
|
|
"recent_ai_failures": aiFails,
|
|||
|
|
"filters": map[string]any{
|
|||
|
|
"status": statusFilter,
|
|||
|
|
"failures_limit": failLimit,
|
|||
|
|
},
|
|||
|
|
"links": map[string]string{
|
|||
|
|
"stuck_products": "/admin/stuck-products",
|
|||
|
|
"orphan_processed": "/admin/orphan-processed",
|
|||
|
|
"tasks_cleanup": "/admin/tasks-cleanup",
|
|||
|
|
"logs": "/admin/logs",
|
|||
|
|
"bootstrap": "/admin/bootstrap",
|
|||
|
|
"analytics": "/admin/analytics",
|
|||
|
|
"metrics": "/metrics",
|
|||
|
|
"readiness": "/api/admin/readiness",
|
|||
|
|
},
|
|||
|
|
"notes": []string{
|
|||
|
|
"Diagnostics is for troubleshooting, not marketing analytics.",
|
|||
|
|
"Secrets, passwords, and API keys are never included.",
|
|||
|
|
"/admin/logs redirects here; stuck cleanup lives under Stuck products.",
|
|||
|
|
"Orphan processed: /admin/orphan-processed (dry-run report → confirm delete). API: GET/POST /api/admin/jobs/orphan-processed(-cleanup); POST needs confirm=true.",
|
|||
|
|
"Prometheus scrape: GET /metrics (HTTP RED). In production: loopback only unless METRICS_PUBLIC=1. Worker sync series need METRICS_ADDR on the worker process.",
|
|||
|
|
"Cutover block: goose version hints + worker age + companies_without_plan + companies_without_api_keys (reissue inventory; presence/counts only; no live Stripe/SMTP; no fake key migration).",
|
|||
|
|
"migration_inventory: read-only COUNT of metadata-only files + jobs/history tags — not an import path; blob bytes and default job history stay unmigrated unless ops ran optional domain jobs.",
|
|||
|
|
},
|
|||
|
|
})
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
func (s *Server) diagDatabase(ctx context.Context) (check map[string]any, ok bool) {
|
|||
|
|
start := time.Now()
|
|||
|
|
var pinger dbPinger
|
|||
|
|
if s.Pool != nil {
|
|||
|
|
pinger = s.Pool
|
|||
|
|
}
|
|||
|
|
ready, status, errMsg := databaseReady(ctx, pinger)
|
|||
|
|
check = map[string]any{
|
|||
|
|
"name": "database",
|
|||
|
|
"status": status,
|
|||
|
|
"latency_ms": time.Since(start).Milliseconds(),
|
|||
|
|
}
|
|||
|
|
if !ready {
|
|||
|
|
check["status"] = "fail"
|
|||
|
|
if errMsg != "" {
|
|||
|
|
check["detail"] = errMsg
|
|||
|
|
}
|
|||
|
|
return check, false
|
|||
|
|
}
|
|||
|
|
check["status"] = "ok"
|
|||
|
|
check["detail"] = "ping ok"
|
|||
|
|
return check, true
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
func (s *Server) diagQueue(ctx context.Context) (check map[string]any, summary map[string]any, ok bool) {
|
|||
|
|
summary = map[string]any{
|
|||
|
|
"by_status": map[string]int64{},
|
|||
|
|
"stuck_running": int64(0),
|
|||
|
|
"driver": "postgres_processing_jobs",
|
|||
|
|
}
|
|||
|
|
check = map[string]any{
|
|||
|
|
"name": "queue",
|
|||
|
|
"status": "ok",
|
|||
|
|
"detail": "processing_jobs poller (SKIP LOCKED)",
|
|||
|
|
}
|
|||
|
|
if s.Pool == nil {
|
|||
|
|
check["status"] = "fail"
|
|||
|
|
check["detail"] = "database pool unavailable"
|
|||
|
|
return check, summary, false
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
start := time.Now()
|
|||
|
|
rows, err := s.Pool.Query(ctx, `
|
|||
|
|
SELECT status, COUNT(*)::bigint
|
|||
|
|
FROM processing_jobs
|
|||
|
|
GROUP BY status`)
|
|||
|
|
if err != nil {
|
|||
|
|
check["status"] = "fail"
|
|||
|
|
check["detail"] = "queue status query failed"
|
|||
|
|
check["latency_ms"] = time.Since(start).Milliseconds()
|
|||
|
|
return check, summary, false
|
|||
|
|
}
|
|||
|
|
defer rows.Close()
|
|||
|
|
|
|||
|
|
byStatus := map[string]int64{}
|
|||
|
|
var total int64
|
|||
|
|
for rows.Next() {
|
|||
|
|
var st string
|
|||
|
|
var n int64
|
|||
|
|
if err := rows.Scan(&st, &n); err != nil {
|
|||
|
|
check["status"] = "fail"
|
|||
|
|
check["detail"] = "queue status scan failed"
|
|||
|
|
check["latency_ms"] = time.Since(start).Milliseconds()
|
|||
|
|
return check, summary, false
|
|||
|
|
}
|
|||
|
|
byStatus[st] = n
|
|||
|
|
total += n
|
|||
|
|
summary[st] = n
|
|||
|
|
}
|
|||
|
|
if err := rows.Err(); err != nil {
|
|||
|
|
check["status"] = "fail"
|
|||
|
|
check["detail"] = "queue status rows failed"
|
|||
|
|
check["latency_ms"] = time.Since(start).Milliseconds()
|
|||
|
|
return check, summary, false
|
|||
|
|
}
|
|||
|
|
summary["by_status"] = byStatus
|
|||
|
|
summary["total"] = total
|
|||
|
|
|
|||
|
|
var stuck int64
|
|||
|
|
_ = s.Pool.QueryRow(ctx, `
|
|||
|
|
SELECT COUNT(*)::bigint FROM processing_jobs
|
|||
|
|
WHERE status = 'running'
|
|||
|
|
AND updated_at < now() - make_interval(secs => $1)`,
|
|||
|
|
adminDiagnosticsStuckAfter.Seconds(),
|
|||
|
|
).Scan(&stuck)
|
|||
|
|
summary["stuck_running"] = stuck
|
|||
|
|
|
|||
|
|
check["latency_ms"] = time.Since(start).Milliseconds()
|
|||
|
|
if stuck > 0 {
|
|||
|
|
check["status"] = "warn"
|
|||
|
|
check["detail"] = "stuck running jobs detected"
|
|||
|
|
return check, summary, false
|
|||
|
|
}
|
|||
|
|
return check, summary, true
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
func (s *Server) diagCache() map[string]any {
|
|||
|
|
// No Redis/memcached in this stack — support KB uses process-local cache only.
|
|||
|
|
return map[string]any{
|
|||
|
|
"name": "cache",
|
|||
|
|
"status": "ok",
|
|||
|
|
"detail": "in-process only (no external cache)",
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
func (s *Server) diagStorage() (check map[string]any, ok bool) {
|
|||
|
|
check = map[string]any{
|
|||
|
|
"name": "storage",
|
|||
|
|
"status": "ok",
|
|||
|
|
}
|
|||
|
|
dir := strings.TrimSpace(s.Config.UploadDir)
|
|||
|
|
if dir == "" {
|
|||
|
|
check["status"] = "warn"
|
|||
|
|
check["detail"] = "upload dir not configured"
|
|||
|
|
return check, false
|
|||
|
|
}
|
|||
|
|
abs, err := filepath.Abs(dir)
|
|||
|
|
if err != nil {
|
|||
|
|
check["status"] = "fail"
|
|||
|
|
check["detail"] = "upload dir path invalid"
|
|||
|
|
return check, false
|
|||
|
|
}
|
|||
|
|
info, err := os.Stat(abs)
|
|||
|
|
if err != nil {
|
|||
|
|
check["status"] = "fail"
|
|||
|
|
if os.IsNotExist(err) {
|
|||
|
|
check["detail"] = "upload dir missing"
|
|||
|
|
} else {
|
|||
|
|
check["detail"] = "upload dir unavailable"
|
|||
|
|
}
|
|||
|
|
return check, false
|
|||
|
|
}
|
|||
|
|
if !info.IsDir() {
|
|||
|
|
check["status"] = "fail"
|
|||
|
|
check["detail"] = "upload path is not a directory"
|
|||
|
|
return check, false
|
|||
|
|
}
|
|||
|
|
probe := filepath.Join(abs, ".diag_write_probe")
|
|||
|
|
if err := os.WriteFile(probe, []byte("ok"), 0o600); err != nil {
|
|||
|
|
check["status"] = "fail"
|
|||
|
|
check["detail"] = "upload dir not writable"
|
|||
|
|
return check, false
|
|||
|
|
}
|
|||
|
|
_ = os.Remove(probe)
|
|||
|
|
// Never return absolute path (may leak host layout); only configured relative name.
|
|||
|
|
check["detail"] = "upload dir writable"
|
|||
|
|
check["configured"] = true
|
|||
|
|
return check, true
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
func (s *Server) diagMail() map[string]any {
|
|||
|
|
enabled := s.Config.SMTPEnabled
|
|||
|
|
if s.Mail != nil {
|
|||
|
|
enabled = s.Mail.Enabled()
|
|||
|
|
}
|
|||
|
|
dryRun := s.Config.EmailDryRun
|
|||
|
|
hostSet := strings.TrimSpace(s.Config.SMTPHost) != ""
|
|||
|
|
|
|||
|
|
status := "ok"
|
|||
|
|
detail := "smtp disabled (noop)"
|
|||
|
|
switch {
|
|||
|
|
case !enabled:
|
|||
|
|
detail = "smtp disabled (noop)"
|
|||
|
|
case dryRun && hostSet:
|
|||
|
|
detail = "smtp enabled; dry-run; host set"
|
|||
|
|
case dryRun && !hostSet:
|
|||
|
|
detail = "smtp enabled; dry-run; host not set"
|
|||
|
|
status = "warn"
|
|||
|
|
case hostSet:
|
|||
|
|
detail = "smtp enabled; host set"
|
|||
|
|
default:
|
|||
|
|
detail = "smtp enabled; host not set"
|
|||
|
|
status = "warn"
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
// Presence flags only — never host hostname or credentials.
|
|||
|
|
return map[string]any{
|
|||
|
|
"name": "mail",
|
|||
|
|
"status": status,
|
|||
|
|
"detail": detail,
|
|||
|
|
"enabled": enabled,
|
|||
|
|
"dry_run": dryRun,
|
|||
|
|
"host_set": hostSet,
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
// diagCutoverReadiness reports deploy/cutover presence signals for platform admins.
|
|||
|
|
// Goose version hints + worker heartbeat age + cheap companies_without_plan /
|
|||
|
|
// companies_without_api_keys counts (reissue inventory; no fake key migration).
|
|||
|
|
// Never runs live Stripe charges or SMTP sends; never returns secrets/DSNs.
|
|||
|
|
func (s *Server) diagCutoverReadiness(ctx context.Context) map[string]any {
|
|||
|
|
goose := s.diagGooseVersionHints(ctx)
|
|||
|
|
worker := s.diagWorkerAge(ctx)
|
|||
|
|
|
|||
|
|
out := map[string]any{
|
|||
|
|
"status": "ok",
|
|||
|
|
"detail": "cutover presence ok",
|
|||
|
|
"goose": goose,
|
|||
|
|
"worker": worker,
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
if n, ok := s.diagCompaniesWithoutPlan(ctx); ok {
|
|||
|
|
out["companies_without_plan"] = n
|
|||
|
|
}
|
|||
|
|
if n, ok := s.diagCompaniesWithoutAPIKeys(ctx); ok {
|
|||
|
|
out["companies_without_api_keys"] = n
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
status := "ok"
|
|||
|
|
detail := "cutover presence ok"
|
|||
|
|
gooseStatus, _ := goose["status"].(string)
|
|||
|
|
workerStatus, _ := worker["status"].(string)
|
|||
|
|
|
|||
|
|
switch {
|
|||
|
|
case gooseStatus == "fail" || workerStatus == "fail":
|
|||
|
|
status = "fail"
|
|||
|
|
detail = "cutover probe failed"
|
|||
|
|
case gooseStatus == "warn" || gooseStatus == "skip":
|
|||
|
|
status = "warn"
|
|||
|
|
if d, ok := goose["detail"].(string); ok && d != "" {
|
|||
|
|
detail = d
|
|||
|
|
} else {
|
|||
|
|
detail = "goose version hints incomplete"
|
|||
|
|
}
|
|||
|
|
case workerStatus == "missing" || workerStatus == "stale" || workerStatus == "unavailable":
|
|||
|
|
status = "warn"
|
|||
|
|
if d, ok := worker["detail"].(string); ok && d != "" {
|
|||
|
|
detail = d
|
|||
|
|
} else {
|
|||
|
|
detail = "worker heartbeat not fresh"
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
out["status"] = status
|
|||
|
|
out["detail"] = detail
|
|||
|
|
return out
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
func (s *Server) diagGooseVersionHints(ctx context.Context) map[string]any {
|
|||
|
|
required := make(map[string]bool, len(adminDiagnosticsGooseRequired))
|
|||
|
|
ids := make([]int64, 0, len(adminDiagnosticsGooseRequired))
|
|||
|
|
idToName := make(map[int64]string, len(adminDiagnosticsGooseRequired))
|
|||
|
|
for _, m := range adminDiagnosticsGooseRequired {
|
|||
|
|
required[m.Name] = false
|
|||
|
|
ids = append(ids, m.ID)
|
|||
|
|
idToName[m.ID] = m.Name
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
out := map[string]any{
|
|||
|
|
"status": "skip",
|
|||
|
|
"detail": "database unavailable",
|
|||
|
|
"required": required,
|
|||
|
|
"expected_min": adminDiagnosticsGooseExpectedMin,
|
|||
|
|
}
|
|||
|
|
if s.Pool == nil {
|
|||
|
|
return out
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
var versionMax int64
|
|||
|
|
if err := s.Pool.QueryRow(ctx, `
|
|||
|
|
SELECT COALESCE(MAX(version_id), 0)::bigint
|
|||
|
|
FROM goose_db_version
|
|||
|
|
WHERE is_applied = true`).Scan(&versionMax); err != nil {
|
|||
|
|
out["status"] = "warn"
|
|||
|
|
out["detail"] = "goose version query unavailable"
|
|||
|
|
return out
|
|||
|
|
}
|
|||
|
|
out["version_max"] = versionMax
|
|||
|
|
|
|||
|
|
rows, err := s.Pool.Query(ctx, `
|
|||
|
|
SELECT version_id::bigint
|
|||
|
|
FROM goose_db_version
|
|||
|
|
WHERE is_applied = true AND version_id = ANY($1)`, ids)
|
|||
|
|
if err != nil {
|
|||
|
|
out["status"] = "warn"
|
|||
|
|
out["detail"] = "goose required migration query unavailable"
|
|||
|
|
return out
|
|||
|
|
}
|
|||
|
|
defer rows.Close()
|
|||
|
|
|
|||
|
|
for rows.Next() {
|
|||
|
|
var id int64
|
|||
|
|
if err := rows.Scan(&id); err != nil {
|
|||
|
|
out["status"] = "warn"
|
|||
|
|
out["detail"] = "goose required migration scan failed"
|
|||
|
|
return out
|
|||
|
|
}
|
|||
|
|
if name, ok := idToName[id]; ok {
|
|||
|
|
required[name] = true
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
if err := rows.Err(); err != nil {
|
|||
|
|
out["status"] = "warn"
|
|||
|
|
out["detail"] = "goose required migration rows failed"
|
|||
|
|
return out
|
|||
|
|
}
|
|||
|
|
out["required"] = required
|
|||
|
|
|
|||
|
|
allApplied := true
|
|||
|
|
for _, m := range adminDiagnosticsGooseRequired {
|
|||
|
|
if !required[m.Name] {
|
|||
|
|
allApplied = false
|
|||
|
|
break
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
if !allApplied {
|
|||
|
|
out["status"] = "warn"
|
|||
|
|
out["detail"] = "required cutover migrations missing"
|
|||
|
|
return out
|
|||
|
|
}
|
|||
|
|
if versionMax < adminDiagnosticsGooseExpectedMin {
|
|||
|
|
out["status"] = "warn"
|
|||
|
|
out["detail"] = "schema behind expected head"
|
|||
|
|
return out
|
|||
|
|
}
|
|||
|
|
out["status"] = "ok"
|
|||
|
|
out["detail"] = "required migrations applied"
|
|||
|
|
return out
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
func (s *Server) diagWorkerAge(ctx context.Context) map[string]any {
|
|||
|
|
staleAfterS := int64(jobs.DefaultHeartbeatStaleAfter / time.Second)
|
|||
|
|
out := map[string]any{
|
|||
|
|
"status": "unavailable",
|
|||
|
|
"detail": "worker probe unavailable",
|
|||
|
|
"stale_after_s": staleAfterS,
|
|||
|
|
}
|
|||
|
|
var prober jobs.HeartbeatQuerier
|
|||
|
|
if s.Pool != nil {
|
|||
|
|
prober = s.Pool
|
|||
|
|
}
|
|||
|
|
probe := jobs.ProbeWorkerReadiness(ctx, prober, jobs.DefaultHeartbeatStaleAfter)
|
|||
|
|
out["status"] = probe.WorkerCheck
|
|||
|
|
if probe.LastSeenAgeS >= 0 {
|
|||
|
|
out["last_seen_age_s"] = probe.LastSeenAgeS
|
|||
|
|
}
|
|||
|
|
if probe.Reason != "" {
|
|||
|
|
out["reason"] = probe.Reason
|
|||
|
|
}
|
|||
|
|
switch probe.WorkerCheck {
|
|||
|
|
case "ok":
|
|||
|
|
out["detail"] = "worker heartbeat fresh"
|
|||
|
|
case "missing":
|
|||
|
|
out["detail"] = "worker heartbeat missing"
|
|||
|
|
case "stale":
|
|||
|
|
out["detail"] = "worker heartbeat stale"
|
|||
|
|
case "fail":
|
|||
|
|
out["detail"] = "worker heartbeat query failed"
|
|||
|
|
default:
|
|||
|
|
out["detail"] = "worker probe unavailable"
|
|||
|
|
}
|
|||
|
|
return out
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
// diagCompaniesWithoutPlan is the cheap cutover hypercare count (same shape as /api/admin/readiness).
|
|||
|
|
func (s *Server) diagCompaniesWithoutPlan(ctx context.Context) (count int64, ok bool) {
|
|||
|
|
if s.Pool == nil {
|
|||
|
|
return 0, false
|
|||
|
|
}
|
|||
|
|
err := s.Pool.QueryRow(ctx, `
|
|||
|
|
SELECT COUNT(*)::bigint FROM companies c
|
|||
|
|
WHERE c.id <> $1
|
|||
|
|
AND NOT EXISTS (
|
|||
|
|
SELECT 1 FROM company_plans cp
|
|||
|
|
WHERE cp.company_id = c.id AND cp.is_active = true
|
|||
|
|
)`, platformsettings.SystemCompanyID).Scan(&count)
|
|||
|
|
if err != nil {
|
|||
|
|
return 0, false
|
|||
|
|
}
|
|||
|
|
return count, true
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
// diagCompaniesWithoutAPIKeys counts tenants with no non-revoked api_keys.
|
|||
|
|
// Legacy secrets were not migrated — inventory for reissue only (no key invent/import).
|
|||
|
|
func (s *Server) diagCompaniesWithoutAPIKeys(ctx context.Context) (count int64, ok bool) {
|
|||
|
|
if s.Pool == nil {
|
|||
|
|
return 0, false
|
|||
|
|
}
|
|||
|
|
err := s.Pool.QueryRow(ctx, `
|
|||
|
|
SELECT COUNT(*)::bigint FROM companies c
|
|||
|
|
WHERE c.id <> $1
|
|||
|
|
AND NOT EXISTS (
|
|||
|
|
SELECT 1 FROM api_keys k
|
|||
|
|
WHERE k.company_id = c.id AND k.revoked_at IS NULL
|
|||
|
|
)`, platformsettings.SystemCompanyID).Scan(&count)
|
|||
|
|
if err != nil {
|
|||
|
|
return 0, false
|
|||
|
|
}
|
|||
|
|
return count, true
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
// diagMigrationInventory returns cheap read-only COUNTs for accepted ETL gaps
|
|||
|
|
// (metadata-only file blobs, optional jobs-domain backfill, tasks history).
|
|||
|
|
// Never imports or invents data; never exposes paths/secrets.
|
|||
|
|
func (s *Server) diagMigrationInventory(ctx context.Context) map[string]any {
|
|||
|
|
notes := []string{
|
|||
|
|
"File blob bytes were never ETL'd — files_metadata_only tags metadata_only_resync_paths or _legacy_file_id.",
|
|||
|
|
"Cutover default skips domain jobs; processing_jobs_migrated>0 means optional jobs backfill ran (ai_provider_mode=migrated).",
|
|||
|
|
"tasks_total is present history only — no migrated tag on tasks; empty Processing UI after cutover is expected unless jobs ran.",
|
|||
|
|
"Read-only inventory — no fake blob/job import from this endpoint.",
|
|||
|
|
}
|
|||
|
|
out := map[string]any{
|
|||
|
|
"status": "skip",
|
|||
|
|
"detail": "database unavailable",
|
|||
|
|
"files_total": int64(0),
|
|||
|
|
"files_metadata_only": int64(0),
|
|||
|
|
"processing_jobs_total": int64(0),
|
|||
|
|
"processing_jobs_migrated": int64(0),
|
|||
|
|
"tasks_total": int64(0),
|
|||
|
|
"jobs_domain_ran": false,
|
|||
|
|
"notes": notes,
|
|||
|
|
}
|
|||
|
|
if s.Pool == nil {
|
|||
|
|
return out
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
var filesTotal, filesMeta, jobsTotal, jobsMigrated, tasksTotal int64
|
|||
|
|
err := s.Pool.QueryRow(ctx, `
|
|||
|
|
SELECT
|
|||
|
|
(SELECT COUNT(*)::bigint FROM files),
|
|||
|
|
(SELECT COUNT(*)::bigint FROM files
|
|||
|
|
WHERE COALESCE(metadata->>'_blob_strategy', '') = 'metadata_only_resync_paths'
|
|||
|
|
OR metadata ? '_legacy_file_id'),
|
|||
|
|
(SELECT COUNT(*)::bigint FROM processing_jobs),
|
|||
|
|
(SELECT COUNT(*)::bigint FROM processing_jobs
|
|||
|
|
WHERE COALESCE(ai_provider_mode, '') = 'migrated'),
|
|||
|
|
(SELECT COUNT(*)::bigint FROM tasks)`).Scan(
|
|||
|
|
&filesTotal, &filesMeta, &jobsTotal, &jobsMigrated, &tasksTotal,
|
|||
|
|
)
|
|||
|
|
if err != nil {
|
|||
|
|
out["status"] = "warn"
|
|||
|
|
out["detail"] = "migration inventory query failed"
|
|||
|
|
return out
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
out["files_total"] = filesTotal
|
|||
|
|
out["files_metadata_only"] = filesMeta
|
|||
|
|
out["processing_jobs_total"] = jobsTotal
|
|||
|
|
out["processing_jobs_migrated"] = jobsMigrated
|
|||
|
|
out["tasks_total"] = tasksTotal
|
|||
|
|
out["jobs_domain_ran"] = jobsMigrated > 0
|
|||
|
|
out["status"] = "ok"
|
|||
|
|
out["detail"] = "read-only ETL gap inventory"
|
|||
|
|
return out
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
// resolveStripeDiagCfg merges env bootstrap with platform_settings when available.
|
|||
|
|
// Presence flags only — never returns secret values to callers that stringify cfg.
|
|||
|
|
func (s *Server) resolveStripeDiagCfg(ctx context.Context) billing.StripeConfig {
|
|||
|
|
if s.Stripe != nil {
|
|||
|
|
base := s.Stripe.Cfg
|
|||
|
|
if s.Stripe.ResolveCfg != nil {
|
|||
|
|
if cfg, err := s.Stripe.ResolveCfg(ctx, base); err == nil {
|
|||
|
|
return cfg
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
return base
|
|||
|
|
}
|
|||
|
|
return billing.StripeConfig{
|
|||
|
|
SecretKey: s.Config.StripeSecretKey,
|
|||
|
|
WebhookSecret: s.Config.StripeWebhookSecret,
|
|||
|
|
ForceMock: s.Config.StripeMock,
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
// diagStripeReadiness reports Stripe ops readiness without leaking secret values.
|
|||
|
|
// Production: mock must be off; missing secret/webhook keys degrade (fail-closed at use).
|
|||
|
|
func diagStripeReadiness(prod bool, cfg billing.StripeConfig) (check map[string]any, ok bool) {
|
|||
|
|
secretSet := strings.TrimSpace(cfg.SecretKey) != ""
|
|||
|
|
webhookSet := strings.TrimSpace(cfg.WebhookSecret) != ""
|
|||
|
|
mock := cfg.ForceMock
|
|||
|
|
mockRejectedInProd := !prod || !mock
|
|||
|
|
|
|||
|
|
check = map[string]any{
|
|||
|
|
"name": "stripe",
|
|||
|
|
"status": "ok",
|
|||
|
|
"secret_key_set": secretSet,
|
|||
|
|
"webhook_secret_set": webhookSet,
|
|||
|
|
"mock": mock,
|
|||
|
|
"mock_rejected_in_prod": mockRejectedInProd,
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
if prod && mock {
|
|||
|
|
check["status"] = "fail"
|
|||
|
|
check["detail"] = "STRIPE_MOCK must be false in production"
|
|||
|
|
return check, false
|
|||
|
|
}
|
|||
|
|
if prod && (!secretSet || !webhookSet) {
|
|||
|
|
check["status"] = "warn"
|
|||
|
|
parts := make([]string, 0, 2)
|
|||
|
|
if !secretSet {
|
|||
|
|
parts = append(parts, "secret key")
|
|||
|
|
}
|
|||
|
|
if !webhookSet {
|
|||
|
|
parts = append(parts, "webhook secret")
|
|||
|
|
}
|
|||
|
|
check["detail"] = "missing " + strings.Join(parts, " and ") + " (checkout/webhooks fail closed)"
|
|||
|
|
return check, false
|
|||
|
|
}
|
|||
|
|
if mock {
|
|||
|
|
check["detail"] = "mock mode enabled"
|
|||
|
|
return check, true
|
|||
|
|
}
|
|||
|
|
if !secretSet {
|
|||
|
|
check["status"] = "warn"
|
|||
|
|
check["detail"] = "secret key not set (mock purchases require STRIPE_MOCK)"
|
|||
|
|
return check, false
|
|||
|
|
}
|
|||
|
|
if !webhookSet {
|
|||
|
|
check["status"] = "warn"
|
|||
|
|
check["detail"] = "webhook secret not set"
|
|||
|
|
return check, false
|
|||
|
|
}
|
|||
|
|
check["detail"] = "keys present"
|
|||
|
|
return check, true
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
func (s *Server) diagConfigSanity(stripe billing.StripeConfig) map[string]any {
|
|||
|
|
smtpEnabled := s.Config.SMTPEnabled
|
|||
|
|
if s.Mail != nil {
|
|||
|
|
smtpEnabled = s.Mail.Enabled()
|
|||
|
|
}
|
|||
|
|
secretSet := strings.TrimSpace(stripe.SecretKey) != ""
|
|||
|
|
webhookSet := strings.TrimSpace(stripe.WebhookSecret) != ""
|
|||
|
|
return map[string]any{
|
|||
|
|
"app_env": s.Config.AppEnv,
|
|||
|
|
"maintenance_mode": s.Config.MaintenanceMode,
|
|||
|
|
"read_only_mode": s.Config.ReadOnlyMode,
|
|||
|
|
"session_secure": s.Config.SessionSecure,
|
|||
|
|
"smtp_enabled": smtpEnabled,
|
|||
|
|
"email_dry_run": s.Config.EmailDryRun,
|
|||
|
|
"smtp_host_set": strings.TrimSpace(s.Config.SMTPHost) != "",
|
|||
|
|
"stripe_mock": stripe.ForceMock,
|
|||
|
|
"eprel_enabled": s.Config.EPRELEnabled,
|
|||
|
|
"processing_rpm": s.Config.ProcessingRPM,
|
|||
|
|
"processing_batch_size": s.Config.ProcessingBatchSize,
|
|||
|
|
"processing_max_retries": s.Config.ProcessingMaxRetries,
|
|||
|
|
"upload_dir_configured": strings.TrimSpace(s.Config.UploadDir) != "",
|
|||
|
|
"trusted_proxies_configured": len(s.Config.TrustedProxies) > 0,
|
|||
|
|
"web_origin_set": strings.TrimSpace(s.Config.WebOrigin) != "",
|
|||
|
|
"public_api_url_set": strings.TrimSpace(s.Config.PublicAPIURL) != "",
|
|||
|
|
// Presence flags only — never the secret values.
|
|||
|
|
"token_signing_secret_set": strings.TrimSpace(s.Config.TokenSigningSecret) != "",
|
|||
|
|
"openai_key_set": strings.TrimSpace(s.Config.OpenAIAPIKey) != "",
|
|||
|
|
"pinecone_key_set": strings.TrimSpace(s.Config.PineconeAPIKey) != "",
|
|||
|
|
"stripe_secret_set": secretSet,
|
|||
|
|
"stripe_webhook_secret_set": webhookSet,
|
|||
|
|
"stripe_mock_rejected_in_prod": !s.Config.IsProduction() || !stripe.ForceMock,
|
|||
|
|
"credentials_encryption_key_set": strings.TrimSpace(s.Config.CredentialsEncryptionKey) != "",
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
func (s *Server) diagRecentJobFailures(ctx context.Context, limit int, statusFilter string) ([]map[string]any, error) {
|
|||
|
|
out := make([]map[string]any, 0)
|
|||
|
|
if s.Pool == nil {
|
|||
|
|
return out, errors.New("database unavailable")
|
|||
|
|
}
|
|||
|
|
status := statusFilter
|
|||
|
|
if status == "" {
|
|||
|
|
status = "failed"
|
|||
|
|
}
|
|||
|
|
rows, err := s.Pool.Query(ctx, `
|
|||
|
|
SELECT id, company_id, status, total_products, processed_products, error, created_at, updated_at
|
|||
|
|
FROM processing_jobs
|
|||
|
|
WHERE status = $1
|
|||
|
|
ORDER BY updated_at DESC
|
|||
|
|
LIMIT $2`, status, limit)
|
|||
|
|
if err != nil {
|
|||
|
|
return out, err
|
|||
|
|
}
|
|||
|
|
defer rows.Close()
|
|||
|
|
for rows.Next() {
|
|||
|
|
var (
|
|||
|
|
id, companyID uuid.UUID
|
|||
|
|
st string
|
|||
|
|
total, processed int
|
|||
|
|
errMsg *string
|
|||
|
|
createdAt, updatedAt time.Time
|
|||
|
|
)
|
|||
|
|
if err := rows.Scan(&id, &companyID, &st, &total, &processed, &errMsg, &createdAt, &updatedAt); err != nil {
|
|||
|
|
return out, err
|
|||
|
|
}
|
|||
|
|
safeErr := ""
|
|||
|
|
if errMsg != nil && *errMsg != "" {
|
|||
|
|
safeErr = processing.TruncateError(errors.New(*errMsg))
|
|||
|
|
}
|
|||
|
|
out = append(out, map[string]any{
|
|||
|
|
"id": id,
|
|||
|
|
"company_id": companyID,
|
|||
|
|
"status": st,
|
|||
|
|
"total_products": total,
|
|||
|
|
"processed_products": processed,
|
|||
|
|
"error": safeErr,
|
|||
|
|
"created_at": createdAt.UTC().Format(time.RFC3339),
|
|||
|
|
"updated_at": updatedAt.UTC().Format(time.RFC3339),
|
|||
|
|
})
|
|||
|
|
}
|
|||
|
|
return out, rows.Err()
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
func (s *Server) diagRecentAIFailures(ctx context.Context, limit int) ([]map[string]any, error) {
|
|||
|
|
out := make([]map[string]any, 0)
|
|||
|
|
if s.Pool == nil {
|
|||
|
|
return out, nil
|
|||
|
|
}
|
|||
|
|
if limit <= 0 {
|
|||
|
|
limit = adminDiagnosticsAIFailDefault
|
|||
|
|
}
|
|||
|
|
if limit > adminDiagnosticsAIFailMax {
|
|||
|
|
limit = adminDiagnosticsAIFailMax
|
|||
|
|
}
|
|||
|
|
rows, err := s.Pool.Query(ctx, `
|
|||
|
|
SELECT id, ticket_id, company_id, kind, created_at
|
|||
|
|
FROM support_ticket_activity
|
|||
|
|
WHERE kind = 'ai_failed'
|
|||
|
|
ORDER BY created_at DESC
|
|||
|
|
LIMIT $1`, limit)
|
|||
|
|
if err != nil {
|
|||
|
|
// Table may be absent on older DBs — soft-skip.
|
|||
|
|
return out, nil
|
|||
|
|
}
|
|||
|
|
defer rows.Close()
|
|||
|
|
for rows.Next() {
|
|||
|
|
var (
|
|||
|
|
id, ticketID, companyID uuid.UUID
|
|||
|
|
kind string
|
|||
|
|
createdAt time.Time
|
|||
|
|
)
|
|||
|
|
if err := rows.Scan(&id, &ticketID, &companyID, &kind, &createdAt); err != nil {
|
|||
|
|
return out, nil
|
|||
|
|
}
|
|||
|
|
out = append(out, map[string]any{
|
|||
|
|
"id": id,
|
|||
|
|
"ticket_id": ticketID,
|
|||
|
|
"company_id": companyID,
|
|||
|
|
"kind": kind,
|
|||
|
|
"created_at": createdAt.UTC().Format(time.RFC3339),
|
|||
|
|
})
|
|||
|
|
}
|
|||
|
|
return out, nil
|
|||
|
|
}
|