package httpapi import ( "context" "net/http" "strconv" "strings" "time" "github.com/descrybe/descrybe-v2/apps/api/internal/aiprovider" "github.com/google/uuid" ) const ( adminAnalyticsDefaultDays = 30 adminAnalyticsMinDays = 7 adminAnalyticsMaxDays = 90 adminAnalyticsTopCompanies = 25 adminAnalyticsRecentCycles = 40 adminAnalyticsProviderDetailMax = 40 adminAnalyticsStuckAfter = 2 * time.Hour ) func adminAnalyticsSummaryOnly(r *http.Request) bool { v := strings.TrimSpace(strings.ToLower(r.URL.Query().Get("summary"))) if v == "" { v = strings.TrimSpace(strings.ToLower(r.URL.Query().Get("summary_only"))) } return v == "1" || v == "true" || v == "yes" } type adminDayPoint struct { Date string `json:"date"` Tokens int64 `json:"tokens"` Products int64 `json:"products,omitempty"` Created int64 `json:"created,omitempty"` Completed int64 `json:"completed,omitempty"` Failed int64 `json:"failed,omitempty"` } type adminAnalyticsSummary struct { Users int64 `json:"users"` Companies int64 `json:"companies"` UsersPeriod int64 `json:"users_period"` CompaniesPeriod int64 `json:"companies_period"` CreditsAllocated int64 `json:"credits_allocated"` CreditsUsed int64 `json:"credits_used"` CreditsRemaining int64 `json:"credits_remaining"` TokensTotal int64 `json:"tokens_total"` TokensPeriod int64 `json:"tokens_period"` JobsTotal int64 `json:"jobs_total"` JobsByStatus map[string]int64 `json:"jobs_by_status"` JobsStuck int64 `json:"jobs_stuck"` JobsFailedPeriod int64 `json:"jobs_failed_period"` JobsCompletedPeriod int64 `json:"jobs_completed_period"` ProductsProcessed int64 `json:"products_processed"` ProductsRaw int64 `json:"products_raw"` FeedsInput int64 `json:"feeds_input"` FeedsExport int64 `json:"feeds_export"` ApiKeysActive int64 `json:"api_keys_active"` ApiKeysTotal int64 `json:"api_keys_total"` FeedSyncByStatus map[string]int64 `json:"feed_sync_by_status"` TicketsByStatus map[string]int64 `json:"tickets_by_status"` } type adminSignupDayPoint struct { Date string `json:"date"` Users int64 `json:"users"` Companies int64 `json:"companies"` } type adminCompanyUsage struct { ID uuid.UUID `json:"id"` Name string `json:"name"` TotalCredits int64 `json:"total_credits"` UsedCredits int64 `json:"used_credits"` Remaining int64 `json:"credits_remaining"` Tokens int64 `json:"tokens"` Jobs int64 `json:"jobs"` Providers adminProviderBreakdown `json:"providers"` } type adminProviderDayPoint struct { Date string `json:"date"` Internal int64 `json:"internal"` Popular int64 `json:"popular"` Custom int64 `json:"custom"` Unknown int64 `json:"unknown"` Total int64 `json:"total"` } type adminProviderDetail struct { Provider string `json:"provider"` Class string `json:"class"` Tokens int64 `json:"tokens"` Products int64 `json:"products"` } type adminBillingCycleRow struct { CompanyID uuid.UUID `json:"company_id"` CompanyName string `json:"company_name"` StartDate time.Time `json:"start_date"` EndDate time.Time `json:"end_date"` CreditsUsed int64 `json:"credits_used"` ProductsProcessed int64 `json:"products_processed"` } // adminProviderBucket is per-mode usage (internal / popular / custom / unknown). type adminProviderBucket struct { Tokens int64 `json:"tokens"` Jobs int64 `json:"jobs"` Products int64 `json:"products"` } type adminProviderBreakdown struct { Internal adminProviderBucket `json:"internal"` Popular adminProviderBucket `json:"popular"` Custom adminProviderBucket `json:"custom"` Unknown adminProviderBucket `json:"unknown"` } // handleAdminAnalytics returns platform-wide aggregates from live tables only. // GET /api/admin/analytics?days=30 // Optional: summary=1|true — dashboard cards only (skips series/companies/cycles). func (s *Server) handleAdminAnalytics(w http.ResponseWriter, r *http.Request) { days := adminAnalyticsDefaultDays if raw := r.URL.Query().Get("days"); raw != "" { if n, err := strconv.Atoi(raw); err == nil { days = clampAdminAnalyticsDays(n) } } summaryOnly := adminAnalyticsSummaryOnly(r) ctx := r.Context() since := time.Now().UTC().Truncate(24*time.Hour).AddDate(0, 0, -(days - 1)) summary, err := s.loadAdminAnalyticsSummary(ctx, since) if err != nil { Error(w, http.StatusInternalServerError, "analytics summary failed") return } providers, providerDetail := s.loadAdminProviderBreakdown(ctx) if summaryOnly { JSON(w, http.StatusOK, map[string]any{ "days": days, "summary": summary, "providers": providers, }) return } tokenByDay := map[string]adminDayPoint{} tokRows, err := s.Pool.Query(ctx, ` SELECT (created_at AT TIME ZONE 'UTC')::date AS d, COALESCE(SUM(COALESCE(total_tokens, 0)), 0)::bigint, COUNT(*)::bigint FROM processed_products WHERE created_at >= $1 GROUP BY 1 ORDER BY 1`, since) if err != nil { Error(w, http.StatusInternalServerError, "analytics token series failed") return } for tokRows.Next() { var d time.Time var tokens, products int64 if err := tokRows.Scan(&d, &tokens, &products); err != nil { tokRows.Close() Error(w, http.StatusInternalServerError, "analytics token series scan failed") return } key := d.UTC().Format("2006-01-02") tokenByDay[key] = adminDayPoint{Date: key, Tokens: tokens, Products: products} } tokRows.Close() if err := tokRows.Err(); err != nil { Error(w, http.StatusInternalServerError, "analytics token series rows failed") return } jobByDay := map[string]adminDayPoint{} jSeries, err := s.Pool.Query(ctx, ` SELECT (created_at AT TIME ZONE 'UTC')::date AS d, COUNT(*)::bigint, COUNT(*) FILTER (WHERE status = 'completed')::bigint, COUNT(*) FILTER (WHERE status = 'failed')::bigint, COALESCE(SUM(estimated_tokens), 0)::bigint FROM processing_jobs WHERE created_at >= $1 GROUP BY 1 ORDER BY 1`, since) if err != nil { Error(w, http.StatusInternalServerError, "analytics job series failed") return } for jSeries.Next() { var d time.Time var created, completed, failed, tokens int64 if err := jSeries.Scan(&d, &created, &completed, &failed, &tokens); err != nil { jSeries.Close() Error(w, http.StatusInternalServerError, "analytics job series scan failed") return } key := d.UTC().Format("2006-01-02") jobByDay[key] = adminDayPoint{ Date: key, Created: created, Completed: completed, Failed: failed, Tokens: tokens, } } jSeries.Close() if err := jSeries.Err(); err != nil { Error(w, http.StatusInternalServerError, "analytics job series rows failed") return } tokenSeries := fillAdminDaySeries(since, days, tokenByDay, func(p adminDayPoint) adminDayPoint { return adminDayPoint{Date: p.Date, Tokens: p.Tokens, Products: p.Products} }) jobSeries := fillAdminDaySeries(since, days, jobByDay, func(p adminDayPoint) adminDayPoint { return adminDayPoint{ Date: p.Date, Created: p.Created, Completed: p.Completed, Failed: p.Failed, Tokens: p.Tokens, } }) companies := make([]adminCompanyUsage, 0, adminAnalyticsTopCompanies) companyIDs := make([]uuid.UUID, 0, adminAnalyticsTopCompanies) cRows, err := s.Pool.Query(ctx, ` SELECT c.id, c.name, COALESCE(cb.total_credits, 0)::bigint, COALESCE(cb.used_credits, 0)::bigint, COALESCE(tok.tokens, 0)::bigint, COALESCE(jobs.cnt, 0)::bigint FROM companies c LEFT JOIN credit_balances cb ON cb.company_id = c.id LEFT JOIN ( SELECT company_id, SUM(COALESCE(total_tokens, 0))::bigint AS tokens FROM processed_products GROUP BY company_id ) tok ON tok.company_id = c.id LEFT JOIN ( SELECT company_id, COUNT(*)::bigint AS cnt FROM processing_jobs GROUP BY company_id ) jobs ON jobs.company_id = c.id ORDER BY COALESCE(tok.tokens, 0) DESC, COALESCE(cb.used_credits, 0) DESC, c.name ASC LIMIT $1`, adminAnalyticsTopCompanies) if err != nil { Error(w, http.StatusInternalServerError, "analytics companies usage failed") return } for cRows.Next() { var row adminCompanyUsage if err := cRows.Scan(&row.ID, &row.Name, &row.TotalCredits, &row.UsedCredits, &row.Tokens, &row.Jobs); err != nil { cRows.Close() Error(w, http.StatusInternalServerError, "analytics companies scan failed") return } row.Remaining = row.TotalCredits - row.UsedCredits if row.Remaining < 0 { row.Remaining = 0 } companies = append(companies, row) companyIDs = append(companyIDs, row.ID) } cRows.Close() if err := cRows.Err(); err != nil { Error(w, http.StatusInternalServerError, "analytics companies rows failed") return } cycles := make([]adminBillingCycleRow, 0, adminAnalyticsRecentCycles) cyRows, err := s.Pool.Query(ctx, ` SELECT bc.company_id, c.name, bc.start_date, bc.end_date, bc.credits_used::bigint, bc.products_processed::bigint FROM billing_cycles bc JOIN companies c ON c.id = bc.company_id ORDER BY bc.start_date DESC LIMIT $1`, adminAnalyticsRecentCycles) if err != nil { Error(w, http.StatusInternalServerError, "analytics billing cycles failed") return } for cyRows.Next() { var row adminBillingCycleRow if err := cyRows.Scan(&row.CompanyID, &row.CompanyName, &row.StartDate, &row.EndDate, &row.CreditsUsed, &row.ProductsProcessed); err != nil { cyRows.Close() Error(w, http.StatusInternalServerError, "analytics billing cycles scan failed") return } cycles = append(cycles, row) } cyRows.Close() if err := cyRows.Err(); err != nil { Error(w, http.StatusInternalServerError, "analytics billing cycles rows failed") return } providerDaySeries := s.loadAdminProviderDaySeries(ctx, since, days) signupSeries := s.loadAdminSignupDaySeries(ctx, since, days) companyProviders := s.loadAdminCompanyProviderBreakdown(ctx, companyIDs) for i := range companies { if split, ok := companyProviders[companies[i].ID]; ok { companies[i].Providers = split } } JSON(w, http.StatusOK, map[string]any{ "days": days, "summary": summary, "series": map[string]any{ "tokens_by_day": tokenSeries, "jobs_by_day": jobSeries, "tokens_by_provider_day": providerDaySeries, "signups_by_day": signupSeries, }, "companies": companies, "billing_cycles": cycles, "providers": providers, "tokens_by_provider_detail": providerDetail, "notes": []string{ "Tokens come from processed_products.total_tokens (LLM usage recorded per product).", "Provider classes use processed_products.ai_provider_mode: internal | popular: | custom (blank/unknown rolled into the internal card).", "Job tokens use processing_jobs.estimated_tokens (running tally during jobs).", "Jobs stuck = status running with updated_at older than 2 hours (same threshold as diagnostics).", "Credits are live credit_balances snapshots - daily credit debits are not ledgered yet.", "Feeds / feed sync / support tickets / API keys are live table aggregates.", "Billing cycle rows are historical rollups when cycles have been run (may lag the active company_plans window).", "Use /admin/diagnostics for queue health checks and recent failure samples.", }, }) } func (s *Server) loadAdminAnalyticsSummary(ctx context.Context, since time.Time) (adminAnalyticsSummary, error) { summary := adminAnalyticsSummary{ JobsByStatus: map[string]int64{}, FeedSyncByStatus: map[string]int64{}, TicketsByStatus: map[string]int64{}, } err := s.Pool.QueryRow(ctx, ` SELECT (SELECT COUNT(*)::bigint FROM users), (SELECT COUNT(*)::bigint FROM companies), (SELECT COUNT(*)::bigint FROM users WHERE created_at >= $1), (SELECT COUNT(*)::bigint FROM companies WHERE created_at >= $1), (SELECT COALESCE(SUM(total_credits), 0)::bigint FROM credit_balances), (SELECT COALESCE(SUM(used_credits), 0)::bigint FROM credit_balances), (SELECT COALESCE(SUM(COALESCE(total_tokens, 0)), 0)::bigint FROM processed_products), (SELECT COUNT(*)::bigint FROM processed_products), (SELECT COUNT(*)::bigint FROM raw_products), (SELECT COUNT(*)::bigint FROM input_feeds), (SELECT COUNT(*)::bigint FROM export_feeds), (SELECT COALESCE(SUM(COALESCE(total_tokens, 0)), 0)::bigint FROM processed_products WHERE created_at >= $1), (SELECT COUNT(*)::bigint FROM processing_jobs WHERE created_at >= $1 AND status = 'failed'), (SELECT COUNT(*)::bigint FROM processing_jobs WHERE created_at >= $1 AND status = 'completed'), (SELECT COUNT(*)::bigint FROM processing_jobs WHERE status = 'running' AND updated_at < now() - make_interval(secs => $2)), (SELECT COUNT(*)::bigint FROM api_keys), (SELECT COUNT(*)::bigint FROM api_keys WHERE revoked_at IS NULL) `, since, adminAnalyticsStuckAfter.Seconds()).Scan( &summary.Users, &summary.Companies, &summary.UsersPeriod, &summary.CompaniesPeriod, &summary.CreditsAllocated, &summary.CreditsUsed, &summary.TokensTotal, &summary.ProductsProcessed, &summary.ProductsRaw, &summary.FeedsInput, &summary.FeedsExport, &summary.TokensPeriod, &summary.JobsFailedPeriod, &summary.JobsCompletedPeriod, &summary.JobsStuck, &summary.ApiKeysTotal, &summary.ApiKeysActive, ) if err != nil { return summary, err } summary.CreditsRemaining = summary.CreditsAllocated - summary.CreditsUsed if summary.CreditsRemaining < 0 { summary.CreditsRemaining = 0 } jobRows, err := s.Pool.Query(ctx, ` SELECT status, COUNT(*) FROM processing_jobs GROUP BY status`) if err != nil { return summary, err } defer jobRows.Close() for jobRows.Next() { var status string var n int64 if err := jobRows.Scan(&status, &n); err != nil { return summary, err } summary.JobsByStatus[status] = n summary.JobsTotal += n } if err := jobRows.Err(); err != nil { return summary, err } summary.FeedSyncByStatus = s.loadAdminStatusCounts(ctx, `SELECT status, COUNT(*)::bigint FROM feed_sync_jobs GROUP BY status`) summary.TicketsByStatus = s.loadAdminStatusCounts(ctx, `SELECT status, COUNT(*)::bigint FROM support_tickets GROUP BY status`) return summary, nil } func (s *Server) loadAdminStatusCounts(ctx context.Context, query string) map[string]int64 { out := map[string]int64{} rows, err := s.Pool.Query(ctx, query) if err != nil { return out } defer rows.Close() for rows.Next() { var status string var n int64 if err := rows.Scan(&status, &n); err != nil { return out } out[status] = n } return out } func (s *Server) loadAdminSignupDaySeries(ctx context.Context, since time.Time, days int) []adminSignupDayPoint { byDay := map[string]adminSignupDayPoint{} uRows, err := s.Pool.Query(ctx, ` SELECT (created_at AT TIME ZONE 'UTC')::date AS d, COUNT(*)::bigint FROM users WHERE created_at >= $1 GROUP BY 1 ORDER BY 1`, since) if err == nil { for uRows.Next() { var d time.Time var n int64 if err := uRows.Scan(&d, &n); err != nil { break } key := d.UTC().Format("2006-01-02") pt := byDay[key] pt.Date = key pt.Users = n byDay[key] = pt } uRows.Close() } cRows, err := s.Pool.Query(ctx, ` SELECT (created_at AT TIME ZONE 'UTC')::date AS d, COUNT(*)::bigint FROM companies WHERE created_at >= $1 GROUP BY 1 ORDER BY 1`, since) if err == nil { for cRows.Next() { var d time.Time var n int64 if err := cRows.Scan(&d, &n); err != nil { break } key := d.UTC().Format("2006-01-02") pt := byDay[key] pt.Date = key pt.Companies = n byDay[key] = pt } cRows.Close() } out := make([]adminSignupDayPoint, 0, days) for i := 0; i < days; i++ { d := since.AddDate(0, 0, i).UTC().Format("2006-01-02") if p, ok := byDay[d]; ok { p.Date = d out = append(out, p) continue } out = append(out, adminSignupDayPoint{Date: d}) } return out } func (s *Server) loadAdminProviderBreakdown(ctx context.Context) (adminProviderBreakdown, []adminProviderDetail) { out := adminProviderBreakdown{} detail := make([]adminProviderDetail, 0) rows, err := s.Pool.Query(ctx, ` SELECT COALESCE(NULLIF(TRIM(ai_provider_mode), ''), 'unknown') AS mode, COALESCE(SUM(COALESCE(total_tokens, 0)), 0)::bigint, COUNT(*)::bigint FROM processed_products GROUP BY 1 ORDER BY 2 DESC`) if err != nil { return out, detail } defer rows.Close() for rows.Next() { var mode string var tokens, products int64 if err := rows.Scan(&mode, &tokens, &products); err != nil { return out, detail } class := aiprovider.AnalyticsClass(mode) bucket := adminProviderBucket{Tokens: tokens, Products: products} switch class { case aiprovider.ModePopular: out.Popular.Tokens += bucket.Tokens out.Popular.Products += bucket.Products case aiprovider.ModeCustom: out.Custom.Tokens += bucket.Tokens out.Custom.Products += bucket.Products default: // internal + unknown → internal card (legacy/backfill) out.Internal.Tokens += bucket.Tokens out.Internal.Products += bucket.Products } detail = append(detail, adminProviderDetail{ Provider: aiprovider.NormalizeAnalyticsMode(mode), Class: class, Tokens: tokens, Products: products, }) } jobRows, err := s.Pool.Query(ctx, ` SELECT COALESCE(NULLIF(TRIM(ai_provider_mode), ''), 'unknown') AS mode, COUNT(*)::bigint FROM processing_jobs GROUP BY 1`) if err != nil { return out, detail } defer jobRows.Close() for jobRows.Next() { var mode string var jobs int64 if err := jobRows.Scan(&mode, &jobs); err != nil { return out, detail } switch aiprovider.AnalyticsClass(mode) { case aiprovider.ModePopular: out.Popular.Jobs += jobs case aiprovider.ModeCustom: out.Custom.Jobs += jobs default: out.Internal.Jobs += jobs } } if len(detail) > adminAnalyticsProviderDetailMax { detail = detail[:adminAnalyticsProviderDetailMax] } return out, detail } func (s *Server) loadAdminProviderDaySeries(ctx context.Context, since time.Time, days int) []adminProviderDayPoint { byDay := map[string]adminProviderDayPoint{} rows, err := s.Pool.Query(ctx, ` SELECT (created_at AT TIME ZONE 'UTC')::date AS d, COALESCE(NULLIF(TRIM(ai_provider_mode), ''), 'unknown') AS mode, COALESCE(SUM(COALESCE(total_tokens, 0)), 0)::bigint FROM processed_products WHERE created_at >= $1 GROUP BY 1, 2 ORDER BY 1`, since) if err == nil { defer rows.Close() for rows.Next() { var d time.Time var mode string var tokens int64 if err := rows.Scan(&d, &mode, &tokens); err != nil { break } key := d.UTC().Format("2006-01-02") pt := byDay[key] pt.Date = key switch aiprovider.AnalyticsClass(mode) { case aiprovider.ModePopular: pt.Popular += tokens case aiprovider.ModeCustom: pt.Custom += tokens case "unknown": pt.Unknown += tokens default: pt.Internal += tokens } pt.Total += tokens byDay[key] = pt } } out := make([]adminProviderDayPoint, 0, days) for i := 0; i < days; i++ { d := since.AddDate(0, 0, i).UTC().Format("2006-01-02") if p, ok := byDay[d]; ok { p.Date = d out = append(out, p) continue } out = append(out, adminProviderDayPoint{Date: d}) } return out } func (s *Server) loadAdminCompanyProviderBreakdown(ctx context.Context, companyIDs []uuid.UUID) map[uuid.UUID]adminProviderBreakdown { out := map[uuid.UUID]adminProviderBreakdown{} if len(companyIDs) == 0 { return out } rows, err := s.Pool.Query(ctx, ` SELECT company_id, COALESCE(NULLIF(TRIM(ai_provider_mode), ''), 'unknown') AS mode, COALESCE(SUM(COALESCE(total_tokens, 0)), 0)::bigint, COUNT(*)::bigint FROM processed_products WHERE company_id = ANY($1) GROUP BY company_id, 2`, companyIDs) if err != nil { return out } defer rows.Close() for rows.Next() { var companyID uuid.UUID var mode string var tokens, products int64 if err := rows.Scan(&companyID, &mode, &tokens, &products); err != nil { return out } b := out[companyID] switch aiprovider.AnalyticsClass(mode) { case aiprovider.ModePopular: b.Popular.Tokens += tokens b.Popular.Products += products case aiprovider.ModeCustom: b.Custom.Tokens += tokens b.Custom.Products += products default: b.Internal.Tokens += tokens b.Internal.Products += products } out[companyID] = b } jobRows, err := s.Pool.Query(ctx, ` SELECT company_id, COALESCE(NULLIF(TRIM(ai_provider_mode), ''), 'unknown') AS mode, COUNT(*)::bigint FROM processing_jobs WHERE company_id = ANY($1) GROUP BY company_id, 2`, companyIDs) if err != nil { return out } defer jobRows.Close() for jobRows.Next() { var companyID uuid.UUID var mode string var jobs int64 if err := jobRows.Scan(&companyID, &mode, &jobs); err != nil { return out } b := out[companyID] switch aiprovider.AnalyticsClass(mode) { case aiprovider.ModePopular: b.Popular.Jobs += jobs case aiprovider.ModeCustom: b.Custom.Jobs += jobs default: b.Internal.Jobs += jobs } out[companyID] = b } return out } func clampAdminAnalyticsDays(n int) int { if n < adminAnalyticsMinDays { return adminAnalyticsMinDays } if n > adminAnalyticsMaxDays { return adminAnalyticsMaxDays } return n } func fillAdminDaySeries( since time.Time, days int, src map[string]adminDayPoint, mapPoint func(adminDayPoint) adminDayPoint, ) []adminDayPoint { out := make([]adminDayPoint, 0, days) for i := 0; i < days; i++ { d := since.AddDate(0, 0, i).UTC().Format("2006-01-02") if p, ok := src[d]; ok { p.Date = d out = append(out, mapPoint(p)) continue } out = append(out, mapPoint(adminDayPoint{Date: d})) } return out }