This commit is contained in:
2026-08-23 22:52:13 +02:00
parent 8983cfc8a1
commit 296dc3c841
32 changed files with 2025 additions and 52 deletions
+100
View File
@@ -0,0 +1,100 @@
// Command backfill-ai-usage recovers AI cost history for products that were
// processed before prompt/usage capture existed.
//
// It reads the provider usage blocks still stored in processed_products.gpt_response
// and rolls them into ai_usage_daily with source='backfill', so /admin/ai-costs
// shows spend from before the feature shipped. See internal/aiaudit/backfill.go for
// what is and is not recoverable.
//
// go run ./cmd/backfill-ai-usage # dry run, reports what it would write
// go run ./cmd/backfill-ai-usage -apply
// go run ./cmd/backfill-ai-usage -apply -before 2026-08-23
//
// Safe to re-run: each pass replaces its own rows and never touches live capture.
package main
import (
"context"
"encoding/json"
"flag"
"fmt"
"log"
"os"
"strings"
"time"
"github.com/descrybe/descrybe-v2/apps/api/internal/aiaudit"
"github.com/descrybe/descrybe-v2/apps/api/internal/config"
"github.com/jackc/pgx/v5/pgxpool"
)
func main() {
apply := flag.Bool("apply", false, "write rows (default is a dry run)")
before := flag.String("before", "", "only products last updated before this date (YYYY-MM-DD); default: the day live capture started")
dsn := flag.String("postgres", os.Getenv("DATABASE_URL"), "Postgres URL (defaults to DATABASE_URL / repo .env)")
flag.Parse()
ctx, cancel := context.WithTimeout(context.Background(), 15*time.Minute)
defer cancel()
url := strings.TrimSpace(*dsn)
if url == "" {
cfg, err := config.Load()
if err != nil {
log.Fatalf("config: %v", err)
}
url = cfg.DatabaseURL
}
pool, err := pgxpool.New(ctx, url)
if err != nil {
log.Fatalf("db: %v", err)
}
defer pool.Close()
cutoff, err := resolveCutoff(ctx, pool, *before)
if err != nil {
log.Fatalf("cutoff: %v", err)
}
res, err := aiaudit.BackfillFromProcessedProducts(ctx, pool, cutoff, !*apply)
if err != nil {
log.Fatalf("backfill: %v", err)
}
b, _ := json.MarshalIndent(res, "", " ")
fmt.Println(string(b))
fmt.Printf("\nrecovered %s across %d product(s); %d product(s) had no stored usage block\n",
usd(res.CostUSD()), res.ProductsUsed, res.ProductsNoUsage)
if !*apply {
fmt.Println("dry run — re-run with -apply to write these rows")
}
}
// resolveCutoff keeps the backfill from double-counting calls the live recorder
// already logged: by default it stops at the first day live capture wrote a row.
func resolveCutoff(ctx context.Context, pool *pgxpool.Pool, before string) (time.Time, error) {
if s := strings.TrimSpace(before); s != "" {
t, err := time.Parse("2006-01-02", s)
if err != nil {
return time.Time{}, fmt.Errorf("invalid -before date (YYYY-MM-DD): %w", err)
}
return t.UTC(), nil
}
start, err := aiaudit.LiveCaptureStart(ctx, pool)
if err != nil {
return time.Time{}, err
}
if start.IsZero() {
log.Println("no live capture recorded yet — scanning all history")
return time.Time{}, nil
}
log.Printf("live capture starts %s — backfilling products updated before that", start.Format("2006-01-02"))
return start, nil
}
func usd(v float64) string {
if v != 0 && v < 0.01 {
return fmt.Sprintf("$%.6f", v)
}
return fmt.Sprintf("$%.2f", v)
}
+6
View File
@@ -380,6 +380,12 @@ func main() {
} else if n > 0 {
log.Printf("worker: ai call log cleanup removed=%d older_than_days=%d", n, aiaudit.RetentionDays)
}
// Cost rollup is kept for years, unlike the prompt bodies above.
if n, err := aiaudit.CleanupExpiredUsage(ctx, pool); err != nil {
log.Printf("worker: ai usage cleanup: %v", err)
} else if n > 0 {
log.Printf("worker: ai usage cleanup removed=%d older_than_days=%d", n, aiaudit.UsageRetentionDays)
}
if res, err := processing.CleanupExpiredSyncJobs(ctx, pool); err != nil {
log.Printf("expired sync cleanup: %v", err)
} else if res.SyncJobsDeleted > 0 {
+12 -3
View File
@@ -54,9 +54,12 @@ type Call struct {
Error string
FinishReason string
PromptTokens int
OutputTokens int
TotalTokens int
Duration time.Duration
// CachedPromptTokens is the cached subset of PromptTokens — billed far cheaper,
// so cost accounting must know about it.
CachedPromptTokens int
OutputTokens int
TotalTokens int
Duration time.Duration
}
// CallContext is the per-call metadata processing/campaigns attach to ctx before
@@ -110,6 +113,9 @@ type Recorder struct {
mu sync.Mutex
failed bool // stop log-spamming once writes are known to fail
// prices caches ai_model_prices for cost rollup (see cost.go).
prices priceCache
}
// NewRecorder returns a Recorder writing to pool. nil pool yields a no-op recorder.
@@ -170,6 +176,9 @@ func (r *Recorder) Record(ctx context.Context, c Call) {
if err != nil {
r.noteFailure(err)
}
// Roll the same call into the durable daily cost aggregate. ai_call_logs is
// pruned after a week; spend has to survive for years (see cost.go).
r.recordUsage(ctx, c)
}
func (r *Recorder) noteFailure(err error) {
+305
View File
@@ -0,0 +1,305 @@
package aiaudit
import (
"context"
"encoding/json"
"fmt"
"time"
"github.com/google/uuid"
"github.com/jackc/pgx/v5/pgxpool"
)
// Recovering cost for products processed before capture existed.
//
// processed_products.gpt_response keeps the provider's usage block for some rows
// (model + prompt/completion tokens). That is enough to reconstruct spend per
// company and day, so history is not simply lost. It is explicitly best effort:
//
// - Only rows that still carry a usage block are recoverable. gpt_response is
// overwritten on every reprocess and absent on older/failed rows.
// - There is no per-user attribution: processed_products.user_id is null on these
// rows, so backfilled spend lands under "Unattributed" in the by-user view.
// - The day is processed_products.updated_at, which is the LAST write to the row,
// not necessarily when the call was made.
// - Cached-token discounts cannot be recovered (the field did not exist), so a
// backfilled call is priced as if none of its input was cached — an
// over-estimate, never an under-estimate.
//
// Rows are written with source='backfill' so they never collide with live capture
// and a re-run replaces rather than doubles them.
// BackfillResult reports what a backfill pass recovered.
type BackfillResult struct {
ProductsScanned int `json:"products_scanned"`
ProductsUsed int `json:"products_used"`
ProductsNoUsage int `json:"products_without_usage"`
CallsRecovered int `json:"calls_recovered"`
PromptTokens int64 `json:"prompt_tokens"`
CompletionTokens int64 `json:"completion_tokens"`
CostMicros int64 `json:"cost_micros"`
RowsWritten int `json:"rows_written"`
RowsDeleted int64 `json:"rows_deleted"`
Cutoff string `json:"cutoff"`
DryRun bool `json:"dry_run"`
}
// CostUSD renders the recovered spend for CLI output.
func (r BackfillResult) CostUSD() float64 {
return float64(r.CostMicros) / float64(MicrosPerUSD)
}
type backfillKey struct {
day time.Time
companyID uuid.UUID
model string
role string
}
type backfillAgg struct {
calls int
promptTokens int64
completionTokens int64
totalTokens int64
costMicros int64
}
// LiveCaptureStart returns the earliest day live capture recorded, or zero when
// nothing has been captured yet. The backfill uses it as a default cutoff so a
// product processed after capture began is never counted twice.
func LiveCaptureStart(ctx context.Context, pool *pgxpool.Pool) (time.Time, error) {
if pool == nil {
return time.Time{}, nil
}
var day *time.Time
err := pool.QueryRow(ctx, `
SELECT min(day) FROM ai_usage_daily WHERE source = $1`, SourceLive).Scan(&day)
if err != nil {
return time.Time{}, err
}
if day == nil {
return time.Time{}, nil
}
return *day, nil
}
// BackfillFromProcessedProducts reconstructs cost history from stored provider
// responses for products last written before cutoff. Passing a zero cutoff scans
// everything, which is only safe when live capture has recorded nothing yet.
func BackfillFromProcessedProducts(ctx context.Context, pool *pgxpool.Pool, cutoff time.Time, dryRun bool) (BackfillResult, error) {
out := BackfillResult{DryRun: dryRun}
if pool == nil {
return out, fmt.Errorf("nil pool")
}
if cutoff.IsZero() {
// Far future = no bound.
cutoff = time.Now().UTC().AddDate(100, 0, 0)
}
out.Cutoff = cutoff.Format("2006-01-02")
prices, err := loadPrices(ctx, pool)
if err != nil {
return out, err
}
rows, err := pool.Query(ctx, `
SELECT company_id, updated_at, COALESCE(gpt_response, '{}'::jsonb)
FROM processed_products
WHERE gpt_response IS NOT NULL
AND updated_at < $1
ORDER BY updated_at`, cutoff)
if err != nil {
return out, err
}
defer rows.Close()
agg := map[backfillKey]*backfillAgg{}
for rows.Next() {
var companyID uuid.UUID
var updatedAt time.Time
var raw []byte
if err := rows.Scan(&companyID, &updatedAt, &raw); err != nil {
return out, err
}
out.ProductsScanned++
calls := extractUsageCalls(raw)
if len(calls) == 0 {
out.ProductsNoUsage++
continue
}
out.ProductsUsed++
day := updatedAt.UTC().Truncate(24 * time.Hour)
for _, c := range calls {
price := priceFromMap(prices, c.Model)
// Cached tokens are unknown for history — price all input as fresh.
cost := CostMicros(price, c.PromptTokens, 0, c.CompletionTokens)
k := backfillKey{day: day, companyID: companyID, model: NormalizeModel(c.Model), role: c.Role}
a := agg[k]
if a == nil {
a = &backfillAgg{}
agg[k] = a
}
a.calls++
a.promptTokens += int64(c.PromptTokens)
a.completionTokens += int64(c.CompletionTokens)
a.totalTokens += int64(c.TotalTokens)
a.costMicros += cost
out.CallsRecovered++
out.PromptTokens += int64(c.PromptTokens)
out.CompletionTokens += int64(c.CompletionTokens)
out.CostMicros += cost
}
}
if err := rows.Err(); err != nil {
return out, err
}
out.RowsWritten = len(agg)
if dryRun {
return out, nil
}
tx, err := pool.Begin(ctx)
if err != nil {
return out, err
}
defer func() { _ = tx.Rollback(ctx) }()
// Replace rather than add: a re-run must not double the recovered history.
tag, err := tx.Exec(ctx, `DELETE FROM ai_usage_daily WHERE source = $1`, SourceBackfill)
if err != nil {
return out, err
}
out.RowsDeleted = tag.RowsAffected()
for k, a := range agg {
if _, err := tx.Exec(ctx, `
INSERT INTO ai_usage_daily (
day, company_id, user_id, model, role, source,
calls, failed_calls, prompt_tokens, cached_prompt_tokens,
completion_tokens, total_tokens, cost_micros, updated_at)
VALUES ($1, $2, '00000000-0000-0000-0000-000000000000', $3, $4, $5,
$6, 0, $7, 0, $8, $9, $10, now())`,
k.day, k.companyID, k.model, k.role, SourceBackfill,
a.calls, a.promptTokens, a.completionTokens, a.totalTokens, a.costMicros); err != nil {
return out, err
}
}
if err := tx.Commit(ctx); err != nil {
return out, err
}
return out, nil
}
func loadPrices(ctx context.Context, pool *pgxpool.Pool) (map[string]ModelPrice, error) {
out := map[string]ModelPrice{}
rows, err := pool.Query(ctx, `
SELECT model, input_micros_per_mtok, cached_input_micros_per_mtok, output_micros_per_mtok
FROM ai_model_prices`)
if err != nil {
return out, err
}
defer rows.Close()
for rows.Next() {
var model string
var in, cached, o int64
if err := rows.Scan(&model, &in, &cached, &o); err != nil {
return out, err
}
out[NormalizeModel(model)] = ModelPrice{InputPerMTok: in, CachedInputPerMTok: cached, OutputPerMTok: o}
}
return out, rows.Err()
}
func priceFromMap(prices map[string]ModelPrice, model string) ModelPrice {
key := NormalizeModel(model)
if p, ok := prices[key]; ok {
return p
}
if p, ok := DefaultModelPrice(key); ok {
return p
}
return ModelPrice{}
}
// recoveredCall is one provider response found inside gpt_response.
type recoveredCall struct {
Model string
Role string
PromptTokens int
CompletionTokens int
TotalTokens int
}
// extractUsageCalls walks a stored gpt_response and returns every provider usage
// block it holds. The shape varies by pipeline version, so this looks for any
// object with a "usage" child rather than assuming one fixed path — older rows
// nest it differently and a rigid path would silently recover nothing.
func extractUsageCalls(raw []byte) []recoveredCall {
var doc any
if err := json.Unmarshal(raw, &doc); err != nil {
return nil
}
var out []recoveredCall
walkUsage(doc, "", &out)
return out
}
func walkUsage(node any, stepHint string, out *[]recoveredCall) {
switch v := node.(type) {
case map[string]any:
if s, ok := v["step"].(string); ok && s != "" {
stepHint = s
}
if usage, ok := v["usage"].(map[string]any); ok {
call := recoveredCall{
Model: stringField(v, "model"),
Role: roleFromStep(stepHint),
PromptTokens: intField(usage, "prompt_tokens"),
CompletionTokens: intField(usage, "completion_tokens"),
TotalTokens: intField(usage, "total_tokens"),
}
if call.PromptTokens > 0 || call.CompletionTokens > 0 {
*out = append(*out, call)
}
}
for _, child := range v {
walkUsage(child, stepHint, out)
}
case []any:
for _, child := range v {
walkUsage(child, stepHint, out)
}
}
}
func roleFromStep(step string) string {
switch step {
case "categorize":
return RoleCategorize
case "ai_enhance", "":
return RoleProcessing
default:
return RoleProcessing
}
}
func stringField(m map[string]any, key string) string {
if s, ok := m[key].(string); ok {
return s
}
return ""
}
func intField(m map[string]any, key string) int {
switch v := m[key].(type) {
case float64:
return int(v)
case int:
return v
case json.Number:
n, _ := v.Int64()
return int(n)
}
return 0
}
@@ -0,0 +1,81 @@
package aiaudit
import (
"testing"
)
// gpt_response nests the usage block differently across pipeline versions, so the
// walker looks for any object carrying "usage" rather than one fixed path. A rigid
// path would silently recover nothing from older rows.
func TestExtractUsageCalls_findsNestedUsage(t *testing.T) {
t.Parallel()
raw := []byte(`{
"steps": [
{"step": "normalize", "raw": {"keys": 21}},
{"step": "categorize", "raw": {"model": "gpt-5.6-luna",
"usage": {"prompt_tokens": 700, "completion_tokens": 5, "total_tokens": 705}}},
{"step": "ai_enhance", "raw": {"languages": [
{"language": "sl", "raw": {"raw": {"model": "gpt-5.6-luna", "status": 200,
"usage": {"prompt_tokens": 4000, "completion_tokens": 800, "total_tokens": 4800}}}}
]}}
]
}`)
calls := extractUsageCalls(raw)
if len(calls) != 2 {
t.Fatalf("expected 2 recovered calls, got %d: %+v", len(calls), calls)
}
byRole := map[string]recoveredCall{}
for _, c := range calls {
byRole[c.Role] = c
}
cat, ok := byRole[RoleCategorize]
if !ok {
t.Fatalf("categorize call not recovered: %+v", calls)
}
if cat.PromptTokens != 700 || cat.Model != "gpt-5.6-luna" {
t.Fatalf("categorize call wrong: %+v", cat)
}
enh, ok := byRole[RoleProcessing]
if !ok {
t.Fatalf("enhance call not recovered: %+v", calls)
}
if enh.PromptTokens != 4000 || enh.CompletionTokens != 800 {
t.Fatalf("enhance call wrong: %+v", enh)
}
}
func TestExtractUsageCalls_ignoresRowsWithoutUsage(t *testing.T) {
t.Parallel()
for name, raw := range map[string]string{
"empty": `{}`,
"steps only": `{"steps":[{"step":"normalize","raw":{"keys":3}}]}`,
"zero tokens": `{"raw":{"usage":{"prompt_tokens":0,"completion_tokens":0}}}`,
"not json": `not json at all`,
"unchanged skip": `{"steps":[{"step":"ai_enhance","raw":{"languages":[{"raw":{"status":"unchanged"}}]}}]}`,
} {
t.Run(name, func(t *testing.T) {
if got := extractUsageCalls([]byte(raw)); len(got) != 0 {
t.Fatalf("expected no recovered calls, got %+v", got)
}
})
}
}
// Backfilled history has no cached-token detail, so input is priced as fully
// fresh. That must over-estimate, never under-estimate.
func TestBackfillPricing_treatsInputAsUncached(t *testing.T) {
t.Parallel()
luna, _ := DefaultModelPrice("gpt-5.6-luna")
backfilled := CostMicros(luna, 10_000, 0, 1_000)
ifHalfCached := CostMicros(luna, 10_000, 5_000, 1_000)
if backfilled <= ifHalfCached {
t.Fatalf("backfill cost %d should exceed the cached-aware cost %d", backfilled, ifHalfCached)
}
}
func TestBackfillSourcesAreDistinct(t *testing.T) {
t.Parallel()
if SourceLive == SourceBackfill {
t.Fatal("live and backfill rows must be distinguishable")
}
}
+248
View File
@@ -0,0 +1,248 @@
package aiaudit
import (
"context"
"strings"
"sync"
"time"
"github.com/google/uuid"
"github.com/jackc/pgx/v5/pgxpool"
)
// Cost accounting. Kept apart from the prompt capture above on purpose:
// ai_call_logs is a 7-day debugging buffer, while spend has to answer "what did
// this tenant cost me last quarter" for years. Usage is therefore rolled up per
// day / company / user / model / role as each call completes.
//
// Money is micro-USD integers end to end. Summing millions of fractional-cent
// calls in float drifts; integers add up exactly.
// UsageRetentionDays bounds the rollup. Three years plus a leap day.
const UsageRetentionDays = 3*365 + 1
// MicrosPerUSD converts the stored integer unit to dollars.
const MicrosPerUSD = 1_000_000
// ModelPrice is micro-USD per 1,000,000 tokens.
type ModelPrice struct {
InputPerMTok int64
CachedInputPerMTok int64
OutputPerMTok int64
}
// defaultModelPrices is the fallback when ai_model_prices has no row for a model
// (a provider the admin pointed at without adding a price). Keeping the shipped
// GPT-5.6 rates here means a fresh database still costs correctly before the
// migration seed is edited. Rates: OpenAI list price, 2026-07-30.
var defaultModelPrices = map[string]ModelPrice{
"gpt-5.6-luna": {InputPerMTok: 200_000, CachedInputPerMTok: 20_000, OutputPerMTok: 1_200_000},
"gpt-5.6-terra": {InputPerMTok: 2_000_000, CachedInputPerMTok: 200_000, OutputPerMTok: 12_000_000},
"gpt-5.6-sol": {InputPerMTok: 5_000_000, CachedInputPerMTok: 500_000, OutputPerMTok: 30_000_000},
}
// DefaultModelPrice returns the shipped price for a model, ok=false when unknown.
func DefaultModelPrice(model string) (ModelPrice, bool) {
p, ok := defaultModelPrices[NormalizeModel(model)]
return p, ok
}
// NormalizeModel folds provider prefixes and dated suffixes onto the price key, so
// "openai/gpt-5.6-luna" and "gpt-5.6-luna-2026-07-30" both price as gpt-5.6-luna.
func NormalizeModel(model string) string {
m := strings.ToLower(strings.TrimSpace(model))
if i := strings.LastIndex(m, "/"); i >= 0 {
m = m[i+1:]
}
// Trim a trailing -YYYY-MM-DD snapshot tag.
if len(m) > 11 {
tail := m[len(m)-11:]
if tail[0] == '-' && isDigits(tail[1:5]) && tail[5] == '-' && isDigits(tail[6:8]) && tail[8] == '-' && isDigits(tail[9:11]) {
m = m[:len(m)-11]
}
}
return m
}
func isDigits(s string) bool {
for _, r := range s {
if r < '0' || r > '9' {
return false
}
}
return len(s) > 0
}
// CostMicros prices one call. Cached prompt tokens are billed at the cached rate
// and subtracted from fresh input — they are a subset of prompt tokens, not extra.
func CostMicros(p ModelPrice, promptTokens, cachedPromptTokens, completionTokens int) int64 {
if cachedPromptTokens < 0 {
cachedPromptTokens = 0
}
if cachedPromptTokens > promptTokens {
cachedPromptTokens = promptTokens
}
fresh := int64(promptTokens - cachedPromptTokens)
cached := int64(cachedPromptTokens)
out := int64(completionTokens)
if out < 0 {
out = 0
}
if fresh < 0 {
fresh = 0
}
// Rounded division keeps small calls from always costing zero.
return divRound(fresh*p.InputPerMTok, 1_000_000) +
divRound(cached*p.CachedInputPerMTok, 1_000_000) +
divRound(out*p.OutputPerMTok, 1_000_000)
}
func divRound(n, d int64) int64 {
if d == 0 {
return 0
}
return (n + d/2) / d
}
// priceCache avoids a DB read per call; prices change at most a few times a year.
type priceCache struct {
mu sync.RWMutex
prices map[string]ModelPrice
loadedAt time.Time
}
const priceCacheTTL = 5 * time.Minute
func (r *Recorder) priceFor(ctx context.Context, model string) ModelPrice {
key := NormalizeModel(model)
if r != nil {
r.prices.mu.RLock()
fresh := time.Since(r.prices.loadedAt) < priceCacheTTL
p, ok := r.prices.prices[key]
r.prices.mu.RUnlock()
if fresh && ok {
return p
}
if !fresh {
r.reloadPrices(ctx)
r.prices.mu.RLock()
p, ok = r.prices.prices[key]
r.prices.mu.RUnlock()
if ok {
return p
}
}
}
if p, ok := DefaultModelPrice(key); ok {
return p
}
// Unknown model: record usage with zero cost rather than guessing a rate.
return ModelPrice{}
}
func (r *Recorder) reloadPrices(ctx context.Context) {
if r == nil || r.pool == nil {
return
}
rows, err := r.pool.Query(ctx, `
SELECT model, input_micros_per_mtok, cached_input_micros_per_mtok, output_micros_per_mtok
FROM ai_model_prices`)
if err != nil {
// Keep whatever is cached; defaults still apply for misses.
r.prices.mu.Lock()
r.prices.loadedAt = time.Now()
r.prices.mu.Unlock()
return
}
defer rows.Close()
loaded := map[string]ModelPrice{}
for rows.Next() {
var model string
var in, cached, out int64
if err := rows.Scan(&model, &in, &cached, &out); err != nil {
continue
}
loaded[NormalizeModel(model)] = ModelPrice{
InputPerMTok: in, CachedInputPerMTok: cached, OutputPerMTok: out,
}
}
r.prices.mu.Lock()
r.prices.prices = loaded
r.prices.loadedAt = time.Now()
r.prices.mu.Unlock()
}
// SourceLive marks rows written by the recorder from a provider usage block.
// SourceBackfill marks history reconstructed from processed_products (see
// BackfillFromProcessedProducts) — no user attribution, best effort.
const (
SourceLive = "live"
SourceBackfill = "backfill"
)
const upsertUsageSQL = `
INSERT INTO ai_usage_daily (
day, company_id, user_id, model, role, source,
calls, failed_calls, prompt_tokens, cached_prompt_tokens, completion_tokens,
total_tokens, cost_micros, updated_at)
VALUES (
(now() AT TIME ZONE 'UTC')::date, $1, $2, $3, $4, 'live',
1, $5, $6, $7, $8,
$9, $10, now())
ON CONFLICT (day, company_id, user_id, model, role, source) DO UPDATE SET
calls = ai_usage_daily.calls + 1,
failed_calls = ai_usage_daily.failed_calls + EXCLUDED.failed_calls,
prompt_tokens = ai_usage_daily.prompt_tokens + EXCLUDED.prompt_tokens,
cached_prompt_tokens = ai_usage_daily.cached_prompt_tokens + EXCLUDED.cached_prompt_tokens,
completion_tokens = ai_usage_daily.completion_tokens + EXCLUDED.completion_tokens,
total_tokens = ai_usage_daily.total_tokens + EXCLUDED.total_tokens,
cost_micros = ai_usage_daily.cost_micros + EXCLUDED.cost_micros,
updated_at = now()`
// recordUsage rolls one call into the daily aggregate. Like Record it must never
// break the call it observes.
func (r *Recorder) recordUsage(ctx context.Context, c Call) {
if r == nil || r.pool == nil {
return
}
price := r.priceFor(ctx, c.Model)
cost := CostMicros(price, c.PromptTokens, c.CachedPromptTokens, c.OutputTokens)
failed := int64(0)
if strings.TrimSpace(c.Error) != "" {
failed = 1
}
writeCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 5*time.Second)
defer cancel()
_, err := r.pool.Exec(writeCtx, upsertUsageSQL,
orNilUUID(c.CompanyID), orNilUUID(c.UserID), NormalizeModel(c.Model), roleOrOther(c.Role),
failed, c.PromptTokens, c.CachedPromptTokens, c.OutputTokens,
c.TotalTokens, cost,
)
if err != nil {
r.noteFailure(err)
}
}
// orNilUUID keeps the primary key non-NULL (see the migration comment).
func orNilUUID(id uuid.UUID) uuid.UUID { return id }
func roleOrOther(role string) string {
if r := strings.TrimSpace(role); r != "" {
return r
}
return RoleOther
}
// CleanupExpiredUsage drops rollup rows past UsageRetentionDays.
func CleanupExpiredUsage(ctx context.Context, pool *pgxpool.Pool) (int64, error) {
if pool == nil {
return 0, nil
}
tag, err := pool.Exec(ctx, `
DELETE FROM ai_usage_daily
WHERE day < ((now() AT TIME ZONE 'UTC')::date - ($1::int))`, UsageRetentionDays)
if err != nil {
return 0, err
}
return tag.RowsAffected(), nil
}
+104
View File
@@ -0,0 +1,104 @@
package aiaudit
import "testing"
// Luna list price, 2026-07-30: $0.20 / $0.02 cached / $1.20 per 1M tokens.
func TestCostMicros_lunaListPrice(t *testing.T) {
t.Parallel()
luna, ok := DefaultModelPrice("gpt-5.6-luna")
if !ok {
t.Fatal("gpt-5.6-luna must have a shipped price")
}
if luna.InputPerMTok != 200_000 || luna.CachedInputPerMTok != 20_000 || luna.OutputPerMTok != 1_200_000 {
t.Fatalf("unexpected Luna price: %+v", luna)
}
// 1M input + 1M output = $0.20 + $1.20 = $1.40 = 1_400_000 micros.
if got := CostMicros(luna, 1_000_000, 0, 1_000_000); got != 1_400_000 {
t.Fatalf("got %d micros want 1400000", got)
}
// A realistic enhance call: 4k prompt, 800 completion.
// 4000*0.2 + 800*1.2 per 1M = 800 + 960 = 1760 micros ($0.00176).
if got := CostMicros(luna, 4000, 0, 800); got != 1760 {
t.Fatalf("got %d micros want 1760", got)
}
}
// Cached tokens are a SUBSET of prompt tokens, not extra ones. Billing them on top
// of full input would overstate every repeated system prompt.
func TestCostMicros_cachedTokensAreDiscountedNotAdded(t *testing.T) {
t.Parallel()
luna, _ := DefaultModelPrice("gpt-5.6-luna")
full := CostMicros(luna, 1_000_000, 0, 0) // 1M fresh input = 200000
allCached := CostMicros(luna, 1_000_000, 1_000_000, 0) // 1M cached = 20000
if full != 200_000 {
t.Fatalf("fresh input = %d want 200000", full)
}
if allCached != 20_000 {
t.Fatalf("fully cached input = %d want 20000", allCached)
}
if allCached >= full {
t.Fatal("cached input must be cheaper than fresh input")
}
// Half cached: 500k*0.2 + 500k*0.02 per 1M = 100000 + 10000.
if got := CostMicros(luna, 1_000_000, 500_000, 0); got != 110_000 {
t.Fatalf("half cached = %d want 110000", got)
}
// A provider reporting more cached than prompt tokens must not go negative.
if got := CostMicros(luna, 100, 900, 0); got < 0 {
t.Fatalf("cached > prompt produced negative cost %d", got)
}
}
// Small calls must not round to zero, or per-call spend vanishes at scale.
func TestCostMicros_smallCallsStillCost(t *testing.T) {
t.Parallel()
luna, _ := DefaultModelPrice("gpt-5.6-luna")
if got := CostMicros(luna, 500, 0, 100); got <= 0 {
t.Fatalf("small call cost %d, expected > 0", got)
}
}
func TestCostMicros_unknownModelIsFree(t *testing.T) {
t.Parallel()
if _, ok := DefaultModelPrice("some-local-model"); ok {
t.Fatal("unknown model should not have a shipped price")
}
// Zero price rather than a guessed rate: better to under-report than invent spend.
if got := CostMicros(ModelPrice{}, 1_000_000, 0, 1_000_000); got != 0 {
t.Fatalf("unpriced model cost %d want 0", got)
}
}
func TestNormalizeModel(t *testing.T) {
t.Parallel()
cases := map[string]string{
"gpt-5.6-luna": "gpt-5.6-luna",
"GPT-5.6-Luna": "gpt-5.6-luna",
"openai/gpt-5.6-luna": "gpt-5.6-luna",
"gpt-5.6-luna-2026-07-30": "gpt-5.6-luna",
" openai/GPT-5.6-Terra ": "gpt-5.6-terra",
"mock-llm": "mock-llm",
}
for in, want := range cases {
if got := NormalizeModel(in); got != want {
t.Fatalf("NormalizeModel(%q) = %q want %q", in, got, want)
}
}
// A dated snapshot must still price like its base model.
if _, ok := DefaultModelPrice("gpt-5.6-luna-2026-07-30"); !ok {
t.Fatal("dated snapshot should resolve to the base model price")
}
}
// Three years, so quarter- and year-scale cost questions still have data.
func TestUsageRetentionCoversThreeYears(t *testing.T) {
t.Parallel()
if UsageRetentionDays < 3*365 {
t.Fatalf("usage retention %d days is under 3 years", UsageRetentionDays)
}
if RetentionDays >= UsageRetentionDays {
t.Fatal("prompt bodies must expire long before the cost rollup")
}
}
+13 -12
View File
@@ -84,18 +84,19 @@ func (a *auditingCompleter) record(ctx context.Context, system, user string, cal
role = cc.Role
}
rec := aiaudit.Call{
CompanyID: a.companyID,
Role: role,
ProviderMode: a.providerMode,
Model: comp.Model,
System: system,
User: user,
Response: comp.Text,
FinishReason: finishReason(comp),
PromptTokens: comp.PromptTokens,
OutputTokens: comp.OutputTokens,
TotalTokens: comp.TotalTokens,
Duration: time.Since(started),
CompanyID: a.companyID,
Role: role,
ProviderMode: a.providerMode,
Model: comp.Model,
System: system,
User: user,
Response: comp.Text,
FinishReason: finishReason(comp),
PromptTokens: comp.PromptTokens,
CachedPromptTokens: comp.CachedPromptTokens,
OutputTokens: comp.OutputTokens,
TotalTokens: comp.TotalTokens,
Duration: time.Since(started),
}
if err != nil {
// The provider error text is already redacted by processing's OpenAI client.
+22 -5
View File
@@ -679,6 +679,14 @@ const (
// Prefer processed_* so dashboard product fields match V1 process items for the same EAN.
processedPreferredNameSQL = `COALESCE(NULLIF(p.processed_name, ''), NULLIF(p.name, ''), NULLIF(r.mapped_data->>'name', ''), NULLIF(r.mapped_data->>'title', ''), '')`
processedPreferredDescriptionSQL = `COALESCE(NULLIF(p.processed_description, ''), NULLIF(p.description, ''), NULLIF(r.mapped_data->>'description', ''), '')`
// Pre-enrichment values, with NO processed_* fallback. The preferred SQL above
// resolves to the enriched copy whenever it exists, so a review screen that read
// "name"/"description" showed the AI output on BOTH sides and every product
// looked unchanged. Review must compare against these instead.
processedOriginalNameSQL = `COALESCE(NULLIF(p.name, ''), NULLIF(r.mapped_data->>'name', ''), NULLIF(r.mapped_data->>'title', ''), '')`
processedOriginalDescriptionSQL = `COALESCE(NULLIF(p.description, ''), NULLIF(r.mapped_data->>'description', ''), '')`
// Category the feed supplied (before categorize assigned one).
processedFeedCategorySQL = `COALESCE(NULLIF(BTRIM(r.mapped_data->>'category'), ''), NULLIF(BTRIM(r.mapped_data->>'category_unique_id'), ''), '')`
processedHasNameSQL = `(` + processedPreferredNameSQL + ` <> '')`
processedHasDescriptionSQL = `(` + processedPreferredDescriptionSQL + ` <> '')`
processedHasCategorySQL = `(COALESCE(NULLIF(p.category, ''), '') <> '' AND lower(p.category) <> 'none')`
@@ -1133,6 +1141,7 @@ func (s *Service) ListProcessedProducts(ctx context.Context, companyID uuid.UUID
rows, err := s.Pool.Query(ctx, fmt.Sprintf(`
SELECT p.id, p.product_id,
`+processedPreferredNameSQL+` AS name,
`+processedOriginalNameSQL+` AS original_name,
COALESCE(NULLIF(p.processed_name, ''), '') AS processed_name,
p.category,
cat.name AS category_name,
@@ -1161,7 +1170,7 @@ func (s *Service) ListProcessedProducts(ctx context.Context, companyID uuid.UUID
}
defer rows.Close()
items, err := scanMaps(rows, []string{
"id", "product_id", "name", "processed_name", "category", "category_name", "category_unique_id",
"id", "product_id", "name", "original_name", "processed_name", "category", "category_name", "category_unique_id",
"status", "raw_product_id", "feed_id", "gtin",
"feed_name", "feed_last_synced_at", "raw_updated_at",
"has_name", "has_processed_name", "has_description", "has_processed_description", "has_category", "has_attributes", "has_processed_attributes",
@@ -1223,11 +1232,14 @@ func (s *Service) ListProcessedProductsDetailed(ctx context.Context, companyID u
rows, err := s.Pool.Query(ctx, fmt.Sprintf(`
SELECT p.id, p.product_id,
`+processedPreferredNameSQL+` AS name,
`+processedOriginalNameSQL+` AS original_name,
p.category,
cat.name AS category_name,
COALESCE(cat.unique_id, NULLIF(BTRIM(p.category), '')) AS category_unique_id,
`+processedFeedCategorySQL+` AS feed_category,
p.status, p.raw_product_id, p.feed_id, r.gtin,
`+processedPreferredDescriptionSQL+` AS description,
`+processedOriginalDescriptionSQL+` AS original_description,
p.processed_name, p.processed_description,
COALESCE(p.meta_title, ''), COALESCE(p.meta_description, ''),
p.attributes, p.processed_attributes, r.mapped_data,
@@ -1241,9 +1253,10 @@ func (s *Service) ListProcessedProductsDetailed(ctx context.Context, companyID u
}
defer rows.Close()
items, err := scanMaps(rows, []string{
"id", "product_id", "name", "category", "category_name", "category_unique_id",
"id", "product_id", "name", "original_name", "category", "category_name", "category_unique_id",
"feed_category",
"status", "raw_product_id", "feed_id", "gtin",
"description", "processed_name", "processed_description",
"description", "original_description", "processed_name", "processed_description",
"meta_title", "meta_description",
"attributes", "processed_attributes", "mapped_data",
"created_at", "updated_at",
@@ -1263,10 +1276,13 @@ func (s *Service) GetProcessedProduct(ctx context.Context, companyID, id uuid.UU
row := s.Pool.QueryRow(ctx, `
SELECT p.id, p.product_id,
`+processedPreferredNameSQL+` AS name,
`+processedOriginalNameSQL+` AS original_name,
p.category,
cat.name AS category_name,
COALESCE(cat.unique_id, NULLIF(BTRIM(p.category), '')) AS category_unique_id,
`+processedFeedCategorySQL+` AS feed_category,
`+processedPreferredDescriptionSQL+` AS description,
`+processedOriginalDescriptionSQL+` AS original_description,
p.processed_name, p.processed_description,
p.status, p.attributes, p.processed_attributes, r.gtin, r.mapped_data,
COALESCE(p.feed_id, r.feed_id) AS feed_id,
@@ -1285,8 +1301,9 @@ func (s *Service) GetProcessedProduct(ctx context.Context, companyID, id uuid.UU
LEFT JOIN input_feeds f ON f.id = COALESCE(p.feed_id, r.feed_id)`+processedCategoryResolveJoin+`
WHERE p.id = $1 AND p.company_id = $2`, id, companyID)
item, err := scanMap(row, []string{
"id", "product_id", "name", "category", "category_name", "category_unique_id",
"description", "processed_name", "processed_description",
"id", "product_id", "name", "original_name", "category", "category_name", "category_unique_id",
"feed_category",
"description", "original_description", "processed_name", "processed_description",
"status", "attributes", "processed_attributes", "gtin", "mapped_data",
"feed_id", "feed_name", "feed_last_synced_at", "raw_updated_at",
"has_processed_name", "has_processed_description", "has_attributes", "has_processed_attributes",
@@ -0,0 +1,243 @@
package httpapi
import (
"context"
"net/http"
"strconv"
"strings"
"time"
"github.com/descrybe/descrybe-v2/apps/api/internal/aiaudit"
"github.com/google/uuid"
)
// Admin AI cost report: what each tenant and each user costs, plus the total.
//
// Reads ai_usage_daily, the durable rollup written alongside every captured call.
// ai_call_logs (full prompts) is pruned after a week; this survives for
// aiaudit.UsageRetentionDays (3 years), so quarter- and year-scale questions work.
const (
adminAICostsTimeout = 30 * time.Second
adminAICostsMaxDays = 1200 // slightly over the 3-year retention
adminAICostsMaxRows = 500
adminAICostsDefDays = 30
)
// handleAdminAICosts returns spend grouped by company and by user, with totals.
// GET /api/admin/ai-costs?days=30&from=&to=&company_id=
func (s *Server) handleAdminAICosts(w http.ResponseWriter, r *http.Request) {
ctx, cancel := context.WithTimeout(r.Context(), adminAICostsTimeout)
defer cancel()
if s.Pool == nil {
Error(w, http.StatusServiceUnavailable, "database unavailable")
return
}
q := r.URL.Query()
from, to, err := adminCostRange(q.Get("from"), q.Get("to"), q.Get("days"))
if err != nil {
Error(w, http.StatusBadRequest, err.Error())
return
}
where := []string{"day >= $1", "day <= $2"}
args := []any{from, to}
if raw := strings.TrimSpace(q.Get("company_id")); raw != "" {
id, perr := uuid.Parse(raw)
if perr != nil {
Error(w, http.StatusBadRequest, "invalid company_id")
return
}
args = append(args, id)
where = append(where, "company_id = $"+strconv.Itoa(len(args)))
}
whereSQL := strings.Join(where, " AND ")
totals, err := s.scanCostTotals(ctx, whereSQL, args)
if err != nil {
Error(w, http.StatusInternalServerError, "load ai costs failed")
return
}
byCompany, err := s.scanCostGroup(ctx, costGroupCompany, whereSQL, args)
if err != nil {
Error(w, http.StatusInternalServerError, "load ai costs by company failed")
return
}
byUser, err := s.scanCostGroup(ctx, costGroupUser, whereSQL, args)
if err != nil {
Error(w, http.StatusInternalServerError, "load ai costs by user failed")
return
}
byModel, err := s.scanCostGroup(ctx, costGroupModel, whereSQL, args)
if err != nil {
Error(w, http.StatusInternalServerError, "load ai costs by model failed")
return
}
JSON(w, http.StatusOK, map[string]any{
"from": from.Format("2006-01-02"),
"to": to.Format("2006-01-02"),
"currency": "USD",
"totals": totals,
"by_company": byCompany,
"by_user": byUser,
"by_model": byModel,
"usage_retention_days": aiaudit.UsageRetentionDays,
"prompt_retention_days": aiaudit.RetentionDays,
})
}
func adminCostRange(fromRaw, toRaw, daysRaw string) (time.Time, time.Time, error) {
today := time.Now().UTC().Truncate(24 * time.Hour)
to := today
if s := strings.TrimSpace(toRaw); s != "" {
t, err := time.Parse("2006-01-02", s)
if err != nil {
return time.Time{}, time.Time{}, errBadRequest("invalid to date (YYYY-MM-DD)")
}
to = t.UTC()
}
if s := strings.TrimSpace(fromRaw); s != "" {
f, err := time.Parse("2006-01-02", s)
if err != nil {
return time.Time{}, time.Time{}, errBadRequest("invalid from date (YYYY-MM-DD)")
}
from := f.UTC()
if from.After(to) {
return time.Time{}, time.Time{}, errBadRequest("from must not be after to")
}
if to.Sub(from) > time.Duration(adminAICostsMaxDays)*24*time.Hour {
return time.Time{}, time.Time{}, errBadRequest("range too large")
}
return from, to, nil
}
days := adminAICostsDefDays
if s := strings.TrimSpace(daysRaw); s != "" {
n, err := strconv.Atoi(s)
if err != nil || n <= 0 {
return time.Time{}, time.Time{}, errBadRequest("invalid days")
}
days = n
}
if days > adminAICostsMaxDays {
days = adminAICostsMaxDays
}
return to.AddDate(0, 0, -(days - 1)), to, nil
}
type badRequestErr string
func (e badRequestErr) Error() string { return string(e) }
func errBadRequest(msg string) error { return badRequestErr(msg) }
func (s *Server) scanCostTotals(ctx context.Context, whereSQL string, args []any) (map[string]any, error) {
var calls, failed, promptTok, cachedTok, outTok, totalTok, costMicros int64
var companies, users int64
err := s.Pool.QueryRow(ctx, `
SELECT COALESCE(sum(calls),0), COALESCE(sum(failed_calls),0),
COALESCE(sum(prompt_tokens),0), COALESCE(sum(cached_prompt_tokens),0),
COALESCE(sum(completion_tokens),0), COALESCE(sum(total_tokens),0),
COALESCE(sum(cost_micros),0),
count(DISTINCT company_id), count(DISTINCT user_id)
FROM ai_usage_daily WHERE `+whereSQL, args...).
Scan(&calls, &failed, &promptTok, &cachedTok, &outTok, &totalTok, &costMicros, &companies, &users)
if err != nil {
return nil, err
}
return costRow(map[string]any{
"companies": companies,
"users": users,
}, calls, failed, promptTok, cachedTok, outTok, totalTok, costMicros), nil
}
type costGroup string
const (
costGroupCompany costGroup = "company"
costGroupUser costGroup = "user"
costGroupModel costGroup = "model"
)
func (s *Server) scanCostGroup(ctx context.Context, group costGroup, whereSQL string, args []any) ([]map[string]any, error) {
var sql string
switch group {
case costGroupCompany:
sql = `
SELECT u.company_id::text, COALESCE(c.name, ''), '', '',
COALESCE(sum(u.calls),0), COALESCE(sum(u.failed_calls),0),
COALESCE(sum(u.prompt_tokens),0), COALESCE(sum(u.cached_prompt_tokens),0),
COALESCE(sum(u.completion_tokens),0), COALESCE(sum(u.total_tokens),0),
COALESCE(sum(u.cost_micros),0)
FROM ai_usage_daily u
LEFT JOIN companies c ON c.id = u.company_id
WHERE ` + whereSQL + `
GROUP BY u.company_id, c.name
ORDER BY COALESCE(sum(u.cost_micros),0) DESC
LIMIT ` + strconv.Itoa(adminAICostsMaxRows)
case costGroupUser:
sql = `
SELECT u.user_id::text, COALESCE(usr.email, ''), u.company_id::text, COALESCE(c.name, ''),
COALESCE(sum(u.calls),0), COALESCE(sum(u.failed_calls),0),
COALESCE(sum(u.prompt_tokens),0), COALESCE(sum(u.cached_prompt_tokens),0),
COALESCE(sum(u.completion_tokens),0), COALESCE(sum(u.total_tokens),0),
COALESCE(sum(u.cost_micros),0)
FROM ai_usage_daily u
LEFT JOIN users usr ON usr.id = u.user_id
LEFT JOIN companies c ON c.id = u.company_id
WHERE ` + whereSQL + `
GROUP BY u.user_id, usr.email, u.company_id, c.name
ORDER BY COALESCE(sum(u.cost_micros),0) DESC
LIMIT ` + strconv.Itoa(adminAICostsMaxRows)
default:
sql = `
SELECT u.model, u.model, '', '',
COALESCE(sum(u.calls),0), COALESCE(sum(u.failed_calls),0),
COALESCE(sum(u.prompt_tokens),0), COALESCE(sum(u.cached_prompt_tokens),0),
COALESCE(sum(u.completion_tokens),0), COALESCE(sum(u.total_tokens),0),
COALESCE(sum(u.cost_micros),0)
FROM ai_usage_daily u
WHERE ` + whereSQL + `
GROUP BY u.model
ORDER BY COALESCE(sum(u.cost_micros),0) DESC
LIMIT ` + strconv.Itoa(adminAICostsMaxRows)
}
rows, err := s.Pool.Query(ctx, sql, args...)
if err != nil {
return nil, err
}
defer rows.Close()
out := []map[string]any{}
for rows.Next() {
var id, label, companyID, companyName string
var calls, failed, promptTok, cachedTok, outTok, totalTok, costMicros int64
if err := rows.Scan(&id, &label, &companyID, &companyName,
&calls, &failed, &promptTok, &cachedTok, &outTok, &totalTok, &costMicros); err != nil {
return nil, err
}
base := map[string]any{"id": id, "label": label}
if group == costGroupUser {
base["company_id"] = companyID
base["company_name"] = companyName
}
out = append(out, costRow(base, calls, failed, promptTok, cachedTok, outTok, totalTok, costMicros))
}
return out, rows.Err()
}
// costRow returns cost as integer micros AND a display-ready USD float. Callers
// must sum micros, never the float — that is the whole reason cost is stored as an
// integer.
func costRow(base map[string]any, calls, failed, promptTok, cachedTok, outTok, totalTok, costMicros int64) map[string]any {
base["calls"] = calls
base["failed_calls"] = failed
base["prompt_tokens"] = promptTok
base["cached_prompt_tokens"] = cachedTok
base["completion_tokens"] = outTok
base["total_tokens"] = totalTok
base["cost_micros"] = costMicros
base["cost_usd"] = float64(costMicros) / float64(aiaudit.MicrosPerUSD)
return base
}
+1
View File
@@ -405,6 +405,7 @@ func (s *Server) Router() http.Handler {
r.Get("/readiness", s.handleAdminReadiness)
r.Get("/diagnostics", s.handleAdminDiagnostics)
r.Get("/ai-calls", s.handleAdminListAICalls)
r.Get("/ai-costs", s.handleAdminAICosts)
r.Get("/ai-calls/{id}", s.handleAdminGetAICall)
r.Get("/analytics", s.handleAdminAnalytics)
r.Get("/jobs", s.handleAdminListJobs)
+8 -4
View File
@@ -43,10 +43,14 @@ type EnableChecker interface {
type Completion struct {
Text string
PromptTokens int
OutputTokens int
TotalTokens int
Model string
Raw any
// CachedPromptTokens is the cached subset of PromptTokens (OpenAI
// usage.prompt_tokens_details.cached_tokens). Billed far cheaper than fresh
// input, so cost accounting must subtract it. 0 when the provider omits it.
CachedPromptTokens int
OutputTokens int
TotalTokens int
Model string
Raw any
}
// Embedder turns text into vectors (OpenAI-compatible /v1/embeddings).
+11 -5
View File
@@ -242,6 +242,11 @@ type chatResponse struct {
PromptTokens int `json:"prompt_tokens"`
CompletionTokens int `json:"completion_tokens"`
TotalTokens int `json:"total_tokens"`
// Cached prompt tokens bill at roughly a tenth of fresh input, and enhance
// re-sends a large shared system prompt, so cost is wrong without them.
PromptTokensDetails struct {
CachedTokens int `json:"cached_tokens"`
} `json:"prompt_tokens_details"`
} `json:"usage"`
Error *openAIErrorBody `json:"error"`
}
@@ -557,11 +562,12 @@ func (c *OpenAIClient) doComplete(ctx context.Context, system, user string, temp
log.Printf("openai: chat content ok model=%s finish_reason=%s content_runes=%d prompt_tokens=%d completion_tokens=%d total_tokens=%d",
c.Model, finishReason, len([]rune(text)), usagePrompt, usageOut, usageTotal)
return Completion{
Text: text,
PromptTokens: usagePrompt,
OutputTokens: usageOut,
TotalTokens: usageTotal,
Model: parsed.Model,
Text: text,
PromptTokens: usagePrompt,
CachedPromptTokens: parsed.Usage.PromptTokensDetails.CachedTokens,
OutputTokens: usageOut,
TotalTokens: usageTotal,
Model: parsed.Model,
Raw: map[string]any{
"model": parsed.Model,
"usage": parsed.Usage,
+65
View File
@@ -0,0 +1,65 @@
-- +goose Up
-- Durable AI cost accounting, deliberately separate from ai_call_logs.
--
-- ai_call_logs holds full prompts and is pruned after 7 days. Cost has to outlive
-- that by years, so usage is rolled up per day / company / user / model / role at
-- capture time. One row per combination per day keeps three years small: a busy
-- tenant with 5 users and 3 roles adds ~45 rows a day, not one per product.
--
-- Cost is stored in micro-USD integers, never floats: summing millions of
-- fractional-cent calls in float drifts, and money must add up exactly.
CREATE TABLE IF NOT EXISTS ai_model_prices (
model TEXT PRIMARY KEY,
-- Micro-USD per 1,000,000 tokens. $0.20/1M = 200000 micros.
input_micros_per_mtok BIGINT NOT NULL DEFAULT 0,
cached_input_micros_per_mtok BIGINT NOT NULL DEFAULT 0,
output_micros_per_mtok BIGINT NOT NULL DEFAULT 0,
source TEXT NOT NULL DEFAULT '',
updated_at TIMESTAMPTZ NOT NULL DEFAULT now()
);
-- OpenAI GPT-5.6 family, rates effective 2026-07-30.
-- Admin can UPDATE these when list prices change; already-recorded cost is never
-- recalculated, so history keeps the price that applied at the time.
INSERT INTO ai_model_prices (model, input_micros_per_mtok, cached_input_micros_per_mtok, output_micros_per_mtok, source)
VALUES
('gpt-5.6-luna', 200000, 20000, 1200000, 'openai list price 2026-07-30'),
('gpt-5.6-terra', 2000000, 200000, 12000000, 'openai list price 2026-07-30'),
('gpt-5.6-sol', 5000000, 500000, 30000000, 'openai list price 2026-07-30')
ON CONFLICT (model) DO NOTHING;
CREATE TABLE IF NOT EXISTS ai_usage_daily (
day DATE NOT NULL,
-- Nil UUID rather than NULL: these are primary-key columns and NULL would let
-- duplicate rows accumulate instead of upserting.
company_id UUID NOT NULL DEFAULT '00000000-0000-0000-0000-000000000000',
user_id UUID NOT NULL DEFAULT '00000000-0000-0000-0000-000000000000',
model TEXT NOT NULL DEFAULT '',
role TEXT NOT NULL DEFAULT 'other',
calls BIGINT NOT NULL DEFAULT 0,
failed_calls BIGINT NOT NULL DEFAULT 0,
prompt_tokens BIGINT NOT NULL DEFAULT 0,
cached_prompt_tokens BIGINT NOT NULL DEFAULT 0,
completion_tokens BIGINT NOT NULL DEFAULT 0,
total_tokens BIGINT NOT NULL DEFAULT 0,
cost_micros BIGINT NOT NULL DEFAULT 0,
updated_at TIMESTAMPTZ NOT NULL DEFAULT now(),
PRIMARY KEY (day, company_id, user_id, model, role)
);
-- No FK to companies/users: cost history must survive a tenant or user being
-- deleted, otherwise the yearly total silently drops.
CREATE INDEX IF NOT EXISTS ai_usage_daily_company_day_idx
ON ai_usage_daily (company_id, day DESC);
CREATE INDEX IF NOT EXISTS ai_usage_daily_user_day_idx
ON ai_usage_daily (user_id, day DESC);
CREATE INDEX IF NOT EXISTS ai_usage_daily_day_idx
ON ai_usage_daily (day DESC);
-- +goose Down
DROP TABLE IF EXISTS ai_usage_daily;
DROP TABLE IF EXISTS ai_model_prices;
@@ -0,0 +1,25 @@
-- +goose Up
-- Distinguish live capture from history reconstructed after the fact.
--
-- ai_usage_daily rows written by the recorder are exact: they come from the
-- provider's usage block at call time. Rows recovered from
-- processed_products.gpt_response are best-effort — that JSON has no user
-- attribution and only survives for products whose response metadata was kept.
--
-- Keeping the two apart (and putting source in the primary key) makes the
-- backfill idempotent: it deletes and rewrites only its own rows, and can never
-- double-count a call the recorder already logged.
ALTER TABLE ai_usage_daily
ADD COLUMN IF NOT EXISTS source TEXT NOT NULL DEFAULT 'live';
ALTER TABLE ai_usage_daily DROP CONSTRAINT IF EXISTS ai_usage_daily_pkey;
ALTER TABLE ai_usage_daily
ADD CONSTRAINT ai_usage_daily_pkey PRIMARY KEY (day, company_id, user_id, model, role, source);
-- +goose Down
ALTER TABLE ai_usage_daily DROP CONSTRAINT IF EXISTS ai_usage_daily_pkey;
DELETE FROM ai_usage_daily WHERE source <> 'live';
ALTER TABLE ai_usage_daily
ADD CONSTRAINT ai_usage_daily_pkey PRIMARY KEY (day, company_id, user_id, model, role);
ALTER TABLE ai_usage_daily DROP COLUMN IF EXISTS source;
+6
View File
@@ -38,6 +38,12 @@ export const ADMIN_NAV_ROUTES: readonly AdminNavRoute[] = [
group: "ops",
fullAdminOnly: true
},
{
titleKey: "admin.nav.aiCosts",
href: "/admin/ai-costs",
group: "ops",
fullAdminOnly: true
},
{
titleKey: "admin.nav.stuckProducts",
href: "/admin/stuck-products",
@@ -30,6 +30,7 @@
ArrowLeft,
FileWarning,
MessagesSquare,
Wallet,
LogOut,
X
} from "@lucide/svelte";
@@ -42,6 +43,7 @@
"/admin/support/knowledge": BookOpen,
"/admin/diagnostics": Activity,
"/admin/ai-calls": MessagesSquare,
"/admin/ai-costs": Wallet,
"/admin/stuck-products": ClipboardList,
"/admin/orphan-processed": FileWarning,
"/admin/billing": CreditCard,
@@ -411,6 +411,47 @@
return unique.length === rows.length ? rows : unique;
});
// --- Review diffs -------------------------------------------------------
// Category the feed supplied vs the one processing assigned. Most catalogs ship
// no category at all, so "(none) -> Stolcki" is the change worth showing.
const feedCategoryRaw = $derived(String(product?.feed_category ?? "").trim());
const feedCategoryLabel = $derived.by(() => {
if (!feedCategoryRaw) return "";
const match = findCategoryOption(categories, feedCategoryRaw);
return match?.name || feedCategoryRaw;
});
const assignedCategoryLabel = $derived.by(() => {
if (!category || category === "none") return "";
const match = findCategoryOption(categories, category);
return match?.name || categoryFieldDisplayValue(categories, product, category) || category;
});
const categoryChanged = $derived(
kind === "processed" &&
assignedCategoryLabel.trim().toLowerCase() !== feedCategoryLabel.trim().toLowerCase()
);
/** Per-key attribute diff: what enrichment added or rewrote versus the feed. */
const attrDiff = $derived.by(() => {
const before = new Map(
productAttrEntries(product?.attributes).map((a) => [a.key.toLowerCase(), a])
);
const after = new Map(
productAttrEntries(product?.processed_attributes).map((a) => [a.key.toLowerCase(), a])
);
const keys = [...new Set([...before.keys(), ...after.keys()])].sort();
const rows = keys.map((k) => {
const b = before.get(k);
const a = after.get(k);
const state = !b ? "added" : !a ? "removed" : b.value === a.value ? "same" : "changed";
return { key: (a ?? b)?.key ?? k, before: b?.value ?? "", after: a?.value ?? "", state };
});
return {
rows,
changed: rows.filter((r) => r.state !== "same")
};
});
const attrsChanged = $derived(kind === "processed" && attrDiff.changed.length > 0);
const feedAttrs = $derived.by(() => {
const rows = productFeedAttributeEntries(product);
const covered = new Set(
@@ -748,26 +789,97 @@
{/if}
</div>
<div class="space-y-2">
<Label for="review-category">{i18n.t("products.enrichment.piece.category")}</Label>
<select
id="review-category"
class="flex h-10 w-full rounded-md border border-border bg-background px-3 py-2 text-sm shadow-sm focus-visible:border-ring focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-ring/30"
bind:value={category}
onchange={markDirty}
>
<option value="none">{i18n.t("products.edit.noCategory")}</option>
{#each categories as cat}
<option value={cat.uniqueId}>{cat.name}</option>
{/each}
{#if category !== "none" && !categories.some((c) => c.uniqueId === category)}
<option value={category}
>{categoryFieldDisplayValue(categories, product, category)}</option
>
<div
class="space-y-2 rounded-md border border-transparent p-1 {categoryChanged
? 'border-chart-amber/40 bg-chart-amber/10'
: ''}"
>
<div class="flex items-center justify-between gap-2">
<Label for="review-category">{i18n.t("products.enrichment.piece.category")}</Label>
{#if categoryChanged}
<Badge variant="outline" class="text-[10px]">{i18n.t("products.edit.changed")}</Badge>
{:else}
<Badge variant="secondary" class="text-[10px]">{i18n.t("products.edit.matched")}</Badge>
{/if}
</div>
<div class="grid gap-3 sm:grid-cols-2">
<div class="space-y-1.5">
<p class="text-xs font-medium uppercase tracking-wide text-muted-foreground">
{i18n.t("products.edit.original")}
</p>
<div
class="min-h-10 rounded-md border border-border bg-muted/30 px-3 py-2 text-sm text-muted-foreground"
>
{feedCategoryLabel || i18n.t("products.edit.noCategoryFromFeed")}
</div>
</div>
<div class="space-y-1.5">
<p class="text-xs font-medium uppercase tracking-wide text-muted-foreground">
{i18n.t("products.edit.enriched")}
</p>
<select
id="review-category"
class="flex h-10 w-full rounded-md border border-border bg-muted/50 px-3 py-2 text-sm shadow-sm focus-visible:border-ring focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-ring/30"
bind:value={category}
onchange={markDirty}
>
<option value="none">{i18n.t("products.edit.noCategory")}</option>
{#each categories as cat}
<option value={cat.uniqueId}>{cat.name}</option>
{/each}
{#if category !== "none" && !categories.some((c) => c.uniqueId === category)}
<option value={category}
>{categoryFieldDisplayValue(categories, product, category)}</option
>
{/if}
</select>
</div>
</div>
</div>
<div
class="space-y-2 rounded-md border border-transparent p-1 {attrsChanged
? 'border-chart-amber/40 bg-chart-amber/10'
: ''}"
>
<div class="flex items-center justify-between gap-2">
<Label>{i18n.t("products.enrichment.piece.attributes")}</Label>
{#if attrsChanged}
<Badge variant="outline" class="text-[10px]">
{i18n.t("products.edit.attrsChangedCount", { count: attrDiff.changed.length })}
</Badge>
{:else}
<Badge variant="secondary" class="text-[10px]">{i18n.t("products.edit.matched")}</Badge>
{/if}
</div>
{#if attrDiff.rows.length === 0}
<p class="text-xs text-muted-foreground">{i18n.t("products.edit.noAttributes")}</p>
{:else}
<div class="overflow-x-auto rounded-md border border-border">
<table class="w-full text-xs">
<thead class="bg-muted/40 text-muted-foreground">
<tr>
<th class="px-2 py-1.5 text-left font-medium">{i18n.t("products.edit.attrKey")}</th>
<th class="px-2 py-1.5 text-left font-medium">{i18n.t("products.edit.original")}</th>
<th class="px-2 py-1.5 text-left font-medium">{i18n.t("products.edit.enriched")}</th>
</tr>
</thead>
<tbody>
{#each attrDiff.rows as row (row.key)}
<tr class="border-t border-border {row.state === 'same' ? '' : 'bg-chart-amber/5'}">
<td class="px-2 py-1.5 font-mono">{row.key}</td>
<td class="px-2 py-1.5 text-muted-foreground">{row.before || "—"}</td>
<td class="px-2 py-1.5 {row.state === 'same' ? 'text-muted-foreground' : 'font-medium text-foreground'}">
{row.after || "—"}
</td>
</tr>
{/each}
</tbody>
</table>
</div>
{/if}
</select>
</div>
</TabsContent>
</div>
</TabsContent>
{/if}
<TabsContent value="details" class="mt-4 space-y-5">
+31 -2
View File
@@ -42,6 +42,12 @@ export type ProductRow = {
id: string | number;
product_id?: string | null;
name?: string | null;
/**
* Pre-enrichment name. `name` is the API's display-preferred value and resolves
* to the enriched copy when one exists, so a review diff MUST use this instead
* comparing against `name` showed the AI output on both sides.
*/
original_name?: string | null;
processed_name?: string | null;
title?: string | null;
sku?: string | null;
@@ -51,6 +57,8 @@ export type ProductRow = {
category_name?: string | null;
/** Canonical categories.unique_id when resolvable. */
category_unique_id?: string | null;
/** Category the feed supplied, before categorize assigned one. */
feed_category?: string | null;
status?: string | null;
processing_status?: string | null;
raw_product_id?: string | null;
@@ -59,6 +67,8 @@ export type ProductRow = {
feed_last_synced_at?: string | null;
raw_updated_at?: string | null;
description?: string | null;
/** Pre-enrichment description (see original_name). */
original_description?: string | null;
processed_description?: string | null;
meta_title?: string | null;
meta_description?: string | null;
@@ -510,7 +520,16 @@ export function productAttrEntries(raw: unknown): { key: string; value: string }
/** Original feed description when processed_products.description was never filled (legacy). */
export function resolveOriginalDescription(product: ProductRow | null | undefined): string {
if (!product) return "";
if (typeof product.description === "string" && product.description.trim() !== "") {
if (typeof product.original_description === "string" && product.original_description.trim() !== "") {
return product.original_description;
}
// Fall back to `description` only for rows that carry no enrichment (raw products,
// older API shapes) — for processed rows it is the enriched copy.
if (
typeof product.description === "string" &&
product.description.trim() !== "" &&
product.description !== product.processed_description
) {
return product.description;
}
const mapped = asRecord(product.mapped_data);
@@ -525,7 +544,17 @@ export function resolveOriginalDescription(product: ProductRow | null | undefine
/** Original feed name when processed_products.name was never filled (legacy). */
export function resolveOriginalName(product: ProductRow | null | undefined): string {
if (!product) return "";
if (typeof product.name === "string" && product.name.trim() !== "") return product.name;
if (typeof product.original_name === "string" && product.original_name.trim() !== "") {
return product.original_name;
}
// See resolveOriginalDescription: `name` is display-preferred, not the original.
if (
typeof product.name === "string" &&
product.name.trim() !== "" &&
product.name !== product.processed_name
) {
return product.name;
}
if (typeof product.title === "string" && product.title.trim() !== "") return product.title;
const mapped = asRecord(product.mapped_data);
if (!mapped) return "";
+30
View File
@@ -5742,4 +5742,34 @@ export const de: MessageDict = {
"admin.aiCalls.loadFailed": "KI-Aufrufe konnten nicht geladen werden.",
"admin.aiCalls.copyFailed": "Kopieren fehlgeschlagen.",
"common.copy": "Kopieren",
"products.edit.noCategoryFromFeed": "Keine Kategorie im Feed",
"products.edit.attrsChangedCount": "{count} geaendert",
"products.edit.attrKey": "Attribut",
"products.edit.noAttributes": "Dieses Produkt hat keine Attribute.",
"admin.nav.aiCosts": "KI-Kosten",
"admin.aiCosts.title": "KI-Kosten",
"admin.aiCosts.description": "Was jeder Mandant und Benutzer kostet. {days} Tage aufbewahrt.",
"admin.aiCosts.loadFailed": "KI-Kosten konnten nicht geladen werden.",
"admin.aiCosts.totalTitle": "Gesamtausgaben",
"admin.aiCosts.rangeLabel": "{from} bis {to}",
"admin.aiCosts.range": "Zeitraum",
"admin.aiCosts.lastNDays": "Letzte {count} Tage",
"admin.aiCosts.grandTotal": "Gesamtsumme",
"admin.aiCosts.calls": "Aufrufe",
"admin.aiCosts.failedCalls": "{count} fehlgeschlagen",
"admin.aiCosts.tokens": "Token",
"admin.aiCosts.cachedTokens": "{count} zwischengespeichert",
"admin.aiCosts.tenants": "Mandanten",
"admin.aiCosts.usersCount": "{count} Benutzer",
"admin.aiCosts.byUser": "Kosten pro Benutzer",
"admin.aiCosts.byCompany": "Kosten pro Firma",
"admin.aiCosts.byModel": "Kosten pro Modell",
"admin.aiCosts.emptyTitle": "Keine Ausgaben erfasst",
"admin.aiCosts.emptyBody": "In diesem Zeitraum wurde nichts berechnet.",
"admin.aiCosts.colName": "Name",
"admin.aiCosts.colCompany": "Firma",
"admin.aiCosts.colIn": "Eingabe",
"admin.aiCosts.colOut": "Ausgabe",
"admin.aiCosts.colCost": "Kosten",
"admin.aiCosts.unattributed": "Nicht zugeordnet",
};
+30
View File
@@ -5828,4 +5828,34 @@ export const en: MessageDict = {
"admin.aiCalls.loadFailed": "Could not load AI calls.",
"admin.aiCalls.copyFailed": "Could not copy to clipboard.",
"common.copy": "Copy",
"products.edit.noCategoryFromFeed": "No category in feed",
"products.edit.attrsChangedCount": "{count} changed",
"products.edit.attrKey": "Attribute",
"products.edit.noAttributes": "No attributes on this product.",
"admin.nav.aiCosts": "AI costs",
"admin.aiCosts.title": "AI costs",
"admin.aiCosts.description": "What each tenant and user costs to run. Kept for {days} days.",
"admin.aiCosts.loadFailed": "Could not load AI costs.",
"admin.aiCosts.totalTitle": "Total spend",
"admin.aiCosts.rangeLabel": "{from} to {to}",
"admin.aiCosts.range": "Range",
"admin.aiCosts.lastNDays": "Last {count} days",
"admin.aiCosts.grandTotal": "Grand total",
"admin.aiCosts.calls": "Calls",
"admin.aiCosts.failedCalls": "{count} failed",
"admin.aiCosts.tokens": "Tokens",
"admin.aiCosts.cachedTokens": "{count} cached",
"admin.aiCosts.tenants": "Tenants",
"admin.aiCosts.usersCount": "{count} users",
"admin.aiCosts.byUser": "Cost by user",
"admin.aiCosts.byCompany": "Cost by company",
"admin.aiCosts.byModel": "Cost by model",
"admin.aiCosts.emptyTitle": "No spend recorded",
"admin.aiCosts.emptyBody": "Nothing was billed in this range.",
"admin.aiCosts.colName": "Name",
"admin.aiCosts.colCompany": "Company",
"admin.aiCosts.colIn": "Input",
"admin.aiCosts.colOut": "Output",
"admin.aiCosts.colCost": "Cost",
"admin.aiCosts.unattributed": "Unattributed",
};
+30
View File
@@ -5751,4 +5751,34 @@ export const es: MessageDict = {
"admin.aiCalls.loadFailed": "No se pudieron cargar las llamadas de IA.",
"admin.aiCalls.copyFailed": "No se pudo copiar al portapapeles.",
"common.copy": "Copiar",
"products.edit.noCategoryFromFeed": "Sin categoria en el feed",
"products.edit.attrsChangedCount": "{count} modificados",
"products.edit.attrKey": "Atributo",
"products.edit.noAttributes": "Este producto no tiene atributos.",
"admin.nav.aiCosts": "Costes de IA",
"admin.aiCosts.title": "Costes de IA",
"admin.aiCosts.description": "Lo que cuesta cada inquilino y usuario. Se guarda {days} dias.",
"admin.aiCosts.loadFailed": "No se pudieron cargar los costes de IA.",
"admin.aiCosts.totalTitle": "Gasto total",
"admin.aiCosts.rangeLabel": "{from} a {to}",
"admin.aiCosts.range": "Periodo",
"admin.aiCosts.lastNDays": "Ultimos {count} dias",
"admin.aiCosts.grandTotal": "Total general",
"admin.aiCosts.calls": "Llamadas",
"admin.aiCosts.failedCalls": "{count} fallidas",
"admin.aiCosts.tokens": "Tokens",
"admin.aiCosts.cachedTokens": "{count} en cache",
"admin.aiCosts.tenants": "Inquilinos",
"admin.aiCosts.usersCount": "{count} usuarios",
"admin.aiCosts.byUser": "Coste por usuario",
"admin.aiCosts.byCompany": "Coste por empresa",
"admin.aiCosts.byModel": "Coste por modelo",
"admin.aiCosts.emptyTitle": "Sin gasto registrado",
"admin.aiCosts.emptyBody": "No se facturo nada en este periodo.",
"admin.aiCosts.colName": "Nombre",
"admin.aiCosts.colCompany": "Empresa",
"admin.aiCosts.colIn": "Entrada",
"admin.aiCosts.colOut": "Salida",
"admin.aiCosts.colCost": "Coste",
"admin.aiCosts.unattributed": "Sin asignar",
};
+30
View File
@@ -5751,4 +5751,34 @@ export const fr: MessageDict = {
"admin.aiCalls.loadFailed": "Impossible de charger les appels IA.",
"admin.aiCalls.copyFailed": "Copie dans le presse-papiers impossible.",
"common.copy": "Copier",
"products.edit.noCategoryFromFeed": "Aucune categorie dans le flux",
"products.edit.attrsChangedCount": "{count} modifies",
"products.edit.attrKey": "Attribut",
"products.edit.noAttributes": "Ce produit n a pas d attributs.",
"admin.nav.aiCosts": "Couts IA",
"admin.aiCosts.title": "Couts IA",
"admin.aiCosts.description": "Ce que coute chaque locataire et utilisateur. Conserve {days} jours.",
"admin.aiCosts.loadFailed": "Impossible de charger les couts IA.",
"admin.aiCosts.totalTitle": "Depense totale",
"admin.aiCosts.rangeLabel": "Du {from} au {to}",
"admin.aiCosts.range": "Periode",
"admin.aiCosts.lastNDays": "{count} derniers jours",
"admin.aiCosts.grandTotal": "Total general",
"admin.aiCosts.calls": "Appels",
"admin.aiCosts.failedCalls": "{count} en echec",
"admin.aiCosts.tokens": "Jetons",
"admin.aiCosts.cachedTokens": "{count} en cache",
"admin.aiCosts.tenants": "Locataires",
"admin.aiCosts.usersCount": "{count} utilisateurs",
"admin.aiCosts.byUser": "Cout par utilisateur",
"admin.aiCosts.byCompany": "Cout par entreprise",
"admin.aiCosts.byModel": "Cout par modele",
"admin.aiCosts.emptyTitle": "Aucune depense enregistree",
"admin.aiCosts.emptyBody": "Rien n a ete facture sur cette periode.",
"admin.aiCosts.colName": "Nom",
"admin.aiCosts.colCompany": "Entreprise",
"admin.aiCosts.colIn": "Entree",
"admin.aiCosts.colOut": "Sortie",
"admin.aiCosts.colCost": "Cout",
"admin.aiCosts.unattributed": "Non attribue",
};
+30
View File
@@ -5751,4 +5751,34 @@ export const it: MessageDict = {
"admin.aiCalls.loadFailed": "Impossibile caricare le chiamate IA.",
"admin.aiCalls.copyFailed": "Impossibile copiare negli appunti.",
"common.copy": "Copia",
"products.edit.noCategoryFromFeed": "Nessuna categoria nel feed",
"products.edit.attrsChangedCount": "{count} modificati",
"products.edit.attrKey": "Attributo",
"products.edit.noAttributes": "Questo prodotto non ha attributi.",
"admin.nav.aiCosts": "Costi IA",
"admin.aiCosts.title": "Costi IA",
"admin.aiCosts.description": "Quanto costa ogni tenant e utente. Conservato {days} giorni.",
"admin.aiCosts.loadFailed": "Impossibile caricare i costi IA.",
"admin.aiCosts.totalTitle": "Spesa totale",
"admin.aiCosts.rangeLabel": "Dal {from} al {to}",
"admin.aiCosts.range": "Periodo",
"admin.aiCosts.lastNDays": "Ultimi {count} giorni",
"admin.aiCosts.grandTotal": "Totale generale",
"admin.aiCosts.calls": "Chiamate",
"admin.aiCosts.failedCalls": "{count} fallite",
"admin.aiCosts.tokens": "Token",
"admin.aiCosts.cachedTokens": "{count} in cache",
"admin.aiCosts.tenants": "Tenant",
"admin.aiCosts.usersCount": "{count} utenti",
"admin.aiCosts.byUser": "Costo per utente",
"admin.aiCosts.byCompany": "Costo per azienda",
"admin.aiCosts.byModel": "Costo per modello",
"admin.aiCosts.emptyTitle": "Nessuna spesa registrata",
"admin.aiCosts.emptyBody": "Nulla e stato addebitato in questo periodo.",
"admin.aiCosts.colName": "Nome",
"admin.aiCosts.colCompany": "Azienda",
"admin.aiCosts.colIn": "Input",
"admin.aiCosts.colOut": "Output",
"admin.aiCosts.colCost": "Costo",
"admin.aiCosts.unattributed": "Non attribuito",
};
+30
View File
@@ -5751,4 +5751,34 @@ export const ja: MessageDict = {
"admin.aiCalls.loadFailed": "AI 呼び出しを読み込めませんでした。",
"admin.aiCalls.copyFailed": "クリップボードにコピーできませんでした。",
"common.copy": "コピー",
"products.edit.noCategoryFromFeed": "フィードにカテゴリーがありません",
"products.edit.attrsChangedCount": "{count} 件変更",
"products.edit.attrKey": "属性",
"products.edit.noAttributes": "この商品には属性がありません。",
"admin.nav.aiCosts": "AI コスト",
"admin.aiCosts.title": "AI コスト",
"admin.aiCosts.description": "各テナントとユーザーのコスト。{days} 日間保持されます。",
"admin.aiCosts.loadFailed": "AI コストを読み込めませんでした。",
"admin.aiCosts.totalTitle": "合計支出",
"admin.aiCosts.rangeLabel": "{from} 〜 {to}",
"admin.aiCosts.range": "期間",
"admin.aiCosts.lastNDays": "過去 {count} 日",
"admin.aiCosts.grandTotal": "総計",
"admin.aiCosts.calls": "呼び出し",
"admin.aiCosts.failedCalls": "{count} 件失敗",
"admin.aiCosts.tokens": "トークン",
"admin.aiCosts.cachedTokens": "{count} 件キャッシュ",
"admin.aiCosts.tenants": "テナント",
"admin.aiCosts.usersCount": "{count} ユーザー",
"admin.aiCosts.byUser": "ユーザー別コスト",
"admin.aiCosts.byCompany": "会社別コスト",
"admin.aiCosts.byModel": "モデル別コスト",
"admin.aiCosts.emptyTitle": "支出の記録なし",
"admin.aiCosts.emptyBody": "この期間の課金はありません。",
"admin.aiCosts.colName": "名前",
"admin.aiCosts.colCompany": "会社",
"admin.aiCosts.colIn": "入力",
"admin.aiCosts.colOut": "出力",
"admin.aiCosts.colCost": "コスト",
"admin.aiCosts.unattributed": "未割り当て",
};
+30
View File
@@ -5751,4 +5751,34 @@ export const nl: MessageDict = {
"admin.aiCalls.loadFailed": "AI-aanroepen konden niet worden geladen.",
"admin.aiCalls.copyFailed": "Kopieren naar klembord mislukt.",
"common.copy": "Kopieren",
"products.edit.noCategoryFromFeed": "Geen categorie in de feed",
"products.edit.attrsChangedCount": "{count} gewijzigd",
"products.edit.attrKey": "Attribuut",
"products.edit.noAttributes": "Dit product heeft geen attributen.",
"admin.nav.aiCosts": "AI-kosten",
"admin.aiCosts.title": "AI-kosten",
"admin.aiCosts.description": "Wat elke tenant en gebruiker kost. {days} dagen bewaard.",
"admin.aiCosts.loadFailed": "AI-kosten konden niet worden geladen.",
"admin.aiCosts.totalTitle": "Totale uitgaven",
"admin.aiCosts.rangeLabel": "{from} tot {to}",
"admin.aiCosts.range": "Periode",
"admin.aiCosts.lastNDays": "Laatste {count} dagen",
"admin.aiCosts.grandTotal": "Eindtotaal",
"admin.aiCosts.calls": "Aanroepen",
"admin.aiCosts.failedCalls": "{count} mislukt",
"admin.aiCosts.tokens": "Tokens",
"admin.aiCosts.cachedTokens": "{count} uit cache",
"admin.aiCosts.tenants": "Tenants",
"admin.aiCosts.usersCount": "{count} gebruikers",
"admin.aiCosts.byUser": "Kosten per gebruiker",
"admin.aiCosts.byCompany": "Kosten per bedrijf",
"admin.aiCosts.byModel": "Kosten per model",
"admin.aiCosts.emptyTitle": "Geen uitgaven vastgelegd",
"admin.aiCosts.emptyBody": "Er is niets in rekening gebracht in deze periode.",
"admin.aiCosts.colName": "Naam",
"admin.aiCosts.colCompany": "Bedrijf",
"admin.aiCosts.colIn": "Invoer",
"admin.aiCosts.colOut": "Uitvoer",
"admin.aiCosts.colCost": "Kosten",
"admin.aiCosts.unattributed": "Niet toegewezen",
};
+30
View File
@@ -5751,4 +5751,34 @@ export const pl: MessageDict = {
"admin.aiCalls.loadFailed": "Nie udalo sie wczytac wywolan AI.",
"admin.aiCalls.copyFailed": "Nie udalo sie skopiowac do schowka.",
"common.copy": "Kopiuj",
"products.edit.noCategoryFromFeed": "Brak kategorii w zrodle",
"products.edit.attrsChangedCount": "Zmienionych: {count}",
"products.edit.attrKey": "Atrybut",
"products.edit.noAttributes": "Ten produkt nie ma atrybutow.",
"admin.nav.aiCosts": "Koszty AI",
"admin.aiCosts.title": "Koszty AI",
"admin.aiCosts.description": "Ile kosztuje kazdy najemca i uzytkownik. Przechowywane {days} dni.",
"admin.aiCosts.loadFailed": "Nie udalo sie wczytac kosztow AI.",
"admin.aiCosts.totalTitle": "Laczne wydatki",
"admin.aiCosts.rangeLabel": "Od {from} do {to}",
"admin.aiCosts.range": "Zakres",
"admin.aiCosts.lastNDays": "Ostatnie {count} dni",
"admin.aiCosts.grandTotal": "Suma calkowita",
"admin.aiCosts.calls": "Wywolania",
"admin.aiCosts.failedCalls": "Nieudanych: {count}",
"admin.aiCosts.tokens": "Tokeny",
"admin.aiCosts.cachedTokens": "W pamieci podrecznej: {count}",
"admin.aiCosts.tenants": "Najemcy",
"admin.aiCosts.usersCount": "Uzytkownikow: {count}",
"admin.aiCosts.byUser": "Koszt wedlug uzytkownika",
"admin.aiCosts.byCompany": "Koszt wedlug firmy",
"admin.aiCosts.byModel": "Koszt wedlug modelu",
"admin.aiCosts.emptyTitle": "Brak zarejestrowanych wydatkow",
"admin.aiCosts.emptyBody": "W tym okresie nic nie naliczono.",
"admin.aiCosts.colName": "Nazwa",
"admin.aiCosts.colCompany": "Firma",
"admin.aiCosts.colIn": "Wejscie",
"admin.aiCosts.colOut": "Wyjscie",
"admin.aiCosts.colCost": "Koszt",
"admin.aiCosts.unattributed": "Nieprzypisane",
};
+30
View File
@@ -5751,4 +5751,34 @@ export const pt: MessageDict = {
"admin.aiCalls.loadFailed": "Nao foi possivel carregar as chamadas de IA.",
"admin.aiCalls.copyFailed": "Nao foi possivel copiar.",
"common.copy": "Copiar",
"products.edit.noCategoryFromFeed": "Sem categoria no feed",
"products.edit.attrsChangedCount": "{count} alterados",
"products.edit.attrKey": "Atributo",
"products.edit.noAttributes": "Este produto nao tem atributos.",
"admin.nav.aiCosts": "Custos de IA",
"admin.aiCosts.title": "Custos de IA",
"admin.aiCosts.description": "Quanto custa cada inquilino e utilizador. Guardado {days} dias.",
"admin.aiCosts.loadFailed": "Nao foi possivel carregar os custos de IA.",
"admin.aiCosts.totalTitle": "Despesa total",
"admin.aiCosts.rangeLabel": "{from} a {to}",
"admin.aiCosts.range": "Periodo",
"admin.aiCosts.lastNDays": "Ultimos {count} dias",
"admin.aiCosts.grandTotal": "Total geral",
"admin.aiCosts.calls": "Chamadas",
"admin.aiCosts.failedCalls": "{count} falhadas",
"admin.aiCosts.tokens": "Tokens",
"admin.aiCosts.cachedTokens": "{count} em cache",
"admin.aiCosts.tenants": "Inquilinos",
"admin.aiCosts.usersCount": "{count} utilizadores",
"admin.aiCosts.byUser": "Custo por utilizador",
"admin.aiCosts.byCompany": "Custo por empresa",
"admin.aiCosts.byModel": "Custo por modelo",
"admin.aiCosts.emptyTitle": "Sem despesa registada",
"admin.aiCosts.emptyBody": "Nada foi cobrado neste periodo.",
"admin.aiCosts.colName": "Nome",
"admin.aiCosts.colCompany": "Empresa",
"admin.aiCosts.colIn": "Entrada",
"admin.aiCosts.colOut": "Saida",
"admin.aiCosts.colCost": "Custo",
"admin.aiCosts.unattributed": "Nao atribuido",
};
+30
View File
@@ -62,4 +62,34 @@ export const sl: MessageDict = {
"admin.aiCalls.loadFailed": "Klicev AI ni bilo mogoce naloziti.",
"admin.aiCalls.copyFailed": "Kopiranje v odlozisce ni uspelo.",
"common.copy": "Kopiraj",
"products.edit.noCategoryFromFeed": "V viru ni kategorije",
"products.edit.attrsChangedCount": "Spremenjenih: {count}",
"products.edit.attrKey": "Atribut",
"products.edit.noAttributes": "Ta izdelek nima atributov.",
"admin.nav.aiCosts": "Stroski AI",
"admin.aiCosts.title": "Stroski AI",
"admin.aiCosts.description": "Koliko stane vsak najemnik in uporabnik. Hranjeno {days} dni.",
"admin.aiCosts.loadFailed": "Stroskov AI ni bilo mogoce naloziti.",
"admin.aiCosts.totalTitle": "Skupna poraba",
"admin.aiCosts.rangeLabel": "Od {from} do {to}",
"admin.aiCosts.range": "Obdobje",
"admin.aiCosts.lastNDays": "Zadnjih {count} dni",
"admin.aiCosts.grandTotal": "Skupaj",
"admin.aiCosts.calls": "Klici",
"admin.aiCosts.failedCalls": "Neuspesnih: {count}",
"admin.aiCosts.tokens": "Zetoni",
"admin.aiCosts.cachedTokens": "V predpomnilniku: {count}",
"admin.aiCosts.tenants": "Najemniki",
"admin.aiCosts.usersCount": "Uporabnikov: {count}",
"admin.aiCosts.byUser": "Stroski po uporabniku",
"admin.aiCosts.byCompany": "Stroski po podjetju",
"admin.aiCosts.byModel": "Stroski po modelu",
"admin.aiCosts.emptyTitle": "Ni zabelezene porabe",
"admin.aiCosts.emptyBody": "V tem obdobju ni bilo obracunano nic.",
"admin.aiCosts.colName": "Ime",
"admin.aiCosts.colCompany": "Podjetje",
"admin.aiCosts.colIn": "Vhod",
"admin.aiCosts.colOut": "Izhod",
"admin.aiCosts.colCost": "Strosek",
"admin.aiCosts.unattributed": "Nedodeljeno",
};
@@ -168,8 +168,9 @@
return;
}
try {
const res = await api<{ items?: Company[] }>("/api/admin/companies?limit=200");
companies = res.items ?? [];
// /api/admin/companies returns { companies: [...] }, not { items }.
const res = await api<{ companies?: Company[] }>("/api/admin/companies?limit=200");
companies = res.companies ?? [];
} catch {
// Company list is a convenience filter; the page still works without it.
}
@@ -0,0 +1,246 @@
<script lang="ts">
import { i18n } from "$lib/i18n";
import { onMount } from "svelte";
import { api, failureMessage } from "$lib/api";
import { requirePlatformAdmin } from "$lib/admin-gate";
import PageShell from "$lib/components/PageShell.svelte";
import Alert from "$lib/components/Alert.svelte";
import ForbiddenEmptyState from "$lib/components/ForbiddenEmptyState.svelte";
import Spinner from "$lib/components/Spinner.svelte";
import EmptyState from "$lib/components/EmptyState.svelte";
import {
Button,
Card,
CardContent,
CardDescription,
CardHeader,
CardTitle,
TableBody,
TableCell,
TableHead,
TableHeader,
TableRow,
TableShell
} from "$lib/components/ui";
import { RefreshCw } from "@lucide/svelte";
type CostRow = {
id?: string;
label?: string;
company_id?: string;
company_name?: string;
calls?: number;
failed_calls?: number;
prompt_tokens?: number;
cached_prompt_tokens?: number;
completion_tokens?: number;
total_tokens?: number;
cost_micros?: number;
cost_usd?: number;
companies?: number;
users?: number;
};
type CostReport = {
from?: string;
to?: string;
currency?: string;
totals?: CostRow;
by_company?: CostRow[];
by_user?: CostRow[];
by_model?: CostRow[];
usage_retention_days?: number;
};
let loading = $state(true);
let accessDenied = $state(false);
let refreshing = $state(false);
let error = $state("");
let days = $state("30");
let report = $state<CostReport | null>(null);
const RANGES = ["7", "30", "90", "365", "1095"];
const totals = $derived(report?.totals ?? {});
const retentionDays = $derived(report?.usage_retention_days ?? 1096);
/**
* Cost is stored as integer micro-USD; cost_usd is a display convenience.
* Format from micros so the page never re-sums floats.
*/
function usd(row: CostRow | undefined): string {
const micros = Number(row?.cost_micros ?? 0);
const value = micros / 1_000_000;
// Sub-cent spend is normal per user per day — show enough precision to see it.
const digits = value !== 0 && Math.abs(value) < 0.01 ? 4 : 2;
return new Intl.NumberFormat(undefined, {
style: "currency",
currency: report?.currency || "USD",
minimumFractionDigits: digits,
maximumFractionDigits: digits
}).format(value);
}
function num(n: number | undefined): string {
return new Intl.NumberFormat().format(Number(n ?? 0));
}
async function load() {
error = "";
refreshing = true;
try {
report = await api<CostReport>(`/api/admin/ai-costs?days=${encodeURIComponent(days)}`);
} catch (err) {
error = failureMessage(err, i18n.t("admin.aiCosts.loadFailed"));
} finally {
refreshing = false;
loading = false;
}
}
onMount(async () => {
if (!(await requirePlatformAdmin())) {
accessDenied = true;
loading = false;
return;
}
await load();
});
</script>
<PageShell
title={i18n.t("admin.aiCosts.title")}
description={i18n.t("admin.aiCosts.description", { days: retentionDays })}
>
{#if accessDenied}
<ForbiddenEmptyState />
{:else if loading}
<Spinner />
{:else}
{#if error}
<Alert tone="error" message={error} />
{/if}
<Card>
<CardHeader>
<CardTitle>{i18n.t("admin.aiCosts.totalTitle")}</CardTitle>
<CardDescription>
{i18n.t("admin.aiCosts.rangeLabel", { from: report?.from ?? "—", to: report?.to ?? "—" })}
</CardDescription>
</CardHeader>
<CardContent class="space-y-4">
<div class="flex flex-wrap items-center gap-2">
<label class="text-sm">
<span class="mr-2 text-muted-foreground">{i18n.t("admin.aiCosts.range")}</span>
<select
class="rounded-md border bg-background p-2 text-sm"
bind:value={days}
onchange={() => load()}
>
{#each RANGES as r (r)}
<option value={r}>{i18n.t("admin.aiCosts.lastNDays", { count: r })}</option>
{/each}
</select>
</label>
<Button onclick={() => load()} disabled={refreshing}>
<RefreshCw class="mr-2 size-4" />
{i18n.t("common.refresh")}
</Button>
</div>
<div class="grid gap-3 sm:grid-cols-2 lg:grid-cols-4">
<div class="rounded-lg border bg-muted/30 p-4">
<p class="text-xs uppercase tracking-wide text-muted-foreground">
{i18n.t("admin.aiCosts.grandTotal")}
</p>
<p class="mt-1 text-2xl font-semibold tabular-nums">{usd(totals)}</p>
</div>
<div class="rounded-lg border bg-muted/30 p-4">
<p class="text-xs uppercase tracking-wide text-muted-foreground">
{i18n.t("admin.aiCosts.calls")}
</p>
<p class="mt-1 text-2xl font-semibold tabular-nums">{num(totals.calls)}</p>
{#if Number(totals.failed_calls ?? 0) > 0}
<p class="text-xs text-destructive">
{i18n.t("admin.aiCosts.failedCalls", { count: num(totals.failed_calls) })}
</p>
{/if}
</div>
<div class="rounded-lg border bg-muted/30 p-4">
<p class="text-xs uppercase tracking-wide text-muted-foreground">
{i18n.t("admin.aiCosts.tokens")}
</p>
<p class="mt-1 text-2xl font-semibold tabular-nums">{num(totals.total_tokens)}</p>
<p class="text-xs text-muted-foreground">
{i18n.t("admin.aiCosts.cachedTokens", { count: num(totals.cached_prompt_tokens) })}
</p>
</div>
<div class="rounded-lg border bg-muted/30 p-4">
<p class="text-xs uppercase tracking-wide text-muted-foreground">
{i18n.t("admin.aiCosts.tenants")}
</p>
<p class="mt-1 text-2xl font-semibold tabular-nums">{num(totals.companies)}</p>
<p class="text-xs text-muted-foreground">
{i18n.t("admin.aiCosts.usersCount", { count: num(totals.users) })}
</p>
</div>
</div>
</CardContent>
</Card>
{#each [{ key: "by_user", titleKey: "admin.aiCosts.byUser", rows: report?.by_user ?? [], showCompany: true }, { key: "by_company", titleKey: "admin.aiCosts.byCompany", rows: report?.by_company ?? [], showCompany: false }, { key: "by_model", titleKey: "admin.aiCosts.byModel", rows: report?.by_model ?? [], showCompany: false }] as section (section.key)}
<Card class="mt-4">
<CardHeader>
<CardTitle>{i18n.t(section.titleKey)}</CardTitle>
</CardHeader>
<CardContent>
{#if section.rows.length === 0}
<EmptyState
title={i18n.t("admin.aiCosts.emptyTitle")}
message={i18n.t("admin.aiCosts.emptyBody")}
/>
{:else}
<TableShell>
<TableHeader>
<TableRow>
<TableHead>{i18n.t("admin.aiCosts.colName")}</TableHead>
{#if section.showCompany}
<TableHead>{i18n.t("admin.aiCosts.colCompany")}</TableHead>
{/if}
<TableHead class="text-right">{i18n.t("admin.aiCosts.calls")}</TableHead>
<TableHead class="text-right">{i18n.t("admin.aiCosts.colIn")}</TableHead>
<TableHead class="text-right">{i18n.t("admin.aiCosts.colOut")}</TableHead>
<TableHead class="text-right">{i18n.t("admin.aiCosts.colCost")}</TableHead>
</TableRow>
</TableHeader>
<TableBody>
{#each section.rows as row (row.id)}
<TableRow>
<TableCell class="text-xs">
{row.label || i18n.t("admin.aiCosts.unattributed")}
</TableCell>
{#if section.showCompany}
<TableCell class="text-xs text-muted-foreground">
{row.company_name || "—"}
</TableCell>
{/if}
<TableCell class="text-right text-xs tabular-nums">{num(row.calls)}</TableCell>
<TableCell class="text-right text-xs tabular-nums">
{num(row.prompt_tokens)}
</TableCell>
<TableCell class="text-right text-xs tabular-nums">
{num(row.completion_tokens)}
</TableCell>
<TableCell class="text-right text-xs font-medium tabular-nums">
{usd(row)}
</TableCell>
</TableRow>
{/each}
</TableBody>
</TableShell>
{/if}
</CardContent>
</Card>
{/each}
{/if}
</PageShell>