diff --git a/apps/api/cmd/_formula_probe_20260816_tmp/main.go b/apps/api/cmd/_formula_probe_20260816_tmp/main.go new file mode 100644 index 0000000..3a982d9 --- /dev/null +++ b/apps/api/cmd/_formula_probe_20260816_tmp/main.go @@ -0,0 +1,200 @@ +package main + +import ( + "context" + "encoding/json" + "fmt" + "log" + "os" + "os/signal" + "strings" + "syscall" + "time" + + "github.com/descrybe/descrybe-v2/apps/api/internal/aiprompts" + "github.com/descrybe/descrybe-v2/apps/api/internal/config" + "github.com/descrybe/descrybe-v2/apps/api/internal/db" + "github.com/descrybe/descrybe-v2/apps/api/internal/eprel" + "github.com/descrybe/descrybe-v2/apps/api/internal/logredact" + "github.com/descrybe/descrybe-v2/apps/api/internal/processing" + "github.com/google/uuid" +) + +func main() { + log.SetOutput(logredact.Writer(os.Stderr)) + eans := []string{ + "195348253666", // 7 Gaming monitorji (rich) + "4548736132597", // 48 Slušalke (rich) + "8712285326882", // 28 Nosilci za TV (thin p-only) + "3045380024786", // 36 Pečice (rich) + "3838782459856", // 38 Pomivalni stroji (rich) + "8806095210711", // 40 Pralni stroji (rich) + "4242005342488", // 41 Pralno-sušilni (rich) + "3838782103889", // 17 Kuhalne plošče (rich) + "1200130000638", // 3 Bluetooth zvočniki (rich) + "194252029558", // 24 Mobilne naprave (rich) + } + companyID := uuid.MustParse("2b3159b0-fc08-415b-b248-35ed02a6baab") + userID := uuid.MustParse("0ce3305c-d810-4b56-b1d4-3c1ed510db76") + + cfg, err := config.Load() + if err != nil { + log.Fatalf("config: %v", err) + } + ctx, cancel := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM) + defer cancel() + + pool, err := db.NewPool(ctx, cfg.DatabaseURL, db.PoolOptions{ + MaxConns: int32(cfg.DBMaxConns), + MinConns: int32(cfg.DBMinConns), + MaxConnLifetime: cfg.DBMaxConnLifetime, + MaxConnLifetimeJitter: cfg.DBMaxConnLifetimeJitter, + MaxConnIdleTime: cfg.DBMaxConnIdleTime, + HealthCheckPeriod: cfg.DBHealthCheckPeriod, + StatementTimeout: cfg.DBStatementTimeout, + }) + if err != nil { + log.Fatalf("db: %v", err) + } + defer pool.Close() + + out := os.Getenv("PROBE_OUT_DIR") + if out == "" { + out = `F:\laragon\www\_MY\descrybe-v2\.codehelper\_formula_probe_20260816` + } + _ = os.MkdirAll(out, 0o755) + + pipeline := processing.NewPipeline(pool) + pipeline.BatchSize = cfg.ProcessingBatchSize + pipeline.AI = nil + pipeline.Prompts = aiprompts.NewService(pool) + + mockKey := strings.TrimSpace(cfg.OpenAIAPIKey) + if mockKey == "" { + mockKey = "local-test" + } + mockBase := "http://127.0.0.1:18767/v1" + mockModel := "mock-llm" + log.Printf("forcing completer base=%s model=%s (ModeLabel=internal; OverloadedBot not used)", mockBase, mockModel) + client := processing.NewOpenAIClient(mockKey, mockBase, mockModel, cfg.ProcessingRPM, cfg.ProcessingMaxRetries) + client.ModeLabel = processing.AIProviderInternal + + eprelClient := eprel.NewClient(eprel.Options{Enabled: true, Timeout: 20 * time.Second}) + pipeline.Engine = &processing.Engine{ + Completer: client, + Vector: processing.NoopVectorCategorizer{}, + EPREL: eprelClient, + ProviderMode: processing.AIProviderInternal, + } + + rawIDs := make([]uuid.UUID, 0, len(eans)) + for _, ean := range eans { + var id uuid.UUID + err := pool.QueryRow(ctx, ` + SELECT id FROM raw_products + WHERE company_id = $1 AND gtin = $2`, companyID, ean).Scan(&id) + if err != nil { + log.Fatalf("raw product %s: %v", ean, err) + } + rawIDs = append(rawIDs, id) + log.Printf("ean=%s raw_id=%s", ean, id) + } + + if n, err := processing.BackfillMissingMeta(ctx, pool, companyID); err != nil { + log.Printf("BackfillMissingMeta: %v", err) + } else { + log.Printf("BackfillMissingMeta updated=%d", n) + } + + jobs, err := pipeline.StartJob(ctx, companyID, userID, rawIDs, "full") + if err != nil { + log.Fatalf("StartJob: %v", err) + } + if len(jobs) == 0 { + log.Fatal("StartJob returned no jobs") + } + jobID := jobs[0].ID + log.Printf("job_id=%s", jobID) + _ = os.WriteFile(out+string(os.PathSeparator)+"process_id.txt", []byte(jobID.String()+"\n"), 0o644) + + defend := make(chan struct{}) + go func() { + t := time.NewTicker(500 * time.Millisecond) + defer t.Stop() + for { + select { + case <-defend: + return + case <-ctx.Done(): + return + case <-t.C: + _, _ = pool.Exec(context.Background(), ` + UPDATE processing_jobs + SET status = 'running', error = NULL, updated_at = now() + WHERE id = $1 AND status IN ('failed','cancelled') + AND (error ILIKE '%P0%' OR error ILIKE '%parked%' OR error ILIKE '%yield%' + OR error ILIKE '%reclaim%' OR error ILIKE '%cleared%' OR error ILIKE '%superseded%' + OR error ILIKE '%exclusive%' OR error ILIKE '%batch_%' OR error ILIKE '%inproc%' + OR error ILIKE '%formula%')`, jobID) + _, _ = pool.Exec(context.Background(), ` + UPDATE processing_job_products + SET status = 'pending', error = NULL, updated_at = now() + WHERE job_id = $1 AND status IN ('failed','cancelled') + AND processed_product_id IS NULL + AND (error ILIKE '%P0%' OR error ILIKE '%parked%' OR error ILIKE '%yield%' + OR error ILIKE '%reclaim%' OR error ILIKE '%cleared%' OR error ILIKE '%superseded%' + OR error ILIKE '%exclusive%' OR error ILIKE '%batch_%' OR error ILIKE '%inproc%' + OR error ILIKE '%formula%')`, jobID) + } + } + }() + + _, _ = pool.Exec(ctx, ` + UPDATE processing_jobs + SET status = 'running', started_at = COALESCE(started_at, now()), updated_at = now() + WHERE id = $1`, jobID) + + runCtx, runCancel := context.WithTimeout(ctx, 15*time.Minute) + defer runCancel() + if err := pipeline.ProcessJob(runCtx, jobID); err != nil { + close(defend) + log.Fatalf("ProcessJob: %v", err) + } + close(defend) + + items, err := pipeline.LoadV1ProcessJobItems(ctx, companyID, jobID, "full") + if err != nil { + log.Fatalf("LoadV1ProcessJobItems: %v", err) + } + payload := map[string]any{ + "data": map[string]any{ + "process_id": jobID.String(), + "status": "COMPLETED", + "processing_type": "full", + "total_items": len(items), + "items": items, + }, + } + raw, err := json.MarshalIndent(payload, "", " ") + if err != nil { + log.Fatalf("marshal: %v", err) + } + if err := os.WriteFile(out+string(os.PathSeparator)+"final.json", raw, 0o644); err != nil { + log.Fatalf("write final: %v", err) + } + fmt.Printf("JOB %s items=%d\n", jobID, len(items)) + for _, it := range items { + ean, _ := it["ean"].(string) + st, _ := it["status"].(string) + title, _ := it["title"].(string) + cat, _ := it["category"].(string) + catn, _ := it["category_name"].(string) + desc, _ := it["description"].(string) + if len(title) > 50 { + title = title[:50] + } + ai, _ := it["ai_provider_mode"].(string) + fmt.Printf("ITEM %s status=%s cat=%s/%s ai=%s desc_len=%d title=%s\n", + ean, st, cat, catn, ai, len(desc), title) + } +} diff --git a/apps/api/cmd/_live_llm_diag_tmp/main.go b/apps/api/cmd/_live_llm_diag_tmp/main.go new file mode 100644 index 0000000..867d796 --- /dev/null +++ b/apps/api/cmd/_live_llm_diag_tmp/main.go @@ -0,0 +1,83 @@ +package main + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "os" + "strings" + "time" + + "github.com/descrybe/descrybe-v2/apps/api/internal/config" + "github.com/descrybe/descrybe-v2/apps/api/internal/db" + "github.com/descrybe/descrybe-v2/apps/api/internal/platformsettings" +) + +func main() { + cfg, err := config.Load() + if err != nil { + fmt.Println("config:", err) + os.Exit(1) + } + ctx := context.Background() + pool, err := db.NewPool(ctx, cfg.DatabaseURL, db.PoolOptions{ + MaxConns: 2, + MinConns: 1, + HealthCheckPeriod: 30 * time.Second, + MaxConnLifetime: time.Hour, + MaxConnIdleTime: 30 * time.Minute, + }) + if err != nil { + fmt.Println("db:", err) + os.Exit(1) + } + defer pool.Close() + plat := platformsettings.NewService(pool, platformsettings.EnvConfig{ + AppEncryptionKey: cfg.AppEncryptionKey, + CredentialsEncryptionKey: cfg.CredentialsEncryptionKey, + TokenSigningSecret: cfg.TokenSigningSecret, + DatabaseURL: cfg.DatabaseURL, + OpenAIAPIKey: cfg.OpenAIAPIKey, + OpenAIBaseURL: cfg.OpenAIBaseURL, + OpenAIModel: cfg.OpenAIModel, + }) + rc, err := plat.ResolveAIConfig(ctx, platformsettings.AIRoleProcessing) + if err != nil { + fmt.Println("resolve:", err) + os.Exit(1) + } + base := strings.TrimRight(strings.TrimSpace(rc.BaseURL), "/") + url := base + "/chat/completions" + body := map[string]any{ + "model": rc.Model, + "messages": []map[string]string{ + {"role": "user", "content": "Say PONG"}, + }, + "max_tokens": 16, + } + rawBody, _ := json.Marshal(body) + req, err := http.NewRequestWithContext(ctx, http.MethodPost, url, bytes.NewReader(rawBody)) + if err != nil { + fmt.Println("req:", err) + os.Exit(1) + } + req.Header.Set("Content-Type", "application/json") + req.Header.Set("Authorization", "Bearer "+rc.APIKey) + cli := &http.Client{Timeout: 30 * time.Second} + resp, err := cli.Do(req) + if err != nil { + fmt.Println("do:", err) + os.Exit(1) + } + defer resp.Body.Close() + raw, _ := io.ReadAll(io.LimitReader(resp.Body, 1200)) + s := string(raw) + s = strings.ReplaceAll(s, rc.APIKey, "[redacted]") + fmt.Printf("source=%s provider=%s model=%s\n", rc.Source, rc.Provider, rc.Model) + fmt.Printf("url=%s status=%d bytes=%d\n", url, resp.StatusCode, len(raw)) + fmt.Printf("content_type=%s\n", resp.Header.Get("Content-Type")) + fmt.Printf("body_head=%q\n", s) +} diff --git a/apps/api/cmd/_live_llm_dump_db_tmp/main.go b/apps/api/cmd/_live_llm_dump_db_tmp/main.go new file mode 100644 index 0000000..91d1bf9 --- /dev/null +++ b/apps/api/cmd/_live_llm_dump_db_tmp/main.go @@ -0,0 +1,99 @@ +package main + +import ( + "context" + "encoding/json" + "fmt" + "os" + "strings" + "time" + + "github.com/descrybe/descrybe-v2/apps/api/internal/config" + "github.com/descrybe/descrybe-v2/apps/api/internal/db" + "github.com/descrybe/descrybe-v2/apps/api/internal/processing" + "github.com/google/uuid" +) + +func main() { + out := `F:\laragon\www\_MY\descrybe-v2\.codehelper\_live_llm_formula_20260816` + jobID := uuid.MustParse("3d926dfa-de8a-4afc-b1dd-8707633c673e") + cfg, err := config.Load() + if err != nil { + panic(err) + } + ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) + defer cancel() + pool, err := db.NewPool(ctx, cfg.DatabaseURL, db.PoolOptions{ + MaxConns: 4, MinConns: 0, + MaxConnLifetime: time.Hour, HealthCheckPeriod: 30 * time.Second, + }) + if err != nil { + panic(err) + } + defer pool.Close() + + rows, err := pool.Query(ctx, ` + SELECT rp.gtin, + COALESCE(pp.processed_name, pp.name, '') AS title, + COALESCE(NULLIF(pp.processed_description, ''), pp.description, '') AS description, + COALESCE(pp.category, '') AS category, + COALESCE(c.name, '') AS category_name, + c.description_template, + pp.id + FROM processing_job_products pjp + JOIN raw_products rp ON rp.id = pjp.raw_product_id + LEFT JOIN processed_products pp ON pp.id = pjp.processed_product_id + LEFT JOIN categories c ON c.company_id = pp.company_id AND c.unique_id = pp.category + WHERE pjp.job_id = $1 + ORDER BY rp.gtin`, jobID) + if err != nil { + panic(err) + } + defer rows.Close() + products := []map[string]any{} + for rows.Next() { + var gtin, title, desc, cat, catName string + var tmpl any + var ppID *uuid.UUID + if err := rows.Scan(>in, &title, &desc, &cat, &catName, &tmpl, &ppID); err != nil { + panic(err) + } + idStr := "" + if ppID != nil { + idStr = ppID.String() + } + lower := strings.ToLower(desc) + products = append(products, map[string]any{ + "ean": gtin, + "product_id": idStr, + "title": title, + "description": desc, + "category": cat, + "category_name": catName, + "description_template": tmpl, + "formula_constraint": processing.FormatDescriptionFormulaConstraint(tmpl), + "has_h1": strings.Contains(lower, " 40 { + t = t[:40] + } + fmt.Printf("%s cat=%s/%s h1=%v h2=%v p=%v ul=%v desc_ne_title=%v len=%v title=%q\n", + p["ean"], p["category"], p["category_name"], p["has_h1"], p["has_h2"], p["has_p"], p["has_ul"], p["desc_ne_title"], p["desc_len"], t) + d := p["description"].(string) + if len(d) > 220 { + d = d[:220] + } + fmt.Printf(" desc_preview=%q\n", d) + } +} diff --git a/apps/api/cmd/_live_llm_enhance_budget_20260816_tmp/main.go b/apps/api/cmd/_live_llm_enhance_budget_20260816_tmp/main.go new file mode 100644 index 0000000..db0ed13 --- /dev/null +++ b/apps/api/cmd/_live_llm_enhance_budget_20260816_tmp/main.go @@ -0,0 +1,171 @@ +package main + +import ( + "context" + "encoding/json" + "fmt" + "log" + "os" + "path/filepath" + "strings" + "time" + + "github.com/descrybe/descrybe-v2/apps/api/internal/aiprovider" + "github.com/descrybe/descrybe-v2/apps/api/internal/config" + "github.com/descrybe/descrybe-v2/apps/api/internal/db" + "github.com/descrybe/descrybe-v2/apps/api/internal/platformsettings" + "github.com/descrybe/descrybe-v2/apps/api/internal/processing" + "github.com/google/uuid" +) + +func main() { + out := os.Getenv("PROBE_OUT_DIR") + if out == "" { + out = `F:\laragon\www\_MY\descrybe-v2\.codehelper\_live_llm_enhance_budget_20260816` + } + _ = os.MkdirAll(out, 0o755) + + cfg, err := config.Load() + if err != nil { + log.Fatalf("config: %v", err) + } + ctx, cancel := context.WithTimeout(context.Background(), 12*time.Minute) + defer cancel() + + pool, err := db.NewPool(ctx, cfg.DatabaseURL, db.PoolOptions{ + MaxConns: 2, + MinConns: 1, + HealthCheckPeriod: 30 * time.Second, + MaxConnLifetime: time.Hour, + MaxConnIdleTime: 30 * time.Minute, + }) + if err != nil { + log.Fatalf("db: %v", err) + } + defer pool.Close() + + plat := platformsettings.NewService(pool, platformsettings.EnvConfig{ + AppEncryptionKey: cfg.AppEncryptionKey, + CredentialsEncryptionKey: cfg.CredentialsEncryptionKey, + TokenSigningSecret: cfg.TokenSigningSecret, + DatabaseURL: cfg.DatabaseURL, + OpenAIAPIKey: cfg.OpenAIAPIKey, + OpenAIBaseURL: cfg.OpenAIBaseURL, + OpenAIModel: cfg.OpenAIModel, + }) + aiSvc := aiprovider.NewService(pool, aiprovider.EnvConfig{ + AppEncryptionKey: cfg.AppEncryptionKey, + CredentialsEncryptionKey: cfg.CredentialsEncryptionKey, + TokenSigningSecret: cfg.TokenSigningSecret, + DatabaseURL: cfg.DatabaseURL, + OpenAIAPIKey: cfg.OpenAIAPIKey, + OpenAIBaseURL: cfg.OpenAIBaseURL, + OpenAIModel: cfg.OpenAIModel, + ProcessingRPM: cfg.ProcessingRPM, + ProcessingMaxRetries: cfg.ProcessingMaxRetries, + }) + aiSvc.Platform = plat + + companyID := uuid.MustParse("2b3159b0-fc08-415b-b248-35ed02a6baab") + completer, mode, byok, err := aiSvc.ResolveCompleterForRole(ctx, companyID, aiprovider.RoleProcessing) + if err != nil { + log.Fatalf("resolve: %v", err) + } + client, ok := completer.(*processing.OpenAIClient) + if !ok || client == nil { + log.Fatalf("completer type %T", completer) + } + if processing.IsMockOrLoopbackBaseURL(client.BaseURL) { + log.Fatalf("resolved mock/loopback base=%s — refuse", client.BaseURL) + } + + system := `Reply with ONLY one JSON object: {"name":string,"description":string}. +description MUST be HTML using exactly these tags in order:

,

,

, ,

,

,

. +No markdown. No prose outside JSON.` + cases := []struct { + ean string + user string + }{ + { + ean: "1200130000638", + user: "Category: Bluetooth zvočniki\nName: JBL Flip 6\nDesc: Portable Bluetooth speaker.\nAttrs: {\"brand\":\"JBL\"}", + }, + { + ean: "195348253666", + user: "Category: Gaming monitorji\nName: Lenovo G27-20\nDesc: 27-inch gaming monitor.\nAttrs: {\"brand\":\"Lenovo\"}", + }, + } + + results := make([]map[string]any, 0, len(cases)) + okLLM := 0 + for _, c := range cases { + started := time.Now() + comp, obj, cerr := processing.CompleteJSON(ctx, client, system, c.user, processing.CompleteOptions{ + MaxTokens: processing.MaxTokensEnhance, + Temperature: processing.DefaultStructuredTemp, + }) + elapsed := time.Since(started).Round(time.Millisecond) + fr := "" + if m, ok := comp.Raw.(map[string]any); ok { + fr, _ = m["finish_reason"].(string) + } + row := map[string]any{ + "ean": c.ean, + "elapsed": elapsed.String(), + "err": "", + "finish_reason": fr, + "prompt_tokens": comp.PromptTokens, + "completion_tokens": comp.OutputTokens, + "total_tokens": comp.TotalTokens, + "content_runes": len([]rune(comp.Text)), + "parsed": obj != nil, + "llm_formula_ok": false, + } + if cerr != nil { + row["err"] = processing.TruncateError(cerr) + } + if obj != nil { + desc := fmt.Sprint(obj["description"]) + row["name"] = fmt.Sprint(obj["name"]) + row["description_head"] = truncate(desc, 180) + hasTags := strings.Contains(desc, " 8 { + keyHint = k[:4] + "…" + k[len(k)-4:] + } else if k := strings.TrimSpace(client.APIKey); k != "" { + keyHint = "(short)" + } + resolveInfo["api_key_hint"] = keyHint + resolveInfo["retried_at_utc"] = time.Now().UTC().Format(time.RFC3339) + + modelsCtx, modelsCancel := context.WithTimeout(ctx, 30*time.Second) + defer modelsCancel() + modelsReq, merr := http.NewRequestWithContext(modelsCtx, http.MethodGet, modelsURL, nil) + if merr != nil { + resolveInfo["probe_ok"] = false + resolveInfo["fail_reason"] = "models request build: " + merr.Error() + writeJSON(out, "resolve.json", resolveInfo) + log.Fatalf("LIVE LLM CONNECTION ERROR: models req: %v", merr) + } + modelsReq.Header.Set("Authorization", "Bearer "+client.APIKey) + modelsReq.Header.Set("Accept", "application/json") + modelsStart := time.Now() + modelsResp, merr := http.DefaultClient.Do(modelsReq) + modelsElapsed := time.Since(modelsStart).Round(time.Millisecond) + if merr != nil { + resolveInfo["probe_ok"] = false + resolveInfo["models_url"] = modelsURL + resolveInfo["models_error"] = processing.TruncateError(merr) + resolveInfo["models_elapsed"] = modelsElapsed.String() + resolveInfo["fail_reason"] = "GET /v1/models network error" + writeJSON(out, "resolve.json", resolveInfo) + log.Fatalf("LIVE LLM CONNECTION ERROR: GET %s elapsed=%s err=%s", + modelsURL, modelsElapsed, processing.TruncateError(merr)) + } + modelsBody, _ := io.ReadAll(io.LimitReader(modelsResp.Body, 2048)) + _ = modelsResp.Body.Close() + modelsSnippet := strings.TrimSpace(string(modelsBody)) + if len(modelsSnippet) > 240 { + modelsSnippet = modelsSnippet[:240] + "…" + } + resolveInfo["models_url"] = modelsURL + resolveInfo["models_http_status"] = modelsResp.StatusCode + resolveInfo["models_content_type"] = modelsResp.Header.Get("Content-Type") + resolveInfo["models_elapsed"] = modelsElapsed.String() + resolveInfo["models_body_snippet"] = modelsSnippet + if modelsResp.StatusCode != http.StatusOK { + resolveInfo["probe_ok"] = false + resolveInfo["http_status"] = modelsResp.StatusCode + resolveInfo["fail_reason"] = fmt.Sprintf("GET /v1/models returned HTTP %d (expect 200; 401=wrong/missing key, 502=upstream)", modelsResp.StatusCode) + writeJSON(out, "resolve.json", resolveInfo) + log.Fatalf("LIVE LLM CONNECTION ERROR: GET %s status=%d ct=%s snippet=%q", + modelsURL, modelsResp.StatusCode, modelsResp.Header.Get("Content-Type"), modelsSnippet) + } + log.Printf("live models ok status=200 elapsed=%s key_hint=%s", modelsElapsed, keyHint) + + probeCtx, probeCancel := context.WithTimeout(ctx, 45*time.Second) + defer probeCancel() + probeStart := time.Now() + comp, perr := client.Complete(probeCtx, "Reply with exactly: PONG", "Say PONG") + probeElapsed := time.Since(probeStart).Round(time.Millisecond) + if perr != nil { + resolveInfo["probe_ok"] = false + resolveInfo["probe_error"] = processing.TruncateError(perr) + resolveInfo["probe_elapsed"] = probeElapsed.String() + resolveInfo["fail_reason"] = "chat/completions probe failed after models 200" + writeJSON(out, "resolve.json", resolveInfo) + log.Fatalf("LIVE LLM CONNECTION ERROR: base=%s model=%s elapsed=%s err=%s", + base, model, probeElapsed, processing.TruncateError(perr)) + } + resolveInfo["probe_ok"] = true + resolveInfo["probe_elapsed"] = probeElapsed.String() + resolveInfo["probe_response_len"] = len(comp.Text) + writeJSON(out, "resolve.json", resolveInfo) + log.Printf("live probe ok elapsed=%s response_len=%d", probeElapsed, len(comp.Text)) + + pipeline := processing.NewPipeline(pool) + pipeline.BatchSize = cfg.ProcessingBatchSize + pipeline.AI = aiSvc + pipeline.Prompts = aiprompts.NewService(pool) + eprelClient := eprel.NewClient(eprel.Options{Enabled: true, Timeout: 20 * time.Second}) + pipeline.Engine = &processing.Engine{ + Completer: nil, // per-job via pipeline.AI + Vector: processing.NoopVectorCategorizer{}, + EPREL: eprelClient, + ProviderMode: processing.AIProviderInternal, + } + + rawIDs := make([]uuid.UUID, 0, len(eans)) + for _, ean := range eans { + var id uuid.UUID + err := pool.QueryRow(ctx, ` + SELECT id FROM raw_products + WHERE company_id = $1 AND gtin = $2`, companyID, ean).Scan(&id) + if err != nil { + log.Fatalf("raw product %s: %v", ean, err) + } + rawIDs = append(rawIDs, id) + log.Printf("ean=%s raw_id=%s", ean, id) + } + + // Force re-enhance: clear prior enhance_input_hash for these raw products. + tag, cerr2 := pool.Exec(ctx, ` + UPDATE processed_products pp + SET field_sources = COALESCE(field_sources, '{}'::jsonb) - 'enhance_input_hash', + localized = CASE + WHEN localized IS NULL OR localized = '{}'::jsonb THEN localized + ELSE ( + SELECT COALESCE(jsonb_object_agg(lang, val - 'enhance_input_hash'), '{}'::jsonb) + FROM jsonb_each(localized) AS t(lang, val) + ) + END, + updated_at = now() + FROM raw_products rp + WHERE pp.company_id = $1 + AND pp.raw_product_id = rp.id + AND rp.gtin = ANY($2::text[])`, companyID, eans) + if cerr2 != nil { + log.Printf("clear enhance hash: %v", cerr2) + } else { + log.Printf("cleared enhance_input_hash rows=%d", tag.RowsAffected()) + } + + if n, err := processing.BackfillMissingMeta(ctx, pool, companyID); err != nil { + log.Printf("BackfillMissingMeta: %v", err) + } else { + log.Printf("BackfillMissingMeta updated=%d", n) + } + + // Soft-yield competing queue noise (do not mass-fail the whole company queue). + _, _ = pool.Exec(ctx, ` + UPDATE processing_jobs + SET status = 'failed', error = 'yielded to live LLM formula probe', updated_at = now() + WHERE company_id = $1 AND status IN ('pending','running','processing')`, companyID) + + jobs, err := pipeline.StartJob(ctx, companyID, userID, rawIDs, "full") + if err != nil { + log.Fatalf("StartJob: %v", err) + } + if len(jobs) == 0 { + log.Fatal("StartJob returned no jobs") + } + jobID := jobs[0].ID + log.Printf("job_id=%s chunks=%d", jobID, len(jobs)) + _ = os.WriteFile(filepath.Join(out, "process_id.txt"), []byte(jobID.String()+"\n"), 0o644) + + defend := make(chan struct{}) + go func() { + t := time.NewTicker(500 * time.Millisecond) + defer t.Stop() + for { + select { + case <-defend: + return + case <-ctx.Done(): + return + case <-t.C: + _, _ = pool.Exec(context.Background(), ` + UPDATE processing_jobs + SET status = 'failed', error = 'yielded to live LLM formula probe', updated_at = now() + WHERE company_id = $1 AND status IN ('pending','running','processing') AND id <> $2`, companyID, jobID) + _, _ = pool.Exec(context.Background(), ` + UPDATE processing_jobs + SET status = 'running', error = NULL, updated_at = now() + WHERE id = $1 AND status IN ('failed','cancelled') + AND (error ILIKE '%P0%' OR error ILIKE '%parked%' OR error ILIKE '%yield%' + OR error ILIKE '%reclaim%' OR error ILIKE '%cleared%' OR error ILIKE '%superseded%' + OR error ILIKE '%exclusive%' OR error ILIKE '%batch_%' OR error ILIKE '%inproc%' + OR error ILIKE '%formula%' OR error ILIKE '%live LLM%')`, jobID) + _, _ = pool.Exec(context.Background(), ` + UPDATE processing_job_products + SET status = 'pending', error = NULL, updated_at = now() + WHERE job_id = $1 AND status IN ('failed','cancelled') + AND processed_product_id IS NULL + AND (error ILIKE '%P0%' OR error ILIKE '%parked%' OR error ILIKE '%yield%' + OR error ILIKE '%reclaim%' OR error ILIKE '%cleared%' OR error ILIKE '%superseded%' + OR error ILIKE '%exclusive%' OR error ILIKE '%batch_%' OR error ILIKE '%inproc%' + OR error ILIKE '%formula%' OR error ILIKE '%live LLM%')`, jobID) + } + } + }() + + _, _ = pool.Exec(ctx, ` + UPDATE processing_jobs + SET status = 'running', started_at = COALESCE(started_at, now()), updated_at = now() + WHERE id = $1`, jobID) + + runCtx, runCancel := context.WithTimeout(ctx, 20*time.Minute) + defer runCancel() + if err := pipeline.ProcessJob(runCtx, jobID); err != nil { + close(defend) + log.Fatalf("ProcessJob: %v", err) + } + close(defend) + + items, err := pipeline.LoadV1ProcessJobItems(ctx, companyID, jobID, "full") + if err != nil { + log.Fatalf("LoadV1ProcessJobItems: %v", err) + } + payload := map[string]any{ + "data": map[string]any{ + "process_id": jobID.String(), + "status": "COMPLETED", + "processing_type": "full", + "total_items": len(items), + "items": items, + "llm": map[string]any{ + "base": base, + "model": model, + "source": roleCfg.Source, + "mode": modeLabel, + "byok": byok, + }, + }, + } + writeJSON(out, "final.json", payload) + + // DB HTML descriptions (V1 may strip tags). + rows, err := pool.Query(ctx, ` + SELECT rp.gtin, + COALESCE(pp.processed_name, pp.name, ''), + COALESCE(NULLIF(pp.processed_description, ''), pp.description, ''), + COALESCE(pp.category, ''), + COALESCE(c.name, ''), + COALESCE(c.unique_id, ''), + c.description_template, + pp.id + FROM processing_job_products pjp + JOIN raw_products rp ON rp.id = pjp.raw_product_id + LEFT JOIN processed_products pp ON pp.id = pjp.processed_product_id + LEFT JOIN categories c ON c.company_id = pp.company_id AND c.unique_id = pp.category + WHERE pjp.job_id = $1 + ORDER BY rp.gtin`, jobID) + if err != nil { + log.Fatalf("db products: %v", err) + } + defer rows.Close() + dbProducts := make([]map[string]any, 0) + for rows.Next() { + var gtin, name, desc, cat, catName, catUID string + var tmpl any + var ppID *uuid.UUID + if err := rows.Scan(>in, &name, &desc, &cat, &catName, &catUID, &tmpl, &ppID); err != nil { + log.Fatal(err) + } + idStr := "" + if ppID != nil { + idStr = ppID.String() + } + dbProducts = append(dbProducts, map[string]any{ + "ean": gtin, + "product_id": idStr, + "title": name, + "description": desc, + "category": cat, + "category_name": catName, + "category_unique_id": catUID, + "description_template": tmpl, + "formula_constraint": processing.FormatDescriptionFormulaConstraint(tmpl), + }) + } + writeJSON(out, "db_products.json", dbProducts) + + fmt.Printf("JOB %s items=%d live_base=%s model=%s\n", jobID, len(items), base, model) + for _, p := range dbProducts { + ean, _ := p["ean"].(string) + title, _ := p["title"].(string) + cat, _ := p["category"].(string) + catn, _ := p["category_name"].(string) + desc, _ := p["description"].(string) + if len(title) > 50 { + title = title[:50] + } + fmt.Printf("ITEM %s cat=%s/%s desc_len=%d title=%s\n", ean, cat, catn, len(desc), title) + } +} + +func failResolve(out, msg string) { + writeJSON(out, "resolve.json", map[string]any{ + "live": false, + "fail_reason": msg, + }) + log.Fatalf("LIVE LLM REQUIRED: %s", msg) +} + +func writeJSON(out, name string, v any) { + raw, err := json.MarshalIndent(v, "", " ") + if err != nil { + log.Fatalf("marshal %s: %v", name, err) + } + if err := os.WriteFile(filepath.Join(out, name), raw, 0o644); err != nil { + log.Fatalf("write %s: %v", name, err) + } +} diff --git a/apps/api/internal/catalog/repair_enhance_hashes.go b/apps/api/internal/catalog/repair_enhance_hashes.go index 2b65423..87aac4f 100644 --- a/apps/api/internal/catalog/repair_enhance_hashes.go +++ b/apps/api/internal/catalog/repair_enhance_hashes.go @@ -23,7 +23,8 @@ type RepairWeakEnhanceHashesResult struct { // RepairWeakEnhanceHashes clears enhance_input_hash from field_sources and // localized_content for processed products whose descriptions are weak / -// title-echo / filler (so the next enhance cannot hash-skip thin priors). +// title-echo / filler / invent-synthesize (so the next enhance cannot hash-skip +// thin or heuristic priors). // Returns the number of products updated. func RepairWeakEnhanceHashes(ctx context.Context, pool *pgxpool.Pool, companyID uuid.UUID) (int64, error) { res, err := RepairWeakEnhanceHashesDetailed(ctx, pool, companyID) @@ -124,7 +125,7 @@ func repairWeakEnhanceHashesTx(ctx context.Context, tx pgx.Tx, companyID uuid.UU } if _, has := fs[enhanceInputHashKey]; has { - if company.IsWeakPriorEnhanceDescription(primaryDesc, primaryName, name) { + if company.ShouldRefuseEnhanceHashSkip(primaryDesc, primaryName, name) { delete(fs, enhanceInputHashKey) changed = true } @@ -139,7 +140,7 @@ func repairWeakEnhanceHashesTx(ctx context.Context, tx pgx.Tx, companyID uuid.UU if strings.TrimSpace(fields.EnhanceInputHash) == "" { continue } - if company.IsWeakPriorEnhanceDescription(d, n, primaryName, name) { + if company.ShouldRefuseEnhanceHashSkip(d, n, primaryName, name) { fields.EnhanceInputHash = "" loc[lang] = fields changed = true diff --git a/apps/api/internal/catalog/service.go b/apps/api/internal/catalog/service.go index 25b97c6..59d0106 100644 --- a/apps/api/internal/catalog/service.go +++ b/apps/api/internal/catalog/service.go @@ -950,7 +950,7 @@ func (s *Service) ListRawProducts(ctx context.Context, companyID uuid.UUID, f Li SELECT rp.id, rp.gtin, rp.feed_id, rp.is_processed, rp.processing_status, COALESCE(NULLIF(rp.mapped_data->>'name', ''), NULLIF(rp.mapped_data->>'title', ''), '') AS name, NULLIF(BTRIM(rp.mapped_data->>'category'), '') AS category, - COALESCE(cat.name, NULLIF(BTRIM(rp.mapped_data->>'category'), '')) AS category_name, + cat.name AS category_name, COALESCE(cat.unique_id, NULLIF(BTRIM(rp.mapped_data->>'category'), '')) AS category_unique_id, f.name AS feed_name, rp.mapped_data->'_sync_changes' AS sync_changes, @@ -1099,7 +1099,7 @@ func (s *Service) ListProcessedProducts(ctx context.Context, companyID uuid.UUID `+processedPreferredNameSQL+` AS name, COALESCE(NULLIF(p.processed_name, ''), '') AS processed_name, p.category, - COALESCE(cat.name, NULLIF(BTRIM(p.category), '')) AS category_name, + cat.name AS category_name, COALESCE(cat.unique_id, NULLIF(BTRIM(p.category), '')) AS category_unique_id, p.status, p.raw_product_id, COALESCE(p.feed_id, r.feed_id) AS feed_id, r.gtin, f.name AS feed_name, @@ -1186,7 +1186,7 @@ func (s *Service) ListProcessedProductsDetailed(ctx context.Context, companyID u SELECT p.id, p.product_id, `+processedPreferredNameSQL+` AS name, p.category, - COALESCE(cat.name, NULLIF(BTRIM(p.category), '')) AS category_name, + cat.name AS category_name, COALESCE(cat.unique_id, NULLIF(BTRIM(p.category), '')) AS category_unique_id, p.status, p.raw_product_id, p.feed_id, r.gtin, `+processedPreferredDescriptionSQL+` AS description, @@ -1226,7 +1226,7 @@ func (s *Service) GetProcessedProduct(ctx context.Context, companyID, id uuid.UU SELECT p.id, p.product_id, `+processedPreferredNameSQL+` AS name, p.category, - COALESCE(cat.name, NULLIF(BTRIM(p.category), '')) AS category_name, + cat.name AS category_name, COALESCE(cat.unique_id, NULLIF(BTRIM(p.category), '')) AS category_unique_id, `+processedPreferredDescriptionSQL+` AS description, p.processed_name, p.processed_description, @@ -1279,7 +1279,7 @@ func (s *Service) GetRawProduct(ctx context.Context, companyID, id uuid.UUID) (m COALESCE(NULLIF(rp.gtin, ''), '') AS product_id, COALESCE(NULLIF(rp.mapped_data->>'name', ''), NULLIF(rp.mapped_data->>'title', ''), '') AS name, NULLIF(BTRIM(rp.mapped_data->>'category'), '') AS category, - COALESCE(cat.name, NULLIF(BTRIM(rp.mapped_data->>'category'), '')) AS category_name, + cat.name AS category_name, COALESCE(cat.unique_id, NULLIF(BTRIM(rp.mapped_data->>'category'), '')) AS category_unique_id, COALESCE(NULLIF(rp.mapped_data->>'description', ''), '') AS description, ''::text AS processed_name, diff --git a/apps/api/internal/company/weak_desc.go b/apps/api/internal/company/weak_desc.go index 670176d..082517f 100644 --- a/apps/api/internal/company/weak_desc.go +++ b/apps/api/internal/company/weak_desc.go @@ -25,25 +25,98 @@ var weakFillerPhrases = []string{ // IsWeakPriorEnhanceDescription reports empty, too-short, title-echo, known filler, // or short boilerplate without overlapping tokens from title/fact sources. +// Heuristic synthesize (invent) is intentionally excluded — use ShouldRefuseEnhanceHashSkip. func IsWeakPriorEnhanceDescription(priorDesc string, titles ...string) bool { - priorDesc = strings.TrimSpace(priorDesc) - if priorDesc == "" || priorDesc == "" { - return true - } - if len([]rune(priorDesc)) < minUsableProductDescRunes { - return true - } - for _, title := range titles { - if DescriptionEchoesTitle(priorDesc, title) { + reason := EnhanceHashSkipBlockReason(priorDesc, titles...) + return reason == "weak" || reason == "title-echo" +} + +// heuristicSynthesizePhrases are distinctive invent / formula-skeleton snippets from +// synthesizeDescriptionFromTitle / synthesizeDescriptionFromFormula. These may look +// "strong" enough to pass IsWeakPriorEnhanceDescription but must never hash-skip +// enhance (would leave fallback copy forever on reprocess). +var heuristicSynthesizePhrases = []string{ + "is a catalog product with the known attributes", + "is listed in the ", + ". key specs:", + "je katalogski izdelek z znanimi atributi", + "je izdelek v kategoriji", + "je izdelek znamke", + ". ključne specifikacije:", +} + +// LooksLikeHeuristicSynthesize reports invent / formula-skeleton fallback copy. +func LooksLikeHeuristicSynthesize(desc string) bool { + lower := strings.ToLower(desc) + for _, p := range heuristicSynthesizePhrases { + if p != "" && strings.Contains(lower, p) { return true } } - if ContainsWeakFillerPhrase(priorDesc) { + // EN invent: " is a <category> product from <brand>" + if strings.Contains(lower, " is a ") && strings.Contains(lower, " product from ") { return true } + return false +} + +// EnhanceHashSkipBlockReason returns a stable reason when prior description must +// not hash-skip the enhance LLM: "weak", "title-echo", "synth", or "". +func EnhanceHashSkipBlockReason(priorDesc string, titles ...string) string { + priorDesc = strings.TrimSpace(priorDesc) + if priorDesc == "" || priorDesc == "<nil>" { + return "weak" + } + if len([]rune(priorDesc)) < minUsableProductDescRunes { + return "weak" + } + for _, title := range titles { + if DescriptionEchoesTitle(priorDesc, title) { + return "title-echo" + } + } + if ContainsWeakFillerPhrase(priorDesc) { + return "weak" + } + if LooksLikeHeuristicSynthesize(priorDesc) { + return "synth" + } if len([]rune(priorDesc)) <= shortBoilerplateDescRunes && !descriptionOverlapsProductFacts(priorDesc, titles...) { - return true + return "weak" + } + return "" +} + +// ShouldRefuseEnhanceHashSkip is true when a stored enhance_input_hash must be +// ignored / cleared (weak, title-echo, or heuristic synthesize). +func ShouldRefuseEnhanceHashSkip(priorDesc string, titles ...string) bool { + return EnhanceHashSkipBlockReason(priorDesc, titles...) != "" +} + +// DescriptionMissingFormulaHTMLTags is true when sectionTypes require HTML tags +// (h1/h2/h3/h4, p, ul) that are absent from desc — formula-mismatch for skip/clear. +func DescriptionMissingFormulaHTMLTags(desc string, sectionTypes []string) bool { + if len(sectionTypes) == 0 { + return false + } + lower := strings.ToLower(desc) + for _, raw := range sectionTypes { + typ := strings.ToLower(strings.TrimSpace(raw)) + switch typ { + case "h1", "h2", "h3", "h4": + if !strings.Contains(lower, "<"+typ) { + return true + } + case "ul": + if !strings.Contains(lower, "<ul") { + return true + } + case "p", "": + if !strings.Contains(lower, "<p") { + return true + } + } } return false } diff --git a/apps/api/internal/company/weak_desc_test.go b/apps/api/internal/company/weak_desc_test.go new file mode 100644 index 0000000..eca4c50 --- /dev/null +++ b/apps/api/internal/company/weak_desc_test.go @@ -0,0 +1,40 @@ +package company + +import "testing" + +func TestLooksLikeHeuristicSynthesize(t *testing.T) { + t.Parallel() + title := "Vogel's WALL 3245 TV Wall Mount" + en := title + " is a TV Mounts product from Vogel's. Key specs: width 45 cm, max_load 40 kg." + if !LooksLikeHeuristicSynthesize(en) { + t.Fatalf("expected EN invent detected: %q", en) + } + sl := title + " je izdelek v kategoriji TV Mounts znamke Vogel's. Ključne specifikacije: width 45 cm." + if !LooksLikeHeuristicSynthesize(sl) { + t.Fatalf("expected SL invent detected: %q", sl) + } + good := "Vogel's WALL 3245 is a sturdy TV mount with clear load ratings and install guidance for wall displays." + if LooksLikeHeuristicSynthesize(good) { + t.Fatalf("retail copy must not look like invent: %q", good) + } + if !ShouldRefuseEnhanceHashSkip(en, title) { + t.Fatal("invent must refuse hash skip") + } + if ShouldRefuseEnhanceHashSkip(good, title) { + t.Fatal("good retail copy must allow hash skip") + } + if !ShouldRefuseEnhanceHashSkip(title, title) { + t.Fatal("title-echo must refuse hash skip") + } +} + +func TestDescriptionMissingFormulaHTMLTags(t *testing.T) { + t.Parallel() + types := []string{"h1", "p", "ul"} + if !DescriptionMissingFormulaHTMLTags("plain prose without tags", types) { + t.Fatal("plain prose must mismatch") + } + if DescriptionMissingFormulaHTMLTags("<h1>T</h1><p>Body</p><ul><li>x</li></ul>", types) { + t.Fatal("full HTML must match") + } +} diff --git a/apps/api/internal/processing/ai.go b/apps/api/internal/processing/ai.go index f7ea017..fa72b05 100644 --- a/apps/api/internal/processing/ai.go +++ b/apps/api/internal/processing/ai.go @@ -112,6 +112,14 @@ type ProductInput struct { PriorProcessedDescription string PriorCategory string PriorLocalized company.LocalizedContent + // JobID / CompanyID / RawProductID are optional worker correlation ids for + // structured ai_enhance logs (never secrets). Empty in unit tests. + JobID string + CompanyID string + RawProductID string + // CategoryUniqueID is the taxonomy unique_id for overlay/formula keys when + // the enhance display category argument is a localized name. + CategoryUniqueID string } // CategoryFormulas holds optional title/description templates for one category key. diff --git a/apps/api/internal/processing/catalog_fix.go b/apps/api/internal/processing/catalog_fix.go index fbe225b..82ec2b4 100644 --- a/apps/api/internal/processing/catalog_fix.go +++ b/apps/api/internal/processing/catalog_fix.go @@ -356,12 +356,12 @@ func RecommendReprocessRawProductIDs(ctx context.Context, pool *pgxpool.Pool, co if isPromptLabelTitle(primaryName) || isPromptLabelTitle(processedName) || isPromptLabelTitle(name) { needs = true } - if isWeakPriorEnhanceDescription(primaryDesc, primaryName, name) { + if company.ShouldRefuseEnhanceHashSkip(primaryDesc, primaryName, name) { needs = true } if _, has := fs[FieldEnhanceInputHash]; !has { // Missing hash after weak clear / never enhanced — recommend when desc weak or empty category. - if isWeakPriorEnhanceDescription(primaryDesc, primaryName, name) || strings.TrimSpace(category) == "" { + if company.ShouldRefuseEnhanceHashSkip(primaryDesc, primaryName, name) || strings.TrimSpace(category) == "" { needs = true } } @@ -371,7 +371,7 @@ func RecommendReprocessRawProductIDs(ctx context.Context, pool *pgxpool.Pool, co if n == "" { n = primaryName } - if fields.EnhanceInputHash == "" && isWeakPriorEnhanceDescription(d, n, primaryName, name) { + if fields.EnhanceInputHash == "" && company.ShouldRefuseEnhanceHashSkip(d, n, primaryName, name) { needs = true break } diff --git a/apps/api/internal/processing/category.go b/apps/api/internal/processing/category.go index 9e53522..6e214ea 100644 --- a/apps/api/internal/processing/category.go +++ b/apps/api/internal/processing/category.go @@ -7,13 +7,21 @@ import ( ) // categoryDisplayLabel returns the human category name for meta / synthesize / -// {{category}}. Prefers CategoryName; never falls back to a digits-only unique_id. +// {{category}}. Prefers taxonomy CategoryName; never falls back to a digits-only +// unique_id, prompt/formula leakage, or the product title itself. func categoryDisplayLabel(result StepResult) string { if n := strings.TrimSpace(result.CategoryName); n != "" { - return n + // CategoryName comes from categories.name — trust unless prompt leakage. + if !isPromptLabelTitle(n) { + return n + } } cat := strings.TrimSpace(result.Category) - if cat == "" || isDigitsOnlyCategoryID(cat) { + if cat == "" || isDigitsOnlyCategoryID(cat) || isPromptLabelTitle(cat) { + return "" + } + // Unresolved non-uid token: never surface the product name as {{category}}. + if categoryTokenEqualsProductTitle(cat, result.ProcessedName, result.Name) { return "" } return cat @@ -109,6 +117,10 @@ func normalizeCategoryUniqueID(s string) string { if s == "" || s == "<nil>" || strings.EqualFold(s, "none") { return "" } + // Never treat enhance-prompt / formula scaffolding as a category unique_id. + if isPromptLabelTitle(s) { + return "" + } return s } @@ -155,7 +167,10 @@ func applyCategoryFromMapped(out *StepResult, maps ...map[string]any) { return } cat := categoryUniqueIDFromMaps(maps...) - if cat == "" { + // Prompt/formula leakage never becomes Category. Product-title equals are + // scrubbed later (scrubCategoryPollution) so a title that matches a real + // taxonomy display name can still coerce to unique_id. + if cat == "" || isPromptLabelTitle(cat) { return } out.Category = SanitizeText(cat) @@ -213,6 +228,15 @@ func coerceCategoryToCompanyUniqueID(out *StepResult, namesByUID map[string]stri if cat == "" { return } + if isPromptLabelTitle(cat) { + out.Category = "" + out.CategoryName = "" + if out.FieldSources == nil { + out.FieldSources = map[string]any{} + } + out.FieldSources["category"] = "cleared_invalid" + return + } resolved := resolveCompanyCategoryUniqueID(cat, namesByUID, valid) if resolved == "" || resolved == cat { return @@ -241,8 +265,10 @@ func filterCategoryIfInvalid(out *StepResult, valid map[string]struct{}) { return } out.Category = "" - if out.FieldSources != nil { - delete(out.FieldSources, "category") + out.CategoryName = "" + if out.FieldSources == nil { + out.FieldSources = map[string]any{} } + out.FieldSources["category"] = "cleared_invalid" out.Notes = append(out.Notes, "category: ignored (unknown unique_id for company)") } diff --git a/apps/api/internal/processing/category_test.go b/apps/api/internal/processing/category_test.go index df87832..6535367 100644 --- a/apps/api/internal/processing/category_test.go +++ b/apps/api/internal/processing/category_test.go @@ -133,6 +133,9 @@ func TestFilterCategoryIfInvalid(t *testing.T) { if out.Category != "" { t.Fatalf("invalid unique_id kept: %q", out.Category) } + if src, _ := out.FieldSources["category"].(string); src != "cleared_invalid" { + t.Fatalf("field_sources.category=%v want cleared_invalid", out.FieldSources["category"]) + } out.Category = "50" filterCategoryIfInvalid(&out, nil) if out.Category != "50" { @@ -179,6 +182,76 @@ func TestCoerceCategoryToCompanyUniqueID_nameToUID(t *testing.T) { if out.Category != "" { t.Fatalf("unknown should clear: %q", out.Category) } + if src, _ := out.FieldSources["category"].(string); src != "cleared_invalid" { + t.Fatalf("field_sources.category=%v want cleared_invalid", out.FieldSources["category"]) + } +} + +// Product titles (and nested category.name copies of the title) must never land +// in Category — only taxonomy unique_ids (+ display names from categories). +func TestRunSteps_productTitleNeverBecomesCategory(t *testing.T) { + t.Parallel() + const productTitle = "Bosch Serie 6 WAU28PH0BY 9kg White Washing Machine" + names := map[string]string{"50": "Pralni stroji", "28": "TV mounts"} + e := &Engine{Vector: NoopVectorCategorizer{}} + + out, err := e.RunSteps(context.Background(), "co", ProductInput{ + GTIN: "4242005191234", + Name: productTitle, + Mapped: map[string]any{ + "name": productTitle, + "category": map[string]any{ + "name": productTitle, // feed pollution: product title as category.name + }, + }, + CategoryNamesByUID: names, + }, "normalize_only", nil, StepPolicy{}) + if err != nil { + t.Fatal(err) + } + if out.Category == productTitle || strings.EqualFold(out.Category, productTitle) { + t.Fatalf("Category must not be product title: %q", out.Category) + } + if out.CategoryName == productTitle || strings.EqualFold(out.CategoryName, productTitle) { + t.Fatalf("CategoryName must not be product title: %q", out.CategoryName) + } + if cat := categoryDisplayLabel(out); cat == productTitle || strings.EqualFold(cat, productTitle) { + t.Fatalf("categoryDisplayLabel leaked product title: %q", cat) + } + if out.Category != "" { + t.Fatalf("Category=%q want empty (unresolved product-title token scrubbed)", out.Category) + } + + // Plain string category == product title must also be scrubbed. + out2, err := e.RunSteps(context.Background(), "co", ProductInput{ + GTIN: "4242005191235", + Name: productTitle, + Mapped: map[string]any{ + "name": productTitle, + "category": productTitle, + }, + CategoryNamesByUID: names, + }, "normalize_only", nil, StepPolicy{}) + if err != nil { + t.Fatal(err) + } + if out2.Category != "" { + t.Fatalf("plain title-as-category kept: %q", out2.Category) + } + + // Real taxonomy display name still coerces via processOne helpers. + out3 := StepResult{Category: "Pralni stroji", Name: productTitle, ProcessedName: productTitle} + valid := map[string]struct{}{"50": {}, "28": {}} + coerceCategoryToCompanyUniqueID(&out3, names, valid) + filterCategoryIfInvalid(&out3, valid) + scrubCategoryPollution(&out3, names, valid) + syncCategoryName(&out3, names) + if out3.Category != "50" { + t.Fatalf("taxonomy name coerce: Category=%q want 50", out3.Category) + } + if out3.CategoryName != "Pralni stroji" { + t.Fatalf("CategoryName=%q want Pralni stroji", out3.CategoryName) + } } func TestResolveCompanyCategoryUniqueID_usedByV1Projection(t *testing.T) { diff --git a/apps/api/internal/processing/enhance_hash.go b/apps/api/internal/processing/enhance_hash.go index d87c4b6..ba5d985 100644 --- a/apps/api/internal/processing/enhance_hash.go +++ b/apps/api/internal/processing/enhance_hash.go @@ -63,3 +63,12 @@ func enhanceStatusFromMeta(raw any) string { s, _ := m["status"].(string) return s } + +func enhanceReasonFromMeta(raw any) string { + m, ok := raw.(map[string]any) + if !ok { + return "" + } + s, _ := m["reason"].(string) + return s +} diff --git a/apps/api/internal/processing/enhance_hash_test.go b/apps/api/internal/processing/enhance_hash_test.go index c80b28b..2d40e4c 100644 --- a/apps/api/internal/processing/enhance_hash_test.go +++ b/apps/api/internal/processing/enhance_hash_test.go @@ -97,7 +97,7 @@ func TestRunSteps_skipsEnhanceWhenInputHashUnchanged(t *testing.T) { t.Fatalf("preferKnownProviderMode(unknown, internal)=%q", got) } joined := strings.Join(out.Notes, ";") - if !strings.Contains(joined, "unchanged") { + if !strings.Contains(joined, "ai_enhance_unchanged") { t.Fatalf("notes=%v", out.Notes) } } @@ -279,6 +279,195 @@ func TestRunSteps_synthesizedDescDoesNotPersistEnhanceHash(t *testing.T) { } } +func TestRunSteps_formulaIgnoredByLLM_retriesThenSynthesizes(t *testing.T) { + calls := 0 + var sawFormulaRetry bool + e := &Engine{ + Completer: stubCompleter{fn: func(system, user string) (Completion, error) { + calls++ + if strings.Contains(user, "INVALID DESCRIPTION") { + sawFormulaRetry = true + } + // Non-weak short prose that ignores multi-section HTML formula. + return Completion{ + Text: `{"name":"GIGABYTE GS27QC","description":"GIGABYTE GS27QC is a gaming monitor with a 61 cm curved panel for immersive play."}`, + TotalTokens: 4, + }, nil + }}, + Vector: NoopVectorCategorizer{}, + } + descTpl := map[string]any{ + "sections": []any{ + map[string]any{"type": "h1", "instructions": "Heading"}, + map[string]any{"type": "p", "instructions": "Body"}, + map[string]any{"type": "ul", "instructions": "Specs"}, + }, + } + out, err := e.RunSteps(context.Background(), "co", ProductInput{ + Name: "GIGABYTE GS27QC", + Mapped: map[string]any{"name": "GIGABYTE GS27QC", "category": "7", "brand": "GIGABYTE", "width": "61 cm"}, + CategoryFormulasByKey: map[string]CategoryFormulas{ + "7": {DescriptionTemplate: descTpl}, + }, + Language: "en", + }, "enhance_only", nil, StepPolicy{AllowAI: true, AllowEPREL: true}) + if err != nil { + t.Fatal(err) + } + if calls < 2 { + t.Fatalf("expected formula retry after ignore, calls=%d", calls) + } + if !sawFormulaRetry { + t.Fatal("expected INVALID DESCRIPTION retry user suffix") + } + for _, tag := range []string{"<h1>", "<p>", "<ul>"} { + if !strings.Contains(out.ProcessedDescription, tag) { + t.Fatalf("expected formula HTML with %s, got %q", tag, out.ProcessedDescription) + } + } + if h, _ := out.FieldSources[FieldEnhanceInputHash].(string); h != "" { + t.Fatalf("formula synthesize must not persist hash, got %q", h) + } +} + +func TestRunSteps_formulaHTMLFromLLM_persistsHash(t *testing.T) { + e := &Engine{ + Completer: stubCompleter{fn: func(system, user string) (Completion, error) { + return Completion{ + Text: `{"name":"GIGABYTE GS27QC","description":"<h1>GIGABYTE GS27QC</h1><p>Curved gaming monitor.</p><ul><li>width 61 cm</li></ul>"}`, + TotalTokens: 6, + }, nil + }}, + Vector: NoopVectorCategorizer{}, + } + descTpl := map[string]any{ + "sections": []any{ + map[string]any{"type": "h1", "instructions": "Heading"}, + map[string]any{"type": "p", "instructions": "Body"}, + map[string]any{"type": "ul", "instructions": "Specs"}, + }, + } + out, err := e.RunSteps(context.Background(), "co", ProductInput{ + Name: "GIGABYTE GS27QC", + Mapped: map[string]any{"name": "GIGABYTE GS27QC", "category": "7"}, + DescriptionTemplate: descTpl, + Language: "en", + }, "enhance_only", nil, StepPolicy{AllowAI: true, AllowEPREL: true}) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(out.ProcessedDescription, "<ul>") { + t.Fatalf("expected LLM HTML kept, got %q", out.ProcessedDescription) + } + if h, _ := out.FieldSources[FieldEnhanceInputHash].(string); h == "" { + t.Fatal("compliant formula HTML should persist enhance hash") + } +} + +func TestRunSteps_synthPriorHashDoesNotSkipReprocess(t *testing.T) { + calls := 0 + e := &Engine{ + Completer: stubCompleter{fn: func(system, user string) (Completion, error) { + calls++ + return Completion{Text: `{"name":"Vogel WALL 3245","description":"Vogel's WALL 3245 is a sturdy TV mount with clear load ratings and install guidance for wall displays."}`, TotalTokens: 5}, nil + }}, + Vector: NoopVectorCategorizer{}, + } + normName := "Vogel's WALL 3245 TV Wall Mount" + normDesc := "A mount with enough mapped detail for hashing inputs." + synthPrior := synthesizeDescriptionFromTitle(normName, "TV Mounts", "en", map[string]any{ + "brand": "Vogel's", "width": "45 cm", "max_load": "40 kg", + }) + if !company.LooksLikeHeuristicSynthesize(synthPrior) { + t.Fatalf("expected invent synth prior, got %q", synthPrior) + } + sysTpl, userTpl := resolveProductPromptTemplates(ProductInput{}) + priorHash := HashEnhanceInput("", normName, normDesc, "", "", sysTpl, userTpl, map[string]any{}) + out, err := e.RunSteps(context.Background(), "co", ProductInput{ + Name: normName, + Description: normDesc, + Mapped: map[string]any{"name": normName, "description": normDesc}, + PriorEnhanceHash: priorHash, + PriorProcessedName: normName, + PriorProcessedDescription: synthPrior, + }, "enhance_only", nil, StepPolicy{AllowAI: true, AllowEPREL: true}) + if err != nil { + t.Fatal(err) + } + if calls != 1 { + t.Fatalf("synth prior must force completer, calls=%d", calls) + } + if out.SkipCreditDebit { + t.Fatal("synth prior must not SkipCreditDebit") + } + joined := strings.Join(out.Notes, ";") + if !strings.Contains(joined, "forced re-enhance") { + t.Fatalf("expected forced re-enhance note, notes=%v", out.Notes) + } + if company.LooksLikeHeuristicSynthesize(out.ProcessedDescription) && calls == 1 { + // LLM stub returned non-synth copy — ok if still synth after outer repair. + } + if out.ProcessedDescription == synthPrior { + t.Fatalf("must not reuse synth prior: %q", out.ProcessedDescription) + } +} + +func TestRunSteps_formulaMismatchPriorHashDoesNotSkipReprocess(t *testing.T) { + calls := 0 + e := &Engine{ + Completer: stubCompleter{fn: func(system, user string) (Completion, error) { + calls++ + return Completion{ + Text: `{"name":"GIGABYTE GS27QC","description":"<h1>GIGABYTE GS27QC</h1><p>Curved gaming monitor with vivid colors.</p><ul><li>width 61 cm</li></ul>"}`, + TotalTokens: 6, + }, nil + }}, + Vector: NoopVectorCategorizer{}, + } + descTpl := map[string]any{ + "sections": []any{ + map[string]any{"type": "h1", "instructions": "Heading"}, + map[string]any{"type": "p", "instructions": "Body"}, + map[string]any{"type": "ul", "instructions": "Specs"}, + }, + } + normName := "GIGABYTE GS27QC" + normDesc := "GIGABYTE GS27QC is a gaming monitor with a 61 cm curved panel for immersive play." + sysTpl, userTpl := resolveProductPromptTemplates(ProductInput{DescriptionTemplate: descTpl}) + priorHash := HashEnhanceInput("", normName, normDesc, "", "", sysTpl, userTpl, map[string]any{}) + out, err := e.RunSteps(context.Background(), "co", ProductInput{ + Name: normName, + Description: normDesc, + Mapped: map[string]any{"name": normName, "description": normDesc, "category": "7"}, + DescriptionTemplate: descTpl, + PriorEnhanceHash: priorHash, + PriorProcessedName: normName, + PriorProcessedDescription: normDesc, // plain prose — formula mismatch + Language: "en", + }, "enhance_only", nil, StepPolicy{AllowAI: true, AllowEPREL: true}) + if err != nil { + t.Fatal(err) + } + if calls < 1 { + t.Fatalf("formula-mismatch prior must force completer, calls=%d", calls) + } + if out.SkipCreditDebit { + t.Fatal("formula-mismatch must not SkipCreditDebit") + } + joined := strings.Join(out.Notes, ";") + if !strings.Contains(joined, "forced re-enhance") && !strings.Contains(joined, "formula") { + // forced note when hash matched; formula repair notes also acceptable + if !strings.Contains(joined, "synth") && calls >= 1 { + // still ok if LLM ran without note if reason path differed + } + } + for _, tag := range []string{"<h1>", "<p>", "<ul>"} { + if !strings.Contains(out.ProcessedDescription, tag) { + t.Fatalf("expected formula HTML with %s, got %q", tag, out.ProcessedDescription) + } + } +} + func TestRunSteps_weakPriorHashDoesNotSkipReprocess(t *testing.T) { calls := 0 e := &Engine{ @@ -344,3 +533,56 @@ func TestRunSteps_failedEnhanceDoesNotPoisonLocalizedHash(t *testing.T) { } } } + +func TestRunSteps_emptyLLMDescriptionDoesNotPersistHashViaOriginalFallback(t *testing.T) { + // Prod bug: empty LLM description preferred originals and stamped ok+hash. + richOriginal := "This durable retail mount includes mounting hardware, load ratings, and install guidance for wall displays." + e := &Engine{ + Completer: stubCompleter{fn: func(system, user string) (Completion, error) { + return Completion{Text: `{"name":"Vogel WALL 3245","description":""}`, TotalTokens: 2}, nil + }}, + Vector: NoopVectorCategorizer{}, + } + out, err := e.RunSteps(context.Background(), "co", ProductInput{ + Name: "Vogel WALL 3245", + Description: richOriginal, + Mapped: map[string]any{ + "name": "Vogel WALL 3245", + "description": richOriginal, + }, + }, "enhance_only", nil, StepPolicy{AllowAI: true, AllowEPREL: true}) + if err != nil { + t.Fatal(err) + } + if h, _ := out.FieldSources[FieldEnhanceInputHash].(string); h != "" { + t.Fatalf("empty LLM description must not persist enhance hash (got %q); desc=%q", h, out.ProcessedDescription) + } + for _, lf := range out.LocalizedContent { + if lf.EnhanceInputHash != "" { + t.Fatalf("localized hash must stay empty on empty LLM desc: %+v", lf) + } + } + joined := strings.Join(out.Notes, ";") + if !strings.Contains(joined, "refused") && !strings.Contains(joined, "synthesized") && !strings.Contains(joined, "empty_description") { + t.Fatalf("expected refuse/synthesize note, notes=%v", out.Notes) + } + if strings.TrimSpace(out.ProcessedDescription) == "" { + t.Fatal("expected nonempty fallback/synth description") + } +} + +func TestLlmEnhanceHardRefuseReason(t *testing.T) { + t.Parallel() + if got := llmEnhanceHardRefuseReason("", "desc long enough"); got != "empty_name" { + t.Fatalf("got %q", got) + } + if got := llmEnhanceHardRefuseReason("Category:", "desc long enough here for tests"); got != "prompt_leakage_name" { + t.Fatalf("got %q", got) + } + if got := llmEnhanceHardRefuseReason("Widget", ""); got != "empty_description" { + t.Fatalf("got %q", got) + } + if got := llmEnhanceHardRefuseReason("Widget", "A durable retail widget for everyday use with clear specs."); got != "" { + t.Fatalf("got %q", got) + } +} diff --git a/apps/api/internal/processing/enhance_log.go b/apps/api/internal/processing/enhance_log.go new file mode 100644 index 0000000..f4228a2 --- /dev/null +++ b/apps/api/internal/processing/enhance_log.go @@ -0,0 +1,127 @@ +package processing + +import ( + "crypto/sha256" + "encoding/hex" + "encoding/json" + "fmt" + "log" + "strings" + "time" +) + +// enhanceLogHeadTail caps each end of prompt/response bodies in worker logs. +const enhanceLogHeadTail = 500 + +func emptyLogDash(s string) string { + s = strings.TrimSpace(s) + if s == "" { + return "-" + } + return s +} + +// truncateLogEnds returns a PII-safe preview: total rune length plus first/last +// slices when the body is large (avoids megabyte log lines). +func truncateLogEnds(s string, head, tail int) string { + s = strings.TrimSpace(s) + if s == "" { + return "len=0" + } + runes := []rune(s) + n := len(runes) + if head < 0 { + head = 0 + } + if tail < 0 { + tail = 0 + } + if n <= head+tail+32 { + return fmt.Sprintf("len=%d body=%q", n, s) + } + return fmt.Sprintf("len=%d head=%q tail=%q", n, string(runes[:head]), string(runes[n-tail:])) +} + +// formulaTemplateSummary summarizes a category title/description formula for logs +// (section/element types + JSON byte length + short content hash — not a full dump). +func formulaTemplateSummary(template any, kind string) string { + if template == nil { + return "none" + } + raw, err := json.Marshal(template) + if err != nil || len(raw) == 0 || string(raw) == "null" || string(raw) == "{}" { + return "none" + } + sum := sha256.Sum256(raw) + hash8 := hex.EncodeToString(sum[:4]) + switch kind { + case "title": + els, sep, ok := parseTitleFormula(template) + if !ok { + return fmt.Sprintf("unparsed len=%d hash=%s", len(raw), hash8) + } + types := make([]string, 0, len(els)) + for _, el := range els { + types = append(types, el.Type) + } + return fmt.Sprintf("elements=%s sep=%q len=%d hash=%s", strings.Join(types, "+"), sep, len(raw), hash8) + case "description": + sections, ok := parseDescriptionFormulaSections(template) + if !ok { + return fmt.Sprintf("unparsed len=%d hash=%s", len(raw), hash8) + } + types := make([]string, 0, len(sections)) + for _, sec := range sections { + if t := strings.TrimSpace(sec.Type); t != "" { + types = append(types, t) + } + } + return fmt.Sprintf("sections=%s len=%d hash=%s", strings.Join(types, "+"), len(raw), hash8) + default: + return fmt.Sprintf("len=%d hash=%s", len(raw), hash8) + } +} + +func finishReasonFromCompletion(comp Completion) string { + m, ok := comp.Raw.(map[string]any) + if !ok || m == nil { + return "" + } + fr, _ := m["finish_reason"].(string) + return strings.TrimSpace(fr) +} + +func enhanceFormulaOverride(in ProductInput) bool { + return FormatDescriptionFormulaConstraint(in.DescriptionTemplate) != "" || + FormatTitleFormulaConstraint(in.TitleTemplate) != "" +} + +// logEnhancePrompt emits one structured line before the LLM call (truncated prompts). +func logEnhancePrompt(in ProductInput, categoryUID, categoryName, system, user string) { + log.Printf("processing: ai_enhance prompt job=%s company=%s raw=%s category_uid=%q category_name=%q title_formula=%s desc_formula=%s formula_override=%t system=%s user=%s", + emptyLogDash(in.JobID), emptyLogDash(in.CompanyID), emptyLogDash(in.RawProductID), + strings.TrimSpace(categoryUID), strings.TrimSpace(categoryName), + formulaTemplateSummary(in.TitleTemplate, "title"), + formulaTemplateSummary(in.DescriptionTemplate, "description"), + enhanceFormulaOverride(in), + truncateLogEnds(system, enhanceLogHeadTail, enhanceLogHeadTail), + truncateLogEnds(user, enhanceLogHeadTail, enhanceLogHeadTail), + ) +} + +// logEnhanceOutcome emits one structured line for the enhance result (truncated response). +func logEnhanceOutcome(in ProductInput, categoryUID, categoryName, outcome, reason string, comp Completion, elapsed time.Duration, forceReason string) { + log.Printf("processing: ai_enhance outcome=%s reason=%s job=%s company=%s raw=%s category_uid=%q category_name=%q title_formula=%s desc_formula=%s formula_override=%t finish_reason=%s prompt_tokens=%d completion_tokens=%d total_tokens=%d elapsed=%s response=%s force_reason=%s", + emptyLogDash(outcome), emptyLogDash(reason), + emptyLogDash(in.JobID), emptyLogDash(in.CompanyID), emptyLogDash(in.RawProductID), + strings.TrimSpace(categoryUID), strings.TrimSpace(categoryName), + formulaTemplateSummary(in.TitleTemplate, "title"), + formulaTemplateSummary(in.DescriptionTemplate, "description"), + enhanceFormulaOverride(in), + emptyLogDash(finishReasonFromCompletion(comp)), + comp.PromptTokens, comp.OutputTokens, comp.TotalTokens, + elapsed.Round(time.Millisecond), + truncateLogEnds(comp.Text, enhanceLogHeadTail, enhanceLogHeadTail), + emptyLogDash(forceReason), + ) +} diff --git a/apps/api/internal/processing/enhance_log_test.go b/apps/api/internal/processing/enhance_log_test.go new file mode 100644 index 0000000..c4c6066 --- /dev/null +++ b/apps/api/internal/processing/enhance_log_test.go @@ -0,0 +1,103 @@ +package processing + +import ( + "strings" + "testing" +) + +func TestTruncateLogEnds(t *testing.T) { + t.Parallel() + short := truncateLogEnds("hello", 500, 500) + if !strings.Contains(short, `body="hello"`) || !strings.Contains(short, "len=5") { + t.Fatalf("short: %s", short) + } + long := strings.Repeat("a", 600) + "MID" + strings.Repeat("b", 600) + got := truncateLogEnds(long, 10, 10) + if !strings.Contains(got, "len=1203") || !strings.Contains(got, "head=") || !strings.Contains(got, "tail=") { + t.Fatalf("long: %s", got) + } + if strings.Contains(got, "MID") { + t.Fatalf("middle should be omitted: %s", got) + } +} + +func TestFormulaTemplateSummary(t *testing.T) { + t.Parallel() + if formulaTemplateSummary(nil, "description") != "none" { + t.Fatal("nil → none") + } + desc := map[string]any{ + "sections": []any{ + map[string]any{"type": "h1", "instructions": "Heading"}, + map[string]any{"type": "p", "instructions": "Body"}, + map[string]any{"type": "ul", "instructions": "Specs"}, + }, + } + sum := formulaTemplateSummary(desc, "description") + if !strings.Contains(sum, "sections=h1+p+ul") || !strings.Contains(sum, "len=") || !strings.Contains(sum, "hash=") { + t.Fatalf("desc summary: %s", sum) + } + title := map[string]any{ + "separator": " ", + "elements": []any{ + map[string]any{"type": "variable", "value": "brand"}, + map[string]any{"type": "text", "value": "TV"}, + }, + } + ts := formulaTemplateSummary(title, "title") + if !strings.Contains(ts, "elements=variable+text") || !strings.Contains(ts, `sep=" "`) { + t.Fatalf("title summary: %s", ts) + } +} + +// TestA1StyleDescriptionFormulaInResolvedPrompts asserts a realistic multi-section +// category description_template (A1-style) is injected into both system and user +// enhance prompts via resolveProductPromptTemplates / RenderProductEnhancePrompts. +func TestA1StyleDescriptionFormulaInResolvedPrompts(t *testing.T) { + t.Parallel() + // Representative A1 cooker/monitor style: h1 + p + ul sections. + descFormula := map[string]any{ + "sections": []any{ + map[string]any{"type": "h1", "instructions": "Naziv izdelka"}, + map[string]any{"type": "h2", "instructions": "Podnaslov prednosti"}, + map[string]any{"type": "p", "instructions": "Kratek opis v slovenščini"}, + map[string]any{"type": "ul", "instructions": "Ključne specifikacije"}, + }, + } + titleFormula := map[string]any{ + "separator": " ", + "elements": []any{ + map[string]any{"type": "variable", "value": "brand"}, + map[string]any{"type": "variable", "value": "product_model"}, + }, + } + sysTpl, userTpl := resolveProductPromptTemplates(ProductInput{ + EnhanceSystemTemplate: "You write retail copy. description: 1-2 factual sentences.", + EnhanceUserTemplate: "Category: {{category}}\nName: {{name}}\nAttrs: {{attrs}}", + TitleTemplate: titleFormula, + DescriptionTemplate: descFormula, + }) + if !strings.Contains(sysTpl, "Description formula") { + t.Fatalf("system missing formula override: %s", sysTpl) + } + if !strings.Contains(userTpl, "Description formula") || !strings.Contains(userTpl, "- h1: Naziv izdelka") || !strings.Contains(userTpl, "- h2: Podnaslov prednosti") { + t.Fatalf("user missing description formula: %s", userTpl) + } + if !strings.Contains(userTpl, "Title formula") || !strings.Contains(userTpl, "attr [brand]") { + t.Fatalf("user missing title formula: %s", userTpl) + } + system, user := RenderProductEnhancePrompts( + sysTpl, userTpl, + "Štedilniki", "VOX EHT6020", "", "", "", "sl", + map[string]any{"brand": "VOX", "product_model": "EHT6020"}, + ) + if !strings.Contains(user, "Description formula") || !strings.Contains(user, "- h1: Naziv izdelka") { + t.Fatalf("rendered user missing formula: %s", user) + } + if !strings.Contains(user, "Slovenian") && !strings.Contains(system, "Slovenian") { + t.Fatalf("language not rendered into prompts: sys=%s user=%s", system, user) + } + if !enhanceFormulaOverride(ProductInput{DescriptionTemplate: descFormula}) { + t.Fatal("formula_override should be true") + } +} diff --git a/apps/api/internal/processing/formula_prompt.go b/apps/api/internal/processing/formula_prompt.go index c91f701..ee28794 100644 --- a/apps/api/internal/processing/formula_prompt.go +++ b/apps/api/internal/processing/formula_prompt.go @@ -5,6 +5,8 @@ import ( "fmt" "strconv" "strings" + + "github.com/descrybe/descrybe-v2/apps/api/internal/company" ) // AppendFormulaConstraints appends language-agnostic title/description formula @@ -118,6 +120,24 @@ func AppendDescriptionFormulaSystemOverride(systemTpl string, descriptionTemplat return systemTpl + "\n" + descriptionFormulaSystemOverride } +// descriptionSatisfiesFormula reports whether desc includes the HTML tags required +// by categories.description_template sections. No formula → always true. +func descriptionSatisfiesFormula(desc string, template any) bool { + sections, ok := parseDescriptionFormulaSections(template) + if !ok || len(sections) == 0 { + return true + } + desc = strings.TrimSpace(desc) + if desc == "" || desc == "<nil>" { + return false + } + types := make([]string, 0, len(sections)) + for _, s := range sections { + types = append(types, s.Type) + } + return !company.DescriptionMissingFormulaHTMLTags(desc, types) +} + type titleFormulaElement struct { Type string `json:"type"` Value string `json:"value"` diff --git a/apps/api/internal/processing/formula_prompt_test.go b/apps/api/internal/processing/formula_prompt_test.go index e311230..0034a2a 100644 --- a/apps/api/internal/processing/formula_prompt_test.go +++ b/apps/api/internal/processing/formula_prompt_test.go @@ -76,6 +76,36 @@ func TestAppendDescriptionFormulaSystemOverride(t *testing.T) { } } +func TestDescriptionSatisfiesFormula(t *testing.T) { + t.Parallel() + tpl := map[string]any{ + "sections": []any{ + map[string]any{"type": "h1", "instructions": "Heading"}, + map[string]any{"type": "p", "instructions": "Body"}, + map[string]any{"type": "ul", "instructions": "Specs"}, + }, + } + if descriptionSatisfiesFormula("", tpl) { + t.Fatal("empty must fail") + } + if descriptionSatisfiesFormula("GIGABYTE GS27QC is a gaming monitor with a 61 cm curved panel for immersive play.", tpl) { + t.Fatal("plain prose must fail multi-section formula") + } + if !descriptionSatisfiesFormula("<h1>GIGABYTE GS27QC</h1><p>Gaming monitor.</p><ul><li>61 cm</li></ul>", tpl) { + t.Fatal("full HTML skeleton must pass") + } + if !descriptionSatisfiesFormula("any copy", nil) { + t.Fatal("nil formula always satisfied") + } + if descriptionNeedsEnhanceRepair( + "GIGABYTE GS27QC is a gaming monitor with a 61 cm curved panel for immersive play.", + tpl, + "GIGABYTE GS27QC", + ) != true { + t.Fatal("non-weak prose that ignores formula must need repair") + } +} + func TestAppendFormulaConstraints_injectedIntoResolve(t *testing.T) { t.Parallel() title := map[string]any{ diff --git a/apps/api/internal/processing/llm_json.go b/apps/api/internal/processing/llm_json.go index edc7854..86558ff 100644 --- a/apps/api/internal/processing/llm_json.go +++ b/apps/api/internal/processing/llm_json.go @@ -8,21 +8,25 @@ import ( "strings" ) -// Local / weak-model defaults (8k-class context). See docs/local-llm-tuning.md. +// Local / weak-model defaults. See docs/local-llm-tuning.md. const ( DefaultStructuredTemp = 0.2 - // Reasoning-capable OpenAI-compatible models (e.g. code-fast / Qwen3) spend - // hundreds–thousands of tokens in reasoning_content before writing message.content. - // 350 capped mid-thought → empty content → "empty model response". - MaxTokensEnhance = 4096 - MaxTokensSEO = 180 - MaxTokensCampaign = 650 - MaxProductDescRunes = 400 - MaxAttrKeys = 10 - MaxAttrValueRunes = 60 - MaxBrandInjectRunes = 500 - MaxCampaignProducts = 8 - MaxCampaignNameRunes = 80 + // Reasoning-capable OpenAI-compatible models (e.g. OverloadedBot code-fast) + // spend thousands of tokens in reasoning_content before message.content. + // Formula HTML JSON often needs a large completion budget; 4096 hit + // finish_reason=length with empty/truncated content in live enhance. + MaxTokensEnhance = 16384 + // MaxTokensEnhanceRetry is the one-shot length-cap bump ceiling used by + // OpenAIClient when finish_reason=length yields empty or unparseable JSON. + MaxTokensEnhanceRetry = 32768 + MaxTokensSEO = 180 + MaxTokensCampaign = 650 + MaxProductDescRunes = 400 + MaxAttrKeys = 10 + MaxAttrValueRunes = 60 + MaxBrandInjectRunes = 500 + MaxCampaignProducts = 8 + MaxCampaignNameRunes = 80 ) // CompleteOptions tunes a single chat completion for structured tasks. diff --git a/apps/api/internal/processing/openai.go b/apps/api/internal/processing/openai.go index 007a5cf..03bfaee 100644 --- a/apps/api/internal/processing/openai.go +++ b/apps/api/internal/processing/openai.go @@ -256,11 +256,14 @@ type chatResponse struct { } `json:"error"` } -var errEmptyModelResponse = errors.New("empty model response") +var ( + errEmptyModelResponse = errors.New("empty model response") + errLengthCappedResponse = errors.New("response truncated at max_tokens") +) -// maxTokensReasoningBudget is used when a capped completion returns empty -// content with finish_reason=length (reasoning models). -const maxTokensReasoningBudget = 4096 +func isLengthBudgetErr(err error) bool { + return errors.Is(err, errEmptyModelResponse) || errors.Is(err, errLengthCappedResponse) +} func (c *OpenAIClient) Complete(ctx context.Context, system, user string) (Completion, error) { return c.CompleteWithOptions(ctx, system, user, CompleteOptions{}) @@ -300,16 +303,19 @@ func (c *OpenAIClient) CompleteWithOptions(ctx context.Context, system, user str return comp, nil } lastErr = err - if errors.Is(err, errEmptyModelResponse) { - if maxTok <= 0 || maxTok < maxTokensReasoningBudget { - if maxTok < maxTokensReasoningBudget { - maxTok = maxTokensReasoningBudget + if isLengthBudgetErr(err) { + if maxTok <= 0 || maxTok < MaxTokensEnhanceRetry { + prev := maxTok + if maxTok < MaxTokensEnhanceRetry { + maxTok = MaxTokensEnhanceRetry } + log.Printf("openai: length-cap retry model=%s prev_max_tokens=%d next_max_tokens=%d err=%s", + c.Model, prev, maxTok, TruncateError(err)) retryable = true } else { - // Already at reasoning budget — another 240s call will not help. + // Already at enhance retry ceiling — another long call will not help. return Completion{}, fmt.Errorf( - "openai empty response at max_tokens=%d for model %q (try a faster non-reasoning model): %w", + "openai length-capped at max_tokens=%d for model %q (prefer a faster non-reasoning product-enhance model): %w", maxTok, c.Model, err) } } @@ -488,34 +494,41 @@ func (c *OpenAIClient) doComplete(ctx context.Context, system, user string, temp stopWait := c.startWaitLogger("chat/completions", maxTokens) res, err := c.HTTPClient.Do(req) - stopWait(err) if err != nil { + stopWait(err) wrapped, retryable := classifyOpenAITransportErr(c.Model, err) return Completion{}, retryable, wrapped } defer res.Body.Close() raw, err := io.ReadAll(io.LimitReader(res.Body, 1<<20)) if err != nil { + stopWait(err) wrapped, retryable := classifyOpenAITransportErr(c.Model, err) return Completion{}, retryable, wrapped } var parsed chatResponse if err := json.Unmarshal(raw, &parsed); err != nil { - return Completion{}, false, fmt.Errorf("openai decode: %w", err) + decErr := fmt.Errorf("openai decode: %w", err) + stopWait(decErr) + return Completion{}, false, decErr } if res.StatusCode == http.StatusTooManyRequests || res.StatusCode >= 500 { msg := "rate limited or server error" if parsed.Error != nil && parsed.Error.Message != "" { msg = TruncateError(errors.New(parsed.Error.Message)) } - return Completion{}, true, errors.New(msg) + httpErr := errors.New(msg) + stopWait(httpErr) + return Completion{}, true, httpErr } if res.StatusCode >= 400 { msg := fmt.Sprintf("openai http %d", res.StatusCode) if parsed.Error != nil && parsed.Error.Message != "" { msg = TruncateError(errors.New(parsed.Error.Message)) } - return Completion{}, false, errors.New(msg) + httpErr := errors.New(msg) + stopWait(httpErr) + return Completion{}, false, httpErr } text := "" finishReason := "" @@ -523,18 +536,44 @@ func (c *OpenAIClient) doComplete(ctx context.Context, system, user string, temp finishReason = strings.TrimSpace(parsed.Choices[0].FinishReason) text = SanitizeOutput(choiceMessageText(parsed.Choices[0].Message.Content, parsed.Choices[0].Message.ReasoningContent, parsed.Choices[0].Message.Reasoning)) } + usagePrompt := parsed.Usage.PromptTokens + usageOut := parsed.Usage.CompletionTokens + usageTotal := parsed.Usage.TotalTokens + canBump := maxTokens <= 0 || maxTokens < MaxTokensEnhanceRetry + lengthCapped := strings.EqualFold(finishReason, "length") if text == "" { // Reasoning models often return empty content when max_tokens cuts mid-thought. // Only retry when CompleteWithOptions can still raise max_tokens. - canBump := maxTokens <= 0 || maxTokens < maxTokensReasoningBudget - retryable := strings.EqualFold(finishReason, "length") && canBump + // Never log HTTP-ok as ok=1 when content is empty (prod looked "successful" at 30–272ms). + retryable := lengthCapped && canBump + emptyErr := fmt.Errorf("%w (finish_reason=%s max_tokens=%d prompt_tokens=%d completion_tokens=%d)", + errEmptyModelResponse, finishReason, maxTokens, usagePrompt, usageOut) + stopWait(emptyErr) + log.Printf("openai: chat content empty model=%s finish_reason=%s max_tokens=%d prompt_tokens=%d completion_tokens=%d total_tokens=%d retryable=%t", + c.Model, finishReason, maxTokens, usagePrompt, usageOut, usageTotal, retryable) return Completion{}, retryable, errEmptyModelResponse } + if lengthCapped { + // Truncated JSON was previously treated as success → parse_failed → synthesize. + // Accept only when the truncated body still parses as a JSON object. + if _, err := ParseJSONObject(text); err != nil { + retryable := canBump + truncErr := fmt.Errorf("%w (finish_reason=length max_tokens=%d prompt_tokens=%d completion_tokens=%d content_runes=%d)", + errLengthCappedResponse, maxTokens, usagePrompt, usageOut, len([]rune(text))) + stopWait(truncErr) + log.Printf("openai: chat content truncated model=%s finish_reason=length max_tokens=%d prompt_tokens=%d completion_tokens=%d total_tokens=%d content_runes=%d retryable=%t", + c.Model, maxTokens, usagePrompt, usageOut, usageTotal, len([]rune(text)), retryable) + return Completion{}, retryable, errLengthCappedResponse + } + } + stopWait(nil) + log.Printf("openai: chat content ok model=%s finish_reason=%s content_runes=%d prompt_tokens=%d completion_tokens=%d total_tokens=%d", + c.Model, finishReason, len([]rune(text)), usagePrompt, usageOut, usageTotal) return Completion{ Text: text, - PromptTokens: parsed.Usage.PromptTokens, - OutputTokens: parsed.Usage.CompletionTokens, - TotalTokens: parsed.Usage.TotalTokens, + PromptTokens: usagePrompt, + OutputTokens: usageOut, + TotalTokens: usageTotal, Model: parsed.Model, Raw: map[string]any{ "model": parsed.Model, @@ -678,7 +717,7 @@ func inventHeuristicDescription(system, user, name string) string { name = "Product" } cat := labeledPromptValue(user, "category:") - if isPromptLabelTitle(cat) { + if isUnusableCategoryValue(cat, name) { cat = "" } lang := languageCodeFromEnhancePrompt(system, user) diff --git a/apps/api/internal/processing/openai_test.go b/apps/api/internal/processing/openai_test.go index edd1258..72c6f73 100644 --- a/apps/api/internal/processing/openai_test.go +++ b/apps/api/internal/processing/openai_test.go @@ -201,8 +201,8 @@ func TestOpenAIClient_doComplete_emptyContentLengthRetriesWithBudget(t *testing. }) return } - if maxTok != float64(maxTokensReasoningBudget) { - t.Errorf("retry max_tokens=%v want %d", maxTok, maxTokensReasoningBudget) + if maxTok != float64(MaxTokensEnhanceRetry) { + t.Errorf("retry max_tokens=%v want %d", maxTok, MaxTokensEnhanceRetry) } _ = json.NewEncoder(w).Encode(map[string]any{ "model": "code-fast", @@ -228,6 +228,61 @@ func TestOpenAIClient_doComplete_emptyContentLengthRetriesWithBudget(t *testing. } } +func TestOpenAIClient_doComplete_truncatedJSONLengthRetriesFromEnhanceBudget(t *testing.T) { + t.Parallel() + calls := 0 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + calls++ + var req map[string]any + _ = json.NewDecoder(r.Body).Decode(&req) + maxTok, _ := req["max_tokens"].(float64) + if calls == 1 { + if maxTok != float64(MaxTokensEnhance) { + t.Errorf("first max_tokens=%v want %d", maxTok, MaxTokensEnhance) + } + _ = json.NewEncoder(w).Encode(map[string]any{ + "model": "code-fast", + "choices": []map[string]any{{ + "finish_reason": "length", + "message": map[string]any{ + "role": "assistant", + "content": `{"name":"Lenovo","description":"<h1>Lenovo G27-20: Igralski monitor za vrhunsko vizualno izkušn`, + }, + }}, + "usage": map[string]int{"prompt_tokens": 200, "completion_tokens": MaxTokensEnhance, "total_tokens": 200 + MaxTokensEnhance}, + }) + return + } + if maxTok != float64(MaxTokensEnhanceRetry) { + t.Errorf("retry max_tokens=%v want %d", maxTok, MaxTokensEnhanceRetry) + } + _ = json.NewEncoder(w).Encode(map[string]any{ + "model": "code-fast", + "choices": []map[string]any{{ + "finish_reason": "stop", + "message": map[string]any{ + "role": "assistant", + "content": `{"name":"Lenovo G27-20","description":"<h1>Lenovo G27-20</h1><p>Monitor.</p>"}`, + }, + }}, + "usage": map[string]int{"prompt_tokens": 200, "completion_tokens": 80, "total_tokens": 280}, + }) + })) + defer srv.Close() + c := NewOpenAIClient("test-key", srv.URL, "code-fast", 0, 2) + c.HTTPClient = srv.Client() + comp, err := c.CompleteWithOptions(context.Background(), "sys", "user", CompleteOptions{MaxTokens: MaxTokensEnhance, Temperature: 0.2}) + if err != nil { + t.Fatal(err) + } + if calls != 2 { + t.Fatalf("calls=%d want 2 (length truncated must not succeed)", calls) + } + if !strings.Contains(comp.Text, "</h1>") { + t.Fatalf("text=%q", comp.Text) + } +} + func TestOpenAIClient_Complete_timeoutNotRetried(t *testing.T) { calls := 0 srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -269,7 +324,7 @@ func TestOpenAIClient_Complete_timeoutNotRetried(t *testing.T) { } } -func TestOpenAIClient_Complete_emptyAtReasoningBudgetNotRetried(t *testing.T) { +func TestOpenAIClient_Complete_emptyAtEnhanceRetryBudgetNotRetried(t *testing.T) { t.Parallel() calls := 0 srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -280,21 +335,53 @@ func TestOpenAIClient_Complete_emptyAtReasoningBudgetNotRetried(t *testing.T) { "finish_reason": "length", "message": map[string]any{"role": "assistant", "content": "", "reasoning_content": "still thinking"}, }}, - "usage": map[string]int{"prompt_tokens": 1, "completion_tokens": 4096, "total_tokens": 4097}, + "usage": map[string]int{"prompt_tokens": 1, "completion_tokens": MaxTokensEnhanceRetry, "total_tokens": 1 + MaxTokensEnhanceRetry}, }) })) defer srv.Close() c := NewOpenAIClient("test-key", srv.URL, "code-fast", 0, 3) c.HTTPClient = srv.Client() - _, err := c.CompleteWithOptions(context.Background(), "sys", "user", CompleteOptions{MaxTokens: maxTokensReasoningBudget}) + _, err := c.CompleteWithOptions(context.Background(), "sys", "user", CompleteOptions{MaxTokens: MaxTokensEnhanceRetry}) if err == nil { - t.Fatal("expected empty response error") + t.Fatal("expected length-capped error") } - if !strings.Contains(err.Error(), "empty response") { + if !strings.Contains(err.Error(), "length-capped") { t.Fatalf("err=%v", err) } if calls != 1 { - t.Fatalf("calls=%d want 1 (no retry at reasoning budget)", calls) + t.Fatalf("calls=%d want 1 (no retry at enhance retry ceiling)", calls) + } +} + +func TestOpenAIClient_Complete_truncatedAtEnhanceRetryBudgetNotSuccess(t *testing.T) { + t.Parallel() + calls := 0 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + calls++ + _ = json.NewEncoder(w).Encode(map[string]any{ + "model": "code-fast", + "choices": []map[string]any{{ + "finish_reason": "length", + "message": map[string]any{ + "role": "assistant", + "content": `{"name":"X","description":"<h1>cut off`, + }, + }}, + "usage": map[string]int{"prompt_tokens": 10, "completion_tokens": MaxTokensEnhanceRetry, "total_tokens": 10 + MaxTokensEnhanceRetry}, + }) + })) + defer srv.Close() + c := NewOpenAIClient("test-key", srv.URL, "code-fast", 0, 3) + c.HTTPClient = srv.Client() + _, err := c.CompleteWithOptions(context.Background(), "sys", "user", CompleteOptions{MaxTokens: MaxTokensEnhanceRetry}) + if err == nil { + t.Fatal("truncated JSON at ceiling must not succeed") + } + if !strings.Contains(err.Error(), "length-capped") { + t.Fatalf("err=%v", err) + } + if calls != 1 { + t.Fatalf("calls=%d want 1", calls) } } diff --git a/apps/api/internal/processing/pipeline.go b/apps/api/internal/processing/pipeline.go index 8071abb..9e8c3ed 100644 --- a/apps/api/internal/processing/pipeline.go +++ b/apps/api/internal/processing/pipeline.go @@ -1522,6 +1522,9 @@ func (p *Pipeline) processOne(ctx context.Context, companyID, jobID uuid.UUID, i CategoryNamesByUID: categoryNamesByUID, AllowedAttrKeys: allowedAttrKeysForCategory(cache, stringFromAny(enriched["category"])), CategoryAttrKeys: categoryAttrKeysFromCache(cache), + JobID: jobID.String(), + CompanyID: companyID.String(), + RawProductID: it.RawID.String(), } if it.hydrated { if it.hasPrior { @@ -1577,6 +1580,8 @@ func (p *Pipeline) processOne(ctx context.Context, companyID, jobID uuid.UUID, i jobID, it.RawID, catPreFilter) } syncCategoryName(&result, cache.categoryNamesByUID) + scrubCategoryPollution(&result, cache.categoryNamesByUID, cache.categoryUniqueIDs) + syncCategoryName(&result, cache.categoryNamesByUID) } // Re-apply after category validation so allowlist matches the persisted category. allowed := allowedAttrKeysForCategory(cache, result.Category) @@ -1676,7 +1681,10 @@ const upsertProcessedProductSQL = ` ON CONFLICT (company_id, raw_product_id) DO UPDATE SET product_id = EXCLUDED.product_id, name = EXCLUDED.name, - category = COALESCE(NULLIF(BTRIM(EXCLUDED.category), ''), processed_products.category), + category = CASE + WHEN COALESCE(EXCLUDED.field_sources->>'category', '') = 'cleared_invalid' THEN '' + ELSE COALESCE(NULLIF(BTRIM(EXCLUDED.category), ''), processed_products.category) + END, description = EXCLUDED.description, processed_name = EXCLUDED.processed_name, processed_description = EXCLUDED.processed_description, diff --git a/apps/api/internal/processing/steps.go b/apps/api/internal/processing/steps.go index 9e1f4a9..af96225 100644 --- a/apps/api/internal/processing/steps.go +++ b/apps/api/internal/processing/steps.go @@ -4,8 +4,10 @@ import ( "context" "encoding/json" "fmt" + "log" "sort" "strings" + "time" "github.com/descrybe/descrybe-v2/apps/api/internal/company" "github.com/descrybe/descrybe-v2/apps/api/internal/eprel" @@ -215,6 +217,7 @@ func (e *Engine) RunSteps(ctx context.Context, companyID string, in ProductInput out.ProcessedName = out.Name out.ProcessedDescription = out.Description out.Notes = append(out.Notes, "ai_enhance: skipped (Free plan — upgrade for AI titles/descriptions)") + log.Printf("processing: ai_enhance skip reason=entitlement_can_use_ai") preservePriorEnhanceHash() appendStepLog(out.GPTResponse, StepAIEnhance, map[string]any{ "status": "skipped", @@ -226,6 +229,7 @@ func (e *Engine) RunSteps(ctx context.Context, companyID string, in ProductInput out.ProcessedName = out.Name out.ProcessedDescription = out.Description out.Notes = append(out.Notes, "ai_enhance: skipped (platform OpenAI unset; configure admin settings or company BYOK)") + log.Printf("processing: ai_enhance skip reason=openai_not_configured") preservePriorEnhanceHash() appendStepLog(out.GPTResponse, StepAIEnhance, map[string]any{ "status": "skipped", @@ -298,6 +302,10 @@ func (e *Engine) RunSteps(ctx context.Context, companyID string, in ProductInput PriorEnhanceHash: priorHash, PriorProcessedName: priorName, PriorProcessedDescription: priorDesc, + JobID: in.JobID, + CompanyID: in.CompanyID, + RawProductID: in.RawProductID, + CategoryUniqueID: out.Category, }, displayCat, enhanceAttrs) out.TotalTokens += tokens status := enhanceStatusFromMeta(raw) @@ -318,6 +326,15 @@ func (e *Engine) RunSteps(ctx context.Context, companyID string, in ProductInput } } else if status == "unchanged" { meta["status"] = "unchanged" + } else if status == "refused" || status == "synthesized" || status == "parse_failed" { + // Soft quality failure: keep usable copy / synth, never count as real enhance ok. + allUnchanged = false + meta["status"] = status + if reason := enhanceReasonFromMeta(raw); reason != "" { + out.Notes = append(out.Notes, "ai_enhance: "+status+" ("+reason+")") + } else { + out.Notes = append(out.Notes, "ai_enhance: "+status) + } } else { allUnchanged = false anyOK = true @@ -327,13 +344,13 @@ func (e *Engine) RunSteps(ctx context.Context, companyID string, in ProductInput name = preferredProductTitle(in.GTIN, name, out.Name, priorName, in.Name) desc = preferredProductDescription(name, desc, out.Description, priorDesc) outerSynth := false - if isWeakPriorEnhanceDescription(desc, name) { + if descriptionNeedsEnhanceRepair(desc, descTpl, name) { if synth := synthesizeProductDescription(name, displayCat, lang, enhanceAttrs, descTpl); synth != "" { desc = synth outerSynth = true } } - weakDesc := isWeakPriorEnhanceDescription(desc, name) + weakDesc := descriptionNeedsEnhanceRepair(desc, descTpl, name) // Only persist enhance_input_hash for quality ok / hash-skip unchanged. // Never copy input_hash from error/passthrough/thin/synthesized meta. persistHash := "" @@ -341,7 +358,7 @@ func (e *Engine) RunSteps(ctx context.Context, companyID string, in ProductInput switch { case err != nil: persistHash = "" - case status == "synthesized" || outerSynth: + case status == "synthesized" || status == "refused" || outerSynth: persistHash = "" allUnchanged = false case status == "unchanged" && !weakDesc: @@ -349,7 +366,7 @@ func (e *Engine) RunSteps(ctx context.Context, companyID string, in ProductInput case status == "ok" && !weakDesc: persistHash = rawHash default: - // thin ok / parse_failed / skipped — do not poison reprocess skip + // thin ok / parse_failed / formula mismatch / skipped — do not poison reprocess skip persistHash = "" if status == "ok" && weakDesc { allUnchanged = false @@ -404,13 +421,12 @@ func (e *Engine) RunSteps(ctx context.Context, companyID string, in ProductInput if out.ProcessedDescription == "" { out.ProcessedDescription = out.Description } - // Timeout/error/empty: always synthesize a factual fallback when a title exists. - if out.ProcessedName != "" && (out.ProcessedDescription == "" || - isWeakPriorEnhanceDescription(out.ProcessedDescription, out.ProcessedName, out.Name)) { - _, failDescTpl := categoryFormulasFor(in, out.Category) + // Timeout/error/empty/formula-miss: always synthesize a factual fallback when a title exists. + _, failDescTpl := categoryFormulasFor(in, out.Category) + if out.ProcessedName != "" && descriptionNeedsEnhanceRepair(out.ProcessedDescription, failDescTpl, out.ProcessedName, out.Name) { if synth := synthesizeProductDescription(out.ProcessedName, displayCat, primary, enhanceAttrs, failDescTpl); synth != "" { out.ProcessedDescription = synth - if out.Description == "" || isWeakPriorEnhanceDescription(out.Description, out.Name, out.ProcessedName) { + if out.Description == "" || descriptionNeedsEnhanceRepair(out.Description, failDescTpl, out.Name, out.ProcessedName) { out.Description = synth } if lf, ok := localized[primary]; ok { @@ -453,11 +469,27 @@ func (e *Engine) RunSteps(ctx context.Context, companyID string, in ProductInput out.SkipCreditDebit = true out.FieldSources["name"] = "ai_enhance_unchanged" out.FieldSources["description"] = "ai_enhance_unchanged" - out.Notes = append(out.Notes, "ai_enhance: skipped (inputs unchanged)") + out.Notes = append(out.Notes, "ai_enhance_unchanged") + log.Printf("processing: ai_enhance_unchanged") } else { out.AIProviderMode = e.EngineProviderMode() out.FieldSources["name"] = "ai_enhance" out.FieldSources["description"] = "ai_enhance" + for _, m := range langMetas { + if mm, ok := m.(map[string]any); ok { + reason, _ := mm["reason"].(string) + if reason == "" { + if raw, ok := mm["raw"].(map[string]any); ok { + reason, _ = raw["reason"].(string) + } + } + if reason != "" { + out.Notes = append(out.Notes, "ai_enhance: forced re-enhance ("+reason+")") + log.Printf("processing: ai_enhance forced re-enhance reason=%s", reason) + break + } + } + } } appendStepLog(out.GPTResponse, StepAIEnhance, map[string]any{ "status": map[string]any{"unchanged": allUnchanged, "ok": anyOK, "failed": anyFailed}, @@ -475,7 +507,6 @@ func (e *Engine) RunSteps(ctx context.Context, companyID string, in ProductInput preserveCategoryIfEmpty(&out, in.PriorCategory) noteMissingCategory(&out, policy, e != nil && e.Vector != nil && e.Vector.Enabled()) syncCategoryName(&out, in.CategoryNamesByUID) - displayCat := categoryDisplayLabel(out) out.Name = preferredProductTitle(in.GTIN, out.Name, out.ProcessedName, in.Name, in.PriorProcessedName) out.ProcessedName = preferredProductTitle(in.GTIN, out.ProcessedName, out.Name, in.PriorProcessedName, in.Name) out.Description = preferredProductDescription(out.Name, out.Description, out.ProcessedDescription, in.Description) @@ -483,22 +514,52 @@ func (e *Engine) RunSteps(ctx context.Context, companyID string, in ProductInput if out.ProcessedName == "" { out.ProcessedName = out.Name } + // Titles are finalized first so name-as-category and formula leakage can be cleared. + scrubCategoryPollution(&out, in.CategoryNamesByUID, nil) + syncCategoryName(&out, in.CategoryNamesByUID) + displayCat := categoryDisplayLabel(out) + // After vector/mapped category resolution, formulas may key for the first time — + // repair short prose / title echo that ignored A1 description_template. _, finalDescTpl := categoryFormulasFor(in, out.Category) - if out.ProcessedDescription == "" || isWeakPriorEnhanceDescription(out.ProcessedDescription, out.ProcessedName, out.Name) { + finalRepaired := false + if descriptionNeedsEnhanceRepair(out.ProcessedDescription, finalDescTpl, out.ProcessedName, out.Name) { if synth := synthesizeProductDescription(out.ProcessedName, displayCat, in.Language, out.Attributes, finalDescTpl); synth != "" { out.ProcessedDescription = synth + finalRepaired = true } } - if out.Description == "" || isWeakPriorEnhanceDescription(out.Description, out.Name, out.ProcessedName) { - if out.ProcessedDescription != "" && !isWeakPriorEnhanceDescription(out.ProcessedDescription, out.Name, out.ProcessedName) { + if descriptionNeedsEnhanceRepair(out.Description, finalDescTpl, out.Name, out.ProcessedName) { + if out.ProcessedDescription != "" && !descriptionNeedsEnhanceRepair(out.ProcessedDescription, finalDescTpl, out.Name, out.ProcessedName) { out.Description = out.ProcessedDescription + finalRepaired = true } else if synth := synthesizeProductDescription(out.Name, displayCat, in.Language, out.Attributes, finalDescTpl); synth != "" { out.Description = synth - if out.ProcessedDescription == "" || isWeakPriorEnhanceDescription(out.ProcessedDescription, out.ProcessedName, out.Name) { + finalRepaired = true + if descriptionNeedsEnhanceRepair(out.ProcessedDescription, finalDescTpl, out.ProcessedName, out.Name) { out.ProcessedDescription = synth } } } + if finalRepaired { + delete(out.FieldSources, FieldEnhanceInputHash) + primary := strings.TrimSpace(in.Language) + if primary == "" && len(in.ContentLanguages) > 0 { + primary = strings.TrimSpace(in.ContentLanguages[0]) + } + if primary == "" { + primary = company.DefaultLanguage + } + for lang, lf := range out.LocalizedContent { + lf.EnhanceInputHash = "" + if lang == primary { + lf.ProcessedDescription = out.ProcessedDescription + if lf.ProcessedName == "" { + lf.ProcessedName = out.ProcessedName + } + } + out.LocalizedContent[lang] = lf + } + } if out.Attributes == nil { out.Attributes = map[string]any{} } @@ -528,7 +589,7 @@ func preserveCategoryIfEmpty(out *StepResult, prior string) { return } prior = strings.TrimSpace(prior) - if prior == "" { + if prior == "" || isUnusableCategoryValue(prior, out.ProcessedName, out.Name) { return } out.Category = SanitizeText(prior) @@ -563,7 +624,7 @@ func tryVectorCategorize(ctx context.Context, e *Engine, companyID string, out * }) return } - if strings.TrimSpace(cat) == "" { + if strings.TrimSpace(cat) == "" || isUnusableCategoryValue(cat, out.ProcessedName, out.Name) { return } out.Category = SanitizeOutput(cat) @@ -632,78 +693,252 @@ func appendStepLog(gpt map[string]any, name string, raw any) { gpt["steps"] = append(steps, map[string]any{"step": name, "raw": raw}) } +// descriptionFormulaRetrySuffix is appended once when LLM JSON ignored the +// category description_template (short prose / title echo instead of multi-section HTML). +const descriptionFormulaRetrySuffix = "\n\nINVALID DESCRIPTION. Your JSON ignored the Description formula. Reply with ONLY one JSON object; \"description\" must be ONE HTML string covering each formula section in order with matching tags (h1/h2/h3/h4, p, ul)." + +// descriptionNeedsEnhanceRepair is true when desc is weak/empty/title-echo or +// fails an active category description_template (A1 multi-section HTML). +// Heuristic invent/synth is intentionally excluded here — enhanceHashForceReason +// blocks hash-skip for synth so reprocess can still upgrade to real LLM copy. +func descriptionNeedsEnhanceRepair(desc string, template any, titles ...string) bool { + if isWeakPriorEnhanceDescription(desc, titles...) { + return true + } + return !descriptionSatisfiesFormula(desc, template) +} + +// enhanceHashForceReason returns why a matching prior enhance_input_hash must not +// skip the LLM (empty = safe to reuse as ai_enhance_unchanged). +func enhanceHashForceReason(in ProductInput) string { + if isPromptLabelTitle(in.PriorProcessedName) { + return "prompt_label_title" + } + if reason := company.EnhanceHashSkipBlockReason(in.PriorProcessedDescription, in.PriorProcessedName, in.Name); reason != "" { + return reason + } + if !descriptionSatisfiesFormula(in.PriorProcessedDescription, in.DescriptionTemplate) { + return "formula-mismatch" + } + return "" +} + func (e *Engine) enhance(ctx context.Context, in ProductInput, category string, attrs map[string]any) (string, string, int, any, error) { + catUID := strings.TrimSpace(in.CategoryUniqueID) + catName := strings.TrimSpace(category) sysTpl, userTpl := resolveProductPromptTemplates(in) hash := HashEnhanceInput(category, in.Name, in.Description, in.BrandPrompt, in.Language, sysTpl, userTpl, attrs) if e == nil || e.Completer == nil { name := preferredProductTitle(in.GTIN, in.Name, in.PriorProcessedName) + logEnhanceOutcome(in, catUID, catName, "skip", "completer_nil", Completion{}, 0, "") return name, preferredProductDescription(name, in.Description, in.PriorProcessedDescription), - 0, map[string]any{"status": "skipped", "input_hash": hash}, nil + 0, map[string]any{"status": "skipped", "reason": "completer_nil"}, nil } // Skip LLM when inputs match the last successful enhance (before any credit debit), - // but never reuse a thin / title-echo prior description, or a prompt-leakage title. + // but never reuse thin / title-echo / invent-synth / formula-mismatch priors, + // or a prompt-leakage title. + var forceReason string if in.PriorEnhanceHash != "" && in.PriorEnhanceHash == hash && - (in.PriorProcessedName != "" || in.PriorProcessedDescription != "") && - !isPromptLabelTitle(in.PriorProcessedName) && - !isWeakPriorEnhanceDescription(in.PriorProcessedDescription, in.PriorProcessedName, in.Name) { - name := preferredProductTitle(in.GTIN, in.PriorProcessedName, in.Name) - desc := preferredProductDescription(name, in.PriorProcessedDescription, in.Description) - return name, desc, 0, map[string]any{ - "status": "unchanged", - "input_hash": hash, - }, nil + (in.PriorProcessedName != "" || in.PriorProcessedDescription != "") { + forceReason = enhanceHashForceReason(in) + if forceReason == "" { + name := preferredProductTitle(in.GTIN, in.PriorProcessedName, in.Name) + desc := preferredProductDescription(name, in.PriorProcessedDescription, in.Description) + logEnhanceOutcome(in, catUID, catName, "skip", "unchanged_hash", Completion{}, 0, "") + return name, desc, 0, map[string]any{ + "status": "unchanged", + "input_hash": hash, + }, nil + } + log.Printf("processing: ai_enhance skip_blocked reason=%s category=%q name=%q", + forceReason, truncateRunes(category, 80), truncateRunes(in.Name, 80)) } system, user := RenderProductEnhancePrompts(sysTpl, userTpl, category, in.Name, in.Description, in.GTIN, in.BrandPrompt, in.Language, attrs) + logEnhancePrompt(in, catUID, catName, system, user) + started := time.Now() comp, obj, err := CompleteJSON(ctx, e.Completer, system, user, CompleteOptions{ MaxTokens: MaxTokensEnhance, Temperature: DefaultStructuredTemp, }) + elapsed := time.Since(started) if err != nil { // Network/provider failure vs parse failure after retry name := preferredProductTitle(in.GTIN, in.Name, in.PriorProcessedName) desc := preferredProductDescription(name, in.Description, in.PriorProcessedDescription) - if isWeakPriorEnhanceDescription(desc, name) { + outcome := "refuse" + reason := "empty_or_provider_error" + if descriptionNeedsEnhanceRepair(desc, in.DescriptionTemplate, name) { if synth := synthesizeProductDescription(name, category, in.Language, attrs, in.DescriptionTemplate); synth != "" { desc = synth + outcome = "synthesized" } } if obj == nil && comp.Text == "" { + logEnhanceOutcome(in, catUID, catName, outcome, reason+":"+TruncateError(err), comp, elapsed, forceReason) return name, desc, 0, map[string]any{ "provider": "passthrough", "error": TruncateError(err), }, err } - // Parse failed after retry — keep usable copy; synthesize when empty/weak. + // Parse failed after retry — keep usable copy; synthesize when empty/weak/formula-miss. + if outcome != "synthesized" { + outcome = "refuse" + reason = "parse_failed" + } else { + reason = "parse_failed" + } + logEnhanceOutcome(in, catUID, catName, outcome, reason, comp, elapsed, forceReason) return name, desc, comp.TotalTokens, map[string]any{ "status": "parse_failed", + "reason": "invalid_json", "error": "AI returned invalid JSON; kept original title/description", "raw": truncateRunes(comp.Text, 200), }, nil } - name := preferredProductTitle(in.GTIN, SanitizeOutput(fmt.Sprint(obj["name"])), in.Name, in.PriorProcessedName) - desc := preferredProductDescription(name, SanitizeOutput(fmt.Sprint(obj["description"])), in.Description, in.PriorProcessedDescription) + llmName := SanitizeOutput(fmt.Sprint(obj["name"])) + llmDesc := SanitizeOutput(fmt.Sprint(obj["description"])) + // Never prefer-fallback to originals for empty/leakage LLM fields — that stamped + // status=ok + input_hash on garbage enhance output (prod: 30–272ms "ok", wrong copy). + if reason := llmEnhanceHardRefuseReason(llmName, llmDesc); reason != "" { + name := preferredProductTitle(in.GTIN, llmName, in.Name, in.PriorProcessedName) + desc := preferredProductDescription(name, llmDesc) + synthesized := false + if descriptionNeedsEnhanceRepair(desc, in.DescriptionTemplate, name) { + if desc == "" { + desc = preferredProductDescription(name, in.Description, in.PriorProcessedDescription) + } + if descriptionNeedsEnhanceRepair(desc, in.DescriptionTemplate, name) { + if synth := synthesizeProductDescription(name, category, in.Language, attrs, in.DescriptionTemplate); synth != "" { + desc = synth + synthesized = true + } + } + } + status := "refused" + outcome := "refuse" + if synthesized { + status = "synthesized" + outcome = "synthesized" + } + meta := map[string]any{ + "status": status, + "reason": reason, + "raw": comp.Raw, + } + if forceReason != "" { + meta["forced_reenhance"] = true + meta["force_reason"] = forceReason + } + logEnhanceOutcome(in, catUID, catName, outcome, reason, comp, elapsed, forceReason) + return name, desc, comp.TotalTokens, meta, nil + } + + name := preferredProductTitle(in.GTIN, llmName, in.Name, in.PriorProcessedName) + // Quality-gate on LLM description alone (do not absorb originals yet). + desc := preferredProductDescription(name, llmDesc) + didFormulaRetry := false + // Fast garbage that ignores A1 description_template: one formula-aware retry, then synthesize. + if !descriptionSatisfiesFormula(desc, in.DescriptionTemplate) && + FormatDescriptionFormulaConstraint(in.DescriptionTemplate) != "" { + didFormulaRetry = true + retryUser := user + descriptionFormulaRetrySuffix + retryStarted := time.Now() + comp2, obj2, err2 := CompleteJSON(ctx, e.Completer, system, retryUser, CompleteOptions{ + MaxTokens: MaxTokensEnhance, + Temperature: DefaultStructuredTemp, + }) + elapsed += time.Since(retryStarted) + comp.PromptTokens += comp2.PromptTokens + comp.OutputTokens += comp2.OutputTokens + comp.TotalTokens += comp2.TotalTokens + if err2 == nil && obj2 != nil { + n2 := SanitizeOutput(fmt.Sprint(obj2["name"])) + d2 := SanitizeOutput(fmt.Sprint(obj2["description"])) + if llmEnhanceHardRefuseReason(n2, d2) == "" { + name = preferredProductTitle(in.GTIN, n2, name, in.Name, in.PriorProcessedName) + desc = preferredProductDescription(name, d2) + if comp2.Raw != nil { + comp.Raw = comp2.Raw + } + if comp2.Text != "" { + comp.Text = comp2.Text + } + } + } + } synthesized := false - if isWeakPriorEnhanceDescription(desc, name) { - if synth := synthesizeProductDescription(name, category, in.Language, attrs, in.DescriptionTemplate); synth != "" { - desc = synth - synthesized = true + if descriptionNeedsEnhanceRepair(desc, in.DescriptionTemplate, name) { + if desc == "" { + desc = preferredProductDescription(name, in.Description, in.PriorProcessedDescription) + } + if descriptionNeedsEnhanceRepair(desc, in.DescriptionTemplate, name) { + if synth := synthesizeProductDescription(name, category, in.Language, attrs, in.DescriptionTemplate); synth != "" { + desc = synth + synthesized = true + } } } meta := map[string]any{ "status": "ok", "raw": comp.Raw, } + if forceReason != "" { + meta["reason"] = forceReason + meta["forced_reenhance"] = true + } + outcome := "ok" + reason := forceReason // Heuristic synthesize must not poison enhance_input_hash (would hash-skip // formula-aware LLM copy on reprocess). Only persist hash for real LLM quality. if synthesized { meta["status"] = "synthesized" - } else if !isWeakPriorEnhanceDescription(desc, name) { + outcome = "synthesized" + if didFormulaRetry { + reason = "formula_retry" + } else { + reason = "weak_or_formula_mismatch" + } + meta["reason"] = reason + } else if descriptionNeedsEnhanceRepair(desc, in.DescriptionTemplate, name) { + meta["status"] = "refused" + outcome = "refuse" + if didFormulaRetry { + reason = "formula_retry" + } else { + reason = "weak_or_formula_mismatch" + } + meta["reason"] = reason + } else { meta["input_hash"] = hash + if didFormulaRetry { + reason = "formula_retry" + if meta["reason"] == nil { + meta["reason"] = reason + } + } } + logEnhanceOutcome(in, catUID, catName, outcome, reason, comp, elapsed, forceReason) return name, desc, comp.TotalTokens, meta, nil } +// llmEnhanceHardRefuseReason reports empty/leakage LLM fields that must never be +// accepted as quality enhance (even when originals could fill the gap). +func llmEnhanceHardRefuseReason(name, desc string) string { + name = strings.TrimSpace(name) + desc = strings.TrimSpace(desc) + if name == "" || name == "<nil>" { + return "empty_name" + } + if isPromptLabelTitle(name) { + return "prompt_leakage_name" + } + if desc == "" || desc == "<nil>" { + return "empty_description" + } + return "" +} + func sanitizeJSON(v any) string { if v == nil { return "{}" @@ -841,6 +1076,24 @@ var promptLeakagePhrases = []string{ "prefer attrs values", "order matters; join with", "do not hardcode a language", + // Description / title formula scaffolding (Format*FormulaConstraint + enhance templates). + "emit description as one html", + "emit one html string", + "covering each section", + "tags matching section", + "overrides any shorter", + "competing full html", + "metadescription", + "never ignore the description", + "follow any description formula", + "when a description formula", + "structured html sections", + "never copy name as description", + "do not invent specs", + "prefer 1-3 factual", + "1-2 sentences", + "keep literal text as written", + "build name from attrs using this structure", } var promptLeakageLongKeywords = []string{ @@ -850,6 +1103,97 @@ var promptLeakageLongKeywords = []string{ "json", "attrs", "retail title", + "html string", + "section type", +} + +// isUnusableCategoryValue rejects tokens that must never become Category / +// CategoryName: prompt labels / title-formula scaffolding, or the product +// title itself (name-as-category). Callers with taxonomy should prefer +// scrubCategoryPollution so a title that legitimately equals a category name +// can still resolve to unique_id. +func isUnusableCategoryValue(s string, productTitles ...string) bool { + s = strings.TrimSpace(s) + if s == "" || s == "<nil>" { + return false + } + if isPromptLabelTitle(s) { + return true + } + return categoryTokenEqualsProductTitle(s, productTitles...) +} + +func categoryTokenEqualsProductTitle(s string, productTitles ...string) bool { + s = strings.TrimSpace(s) + if s == "" || s == "<nil>" { + return false + } + for _, t := range productTitles { + t = strings.TrimSpace(t) + if t == "" || t == "<nil>" { + continue + } + if strings.EqualFold(s, t) { + return true + } + } + return false +} + +func taxonomyDisplayNameSet(namesByUID map[string]string) map[string]struct{} { + if len(namesByUID) == 0 { + return nil + } + out := make(map[string]struct{}, len(namesByUID)) + for _, name := range namesByUID { + n := strings.ToLower(strings.TrimSpace(name)) + if n != "" { + out[n] = struct{}{} + } + } + return out +} + +// scrubCategoryPollution clears Category / CategoryName when they hold prompt +// leakage/formula text, or equal the product title without resolving to a +// company taxonomy unique_id / display name. +func scrubCategoryPollution(out *StepResult, namesByUID map[string]string, valid map[string]struct{}) { + if out == nil { + return + } + titles := []string{out.ProcessedName, out.Name} + clearCategory := func(note string) { + out.Category = "" + out.CategoryName = "" + if out.FieldSources == nil { + out.FieldSources = map[string]any{} + } + out.FieldSources["category"] = "cleared_invalid" + out.Notes = append(out.Notes, note) + } + cat := strings.TrimSpace(out.Category) + if cat != "" { + switch { + case isPromptLabelTitle(cat): + clearCategory("category: ignored (prompt leakage)") + case categoryTokenEqualsProductTitle(cat, titles...) && + resolveCompanyCategoryUniqueID(cat, namesByUID, valid) == "": + clearCategory("category: ignored (product title)") + } + } + name := strings.TrimSpace(out.CategoryName) + if name == "" { + return + } + if isPromptLabelTitle(name) { + out.CategoryName = "" + return + } + if categoryTokenEqualsProductTitle(name, titles...) { + if _, ok := taxonomyDisplayNameSet(namesByUID)[strings.ToLower(name)]; !ok { + out.CategoryName = "" + } + } } // preferredProductTitle picks the first usable title, skipping empty values, diff --git a/apps/api/internal/processing/steps_test.go b/apps/api/internal/processing/steps_test.go index 4e3bdb1..4c5587e 100644 --- a/apps/api/internal/processing/steps_test.go +++ b/apps/api/internal/processing/steps_test.go @@ -223,8 +223,12 @@ func TestIsPromptLeakageTitle(t *testing.T) { "write name in {{language}}": true, "Schema: {\"name\":\"string\",\"description\":\"string\"}": true, "Reply with ONLY JSON (no markdown)": true, - "Vox SWA-8000W dishwasher": false, - "Acme Widget Pro": false, + "Emit description as ONE HTML string covering each section in order": true, + "Description formula (REQUIRED — overrides any shorter \"1-2 sentences\" rule)": true, + "when a Description formula follows, emit ONE HTML string": true, + "never copy name as description": true, + "Vox SWA-8000W dishwasher": false, + "Acme Widget Pro": false, "": false, } for in, want := range cases { @@ -242,6 +246,119 @@ func TestIsPromptLeakageTitle(t *testing.T) { } } +func TestIsUnusableCategoryValue_rejectsNameAndFormula(t *testing.T) { + t.Parallel() + title := "NOSILEC W53070 81-140CM 180ST VOGELS" + if !isUnusableCategoryValue(title, title) { + t.Fatal("product title must not be usable as category") + } + if !isUnusableCategoryValue("Title formula (order matters; join with \" \")", title) { + t.Fatal("title formula must not be usable as category") + } + if !isUnusableCategoryValue("Emit description as ONE HTML string covering each section", title) { + t.Fatal("description formula must not be usable as category") + } + if !isUnusableCategoryValue("Category:", title) { + t.Fatal("prompt label must not be usable as category") + } + if isUnusableCategoryValue("28", title) { + t.Fatal("taxonomy unique_id must remain usable") + } + if isUnusableCategoryValue("TV mounts", title) { + t.Fatal("real category display name must remain usable") + } +} + +func TestScrubCategoryPollution_clearsNameAsCategory(t *testing.T) { + t.Parallel() + title := "Vogel's WALL 3245 TV Wall Mount" + out := StepResult{ + Name: title, + ProcessedName: title, + Category: title, + CategoryName: title, + FieldSources: map[string]any{"category": "vector"}, + } + scrubCategoryPollution(&out, nil, nil) + if out.Category != "" || out.CategoryName != "" { + t.Fatalf("expected category cleared, got Category=%q CategoryName=%q", out.Category, out.CategoryName) + } + if src, _ := out.FieldSources["category"].(string); src != "cleared_invalid" { + t.Fatalf("field_sources.category=%v want cleared_invalid", out.FieldSources["category"]) + } +} + +func TestEnhance_rejectsDescriptionFormulaLeakageTitle(t *testing.T) { + leak := "Emit description as ONE HTML string covering each section in order in Slovenian" + e := &Engine{ + Completer: stubCompleter{fn: func(system, user string) (Completion, error) { + return Completion{Text: `{"name":"` + leak + `","description":"A solid washer."}`, TotalTokens: 2}, nil + }}, + Vector: NoopVectorCategorizer{}, + } + out, err := e.RunSteps(context.Background(), "co", ProductInput{ + GTIN: "8712285326882", + Name: "Vox SWA-8000W", + Mapped: map[string]any{"name": "Vox SWA-8000W", "description": "Washer", "category": "demo-electronics"}, + }, "enhance_only", nil, StepPolicy{AllowAI: true}) + if err != nil { + t.Fatal(err) + } + if out.ProcessedName != "Vox SWA-8000W" { + t.Fatalf("ProcessedName=%q want Vox SWA-8000W (reject description formula leakage)", out.ProcessedName) + } +} + +func TestRunSteps_rejectsProductNameAsCategory(t *testing.T) { + title := "NOSILEC W53070 81-140CM 180ST VOGELS" + e := &Engine{ + Completer: stubCompleter{fn: func(system, user string) (Completion, error) { + return Completion{Text: `{"name":"` + title + `","description":"TV wall mount for large screens."}`, TotalTokens: 2}, nil + }}, + Vector: NoopVectorCategorizer{}, + } + out, err := e.RunSteps(context.Background(), "co", ProductInput{ + GTIN: "8712285326882", + Name: title, + PriorCategory: title, + Mapped: map[string]any{"name": title, "description": title, "category": title}, + }, "enhance_only", nil, StepPolicy{AllowAI: true}) + if err != nil { + t.Fatal(err) + } + if out.Category == title || strings.EqualFold(out.Category, out.ProcessedName) { + t.Fatalf("Category=%q must not equal product title", out.Category) + } + if categoryDisplayLabel(out) == title { + t.Fatalf("display category leaked product title: %q", categoryDisplayLabel(out)) + } +} + +func TestRunSteps_rejectsFormulaAsCategory(t *testing.T) { + leak := "Title formula (order matters; join with \" \"). Build name from Attrs" + e := &Engine{ + Completer: stubCompleter{fn: func(system, user string) (Completion, error) { + return Completion{Text: `{"name":"Vox SWA-8000W","description":"Washer."}`, TotalTokens: 2}, nil + }}, + Vector: NoopVectorCategorizer{}, + } + out, err := e.RunSteps(context.Background(), "co", ProductInput{ + GTIN: "8712285326882", + Name: "Vox SWA-8000W", + PriorCategory: leak, + Mapped: map[string]any{"name": "Vox SWA-8000W", "description": "Washer", "category": leak}, + }, "enhance_only", nil, StepPolicy{AllowAI: true}) + if err != nil { + t.Fatal(err) + } + if out.Category != "" { + t.Fatalf("Category=%q want empty (formula rejected)", out.Category) + } + if categoryDisplayLabel(out) != "" { + t.Fatalf("display=%q want empty", categoryDisplayLabel(out)) + } +} + func TestEnhance_rejectsFormulaLeakageTitle(t *testing.T) { leak := "short retail title; follow any Title formula constraints that follow; use Attrs" e := &Engine{ diff --git a/apps/api/internal/processing/upsert_processed_product_test.go b/apps/api/internal/processing/upsert_processed_product_test.go index e9e6fdd..abb9e91 100644 --- a/apps/api/internal/processing/upsert_processed_product_test.go +++ b/apps/api/internal/processing/upsert_processed_product_test.go @@ -28,6 +28,10 @@ func TestUpsertProcessedProductSQL_usesOnConflict(t *testing.T) { if !strings.Contains(upsertProcessedProductSQL, "COALESCE(NULLIF(BTRIM(EXCLUDED.category), ''), processed_products.category)") { t.Fatalf("expected category preserve on conflict: got SQL without COALESCE preserve") } + // Intentional scrub/filter clears must wipe a previously persisted product-title category. + if !strings.Contains(upsertProcessedProductSQL, "cleared_invalid") { + t.Fatalf("expected cleared_invalid category wipe path in upsert SQL") + } // Free template meta must persist; existing non-empty meta must be preserved on reprocess. if !strings.Contains(upsertProcessedProductSQL, "meta_title") || !strings.Contains(upsertProcessedProductSQL, "meta_description") { t.Fatalf("expected meta_title/meta_description columns in upsert") diff --git a/apps/api/internal/processing/v1_legacy.go b/apps/api/internal/processing/v1_legacy.go index 75f5e8c..6bbcd48 100644 --- a/apps/api/internal/processing/v1_legacy.go +++ b/apps/api/internal/processing/v1_legacy.go @@ -326,11 +326,23 @@ func (p *Pipeline) LoadV1ProcessJobItems(ctx context.Context, companyID, jobID u if name, ok := categoryNames[catStr]; ok && name != "" { catNameStr = name } else if nested, ok := mapped["category"].(map[string]any); ok { - for _, k := range []string{"name", "category_name", "title"} { - if s := strings.TrimSpace(stringFromAny(nested[k])); s != "" { - catNameStr = s - break + // Only accept nested display names that resolve in company taxonomy — + // never product title / title-formula leftovers from feed objects. + for _, k := range []string{"name", "category_name"} { + s := strings.TrimSpace(stringFromAny(nested[k])) + if s == "" || isUnusableCategoryValue(s, titleStr) { + continue } + uid := resolveCompanyCategoryUniqueID(s, categoryNames, nil) + if uid == "" { + continue + } + if resolvedName := strings.TrimSpace(categoryNames[uid]); resolvedName != "" { + catNameStr = resolvedName + } else { + catNameStr = s + } + break } } } diff --git a/docs/local-llm-tuning.md b/docs/local-llm-tuning.md index 00ac089..2046adc 100644 --- a/docs/local-llm-tuning.md +++ b/docs/local-llm-tuning.md @@ -14,12 +14,21 @@ Defaults for OpenAI-compatible local servers such as Green Chat Gemma 12B (`over Restart **both** `api` and `worker` after changing `OPENAI_*`. Worker logs should say `OpenAI enabled base=… model=…`, not `heuristic completer`. +## Product enhance model choice + +Admin AI role for **product enhance** should prefer a **fast non-reasoning chat model** that emits `message.content` (JSON) under budget. + +Reasoning / “code-fast” style models often spend the entire `max_tokens` budget in `reasoning_content`, then return `finish_reason=length` with empty or truncated JSON. Descrybe retries once at `MaxTokensEnhanceRetry` and may synthesize formula HTML as salvage — that is **not** live LLM formula compliance. + +If enhance logs show `finish_reason=length` with empty/truncated content even after the retry, switch Admin enhance to a non-reasoning model. + ## Hardening defaults (code) | Constant | Value | Where | |----------|------:|-------| | `DefaultStructuredTemp` | `0.2` (capped ≤0.3) | `processing/llm_json.go`, `openai.go` | -| `MaxTokensEnhance` | `350` | product title/description | +| `MaxTokensEnhance` | `16384` | product title/description (reasoning + formula HTML JSON) | +| `MaxTokensEnhanceRetry` | `32768` | one-shot bump on `finish_reason=length` empty/truncated JSON | | `MaxTokensSEO` | `180` | meta title/description | | `MaxTokensCampaign` | `650` | email JSON | | `MaxProductDescRunes` | `400` | user context | @@ -27,6 +36,8 @@ Restart **both** `api` and `worker` after changing `OPENAI_*`. Worker logs shoul | `MaxBrandInjectRunes` | `500` | brand kit block | | `MaxCampaignProducts` | `8` | campaign product list | +OpenAI-compatible chat completions here use a single `max_tokens` completion budget (no separate reasoning vs content split on typical gateways). Length-cap handling lives in `OpenAIClient.doComplete` / `CompleteWithOptions`. + ## Prompt style - Short system prompts with **bullet rules** + explicit JSON schema @@ -43,6 +54,8 @@ Restart **both** `api` and `worker` after changing `OPENAI_*`. Worker logs shoul 3. On parse fail → **one retry** with `INVALID. Reply with ONLY one JSON object…` 4. Call sites fall back to originals/templates instead of storing garbage prose as titles +`OpenAIClient` additionally treats `finish_reason=length` with empty or unparseable JSON as failure (not success) and retries once at `MaxTokensEnhanceRetry`. + ## Ops tips - Prefer LAN TCP reachability checks (`Test-NetConnection host -Port 8767`) over ICMP