This commit is contained in:
2026-08-16 21:35:48 +02:00
parent 106f602232
commit 389608ea56
28 changed files with 2491 additions and 115 deletions
@@ -0,0 +1,200 @@
package main
import (
"context"
"encoding/json"
"fmt"
"log"
"os"
"os/signal"
"strings"
"syscall"
"time"
"github.com/descrybe/descrybe-v2/apps/api/internal/aiprompts"
"github.com/descrybe/descrybe-v2/apps/api/internal/config"
"github.com/descrybe/descrybe-v2/apps/api/internal/db"
"github.com/descrybe/descrybe-v2/apps/api/internal/eprel"
"github.com/descrybe/descrybe-v2/apps/api/internal/logredact"
"github.com/descrybe/descrybe-v2/apps/api/internal/processing"
"github.com/google/uuid"
)
func main() {
log.SetOutput(logredact.Writer(os.Stderr))
eans := []string{
"195348253666", // 7 Gaming monitorji (rich)
"4548736132597", // 48 Slušalke (rich)
"8712285326882", // 28 Nosilci za TV (thin p-only)
"3045380024786", // 36 Pečice (rich)
"3838782459856", // 38 Pomivalni stroji (rich)
"8806095210711", // 40 Pralni stroji (rich)
"4242005342488", // 41 Pralno-sušilni (rich)
"3838782103889", // 17 Kuhalne plošče (rich)
"1200130000638", // 3 Bluetooth zvočniki (rich)
"194252029558", // 24 Mobilne naprave (rich)
}
companyID := uuid.MustParse("2b3159b0-fc08-415b-b248-35ed02a6baab")
userID := uuid.MustParse("0ce3305c-d810-4b56-b1d4-3c1ed510db76")
cfg, err := config.Load()
if err != nil {
log.Fatalf("config: %v", err)
}
ctx, cancel := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
defer cancel()
pool, err := db.NewPool(ctx, cfg.DatabaseURL, db.PoolOptions{
MaxConns: int32(cfg.DBMaxConns),
MinConns: int32(cfg.DBMinConns),
MaxConnLifetime: cfg.DBMaxConnLifetime,
MaxConnLifetimeJitter: cfg.DBMaxConnLifetimeJitter,
MaxConnIdleTime: cfg.DBMaxConnIdleTime,
HealthCheckPeriod: cfg.DBHealthCheckPeriod,
StatementTimeout: cfg.DBStatementTimeout,
})
if err != nil {
log.Fatalf("db: %v", err)
}
defer pool.Close()
out := os.Getenv("PROBE_OUT_DIR")
if out == "" {
out = `F:\laragon\www\_MY\descrybe-v2\.codehelper\_formula_probe_20260816`
}
_ = os.MkdirAll(out, 0o755)
pipeline := processing.NewPipeline(pool)
pipeline.BatchSize = cfg.ProcessingBatchSize
pipeline.AI = nil
pipeline.Prompts = aiprompts.NewService(pool)
mockKey := strings.TrimSpace(cfg.OpenAIAPIKey)
if mockKey == "" {
mockKey = "local-test"
}
mockBase := "http://127.0.0.1:18767/v1"
mockModel := "mock-llm"
log.Printf("forcing completer base=%s model=%s (ModeLabel=internal; OverloadedBot not used)", mockBase, mockModel)
client := processing.NewOpenAIClient(mockKey, mockBase, mockModel, cfg.ProcessingRPM, cfg.ProcessingMaxRetries)
client.ModeLabel = processing.AIProviderInternal
eprelClient := eprel.NewClient(eprel.Options{Enabled: true, Timeout: 20 * time.Second})
pipeline.Engine = &processing.Engine{
Completer: client,
Vector: processing.NoopVectorCategorizer{},
EPREL: eprelClient,
ProviderMode: processing.AIProviderInternal,
}
rawIDs := make([]uuid.UUID, 0, len(eans))
for _, ean := range eans {
var id uuid.UUID
err := pool.QueryRow(ctx, `
SELECT id FROM raw_products
WHERE company_id = $1 AND gtin = $2`, companyID, ean).Scan(&id)
if err != nil {
log.Fatalf("raw product %s: %v", ean, err)
}
rawIDs = append(rawIDs, id)
log.Printf("ean=%s raw_id=%s", ean, id)
}
if n, err := processing.BackfillMissingMeta(ctx, pool, companyID); err != nil {
log.Printf("BackfillMissingMeta: %v", err)
} else {
log.Printf("BackfillMissingMeta updated=%d", n)
}
jobs, err := pipeline.StartJob(ctx, companyID, userID, rawIDs, "full")
if err != nil {
log.Fatalf("StartJob: %v", err)
}
if len(jobs) == 0 {
log.Fatal("StartJob returned no jobs")
}
jobID := jobs[0].ID
log.Printf("job_id=%s", jobID)
_ = os.WriteFile(out+string(os.PathSeparator)+"process_id.txt", []byte(jobID.String()+"\n"), 0o644)
defend := make(chan struct{})
go func() {
t := time.NewTicker(500 * time.Millisecond)
defer t.Stop()
for {
select {
case <-defend:
return
case <-ctx.Done():
return
case <-t.C:
_, _ = pool.Exec(context.Background(), `
UPDATE processing_jobs
SET status = 'running', error = NULL, updated_at = now()
WHERE id = $1 AND status IN ('failed','cancelled')
AND (error ILIKE '%P0%' OR error ILIKE '%parked%' OR error ILIKE '%yield%'
OR error ILIKE '%reclaim%' OR error ILIKE '%cleared%' OR error ILIKE '%superseded%'
OR error ILIKE '%exclusive%' OR error ILIKE '%batch_%' OR error ILIKE '%inproc%'
OR error ILIKE '%formula%')`, jobID)
_, _ = pool.Exec(context.Background(), `
UPDATE processing_job_products
SET status = 'pending', error = NULL, updated_at = now()
WHERE job_id = $1 AND status IN ('failed','cancelled')
AND processed_product_id IS NULL
AND (error ILIKE '%P0%' OR error ILIKE '%parked%' OR error ILIKE '%yield%'
OR error ILIKE '%reclaim%' OR error ILIKE '%cleared%' OR error ILIKE '%superseded%'
OR error ILIKE '%exclusive%' OR error ILIKE '%batch_%' OR error ILIKE '%inproc%'
OR error ILIKE '%formula%')`, jobID)
}
}
}()
_, _ = pool.Exec(ctx, `
UPDATE processing_jobs
SET status = 'running', started_at = COALESCE(started_at, now()), updated_at = now()
WHERE id = $1`, jobID)
runCtx, runCancel := context.WithTimeout(ctx, 15*time.Minute)
defer runCancel()
if err := pipeline.ProcessJob(runCtx, jobID); err != nil {
close(defend)
log.Fatalf("ProcessJob: %v", err)
}
close(defend)
items, err := pipeline.LoadV1ProcessJobItems(ctx, companyID, jobID, "full")
if err != nil {
log.Fatalf("LoadV1ProcessJobItems: %v", err)
}
payload := map[string]any{
"data": map[string]any{
"process_id": jobID.String(),
"status": "COMPLETED",
"processing_type": "full",
"total_items": len(items),
"items": items,
},
}
raw, err := json.MarshalIndent(payload, "", " ")
if err != nil {
log.Fatalf("marshal: %v", err)
}
if err := os.WriteFile(out+string(os.PathSeparator)+"final.json", raw, 0o644); err != nil {
log.Fatalf("write final: %v", err)
}
fmt.Printf("JOB %s items=%d\n", jobID, len(items))
for _, it := range items {
ean, _ := it["ean"].(string)
st, _ := it["status"].(string)
title, _ := it["title"].(string)
cat, _ := it["category"].(string)
catn, _ := it["category_name"].(string)
desc, _ := it["description"].(string)
if len(title) > 50 {
title = title[:50]
}
ai, _ := it["ai_provider_mode"].(string)
fmt.Printf("ITEM %s status=%s cat=%s/%s ai=%s desc_len=%d title=%s\n",
ean, st, cat, catn, ai, len(desc), title)
}
}
+83
View File
@@ -0,0 +1,83 @@
package main
import (
"bytes"
"context"
"encoding/json"
"fmt"
"io"
"net/http"
"os"
"strings"
"time"
"github.com/descrybe/descrybe-v2/apps/api/internal/config"
"github.com/descrybe/descrybe-v2/apps/api/internal/db"
"github.com/descrybe/descrybe-v2/apps/api/internal/platformsettings"
)
func main() {
cfg, err := config.Load()
if err != nil {
fmt.Println("config:", err)
os.Exit(1)
}
ctx := context.Background()
pool, err := db.NewPool(ctx, cfg.DatabaseURL, db.PoolOptions{
MaxConns: 2,
MinConns: 1,
HealthCheckPeriod: 30 * time.Second,
MaxConnLifetime: time.Hour,
MaxConnIdleTime: 30 * time.Minute,
})
if err != nil {
fmt.Println("db:", err)
os.Exit(1)
}
defer pool.Close()
plat := platformsettings.NewService(pool, platformsettings.EnvConfig{
AppEncryptionKey: cfg.AppEncryptionKey,
CredentialsEncryptionKey: cfg.CredentialsEncryptionKey,
TokenSigningSecret: cfg.TokenSigningSecret,
DatabaseURL: cfg.DatabaseURL,
OpenAIAPIKey: cfg.OpenAIAPIKey,
OpenAIBaseURL: cfg.OpenAIBaseURL,
OpenAIModel: cfg.OpenAIModel,
})
rc, err := plat.ResolveAIConfig(ctx, platformsettings.AIRoleProcessing)
if err != nil {
fmt.Println("resolve:", err)
os.Exit(1)
}
base := strings.TrimRight(strings.TrimSpace(rc.BaseURL), "/")
url := base + "/chat/completions"
body := map[string]any{
"model": rc.Model,
"messages": []map[string]string{
{"role": "user", "content": "Say PONG"},
},
"max_tokens": 16,
}
rawBody, _ := json.Marshal(body)
req, err := http.NewRequestWithContext(ctx, http.MethodPost, url, bytes.NewReader(rawBody))
if err != nil {
fmt.Println("req:", err)
os.Exit(1)
}
req.Header.Set("Content-Type", "application/json")
req.Header.Set("Authorization", "Bearer "+rc.APIKey)
cli := &http.Client{Timeout: 30 * time.Second}
resp, err := cli.Do(req)
if err != nil {
fmt.Println("do:", err)
os.Exit(1)
}
defer resp.Body.Close()
raw, _ := io.ReadAll(io.LimitReader(resp.Body, 1200))
s := string(raw)
s = strings.ReplaceAll(s, rc.APIKey, "[redacted]")
fmt.Printf("source=%s provider=%s model=%s\n", rc.Source, rc.Provider, rc.Model)
fmt.Printf("url=%s status=%d bytes=%d\n", url, resp.StatusCode, len(raw))
fmt.Printf("content_type=%s\n", resp.Header.Get("Content-Type"))
fmt.Printf("body_head=%q\n", s)
}
@@ -0,0 +1,99 @@
package main
import (
"context"
"encoding/json"
"fmt"
"os"
"strings"
"time"
"github.com/descrybe/descrybe-v2/apps/api/internal/config"
"github.com/descrybe/descrybe-v2/apps/api/internal/db"
"github.com/descrybe/descrybe-v2/apps/api/internal/processing"
"github.com/google/uuid"
)
func main() {
out := `F:\laragon\www\_MY\descrybe-v2\.codehelper\_live_llm_formula_20260816`
jobID := uuid.MustParse("3d926dfa-de8a-4afc-b1dd-8707633c673e")
cfg, err := config.Load()
if err != nil {
panic(err)
}
ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
defer cancel()
pool, err := db.NewPool(ctx, cfg.DatabaseURL, db.PoolOptions{
MaxConns: 4, MinConns: 0,
MaxConnLifetime: time.Hour, HealthCheckPeriod: 30 * time.Second,
})
if err != nil {
panic(err)
}
defer pool.Close()
rows, err := pool.Query(ctx, `
SELECT rp.gtin,
COALESCE(pp.processed_name, pp.name, '') AS title,
COALESCE(NULLIF(pp.processed_description, ''), pp.description, '') AS description,
COALESCE(pp.category, '') AS category,
COALESCE(c.name, '') AS category_name,
c.description_template,
pp.id
FROM processing_job_products pjp
JOIN raw_products rp ON rp.id = pjp.raw_product_id
LEFT JOIN processed_products pp ON pp.id = pjp.processed_product_id
LEFT JOIN categories c ON c.company_id = pp.company_id AND c.unique_id = pp.category
WHERE pjp.job_id = $1
ORDER BY rp.gtin`, jobID)
if err != nil {
panic(err)
}
defer rows.Close()
products := []map[string]any{}
for rows.Next() {
var gtin, title, desc, cat, catName string
var tmpl any
var ppID *uuid.UUID
if err := rows.Scan(&gtin, &title, &desc, &cat, &catName, &tmpl, &ppID); err != nil {
panic(err)
}
idStr := ""
if ppID != nil {
idStr = ppID.String()
}
lower := strings.ToLower(desc)
products = append(products, map[string]any{
"ean": gtin,
"product_id": idStr,
"title": title,
"description": desc,
"category": cat,
"category_name": catName,
"description_template": tmpl,
"formula_constraint": processing.FormatDescriptionFormulaConstraint(tmpl),
"has_h1": strings.Contains(lower, "<h1"),
"has_h2": strings.Contains(lower, "<h2"),
"has_p": strings.Contains(lower, "<p"),
"has_ul": strings.Contains(lower, "<ul"),
"desc_len": len(desc),
"desc_ne_title": strings.TrimSpace(desc) != "" && strings.TrimSpace(desc) != strings.TrimSpace(title),
})
}
raw, _ := json.MarshalIndent(products, "", " ")
_ = os.WriteFile(out+`\db_products.json`, raw, 0o644)
fmt.Printf("wrote %d products\n", len(products))
for _, p := range products {
t := p["title"].(string)
if len(t) > 40 {
t = t[:40]
}
fmt.Printf("%s cat=%s/%s h1=%v h2=%v p=%v ul=%v desc_ne_title=%v len=%v title=%q\n",
p["ean"], p["category"], p["category_name"], p["has_h1"], p["has_h2"], p["has_p"], p["has_ul"], p["desc_ne_title"], p["desc_len"], t)
d := p["description"].(string)
if len(d) > 220 {
d = d[:220]
}
fmt.Printf(" desc_preview=%q\n", d)
}
}
@@ -0,0 +1,171 @@
package main
import (
"context"
"encoding/json"
"fmt"
"log"
"os"
"path/filepath"
"strings"
"time"
"github.com/descrybe/descrybe-v2/apps/api/internal/aiprovider"
"github.com/descrybe/descrybe-v2/apps/api/internal/config"
"github.com/descrybe/descrybe-v2/apps/api/internal/db"
"github.com/descrybe/descrybe-v2/apps/api/internal/platformsettings"
"github.com/descrybe/descrybe-v2/apps/api/internal/processing"
"github.com/google/uuid"
)
func main() {
out := os.Getenv("PROBE_OUT_DIR")
if out == "" {
out = `F:\laragon\www\_MY\descrybe-v2\.codehelper\_live_llm_enhance_budget_20260816`
}
_ = os.MkdirAll(out, 0o755)
cfg, err := config.Load()
if err != nil {
log.Fatalf("config: %v", err)
}
ctx, cancel := context.WithTimeout(context.Background(), 12*time.Minute)
defer cancel()
pool, err := db.NewPool(ctx, cfg.DatabaseURL, db.PoolOptions{
MaxConns: 2,
MinConns: 1,
HealthCheckPeriod: 30 * time.Second,
MaxConnLifetime: time.Hour,
MaxConnIdleTime: 30 * time.Minute,
})
if err != nil {
log.Fatalf("db: %v", err)
}
defer pool.Close()
plat := platformsettings.NewService(pool, platformsettings.EnvConfig{
AppEncryptionKey: cfg.AppEncryptionKey,
CredentialsEncryptionKey: cfg.CredentialsEncryptionKey,
TokenSigningSecret: cfg.TokenSigningSecret,
DatabaseURL: cfg.DatabaseURL,
OpenAIAPIKey: cfg.OpenAIAPIKey,
OpenAIBaseURL: cfg.OpenAIBaseURL,
OpenAIModel: cfg.OpenAIModel,
})
aiSvc := aiprovider.NewService(pool, aiprovider.EnvConfig{
AppEncryptionKey: cfg.AppEncryptionKey,
CredentialsEncryptionKey: cfg.CredentialsEncryptionKey,
TokenSigningSecret: cfg.TokenSigningSecret,
DatabaseURL: cfg.DatabaseURL,
OpenAIAPIKey: cfg.OpenAIAPIKey,
OpenAIBaseURL: cfg.OpenAIBaseURL,
OpenAIModel: cfg.OpenAIModel,
ProcessingRPM: cfg.ProcessingRPM,
ProcessingMaxRetries: cfg.ProcessingMaxRetries,
})
aiSvc.Platform = plat
companyID := uuid.MustParse("2b3159b0-fc08-415b-b248-35ed02a6baab")
completer, mode, byok, err := aiSvc.ResolveCompleterForRole(ctx, companyID, aiprovider.RoleProcessing)
if err != nil {
log.Fatalf("resolve: %v", err)
}
client, ok := completer.(*processing.OpenAIClient)
if !ok || client == nil {
log.Fatalf("completer type %T", completer)
}
if processing.IsMockOrLoopbackBaseURL(client.BaseURL) {
log.Fatalf("resolved mock/loopback base=%s — refuse", client.BaseURL)
}
system := `Reply with ONLY one JSON object: {"name":string,"description":string}.
description MUST be HTML using exactly these tags in order: <h1>, <p>, <h2>, <ul><li>…</li></ul>, <p>, <p>, <p>.
No markdown. No prose outside JSON.`
cases := []struct {
ean string
user string
}{
{
ean: "1200130000638",
user: "Category: Bluetooth zvočniki\nName: JBL Flip 6\nDesc: Portable Bluetooth speaker.\nAttrs: {\"brand\":\"JBL\"}",
},
{
ean: "195348253666",
user: "Category: Gaming monitorji\nName: Lenovo G27-20\nDesc: 27-inch gaming monitor.\nAttrs: {\"brand\":\"Lenovo\"}",
},
}
results := make([]map[string]any, 0, len(cases))
okLLM := 0
for _, c := range cases {
started := time.Now()
comp, obj, cerr := processing.CompleteJSON(ctx, client, system, c.user, processing.CompleteOptions{
MaxTokens: processing.MaxTokensEnhance,
Temperature: processing.DefaultStructuredTemp,
})
elapsed := time.Since(started).Round(time.Millisecond)
fr := ""
if m, ok := comp.Raw.(map[string]any); ok {
fr, _ = m["finish_reason"].(string)
}
row := map[string]any{
"ean": c.ean,
"elapsed": elapsed.String(),
"err": "",
"finish_reason": fr,
"prompt_tokens": comp.PromptTokens,
"completion_tokens": comp.OutputTokens,
"total_tokens": comp.TotalTokens,
"content_runes": len([]rune(comp.Text)),
"parsed": obj != nil,
"llm_formula_ok": false,
}
if cerr != nil {
row["err"] = processing.TruncateError(cerr)
}
if obj != nil {
desc := fmt.Sprint(obj["description"])
row["name"] = fmt.Sprint(obj["name"])
row["description_head"] = truncate(desc, 180)
hasTags := strings.Contains(desc, "<h1") && strings.Contains(desc, "<h2") &&
strings.Contains(desc, "<p") && strings.Contains(desc, "<ul")
row["has_h1_h2_p_ul"] = hasTags
row["llm_formula_ok"] = cerr == nil && hasTags && strings.EqualFold(fr, "stop")
if row["llm_formula_ok"].(bool) {
okLLM++
}
}
results = append(results, row)
log.Printf("live enhance ean=%s finish_reason=%s tokens=%d/%d/%d elapsed=%s parsed=%v formula_ok=%v err=%v",
c.ean, fr, comp.PromptTokens, comp.OutputTokens, comp.TotalTokens, elapsed, obj != nil, row["llm_formula_ok"], row["err"])
}
summary := map[string]any{
"at_utc": time.Now().UTC().Format(time.RFC3339),
"base": client.BaseURL,
"model": client.Model,
"mode": mode,
"byok": byok,
"max_tokens_enhance": processing.MaxTokensEnhance,
"max_tokens_retry": processing.MaxTokensEnhanceRetry,
"products": len(cases),
"llm_formula_ok_count": okLLM,
"live_llm_formula_pass": okLLM == len(cases),
"results": results,
}
raw, _ := json.MarshalIndent(summary, "", " ")
_ = os.WriteFile(filepath.Join(out, "summary.json"), raw, 0o644)
fmt.Println(string(raw))
if okLLM != len(cases) {
os.Exit(2)
}
}
func truncate(s string, n int) string {
r := []rune(strings.TrimSpace(s))
if len(r) <= n {
return string(r)
}
return string(r[:n]) + "…"
}
@@ -0,0 +1,443 @@
package main
import (
"context"
"encoding/json"
"fmt"
"io"
"log"
"net/http"
"os"
"os/signal"
"path/filepath"
"strings"
"syscall"
"time"
"github.com/descrybe/descrybe-v2/apps/api/internal/aiprompts"
"github.com/descrybe/descrybe-v2/apps/api/internal/aiprovider"
"github.com/descrybe/descrybe-v2/apps/api/internal/config"
"github.com/descrybe/descrybe-v2/apps/api/internal/db"
"github.com/descrybe/descrybe-v2/apps/api/internal/eprel"
"github.com/descrybe/descrybe-v2/apps/api/internal/logredact"
"github.com/descrybe/descrybe-v2/apps/api/internal/platformsettings"
"github.com/descrybe/descrybe-v2/apps/api/internal/processing"
"github.com/google/uuid"
)
func main() {
out := os.Getenv("PROBE_OUT_DIR")
if out == "" {
out = `F:\laragon\www\_MY\descrybe-v2\.codehelper\_live_llm_formula_20260816`
}
_ = os.MkdirAll(out, 0o755)
logPath := filepath.Join(out, "inproc_run.log")
logFile, err := os.Create(logPath)
if err != nil {
log.Fatalf("log file: %v", err)
}
defer logFile.Close()
log.SetOutput(logredact.Writer(io.MultiWriter(os.Stderr, logFile)))
// 5 rich A1 description formulas (h1/h2/p/ul) — Platform Demo.
eans := []string{
"195348253666", // 7 Gaming monitorji
"4548736132597", // 48 Slušalke
"3045380024786", // 36 Pečice
"3838782459856", // 38 Pomivalni stroji
"1200130000638", // 3 Bluetooth zvočniki
}
companyID := uuid.MustParse("2b3159b0-fc08-415b-b248-35ed02a6baab")
userID := uuid.MustParse("0ce3305c-d810-4b56-b1d4-3c1ed510db76")
cfg, err := config.Load()
if err != nil {
log.Fatalf("config: %v", err)
}
ctx, cancel := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
defer cancel()
pool, err := db.NewPool(ctx, cfg.DatabaseURL, db.PoolOptions{
MaxConns: int32(cfg.DBMaxConns),
MinConns: int32(cfg.DBMinConns),
MaxConnLifetime: cfg.DBMaxConnLifetime,
MaxConnLifetimeJitter: cfg.DBMaxConnLifetimeJitter,
MaxConnIdleTime: cfg.DBMaxConnIdleTime,
HealthCheckPeriod: cfg.DBHealthCheckPeriod,
StatementTimeout: cfg.DBStatementTimeout,
})
if err != nil {
log.Fatalf("db: %v", err)
}
defer pool.Close()
platEnv := platformsettings.EnvConfig{
AppEncryptionKey: cfg.AppEncryptionKey,
CredentialsEncryptionKey: cfg.CredentialsEncryptionKey,
TokenSigningSecret: cfg.TokenSigningSecret,
DatabaseURL: cfg.DatabaseURL,
OpenAIAPIKey: cfg.OpenAIAPIKey,
OpenAIBaseURL: cfg.OpenAIBaseURL,
OpenAIModel: cfg.OpenAIModel,
OpenAIEmbeddingAPIKey: cfg.OpenAIEmbeddingAPIKey,
OpenAIEmbeddingBaseURL: cfg.OpenAIEmbeddingBaseURL,
OpenAIEmbeddingModel: cfg.OpenAIEmbeddingModel,
}
platSettings := platformsettings.NewService(pool, platEnv)
aiSvc := aiprovider.NewService(pool, aiprovider.EnvConfig{
AppEncryptionKey: cfg.AppEncryptionKey,
CredentialsEncryptionKey: cfg.CredentialsEncryptionKey,
TokenSigningSecret: cfg.TokenSigningSecret,
DatabaseURL: cfg.DatabaseURL,
OpenAIAPIKey: cfg.OpenAIAPIKey,
OpenAIBaseURL: cfg.OpenAIBaseURL,
OpenAIModel: cfg.OpenAIModel,
ProcessingRPM: cfg.ProcessingRPM,
ProcessingMaxRetries: cfg.ProcessingMaxRetries,
})
aiSvc.Platform = platSettings
// Resolve like worker (DB admin AI / company BYOK preferred over mock .env).
roleCfg, rerr := platSettings.ResolveAIConfig(ctx, platformsettings.AIRoleProcessing)
if rerr != nil {
failResolve(out, fmt.Sprintf("ResolveAIConfig: %v", rerr))
}
completer, modeLabel, byok, cerr := aiSvc.ResolveCompleterForRole(ctx, companyID, aiprovider.RoleProcessing)
if cerr != nil {
failResolve(out, fmt.Sprintf("ResolveCompleterForRole: %v", cerr))
}
if completer == nil {
failResolve(out, "ResolveCompleterForRole returned nil completer (OpenAI not configured in admin/BYOK/env)")
}
client, ok := completer.(*processing.OpenAIClient)
if !ok {
failResolve(out, fmt.Sprintf("completer type %T is not *OpenAIClient", completer))
}
base := strings.TrimSpace(client.BaseURL)
model := strings.TrimSpace(client.Model)
resolveInfo := map[string]any{
"role_source": roleCfg.Source,
"role_provider": roleCfg.Provider,
"role_base_url": roleCfg.BaseURL,
"role_model": roleCfg.Model,
"role_enabled": roleCfg.Enabled,
"completer_base": base,
"completer_model": model,
"mode_label": modeLabel,
"byok": byok,
"env_base_url": cfg.OpenAIBaseURL,
"env_model": cfg.OpenAIModel,
"live": true,
}
if processing.IsMockOrLoopbackBaseURL(base) || strings.Contains(strings.ToLower(base), "18767") {
resolveInfo["live"] = false
resolveInfo["fail_reason"] = "resolved completer is mock/loopback — refuse silent mock fallback"
writeJSON(out, "resolve.json", resolveInfo)
log.Fatalf("LIVE LLM REQUIRED: resolved base=%s model=%s source=%s mode=%s — refuse mock :18767 / loopback",
base, model, roleCfg.Source, modeLabel)
}
if strings.TrimSpace(client.APIKey) == "" {
resolveInfo["live"] = false
resolveInfo["fail_reason"] = "empty API key"
writeJSON(out, "resolve.json", resolveInfo)
log.Fatal("LIVE LLM REQUIRED: API key empty after resolve")
}
writeJSON(out, "resolve.json", resolveInfo)
log.Printf("live resolve source=%s base=%s model=%s mode=%s byok=%v (env was base=%s model=%s)",
roleCfg.Source, base, model, modeLabel, byok, cfg.OpenAIBaseURL, cfg.OpenAIModel)
// Connectivity probe — GET /v1/models then tiny chat (admin DB key; never log full key).
modelsURL := strings.TrimRight(base, "/") + "/models"
keyHint := ""
if k := strings.TrimSpace(client.APIKey); len(k) > 8 {
keyHint = k[:4] + "…" + k[len(k)-4:]
} else if k := strings.TrimSpace(client.APIKey); k != "" {
keyHint = "(short)"
}
resolveInfo["api_key_hint"] = keyHint
resolveInfo["retried_at_utc"] = time.Now().UTC().Format(time.RFC3339)
modelsCtx, modelsCancel := context.WithTimeout(ctx, 30*time.Second)
defer modelsCancel()
modelsReq, merr := http.NewRequestWithContext(modelsCtx, http.MethodGet, modelsURL, nil)
if merr != nil {
resolveInfo["probe_ok"] = false
resolveInfo["fail_reason"] = "models request build: " + merr.Error()
writeJSON(out, "resolve.json", resolveInfo)
log.Fatalf("LIVE LLM CONNECTION ERROR: models req: %v", merr)
}
modelsReq.Header.Set("Authorization", "Bearer "+client.APIKey)
modelsReq.Header.Set("Accept", "application/json")
modelsStart := time.Now()
modelsResp, merr := http.DefaultClient.Do(modelsReq)
modelsElapsed := time.Since(modelsStart).Round(time.Millisecond)
if merr != nil {
resolveInfo["probe_ok"] = false
resolveInfo["models_url"] = modelsURL
resolveInfo["models_error"] = processing.TruncateError(merr)
resolveInfo["models_elapsed"] = modelsElapsed.String()
resolveInfo["fail_reason"] = "GET /v1/models network error"
writeJSON(out, "resolve.json", resolveInfo)
log.Fatalf("LIVE LLM CONNECTION ERROR: GET %s elapsed=%s err=%s",
modelsURL, modelsElapsed, processing.TruncateError(merr))
}
modelsBody, _ := io.ReadAll(io.LimitReader(modelsResp.Body, 2048))
_ = modelsResp.Body.Close()
modelsSnippet := strings.TrimSpace(string(modelsBody))
if len(modelsSnippet) > 240 {
modelsSnippet = modelsSnippet[:240] + "…"
}
resolveInfo["models_url"] = modelsURL
resolveInfo["models_http_status"] = modelsResp.StatusCode
resolveInfo["models_content_type"] = modelsResp.Header.Get("Content-Type")
resolveInfo["models_elapsed"] = modelsElapsed.String()
resolveInfo["models_body_snippet"] = modelsSnippet
if modelsResp.StatusCode != http.StatusOK {
resolveInfo["probe_ok"] = false
resolveInfo["http_status"] = modelsResp.StatusCode
resolveInfo["fail_reason"] = fmt.Sprintf("GET /v1/models returned HTTP %d (expect 200; 401=wrong/missing key, 502=upstream)", modelsResp.StatusCode)
writeJSON(out, "resolve.json", resolveInfo)
log.Fatalf("LIVE LLM CONNECTION ERROR: GET %s status=%d ct=%s snippet=%q",
modelsURL, modelsResp.StatusCode, modelsResp.Header.Get("Content-Type"), modelsSnippet)
}
log.Printf("live models ok status=200 elapsed=%s key_hint=%s", modelsElapsed, keyHint)
probeCtx, probeCancel := context.WithTimeout(ctx, 45*time.Second)
defer probeCancel()
probeStart := time.Now()
comp, perr := client.Complete(probeCtx, "Reply with exactly: PONG", "Say PONG")
probeElapsed := time.Since(probeStart).Round(time.Millisecond)
if perr != nil {
resolveInfo["probe_ok"] = false
resolveInfo["probe_error"] = processing.TruncateError(perr)
resolveInfo["probe_elapsed"] = probeElapsed.String()
resolveInfo["fail_reason"] = "chat/completions probe failed after models 200"
writeJSON(out, "resolve.json", resolveInfo)
log.Fatalf("LIVE LLM CONNECTION ERROR: base=%s model=%s elapsed=%s err=%s",
base, model, probeElapsed, processing.TruncateError(perr))
}
resolveInfo["probe_ok"] = true
resolveInfo["probe_elapsed"] = probeElapsed.String()
resolveInfo["probe_response_len"] = len(comp.Text)
writeJSON(out, "resolve.json", resolveInfo)
log.Printf("live probe ok elapsed=%s response_len=%d", probeElapsed, len(comp.Text))
pipeline := processing.NewPipeline(pool)
pipeline.BatchSize = cfg.ProcessingBatchSize
pipeline.AI = aiSvc
pipeline.Prompts = aiprompts.NewService(pool)
eprelClient := eprel.NewClient(eprel.Options{Enabled: true, Timeout: 20 * time.Second})
pipeline.Engine = &processing.Engine{
Completer: nil, // per-job via pipeline.AI
Vector: processing.NoopVectorCategorizer{},
EPREL: eprelClient,
ProviderMode: processing.AIProviderInternal,
}
rawIDs := make([]uuid.UUID, 0, len(eans))
for _, ean := range eans {
var id uuid.UUID
err := pool.QueryRow(ctx, `
SELECT id FROM raw_products
WHERE company_id = $1 AND gtin = $2`, companyID, ean).Scan(&id)
if err != nil {
log.Fatalf("raw product %s: %v", ean, err)
}
rawIDs = append(rawIDs, id)
log.Printf("ean=%s raw_id=%s", ean, id)
}
// Force re-enhance: clear prior enhance_input_hash for these raw products.
tag, cerr2 := pool.Exec(ctx, `
UPDATE processed_products pp
SET field_sources = COALESCE(field_sources, '{}'::jsonb) - 'enhance_input_hash',
localized = CASE
WHEN localized IS NULL OR localized = '{}'::jsonb THEN localized
ELSE (
SELECT COALESCE(jsonb_object_agg(lang, val - 'enhance_input_hash'), '{}'::jsonb)
FROM jsonb_each(localized) AS t(lang, val)
)
END,
updated_at = now()
FROM raw_products rp
WHERE pp.company_id = $1
AND pp.raw_product_id = rp.id
AND rp.gtin = ANY($2::text[])`, companyID, eans)
if cerr2 != nil {
log.Printf("clear enhance hash: %v", cerr2)
} else {
log.Printf("cleared enhance_input_hash rows=%d", tag.RowsAffected())
}
if n, err := processing.BackfillMissingMeta(ctx, pool, companyID); err != nil {
log.Printf("BackfillMissingMeta: %v", err)
} else {
log.Printf("BackfillMissingMeta updated=%d", n)
}
// Soft-yield competing queue noise (do not mass-fail the whole company queue).
_, _ = pool.Exec(ctx, `
UPDATE processing_jobs
SET status = 'failed', error = 'yielded to live LLM formula probe', updated_at = now()
WHERE company_id = $1 AND status IN ('pending','running','processing')`, companyID)
jobs, err := pipeline.StartJob(ctx, companyID, userID, rawIDs, "full")
if err != nil {
log.Fatalf("StartJob: %v", err)
}
if len(jobs) == 0 {
log.Fatal("StartJob returned no jobs")
}
jobID := jobs[0].ID
log.Printf("job_id=%s chunks=%d", jobID, len(jobs))
_ = os.WriteFile(filepath.Join(out, "process_id.txt"), []byte(jobID.String()+"\n"), 0o644)
defend := make(chan struct{})
go func() {
t := time.NewTicker(500 * time.Millisecond)
defer t.Stop()
for {
select {
case <-defend:
return
case <-ctx.Done():
return
case <-t.C:
_, _ = pool.Exec(context.Background(), `
UPDATE processing_jobs
SET status = 'failed', error = 'yielded to live LLM formula probe', updated_at = now()
WHERE company_id = $1 AND status IN ('pending','running','processing') AND id <> $2`, companyID, jobID)
_, _ = pool.Exec(context.Background(), `
UPDATE processing_jobs
SET status = 'running', error = NULL, updated_at = now()
WHERE id = $1 AND status IN ('failed','cancelled')
AND (error ILIKE '%P0%' OR error ILIKE '%parked%' OR error ILIKE '%yield%'
OR error ILIKE '%reclaim%' OR error ILIKE '%cleared%' OR error ILIKE '%superseded%'
OR error ILIKE '%exclusive%' OR error ILIKE '%batch_%' OR error ILIKE '%inproc%'
OR error ILIKE '%formula%' OR error ILIKE '%live LLM%')`, jobID)
_, _ = pool.Exec(context.Background(), `
UPDATE processing_job_products
SET status = 'pending', error = NULL, updated_at = now()
WHERE job_id = $1 AND status IN ('failed','cancelled')
AND processed_product_id IS NULL
AND (error ILIKE '%P0%' OR error ILIKE '%parked%' OR error ILIKE '%yield%'
OR error ILIKE '%reclaim%' OR error ILIKE '%cleared%' OR error ILIKE '%superseded%'
OR error ILIKE '%exclusive%' OR error ILIKE '%batch_%' OR error ILIKE '%inproc%'
OR error ILIKE '%formula%' OR error ILIKE '%live LLM%')`, jobID)
}
}
}()
_, _ = pool.Exec(ctx, `
UPDATE processing_jobs
SET status = 'running', started_at = COALESCE(started_at, now()), updated_at = now()
WHERE id = $1`, jobID)
runCtx, runCancel := context.WithTimeout(ctx, 20*time.Minute)
defer runCancel()
if err := pipeline.ProcessJob(runCtx, jobID); err != nil {
close(defend)
log.Fatalf("ProcessJob: %v", err)
}
close(defend)
items, err := pipeline.LoadV1ProcessJobItems(ctx, companyID, jobID, "full")
if err != nil {
log.Fatalf("LoadV1ProcessJobItems: %v", err)
}
payload := map[string]any{
"data": map[string]any{
"process_id": jobID.String(),
"status": "COMPLETED",
"processing_type": "full",
"total_items": len(items),
"items": items,
"llm": map[string]any{
"base": base,
"model": model,
"source": roleCfg.Source,
"mode": modeLabel,
"byok": byok,
},
},
}
writeJSON(out, "final.json", payload)
// DB HTML descriptions (V1 may strip tags).
rows, err := pool.Query(ctx, `
SELECT rp.gtin,
COALESCE(pp.processed_name, pp.name, ''),
COALESCE(NULLIF(pp.processed_description, ''), pp.description, ''),
COALESCE(pp.category, ''),
COALESCE(c.name, ''),
COALESCE(c.unique_id, ''),
c.description_template,
pp.id
FROM processing_job_products pjp
JOIN raw_products rp ON rp.id = pjp.raw_product_id
LEFT JOIN processed_products pp ON pp.id = pjp.processed_product_id
LEFT JOIN categories c ON c.company_id = pp.company_id AND c.unique_id = pp.category
WHERE pjp.job_id = $1
ORDER BY rp.gtin`, jobID)
if err != nil {
log.Fatalf("db products: %v", err)
}
defer rows.Close()
dbProducts := make([]map[string]any, 0)
for rows.Next() {
var gtin, name, desc, cat, catName, catUID string
var tmpl any
var ppID *uuid.UUID
if err := rows.Scan(&gtin, &name, &desc, &cat, &catName, &catUID, &tmpl, &ppID); err != nil {
log.Fatal(err)
}
idStr := ""
if ppID != nil {
idStr = ppID.String()
}
dbProducts = append(dbProducts, map[string]any{
"ean": gtin,
"product_id": idStr,
"title": name,
"description": desc,
"category": cat,
"category_name": catName,
"category_unique_id": catUID,
"description_template": tmpl,
"formula_constraint": processing.FormatDescriptionFormulaConstraint(tmpl),
})
}
writeJSON(out, "db_products.json", dbProducts)
fmt.Printf("JOB %s items=%d live_base=%s model=%s\n", jobID, len(items), base, model)
for _, p := range dbProducts {
ean, _ := p["ean"].(string)
title, _ := p["title"].(string)
cat, _ := p["category"].(string)
catn, _ := p["category_name"].(string)
desc, _ := p["description"].(string)
if len(title) > 50 {
title = title[:50]
}
fmt.Printf("ITEM %s cat=%s/%s desc_len=%d title=%s\n", ean, cat, catn, len(desc), title)
}
}
func failResolve(out, msg string) {
writeJSON(out, "resolve.json", map[string]any{
"live": false,
"fail_reason": msg,
})
log.Fatalf("LIVE LLM REQUIRED: %s", msg)
}
func writeJSON(out, name string, v any) {
raw, err := json.MarshalIndent(v, "", " ")
if err != nil {
log.Fatalf("marshal %s: %v", name, err)
}
if err := os.WriteFile(filepath.Join(out, name), raw, 0o644); err != nil {
log.Fatalf("write %s: %v", name, err)
}
}
@@ -23,7 +23,8 @@ type RepairWeakEnhanceHashesResult struct {
// RepairWeakEnhanceHashes clears enhance_input_hash from field_sources and // RepairWeakEnhanceHashes clears enhance_input_hash from field_sources and
// localized_content for processed products whose descriptions are weak / // localized_content for processed products whose descriptions are weak /
// title-echo / filler (so the next enhance cannot hash-skip thin priors). // title-echo / filler / invent-synthesize (so the next enhance cannot hash-skip
// thin or heuristic priors).
// Returns the number of products updated. // Returns the number of products updated.
func RepairWeakEnhanceHashes(ctx context.Context, pool *pgxpool.Pool, companyID uuid.UUID) (int64, error) { func RepairWeakEnhanceHashes(ctx context.Context, pool *pgxpool.Pool, companyID uuid.UUID) (int64, error) {
res, err := RepairWeakEnhanceHashesDetailed(ctx, pool, companyID) res, err := RepairWeakEnhanceHashesDetailed(ctx, pool, companyID)
@@ -124,7 +125,7 @@ func repairWeakEnhanceHashesTx(ctx context.Context, tx pgx.Tx, companyID uuid.UU
} }
if _, has := fs[enhanceInputHashKey]; has { if _, has := fs[enhanceInputHashKey]; has {
if company.IsWeakPriorEnhanceDescription(primaryDesc, primaryName, name) { if company.ShouldRefuseEnhanceHashSkip(primaryDesc, primaryName, name) {
delete(fs, enhanceInputHashKey) delete(fs, enhanceInputHashKey)
changed = true changed = true
} }
@@ -139,7 +140,7 @@ func repairWeakEnhanceHashesTx(ctx context.Context, tx pgx.Tx, companyID uuid.UU
if strings.TrimSpace(fields.EnhanceInputHash) == "" { if strings.TrimSpace(fields.EnhanceInputHash) == "" {
continue continue
} }
if company.IsWeakPriorEnhanceDescription(d, n, primaryName, name) { if company.ShouldRefuseEnhanceHashSkip(d, n, primaryName, name) {
fields.EnhanceInputHash = "" fields.EnhanceInputHash = ""
loc[lang] = fields loc[lang] = fields
changed = true changed = true
+5 -5
View File
@@ -950,7 +950,7 @@ func (s *Service) ListRawProducts(ctx context.Context, companyID uuid.UUID, f Li
SELECT rp.id, rp.gtin, rp.feed_id, rp.is_processed, rp.processing_status, SELECT rp.id, rp.gtin, rp.feed_id, rp.is_processed, rp.processing_status,
COALESCE(NULLIF(rp.mapped_data->>'name', ''), NULLIF(rp.mapped_data->>'title', ''), '') AS name, COALESCE(NULLIF(rp.mapped_data->>'name', ''), NULLIF(rp.mapped_data->>'title', ''), '') AS name,
NULLIF(BTRIM(rp.mapped_data->>'category'), '') AS category, NULLIF(BTRIM(rp.mapped_data->>'category'), '') AS category,
COALESCE(cat.name, NULLIF(BTRIM(rp.mapped_data->>'category'), '')) AS category_name, cat.name AS category_name,
COALESCE(cat.unique_id, NULLIF(BTRIM(rp.mapped_data->>'category'), '')) AS category_unique_id, COALESCE(cat.unique_id, NULLIF(BTRIM(rp.mapped_data->>'category'), '')) AS category_unique_id,
f.name AS feed_name, f.name AS feed_name,
rp.mapped_data->'_sync_changes' AS sync_changes, rp.mapped_data->'_sync_changes' AS sync_changes,
@@ -1099,7 +1099,7 @@ func (s *Service) ListProcessedProducts(ctx context.Context, companyID uuid.UUID
`+processedPreferredNameSQL+` AS name, `+processedPreferredNameSQL+` AS name,
COALESCE(NULLIF(p.processed_name, ''), '') AS processed_name, COALESCE(NULLIF(p.processed_name, ''), '') AS processed_name,
p.category, p.category,
COALESCE(cat.name, NULLIF(BTRIM(p.category), '')) AS category_name, cat.name AS category_name,
COALESCE(cat.unique_id, NULLIF(BTRIM(p.category), '')) AS category_unique_id, COALESCE(cat.unique_id, NULLIF(BTRIM(p.category), '')) AS category_unique_id,
p.status, p.raw_product_id, COALESCE(p.feed_id, r.feed_id) AS feed_id, r.gtin, p.status, p.raw_product_id, COALESCE(p.feed_id, r.feed_id) AS feed_id, r.gtin,
f.name AS feed_name, f.name AS feed_name,
@@ -1186,7 +1186,7 @@ func (s *Service) ListProcessedProductsDetailed(ctx context.Context, companyID u
SELECT p.id, p.product_id, SELECT p.id, p.product_id,
`+processedPreferredNameSQL+` AS name, `+processedPreferredNameSQL+` AS name,
p.category, p.category,
COALESCE(cat.name, NULLIF(BTRIM(p.category), '')) AS category_name, cat.name AS category_name,
COALESCE(cat.unique_id, NULLIF(BTRIM(p.category), '')) AS category_unique_id, COALESCE(cat.unique_id, NULLIF(BTRIM(p.category), '')) AS category_unique_id,
p.status, p.raw_product_id, p.feed_id, r.gtin, p.status, p.raw_product_id, p.feed_id, r.gtin,
`+processedPreferredDescriptionSQL+` AS description, `+processedPreferredDescriptionSQL+` AS description,
@@ -1226,7 +1226,7 @@ func (s *Service) GetProcessedProduct(ctx context.Context, companyID, id uuid.UU
SELECT p.id, p.product_id, SELECT p.id, p.product_id,
`+processedPreferredNameSQL+` AS name, `+processedPreferredNameSQL+` AS name,
p.category, p.category,
COALESCE(cat.name, NULLIF(BTRIM(p.category), '')) AS category_name, cat.name AS category_name,
COALESCE(cat.unique_id, NULLIF(BTRIM(p.category), '')) AS category_unique_id, COALESCE(cat.unique_id, NULLIF(BTRIM(p.category), '')) AS category_unique_id,
`+processedPreferredDescriptionSQL+` AS description, `+processedPreferredDescriptionSQL+` AS description,
p.processed_name, p.processed_description, p.processed_name, p.processed_description,
@@ -1279,7 +1279,7 @@ func (s *Service) GetRawProduct(ctx context.Context, companyID, id uuid.UUID) (m
COALESCE(NULLIF(rp.gtin, ''), '') AS product_id, COALESCE(NULLIF(rp.gtin, ''), '') AS product_id,
COALESCE(NULLIF(rp.mapped_data->>'name', ''), NULLIF(rp.mapped_data->>'title', ''), '') AS name, COALESCE(NULLIF(rp.mapped_data->>'name', ''), NULLIF(rp.mapped_data->>'title', ''), '') AS name,
NULLIF(BTRIM(rp.mapped_data->>'category'), '') AS category, NULLIF(BTRIM(rp.mapped_data->>'category'), '') AS category,
COALESCE(cat.name, NULLIF(BTRIM(rp.mapped_data->>'category'), '')) AS category_name, cat.name AS category_name,
COALESCE(cat.unique_id, NULLIF(BTRIM(rp.mapped_data->>'category'), '')) AS category_unique_id, COALESCE(cat.unique_id, NULLIF(BTRIM(rp.mapped_data->>'category'), '')) AS category_unique_id,
COALESCE(NULLIF(rp.mapped_data->>'description', ''), '') AS description, COALESCE(NULLIF(rp.mapped_data->>'description', ''), '') AS description,
''::text AS processed_name, ''::text AS processed_name,
+78 -5
View File
@@ -25,26 +25,99 @@ var weakFillerPhrases = []string{
// IsWeakPriorEnhanceDescription reports empty, too-short, title-echo, known filler, // IsWeakPriorEnhanceDescription reports empty, too-short, title-echo, known filler,
// or short boilerplate without overlapping tokens from title/fact sources. // or short boilerplate without overlapping tokens from title/fact sources.
// Heuristic synthesize (invent) is intentionally excluded — use ShouldRefuseEnhanceHashSkip.
func IsWeakPriorEnhanceDescription(priorDesc string, titles ...string) bool { func IsWeakPriorEnhanceDescription(priorDesc string, titles ...string) bool {
priorDesc = strings.TrimSpace(priorDesc) reason := EnhanceHashSkipBlockReason(priorDesc, titles...)
if priorDesc == "" || priorDesc == "<nil>" { return reason == "weak" || reason == "title-echo"
}
// heuristicSynthesizePhrases are distinctive invent / formula-skeleton snippets from
// synthesizeDescriptionFromTitle / synthesizeDescriptionFromFormula. These may look
// "strong" enough to pass IsWeakPriorEnhanceDescription but must never hash-skip
// enhance (would leave fallback copy forever on reprocess).
var heuristicSynthesizePhrases = []string{
"is a catalog product with the known attributes",
"is listed in the ",
". key specs:",
"je katalogski izdelek z znanimi atributi",
"je izdelek v kategoriji",
"je izdelek znamke",
". ključne specifikacije:",
}
// LooksLikeHeuristicSynthesize reports invent / formula-skeleton fallback copy.
func LooksLikeHeuristicSynthesize(desc string) bool {
lower := strings.ToLower(desc)
for _, p := range heuristicSynthesizePhrases {
if p != "" && strings.Contains(lower, p) {
return true return true
} }
if len([]rune(priorDesc)) < minUsableProductDescRunes { }
// EN invent: "<title> is a <category> product from <brand>"
if strings.Contains(lower, " is a ") && strings.Contains(lower, " product from ") {
return true return true
} }
return false
}
// EnhanceHashSkipBlockReason returns a stable reason when prior description must
// not hash-skip the enhance LLM: "weak", "title-echo", "synth", or "".
func EnhanceHashSkipBlockReason(priorDesc string, titles ...string) string {
priorDesc = strings.TrimSpace(priorDesc)
if priorDesc == "" || priorDesc == "<nil>" {
return "weak"
}
if len([]rune(priorDesc)) < minUsableProductDescRunes {
return "weak"
}
for _, title := range titles { for _, title := range titles {
if DescriptionEchoesTitle(priorDesc, title) { if DescriptionEchoesTitle(priorDesc, title) {
return true return "title-echo"
} }
} }
if ContainsWeakFillerPhrase(priorDesc) { if ContainsWeakFillerPhrase(priorDesc) {
return true return "weak"
}
if LooksLikeHeuristicSynthesize(priorDesc) {
return "synth"
} }
if len([]rune(priorDesc)) <= shortBoilerplateDescRunes && if len([]rune(priorDesc)) <= shortBoilerplateDescRunes &&
!descriptionOverlapsProductFacts(priorDesc, titles...) { !descriptionOverlapsProductFacts(priorDesc, titles...) {
return "weak"
}
return ""
}
// ShouldRefuseEnhanceHashSkip is true when a stored enhance_input_hash must be
// ignored / cleared (weak, title-echo, or heuristic synthesize).
func ShouldRefuseEnhanceHashSkip(priorDesc string, titles ...string) bool {
return EnhanceHashSkipBlockReason(priorDesc, titles...) != ""
}
// DescriptionMissingFormulaHTMLTags is true when sectionTypes require HTML tags
// (h1/h2/h3/h4, p, ul) that are absent from desc — formula-mismatch for skip/clear.
func DescriptionMissingFormulaHTMLTags(desc string, sectionTypes []string) bool {
if len(sectionTypes) == 0 {
return false
}
lower := strings.ToLower(desc)
for _, raw := range sectionTypes {
typ := strings.ToLower(strings.TrimSpace(raw))
switch typ {
case "h1", "h2", "h3", "h4":
if !strings.Contains(lower, "<"+typ) {
return true return true
} }
case "ul":
if !strings.Contains(lower, "<ul") {
return true
}
case "p", "":
if !strings.Contains(lower, "<p") {
return true
}
}
}
return false return false
} }
@@ -0,0 +1,40 @@
package company
import "testing"
func TestLooksLikeHeuristicSynthesize(t *testing.T) {
t.Parallel()
title := "Vogel's WALL 3245 TV Wall Mount"
en := title + " is a TV Mounts product from Vogel's. Key specs: width 45 cm, max_load 40 kg."
if !LooksLikeHeuristicSynthesize(en) {
t.Fatalf("expected EN invent detected: %q", en)
}
sl := title + " je izdelek v kategoriji TV Mounts znamke Vogel's. Ključne specifikacije: width 45 cm."
if !LooksLikeHeuristicSynthesize(sl) {
t.Fatalf("expected SL invent detected: %q", sl)
}
good := "Vogel's WALL 3245 is a sturdy TV mount with clear load ratings and install guidance for wall displays."
if LooksLikeHeuristicSynthesize(good) {
t.Fatalf("retail copy must not look like invent: %q", good)
}
if !ShouldRefuseEnhanceHashSkip(en, title) {
t.Fatal("invent must refuse hash skip")
}
if ShouldRefuseEnhanceHashSkip(good, title) {
t.Fatal("good retail copy must allow hash skip")
}
if !ShouldRefuseEnhanceHashSkip(title, title) {
t.Fatal("title-echo must refuse hash skip")
}
}
func TestDescriptionMissingFormulaHTMLTags(t *testing.T) {
t.Parallel()
types := []string{"h1", "p", "ul"}
if !DescriptionMissingFormulaHTMLTags("plain prose without tags", types) {
t.Fatal("plain prose must mismatch")
}
if DescriptionMissingFormulaHTMLTags("<h1>T</h1><p>Body</p><ul><li>x</li></ul>", types) {
t.Fatal("full HTML must match")
}
}
+8
View File
@@ -112,6 +112,14 @@ type ProductInput struct {
PriorProcessedDescription string PriorProcessedDescription string
PriorCategory string PriorCategory string
PriorLocalized company.LocalizedContent PriorLocalized company.LocalizedContent
// JobID / CompanyID / RawProductID are optional worker correlation ids for
// structured ai_enhance logs (never secrets). Empty in unit tests.
JobID string
CompanyID string
RawProductID string
// CategoryUniqueID is the taxonomy unique_id for overlay/formula keys when
// the enhance display category argument is a localized name.
CategoryUniqueID string
} }
// CategoryFormulas holds optional title/description templates for one category key. // CategoryFormulas holds optional title/description templates for one category key.
+3 -3
View File
@@ -356,12 +356,12 @@ func RecommendReprocessRawProductIDs(ctx context.Context, pool *pgxpool.Pool, co
if isPromptLabelTitle(primaryName) || isPromptLabelTitle(processedName) || isPromptLabelTitle(name) { if isPromptLabelTitle(primaryName) || isPromptLabelTitle(processedName) || isPromptLabelTitle(name) {
needs = true needs = true
} }
if isWeakPriorEnhanceDescription(primaryDesc, primaryName, name) { if company.ShouldRefuseEnhanceHashSkip(primaryDesc, primaryName, name) {
needs = true needs = true
} }
if _, has := fs[FieldEnhanceInputHash]; !has { if _, has := fs[FieldEnhanceInputHash]; !has {
// Missing hash after weak clear / never enhanced — recommend when desc weak or empty category. // Missing hash after weak clear / never enhanced — recommend when desc weak or empty category.
if isWeakPriorEnhanceDescription(primaryDesc, primaryName, name) || strings.TrimSpace(category) == "" { if company.ShouldRefuseEnhanceHashSkip(primaryDesc, primaryName, name) || strings.TrimSpace(category) == "" {
needs = true needs = true
} }
} }
@@ -371,7 +371,7 @@ func RecommendReprocessRawProductIDs(ctx context.Context, pool *pgxpool.Pool, co
if n == "" { if n == "" {
n = primaryName n = primaryName
} }
if fields.EnhanceInputHash == "" && isWeakPriorEnhanceDescription(d, n, primaryName, name) { if fields.EnhanceInputHash == "" && company.ShouldRefuseEnhanceHashSkip(d, n, primaryName, name) {
needs = true needs = true
break break
} }
+31 -5
View File
@@ -7,13 +7,21 @@ import (
) )
// categoryDisplayLabel returns the human category name for meta / synthesize / // categoryDisplayLabel returns the human category name for meta / synthesize /
// {{category}}. Prefers CategoryName; never falls back to a digits-only unique_id. // {{category}}. Prefers taxonomy CategoryName; never falls back to a digits-only
// unique_id, prompt/formula leakage, or the product title itself.
func categoryDisplayLabel(result StepResult) string { func categoryDisplayLabel(result StepResult) string {
if n := strings.TrimSpace(result.CategoryName); n != "" { if n := strings.TrimSpace(result.CategoryName); n != "" {
// CategoryName comes from categories.name — trust unless prompt leakage.
if !isPromptLabelTitle(n) {
return n return n
} }
}
cat := strings.TrimSpace(result.Category) cat := strings.TrimSpace(result.Category)
if cat == "" || isDigitsOnlyCategoryID(cat) { if cat == "" || isDigitsOnlyCategoryID(cat) || isPromptLabelTitle(cat) {
return ""
}
// Unresolved non-uid token: never surface the product name as {{category}}.
if categoryTokenEqualsProductTitle(cat, result.ProcessedName, result.Name) {
return "" return ""
} }
return cat return cat
@@ -109,6 +117,10 @@ func normalizeCategoryUniqueID(s string) string {
if s == "" || s == "<nil>" || strings.EqualFold(s, "none") { if s == "" || s == "<nil>" || strings.EqualFold(s, "none") {
return "" return ""
} }
// Never treat enhance-prompt / formula scaffolding as a category unique_id.
if isPromptLabelTitle(s) {
return ""
}
return s return s
} }
@@ -155,7 +167,10 @@ func applyCategoryFromMapped(out *StepResult, maps ...map[string]any) {
return return
} }
cat := categoryUniqueIDFromMaps(maps...) cat := categoryUniqueIDFromMaps(maps...)
if cat == "" { // Prompt/formula leakage never becomes Category. Product-title equals are
// scrubbed later (scrubCategoryPollution) so a title that matches a real
// taxonomy display name can still coerce to unique_id.
if cat == "" || isPromptLabelTitle(cat) {
return return
} }
out.Category = SanitizeText(cat) out.Category = SanitizeText(cat)
@@ -213,6 +228,15 @@ func coerceCategoryToCompanyUniqueID(out *StepResult, namesByUID map[string]stri
if cat == "" { if cat == "" {
return return
} }
if isPromptLabelTitle(cat) {
out.Category = ""
out.CategoryName = ""
if out.FieldSources == nil {
out.FieldSources = map[string]any{}
}
out.FieldSources["category"] = "cleared_invalid"
return
}
resolved := resolveCompanyCategoryUniqueID(cat, namesByUID, valid) resolved := resolveCompanyCategoryUniqueID(cat, namesByUID, valid)
if resolved == "" || resolved == cat { if resolved == "" || resolved == cat {
return return
@@ -241,8 +265,10 @@ func filterCategoryIfInvalid(out *StepResult, valid map[string]struct{}) {
return return
} }
out.Category = "" out.Category = ""
if out.FieldSources != nil { out.CategoryName = ""
delete(out.FieldSources, "category") if out.FieldSources == nil {
out.FieldSources = map[string]any{}
} }
out.FieldSources["category"] = "cleared_invalid"
out.Notes = append(out.Notes, "category: ignored (unknown unique_id for company)") out.Notes = append(out.Notes, "category: ignored (unknown unique_id for company)")
} }
@@ -133,6 +133,9 @@ func TestFilterCategoryIfInvalid(t *testing.T) {
if out.Category != "" { if out.Category != "" {
t.Fatalf("invalid unique_id kept: %q", out.Category) t.Fatalf("invalid unique_id kept: %q", out.Category)
} }
if src, _ := out.FieldSources["category"].(string); src != "cleared_invalid" {
t.Fatalf("field_sources.category=%v want cleared_invalid", out.FieldSources["category"])
}
out.Category = "50" out.Category = "50"
filterCategoryIfInvalid(&out, nil) filterCategoryIfInvalid(&out, nil)
if out.Category != "50" { if out.Category != "50" {
@@ -179,6 +182,76 @@ func TestCoerceCategoryToCompanyUniqueID_nameToUID(t *testing.T) {
if out.Category != "" { if out.Category != "" {
t.Fatalf("unknown should clear: %q", out.Category) t.Fatalf("unknown should clear: %q", out.Category)
} }
if src, _ := out.FieldSources["category"].(string); src != "cleared_invalid" {
t.Fatalf("field_sources.category=%v want cleared_invalid", out.FieldSources["category"])
}
}
// Product titles (and nested category.name copies of the title) must never land
// in Category — only taxonomy unique_ids (+ display names from categories).
func TestRunSteps_productTitleNeverBecomesCategory(t *testing.T) {
t.Parallel()
const productTitle = "Bosch Serie 6 WAU28PH0BY 9kg White Washing Machine"
names := map[string]string{"50": "Pralni stroji", "28": "TV mounts"}
e := &Engine{Vector: NoopVectorCategorizer{}}
out, err := e.RunSteps(context.Background(), "co", ProductInput{
GTIN: "4242005191234",
Name: productTitle,
Mapped: map[string]any{
"name": productTitle,
"category": map[string]any{
"name": productTitle, // feed pollution: product title as category.name
},
},
CategoryNamesByUID: names,
}, "normalize_only", nil, StepPolicy{})
if err != nil {
t.Fatal(err)
}
if out.Category == productTitle || strings.EqualFold(out.Category, productTitle) {
t.Fatalf("Category must not be product title: %q", out.Category)
}
if out.CategoryName == productTitle || strings.EqualFold(out.CategoryName, productTitle) {
t.Fatalf("CategoryName must not be product title: %q", out.CategoryName)
}
if cat := categoryDisplayLabel(out); cat == productTitle || strings.EqualFold(cat, productTitle) {
t.Fatalf("categoryDisplayLabel leaked product title: %q", cat)
}
if out.Category != "" {
t.Fatalf("Category=%q want empty (unresolved product-title token scrubbed)", out.Category)
}
// Plain string category == product title must also be scrubbed.
out2, err := e.RunSteps(context.Background(), "co", ProductInput{
GTIN: "4242005191235",
Name: productTitle,
Mapped: map[string]any{
"name": productTitle,
"category": productTitle,
},
CategoryNamesByUID: names,
}, "normalize_only", nil, StepPolicy{})
if err != nil {
t.Fatal(err)
}
if out2.Category != "" {
t.Fatalf("plain title-as-category kept: %q", out2.Category)
}
// Real taxonomy display name still coerces via processOne helpers.
out3 := StepResult{Category: "Pralni stroji", Name: productTitle, ProcessedName: productTitle}
valid := map[string]struct{}{"50": {}, "28": {}}
coerceCategoryToCompanyUniqueID(&out3, names, valid)
filterCategoryIfInvalid(&out3, valid)
scrubCategoryPollution(&out3, names, valid)
syncCategoryName(&out3, names)
if out3.Category != "50" {
t.Fatalf("taxonomy name coerce: Category=%q want 50", out3.Category)
}
if out3.CategoryName != "Pralni stroji" {
t.Fatalf("CategoryName=%q want Pralni stroji", out3.CategoryName)
}
} }
func TestResolveCompanyCategoryUniqueID_usedByV1Projection(t *testing.T) { func TestResolveCompanyCategoryUniqueID_usedByV1Projection(t *testing.T) {
@@ -63,3 +63,12 @@ func enhanceStatusFromMeta(raw any) string {
s, _ := m["status"].(string) s, _ := m["status"].(string)
return s return s
} }
func enhanceReasonFromMeta(raw any) string {
m, ok := raw.(map[string]any)
if !ok {
return ""
}
s, _ := m["reason"].(string)
return s
}
@@ -97,7 +97,7 @@ func TestRunSteps_skipsEnhanceWhenInputHashUnchanged(t *testing.T) {
t.Fatalf("preferKnownProviderMode(unknown, internal)=%q", got) t.Fatalf("preferKnownProviderMode(unknown, internal)=%q", got)
} }
joined := strings.Join(out.Notes, ";") joined := strings.Join(out.Notes, ";")
if !strings.Contains(joined, "unchanged") { if !strings.Contains(joined, "ai_enhance_unchanged") {
t.Fatalf("notes=%v", out.Notes) t.Fatalf("notes=%v", out.Notes)
} }
} }
@@ -279,6 +279,195 @@ func TestRunSteps_synthesizedDescDoesNotPersistEnhanceHash(t *testing.T) {
} }
} }
func TestRunSteps_formulaIgnoredByLLM_retriesThenSynthesizes(t *testing.T) {
calls := 0
var sawFormulaRetry bool
e := &Engine{
Completer: stubCompleter{fn: func(system, user string) (Completion, error) {
calls++
if strings.Contains(user, "INVALID DESCRIPTION") {
sawFormulaRetry = true
}
// Non-weak short prose that ignores multi-section HTML formula.
return Completion{
Text: `{"name":"GIGABYTE GS27QC","description":"GIGABYTE GS27QC is a gaming monitor with a 61 cm curved panel for immersive play."}`,
TotalTokens: 4,
}, nil
}},
Vector: NoopVectorCategorizer{},
}
descTpl := map[string]any{
"sections": []any{
map[string]any{"type": "h1", "instructions": "Heading"},
map[string]any{"type": "p", "instructions": "Body"},
map[string]any{"type": "ul", "instructions": "Specs"},
},
}
out, err := e.RunSteps(context.Background(), "co", ProductInput{
Name: "GIGABYTE GS27QC",
Mapped: map[string]any{"name": "GIGABYTE GS27QC", "category": "7", "brand": "GIGABYTE", "width": "61 cm"},
CategoryFormulasByKey: map[string]CategoryFormulas{
"7": {DescriptionTemplate: descTpl},
},
Language: "en",
}, "enhance_only", nil, StepPolicy{AllowAI: true, AllowEPREL: true})
if err != nil {
t.Fatal(err)
}
if calls < 2 {
t.Fatalf("expected formula retry after ignore, calls=%d", calls)
}
if !sawFormulaRetry {
t.Fatal("expected INVALID DESCRIPTION retry user suffix")
}
for _, tag := range []string{"<h1>", "<p>", "<ul>"} {
if !strings.Contains(out.ProcessedDescription, tag) {
t.Fatalf("expected formula HTML with %s, got %q", tag, out.ProcessedDescription)
}
}
if h, _ := out.FieldSources[FieldEnhanceInputHash].(string); h != "" {
t.Fatalf("formula synthesize must not persist hash, got %q", h)
}
}
func TestRunSteps_formulaHTMLFromLLM_persistsHash(t *testing.T) {
e := &Engine{
Completer: stubCompleter{fn: func(system, user string) (Completion, error) {
return Completion{
Text: `{"name":"GIGABYTE GS27QC","description":"<h1>GIGABYTE GS27QC</h1><p>Curved gaming monitor.</p><ul><li>width 61 cm</li></ul>"}`,
TotalTokens: 6,
}, nil
}},
Vector: NoopVectorCategorizer{},
}
descTpl := map[string]any{
"sections": []any{
map[string]any{"type": "h1", "instructions": "Heading"},
map[string]any{"type": "p", "instructions": "Body"},
map[string]any{"type": "ul", "instructions": "Specs"},
},
}
out, err := e.RunSteps(context.Background(), "co", ProductInput{
Name: "GIGABYTE GS27QC",
Mapped: map[string]any{"name": "GIGABYTE GS27QC", "category": "7"},
DescriptionTemplate: descTpl,
Language: "en",
}, "enhance_only", nil, StepPolicy{AllowAI: true, AllowEPREL: true})
if err != nil {
t.Fatal(err)
}
if !strings.Contains(out.ProcessedDescription, "<ul>") {
t.Fatalf("expected LLM HTML kept, got %q", out.ProcessedDescription)
}
if h, _ := out.FieldSources[FieldEnhanceInputHash].(string); h == "" {
t.Fatal("compliant formula HTML should persist enhance hash")
}
}
func TestRunSteps_synthPriorHashDoesNotSkipReprocess(t *testing.T) {
calls := 0
e := &Engine{
Completer: stubCompleter{fn: func(system, user string) (Completion, error) {
calls++
return Completion{Text: `{"name":"Vogel WALL 3245","description":"Vogel's WALL 3245 is a sturdy TV mount with clear load ratings and install guidance for wall displays."}`, TotalTokens: 5}, nil
}},
Vector: NoopVectorCategorizer{},
}
normName := "Vogel's WALL 3245 TV Wall Mount"
normDesc := "A mount with enough mapped detail for hashing inputs."
synthPrior := synthesizeDescriptionFromTitle(normName, "TV Mounts", "en", map[string]any{
"brand": "Vogel's", "width": "45 cm", "max_load": "40 kg",
})
if !company.LooksLikeHeuristicSynthesize(synthPrior) {
t.Fatalf("expected invent synth prior, got %q", synthPrior)
}
sysTpl, userTpl := resolveProductPromptTemplates(ProductInput{})
priorHash := HashEnhanceInput("", normName, normDesc, "", "", sysTpl, userTpl, map[string]any{})
out, err := e.RunSteps(context.Background(), "co", ProductInput{
Name: normName,
Description: normDesc,
Mapped: map[string]any{"name": normName, "description": normDesc},
PriorEnhanceHash: priorHash,
PriorProcessedName: normName,
PriorProcessedDescription: synthPrior,
}, "enhance_only", nil, StepPolicy{AllowAI: true, AllowEPREL: true})
if err != nil {
t.Fatal(err)
}
if calls != 1 {
t.Fatalf("synth prior must force completer, calls=%d", calls)
}
if out.SkipCreditDebit {
t.Fatal("synth prior must not SkipCreditDebit")
}
joined := strings.Join(out.Notes, ";")
if !strings.Contains(joined, "forced re-enhance") {
t.Fatalf("expected forced re-enhance note, notes=%v", out.Notes)
}
if company.LooksLikeHeuristicSynthesize(out.ProcessedDescription) && calls == 1 {
// LLM stub returned non-synth copy — ok if still synth after outer repair.
}
if out.ProcessedDescription == synthPrior {
t.Fatalf("must not reuse synth prior: %q", out.ProcessedDescription)
}
}
func TestRunSteps_formulaMismatchPriorHashDoesNotSkipReprocess(t *testing.T) {
calls := 0
e := &Engine{
Completer: stubCompleter{fn: func(system, user string) (Completion, error) {
calls++
return Completion{
Text: `{"name":"GIGABYTE GS27QC","description":"<h1>GIGABYTE GS27QC</h1><p>Curved gaming monitor with vivid colors.</p><ul><li>width 61 cm</li></ul>"}`,
TotalTokens: 6,
}, nil
}},
Vector: NoopVectorCategorizer{},
}
descTpl := map[string]any{
"sections": []any{
map[string]any{"type": "h1", "instructions": "Heading"},
map[string]any{"type": "p", "instructions": "Body"},
map[string]any{"type": "ul", "instructions": "Specs"},
},
}
normName := "GIGABYTE GS27QC"
normDesc := "GIGABYTE GS27QC is a gaming monitor with a 61 cm curved panel for immersive play."
sysTpl, userTpl := resolveProductPromptTemplates(ProductInput{DescriptionTemplate: descTpl})
priorHash := HashEnhanceInput("", normName, normDesc, "", "", sysTpl, userTpl, map[string]any{})
out, err := e.RunSteps(context.Background(), "co", ProductInput{
Name: normName,
Description: normDesc,
Mapped: map[string]any{"name": normName, "description": normDesc, "category": "7"},
DescriptionTemplate: descTpl,
PriorEnhanceHash: priorHash,
PriorProcessedName: normName,
PriorProcessedDescription: normDesc, // plain prose — formula mismatch
Language: "en",
}, "enhance_only", nil, StepPolicy{AllowAI: true, AllowEPREL: true})
if err != nil {
t.Fatal(err)
}
if calls < 1 {
t.Fatalf("formula-mismatch prior must force completer, calls=%d", calls)
}
if out.SkipCreditDebit {
t.Fatal("formula-mismatch must not SkipCreditDebit")
}
joined := strings.Join(out.Notes, ";")
if !strings.Contains(joined, "forced re-enhance") && !strings.Contains(joined, "formula") {
// forced note when hash matched; formula repair notes also acceptable
if !strings.Contains(joined, "synth") && calls >= 1 {
// still ok if LLM ran without note if reason path differed
}
}
for _, tag := range []string{"<h1>", "<p>", "<ul>"} {
if !strings.Contains(out.ProcessedDescription, tag) {
t.Fatalf("expected formula HTML with %s, got %q", tag, out.ProcessedDescription)
}
}
}
func TestRunSteps_weakPriorHashDoesNotSkipReprocess(t *testing.T) { func TestRunSteps_weakPriorHashDoesNotSkipReprocess(t *testing.T) {
calls := 0 calls := 0
e := &Engine{ e := &Engine{
@@ -344,3 +533,56 @@ func TestRunSteps_failedEnhanceDoesNotPoisonLocalizedHash(t *testing.T) {
} }
} }
} }
func TestRunSteps_emptyLLMDescriptionDoesNotPersistHashViaOriginalFallback(t *testing.T) {
// Prod bug: empty LLM description preferred originals and stamped ok+hash.
richOriginal := "This durable retail mount includes mounting hardware, load ratings, and install guidance for wall displays."
e := &Engine{
Completer: stubCompleter{fn: func(system, user string) (Completion, error) {
return Completion{Text: `{"name":"Vogel WALL 3245","description":""}`, TotalTokens: 2}, nil
}},
Vector: NoopVectorCategorizer{},
}
out, err := e.RunSteps(context.Background(), "co", ProductInput{
Name: "Vogel WALL 3245",
Description: richOriginal,
Mapped: map[string]any{
"name": "Vogel WALL 3245",
"description": richOriginal,
},
}, "enhance_only", nil, StepPolicy{AllowAI: true, AllowEPREL: true})
if err != nil {
t.Fatal(err)
}
if h, _ := out.FieldSources[FieldEnhanceInputHash].(string); h != "" {
t.Fatalf("empty LLM description must not persist enhance hash (got %q); desc=%q", h, out.ProcessedDescription)
}
for _, lf := range out.LocalizedContent {
if lf.EnhanceInputHash != "" {
t.Fatalf("localized hash must stay empty on empty LLM desc: %+v", lf)
}
}
joined := strings.Join(out.Notes, ";")
if !strings.Contains(joined, "refused") && !strings.Contains(joined, "synthesized") && !strings.Contains(joined, "empty_description") {
t.Fatalf("expected refuse/synthesize note, notes=%v", out.Notes)
}
if strings.TrimSpace(out.ProcessedDescription) == "" {
t.Fatal("expected nonempty fallback/synth description")
}
}
func TestLlmEnhanceHardRefuseReason(t *testing.T) {
t.Parallel()
if got := llmEnhanceHardRefuseReason("", "desc long enough"); got != "empty_name" {
t.Fatalf("got %q", got)
}
if got := llmEnhanceHardRefuseReason("Category:", "desc long enough here for tests"); got != "prompt_leakage_name" {
t.Fatalf("got %q", got)
}
if got := llmEnhanceHardRefuseReason("Widget", ""); got != "empty_description" {
t.Fatalf("got %q", got)
}
if got := llmEnhanceHardRefuseReason("Widget", "A durable retail widget for everyday use with clear specs."); got != "" {
t.Fatalf("got %q", got)
}
}
+127
View File
@@ -0,0 +1,127 @@
package processing
import (
"crypto/sha256"
"encoding/hex"
"encoding/json"
"fmt"
"log"
"strings"
"time"
)
// enhanceLogHeadTail caps each end of prompt/response bodies in worker logs.
const enhanceLogHeadTail = 500
func emptyLogDash(s string) string {
s = strings.TrimSpace(s)
if s == "" {
return "-"
}
return s
}
// truncateLogEnds returns a PII-safe preview: total rune length plus first/last
// slices when the body is large (avoids megabyte log lines).
func truncateLogEnds(s string, head, tail int) string {
s = strings.TrimSpace(s)
if s == "" {
return "len=0"
}
runes := []rune(s)
n := len(runes)
if head < 0 {
head = 0
}
if tail < 0 {
tail = 0
}
if n <= head+tail+32 {
return fmt.Sprintf("len=%d body=%q", n, s)
}
return fmt.Sprintf("len=%d head=%q tail=%q", n, string(runes[:head]), string(runes[n-tail:]))
}
// formulaTemplateSummary summarizes a category title/description formula for logs
// (section/element types + JSON byte length + short content hash — not a full dump).
func formulaTemplateSummary(template any, kind string) string {
if template == nil {
return "none"
}
raw, err := json.Marshal(template)
if err != nil || len(raw) == 0 || string(raw) == "null" || string(raw) == "{}" {
return "none"
}
sum := sha256.Sum256(raw)
hash8 := hex.EncodeToString(sum[:4])
switch kind {
case "title":
els, sep, ok := parseTitleFormula(template)
if !ok {
return fmt.Sprintf("unparsed len=%d hash=%s", len(raw), hash8)
}
types := make([]string, 0, len(els))
for _, el := range els {
types = append(types, el.Type)
}
return fmt.Sprintf("elements=%s sep=%q len=%d hash=%s", strings.Join(types, "+"), sep, len(raw), hash8)
case "description":
sections, ok := parseDescriptionFormulaSections(template)
if !ok {
return fmt.Sprintf("unparsed len=%d hash=%s", len(raw), hash8)
}
types := make([]string, 0, len(sections))
for _, sec := range sections {
if t := strings.TrimSpace(sec.Type); t != "" {
types = append(types, t)
}
}
return fmt.Sprintf("sections=%s len=%d hash=%s", strings.Join(types, "+"), len(raw), hash8)
default:
return fmt.Sprintf("len=%d hash=%s", len(raw), hash8)
}
}
func finishReasonFromCompletion(comp Completion) string {
m, ok := comp.Raw.(map[string]any)
if !ok || m == nil {
return ""
}
fr, _ := m["finish_reason"].(string)
return strings.TrimSpace(fr)
}
func enhanceFormulaOverride(in ProductInput) bool {
return FormatDescriptionFormulaConstraint(in.DescriptionTemplate) != "" ||
FormatTitleFormulaConstraint(in.TitleTemplate) != ""
}
// logEnhancePrompt emits one structured line before the LLM call (truncated prompts).
func logEnhancePrompt(in ProductInput, categoryUID, categoryName, system, user string) {
log.Printf("processing: ai_enhance prompt job=%s company=%s raw=%s category_uid=%q category_name=%q title_formula=%s desc_formula=%s formula_override=%t system=%s user=%s",
emptyLogDash(in.JobID), emptyLogDash(in.CompanyID), emptyLogDash(in.RawProductID),
strings.TrimSpace(categoryUID), strings.TrimSpace(categoryName),
formulaTemplateSummary(in.TitleTemplate, "title"),
formulaTemplateSummary(in.DescriptionTemplate, "description"),
enhanceFormulaOverride(in),
truncateLogEnds(system, enhanceLogHeadTail, enhanceLogHeadTail),
truncateLogEnds(user, enhanceLogHeadTail, enhanceLogHeadTail),
)
}
// logEnhanceOutcome emits one structured line for the enhance result (truncated response).
func logEnhanceOutcome(in ProductInput, categoryUID, categoryName, outcome, reason string, comp Completion, elapsed time.Duration, forceReason string) {
log.Printf("processing: ai_enhance outcome=%s reason=%s job=%s company=%s raw=%s category_uid=%q category_name=%q title_formula=%s desc_formula=%s formula_override=%t finish_reason=%s prompt_tokens=%d completion_tokens=%d total_tokens=%d elapsed=%s response=%s force_reason=%s",
emptyLogDash(outcome), emptyLogDash(reason),
emptyLogDash(in.JobID), emptyLogDash(in.CompanyID), emptyLogDash(in.RawProductID),
strings.TrimSpace(categoryUID), strings.TrimSpace(categoryName),
formulaTemplateSummary(in.TitleTemplate, "title"),
formulaTemplateSummary(in.DescriptionTemplate, "description"),
enhanceFormulaOverride(in),
emptyLogDash(finishReasonFromCompletion(comp)),
comp.PromptTokens, comp.OutputTokens, comp.TotalTokens,
elapsed.Round(time.Millisecond),
truncateLogEnds(comp.Text, enhanceLogHeadTail, enhanceLogHeadTail),
emptyLogDash(forceReason),
)
}
@@ -0,0 +1,103 @@
package processing
import (
"strings"
"testing"
)
func TestTruncateLogEnds(t *testing.T) {
t.Parallel()
short := truncateLogEnds("hello", 500, 500)
if !strings.Contains(short, `body="hello"`) || !strings.Contains(short, "len=5") {
t.Fatalf("short: %s", short)
}
long := strings.Repeat("a", 600) + "MID" + strings.Repeat("b", 600)
got := truncateLogEnds(long, 10, 10)
if !strings.Contains(got, "len=1203") || !strings.Contains(got, "head=") || !strings.Contains(got, "tail=") {
t.Fatalf("long: %s", got)
}
if strings.Contains(got, "MID") {
t.Fatalf("middle should be omitted: %s", got)
}
}
func TestFormulaTemplateSummary(t *testing.T) {
t.Parallel()
if formulaTemplateSummary(nil, "description") != "none" {
t.Fatal("nil → none")
}
desc := map[string]any{
"sections": []any{
map[string]any{"type": "h1", "instructions": "Heading"},
map[string]any{"type": "p", "instructions": "Body"},
map[string]any{"type": "ul", "instructions": "Specs"},
},
}
sum := formulaTemplateSummary(desc, "description")
if !strings.Contains(sum, "sections=h1+p+ul") || !strings.Contains(sum, "len=") || !strings.Contains(sum, "hash=") {
t.Fatalf("desc summary: %s", sum)
}
title := map[string]any{
"separator": " ",
"elements": []any{
map[string]any{"type": "variable", "value": "brand"},
map[string]any{"type": "text", "value": "TV"},
},
}
ts := formulaTemplateSummary(title, "title")
if !strings.Contains(ts, "elements=variable+text") || !strings.Contains(ts, `sep=" "`) {
t.Fatalf("title summary: %s", ts)
}
}
// TestA1StyleDescriptionFormulaInResolvedPrompts asserts a realistic multi-section
// category description_template (A1-style) is injected into both system and user
// enhance prompts via resolveProductPromptTemplates / RenderProductEnhancePrompts.
func TestA1StyleDescriptionFormulaInResolvedPrompts(t *testing.T) {
t.Parallel()
// Representative A1 cooker/monitor style: h1 + p + ul sections.
descFormula := map[string]any{
"sections": []any{
map[string]any{"type": "h1", "instructions": "Naziv izdelka"},
map[string]any{"type": "h2", "instructions": "Podnaslov prednosti"},
map[string]any{"type": "p", "instructions": "Kratek opis v slovenščini"},
map[string]any{"type": "ul", "instructions": "Ključne specifikacije"},
},
}
titleFormula := map[string]any{
"separator": " ",
"elements": []any{
map[string]any{"type": "variable", "value": "brand"},
map[string]any{"type": "variable", "value": "product_model"},
},
}
sysTpl, userTpl := resolveProductPromptTemplates(ProductInput{
EnhanceSystemTemplate: "You write retail copy. description: 1-2 factual sentences.",
EnhanceUserTemplate: "Category: {{category}}\nName: {{name}}\nAttrs: {{attrs}}",
TitleTemplate: titleFormula,
DescriptionTemplate: descFormula,
})
if !strings.Contains(sysTpl, "Description formula") {
t.Fatalf("system missing formula override: %s", sysTpl)
}
if !strings.Contains(userTpl, "Description formula") || !strings.Contains(userTpl, "- h1: Naziv izdelka") || !strings.Contains(userTpl, "- h2: Podnaslov prednosti") {
t.Fatalf("user missing description formula: %s", userTpl)
}
if !strings.Contains(userTpl, "Title formula") || !strings.Contains(userTpl, "attr [brand]") {
t.Fatalf("user missing title formula: %s", userTpl)
}
system, user := RenderProductEnhancePrompts(
sysTpl, userTpl,
"Štedilniki", "VOX EHT6020", "", "", "", "sl",
map[string]any{"brand": "VOX", "product_model": "EHT6020"},
)
if !strings.Contains(user, "Description formula") || !strings.Contains(user, "- h1: Naziv izdelka") {
t.Fatalf("rendered user missing formula: %s", user)
}
if !strings.Contains(user, "Slovenian") && !strings.Contains(system, "Slovenian") {
t.Fatalf("language not rendered into prompts: sys=%s user=%s", system, user)
}
if !enhanceFormulaOverride(ProductInput{DescriptionTemplate: descFormula}) {
t.Fatal("formula_override should be true")
}
}
@@ -5,6 +5,8 @@ import (
"fmt" "fmt"
"strconv" "strconv"
"strings" "strings"
"github.com/descrybe/descrybe-v2/apps/api/internal/company"
) )
// AppendFormulaConstraints appends language-agnostic title/description formula // AppendFormulaConstraints appends language-agnostic title/description formula
@@ -118,6 +120,24 @@ func AppendDescriptionFormulaSystemOverride(systemTpl string, descriptionTemplat
return systemTpl + "\n" + descriptionFormulaSystemOverride return systemTpl + "\n" + descriptionFormulaSystemOverride
} }
// descriptionSatisfiesFormula reports whether desc includes the HTML tags required
// by categories.description_template sections. No formula → always true.
func descriptionSatisfiesFormula(desc string, template any) bool {
sections, ok := parseDescriptionFormulaSections(template)
if !ok || len(sections) == 0 {
return true
}
desc = strings.TrimSpace(desc)
if desc == "" || desc == "<nil>" {
return false
}
types := make([]string, 0, len(sections))
for _, s := range sections {
types = append(types, s.Type)
}
return !company.DescriptionMissingFormulaHTMLTags(desc, types)
}
type titleFormulaElement struct { type titleFormulaElement struct {
Type string `json:"type"` Type string `json:"type"`
Value string `json:"value"` Value string `json:"value"`
@@ -76,6 +76,36 @@ func TestAppendDescriptionFormulaSystemOverride(t *testing.T) {
} }
} }
func TestDescriptionSatisfiesFormula(t *testing.T) {
t.Parallel()
tpl := map[string]any{
"sections": []any{
map[string]any{"type": "h1", "instructions": "Heading"},
map[string]any{"type": "p", "instructions": "Body"},
map[string]any{"type": "ul", "instructions": "Specs"},
},
}
if descriptionSatisfiesFormula("", tpl) {
t.Fatal("empty must fail")
}
if descriptionSatisfiesFormula("GIGABYTE GS27QC is a gaming monitor with a 61 cm curved panel for immersive play.", tpl) {
t.Fatal("plain prose must fail multi-section formula")
}
if !descriptionSatisfiesFormula("<h1>GIGABYTE GS27QC</h1><p>Gaming monitor.</p><ul><li>61 cm</li></ul>", tpl) {
t.Fatal("full HTML skeleton must pass")
}
if !descriptionSatisfiesFormula("any copy", nil) {
t.Fatal("nil formula always satisfied")
}
if descriptionNeedsEnhanceRepair(
"GIGABYTE GS27QC is a gaming monitor with a 61 cm curved panel for immersive play.",
tpl,
"GIGABYTE GS27QC",
) != true {
t.Fatal("non-weak prose that ignores formula must need repair")
}
}
func TestAppendFormulaConstraints_injectedIntoResolve(t *testing.T) { func TestAppendFormulaConstraints_injectedIntoResolve(t *testing.T) {
t.Parallel() t.Parallel()
title := map[string]any{ title := map[string]any{
+9 -5
View File
@@ -8,13 +8,17 @@ import (
"strings" "strings"
) )
// Local / weak-model defaults (8k-class context). See docs/local-llm-tuning.md. // Local / weak-model defaults. See docs/local-llm-tuning.md.
const ( const (
DefaultStructuredTemp = 0.2 DefaultStructuredTemp = 0.2
// Reasoning-capable OpenAI-compatible models (e.g. code-fast / Qwen3) spend // Reasoning-capable OpenAI-compatible models (e.g. OverloadedBot code-fast)
// hundredsthousands of tokens in reasoning_content before writing message.content. // spend thousands of tokens in reasoning_content before message.content.
// 350 capped mid-thought → empty content → "empty model response". // Formula HTML JSON often needs a large completion budget; 4096 hit
MaxTokensEnhance = 4096 // finish_reason=length with empty/truncated content in live enhance.
MaxTokensEnhance = 16384
// MaxTokensEnhanceRetry is the one-shot length-cap bump ceiling used by
// OpenAIClient when finish_reason=length yields empty or unparseable JSON.
MaxTokensEnhanceRetry = 32768
MaxTokensSEO = 180 MaxTokensSEO = 180
MaxTokensCampaign = 650 MaxTokensCampaign = 650
MaxProductDescRunes = 400 MaxProductDescRunes = 400
+59 -20
View File
@@ -256,11 +256,14 @@ type chatResponse struct {
} `json:"error"` } `json:"error"`
} }
var errEmptyModelResponse = errors.New("empty model response") var (
errEmptyModelResponse = errors.New("empty model response")
errLengthCappedResponse = errors.New("response truncated at max_tokens")
)
// maxTokensReasoningBudget is used when a capped completion returns empty func isLengthBudgetErr(err error) bool {
// content with finish_reason=length (reasoning models). return errors.Is(err, errEmptyModelResponse) || errors.Is(err, errLengthCappedResponse)
const maxTokensReasoningBudget = 4096 }
func (c *OpenAIClient) Complete(ctx context.Context, system, user string) (Completion, error) { func (c *OpenAIClient) Complete(ctx context.Context, system, user string) (Completion, error) {
return c.CompleteWithOptions(ctx, system, user, CompleteOptions{}) return c.CompleteWithOptions(ctx, system, user, CompleteOptions{})
@@ -300,16 +303,19 @@ func (c *OpenAIClient) CompleteWithOptions(ctx context.Context, system, user str
return comp, nil return comp, nil
} }
lastErr = err lastErr = err
if errors.Is(err, errEmptyModelResponse) { if isLengthBudgetErr(err) {
if maxTok <= 0 || maxTok < maxTokensReasoningBudget { if maxTok <= 0 || maxTok < MaxTokensEnhanceRetry {
if maxTok < maxTokensReasoningBudget { prev := maxTok
maxTok = maxTokensReasoningBudget if maxTok < MaxTokensEnhanceRetry {
maxTok = MaxTokensEnhanceRetry
} }
log.Printf("openai: length-cap retry model=%s prev_max_tokens=%d next_max_tokens=%d err=%s",
c.Model, prev, maxTok, TruncateError(err))
retryable = true retryable = true
} else { } else {
// Already at reasoning budget — another 240s call will not help. // Already at enhance retry ceiling — another long call will not help.
return Completion{}, fmt.Errorf( return Completion{}, fmt.Errorf(
"openai empty response at max_tokens=%d for model %q (try a faster non-reasoning model): %w", "openai length-capped at max_tokens=%d for model %q (prefer a faster non-reasoning product-enhance model): %w",
maxTok, c.Model, err) maxTok, c.Model, err)
} }
} }
@@ -488,34 +494,41 @@ func (c *OpenAIClient) doComplete(ctx context.Context, system, user string, temp
stopWait := c.startWaitLogger("chat/completions", maxTokens) stopWait := c.startWaitLogger("chat/completions", maxTokens)
res, err := c.HTTPClient.Do(req) res, err := c.HTTPClient.Do(req)
stopWait(err)
if err != nil { if err != nil {
stopWait(err)
wrapped, retryable := classifyOpenAITransportErr(c.Model, err) wrapped, retryable := classifyOpenAITransportErr(c.Model, err)
return Completion{}, retryable, wrapped return Completion{}, retryable, wrapped
} }
defer res.Body.Close() defer res.Body.Close()
raw, err := io.ReadAll(io.LimitReader(res.Body, 1<<20)) raw, err := io.ReadAll(io.LimitReader(res.Body, 1<<20))
if err != nil { if err != nil {
stopWait(err)
wrapped, retryable := classifyOpenAITransportErr(c.Model, err) wrapped, retryable := classifyOpenAITransportErr(c.Model, err)
return Completion{}, retryable, wrapped return Completion{}, retryable, wrapped
} }
var parsed chatResponse var parsed chatResponse
if err := json.Unmarshal(raw, &parsed); err != nil { if err := json.Unmarshal(raw, &parsed); err != nil {
return Completion{}, false, fmt.Errorf("openai decode: %w", err) decErr := fmt.Errorf("openai decode: %w", err)
stopWait(decErr)
return Completion{}, false, decErr
} }
if res.StatusCode == http.StatusTooManyRequests || res.StatusCode >= 500 { if res.StatusCode == http.StatusTooManyRequests || res.StatusCode >= 500 {
msg := "rate limited or server error" msg := "rate limited or server error"
if parsed.Error != nil && parsed.Error.Message != "" { if parsed.Error != nil && parsed.Error.Message != "" {
msg = TruncateError(errors.New(parsed.Error.Message)) msg = TruncateError(errors.New(parsed.Error.Message))
} }
return Completion{}, true, errors.New(msg) httpErr := errors.New(msg)
stopWait(httpErr)
return Completion{}, true, httpErr
} }
if res.StatusCode >= 400 { if res.StatusCode >= 400 {
msg := fmt.Sprintf("openai http %d", res.StatusCode) msg := fmt.Sprintf("openai http %d", res.StatusCode)
if parsed.Error != nil && parsed.Error.Message != "" { if parsed.Error != nil && parsed.Error.Message != "" {
msg = TruncateError(errors.New(parsed.Error.Message)) msg = TruncateError(errors.New(parsed.Error.Message))
} }
return Completion{}, false, errors.New(msg) httpErr := errors.New(msg)
stopWait(httpErr)
return Completion{}, false, httpErr
} }
text := "" text := ""
finishReason := "" finishReason := ""
@@ -523,18 +536,44 @@ func (c *OpenAIClient) doComplete(ctx context.Context, system, user string, temp
finishReason = strings.TrimSpace(parsed.Choices[0].FinishReason) finishReason = strings.TrimSpace(parsed.Choices[0].FinishReason)
text = SanitizeOutput(choiceMessageText(parsed.Choices[0].Message.Content, parsed.Choices[0].Message.ReasoningContent, parsed.Choices[0].Message.Reasoning)) text = SanitizeOutput(choiceMessageText(parsed.Choices[0].Message.Content, parsed.Choices[0].Message.ReasoningContent, parsed.Choices[0].Message.Reasoning))
} }
usagePrompt := parsed.Usage.PromptTokens
usageOut := parsed.Usage.CompletionTokens
usageTotal := parsed.Usage.TotalTokens
canBump := maxTokens <= 0 || maxTokens < MaxTokensEnhanceRetry
lengthCapped := strings.EqualFold(finishReason, "length")
if text == "" { if text == "" {
// Reasoning models often return empty content when max_tokens cuts mid-thought. // Reasoning models often return empty content when max_tokens cuts mid-thought.
// Only retry when CompleteWithOptions can still raise max_tokens. // Only retry when CompleteWithOptions can still raise max_tokens.
canBump := maxTokens <= 0 || maxTokens < maxTokensReasoningBudget // Never log HTTP-ok as ok=1 when content is empty (prod looked "successful" at 30272ms).
retryable := strings.EqualFold(finishReason, "length") && canBump retryable := lengthCapped && canBump
emptyErr := fmt.Errorf("%w (finish_reason=%s max_tokens=%d prompt_tokens=%d completion_tokens=%d)",
errEmptyModelResponse, finishReason, maxTokens, usagePrompt, usageOut)
stopWait(emptyErr)
log.Printf("openai: chat content empty model=%s finish_reason=%s max_tokens=%d prompt_tokens=%d completion_tokens=%d total_tokens=%d retryable=%t",
c.Model, finishReason, maxTokens, usagePrompt, usageOut, usageTotal, retryable)
return Completion{}, retryable, errEmptyModelResponse return Completion{}, retryable, errEmptyModelResponse
} }
if lengthCapped {
// Truncated JSON was previously treated as success → parse_failed → synthesize.
// Accept only when the truncated body still parses as a JSON object.
if _, err := ParseJSONObject(text); err != nil {
retryable := canBump
truncErr := fmt.Errorf("%w (finish_reason=length max_tokens=%d prompt_tokens=%d completion_tokens=%d content_runes=%d)",
errLengthCappedResponse, maxTokens, usagePrompt, usageOut, len([]rune(text)))
stopWait(truncErr)
log.Printf("openai: chat content truncated model=%s finish_reason=length max_tokens=%d prompt_tokens=%d completion_tokens=%d total_tokens=%d content_runes=%d retryable=%t",
c.Model, maxTokens, usagePrompt, usageOut, usageTotal, len([]rune(text)), retryable)
return Completion{}, retryable, errLengthCappedResponse
}
}
stopWait(nil)
log.Printf("openai: chat content ok model=%s finish_reason=%s content_runes=%d prompt_tokens=%d completion_tokens=%d total_tokens=%d",
c.Model, finishReason, len([]rune(text)), usagePrompt, usageOut, usageTotal)
return Completion{ return Completion{
Text: text, Text: text,
PromptTokens: parsed.Usage.PromptTokens, PromptTokens: usagePrompt,
OutputTokens: parsed.Usage.CompletionTokens, OutputTokens: usageOut,
TotalTokens: parsed.Usage.TotalTokens, TotalTokens: usageTotal,
Model: parsed.Model, Model: parsed.Model,
Raw: map[string]any{ Raw: map[string]any{
"model": parsed.Model, "model": parsed.Model,
@@ -678,7 +717,7 @@ func inventHeuristicDescription(system, user, name string) string {
name = "Product" name = "Product"
} }
cat := labeledPromptValue(user, "category:") cat := labeledPromptValue(user, "category:")
if isPromptLabelTitle(cat) { if isUnusableCategoryValue(cat, name) {
cat = "" cat = ""
} }
lang := languageCodeFromEnhancePrompt(system, user) lang := languageCodeFromEnhancePrompt(system, user)
+95 -8
View File
@@ -201,8 +201,8 @@ func TestOpenAIClient_doComplete_emptyContentLengthRetriesWithBudget(t *testing.
}) })
return return
} }
if maxTok != float64(maxTokensReasoningBudget) { if maxTok != float64(MaxTokensEnhanceRetry) {
t.Errorf("retry max_tokens=%v want %d", maxTok, maxTokensReasoningBudget) t.Errorf("retry max_tokens=%v want %d", maxTok, MaxTokensEnhanceRetry)
} }
_ = json.NewEncoder(w).Encode(map[string]any{ _ = json.NewEncoder(w).Encode(map[string]any{
"model": "code-fast", "model": "code-fast",
@@ -228,6 +228,61 @@ func TestOpenAIClient_doComplete_emptyContentLengthRetriesWithBudget(t *testing.
} }
} }
func TestOpenAIClient_doComplete_truncatedJSONLengthRetriesFromEnhanceBudget(t *testing.T) {
t.Parallel()
calls := 0
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
calls++
var req map[string]any
_ = json.NewDecoder(r.Body).Decode(&req)
maxTok, _ := req["max_tokens"].(float64)
if calls == 1 {
if maxTok != float64(MaxTokensEnhance) {
t.Errorf("first max_tokens=%v want %d", maxTok, MaxTokensEnhance)
}
_ = json.NewEncoder(w).Encode(map[string]any{
"model": "code-fast",
"choices": []map[string]any{{
"finish_reason": "length",
"message": map[string]any{
"role": "assistant",
"content": `{"name":"Lenovo","description":"<h1>Lenovo G27-20: Igralski monitor za vrhunsko vizualno izkušn`,
},
}},
"usage": map[string]int{"prompt_tokens": 200, "completion_tokens": MaxTokensEnhance, "total_tokens": 200 + MaxTokensEnhance},
})
return
}
if maxTok != float64(MaxTokensEnhanceRetry) {
t.Errorf("retry max_tokens=%v want %d", maxTok, MaxTokensEnhanceRetry)
}
_ = json.NewEncoder(w).Encode(map[string]any{
"model": "code-fast",
"choices": []map[string]any{{
"finish_reason": "stop",
"message": map[string]any{
"role": "assistant",
"content": `{"name":"Lenovo G27-20","description":"<h1>Lenovo G27-20</h1><p>Monitor.</p>"}`,
},
}},
"usage": map[string]int{"prompt_tokens": 200, "completion_tokens": 80, "total_tokens": 280},
})
}))
defer srv.Close()
c := NewOpenAIClient("test-key", srv.URL, "code-fast", 0, 2)
c.HTTPClient = srv.Client()
comp, err := c.CompleteWithOptions(context.Background(), "sys", "user", CompleteOptions{MaxTokens: MaxTokensEnhance, Temperature: 0.2})
if err != nil {
t.Fatal(err)
}
if calls != 2 {
t.Fatalf("calls=%d want 2 (length truncated must not succeed)", calls)
}
if !strings.Contains(comp.Text, "</h1>") {
t.Fatalf("text=%q", comp.Text)
}
}
func TestOpenAIClient_Complete_timeoutNotRetried(t *testing.T) { func TestOpenAIClient_Complete_timeoutNotRetried(t *testing.T) {
calls := 0 calls := 0
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
@@ -269,7 +324,7 @@ func TestOpenAIClient_Complete_timeoutNotRetried(t *testing.T) {
} }
} }
func TestOpenAIClient_Complete_emptyAtReasoningBudgetNotRetried(t *testing.T) { func TestOpenAIClient_Complete_emptyAtEnhanceRetryBudgetNotRetried(t *testing.T) {
t.Parallel() t.Parallel()
calls := 0 calls := 0
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
@@ -280,21 +335,53 @@ func TestOpenAIClient_Complete_emptyAtReasoningBudgetNotRetried(t *testing.T) {
"finish_reason": "length", "finish_reason": "length",
"message": map[string]any{"role": "assistant", "content": "", "reasoning_content": "still thinking"}, "message": map[string]any{"role": "assistant", "content": "", "reasoning_content": "still thinking"},
}}, }},
"usage": map[string]int{"prompt_tokens": 1, "completion_tokens": 4096, "total_tokens": 4097}, "usage": map[string]int{"prompt_tokens": 1, "completion_tokens": MaxTokensEnhanceRetry, "total_tokens": 1 + MaxTokensEnhanceRetry},
}) })
})) }))
defer srv.Close() defer srv.Close()
c := NewOpenAIClient("test-key", srv.URL, "code-fast", 0, 3) c := NewOpenAIClient("test-key", srv.URL, "code-fast", 0, 3)
c.HTTPClient = srv.Client() c.HTTPClient = srv.Client()
_, err := c.CompleteWithOptions(context.Background(), "sys", "user", CompleteOptions{MaxTokens: maxTokensReasoningBudget}) _, err := c.CompleteWithOptions(context.Background(), "sys", "user", CompleteOptions{MaxTokens: MaxTokensEnhanceRetry})
if err == nil { if err == nil {
t.Fatal("expected empty response error") t.Fatal("expected length-capped error")
} }
if !strings.Contains(err.Error(), "empty response") { if !strings.Contains(err.Error(), "length-capped") {
t.Fatalf("err=%v", err) t.Fatalf("err=%v", err)
} }
if calls != 1 { if calls != 1 {
t.Fatalf("calls=%d want 1 (no retry at reasoning budget)", calls) t.Fatalf("calls=%d want 1 (no retry at enhance retry ceiling)", calls)
}
}
func TestOpenAIClient_Complete_truncatedAtEnhanceRetryBudgetNotSuccess(t *testing.T) {
t.Parallel()
calls := 0
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
calls++
_ = json.NewEncoder(w).Encode(map[string]any{
"model": "code-fast",
"choices": []map[string]any{{
"finish_reason": "length",
"message": map[string]any{
"role": "assistant",
"content": `{"name":"X","description":"<h1>cut off`,
},
}},
"usage": map[string]int{"prompt_tokens": 10, "completion_tokens": MaxTokensEnhanceRetry, "total_tokens": 10 + MaxTokensEnhanceRetry},
})
}))
defer srv.Close()
c := NewOpenAIClient("test-key", srv.URL, "code-fast", 0, 3)
c.HTTPClient = srv.Client()
_, err := c.CompleteWithOptions(context.Background(), "sys", "user", CompleteOptions{MaxTokens: MaxTokensEnhanceRetry})
if err == nil {
t.Fatal("truncated JSON at ceiling must not succeed")
}
if !strings.Contains(err.Error(), "length-capped") {
t.Fatalf("err=%v", err)
}
if calls != 1 {
t.Fatalf("calls=%d want 1", calls)
} }
} }
+9 -1
View File
@@ -1522,6 +1522,9 @@ func (p *Pipeline) processOne(ctx context.Context, companyID, jobID uuid.UUID, i
CategoryNamesByUID: categoryNamesByUID, CategoryNamesByUID: categoryNamesByUID,
AllowedAttrKeys: allowedAttrKeysForCategory(cache, stringFromAny(enriched["category"])), AllowedAttrKeys: allowedAttrKeysForCategory(cache, stringFromAny(enriched["category"])),
CategoryAttrKeys: categoryAttrKeysFromCache(cache), CategoryAttrKeys: categoryAttrKeysFromCache(cache),
JobID: jobID.String(),
CompanyID: companyID.String(),
RawProductID: it.RawID.String(),
} }
if it.hydrated { if it.hydrated {
if it.hasPrior { if it.hasPrior {
@@ -1577,6 +1580,8 @@ func (p *Pipeline) processOne(ctx context.Context, companyID, jobID uuid.UUID, i
jobID, it.RawID, catPreFilter) jobID, it.RawID, catPreFilter)
} }
syncCategoryName(&result, cache.categoryNamesByUID) syncCategoryName(&result, cache.categoryNamesByUID)
scrubCategoryPollution(&result, cache.categoryNamesByUID, cache.categoryUniqueIDs)
syncCategoryName(&result, cache.categoryNamesByUID)
} }
// Re-apply after category validation so allowlist matches the persisted category. // Re-apply after category validation so allowlist matches the persisted category.
allowed := allowedAttrKeysForCategory(cache, result.Category) allowed := allowedAttrKeysForCategory(cache, result.Category)
@@ -1676,7 +1681,10 @@ const upsertProcessedProductSQL = `
ON CONFLICT (company_id, raw_product_id) DO UPDATE SET ON CONFLICT (company_id, raw_product_id) DO UPDATE SET
product_id = EXCLUDED.product_id, product_id = EXCLUDED.product_id,
name = EXCLUDED.name, name = EXCLUDED.name,
category = COALESCE(NULLIF(BTRIM(EXCLUDED.category), ''), processed_products.category), category = CASE
WHEN COALESCE(EXCLUDED.field_sources->>'category', '') = 'cleared_invalid' THEN ''
ELSE COALESCE(NULLIF(BTRIM(EXCLUDED.category), ''), processed_products.category)
END,
description = EXCLUDED.description, description = EXCLUDED.description,
processed_name = EXCLUDED.processed_name, processed_name = EXCLUDED.processed_name,
processed_description = EXCLUDED.processed_description, processed_description = EXCLUDED.processed_description,
+372 -28
View File
@@ -4,8 +4,10 @@ import (
"context" "context"
"encoding/json" "encoding/json"
"fmt" "fmt"
"log"
"sort" "sort"
"strings" "strings"
"time"
"github.com/descrybe/descrybe-v2/apps/api/internal/company" "github.com/descrybe/descrybe-v2/apps/api/internal/company"
"github.com/descrybe/descrybe-v2/apps/api/internal/eprel" "github.com/descrybe/descrybe-v2/apps/api/internal/eprel"
@@ -215,6 +217,7 @@ func (e *Engine) RunSteps(ctx context.Context, companyID string, in ProductInput
out.ProcessedName = out.Name out.ProcessedName = out.Name
out.ProcessedDescription = out.Description out.ProcessedDescription = out.Description
out.Notes = append(out.Notes, "ai_enhance: skipped (Free plan — upgrade for AI titles/descriptions)") out.Notes = append(out.Notes, "ai_enhance: skipped (Free plan — upgrade for AI titles/descriptions)")
log.Printf("processing: ai_enhance skip reason=entitlement_can_use_ai")
preservePriorEnhanceHash() preservePriorEnhanceHash()
appendStepLog(out.GPTResponse, StepAIEnhance, map[string]any{ appendStepLog(out.GPTResponse, StepAIEnhance, map[string]any{
"status": "skipped", "status": "skipped",
@@ -226,6 +229,7 @@ func (e *Engine) RunSteps(ctx context.Context, companyID string, in ProductInput
out.ProcessedName = out.Name out.ProcessedName = out.Name
out.ProcessedDescription = out.Description out.ProcessedDescription = out.Description
out.Notes = append(out.Notes, "ai_enhance: skipped (platform OpenAI unset; configure admin settings or company BYOK)") out.Notes = append(out.Notes, "ai_enhance: skipped (platform OpenAI unset; configure admin settings or company BYOK)")
log.Printf("processing: ai_enhance skip reason=openai_not_configured")
preservePriorEnhanceHash() preservePriorEnhanceHash()
appendStepLog(out.GPTResponse, StepAIEnhance, map[string]any{ appendStepLog(out.GPTResponse, StepAIEnhance, map[string]any{
"status": "skipped", "status": "skipped",
@@ -298,6 +302,10 @@ func (e *Engine) RunSteps(ctx context.Context, companyID string, in ProductInput
PriorEnhanceHash: priorHash, PriorEnhanceHash: priorHash,
PriorProcessedName: priorName, PriorProcessedName: priorName,
PriorProcessedDescription: priorDesc, PriorProcessedDescription: priorDesc,
JobID: in.JobID,
CompanyID: in.CompanyID,
RawProductID: in.RawProductID,
CategoryUniqueID: out.Category,
}, displayCat, enhanceAttrs) }, displayCat, enhanceAttrs)
out.TotalTokens += tokens out.TotalTokens += tokens
status := enhanceStatusFromMeta(raw) status := enhanceStatusFromMeta(raw)
@@ -318,6 +326,15 @@ func (e *Engine) RunSteps(ctx context.Context, companyID string, in ProductInput
} }
} else if status == "unchanged" { } else if status == "unchanged" {
meta["status"] = "unchanged" meta["status"] = "unchanged"
} else if status == "refused" || status == "synthesized" || status == "parse_failed" {
// Soft quality failure: keep usable copy / synth, never count as real enhance ok.
allUnchanged = false
meta["status"] = status
if reason := enhanceReasonFromMeta(raw); reason != "" {
out.Notes = append(out.Notes, "ai_enhance: "+status+" ("+reason+")")
} else {
out.Notes = append(out.Notes, "ai_enhance: "+status)
}
} else { } else {
allUnchanged = false allUnchanged = false
anyOK = true anyOK = true
@@ -327,13 +344,13 @@ func (e *Engine) RunSteps(ctx context.Context, companyID string, in ProductInput
name = preferredProductTitle(in.GTIN, name, out.Name, priorName, in.Name) name = preferredProductTitle(in.GTIN, name, out.Name, priorName, in.Name)
desc = preferredProductDescription(name, desc, out.Description, priorDesc) desc = preferredProductDescription(name, desc, out.Description, priorDesc)
outerSynth := false outerSynth := false
if isWeakPriorEnhanceDescription(desc, name) { if descriptionNeedsEnhanceRepair(desc, descTpl, name) {
if synth := synthesizeProductDescription(name, displayCat, lang, enhanceAttrs, descTpl); synth != "" { if synth := synthesizeProductDescription(name, displayCat, lang, enhanceAttrs, descTpl); synth != "" {
desc = synth desc = synth
outerSynth = true outerSynth = true
} }
} }
weakDesc := isWeakPriorEnhanceDescription(desc, name) weakDesc := descriptionNeedsEnhanceRepair(desc, descTpl, name)
// Only persist enhance_input_hash for quality ok / hash-skip unchanged. // Only persist enhance_input_hash for quality ok / hash-skip unchanged.
// Never copy input_hash from error/passthrough/thin/synthesized meta. // Never copy input_hash from error/passthrough/thin/synthesized meta.
persistHash := "" persistHash := ""
@@ -341,7 +358,7 @@ func (e *Engine) RunSteps(ctx context.Context, companyID string, in ProductInput
switch { switch {
case err != nil: case err != nil:
persistHash = "" persistHash = ""
case status == "synthesized" || outerSynth: case status == "synthesized" || status == "refused" || outerSynth:
persistHash = "" persistHash = ""
allUnchanged = false allUnchanged = false
case status == "unchanged" && !weakDesc: case status == "unchanged" && !weakDesc:
@@ -349,7 +366,7 @@ func (e *Engine) RunSteps(ctx context.Context, companyID string, in ProductInput
case status == "ok" && !weakDesc: case status == "ok" && !weakDesc:
persistHash = rawHash persistHash = rawHash
default: default:
// thin ok / parse_failed / skipped — do not poison reprocess skip // thin ok / parse_failed / formula mismatch / skipped — do not poison reprocess skip
persistHash = "" persistHash = ""
if status == "ok" && weakDesc { if status == "ok" && weakDesc {
allUnchanged = false allUnchanged = false
@@ -404,13 +421,12 @@ func (e *Engine) RunSteps(ctx context.Context, companyID string, in ProductInput
if out.ProcessedDescription == "" { if out.ProcessedDescription == "" {
out.ProcessedDescription = out.Description out.ProcessedDescription = out.Description
} }
// Timeout/error/empty: always synthesize a factual fallback when a title exists. // Timeout/error/empty/formula-miss: always synthesize a factual fallback when a title exists.
if out.ProcessedName != "" && (out.ProcessedDescription == "" ||
isWeakPriorEnhanceDescription(out.ProcessedDescription, out.ProcessedName, out.Name)) {
_, failDescTpl := categoryFormulasFor(in, out.Category) _, failDescTpl := categoryFormulasFor(in, out.Category)
if out.ProcessedName != "" && descriptionNeedsEnhanceRepair(out.ProcessedDescription, failDescTpl, out.ProcessedName, out.Name) {
if synth := synthesizeProductDescription(out.ProcessedName, displayCat, primary, enhanceAttrs, failDescTpl); synth != "" { if synth := synthesizeProductDescription(out.ProcessedName, displayCat, primary, enhanceAttrs, failDescTpl); synth != "" {
out.ProcessedDescription = synth out.ProcessedDescription = synth
if out.Description == "" || isWeakPriorEnhanceDescription(out.Description, out.Name, out.ProcessedName) { if out.Description == "" || descriptionNeedsEnhanceRepair(out.Description, failDescTpl, out.Name, out.ProcessedName) {
out.Description = synth out.Description = synth
} }
if lf, ok := localized[primary]; ok { if lf, ok := localized[primary]; ok {
@@ -453,11 +469,27 @@ func (e *Engine) RunSteps(ctx context.Context, companyID string, in ProductInput
out.SkipCreditDebit = true out.SkipCreditDebit = true
out.FieldSources["name"] = "ai_enhance_unchanged" out.FieldSources["name"] = "ai_enhance_unchanged"
out.FieldSources["description"] = "ai_enhance_unchanged" out.FieldSources["description"] = "ai_enhance_unchanged"
out.Notes = append(out.Notes, "ai_enhance: skipped (inputs unchanged)") out.Notes = append(out.Notes, "ai_enhance_unchanged")
log.Printf("processing: ai_enhance_unchanged")
} else { } else {
out.AIProviderMode = e.EngineProviderMode() out.AIProviderMode = e.EngineProviderMode()
out.FieldSources["name"] = "ai_enhance" out.FieldSources["name"] = "ai_enhance"
out.FieldSources["description"] = "ai_enhance" out.FieldSources["description"] = "ai_enhance"
for _, m := range langMetas {
if mm, ok := m.(map[string]any); ok {
reason, _ := mm["reason"].(string)
if reason == "" {
if raw, ok := mm["raw"].(map[string]any); ok {
reason, _ = raw["reason"].(string)
}
}
if reason != "" {
out.Notes = append(out.Notes, "ai_enhance: forced re-enhance ("+reason+")")
log.Printf("processing: ai_enhance forced re-enhance reason=%s", reason)
break
}
}
}
} }
appendStepLog(out.GPTResponse, StepAIEnhance, map[string]any{ appendStepLog(out.GPTResponse, StepAIEnhance, map[string]any{
"status": map[string]any{"unchanged": allUnchanged, "ok": anyOK, "failed": anyFailed}, "status": map[string]any{"unchanged": allUnchanged, "ok": anyOK, "failed": anyFailed},
@@ -475,7 +507,6 @@ func (e *Engine) RunSteps(ctx context.Context, companyID string, in ProductInput
preserveCategoryIfEmpty(&out, in.PriorCategory) preserveCategoryIfEmpty(&out, in.PriorCategory)
noteMissingCategory(&out, policy, e != nil && e.Vector != nil && e.Vector.Enabled()) noteMissingCategory(&out, policy, e != nil && e.Vector != nil && e.Vector.Enabled())
syncCategoryName(&out, in.CategoryNamesByUID) syncCategoryName(&out, in.CategoryNamesByUID)
displayCat := categoryDisplayLabel(out)
out.Name = preferredProductTitle(in.GTIN, out.Name, out.ProcessedName, in.Name, in.PriorProcessedName) out.Name = preferredProductTitle(in.GTIN, out.Name, out.ProcessedName, in.Name, in.PriorProcessedName)
out.ProcessedName = preferredProductTitle(in.GTIN, out.ProcessedName, out.Name, in.PriorProcessedName, in.Name) out.ProcessedName = preferredProductTitle(in.GTIN, out.ProcessedName, out.Name, in.PriorProcessedName, in.Name)
out.Description = preferredProductDescription(out.Name, out.Description, out.ProcessedDescription, in.Description) out.Description = preferredProductDescription(out.Name, out.Description, out.ProcessedDescription, in.Description)
@@ -483,22 +514,52 @@ func (e *Engine) RunSteps(ctx context.Context, companyID string, in ProductInput
if out.ProcessedName == "" { if out.ProcessedName == "" {
out.ProcessedName = out.Name out.ProcessedName = out.Name
} }
// Titles are finalized first so name-as-category and formula leakage can be cleared.
scrubCategoryPollution(&out, in.CategoryNamesByUID, nil)
syncCategoryName(&out, in.CategoryNamesByUID)
displayCat := categoryDisplayLabel(out)
// After vector/mapped category resolution, formulas may key for the first time —
// repair short prose / title echo that ignored A1 description_template.
_, finalDescTpl := categoryFormulasFor(in, out.Category) _, finalDescTpl := categoryFormulasFor(in, out.Category)
if out.ProcessedDescription == "" || isWeakPriorEnhanceDescription(out.ProcessedDescription, out.ProcessedName, out.Name) { finalRepaired := false
if descriptionNeedsEnhanceRepair(out.ProcessedDescription, finalDescTpl, out.ProcessedName, out.Name) {
if synth := synthesizeProductDescription(out.ProcessedName, displayCat, in.Language, out.Attributes, finalDescTpl); synth != "" { if synth := synthesizeProductDescription(out.ProcessedName, displayCat, in.Language, out.Attributes, finalDescTpl); synth != "" {
out.ProcessedDescription = synth out.ProcessedDescription = synth
finalRepaired = true
} }
} }
if out.Description == "" || isWeakPriorEnhanceDescription(out.Description, out.Name, out.ProcessedName) { if descriptionNeedsEnhanceRepair(out.Description, finalDescTpl, out.Name, out.ProcessedName) {
if out.ProcessedDescription != "" && !isWeakPriorEnhanceDescription(out.ProcessedDescription, out.Name, out.ProcessedName) { if out.ProcessedDescription != "" && !descriptionNeedsEnhanceRepair(out.ProcessedDescription, finalDescTpl, out.Name, out.ProcessedName) {
out.Description = out.ProcessedDescription out.Description = out.ProcessedDescription
finalRepaired = true
} else if synth := synthesizeProductDescription(out.Name, displayCat, in.Language, out.Attributes, finalDescTpl); synth != "" { } else if synth := synthesizeProductDescription(out.Name, displayCat, in.Language, out.Attributes, finalDescTpl); synth != "" {
out.Description = synth out.Description = synth
if out.ProcessedDescription == "" || isWeakPriorEnhanceDescription(out.ProcessedDescription, out.ProcessedName, out.Name) { finalRepaired = true
if descriptionNeedsEnhanceRepair(out.ProcessedDescription, finalDescTpl, out.ProcessedName, out.Name) {
out.ProcessedDescription = synth out.ProcessedDescription = synth
} }
} }
} }
if finalRepaired {
delete(out.FieldSources, FieldEnhanceInputHash)
primary := strings.TrimSpace(in.Language)
if primary == "" && len(in.ContentLanguages) > 0 {
primary = strings.TrimSpace(in.ContentLanguages[0])
}
if primary == "" {
primary = company.DefaultLanguage
}
for lang, lf := range out.LocalizedContent {
lf.EnhanceInputHash = ""
if lang == primary {
lf.ProcessedDescription = out.ProcessedDescription
if lf.ProcessedName == "" {
lf.ProcessedName = out.ProcessedName
}
}
out.LocalizedContent[lang] = lf
}
}
if out.Attributes == nil { if out.Attributes == nil {
out.Attributes = map[string]any{} out.Attributes = map[string]any{}
} }
@@ -528,7 +589,7 @@ func preserveCategoryIfEmpty(out *StepResult, prior string) {
return return
} }
prior = strings.TrimSpace(prior) prior = strings.TrimSpace(prior)
if prior == "" { if prior == "" || isUnusableCategoryValue(prior, out.ProcessedName, out.Name) {
return return
} }
out.Category = SanitizeText(prior) out.Category = SanitizeText(prior)
@@ -563,7 +624,7 @@ func tryVectorCategorize(ctx context.Context, e *Engine, companyID string, out *
}) })
return return
} }
if strings.TrimSpace(cat) == "" { if strings.TrimSpace(cat) == "" || isUnusableCategoryValue(cat, out.ProcessedName, out.Name) {
return return
} }
out.Category = SanitizeOutput(cat) out.Category = SanitizeOutput(cat)
@@ -632,78 +693,252 @@ func appendStepLog(gpt map[string]any, name string, raw any) {
gpt["steps"] = append(steps, map[string]any{"step": name, "raw": raw}) gpt["steps"] = append(steps, map[string]any{"step": name, "raw": raw})
} }
// descriptionFormulaRetrySuffix is appended once when LLM JSON ignored the
// category description_template (short prose / title echo instead of multi-section HTML).
const descriptionFormulaRetrySuffix = "\n\nINVALID DESCRIPTION. Your JSON ignored the Description formula. Reply with ONLY one JSON object; \"description\" must be ONE HTML string covering each formula section in order with matching tags (h1/h2/h3/h4, p, ul)."
// descriptionNeedsEnhanceRepair is true when desc is weak/empty/title-echo or
// fails an active category description_template (A1 multi-section HTML).
// Heuristic invent/synth is intentionally excluded here — enhanceHashForceReason
// blocks hash-skip for synth so reprocess can still upgrade to real LLM copy.
func descriptionNeedsEnhanceRepair(desc string, template any, titles ...string) bool {
if isWeakPriorEnhanceDescription(desc, titles...) {
return true
}
return !descriptionSatisfiesFormula(desc, template)
}
// enhanceHashForceReason returns why a matching prior enhance_input_hash must not
// skip the LLM (empty = safe to reuse as ai_enhance_unchanged).
func enhanceHashForceReason(in ProductInput) string {
if isPromptLabelTitle(in.PriorProcessedName) {
return "prompt_label_title"
}
if reason := company.EnhanceHashSkipBlockReason(in.PriorProcessedDescription, in.PriorProcessedName, in.Name); reason != "" {
return reason
}
if !descriptionSatisfiesFormula(in.PriorProcessedDescription, in.DescriptionTemplate) {
return "formula-mismatch"
}
return ""
}
func (e *Engine) enhance(ctx context.Context, in ProductInput, category string, attrs map[string]any) (string, string, int, any, error) { func (e *Engine) enhance(ctx context.Context, in ProductInput, category string, attrs map[string]any) (string, string, int, any, error) {
catUID := strings.TrimSpace(in.CategoryUniqueID)
catName := strings.TrimSpace(category)
sysTpl, userTpl := resolveProductPromptTemplates(in) sysTpl, userTpl := resolveProductPromptTemplates(in)
hash := HashEnhanceInput(category, in.Name, in.Description, in.BrandPrompt, in.Language, sysTpl, userTpl, attrs) hash := HashEnhanceInput(category, in.Name, in.Description, in.BrandPrompt, in.Language, sysTpl, userTpl, attrs)
if e == nil || e.Completer == nil { if e == nil || e.Completer == nil {
name := preferredProductTitle(in.GTIN, in.Name, in.PriorProcessedName) name := preferredProductTitle(in.GTIN, in.Name, in.PriorProcessedName)
logEnhanceOutcome(in, catUID, catName, "skip", "completer_nil", Completion{}, 0, "")
return name, return name,
preferredProductDescription(name, in.Description, in.PriorProcessedDescription), preferredProductDescription(name, in.Description, in.PriorProcessedDescription),
0, map[string]any{"status": "skipped", "input_hash": hash}, nil 0, map[string]any{"status": "skipped", "reason": "completer_nil"}, nil
} }
// Skip LLM when inputs match the last successful enhance (before any credit debit), // Skip LLM when inputs match the last successful enhance (before any credit debit),
// but never reuse a thin / title-echo prior description, or a prompt-leakage title. // but never reuse thin / title-echo / invent-synth / formula-mismatch priors,
// or a prompt-leakage title.
var forceReason string
if in.PriorEnhanceHash != "" && in.PriorEnhanceHash == hash && if in.PriorEnhanceHash != "" && in.PriorEnhanceHash == hash &&
(in.PriorProcessedName != "" || in.PriorProcessedDescription != "") && (in.PriorProcessedName != "" || in.PriorProcessedDescription != "") {
!isPromptLabelTitle(in.PriorProcessedName) && forceReason = enhanceHashForceReason(in)
!isWeakPriorEnhanceDescription(in.PriorProcessedDescription, in.PriorProcessedName, in.Name) { if forceReason == "" {
name := preferredProductTitle(in.GTIN, in.PriorProcessedName, in.Name) name := preferredProductTitle(in.GTIN, in.PriorProcessedName, in.Name)
desc := preferredProductDescription(name, in.PriorProcessedDescription, in.Description) desc := preferredProductDescription(name, in.PriorProcessedDescription, in.Description)
logEnhanceOutcome(in, catUID, catName, "skip", "unchanged_hash", Completion{}, 0, "")
return name, desc, 0, map[string]any{ return name, desc, 0, map[string]any{
"status": "unchanged", "status": "unchanged",
"input_hash": hash, "input_hash": hash,
}, nil }, nil
} }
log.Printf("processing: ai_enhance skip_blocked reason=%s category=%q name=%q",
forceReason, truncateRunes(category, 80), truncateRunes(in.Name, 80))
}
system, user := RenderProductEnhancePrompts(sysTpl, userTpl, category, in.Name, in.Description, in.GTIN, in.BrandPrompt, in.Language, attrs) system, user := RenderProductEnhancePrompts(sysTpl, userTpl, category, in.Name, in.Description, in.GTIN, in.BrandPrompt, in.Language, attrs)
logEnhancePrompt(in, catUID, catName, system, user)
started := time.Now()
comp, obj, err := CompleteJSON(ctx, e.Completer, system, user, CompleteOptions{ comp, obj, err := CompleteJSON(ctx, e.Completer, system, user, CompleteOptions{
MaxTokens: MaxTokensEnhance, MaxTokens: MaxTokensEnhance,
Temperature: DefaultStructuredTemp, Temperature: DefaultStructuredTemp,
}) })
elapsed := time.Since(started)
if err != nil { if err != nil {
// Network/provider failure vs parse failure after retry // Network/provider failure vs parse failure after retry
name := preferredProductTitle(in.GTIN, in.Name, in.PriorProcessedName) name := preferredProductTitle(in.GTIN, in.Name, in.PriorProcessedName)
desc := preferredProductDescription(name, in.Description, in.PriorProcessedDescription) desc := preferredProductDescription(name, in.Description, in.PriorProcessedDescription)
if isWeakPriorEnhanceDescription(desc, name) { outcome := "refuse"
reason := "empty_or_provider_error"
if descriptionNeedsEnhanceRepair(desc, in.DescriptionTemplate, name) {
if synth := synthesizeProductDescription(name, category, in.Language, attrs, in.DescriptionTemplate); synth != "" { if synth := synthesizeProductDescription(name, category, in.Language, attrs, in.DescriptionTemplate); synth != "" {
desc = synth desc = synth
outcome = "synthesized"
} }
} }
if obj == nil && comp.Text == "" { if obj == nil && comp.Text == "" {
logEnhanceOutcome(in, catUID, catName, outcome, reason+":"+TruncateError(err), comp, elapsed, forceReason)
return name, desc, 0, map[string]any{ return name, desc, 0, map[string]any{
"provider": "passthrough", "provider": "passthrough",
"error": TruncateError(err), "error": TruncateError(err),
}, err }, err
} }
// Parse failed after retry — keep usable copy; synthesize when empty/weak. // Parse failed after retry — keep usable copy; synthesize when empty/weak/formula-miss.
if outcome != "synthesized" {
outcome = "refuse"
reason = "parse_failed"
} else {
reason = "parse_failed"
}
logEnhanceOutcome(in, catUID, catName, outcome, reason, comp, elapsed, forceReason)
return name, desc, comp.TotalTokens, map[string]any{ return name, desc, comp.TotalTokens, map[string]any{
"status": "parse_failed", "status": "parse_failed",
"reason": "invalid_json",
"error": "AI returned invalid JSON; kept original title/description", "error": "AI returned invalid JSON; kept original title/description",
"raw": truncateRunes(comp.Text, 200), "raw": truncateRunes(comp.Text, 200),
}, nil }, nil
} }
name := preferredProductTitle(in.GTIN, SanitizeOutput(fmt.Sprint(obj["name"])), in.Name, in.PriorProcessedName) llmName := SanitizeOutput(fmt.Sprint(obj["name"]))
desc := preferredProductDescription(name, SanitizeOutput(fmt.Sprint(obj["description"])), in.Description, in.PriorProcessedDescription) llmDesc := SanitizeOutput(fmt.Sprint(obj["description"]))
// Never prefer-fallback to originals for empty/leakage LLM fields — that stamped
// status=ok + input_hash on garbage enhance output (prod: 30272ms "ok", wrong copy).
if reason := llmEnhanceHardRefuseReason(llmName, llmDesc); reason != "" {
name := preferredProductTitle(in.GTIN, llmName, in.Name, in.PriorProcessedName)
desc := preferredProductDescription(name, llmDesc)
synthesized := false synthesized := false
if isWeakPriorEnhanceDescription(desc, name) { if descriptionNeedsEnhanceRepair(desc, in.DescriptionTemplate, name) {
if desc == "" {
desc = preferredProductDescription(name, in.Description, in.PriorProcessedDescription)
}
if descriptionNeedsEnhanceRepair(desc, in.DescriptionTemplate, name) {
if synth := synthesizeProductDescription(name, category, in.Language, attrs, in.DescriptionTemplate); synth != "" { if synth := synthesizeProductDescription(name, category, in.Language, attrs, in.DescriptionTemplate); synth != "" {
desc = synth desc = synth
synthesized = true synthesized = true
} }
} }
}
status := "refused"
outcome := "refuse"
if synthesized {
status = "synthesized"
outcome = "synthesized"
}
meta := map[string]any{
"status": status,
"reason": reason,
"raw": comp.Raw,
}
if forceReason != "" {
meta["forced_reenhance"] = true
meta["force_reason"] = forceReason
}
logEnhanceOutcome(in, catUID, catName, outcome, reason, comp, elapsed, forceReason)
return name, desc, comp.TotalTokens, meta, nil
}
name := preferredProductTitle(in.GTIN, llmName, in.Name, in.PriorProcessedName)
// Quality-gate on LLM description alone (do not absorb originals yet).
desc := preferredProductDescription(name, llmDesc)
didFormulaRetry := false
// Fast garbage that ignores A1 description_template: one formula-aware retry, then synthesize.
if !descriptionSatisfiesFormula(desc, in.DescriptionTemplate) &&
FormatDescriptionFormulaConstraint(in.DescriptionTemplate) != "" {
didFormulaRetry = true
retryUser := user + descriptionFormulaRetrySuffix
retryStarted := time.Now()
comp2, obj2, err2 := CompleteJSON(ctx, e.Completer, system, retryUser, CompleteOptions{
MaxTokens: MaxTokensEnhance,
Temperature: DefaultStructuredTemp,
})
elapsed += time.Since(retryStarted)
comp.PromptTokens += comp2.PromptTokens
comp.OutputTokens += comp2.OutputTokens
comp.TotalTokens += comp2.TotalTokens
if err2 == nil && obj2 != nil {
n2 := SanitizeOutput(fmt.Sprint(obj2["name"]))
d2 := SanitizeOutput(fmt.Sprint(obj2["description"]))
if llmEnhanceHardRefuseReason(n2, d2) == "" {
name = preferredProductTitle(in.GTIN, n2, name, in.Name, in.PriorProcessedName)
desc = preferredProductDescription(name, d2)
if comp2.Raw != nil {
comp.Raw = comp2.Raw
}
if comp2.Text != "" {
comp.Text = comp2.Text
}
}
}
}
synthesized := false
if descriptionNeedsEnhanceRepair(desc, in.DescriptionTemplate, name) {
if desc == "" {
desc = preferredProductDescription(name, in.Description, in.PriorProcessedDescription)
}
if descriptionNeedsEnhanceRepair(desc, in.DescriptionTemplate, name) {
if synth := synthesizeProductDescription(name, category, in.Language, attrs, in.DescriptionTemplate); synth != "" {
desc = synth
synthesized = true
}
}
}
meta := map[string]any{ meta := map[string]any{
"status": "ok", "status": "ok",
"raw": comp.Raw, "raw": comp.Raw,
} }
if forceReason != "" {
meta["reason"] = forceReason
meta["forced_reenhance"] = true
}
outcome := "ok"
reason := forceReason
// Heuristic synthesize must not poison enhance_input_hash (would hash-skip // Heuristic synthesize must not poison enhance_input_hash (would hash-skip
// formula-aware LLM copy on reprocess). Only persist hash for real LLM quality. // formula-aware LLM copy on reprocess). Only persist hash for real LLM quality.
if synthesized { if synthesized {
meta["status"] = "synthesized" meta["status"] = "synthesized"
} else if !isWeakPriorEnhanceDescription(desc, name) { outcome = "synthesized"
meta["input_hash"] = hash if didFormulaRetry {
reason = "formula_retry"
} else {
reason = "weak_or_formula_mismatch"
} }
meta["reason"] = reason
} else if descriptionNeedsEnhanceRepair(desc, in.DescriptionTemplate, name) {
meta["status"] = "refused"
outcome = "refuse"
if didFormulaRetry {
reason = "formula_retry"
} else {
reason = "weak_or_formula_mismatch"
}
meta["reason"] = reason
} else {
meta["input_hash"] = hash
if didFormulaRetry {
reason = "formula_retry"
if meta["reason"] == nil {
meta["reason"] = reason
}
}
}
logEnhanceOutcome(in, catUID, catName, outcome, reason, comp, elapsed, forceReason)
return name, desc, comp.TotalTokens, meta, nil return name, desc, comp.TotalTokens, meta, nil
} }
// llmEnhanceHardRefuseReason reports empty/leakage LLM fields that must never be
// accepted as quality enhance (even when originals could fill the gap).
func llmEnhanceHardRefuseReason(name, desc string) string {
name = strings.TrimSpace(name)
desc = strings.TrimSpace(desc)
if name == "" || name == "<nil>" {
return "empty_name"
}
if isPromptLabelTitle(name) {
return "prompt_leakage_name"
}
if desc == "" || desc == "<nil>" {
return "empty_description"
}
return ""
}
func sanitizeJSON(v any) string { func sanitizeJSON(v any) string {
if v == nil { if v == nil {
return "{}" return "{}"
@@ -841,6 +1076,24 @@ var promptLeakagePhrases = []string{
"prefer attrs values", "prefer attrs values",
"order matters; join with", "order matters; join with",
"do not hardcode a language", "do not hardcode a language",
// Description / title formula scaffolding (Format*FormulaConstraint + enhance templates).
"emit description as one html",
"emit one html string",
"covering each section",
"tags matching section",
"overrides any shorter",
"competing full html",
"metadescription",
"never ignore the description",
"follow any description formula",
"when a description formula",
"structured html sections",
"never copy name as description",
"do not invent specs",
"prefer 1-3 factual",
"1-2 sentences",
"keep literal text as written",
"build name from attrs using this structure",
} }
var promptLeakageLongKeywords = []string{ var promptLeakageLongKeywords = []string{
@@ -850,6 +1103,97 @@ var promptLeakageLongKeywords = []string{
"json", "json",
"attrs", "attrs",
"retail title", "retail title",
"html string",
"section type",
}
// isUnusableCategoryValue rejects tokens that must never become Category /
// CategoryName: prompt labels / title-formula scaffolding, or the product
// title itself (name-as-category). Callers with taxonomy should prefer
// scrubCategoryPollution so a title that legitimately equals a category name
// can still resolve to unique_id.
func isUnusableCategoryValue(s string, productTitles ...string) bool {
s = strings.TrimSpace(s)
if s == "" || s == "<nil>" {
return false
}
if isPromptLabelTitle(s) {
return true
}
return categoryTokenEqualsProductTitle(s, productTitles...)
}
func categoryTokenEqualsProductTitle(s string, productTitles ...string) bool {
s = strings.TrimSpace(s)
if s == "" || s == "<nil>" {
return false
}
for _, t := range productTitles {
t = strings.TrimSpace(t)
if t == "" || t == "<nil>" {
continue
}
if strings.EqualFold(s, t) {
return true
}
}
return false
}
func taxonomyDisplayNameSet(namesByUID map[string]string) map[string]struct{} {
if len(namesByUID) == 0 {
return nil
}
out := make(map[string]struct{}, len(namesByUID))
for _, name := range namesByUID {
n := strings.ToLower(strings.TrimSpace(name))
if n != "" {
out[n] = struct{}{}
}
}
return out
}
// scrubCategoryPollution clears Category / CategoryName when they hold prompt
// leakage/formula text, or equal the product title without resolving to a
// company taxonomy unique_id / display name.
func scrubCategoryPollution(out *StepResult, namesByUID map[string]string, valid map[string]struct{}) {
if out == nil {
return
}
titles := []string{out.ProcessedName, out.Name}
clearCategory := func(note string) {
out.Category = ""
out.CategoryName = ""
if out.FieldSources == nil {
out.FieldSources = map[string]any{}
}
out.FieldSources["category"] = "cleared_invalid"
out.Notes = append(out.Notes, note)
}
cat := strings.TrimSpace(out.Category)
if cat != "" {
switch {
case isPromptLabelTitle(cat):
clearCategory("category: ignored (prompt leakage)")
case categoryTokenEqualsProductTitle(cat, titles...) &&
resolveCompanyCategoryUniqueID(cat, namesByUID, valid) == "":
clearCategory("category: ignored (product title)")
}
}
name := strings.TrimSpace(out.CategoryName)
if name == "" {
return
}
if isPromptLabelTitle(name) {
out.CategoryName = ""
return
}
if categoryTokenEqualsProductTitle(name, titles...) {
if _, ok := taxonomyDisplayNameSet(namesByUID)[strings.ToLower(name)]; !ok {
out.CategoryName = ""
}
}
} }
// preferredProductTitle picks the first usable title, skipping empty values, // preferredProductTitle picks the first usable title, skipping empty values,
+117
View File
@@ -223,6 +223,10 @@ func TestIsPromptLeakageTitle(t *testing.T) {
"write name in {{language}}": true, "write name in {{language}}": true,
"Schema: {\"name\":\"string\",\"description\":\"string\"}": true, "Schema: {\"name\":\"string\",\"description\":\"string\"}": true,
"Reply with ONLY JSON (no markdown)": true, "Reply with ONLY JSON (no markdown)": true,
"Emit description as ONE HTML string covering each section in order": true,
"Description formula (REQUIRED — overrides any shorter \"1-2 sentences\" rule)": true,
"when a Description formula follows, emit ONE HTML string": true,
"never copy name as description": true,
"Vox SWA-8000W dishwasher": false, "Vox SWA-8000W dishwasher": false,
"Acme Widget Pro": false, "Acme Widget Pro": false,
"": false, "": false,
@@ -242,6 +246,119 @@ func TestIsPromptLeakageTitle(t *testing.T) {
} }
} }
func TestIsUnusableCategoryValue_rejectsNameAndFormula(t *testing.T) {
t.Parallel()
title := "NOSILEC W53070 81-140CM 180ST VOGELS"
if !isUnusableCategoryValue(title, title) {
t.Fatal("product title must not be usable as category")
}
if !isUnusableCategoryValue("Title formula (order matters; join with \" \")", title) {
t.Fatal("title formula must not be usable as category")
}
if !isUnusableCategoryValue("Emit description as ONE HTML string covering each section", title) {
t.Fatal("description formula must not be usable as category")
}
if !isUnusableCategoryValue("Category:", title) {
t.Fatal("prompt label must not be usable as category")
}
if isUnusableCategoryValue("28", title) {
t.Fatal("taxonomy unique_id must remain usable")
}
if isUnusableCategoryValue("TV mounts", title) {
t.Fatal("real category display name must remain usable")
}
}
func TestScrubCategoryPollution_clearsNameAsCategory(t *testing.T) {
t.Parallel()
title := "Vogel's WALL 3245 TV Wall Mount"
out := StepResult{
Name: title,
ProcessedName: title,
Category: title,
CategoryName: title,
FieldSources: map[string]any{"category": "vector"},
}
scrubCategoryPollution(&out, nil, nil)
if out.Category != "" || out.CategoryName != "" {
t.Fatalf("expected category cleared, got Category=%q CategoryName=%q", out.Category, out.CategoryName)
}
if src, _ := out.FieldSources["category"].(string); src != "cleared_invalid" {
t.Fatalf("field_sources.category=%v want cleared_invalid", out.FieldSources["category"])
}
}
func TestEnhance_rejectsDescriptionFormulaLeakageTitle(t *testing.T) {
leak := "Emit description as ONE HTML string covering each section in order in Slovenian"
e := &Engine{
Completer: stubCompleter{fn: func(system, user string) (Completion, error) {
return Completion{Text: `{"name":"` + leak + `","description":"A solid washer."}`, TotalTokens: 2}, nil
}},
Vector: NoopVectorCategorizer{},
}
out, err := e.RunSteps(context.Background(), "co", ProductInput{
GTIN: "8712285326882",
Name: "Vox SWA-8000W",
Mapped: map[string]any{"name": "Vox SWA-8000W", "description": "Washer", "category": "demo-electronics"},
}, "enhance_only", nil, StepPolicy{AllowAI: true})
if err != nil {
t.Fatal(err)
}
if out.ProcessedName != "Vox SWA-8000W" {
t.Fatalf("ProcessedName=%q want Vox SWA-8000W (reject description formula leakage)", out.ProcessedName)
}
}
func TestRunSteps_rejectsProductNameAsCategory(t *testing.T) {
title := "NOSILEC W53070 81-140CM 180ST VOGELS"
e := &Engine{
Completer: stubCompleter{fn: func(system, user string) (Completion, error) {
return Completion{Text: `{"name":"` + title + `","description":"TV wall mount for large screens."}`, TotalTokens: 2}, nil
}},
Vector: NoopVectorCategorizer{},
}
out, err := e.RunSteps(context.Background(), "co", ProductInput{
GTIN: "8712285326882",
Name: title,
PriorCategory: title,
Mapped: map[string]any{"name": title, "description": title, "category": title},
}, "enhance_only", nil, StepPolicy{AllowAI: true})
if err != nil {
t.Fatal(err)
}
if out.Category == title || strings.EqualFold(out.Category, out.ProcessedName) {
t.Fatalf("Category=%q must not equal product title", out.Category)
}
if categoryDisplayLabel(out) == title {
t.Fatalf("display category leaked product title: %q", categoryDisplayLabel(out))
}
}
func TestRunSteps_rejectsFormulaAsCategory(t *testing.T) {
leak := "Title formula (order matters; join with \" \"). Build name from Attrs"
e := &Engine{
Completer: stubCompleter{fn: func(system, user string) (Completion, error) {
return Completion{Text: `{"name":"Vox SWA-8000W","description":"Washer."}`, TotalTokens: 2}, nil
}},
Vector: NoopVectorCategorizer{},
}
out, err := e.RunSteps(context.Background(), "co", ProductInput{
GTIN: "8712285326882",
Name: "Vox SWA-8000W",
PriorCategory: leak,
Mapped: map[string]any{"name": "Vox SWA-8000W", "description": "Washer", "category": leak},
}, "enhance_only", nil, StepPolicy{AllowAI: true})
if err != nil {
t.Fatal(err)
}
if out.Category != "" {
t.Fatalf("Category=%q want empty (formula rejected)", out.Category)
}
if categoryDisplayLabel(out) != "" {
t.Fatalf("display=%q want empty", categoryDisplayLabel(out))
}
}
func TestEnhance_rejectsFormulaLeakageTitle(t *testing.T) { func TestEnhance_rejectsFormulaLeakageTitle(t *testing.T) {
leak := "short retail title; follow any Title formula constraints that follow; use Attrs" leak := "short retail title; follow any Title formula constraints that follow; use Attrs"
e := &Engine{ e := &Engine{
@@ -28,6 +28,10 @@ func TestUpsertProcessedProductSQL_usesOnConflict(t *testing.T) {
if !strings.Contains(upsertProcessedProductSQL, "COALESCE(NULLIF(BTRIM(EXCLUDED.category), ''), processed_products.category)") { if !strings.Contains(upsertProcessedProductSQL, "COALESCE(NULLIF(BTRIM(EXCLUDED.category), ''), processed_products.category)") {
t.Fatalf("expected category preserve on conflict: got SQL without COALESCE preserve") t.Fatalf("expected category preserve on conflict: got SQL without COALESCE preserve")
} }
// Intentional scrub/filter clears must wipe a previously persisted product-title category.
if !strings.Contains(upsertProcessedProductSQL, "cleared_invalid") {
t.Fatalf("expected cleared_invalid category wipe path in upsert SQL")
}
// Free template meta must persist; existing non-empty meta must be preserved on reprocess. // Free template meta must persist; existing non-empty meta must be preserved on reprocess.
if !strings.Contains(upsertProcessedProductSQL, "meta_title") || !strings.Contains(upsertProcessedProductSQL, "meta_description") { if !strings.Contains(upsertProcessedProductSQL, "meta_title") || !strings.Contains(upsertProcessedProductSQL, "meta_description") {
t.Fatalf("expected meta_title/meta_description columns in upsert") t.Fatalf("expected meta_title/meta_description columns in upsert")
+16 -4
View File
@@ -326,11 +326,23 @@ func (p *Pipeline) LoadV1ProcessJobItems(ctx context.Context, companyID, jobID u
if name, ok := categoryNames[catStr]; ok && name != "" { if name, ok := categoryNames[catStr]; ok && name != "" {
catNameStr = name catNameStr = name
} else if nested, ok := mapped["category"].(map[string]any); ok { } else if nested, ok := mapped["category"].(map[string]any); ok {
for _, k := range []string{"name", "category_name", "title"} { // Only accept nested display names that resolve in company taxonomy —
if s := strings.TrimSpace(stringFromAny(nested[k])); s != "" { // never product title / title-formula leftovers from feed objects.
catNameStr = s for _, k := range []string{"name", "category_name"} {
break s := strings.TrimSpace(stringFromAny(nested[k]))
if s == "" || isUnusableCategoryValue(s, titleStr) {
continue
} }
uid := resolveCompanyCategoryUniqueID(s, categoryNames, nil)
if uid == "" {
continue
}
if resolvedName := strings.TrimSpace(categoryNames[uid]); resolvedName != "" {
catNameStr = resolvedName
} else {
catNameStr = s
}
break
} }
} }
} }
+14 -1
View File
@@ -14,12 +14,21 @@ Defaults for OpenAI-compatible local servers such as Green Chat Gemma 12B (`over
Restart **both** `api` and `worker` after changing `OPENAI_*`. Worker logs should say `OpenAI enabled base=… model=…`, not `heuristic completer`. Restart **both** `api` and `worker` after changing `OPENAI_*`. Worker logs should say `OpenAI enabled base=… model=…`, not `heuristic completer`.
## Product enhance model choice
Admin AI role for **product enhance** should prefer a **fast non-reasoning chat model** that emits `message.content` (JSON) under budget.
Reasoning / “code-fast” style models often spend the entire `max_tokens` budget in `reasoning_content`, then return `finish_reason=length` with empty or truncated JSON. Descrybe retries once at `MaxTokensEnhanceRetry` and may synthesize formula HTML as salvage — that is **not** live LLM formula compliance.
If enhance logs show `finish_reason=length` with empty/truncated content even after the retry, switch Admin enhance to a non-reasoning model.
## Hardening defaults (code) ## Hardening defaults (code)
| Constant | Value | Where | | Constant | Value | Where |
|----------|------:|-------| |----------|------:|-------|
| `DefaultStructuredTemp` | `0.2` (capped ≤0.3) | `processing/llm_json.go`, `openai.go` | | `DefaultStructuredTemp` | `0.2` (capped ≤0.3) | `processing/llm_json.go`, `openai.go` |
| `MaxTokensEnhance` | `350` | product title/description | | `MaxTokensEnhance` | `16384` | product title/description (reasoning + formula HTML JSON) |
| `MaxTokensEnhanceRetry` | `32768` | one-shot bump on `finish_reason=length` empty/truncated JSON |
| `MaxTokensSEO` | `180` | meta title/description | | `MaxTokensSEO` | `180` | meta title/description |
| `MaxTokensCampaign` | `650` | email JSON | | `MaxTokensCampaign` | `650` | email JSON |
| `MaxProductDescRunes` | `400` | user context | | `MaxProductDescRunes` | `400` | user context |
@@ -27,6 +36,8 @@ Restart **both** `api` and `worker` after changing `OPENAI_*`. Worker logs shoul
| `MaxBrandInjectRunes` | `500` | brand kit block | | `MaxBrandInjectRunes` | `500` | brand kit block |
| `MaxCampaignProducts` | `8` | campaign product list | | `MaxCampaignProducts` | `8` | campaign product list |
OpenAI-compatible chat completions here use a single `max_tokens` completion budget (no separate reasoning vs content split on typical gateways). Length-cap handling lives in `OpenAIClient.doComplete` / `CompleteWithOptions`.
## Prompt style ## Prompt style
- Short system prompts with **bullet rules** + explicit JSON schema - Short system prompts with **bullet rules** + explicit JSON schema
@@ -43,6 +54,8 @@ Restart **both** `api` and `worker` after changing `OPENAI_*`. Worker logs shoul
3. On parse fail → **one retry** with `INVALID. Reply with ONLY one JSON object…` 3. On parse fail → **one retry** with `INVALID. Reply with ONLY one JSON object…`
4. Call sites fall back to originals/templates instead of storing garbage prose as titles 4. Call sites fall back to originals/templates instead of storing garbage prose as titles
`OpenAIClient` additionally treats `finish_reason=length` with empty or unparseable JSON as failure (not success) and retries once at `MaxTokensEnhanceRetry`.
## Ops tips ## Ops tips
- Prefer LAN TCP reachability checks (`Test-NetConnection host -Port 8767`) over ICMP - Prefer LAN TCP reachability checks (`Test-NetConnection host -Port 8767`) over ICMP