fix
This commit is contained in:
@@ -0,0 +1,200 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"log"
|
||||
"os"
|
||||
"os/signal"
|
||||
"strings"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"github.com/descrybe/descrybe-v2/apps/api/internal/aiprompts"
|
||||
"github.com/descrybe/descrybe-v2/apps/api/internal/config"
|
||||
"github.com/descrybe/descrybe-v2/apps/api/internal/db"
|
||||
"github.com/descrybe/descrybe-v2/apps/api/internal/eprel"
|
||||
"github.com/descrybe/descrybe-v2/apps/api/internal/logredact"
|
||||
"github.com/descrybe/descrybe-v2/apps/api/internal/processing"
|
||||
"github.com/google/uuid"
|
||||
)
|
||||
|
||||
func main() {
|
||||
log.SetOutput(logredact.Writer(os.Stderr))
|
||||
eans := []string{
|
||||
"195348253666", // 7 Gaming monitorji (rich)
|
||||
"4548736132597", // 48 Slušalke (rich)
|
||||
"8712285326882", // 28 Nosilci za TV (thin p-only)
|
||||
"3045380024786", // 36 Pečice (rich)
|
||||
"3838782459856", // 38 Pomivalni stroji (rich)
|
||||
"8806095210711", // 40 Pralni stroji (rich)
|
||||
"4242005342488", // 41 Pralno-sušilni (rich)
|
||||
"3838782103889", // 17 Kuhalne plošče (rich)
|
||||
"1200130000638", // 3 Bluetooth zvočniki (rich)
|
||||
"194252029558", // 24 Mobilne naprave (rich)
|
||||
}
|
||||
companyID := uuid.MustParse("2b3159b0-fc08-415b-b248-35ed02a6baab")
|
||||
userID := uuid.MustParse("0ce3305c-d810-4b56-b1d4-3c1ed510db76")
|
||||
|
||||
cfg, err := config.Load()
|
||||
if err != nil {
|
||||
log.Fatalf("config: %v", err)
|
||||
}
|
||||
ctx, cancel := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
|
||||
defer cancel()
|
||||
|
||||
pool, err := db.NewPool(ctx, cfg.DatabaseURL, db.PoolOptions{
|
||||
MaxConns: int32(cfg.DBMaxConns),
|
||||
MinConns: int32(cfg.DBMinConns),
|
||||
MaxConnLifetime: cfg.DBMaxConnLifetime,
|
||||
MaxConnLifetimeJitter: cfg.DBMaxConnLifetimeJitter,
|
||||
MaxConnIdleTime: cfg.DBMaxConnIdleTime,
|
||||
HealthCheckPeriod: cfg.DBHealthCheckPeriod,
|
||||
StatementTimeout: cfg.DBStatementTimeout,
|
||||
})
|
||||
if err != nil {
|
||||
log.Fatalf("db: %v", err)
|
||||
}
|
||||
defer pool.Close()
|
||||
|
||||
out := os.Getenv("PROBE_OUT_DIR")
|
||||
if out == "" {
|
||||
out = `F:\laragon\www\_MY\descrybe-v2\.codehelper\_formula_probe_20260816`
|
||||
}
|
||||
_ = os.MkdirAll(out, 0o755)
|
||||
|
||||
pipeline := processing.NewPipeline(pool)
|
||||
pipeline.BatchSize = cfg.ProcessingBatchSize
|
||||
pipeline.AI = nil
|
||||
pipeline.Prompts = aiprompts.NewService(pool)
|
||||
|
||||
mockKey := strings.TrimSpace(cfg.OpenAIAPIKey)
|
||||
if mockKey == "" {
|
||||
mockKey = "local-test"
|
||||
}
|
||||
mockBase := "http://127.0.0.1:18767/v1"
|
||||
mockModel := "mock-llm"
|
||||
log.Printf("forcing completer base=%s model=%s (ModeLabel=internal; OverloadedBot not used)", mockBase, mockModel)
|
||||
client := processing.NewOpenAIClient(mockKey, mockBase, mockModel, cfg.ProcessingRPM, cfg.ProcessingMaxRetries)
|
||||
client.ModeLabel = processing.AIProviderInternal
|
||||
|
||||
eprelClient := eprel.NewClient(eprel.Options{Enabled: true, Timeout: 20 * time.Second})
|
||||
pipeline.Engine = &processing.Engine{
|
||||
Completer: client,
|
||||
Vector: processing.NoopVectorCategorizer{},
|
||||
EPREL: eprelClient,
|
||||
ProviderMode: processing.AIProviderInternal,
|
||||
}
|
||||
|
||||
rawIDs := make([]uuid.UUID, 0, len(eans))
|
||||
for _, ean := range eans {
|
||||
var id uuid.UUID
|
||||
err := pool.QueryRow(ctx, `
|
||||
SELECT id FROM raw_products
|
||||
WHERE company_id = $1 AND gtin = $2`, companyID, ean).Scan(&id)
|
||||
if err != nil {
|
||||
log.Fatalf("raw product %s: %v", ean, err)
|
||||
}
|
||||
rawIDs = append(rawIDs, id)
|
||||
log.Printf("ean=%s raw_id=%s", ean, id)
|
||||
}
|
||||
|
||||
if n, err := processing.BackfillMissingMeta(ctx, pool, companyID); err != nil {
|
||||
log.Printf("BackfillMissingMeta: %v", err)
|
||||
} else {
|
||||
log.Printf("BackfillMissingMeta updated=%d", n)
|
||||
}
|
||||
|
||||
jobs, err := pipeline.StartJob(ctx, companyID, userID, rawIDs, "full")
|
||||
if err != nil {
|
||||
log.Fatalf("StartJob: %v", err)
|
||||
}
|
||||
if len(jobs) == 0 {
|
||||
log.Fatal("StartJob returned no jobs")
|
||||
}
|
||||
jobID := jobs[0].ID
|
||||
log.Printf("job_id=%s", jobID)
|
||||
_ = os.WriteFile(out+string(os.PathSeparator)+"process_id.txt", []byte(jobID.String()+"\n"), 0o644)
|
||||
|
||||
defend := make(chan struct{})
|
||||
go func() {
|
||||
t := time.NewTicker(500 * time.Millisecond)
|
||||
defer t.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-defend:
|
||||
return
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-t.C:
|
||||
_, _ = pool.Exec(context.Background(), `
|
||||
UPDATE processing_jobs
|
||||
SET status = 'running', error = NULL, updated_at = now()
|
||||
WHERE id = $1 AND status IN ('failed','cancelled')
|
||||
AND (error ILIKE '%P0%' OR error ILIKE '%parked%' OR error ILIKE '%yield%'
|
||||
OR error ILIKE '%reclaim%' OR error ILIKE '%cleared%' OR error ILIKE '%superseded%'
|
||||
OR error ILIKE '%exclusive%' OR error ILIKE '%batch_%' OR error ILIKE '%inproc%'
|
||||
OR error ILIKE '%formula%')`, jobID)
|
||||
_, _ = pool.Exec(context.Background(), `
|
||||
UPDATE processing_job_products
|
||||
SET status = 'pending', error = NULL, updated_at = now()
|
||||
WHERE job_id = $1 AND status IN ('failed','cancelled')
|
||||
AND processed_product_id IS NULL
|
||||
AND (error ILIKE '%P0%' OR error ILIKE '%parked%' OR error ILIKE '%yield%'
|
||||
OR error ILIKE '%reclaim%' OR error ILIKE '%cleared%' OR error ILIKE '%superseded%'
|
||||
OR error ILIKE '%exclusive%' OR error ILIKE '%batch_%' OR error ILIKE '%inproc%'
|
||||
OR error ILIKE '%formula%')`, jobID)
|
||||
}
|
||||
}
|
||||
}()
|
||||
|
||||
_, _ = pool.Exec(ctx, `
|
||||
UPDATE processing_jobs
|
||||
SET status = 'running', started_at = COALESCE(started_at, now()), updated_at = now()
|
||||
WHERE id = $1`, jobID)
|
||||
|
||||
runCtx, runCancel := context.WithTimeout(ctx, 15*time.Minute)
|
||||
defer runCancel()
|
||||
if err := pipeline.ProcessJob(runCtx, jobID); err != nil {
|
||||
close(defend)
|
||||
log.Fatalf("ProcessJob: %v", err)
|
||||
}
|
||||
close(defend)
|
||||
|
||||
items, err := pipeline.LoadV1ProcessJobItems(ctx, companyID, jobID, "full")
|
||||
if err != nil {
|
||||
log.Fatalf("LoadV1ProcessJobItems: %v", err)
|
||||
}
|
||||
payload := map[string]any{
|
||||
"data": map[string]any{
|
||||
"process_id": jobID.String(),
|
||||
"status": "COMPLETED",
|
||||
"processing_type": "full",
|
||||
"total_items": len(items),
|
||||
"items": items,
|
||||
},
|
||||
}
|
||||
raw, err := json.MarshalIndent(payload, "", " ")
|
||||
if err != nil {
|
||||
log.Fatalf("marshal: %v", err)
|
||||
}
|
||||
if err := os.WriteFile(out+string(os.PathSeparator)+"final.json", raw, 0o644); err != nil {
|
||||
log.Fatalf("write final: %v", err)
|
||||
}
|
||||
fmt.Printf("JOB %s items=%d\n", jobID, len(items))
|
||||
for _, it := range items {
|
||||
ean, _ := it["ean"].(string)
|
||||
st, _ := it["status"].(string)
|
||||
title, _ := it["title"].(string)
|
||||
cat, _ := it["category"].(string)
|
||||
catn, _ := it["category_name"].(string)
|
||||
desc, _ := it["description"].(string)
|
||||
if len(title) > 50 {
|
||||
title = title[:50]
|
||||
}
|
||||
ai, _ := it["ai_provider_mode"].(string)
|
||||
fmt.Printf("ITEM %s status=%s cat=%s/%s ai=%s desc_len=%d title=%s\n",
|
||||
ean, st, cat, catn, ai, len(desc), title)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,83 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"os"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/descrybe/descrybe-v2/apps/api/internal/config"
|
||||
"github.com/descrybe/descrybe-v2/apps/api/internal/db"
|
||||
"github.com/descrybe/descrybe-v2/apps/api/internal/platformsettings"
|
||||
)
|
||||
|
||||
func main() {
|
||||
cfg, err := config.Load()
|
||||
if err != nil {
|
||||
fmt.Println("config:", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
ctx := context.Background()
|
||||
pool, err := db.NewPool(ctx, cfg.DatabaseURL, db.PoolOptions{
|
||||
MaxConns: 2,
|
||||
MinConns: 1,
|
||||
HealthCheckPeriod: 30 * time.Second,
|
||||
MaxConnLifetime: time.Hour,
|
||||
MaxConnIdleTime: 30 * time.Minute,
|
||||
})
|
||||
if err != nil {
|
||||
fmt.Println("db:", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
defer pool.Close()
|
||||
plat := platformsettings.NewService(pool, platformsettings.EnvConfig{
|
||||
AppEncryptionKey: cfg.AppEncryptionKey,
|
||||
CredentialsEncryptionKey: cfg.CredentialsEncryptionKey,
|
||||
TokenSigningSecret: cfg.TokenSigningSecret,
|
||||
DatabaseURL: cfg.DatabaseURL,
|
||||
OpenAIAPIKey: cfg.OpenAIAPIKey,
|
||||
OpenAIBaseURL: cfg.OpenAIBaseURL,
|
||||
OpenAIModel: cfg.OpenAIModel,
|
||||
})
|
||||
rc, err := plat.ResolveAIConfig(ctx, platformsettings.AIRoleProcessing)
|
||||
if err != nil {
|
||||
fmt.Println("resolve:", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
base := strings.TrimRight(strings.TrimSpace(rc.BaseURL), "/")
|
||||
url := base + "/chat/completions"
|
||||
body := map[string]any{
|
||||
"model": rc.Model,
|
||||
"messages": []map[string]string{
|
||||
{"role": "user", "content": "Say PONG"},
|
||||
},
|
||||
"max_tokens": 16,
|
||||
}
|
||||
rawBody, _ := json.Marshal(body)
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodPost, url, bytes.NewReader(rawBody))
|
||||
if err != nil {
|
||||
fmt.Println("req:", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
req.Header.Set("Authorization", "Bearer "+rc.APIKey)
|
||||
cli := &http.Client{Timeout: 30 * time.Second}
|
||||
resp, err := cli.Do(req)
|
||||
if err != nil {
|
||||
fmt.Println("do:", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
raw, _ := io.ReadAll(io.LimitReader(resp.Body, 1200))
|
||||
s := string(raw)
|
||||
s = strings.ReplaceAll(s, rc.APIKey, "[redacted]")
|
||||
fmt.Printf("source=%s provider=%s model=%s\n", rc.Source, rc.Provider, rc.Model)
|
||||
fmt.Printf("url=%s status=%d bytes=%d\n", url, resp.StatusCode, len(raw))
|
||||
fmt.Printf("content_type=%s\n", resp.Header.Get("Content-Type"))
|
||||
fmt.Printf("body_head=%q\n", s)
|
||||
}
|
||||
@@ -0,0 +1,99 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/descrybe/descrybe-v2/apps/api/internal/config"
|
||||
"github.com/descrybe/descrybe-v2/apps/api/internal/db"
|
||||
"github.com/descrybe/descrybe-v2/apps/api/internal/processing"
|
||||
"github.com/google/uuid"
|
||||
)
|
||||
|
||||
func main() {
|
||||
out := `F:\laragon\www\_MY\descrybe-v2\.codehelper\_live_llm_formula_20260816`
|
||||
jobID := uuid.MustParse("3d926dfa-de8a-4afc-b1dd-8707633c673e")
|
||||
cfg, err := config.Load()
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
|
||||
defer cancel()
|
||||
pool, err := db.NewPool(ctx, cfg.DatabaseURL, db.PoolOptions{
|
||||
MaxConns: 4, MinConns: 0,
|
||||
MaxConnLifetime: time.Hour, HealthCheckPeriod: 30 * time.Second,
|
||||
})
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
defer pool.Close()
|
||||
|
||||
rows, err := pool.Query(ctx, `
|
||||
SELECT rp.gtin,
|
||||
COALESCE(pp.processed_name, pp.name, '') AS title,
|
||||
COALESCE(NULLIF(pp.processed_description, ''), pp.description, '') AS description,
|
||||
COALESCE(pp.category, '') AS category,
|
||||
COALESCE(c.name, '') AS category_name,
|
||||
c.description_template,
|
||||
pp.id
|
||||
FROM processing_job_products pjp
|
||||
JOIN raw_products rp ON rp.id = pjp.raw_product_id
|
||||
LEFT JOIN processed_products pp ON pp.id = pjp.processed_product_id
|
||||
LEFT JOIN categories c ON c.company_id = pp.company_id AND c.unique_id = pp.category
|
||||
WHERE pjp.job_id = $1
|
||||
ORDER BY rp.gtin`, jobID)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
defer rows.Close()
|
||||
products := []map[string]any{}
|
||||
for rows.Next() {
|
||||
var gtin, title, desc, cat, catName string
|
||||
var tmpl any
|
||||
var ppID *uuid.UUID
|
||||
if err := rows.Scan(>in, &title, &desc, &cat, &catName, &tmpl, &ppID); err != nil {
|
||||
panic(err)
|
||||
}
|
||||
idStr := ""
|
||||
if ppID != nil {
|
||||
idStr = ppID.String()
|
||||
}
|
||||
lower := strings.ToLower(desc)
|
||||
products = append(products, map[string]any{
|
||||
"ean": gtin,
|
||||
"product_id": idStr,
|
||||
"title": title,
|
||||
"description": desc,
|
||||
"category": cat,
|
||||
"category_name": catName,
|
||||
"description_template": tmpl,
|
||||
"formula_constraint": processing.FormatDescriptionFormulaConstraint(tmpl),
|
||||
"has_h1": strings.Contains(lower, "<h1"),
|
||||
"has_h2": strings.Contains(lower, "<h2"),
|
||||
"has_p": strings.Contains(lower, "<p"),
|
||||
"has_ul": strings.Contains(lower, "<ul"),
|
||||
"desc_len": len(desc),
|
||||
"desc_ne_title": strings.TrimSpace(desc) != "" && strings.TrimSpace(desc) != strings.TrimSpace(title),
|
||||
})
|
||||
}
|
||||
raw, _ := json.MarshalIndent(products, "", " ")
|
||||
_ = os.WriteFile(out+`\db_products.json`, raw, 0o644)
|
||||
fmt.Printf("wrote %d products\n", len(products))
|
||||
for _, p := range products {
|
||||
t := p["title"].(string)
|
||||
if len(t) > 40 {
|
||||
t = t[:40]
|
||||
}
|
||||
fmt.Printf("%s cat=%s/%s h1=%v h2=%v p=%v ul=%v desc_ne_title=%v len=%v title=%q\n",
|
||||
p["ean"], p["category"], p["category_name"], p["has_h1"], p["has_h2"], p["has_p"], p["has_ul"], p["desc_ne_title"], p["desc_len"], t)
|
||||
d := p["description"].(string)
|
||||
if len(d) > 220 {
|
||||
d = d[:220]
|
||||
}
|
||||
fmt.Printf(" desc_preview=%q\n", d)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,171 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"log"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/descrybe/descrybe-v2/apps/api/internal/aiprovider"
|
||||
"github.com/descrybe/descrybe-v2/apps/api/internal/config"
|
||||
"github.com/descrybe/descrybe-v2/apps/api/internal/db"
|
||||
"github.com/descrybe/descrybe-v2/apps/api/internal/platformsettings"
|
||||
"github.com/descrybe/descrybe-v2/apps/api/internal/processing"
|
||||
"github.com/google/uuid"
|
||||
)
|
||||
|
||||
func main() {
|
||||
out := os.Getenv("PROBE_OUT_DIR")
|
||||
if out == "" {
|
||||
out = `F:\laragon\www\_MY\descrybe-v2\.codehelper\_live_llm_enhance_budget_20260816`
|
||||
}
|
||||
_ = os.MkdirAll(out, 0o755)
|
||||
|
||||
cfg, err := config.Load()
|
||||
if err != nil {
|
||||
log.Fatalf("config: %v", err)
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 12*time.Minute)
|
||||
defer cancel()
|
||||
|
||||
pool, err := db.NewPool(ctx, cfg.DatabaseURL, db.PoolOptions{
|
||||
MaxConns: 2,
|
||||
MinConns: 1,
|
||||
HealthCheckPeriod: 30 * time.Second,
|
||||
MaxConnLifetime: time.Hour,
|
||||
MaxConnIdleTime: 30 * time.Minute,
|
||||
})
|
||||
if err != nil {
|
||||
log.Fatalf("db: %v", err)
|
||||
}
|
||||
defer pool.Close()
|
||||
|
||||
plat := platformsettings.NewService(pool, platformsettings.EnvConfig{
|
||||
AppEncryptionKey: cfg.AppEncryptionKey,
|
||||
CredentialsEncryptionKey: cfg.CredentialsEncryptionKey,
|
||||
TokenSigningSecret: cfg.TokenSigningSecret,
|
||||
DatabaseURL: cfg.DatabaseURL,
|
||||
OpenAIAPIKey: cfg.OpenAIAPIKey,
|
||||
OpenAIBaseURL: cfg.OpenAIBaseURL,
|
||||
OpenAIModel: cfg.OpenAIModel,
|
||||
})
|
||||
aiSvc := aiprovider.NewService(pool, aiprovider.EnvConfig{
|
||||
AppEncryptionKey: cfg.AppEncryptionKey,
|
||||
CredentialsEncryptionKey: cfg.CredentialsEncryptionKey,
|
||||
TokenSigningSecret: cfg.TokenSigningSecret,
|
||||
DatabaseURL: cfg.DatabaseURL,
|
||||
OpenAIAPIKey: cfg.OpenAIAPIKey,
|
||||
OpenAIBaseURL: cfg.OpenAIBaseURL,
|
||||
OpenAIModel: cfg.OpenAIModel,
|
||||
ProcessingRPM: cfg.ProcessingRPM,
|
||||
ProcessingMaxRetries: cfg.ProcessingMaxRetries,
|
||||
})
|
||||
aiSvc.Platform = plat
|
||||
|
||||
companyID := uuid.MustParse("2b3159b0-fc08-415b-b248-35ed02a6baab")
|
||||
completer, mode, byok, err := aiSvc.ResolveCompleterForRole(ctx, companyID, aiprovider.RoleProcessing)
|
||||
if err != nil {
|
||||
log.Fatalf("resolve: %v", err)
|
||||
}
|
||||
client, ok := completer.(*processing.OpenAIClient)
|
||||
if !ok || client == nil {
|
||||
log.Fatalf("completer type %T", completer)
|
||||
}
|
||||
if processing.IsMockOrLoopbackBaseURL(client.BaseURL) {
|
||||
log.Fatalf("resolved mock/loopback base=%s — refuse", client.BaseURL)
|
||||
}
|
||||
|
||||
system := `Reply with ONLY one JSON object: {"name":string,"description":string}.
|
||||
description MUST be HTML using exactly these tags in order: <h1>, <p>, <h2>, <ul><li>…</li></ul>, <p>, <p>, <p>.
|
||||
No markdown. No prose outside JSON.`
|
||||
cases := []struct {
|
||||
ean string
|
||||
user string
|
||||
}{
|
||||
{
|
||||
ean: "1200130000638",
|
||||
user: "Category: Bluetooth zvočniki\nName: JBL Flip 6\nDesc: Portable Bluetooth speaker.\nAttrs: {\"brand\":\"JBL\"}",
|
||||
},
|
||||
{
|
||||
ean: "195348253666",
|
||||
user: "Category: Gaming monitorji\nName: Lenovo G27-20\nDesc: 27-inch gaming monitor.\nAttrs: {\"brand\":\"Lenovo\"}",
|
||||
},
|
||||
}
|
||||
|
||||
results := make([]map[string]any, 0, len(cases))
|
||||
okLLM := 0
|
||||
for _, c := range cases {
|
||||
started := time.Now()
|
||||
comp, obj, cerr := processing.CompleteJSON(ctx, client, system, c.user, processing.CompleteOptions{
|
||||
MaxTokens: processing.MaxTokensEnhance,
|
||||
Temperature: processing.DefaultStructuredTemp,
|
||||
})
|
||||
elapsed := time.Since(started).Round(time.Millisecond)
|
||||
fr := ""
|
||||
if m, ok := comp.Raw.(map[string]any); ok {
|
||||
fr, _ = m["finish_reason"].(string)
|
||||
}
|
||||
row := map[string]any{
|
||||
"ean": c.ean,
|
||||
"elapsed": elapsed.String(),
|
||||
"err": "",
|
||||
"finish_reason": fr,
|
||||
"prompt_tokens": comp.PromptTokens,
|
||||
"completion_tokens": comp.OutputTokens,
|
||||
"total_tokens": comp.TotalTokens,
|
||||
"content_runes": len([]rune(comp.Text)),
|
||||
"parsed": obj != nil,
|
||||
"llm_formula_ok": false,
|
||||
}
|
||||
if cerr != nil {
|
||||
row["err"] = processing.TruncateError(cerr)
|
||||
}
|
||||
if obj != nil {
|
||||
desc := fmt.Sprint(obj["description"])
|
||||
row["name"] = fmt.Sprint(obj["name"])
|
||||
row["description_head"] = truncate(desc, 180)
|
||||
hasTags := strings.Contains(desc, "<h1") && strings.Contains(desc, "<h2") &&
|
||||
strings.Contains(desc, "<p") && strings.Contains(desc, "<ul")
|
||||
row["has_h1_h2_p_ul"] = hasTags
|
||||
row["llm_formula_ok"] = cerr == nil && hasTags && strings.EqualFold(fr, "stop")
|
||||
if row["llm_formula_ok"].(bool) {
|
||||
okLLM++
|
||||
}
|
||||
}
|
||||
results = append(results, row)
|
||||
log.Printf("live enhance ean=%s finish_reason=%s tokens=%d/%d/%d elapsed=%s parsed=%v formula_ok=%v err=%v",
|
||||
c.ean, fr, comp.PromptTokens, comp.OutputTokens, comp.TotalTokens, elapsed, obj != nil, row["llm_formula_ok"], row["err"])
|
||||
}
|
||||
|
||||
summary := map[string]any{
|
||||
"at_utc": time.Now().UTC().Format(time.RFC3339),
|
||||
"base": client.BaseURL,
|
||||
"model": client.Model,
|
||||
"mode": mode,
|
||||
"byok": byok,
|
||||
"max_tokens_enhance": processing.MaxTokensEnhance,
|
||||
"max_tokens_retry": processing.MaxTokensEnhanceRetry,
|
||||
"products": len(cases),
|
||||
"llm_formula_ok_count": okLLM,
|
||||
"live_llm_formula_pass": okLLM == len(cases),
|
||||
"results": results,
|
||||
}
|
||||
raw, _ := json.MarshalIndent(summary, "", " ")
|
||||
_ = os.WriteFile(filepath.Join(out, "summary.json"), raw, 0o644)
|
||||
fmt.Println(string(raw))
|
||||
if okLLM != len(cases) {
|
||||
os.Exit(2)
|
||||
}
|
||||
}
|
||||
|
||||
func truncate(s string, n int) string {
|
||||
r := []rune(strings.TrimSpace(s))
|
||||
if len(r) <= n {
|
||||
return string(r)
|
||||
}
|
||||
return string(r[:n]) + "…"
|
||||
}
|
||||
@@ -0,0 +1,443 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"log"
|
||||
"net/http"
|
||||
"os"
|
||||
"os/signal"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"github.com/descrybe/descrybe-v2/apps/api/internal/aiprompts"
|
||||
"github.com/descrybe/descrybe-v2/apps/api/internal/aiprovider"
|
||||
"github.com/descrybe/descrybe-v2/apps/api/internal/config"
|
||||
"github.com/descrybe/descrybe-v2/apps/api/internal/db"
|
||||
"github.com/descrybe/descrybe-v2/apps/api/internal/eprel"
|
||||
"github.com/descrybe/descrybe-v2/apps/api/internal/logredact"
|
||||
"github.com/descrybe/descrybe-v2/apps/api/internal/platformsettings"
|
||||
"github.com/descrybe/descrybe-v2/apps/api/internal/processing"
|
||||
"github.com/google/uuid"
|
||||
)
|
||||
|
||||
func main() {
|
||||
out := os.Getenv("PROBE_OUT_DIR")
|
||||
if out == "" {
|
||||
out = `F:\laragon\www\_MY\descrybe-v2\.codehelper\_live_llm_formula_20260816`
|
||||
}
|
||||
_ = os.MkdirAll(out, 0o755)
|
||||
|
||||
logPath := filepath.Join(out, "inproc_run.log")
|
||||
logFile, err := os.Create(logPath)
|
||||
if err != nil {
|
||||
log.Fatalf("log file: %v", err)
|
||||
}
|
||||
defer logFile.Close()
|
||||
log.SetOutput(logredact.Writer(io.MultiWriter(os.Stderr, logFile)))
|
||||
|
||||
// 5 rich A1 description formulas (h1/h2/p/ul) — Platform Demo.
|
||||
eans := []string{
|
||||
"195348253666", // 7 Gaming monitorji
|
||||
"4548736132597", // 48 Slušalke
|
||||
"3045380024786", // 36 Pečice
|
||||
"3838782459856", // 38 Pomivalni stroji
|
||||
"1200130000638", // 3 Bluetooth zvočniki
|
||||
}
|
||||
companyID := uuid.MustParse("2b3159b0-fc08-415b-b248-35ed02a6baab")
|
||||
userID := uuid.MustParse("0ce3305c-d810-4b56-b1d4-3c1ed510db76")
|
||||
|
||||
cfg, err := config.Load()
|
||||
if err != nil {
|
||||
log.Fatalf("config: %v", err)
|
||||
}
|
||||
ctx, cancel := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
|
||||
defer cancel()
|
||||
|
||||
pool, err := db.NewPool(ctx, cfg.DatabaseURL, db.PoolOptions{
|
||||
MaxConns: int32(cfg.DBMaxConns),
|
||||
MinConns: int32(cfg.DBMinConns),
|
||||
MaxConnLifetime: cfg.DBMaxConnLifetime,
|
||||
MaxConnLifetimeJitter: cfg.DBMaxConnLifetimeJitter,
|
||||
MaxConnIdleTime: cfg.DBMaxConnIdleTime,
|
||||
HealthCheckPeriod: cfg.DBHealthCheckPeriod,
|
||||
StatementTimeout: cfg.DBStatementTimeout,
|
||||
})
|
||||
if err != nil {
|
||||
log.Fatalf("db: %v", err)
|
||||
}
|
||||
defer pool.Close()
|
||||
|
||||
platEnv := platformsettings.EnvConfig{
|
||||
AppEncryptionKey: cfg.AppEncryptionKey,
|
||||
CredentialsEncryptionKey: cfg.CredentialsEncryptionKey,
|
||||
TokenSigningSecret: cfg.TokenSigningSecret,
|
||||
DatabaseURL: cfg.DatabaseURL,
|
||||
OpenAIAPIKey: cfg.OpenAIAPIKey,
|
||||
OpenAIBaseURL: cfg.OpenAIBaseURL,
|
||||
OpenAIModel: cfg.OpenAIModel,
|
||||
OpenAIEmbeddingAPIKey: cfg.OpenAIEmbeddingAPIKey,
|
||||
OpenAIEmbeddingBaseURL: cfg.OpenAIEmbeddingBaseURL,
|
||||
OpenAIEmbeddingModel: cfg.OpenAIEmbeddingModel,
|
||||
}
|
||||
platSettings := platformsettings.NewService(pool, platEnv)
|
||||
aiSvc := aiprovider.NewService(pool, aiprovider.EnvConfig{
|
||||
AppEncryptionKey: cfg.AppEncryptionKey,
|
||||
CredentialsEncryptionKey: cfg.CredentialsEncryptionKey,
|
||||
TokenSigningSecret: cfg.TokenSigningSecret,
|
||||
DatabaseURL: cfg.DatabaseURL,
|
||||
OpenAIAPIKey: cfg.OpenAIAPIKey,
|
||||
OpenAIBaseURL: cfg.OpenAIBaseURL,
|
||||
OpenAIModel: cfg.OpenAIModel,
|
||||
ProcessingRPM: cfg.ProcessingRPM,
|
||||
ProcessingMaxRetries: cfg.ProcessingMaxRetries,
|
||||
})
|
||||
aiSvc.Platform = platSettings
|
||||
|
||||
// Resolve like worker (DB admin AI / company BYOK preferred over mock .env).
|
||||
roleCfg, rerr := platSettings.ResolveAIConfig(ctx, platformsettings.AIRoleProcessing)
|
||||
if rerr != nil {
|
||||
failResolve(out, fmt.Sprintf("ResolveAIConfig: %v", rerr))
|
||||
}
|
||||
completer, modeLabel, byok, cerr := aiSvc.ResolveCompleterForRole(ctx, companyID, aiprovider.RoleProcessing)
|
||||
if cerr != nil {
|
||||
failResolve(out, fmt.Sprintf("ResolveCompleterForRole: %v", cerr))
|
||||
}
|
||||
if completer == nil {
|
||||
failResolve(out, "ResolveCompleterForRole returned nil completer (OpenAI not configured in admin/BYOK/env)")
|
||||
}
|
||||
client, ok := completer.(*processing.OpenAIClient)
|
||||
if !ok {
|
||||
failResolve(out, fmt.Sprintf("completer type %T is not *OpenAIClient", completer))
|
||||
}
|
||||
base := strings.TrimSpace(client.BaseURL)
|
||||
model := strings.TrimSpace(client.Model)
|
||||
resolveInfo := map[string]any{
|
||||
"role_source": roleCfg.Source,
|
||||
"role_provider": roleCfg.Provider,
|
||||
"role_base_url": roleCfg.BaseURL,
|
||||
"role_model": roleCfg.Model,
|
||||
"role_enabled": roleCfg.Enabled,
|
||||
"completer_base": base,
|
||||
"completer_model": model,
|
||||
"mode_label": modeLabel,
|
||||
"byok": byok,
|
||||
"env_base_url": cfg.OpenAIBaseURL,
|
||||
"env_model": cfg.OpenAIModel,
|
||||
"live": true,
|
||||
}
|
||||
if processing.IsMockOrLoopbackBaseURL(base) || strings.Contains(strings.ToLower(base), "18767") {
|
||||
resolveInfo["live"] = false
|
||||
resolveInfo["fail_reason"] = "resolved completer is mock/loopback — refuse silent mock fallback"
|
||||
writeJSON(out, "resolve.json", resolveInfo)
|
||||
log.Fatalf("LIVE LLM REQUIRED: resolved base=%s model=%s source=%s mode=%s — refuse mock :18767 / loopback",
|
||||
base, model, roleCfg.Source, modeLabel)
|
||||
}
|
||||
if strings.TrimSpace(client.APIKey) == "" {
|
||||
resolveInfo["live"] = false
|
||||
resolveInfo["fail_reason"] = "empty API key"
|
||||
writeJSON(out, "resolve.json", resolveInfo)
|
||||
log.Fatal("LIVE LLM REQUIRED: API key empty after resolve")
|
||||
}
|
||||
writeJSON(out, "resolve.json", resolveInfo)
|
||||
log.Printf("live resolve source=%s base=%s model=%s mode=%s byok=%v (env was base=%s model=%s)",
|
||||
roleCfg.Source, base, model, modeLabel, byok, cfg.OpenAIBaseURL, cfg.OpenAIModel)
|
||||
|
||||
// Connectivity probe — GET /v1/models then tiny chat (admin DB key; never log full key).
|
||||
modelsURL := strings.TrimRight(base, "/") + "/models"
|
||||
keyHint := ""
|
||||
if k := strings.TrimSpace(client.APIKey); len(k) > 8 {
|
||||
keyHint = k[:4] + "…" + k[len(k)-4:]
|
||||
} else if k := strings.TrimSpace(client.APIKey); k != "" {
|
||||
keyHint = "(short)"
|
||||
}
|
||||
resolveInfo["api_key_hint"] = keyHint
|
||||
resolveInfo["retried_at_utc"] = time.Now().UTC().Format(time.RFC3339)
|
||||
|
||||
modelsCtx, modelsCancel := context.WithTimeout(ctx, 30*time.Second)
|
||||
defer modelsCancel()
|
||||
modelsReq, merr := http.NewRequestWithContext(modelsCtx, http.MethodGet, modelsURL, nil)
|
||||
if merr != nil {
|
||||
resolveInfo["probe_ok"] = false
|
||||
resolveInfo["fail_reason"] = "models request build: " + merr.Error()
|
||||
writeJSON(out, "resolve.json", resolveInfo)
|
||||
log.Fatalf("LIVE LLM CONNECTION ERROR: models req: %v", merr)
|
||||
}
|
||||
modelsReq.Header.Set("Authorization", "Bearer "+client.APIKey)
|
||||
modelsReq.Header.Set("Accept", "application/json")
|
||||
modelsStart := time.Now()
|
||||
modelsResp, merr := http.DefaultClient.Do(modelsReq)
|
||||
modelsElapsed := time.Since(modelsStart).Round(time.Millisecond)
|
||||
if merr != nil {
|
||||
resolveInfo["probe_ok"] = false
|
||||
resolveInfo["models_url"] = modelsURL
|
||||
resolveInfo["models_error"] = processing.TruncateError(merr)
|
||||
resolveInfo["models_elapsed"] = modelsElapsed.String()
|
||||
resolveInfo["fail_reason"] = "GET /v1/models network error"
|
||||
writeJSON(out, "resolve.json", resolveInfo)
|
||||
log.Fatalf("LIVE LLM CONNECTION ERROR: GET %s elapsed=%s err=%s",
|
||||
modelsURL, modelsElapsed, processing.TruncateError(merr))
|
||||
}
|
||||
modelsBody, _ := io.ReadAll(io.LimitReader(modelsResp.Body, 2048))
|
||||
_ = modelsResp.Body.Close()
|
||||
modelsSnippet := strings.TrimSpace(string(modelsBody))
|
||||
if len(modelsSnippet) > 240 {
|
||||
modelsSnippet = modelsSnippet[:240] + "…"
|
||||
}
|
||||
resolveInfo["models_url"] = modelsURL
|
||||
resolveInfo["models_http_status"] = modelsResp.StatusCode
|
||||
resolveInfo["models_content_type"] = modelsResp.Header.Get("Content-Type")
|
||||
resolveInfo["models_elapsed"] = modelsElapsed.String()
|
||||
resolveInfo["models_body_snippet"] = modelsSnippet
|
||||
if modelsResp.StatusCode != http.StatusOK {
|
||||
resolveInfo["probe_ok"] = false
|
||||
resolveInfo["http_status"] = modelsResp.StatusCode
|
||||
resolveInfo["fail_reason"] = fmt.Sprintf("GET /v1/models returned HTTP %d (expect 200; 401=wrong/missing key, 502=upstream)", modelsResp.StatusCode)
|
||||
writeJSON(out, "resolve.json", resolveInfo)
|
||||
log.Fatalf("LIVE LLM CONNECTION ERROR: GET %s status=%d ct=%s snippet=%q",
|
||||
modelsURL, modelsResp.StatusCode, modelsResp.Header.Get("Content-Type"), modelsSnippet)
|
||||
}
|
||||
log.Printf("live models ok status=200 elapsed=%s key_hint=%s", modelsElapsed, keyHint)
|
||||
|
||||
probeCtx, probeCancel := context.WithTimeout(ctx, 45*time.Second)
|
||||
defer probeCancel()
|
||||
probeStart := time.Now()
|
||||
comp, perr := client.Complete(probeCtx, "Reply with exactly: PONG", "Say PONG")
|
||||
probeElapsed := time.Since(probeStart).Round(time.Millisecond)
|
||||
if perr != nil {
|
||||
resolveInfo["probe_ok"] = false
|
||||
resolveInfo["probe_error"] = processing.TruncateError(perr)
|
||||
resolveInfo["probe_elapsed"] = probeElapsed.String()
|
||||
resolveInfo["fail_reason"] = "chat/completions probe failed after models 200"
|
||||
writeJSON(out, "resolve.json", resolveInfo)
|
||||
log.Fatalf("LIVE LLM CONNECTION ERROR: base=%s model=%s elapsed=%s err=%s",
|
||||
base, model, probeElapsed, processing.TruncateError(perr))
|
||||
}
|
||||
resolveInfo["probe_ok"] = true
|
||||
resolveInfo["probe_elapsed"] = probeElapsed.String()
|
||||
resolveInfo["probe_response_len"] = len(comp.Text)
|
||||
writeJSON(out, "resolve.json", resolveInfo)
|
||||
log.Printf("live probe ok elapsed=%s response_len=%d", probeElapsed, len(comp.Text))
|
||||
|
||||
pipeline := processing.NewPipeline(pool)
|
||||
pipeline.BatchSize = cfg.ProcessingBatchSize
|
||||
pipeline.AI = aiSvc
|
||||
pipeline.Prompts = aiprompts.NewService(pool)
|
||||
eprelClient := eprel.NewClient(eprel.Options{Enabled: true, Timeout: 20 * time.Second})
|
||||
pipeline.Engine = &processing.Engine{
|
||||
Completer: nil, // per-job via pipeline.AI
|
||||
Vector: processing.NoopVectorCategorizer{},
|
||||
EPREL: eprelClient,
|
||||
ProviderMode: processing.AIProviderInternal,
|
||||
}
|
||||
|
||||
rawIDs := make([]uuid.UUID, 0, len(eans))
|
||||
for _, ean := range eans {
|
||||
var id uuid.UUID
|
||||
err := pool.QueryRow(ctx, `
|
||||
SELECT id FROM raw_products
|
||||
WHERE company_id = $1 AND gtin = $2`, companyID, ean).Scan(&id)
|
||||
if err != nil {
|
||||
log.Fatalf("raw product %s: %v", ean, err)
|
||||
}
|
||||
rawIDs = append(rawIDs, id)
|
||||
log.Printf("ean=%s raw_id=%s", ean, id)
|
||||
}
|
||||
|
||||
// Force re-enhance: clear prior enhance_input_hash for these raw products.
|
||||
tag, cerr2 := pool.Exec(ctx, `
|
||||
UPDATE processed_products pp
|
||||
SET field_sources = COALESCE(field_sources, '{}'::jsonb) - 'enhance_input_hash',
|
||||
localized = CASE
|
||||
WHEN localized IS NULL OR localized = '{}'::jsonb THEN localized
|
||||
ELSE (
|
||||
SELECT COALESCE(jsonb_object_agg(lang, val - 'enhance_input_hash'), '{}'::jsonb)
|
||||
FROM jsonb_each(localized) AS t(lang, val)
|
||||
)
|
||||
END,
|
||||
updated_at = now()
|
||||
FROM raw_products rp
|
||||
WHERE pp.company_id = $1
|
||||
AND pp.raw_product_id = rp.id
|
||||
AND rp.gtin = ANY($2::text[])`, companyID, eans)
|
||||
if cerr2 != nil {
|
||||
log.Printf("clear enhance hash: %v", cerr2)
|
||||
} else {
|
||||
log.Printf("cleared enhance_input_hash rows=%d", tag.RowsAffected())
|
||||
}
|
||||
|
||||
if n, err := processing.BackfillMissingMeta(ctx, pool, companyID); err != nil {
|
||||
log.Printf("BackfillMissingMeta: %v", err)
|
||||
} else {
|
||||
log.Printf("BackfillMissingMeta updated=%d", n)
|
||||
}
|
||||
|
||||
// Soft-yield competing queue noise (do not mass-fail the whole company queue).
|
||||
_, _ = pool.Exec(ctx, `
|
||||
UPDATE processing_jobs
|
||||
SET status = 'failed', error = 'yielded to live LLM formula probe', updated_at = now()
|
||||
WHERE company_id = $1 AND status IN ('pending','running','processing')`, companyID)
|
||||
|
||||
jobs, err := pipeline.StartJob(ctx, companyID, userID, rawIDs, "full")
|
||||
if err != nil {
|
||||
log.Fatalf("StartJob: %v", err)
|
||||
}
|
||||
if len(jobs) == 0 {
|
||||
log.Fatal("StartJob returned no jobs")
|
||||
}
|
||||
jobID := jobs[0].ID
|
||||
log.Printf("job_id=%s chunks=%d", jobID, len(jobs))
|
||||
_ = os.WriteFile(filepath.Join(out, "process_id.txt"), []byte(jobID.String()+"\n"), 0o644)
|
||||
|
||||
defend := make(chan struct{})
|
||||
go func() {
|
||||
t := time.NewTicker(500 * time.Millisecond)
|
||||
defer t.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-defend:
|
||||
return
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-t.C:
|
||||
_, _ = pool.Exec(context.Background(), `
|
||||
UPDATE processing_jobs
|
||||
SET status = 'failed', error = 'yielded to live LLM formula probe', updated_at = now()
|
||||
WHERE company_id = $1 AND status IN ('pending','running','processing') AND id <> $2`, companyID, jobID)
|
||||
_, _ = pool.Exec(context.Background(), `
|
||||
UPDATE processing_jobs
|
||||
SET status = 'running', error = NULL, updated_at = now()
|
||||
WHERE id = $1 AND status IN ('failed','cancelled')
|
||||
AND (error ILIKE '%P0%' OR error ILIKE '%parked%' OR error ILIKE '%yield%'
|
||||
OR error ILIKE '%reclaim%' OR error ILIKE '%cleared%' OR error ILIKE '%superseded%'
|
||||
OR error ILIKE '%exclusive%' OR error ILIKE '%batch_%' OR error ILIKE '%inproc%'
|
||||
OR error ILIKE '%formula%' OR error ILIKE '%live LLM%')`, jobID)
|
||||
_, _ = pool.Exec(context.Background(), `
|
||||
UPDATE processing_job_products
|
||||
SET status = 'pending', error = NULL, updated_at = now()
|
||||
WHERE job_id = $1 AND status IN ('failed','cancelled')
|
||||
AND processed_product_id IS NULL
|
||||
AND (error ILIKE '%P0%' OR error ILIKE '%parked%' OR error ILIKE '%yield%'
|
||||
OR error ILIKE '%reclaim%' OR error ILIKE '%cleared%' OR error ILIKE '%superseded%'
|
||||
OR error ILIKE '%exclusive%' OR error ILIKE '%batch_%' OR error ILIKE '%inproc%'
|
||||
OR error ILIKE '%formula%' OR error ILIKE '%live LLM%')`, jobID)
|
||||
}
|
||||
}
|
||||
}()
|
||||
|
||||
_, _ = pool.Exec(ctx, `
|
||||
UPDATE processing_jobs
|
||||
SET status = 'running', started_at = COALESCE(started_at, now()), updated_at = now()
|
||||
WHERE id = $1`, jobID)
|
||||
|
||||
runCtx, runCancel := context.WithTimeout(ctx, 20*time.Minute)
|
||||
defer runCancel()
|
||||
if err := pipeline.ProcessJob(runCtx, jobID); err != nil {
|
||||
close(defend)
|
||||
log.Fatalf("ProcessJob: %v", err)
|
||||
}
|
||||
close(defend)
|
||||
|
||||
items, err := pipeline.LoadV1ProcessJobItems(ctx, companyID, jobID, "full")
|
||||
if err != nil {
|
||||
log.Fatalf("LoadV1ProcessJobItems: %v", err)
|
||||
}
|
||||
payload := map[string]any{
|
||||
"data": map[string]any{
|
||||
"process_id": jobID.String(),
|
||||
"status": "COMPLETED",
|
||||
"processing_type": "full",
|
||||
"total_items": len(items),
|
||||
"items": items,
|
||||
"llm": map[string]any{
|
||||
"base": base,
|
||||
"model": model,
|
||||
"source": roleCfg.Source,
|
||||
"mode": modeLabel,
|
||||
"byok": byok,
|
||||
},
|
||||
},
|
||||
}
|
||||
writeJSON(out, "final.json", payload)
|
||||
|
||||
// DB HTML descriptions (V1 may strip tags).
|
||||
rows, err := pool.Query(ctx, `
|
||||
SELECT rp.gtin,
|
||||
COALESCE(pp.processed_name, pp.name, ''),
|
||||
COALESCE(NULLIF(pp.processed_description, ''), pp.description, ''),
|
||||
COALESCE(pp.category, ''),
|
||||
COALESCE(c.name, ''),
|
||||
COALESCE(c.unique_id, ''),
|
||||
c.description_template,
|
||||
pp.id
|
||||
FROM processing_job_products pjp
|
||||
JOIN raw_products rp ON rp.id = pjp.raw_product_id
|
||||
LEFT JOIN processed_products pp ON pp.id = pjp.processed_product_id
|
||||
LEFT JOIN categories c ON c.company_id = pp.company_id AND c.unique_id = pp.category
|
||||
WHERE pjp.job_id = $1
|
||||
ORDER BY rp.gtin`, jobID)
|
||||
if err != nil {
|
||||
log.Fatalf("db products: %v", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
dbProducts := make([]map[string]any, 0)
|
||||
for rows.Next() {
|
||||
var gtin, name, desc, cat, catName, catUID string
|
||||
var tmpl any
|
||||
var ppID *uuid.UUID
|
||||
if err := rows.Scan(>in, &name, &desc, &cat, &catName, &catUID, &tmpl, &ppID); err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
idStr := ""
|
||||
if ppID != nil {
|
||||
idStr = ppID.String()
|
||||
}
|
||||
dbProducts = append(dbProducts, map[string]any{
|
||||
"ean": gtin,
|
||||
"product_id": idStr,
|
||||
"title": name,
|
||||
"description": desc,
|
||||
"category": cat,
|
||||
"category_name": catName,
|
||||
"category_unique_id": catUID,
|
||||
"description_template": tmpl,
|
||||
"formula_constraint": processing.FormatDescriptionFormulaConstraint(tmpl),
|
||||
})
|
||||
}
|
||||
writeJSON(out, "db_products.json", dbProducts)
|
||||
|
||||
fmt.Printf("JOB %s items=%d live_base=%s model=%s\n", jobID, len(items), base, model)
|
||||
for _, p := range dbProducts {
|
||||
ean, _ := p["ean"].(string)
|
||||
title, _ := p["title"].(string)
|
||||
cat, _ := p["category"].(string)
|
||||
catn, _ := p["category_name"].(string)
|
||||
desc, _ := p["description"].(string)
|
||||
if len(title) > 50 {
|
||||
title = title[:50]
|
||||
}
|
||||
fmt.Printf("ITEM %s cat=%s/%s desc_len=%d title=%s\n", ean, cat, catn, len(desc), title)
|
||||
}
|
||||
}
|
||||
|
||||
func failResolve(out, msg string) {
|
||||
writeJSON(out, "resolve.json", map[string]any{
|
||||
"live": false,
|
||||
"fail_reason": msg,
|
||||
})
|
||||
log.Fatalf("LIVE LLM REQUIRED: %s", msg)
|
||||
}
|
||||
|
||||
func writeJSON(out, name string, v any) {
|
||||
raw, err := json.MarshalIndent(v, "", " ")
|
||||
if err != nil {
|
||||
log.Fatalf("marshal %s: %v", name, err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(out, name), raw, 0o644); err != nil {
|
||||
log.Fatalf("write %s: %v", name, err)
|
||||
}
|
||||
}
|
||||
@@ -23,7 +23,8 @@ type RepairWeakEnhanceHashesResult struct {
|
||||
|
||||
// RepairWeakEnhanceHashes clears enhance_input_hash from field_sources and
|
||||
// localized_content for processed products whose descriptions are weak /
|
||||
// title-echo / filler (so the next enhance cannot hash-skip thin priors).
|
||||
// title-echo / filler / invent-synthesize (so the next enhance cannot hash-skip
|
||||
// thin or heuristic priors).
|
||||
// Returns the number of products updated.
|
||||
func RepairWeakEnhanceHashes(ctx context.Context, pool *pgxpool.Pool, companyID uuid.UUID) (int64, error) {
|
||||
res, err := RepairWeakEnhanceHashesDetailed(ctx, pool, companyID)
|
||||
@@ -124,7 +125,7 @@ func repairWeakEnhanceHashesTx(ctx context.Context, tx pgx.Tx, companyID uuid.UU
|
||||
}
|
||||
|
||||
if _, has := fs[enhanceInputHashKey]; has {
|
||||
if company.IsWeakPriorEnhanceDescription(primaryDesc, primaryName, name) {
|
||||
if company.ShouldRefuseEnhanceHashSkip(primaryDesc, primaryName, name) {
|
||||
delete(fs, enhanceInputHashKey)
|
||||
changed = true
|
||||
}
|
||||
@@ -139,7 +140,7 @@ func repairWeakEnhanceHashesTx(ctx context.Context, tx pgx.Tx, companyID uuid.UU
|
||||
if strings.TrimSpace(fields.EnhanceInputHash) == "" {
|
||||
continue
|
||||
}
|
||||
if company.IsWeakPriorEnhanceDescription(d, n, primaryName, name) {
|
||||
if company.ShouldRefuseEnhanceHashSkip(d, n, primaryName, name) {
|
||||
fields.EnhanceInputHash = ""
|
||||
loc[lang] = fields
|
||||
changed = true
|
||||
|
||||
@@ -950,7 +950,7 @@ func (s *Service) ListRawProducts(ctx context.Context, companyID uuid.UUID, f Li
|
||||
SELECT rp.id, rp.gtin, rp.feed_id, rp.is_processed, rp.processing_status,
|
||||
COALESCE(NULLIF(rp.mapped_data->>'name', ''), NULLIF(rp.mapped_data->>'title', ''), '') AS name,
|
||||
NULLIF(BTRIM(rp.mapped_data->>'category'), '') AS category,
|
||||
COALESCE(cat.name, NULLIF(BTRIM(rp.mapped_data->>'category'), '')) AS category_name,
|
||||
cat.name AS category_name,
|
||||
COALESCE(cat.unique_id, NULLIF(BTRIM(rp.mapped_data->>'category'), '')) AS category_unique_id,
|
||||
f.name AS feed_name,
|
||||
rp.mapped_data->'_sync_changes' AS sync_changes,
|
||||
@@ -1099,7 +1099,7 @@ func (s *Service) ListProcessedProducts(ctx context.Context, companyID uuid.UUID
|
||||
`+processedPreferredNameSQL+` AS name,
|
||||
COALESCE(NULLIF(p.processed_name, ''), '') AS processed_name,
|
||||
p.category,
|
||||
COALESCE(cat.name, NULLIF(BTRIM(p.category), '')) AS category_name,
|
||||
cat.name AS category_name,
|
||||
COALESCE(cat.unique_id, NULLIF(BTRIM(p.category), '')) AS category_unique_id,
|
||||
p.status, p.raw_product_id, COALESCE(p.feed_id, r.feed_id) AS feed_id, r.gtin,
|
||||
f.name AS feed_name,
|
||||
@@ -1186,7 +1186,7 @@ func (s *Service) ListProcessedProductsDetailed(ctx context.Context, companyID u
|
||||
SELECT p.id, p.product_id,
|
||||
`+processedPreferredNameSQL+` AS name,
|
||||
p.category,
|
||||
COALESCE(cat.name, NULLIF(BTRIM(p.category), '')) AS category_name,
|
||||
cat.name AS category_name,
|
||||
COALESCE(cat.unique_id, NULLIF(BTRIM(p.category), '')) AS category_unique_id,
|
||||
p.status, p.raw_product_id, p.feed_id, r.gtin,
|
||||
`+processedPreferredDescriptionSQL+` AS description,
|
||||
@@ -1226,7 +1226,7 @@ func (s *Service) GetProcessedProduct(ctx context.Context, companyID, id uuid.UU
|
||||
SELECT p.id, p.product_id,
|
||||
`+processedPreferredNameSQL+` AS name,
|
||||
p.category,
|
||||
COALESCE(cat.name, NULLIF(BTRIM(p.category), '')) AS category_name,
|
||||
cat.name AS category_name,
|
||||
COALESCE(cat.unique_id, NULLIF(BTRIM(p.category), '')) AS category_unique_id,
|
||||
`+processedPreferredDescriptionSQL+` AS description,
|
||||
p.processed_name, p.processed_description,
|
||||
@@ -1279,7 +1279,7 @@ func (s *Service) GetRawProduct(ctx context.Context, companyID, id uuid.UUID) (m
|
||||
COALESCE(NULLIF(rp.gtin, ''), '') AS product_id,
|
||||
COALESCE(NULLIF(rp.mapped_data->>'name', ''), NULLIF(rp.mapped_data->>'title', ''), '') AS name,
|
||||
NULLIF(BTRIM(rp.mapped_data->>'category'), '') AS category,
|
||||
COALESCE(cat.name, NULLIF(BTRIM(rp.mapped_data->>'category'), '')) AS category_name,
|
||||
cat.name AS category_name,
|
||||
COALESCE(cat.unique_id, NULLIF(BTRIM(rp.mapped_data->>'category'), '')) AS category_unique_id,
|
||||
COALESCE(NULLIF(rp.mapped_data->>'description', ''), '') AS description,
|
||||
''::text AS processed_name,
|
||||
|
||||
@@ -25,25 +25,98 @@ var weakFillerPhrases = []string{
|
||||
|
||||
// IsWeakPriorEnhanceDescription reports empty, too-short, title-echo, known filler,
|
||||
// or short boilerplate without overlapping tokens from title/fact sources.
|
||||
// Heuristic synthesize (invent) is intentionally excluded — use ShouldRefuseEnhanceHashSkip.
|
||||
func IsWeakPriorEnhanceDescription(priorDesc string, titles ...string) bool {
|
||||
priorDesc = strings.TrimSpace(priorDesc)
|
||||
if priorDesc == "" || priorDesc == "<nil>" {
|
||||
return true
|
||||
}
|
||||
if len([]rune(priorDesc)) < minUsableProductDescRunes {
|
||||
return true
|
||||
}
|
||||
for _, title := range titles {
|
||||
if DescriptionEchoesTitle(priorDesc, title) {
|
||||
reason := EnhanceHashSkipBlockReason(priorDesc, titles...)
|
||||
return reason == "weak" || reason == "title-echo"
|
||||
}
|
||||
|
||||
// heuristicSynthesizePhrases are distinctive invent / formula-skeleton snippets from
|
||||
// synthesizeDescriptionFromTitle / synthesizeDescriptionFromFormula. These may look
|
||||
// "strong" enough to pass IsWeakPriorEnhanceDescription but must never hash-skip
|
||||
// enhance (would leave fallback copy forever on reprocess).
|
||||
var heuristicSynthesizePhrases = []string{
|
||||
"is a catalog product with the known attributes",
|
||||
"is listed in the ",
|
||||
". key specs:",
|
||||
"je katalogski izdelek z znanimi atributi",
|
||||
"je izdelek v kategoriji",
|
||||
"je izdelek znamke",
|
||||
". ključne specifikacije:",
|
||||
}
|
||||
|
||||
// LooksLikeHeuristicSynthesize reports invent / formula-skeleton fallback copy.
|
||||
func LooksLikeHeuristicSynthesize(desc string) bool {
|
||||
lower := strings.ToLower(desc)
|
||||
for _, p := range heuristicSynthesizePhrases {
|
||||
if p != "" && strings.Contains(lower, p) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
if ContainsWeakFillerPhrase(priorDesc) {
|
||||
// EN invent: "<title> is a <category> product from <brand>"
|
||||
if strings.Contains(lower, " is a ") && strings.Contains(lower, " product from ") {
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// EnhanceHashSkipBlockReason returns a stable reason when prior description must
|
||||
// not hash-skip the enhance LLM: "weak", "title-echo", "synth", or "".
|
||||
func EnhanceHashSkipBlockReason(priorDesc string, titles ...string) string {
|
||||
priorDesc = strings.TrimSpace(priorDesc)
|
||||
if priorDesc == "" || priorDesc == "<nil>" {
|
||||
return "weak"
|
||||
}
|
||||
if len([]rune(priorDesc)) < minUsableProductDescRunes {
|
||||
return "weak"
|
||||
}
|
||||
for _, title := range titles {
|
||||
if DescriptionEchoesTitle(priorDesc, title) {
|
||||
return "title-echo"
|
||||
}
|
||||
}
|
||||
if ContainsWeakFillerPhrase(priorDesc) {
|
||||
return "weak"
|
||||
}
|
||||
if LooksLikeHeuristicSynthesize(priorDesc) {
|
||||
return "synth"
|
||||
}
|
||||
if len([]rune(priorDesc)) <= shortBoilerplateDescRunes &&
|
||||
!descriptionOverlapsProductFacts(priorDesc, titles...) {
|
||||
return true
|
||||
return "weak"
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// ShouldRefuseEnhanceHashSkip is true when a stored enhance_input_hash must be
|
||||
// ignored / cleared (weak, title-echo, or heuristic synthesize).
|
||||
func ShouldRefuseEnhanceHashSkip(priorDesc string, titles ...string) bool {
|
||||
return EnhanceHashSkipBlockReason(priorDesc, titles...) != ""
|
||||
}
|
||||
|
||||
// DescriptionMissingFormulaHTMLTags is true when sectionTypes require HTML tags
|
||||
// (h1/h2/h3/h4, p, ul) that are absent from desc — formula-mismatch for skip/clear.
|
||||
func DescriptionMissingFormulaHTMLTags(desc string, sectionTypes []string) bool {
|
||||
if len(sectionTypes) == 0 {
|
||||
return false
|
||||
}
|
||||
lower := strings.ToLower(desc)
|
||||
for _, raw := range sectionTypes {
|
||||
typ := strings.ToLower(strings.TrimSpace(raw))
|
||||
switch typ {
|
||||
case "h1", "h2", "h3", "h4":
|
||||
if !strings.Contains(lower, "<"+typ) {
|
||||
return true
|
||||
}
|
||||
case "ul":
|
||||
if !strings.Contains(lower, "<ul") {
|
||||
return true
|
||||
}
|
||||
case "p", "":
|
||||
if !strings.Contains(lower, "<p") {
|
||||
return true
|
||||
}
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
@@ -0,0 +1,40 @@
|
||||
package company
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestLooksLikeHeuristicSynthesize(t *testing.T) {
|
||||
t.Parallel()
|
||||
title := "Vogel's WALL 3245 TV Wall Mount"
|
||||
en := title + " is a TV Mounts product from Vogel's. Key specs: width 45 cm, max_load 40 kg."
|
||||
if !LooksLikeHeuristicSynthesize(en) {
|
||||
t.Fatalf("expected EN invent detected: %q", en)
|
||||
}
|
||||
sl := title + " je izdelek v kategoriji TV Mounts znamke Vogel's. Ključne specifikacije: width 45 cm."
|
||||
if !LooksLikeHeuristicSynthesize(sl) {
|
||||
t.Fatalf("expected SL invent detected: %q", sl)
|
||||
}
|
||||
good := "Vogel's WALL 3245 is a sturdy TV mount with clear load ratings and install guidance for wall displays."
|
||||
if LooksLikeHeuristicSynthesize(good) {
|
||||
t.Fatalf("retail copy must not look like invent: %q", good)
|
||||
}
|
||||
if !ShouldRefuseEnhanceHashSkip(en, title) {
|
||||
t.Fatal("invent must refuse hash skip")
|
||||
}
|
||||
if ShouldRefuseEnhanceHashSkip(good, title) {
|
||||
t.Fatal("good retail copy must allow hash skip")
|
||||
}
|
||||
if !ShouldRefuseEnhanceHashSkip(title, title) {
|
||||
t.Fatal("title-echo must refuse hash skip")
|
||||
}
|
||||
}
|
||||
|
||||
func TestDescriptionMissingFormulaHTMLTags(t *testing.T) {
|
||||
t.Parallel()
|
||||
types := []string{"h1", "p", "ul"}
|
||||
if !DescriptionMissingFormulaHTMLTags("plain prose without tags", types) {
|
||||
t.Fatal("plain prose must mismatch")
|
||||
}
|
||||
if DescriptionMissingFormulaHTMLTags("<h1>T</h1><p>Body</p><ul><li>x</li></ul>", types) {
|
||||
t.Fatal("full HTML must match")
|
||||
}
|
||||
}
|
||||
@@ -112,6 +112,14 @@ type ProductInput struct {
|
||||
PriorProcessedDescription string
|
||||
PriorCategory string
|
||||
PriorLocalized company.LocalizedContent
|
||||
// JobID / CompanyID / RawProductID are optional worker correlation ids for
|
||||
// structured ai_enhance logs (never secrets). Empty in unit tests.
|
||||
JobID string
|
||||
CompanyID string
|
||||
RawProductID string
|
||||
// CategoryUniqueID is the taxonomy unique_id for overlay/formula keys when
|
||||
// the enhance display category argument is a localized name.
|
||||
CategoryUniqueID string
|
||||
}
|
||||
|
||||
// CategoryFormulas holds optional title/description templates for one category key.
|
||||
|
||||
@@ -356,12 +356,12 @@ func RecommendReprocessRawProductIDs(ctx context.Context, pool *pgxpool.Pool, co
|
||||
if isPromptLabelTitle(primaryName) || isPromptLabelTitle(processedName) || isPromptLabelTitle(name) {
|
||||
needs = true
|
||||
}
|
||||
if isWeakPriorEnhanceDescription(primaryDesc, primaryName, name) {
|
||||
if company.ShouldRefuseEnhanceHashSkip(primaryDesc, primaryName, name) {
|
||||
needs = true
|
||||
}
|
||||
if _, has := fs[FieldEnhanceInputHash]; !has {
|
||||
// Missing hash after weak clear / never enhanced — recommend when desc weak or empty category.
|
||||
if isWeakPriorEnhanceDescription(primaryDesc, primaryName, name) || strings.TrimSpace(category) == "" {
|
||||
if company.ShouldRefuseEnhanceHashSkip(primaryDesc, primaryName, name) || strings.TrimSpace(category) == "" {
|
||||
needs = true
|
||||
}
|
||||
}
|
||||
@@ -371,7 +371,7 @@ func RecommendReprocessRawProductIDs(ctx context.Context, pool *pgxpool.Pool, co
|
||||
if n == "" {
|
||||
n = primaryName
|
||||
}
|
||||
if fields.EnhanceInputHash == "" && isWeakPriorEnhanceDescription(d, n, primaryName, name) {
|
||||
if fields.EnhanceInputHash == "" && company.ShouldRefuseEnhanceHashSkip(d, n, primaryName, name) {
|
||||
needs = true
|
||||
break
|
||||
}
|
||||
|
||||
@@ -7,13 +7,21 @@ import (
|
||||
)
|
||||
|
||||
// categoryDisplayLabel returns the human category name for meta / synthesize /
|
||||
// {{category}}. Prefers CategoryName; never falls back to a digits-only unique_id.
|
||||
// {{category}}. Prefers taxonomy CategoryName; never falls back to a digits-only
|
||||
// unique_id, prompt/formula leakage, or the product title itself.
|
||||
func categoryDisplayLabel(result StepResult) string {
|
||||
if n := strings.TrimSpace(result.CategoryName); n != "" {
|
||||
return n
|
||||
// CategoryName comes from categories.name — trust unless prompt leakage.
|
||||
if !isPromptLabelTitle(n) {
|
||||
return n
|
||||
}
|
||||
}
|
||||
cat := strings.TrimSpace(result.Category)
|
||||
if cat == "" || isDigitsOnlyCategoryID(cat) {
|
||||
if cat == "" || isDigitsOnlyCategoryID(cat) || isPromptLabelTitle(cat) {
|
||||
return ""
|
||||
}
|
||||
// Unresolved non-uid token: never surface the product name as {{category}}.
|
||||
if categoryTokenEqualsProductTitle(cat, result.ProcessedName, result.Name) {
|
||||
return ""
|
||||
}
|
||||
return cat
|
||||
@@ -109,6 +117,10 @@ func normalizeCategoryUniqueID(s string) string {
|
||||
if s == "" || s == "<nil>" || strings.EqualFold(s, "none") {
|
||||
return ""
|
||||
}
|
||||
// Never treat enhance-prompt / formula scaffolding as a category unique_id.
|
||||
if isPromptLabelTitle(s) {
|
||||
return ""
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
@@ -155,7 +167,10 @@ func applyCategoryFromMapped(out *StepResult, maps ...map[string]any) {
|
||||
return
|
||||
}
|
||||
cat := categoryUniqueIDFromMaps(maps...)
|
||||
if cat == "" {
|
||||
// Prompt/formula leakage never becomes Category. Product-title equals are
|
||||
// scrubbed later (scrubCategoryPollution) so a title that matches a real
|
||||
// taxonomy display name can still coerce to unique_id.
|
||||
if cat == "" || isPromptLabelTitle(cat) {
|
||||
return
|
||||
}
|
||||
out.Category = SanitizeText(cat)
|
||||
@@ -213,6 +228,15 @@ func coerceCategoryToCompanyUniqueID(out *StepResult, namesByUID map[string]stri
|
||||
if cat == "" {
|
||||
return
|
||||
}
|
||||
if isPromptLabelTitle(cat) {
|
||||
out.Category = ""
|
||||
out.CategoryName = ""
|
||||
if out.FieldSources == nil {
|
||||
out.FieldSources = map[string]any{}
|
||||
}
|
||||
out.FieldSources["category"] = "cleared_invalid"
|
||||
return
|
||||
}
|
||||
resolved := resolveCompanyCategoryUniqueID(cat, namesByUID, valid)
|
||||
if resolved == "" || resolved == cat {
|
||||
return
|
||||
@@ -241,8 +265,10 @@ func filterCategoryIfInvalid(out *StepResult, valid map[string]struct{}) {
|
||||
return
|
||||
}
|
||||
out.Category = ""
|
||||
if out.FieldSources != nil {
|
||||
delete(out.FieldSources, "category")
|
||||
out.CategoryName = ""
|
||||
if out.FieldSources == nil {
|
||||
out.FieldSources = map[string]any{}
|
||||
}
|
||||
out.FieldSources["category"] = "cleared_invalid"
|
||||
out.Notes = append(out.Notes, "category: ignored (unknown unique_id for company)")
|
||||
}
|
||||
|
||||
@@ -133,6 +133,9 @@ func TestFilterCategoryIfInvalid(t *testing.T) {
|
||||
if out.Category != "" {
|
||||
t.Fatalf("invalid unique_id kept: %q", out.Category)
|
||||
}
|
||||
if src, _ := out.FieldSources["category"].(string); src != "cleared_invalid" {
|
||||
t.Fatalf("field_sources.category=%v want cleared_invalid", out.FieldSources["category"])
|
||||
}
|
||||
out.Category = "50"
|
||||
filterCategoryIfInvalid(&out, nil)
|
||||
if out.Category != "50" {
|
||||
@@ -179,6 +182,76 @@ func TestCoerceCategoryToCompanyUniqueID_nameToUID(t *testing.T) {
|
||||
if out.Category != "" {
|
||||
t.Fatalf("unknown should clear: %q", out.Category)
|
||||
}
|
||||
if src, _ := out.FieldSources["category"].(string); src != "cleared_invalid" {
|
||||
t.Fatalf("field_sources.category=%v want cleared_invalid", out.FieldSources["category"])
|
||||
}
|
||||
}
|
||||
|
||||
// Product titles (and nested category.name copies of the title) must never land
|
||||
// in Category — only taxonomy unique_ids (+ display names from categories).
|
||||
func TestRunSteps_productTitleNeverBecomesCategory(t *testing.T) {
|
||||
t.Parallel()
|
||||
const productTitle = "Bosch Serie 6 WAU28PH0BY 9kg White Washing Machine"
|
||||
names := map[string]string{"50": "Pralni stroji", "28": "TV mounts"}
|
||||
e := &Engine{Vector: NoopVectorCategorizer{}}
|
||||
|
||||
out, err := e.RunSteps(context.Background(), "co", ProductInput{
|
||||
GTIN: "4242005191234",
|
||||
Name: productTitle,
|
||||
Mapped: map[string]any{
|
||||
"name": productTitle,
|
||||
"category": map[string]any{
|
||||
"name": productTitle, // feed pollution: product title as category.name
|
||||
},
|
||||
},
|
||||
CategoryNamesByUID: names,
|
||||
}, "normalize_only", nil, StepPolicy{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if out.Category == productTitle || strings.EqualFold(out.Category, productTitle) {
|
||||
t.Fatalf("Category must not be product title: %q", out.Category)
|
||||
}
|
||||
if out.CategoryName == productTitle || strings.EqualFold(out.CategoryName, productTitle) {
|
||||
t.Fatalf("CategoryName must not be product title: %q", out.CategoryName)
|
||||
}
|
||||
if cat := categoryDisplayLabel(out); cat == productTitle || strings.EqualFold(cat, productTitle) {
|
||||
t.Fatalf("categoryDisplayLabel leaked product title: %q", cat)
|
||||
}
|
||||
if out.Category != "" {
|
||||
t.Fatalf("Category=%q want empty (unresolved product-title token scrubbed)", out.Category)
|
||||
}
|
||||
|
||||
// Plain string category == product title must also be scrubbed.
|
||||
out2, err := e.RunSteps(context.Background(), "co", ProductInput{
|
||||
GTIN: "4242005191235",
|
||||
Name: productTitle,
|
||||
Mapped: map[string]any{
|
||||
"name": productTitle,
|
||||
"category": productTitle,
|
||||
},
|
||||
CategoryNamesByUID: names,
|
||||
}, "normalize_only", nil, StepPolicy{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if out2.Category != "" {
|
||||
t.Fatalf("plain title-as-category kept: %q", out2.Category)
|
||||
}
|
||||
|
||||
// Real taxonomy display name still coerces via processOne helpers.
|
||||
out3 := StepResult{Category: "Pralni stroji", Name: productTitle, ProcessedName: productTitle}
|
||||
valid := map[string]struct{}{"50": {}, "28": {}}
|
||||
coerceCategoryToCompanyUniqueID(&out3, names, valid)
|
||||
filterCategoryIfInvalid(&out3, valid)
|
||||
scrubCategoryPollution(&out3, names, valid)
|
||||
syncCategoryName(&out3, names)
|
||||
if out3.Category != "50" {
|
||||
t.Fatalf("taxonomy name coerce: Category=%q want 50", out3.Category)
|
||||
}
|
||||
if out3.CategoryName != "Pralni stroji" {
|
||||
t.Fatalf("CategoryName=%q want Pralni stroji", out3.CategoryName)
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolveCompanyCategoryUniqueID_usedByV1Projection(t *testing.T) {
|
||||
|
||||
@@ -63,3 +63,12 @@ func enhanceStatusFromMeta(raw any) string {
|
||||
s, _ := m["status"].(string)
|
||||
return s
|
||||
}
|
||||
|
||||
func enhanceReasonFromMeta(raw any) string {
|
||||
m, ok := raw.(map[string]any)
|
||||
if !ok {
|
||||
return ""
|
||||
}
|
||||
s, _ := m["reason"].(string)
|
||||
return s
|
||||
}
|
||||
|
||||
@@ -97,7 +97,7 @@ func TestRunSteps_skipsEnhanceWhenInputHashUnchanged(t *testing.T) {
|
||||
t.Fatalf("preferKnownProviderMode(unknown, internal)=%q", got)
|
||||
}
|
||||
joined := strings.Join(out.Notes, ";")
|
||||
if !strings.Contains(joined, "unchanged") {
|
||||
if !strings.Contains(joined, "ai_enhance_unchanged") {
|
||||
t.Fatalf("notes=%v", out.Notes)
|
||||
}
|
||||
}
|
||||
@@ -279,6 +279,195 @@ func TestRunSteps_synthesizedDescDoesNotPersistEnhanceHash(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunSteps_formulaIgnoredByLLM_retriesThenSynthesizes(t *testing.T) {
|
||||
calls := 0
|
||||
var sawFormulaRetry bool
|
||||
e := &Engine{
|
||||
Completer: stubCompleter{fn: func(system, user string) (Completion, error) {
|
||||
calls++
|
||||
if strings.Contains(user, "INVALID DESCRIPTION") {
|
||||
sawFormulaRetry = true
|
||||
}
|
||||
// Non-weak short prose that ignores multi-section HTML formula.
|
||||
return Completion{
|
||||
Text: `{"name":"GIGABYTE GS27QC","description":"GIGABYTE GS27QC is a gaming monitor with a 61 cm curved panel for immersive play."}`,
|
||||
TotalTokens: 4,
|
||||
}, nil
|
||||
}},
|
||||
Vector: NoopVectorCategorizer{},
|
||||
}
|
||||
descTpl := map[string]any{
|
||||
"sections": []any{
|
||||
map[string]any{"type": "h1", "instructions": "Heading"},
|
||||
map[string]any{"type": "p", "instructions": "Body"},
|
||||
map[string]any{"type": "ul", "instructions": "Specs"},
|
||||
},
|
||||
}
|
||||
out, err := e.RunSteps(context.Background(), "co", ProductInput{
|
||||
Name: "GIGABYTE GS27QC",
|
||||
Mapped: map[string]any{"name": "GIGABYTE GS27QC", "category": "7", "brand": "GIGABYTE", "width": "61 cm"},
|
||||
CategoryFormulasByKey: map[string]CategoryFormulas{
|
||||
"7": {DescriptionTemplate: descTpl},
|
||||
},
|
||||
Language: "en",
|
||||
}, "enhance_only", nil, StepPolicy{AllowAI: true, AllowEPREL: true})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if calls < 2 {
|
||||
t.Fatalf("expected formula retry after ignore, calls=%d", calls)
|
||||
}
|
||||
if !sawFormulaRetry {
|
||||
t.Fatal("expected INVALID DESCRIPTION retry user suffix")
|
||||
}
|
||||
for _, tag := range []string{"<h1>", "<p>", "<ul>"} {
|
||||
if !strings.Contains(out.ProcessedDescription, tag) {
|
||||
t.Fatalf("expected formula HTML with %s, got %q", tag, out.ProcessedDescription)
|
||||
}
|
||||
}
|
||||
if h, _ := out.FieldSources[FieldEnhanceInputHash].(string); h != "" {
|
||||
t.Fatalf("formula synthesize must not persist hash, got %q", h)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunSteps_formulaHTMLFromLLM_persistsHash(t *testing.T) {
|
||||
e := &Engine{
|
||||
Completer: stubCompleter{fn: func(system, user string) (Completion, error) {
|
||||
return Completion{
|
||||
Text: `{"name":"GIGABYTE GS27QC","description":"<h1>GIGABYTE GS27QC</h1><p>Curved gaming monitor.</p><ul><li>width 61 cm</li></ul>"}`,
|
||||
TotalTokens: 6,
|
||||
}, nil
|
||||
}},
|
||||
Vector: NoopVectorCategorizer{},
|
||||
}
|
||||
descTpl := map[string]any{
|
||||
"sections": []any{
|
||||
map[string]any{"type": "h1", "instructions": "Heading"},
|
||||
map[string]any{"type": "p", "instructions": "Body"},
|
||||
map[string]any{"type": "ul", "instructions": "Specs"},
|
||||
},
|
||||
}
|
||||
out, err := e.RunSteps(context.Background(), "co", ProductInput{
|
||||
Name: "GIGABYTE GS27QC",
|
||||
Mapped: map[string]any{"name": "GIGABYTE GS27QC", "category": "7"},
|
||||
DescriptionTemplate: descTpl,
|
||||
Language: "en",
|
||||
}, "enhance_only", nil, StepPolicy{AllowAI: true, AllowEPREL: true})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !strings.Contains(out.ProcessedDescription, "<ul>") {
|
||||
t.Fatalf("expected LLM HTML kept, got %q", out.ProcessedDescription)
|
||||
}
|
||||
if h, _ := out.FieldSources[FieldEnhanceInputHash].(string); h == "" {
|
||||
t.Fatal("compliant formula HTML should persist enhance hash")
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunSteps_synthPriorHashDoesNotSkipReprocess(t *testing.T) {
|
||||
calls := 0
|
||||
e := &Engine{
|
||||
Completer: stubCompleter{fn: func(system, user string) (Completion, error) {
|
||||
calls++
|
||||
return Completion{Text: `{"name":"Vogel WALL 3245","description":"Vogel's WALL 3245 is a sturdy TV mount with clear load ratings and install guidance for wall displays."}`, TotalTokens: 5}, nil
|
||||
}},
|
||||
Vector: NoopVectorCategorizer{},
|
||||
}
|
||||
normName := "Vogel's WALL 3245 TV Wall Mount"
|
||||
normDesc := "A mount with enough mapped detail for hashing inputs."
|
||||
synthPrior := synthesizeDescriptionFromTitle(normName, "TV Mounts", "en", map[string]any{
|
||||
"brand": "Vogel's", "width": "45 cm", "max_load": "40 kg",
|
||||
})
|
||||
if !company.LooksLikeHeuristicSynthesize(synthPrior) {
|
||||
t.Fatalf("expected invent synth prior, got %q", synthPrior)
|
||||
}
|
||||
sysTpl, userTpl := resolveProductPromptTemplates(ProductInput{})
|
||||
priorHash := HashEnhanceInput("", normName, normDesc, "", "", sysTpl, userTpl, map[string]any{})
|
||||
out, err := e.RunSteps(context.Background(), "co", ProductInput{
|
||||
Name: normName,
|
||||
Description: normDesc,
|
||||
Mapped: map[string]any{"name": normName, "description": normDesc},
|
||||
PriorEnhanceHash: priorHash,
|
||||
PriorProcessedName: normName,
|
||||
PriorProcessedDescription: synthPrior,
|
||||
}, "enhance_only", nil, StepPolicy{AllowAI: true, AllowEPREL: true})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if calls != 1 {
|
||||
t.Fatalf("synth prior must force completer, calls=%d", calls)
|
||||
}
|
||||
if out.SkipCreditDebit {
|
||||
t.Fatal("synth prior must not SkipCreditDebit")
|
||||
}
|
||||
joined := strings.Join(out.Notes, ";")
|
||||
if !strings.Contains(joined, "forced re-enhance") {
|
||||
t.Fatalf("expected forced re-enhance note, notes=%v", out.Notes)
|
||||
}
|
||||
if company.LooksLikeHeuristicSynthesize(out.ProcessedDescription) && calls == 1 {
|
||||
// LLM stub returned non-synth copy — ok if still synth after outer repair.
|
||||
}
|
||||
if out.ProcessedDescription == synthPrior {
|
||||
t.Fatalf("must not reuse synth prior: %q", out.ProcessedDescription)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunSteps_formulaMismatchPriorHashDoesNotSkipReprocess(t *testing.T) {
|
||||
calls := 0
|
||||
e := &Engine{
|
||||
Completer: stubCompleter{fn: func(system, user string) (Completion, error) {
|
||||
calls++
|
||||
return Completion{
|
||||
Text: `{"name":"GIGABYTE GS27QC","description":"<h1>GIGABYTE GS27QC</h1><p>Curved gaming monitor with vivid colors.</p><ul><li>width 61 cm</li></ul>"}`,
|
||||
TotalTokens: 6,
|
||||
}, nil
|
||||
}},
|
||||
Vector: NoopVectorCategorizer{},
|
||||
}
|
||||
descTpl := map[string]any{
|
||||
"sections": []any{
|
||||
map[string]any{"type": "h1", "instructions": "Heading"},
|
||||
map[string]any{"type": "p", "instructions": "Body"},
|
||||
map[string]any{"type": "ul", "instructions": "Specs"},
|
||||
},
|
||||
}
|
||||
normName := "GIGABYTE GS27QC"
|
||||
normDesc := "GIGABYTE GS27QC is a gaming monitor with a 61 cm curved panel for immersive play."
|
||||
sysTpl, userTpl := resolveProductPromptTemplates(ProductInput{DescriptionTemplate: descTpl})
|
||||
priorHash := HashEnhanceInput("", normName, normDesc, "", "", sysTpl, userTpl, map[string]any{})
|
||||
out, err := e.RunSteps(context.Background(), "co", ProductInput{
|
||||
Name: normName,
|
||||
Description: normDesc,
|
||||
Mapped: map[string]any{"name": normName, "description": normDesc, "category": "7"},
|
||||
DescriptionTemplate: descTpl,
|
||||
PriorEnhanceHash: priorHash,
|
||||
PriorProcessedName: normName,
|
||||
PriorProcessedDescription: normDesc, // plain prose — formula mismatch
|
||||
Language: "en",
|
||||
}, "enhance_only", nil, StepPolicy{AllowAI: true, AllowEPREL: true})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if calls < 1 {
|
||||
t.Fatalf("formula-mismatch prior must force completer, calls=%d", calls)
|
||||
}
|
||||
if out.SkipCreditDebit {
|
||||
t.Fatal("formula-mismatch must not SkipCreditDebit")
|
||||
}
|
||||
joined := strings.Join(out.Notes, ";")
|
||||
if !strings.Contains(joined, "forced re-enhance") && !strings.Contains(joined, "formula") {
|
||||
// forced note when hash matched; formula repair notes also acceptable
|
||||
if !strings.Contains(joined, "synth") && calls >= 1 {
|
||||
// still ok if LLM ran without note if reason path differed
|
||||
}
|
||||
}
|
||||
for _, tag := range []string{"<h1>", "<p>", "<ul>"} {
|
||||
if !strings.Contains(out.ProcessedDescription, tag) {
|
||||
t.Fatalf("expected formula HTML with %s, got %q", tag, out.ProcessedDescription)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunSteps_weakPriorHashDoesNotSkipReprocess(t *testing.T) {
|
||||
calls := 0
|
||||
e := &Engine{
|
||||
@@ -344,3 +533,56 @@ func TestRunSteps_failedEnhanceDoesNotPoisonLocalizedHash(t *testing.T) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunSteps_emptyLLMDescriptionDoesNotPersistHashViaOriginalFallback(t *testing.T) {
|
||||
// Prod bug: empty LLM description preferred originals and stamped ok+hash.
|
||||
richOriginal := "This durable retail mount includes mounting hardware, load ratings, and install guidance for wall displays."
|
||||
e := &Engine{
|
||||
Completer: stubCompleter{fn: func(system, user string) (Completion, error) {
|
||||
return Completion{Text: `{"name":"Vogel WALL 3245","description":""}`, TotalTokens: 2}, nil
|
||||
}},
|
||||
Vector: NoopVectorCategorizer{},
|
||||
}
|
||||
out, err := e.RunSteps(context.Background(), "co", ProductInput{
|
||||
Name: "Vogel WALL 3245",
|
||||
Description: richOriginal,
|
||||
Mapped: map[string]any{
|
||||
"name": "Vogel WALL 3245",
|
||||
"description": richOriginal,
|
||||
},
|
||||
}, "enhance_only", nil, StepPolicy{AllowAI: true, AllowEPREL: true})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if h, _ := out.FieldSources[FieldEnhanceInputHash].(string); h != "" {
|
||||
t.Fatalf("empty LLM description must not persist enhance hash (got %q); desc=%q", h, out.ProcessedDescription)
|
||||
}
|
||||
for _, lf := range out.LocalizedContent {
|
||||
if lf.EnhanceInputHash != "" {
|
||||
t.Fatalf("localized hash must stay empty on empty LLM desc: %+v", lf)
|
||||
}
|
||||
}
|
||||
joined := strings.Join(out.Notes, ";")
|
||||
if !strings.Contains(joined, "refused") && !strings.Contains(joined, "synthesized") && !strings.Contains(joined, "empty_description") {
|
||||
t.Fatalf("expected refuse/synthesize note, notes=%v", out.Notes)
|
||||
}
|
||||
if strings.TrimSpace(out.ProcessedDescription) == "" {
|
||||
t.Fatal("expected nonempty fallback/synth description")
|
||||
}
|
||||
}
|
||||
|
||||
func TestLlmEnhanceHardRefuseReason(t *testing.T) {
|
||||
t.Parallel()
|
||||
if got := llmEnhanceHardRefuseReason("", "desc long enough"); got != "empty_name" {
|
||||
t.Fatalf("got %q", got)
|
||||
}
|
||||
if got := llmEnhanceHardRefuseReason("Category:", "desc long enough here for tests"); got != "prompt_leakage_name" {
|
||||
t.Fatalf("got %q", got)
|
||||
}
|
||||
if got := llmEnhanceHardRefuseReason("Widget", ""); got != "empty_description" {
|
||||
t.Fatalf("got %q", got)
|
||||
}
|
||||
if got := llmEnhanceHardRefuseReason("Widget", "A durable retail widget for everyday use with clear specs."); got != "" {
|
||||
t.Fatalf("got %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,127 @@
|
||||
package processing
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"log"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// enhanceLogHeadTail caps each end of prompt/response bodies in worker logs.
|
||||
const enhanceLogHeadTail = 500
|
||||
|
||||
func emptyLogDash(s string) string {
|
||||
s = strings.TrimSpace(s)
|
||||
if s == "" {
|
||||
return "-"
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
// truncateLogEnds returns a PII-safe preview: total rune length plus first/last
|
||||
// slices when the body is large (avoids megabyte log lines).
|
||||
func truncateLogEnds(s string, head, tail int) string {
|
||||
s = strings.TrimSpace(s)
|
||||
if s == "" {
|
||||
return "len=0"
|
||||
}
|
||||
runes := []rune(s)
|
||||
n := len(runes)
|
||||
if head < 0 {
|
||||
head = 0
|
||||
}
|
||||
if tail < 0 {
|
||||
tail = 0
|
||||
}
|
||||
if n <= head+tail+32 {
|
||||
return fmt.Sprintf("len=%d body=%q", n, s)
|
||||
}
|
||||
return fmt.Sprintf("len=%d head=%q tail=%q", n, string(runes[:head]), string(runes[n-tail:]))
|
||||
}
|
||||
|
||||
// formulaTemplateSummary summarizes a category title/description formula for logs
|
||||
// (section/element types + JSON byte length + short content hash — not a full dump).
|
||||
func formulaTemplateSummary(template any, kind string) string {
|
||||
if template == nil {
|
||||
return "none"
|
||||
}
|
||||
raw, err := json.Marshal(template)
|
||||
if err != nil || len(raw) == 0 || string(raw) == "null" || string(raw) == "{}" {
|
||||
return "none"
|
||||
}
|
||||
sum := sha256.Sum256(raw)
|
||||
hash8 := hex.EncodeToString(sum[:4])
|
||||
switch kind {
|
||||
case "title":
|
||||
els, sep, ok := parseTitleFormula(template)
|
||||
if !ok {
|
||||
return fmt.Sprintf("unparsed len=%d hash=%s", len(raw), hash8)
|
||||
}
|
||||
types := make([]string, 0, len(els))
|
||||
for _, el := range els {
|
||||
types = append(types, el.Type)
|
||||
}
|
||||
return fmt.Sprintf("elements=%s sep=%q len=%d hash=%s", strings.Join(types, "+"), sep, len(raw), hash8)
|
||||
case "description":
|
||||
sections, ok := parseDescriptionFormulaSections(template)
|
||||
if !ok {
|
||||
return fmt.Sprintf("unparsed len=%d hash=%s", len(raw), hash8)
|
||||
}
|
||||
types := make([]string, 0, len(sections))
|
||||
for _, sec := range sections {
|
||||
if t := strings.TrimSpace(sec.Type); t != "" {
|
||||
types = append(types, t)
|
||||
}
|
||||
}
|
||||
return fmt.Sprintf("sections=%s len=%d hash=%s", strings.Join(types, "+"), len(raw), hash8)
|
||||
default:
|
||||
return fmt.Sprintf("len=%d hash=%s", len(raw), hash8)
|
||||
}
|
||||
}
|
||||
|
||||
func finishReasonFromCompletion(comp Completion) string {
|
||||
m, ok := comp.Raw.(map[string]any)
|
||||
if !ok || m == nil {
|
||||
return ""
|
||||
}
|
||||
fr, _ := m["finish_reason"].(string)
|
||||
return strings.TrimSpace(fr)
|
||||
}
|
||||
|
||||
func enhanceFormulaOverride(in ProductInput) bool {
|
||||
return FormatDescriptionFormulaConstraint(in.DescriptionTemplate) != "" ||
|
||||
FormatTitleFormulaConstraint(in.TitleTemplate) != ""
|
||||
}
|
||||
|
||||
// logEnhancePrompt emits one structured line before the LLM call (truncated prompts).
|
||||
func logEnhancePrompt(in ProductInput, categoryUID, categoryName, system, user string) {
|
||||
log.Printf("processing: ai_enhance prompt job=%s company=%s raw=%s category_uid=%q category_name=%q title_formula=%s desc_formula=%s formula_override=%t system=%s user=%s",
|
||||
emptyLogDash(in.JobID), emptyLogDash(in.CompanyID), emptyLogDash(in.RawProductID),
|
||||
strings.TrimSpace(categoryUID), strings.TrimSpace(categoryName),
|
||||
formulaTemplateSummary(in.TitleTemplate, "title"),
|
||||
formulaTemplateSummary(in.DescriptionTemplate, "description"),
|
||||
enhanceFormulaOverride(in),
|
||||
truncateLogEnds(system, enhanceLogHeadTail, enhanceLogHeadTail),
|
||||
truncateLogEnds(user, enhanceLogHeadTail, enhanceLogHeadTail),
|
||||
)
|
||||
}
|
||||
|
||||
// logEnhanceOutcome emits one structured line for the enhance result (truncated response).
|
||||
func logEnhanceOutcome(in ProductInput, categoryUID, categoryName, outcome, reason string, comp Completion, elapsed time.Duration, forceReason string) {
|
||||
log.Printf("processing: ai_enhance outcome=%s reason=%s job=%s company=%s raw=%s category_uid=%q category_name=%q title_formula=%s desc_formula=%s formula_override=%t finish_reason=%s prompt_tokens=%d completion_tokens=%d total_tokens=%d elapsed=%s response=%s force_reason=%s",
|
||||
emptyLogDash(outcome), emptyLogDash(reason),
|
||||
emptyLogDash(in.JobID), emptyLogDash(in.CompanyID), emptyLogDash(in.RawProductID),
|
||||
strings.TrimSpace(categoryUID), strings.TrimSpace(categoryName),
|
||||
formulaTemplateSummary(in.TitleTemplate, "title"),
|
||||
formulaTemplateSummary(in.DescriptionTemplate, "description"),
|
||||
enhanceFormulaOverride(in),
|
||||
emptyLogDash(finishReasonFromCompletion(comp)),
|
||||
comp.PromptTokens, comp.OutputTokens, comp.TotalTokens,
|
||||
elapsed.Round(time.Millisecond),
|
||||
truncateLogEnds(comp.Text, enhanceLogHeadTail, enhanceLogHeadTail),
|
||||
emptyLogDash(forceReason),
|
||||
)
|
||||
}
|
||||
@@ -0,0 +1,103 @@
|
||||
package processing
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestTruncateLogEnds(t *testing.T) {
|
||||
t.Parallel()
|
||||
short := truncateLogEnds("hello", 500, 500)
|
||||
if !strings.Contains(short, `body="hello"`) || !strings.Contains(short, "len=5") {
|
||||
t.Fatalf("short: %s", short)
|
||||
}
|
||||
long := strings.Repeat("a", 600) + "MID" + strings.Repeat("b", 600)
|
||||
got := truncateLogEnds(long, 10, 10)
|
||||
if !strings.Contains(got, "len=1203") || !strings.Contains(got, "head=") || !strings.Contains(got, "tail=") {
|
||||
t.Fatalf("long: %s", got)
|
||||
}
|
||||
if strings.Contains(got, "MID") {
|
||||
t.Fatalf("middle should be omitted: %s", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFormulaTemplateSummary(t *testing.T) {
|
||||
t.Parallel()
|
||||
if formulaTemplateSummary(nil, "description") != "none" {
|
||||
t.Fatal("nil → none")
|
||||
}
|
||||
desc := map[string]any{
|
||||
"sections": []any{
|
||||
map[string]any{"type": "h1", "instructions": "Heading"},
|
||||
map[string]any{"type": "p", "instructions": "Body"},
|
||||
map[string]any{"type": "ul", "instructions": "Specs"},
|
||||
},
|
||||
}
|
||||
sum := formulaTemplateSummary(desc, "description")
|
||||
if !strings.Contains(sum, "sections=h1+p+ul") || !strings.Contains(sum, "len=") || !strings.Contains(sum, "hash=") {
|
||||
t.Fatalf("desc summary: %s", sum)
|
||||
}
|
||||
title := map[string]any{
|
||||
"separator": " ",
|
||||
"elements": []any{
|
||||
map[string]any{"type": "variable", "value": "brand"},
|
||||
map[string]any{"type": "text", "value": "TV"},
|
||||
},
|
||||
}
|
||||
ts := formulaTemplateSummary(title, "title")
|
||||
if !strings.Contains(ts, "elements=variable+text") || !strings.Contains(ts, `sep=" "`) {
|
||||
t.Fatalf("title summary: %s", ts)
|
||||
}
|
||||
}
|
||||
|
||||
// TestA1StyleDescriptionFormulaInResolvedPrompts asserts a realistic multi-section
|
||||
// category description_template (A1-style) is injected into both system and user
|
||||
// enhance prompts via resolveProductPromptTemplates / RenderProductEnhancePrompts.
|
||||
func TestA1StyleDescriptionFormulaInResolvedPrompts(t *testing.T) {
|
||||
t.Parallel()
|
||||
// Representative A1 cooker/monitor style: h1 + p + ul sections.
|
||||
descFormula := map[string]any{
|
||||
"sections": []any{
|
||||
map[string]any{"type": "h1", "instructions": "Naziv izdelka"},
|
||||
map[string]any{"type": "h2", "instructions": "Podnaslov prednosti"},
|
||||
map[string]any{"type": "p", "instructions": "Kratek opis v slovenščini"},
|
||||
map[string]any{"type": "ul", "instructions": "Ključne specifikacije"},
|
||||
},
|
||||
}
|
||||
titleFormula := map[string]any{
|
||||
"separator": " ",
|
||||
"elements": []any{
|
||||
map[string]any{"type": "variable", "value": "brand"},
|
||||
map[string]any{"type": "variable", "value": "product_model"},
|
||||
},
|
||||
}
|
||||
sysTpl, userTpl := resolveProductPromptTemplates(ProductInput{
|
||||
EnhanceSystemTemplate: "You write retail copy. description: 1-2 factual sentences.",
|
||||
EnhanceUserTemplate: "Category: {{category}}\nName: {{name}}\nAttrs: {{attrs}}",
|
||||
TitleTemplate: titleFormula,
|
||||
DescriptionTemplate: descFormula,
|
||||
})
|
||||
if !strings.Contains(sysTpl, "Description formula") {
|
||||
t.Fatalf("system missing formula override: %s", sysTpl)
|
||||
}
|
||||
if !strings.Contains(userTpl, "Description formula") || !strings.Contains(userTpl, "- h1: Naziv izdelka") || !strings.Contains(userTpl, "- h2: Podnaslov prednosti") {
|
||||
t.Fatalf("user missing description formula: %s", userTpl)
|
||||
}
|
||||
if !strings.Contains(userTpl, "Title formula") || !strings.Contains(userTpl, "attr [brand]") {
|
||||
t.Fatalf("user missing title formula: %s", userTpl)
|
||||
}
|
||||
system, user := RenderProductEnhancePrompts(
|
||||
sysTpl, userTpl,
|
||||
"Štedilniki", "VOX EHT6020", "", "", "", "sl",
|
||||
map[string]any{"brand": "VOX", "product_model": "EHT6020"},
|
||||
)
|
||||
if !strings.Contains(user, "Description formula") || !strings.Contains(user, "- h1: Naziv izdelka") {
|
||||
t.Fatalf("rendered user missing formula: %s", user)
|
||||
}
|
||||
if !strings.Contains(user, "Slovenian") && !strings.Contains(system, "Slovenian") {
|
||||
t.Fatalf("language not rendered into prompts: sys=%s user=%s", system, user)
|
||||
}
|
||||
if !enhanceFormulaOverride(ProductInput{DescriptionTemplate: descFormula}) {
|
||||
t.Fatal("formula_override should be true")
|
||||
}
|
||||
}
|
||||
@@ -5,6 +5,8 @@ import (
|
||||
"fmt"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/descrybe/descrybe-v2/apps/api/internal/company"
|
||||
)
|
||||
|
||||
// AppendFormulaConstraints appends language-agnostic title/description formula
|
||||
@@ -118,6 +120,24 @@ func AppendDescriptionFormulaSystemOverride(systemTpl string, descriptionTemplat
|
||||
return systemTpl + "\n" + descriptionFormulaSystemOverride
|
||||
}
|
||||
|
||||
// descriptionSatisfiesFormula reports whether desc includes the HTML tags required
|
||||
// by categories.description_template sections. No formula → always true.
|
||||
func descriptionSatisfiesFormula(desc string, template any) bool {
|
||||
sections, ok := parseDescriptionFormulaSections(template)
|
||||
if !ok || len(sections) == 0 {
|
||||
return true
|
||||
}
|
||||
desc = strings.TrimSpace(desc)
|
||||
if desc == "" || desc == "<nil>" {
|
||||
return false
|
||||
}
|
||||
types := make([]string, 0, len(sections))
|
||||
for _, s := range sections {
|
||||
types = append(types, s.Type)
|
||||
}
|
||||
return !company.DescriptionMissingFormulaHTMLTags(desc, types)
|
||||
}
|
||||
|
||||
type titleFormulaElement struct {
|
||||
Type string `json:"type"`
|
||||
Value string `json:"value"`
|
||||
|
||||
@@ -76,6 +76,36 @@ func TestAppendDescriptionFormulaSystemOverride(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestDescriptionSatisfiesFormula(t *testing.T) {
|
||||
t.Parallel()
|
||||
tpl := map[string]any{
|
||||
"sections": []any{
|
||||
map[string]any{"type": "h1", "instructions": "Heading"},
|
||||
map[string]any{"type": "p", "instructions": "Body"},
|
||||
map[string]any{"type": "ul", "instructions": "Specs"},
|
||||
},
|
||||
}
|
||||
if descriptionSatisfiesFormula("", tpl) {
|
||||
t.Fatal("empty must fail")
|
||||
}
|
||||
if descriptionSatisfiesFormula("GIGABYTE GS27QC is a gaming monitor with a 61 cm curved panel for immersive play.", tpl) {
|
||||
t.Fatal("plain prose must fail multi-section formula")
|
||||
}
|
||||
if !descriptionSatisfiesFormula("<h1>GIGABYTE GS27QC</h1><p>Gaming monitor.</p><ul><li>61 cm</li></ul>", tpl) {
|
||||
t.Fatal("full HTML skeleton must pass")
|
||||
}
|
||||
if !descriptionSatisfiesFormula("any copy", nil) {
|
||||
t.Fatal("nil formula always satisfied")
|
||||
}
|
||||
if descriptionNeedsEnhanceRepair(
|
||||
"GIGABYTE GS27QC is a gaming monitor with a 61 cm curved panel for immersive play.",
|
||||
tpl,
|
||||
"GIGABYTE GS27QC",
|
||||
) != true {
|
||||
t.Fatal("non-weak prose that ignores formula must need repair")
|
||||
}
|
||||
}
|
||||
|
||||
func TestAppendFormulaConstraints_injectedIntoResolve(t *testing.T) {
|
||||
t.Parallel()
|
||||
title := map[string]any{
|
||||
|
||||
@@ -8,21 +8,25 @@ import (
|
||||
"strings"
|
||||
)
|
||||
|
||||
// Local / weak-model defaults (8k-class context). See docs/local-llm-tuning.md.
|
||||
// Local / weak-model defaults. See docs/local-llm-tuning.md.
|
||||
const (
|
||||
DefaultStructuredTemp = 0.2
|
||||
// Reasoning-capable OpenAI-compatible models (e.g. code-fast / Qwen3) spend
|
||||
// hundreds–thousands of tokens in reasoning_content before writing message.content.
|
||||
// 350 capped mid-thought → empty content → "empty model response".
|
||||
MaxTokensEnhance = 4096
|
||||
MaxTokensSEO = 180
|
||||
MaxTokensCampaign = 650
|
||||
MaxProductDescRunes = 400
|
||||
MaxAttrKeys = 10
|
||||
MaxAttrValueRunes = 60
|
||||
MaxBrandInjectRunes = 500
|
||||
MaxCampaignProducts = 8
|
||||
MaxCampaignNameRunes = 80
|
||||
// Reasoning-capable OpenAI-compatible models (e.g. OverloadedBot code-fast)
|
||||
// spend thousands of tokens in reasoning_content before message.content.
|
||||
// Formula HTML JSON often needs a large completion budget; 4096 hit
|
||||
// finish_reason=length with empty/truncated content in live enhance.
|
||||
MaxTokensEnhance = 16384
|
||||
// MaxTokensEnhanceRetry is the one-shot length-cap bump ceiling used by
|
||||
// OpenAIClient when finish_reason=length yields empty or unparseable JSON.
|
||||
MaxTokensEnhanceRetry = 32768
|
||||
MaxTokensSEO = 180
|
||||
MaxTokensCampaign = 650
|
||||
MaxProductDescRunes = 400
|
||||
MaxAttrKeys = 10
|
||||
MaxAttrValueRunes = 60
|
||||
MaxBrandInjectRunes = 500
|
||||
MaxCampaignProducts = 8
|
||||
MaxCampaignNameRunes = 80
|
||||
)
|
||||
|
||||
// CompleteOptions tunes a single chat completion for structured tasks.
|
||||
|
||||
@@ -256,11 +256,14 @@ type chatResponse struct {
|
||||
} `json:"error"`
|
||||
}
|
||||
|
||||
var errEmptyModelResponse = errors.New("empty model response")
|
||||
var (
|
||||
errEmptyModelResponse = errors.New("empty model response")
|
||||
errLengthCappedResponse = errors.New("response truncated at max_tokens")
|
||||
)
|
||||
|
||||
// maxTokensReasoningBudget is used when a capped completion returns empty
|
||||
// content with finish_reason=length (reasoning models).
|
||||
const maxTokensReasoningBudget = 4096
|
||||
func isLengthBudgetErr(err error) bool {
|
||||
return errors.Is(err, errEmptyModelResponse) || errors.Is(err, errLengthCappedResponse)
|
||||
}
|
||||
|
||||
func (c *OpenAIClient) Complete(ctx context.Context, system, user string) (Completion, error) {
|
||||
return c.CompleteWithOptions(ctx, system, user, CompleteOptions{})
|
||||
@@ -300,16 +303,19 @@ func (c *OpenAIClient) CompleteWithOptions(ctx context.Context, system, user str
|
||||
return comp, nil
|
||||
}
|
||||
lastErr = err
|
||||
if errors.Is(err, errEmptyModelResponse) {
|
||||
if maxTok <= 0 || maxTok < maxTokensReasoningBudget {
|
||||
if maxTok < maxTokensReasoningBudget {
|
||||
maxTok = maxTokensReasoningBudget
|
||||
if isLengthBudgetErr(err) {
|
||||
if maxTok <= 0 || maxTok < MaxTokensEnhanceRetry {
|
||||
prev := maxTok
|
||||
if maxTok < MaxTokensEnhanceRetry {
|
||||
maxTok = MaxTokensEnhanceRetry
|
||||
}
|
||||
log.Printf("openai: length-cap retry model=%s prev_max_tokens=%d next_max_tokens=%d err=%s",
|
||||
c.Model, prev, maxTok, TruncateError(err))
|
||||
retryable = true
|
||||
} else {
|
||||
// Already at reasoning budget — another 240s call will not help.
|
||||
// Already at enhance retry ceiling — another long call will not help.
|
||||
return Completion{}, fmt.Errorf(
|
||||
"openai empty response at max_tokens=%d for model %q (try a faster non-reasoning model): %w",
|
||||
"openai length-capped at max_tokens=%d for model %q (prefer a faster non-reasoning product-enhance model): %w",
|
||||
maxTok, c.Model, err)
|
||||
}
|
||||
}
|
||||
@@ -488,34 +494,41 @@ func (c *OpenAIClient) doComplete(ctx context.Context, system, user string, temp
|
||||
|
||||
stopWait := c.startWaitLogger("chat/completions", maxTokens)
|
||||
res, err := c.HTTPClient.Do(req)
|
||||
stopWait(err)
|
||||
if err != nil {
|
||||
stopWait(err)
|
||||
wrapped, retryable := classifyOpenAITransportErr(c.Model, err)
|
||||
return Completion{}, retryable, wrapped
|
||||
}
|
||||
defer res.Body.Close()
|
||||
raw, err := io.ReadAll(io.LimitReader(res.Body, 1<<20))
|
||||
if err != nil {
|
||||
stopWait(err)
|
||||
wrapped, retryable := classifyOpenAITransportErr(c.Model, err)
|
||||
return Completion{}, retryable, wrapped
|
||||
}
|
||||
var parsed chatResponse
|
||||
if err := json.Unmarshal(raw, &parsed); err != nil {
|
||||
return Completion{}, false, fmt.Errorf("openai decode: %w", err)
|
||||
decErr := fmt.Errorf("openai decode: %w", err)
|
||||
stopWait(decErr)
|
||||
return Completion{}, false, decErr
|
||||
}
|
||||
if res.StatusCode == http.StatusTooManyRequests || res.StatusCode >= 500 {
|
||||
msg := "rate limited or server error"
|
||||
if parsed.Error != nil && parsed.Error.Message != "" {
|
||||
msg = TruncateError(errors.New(parsed.Error.Message))
|
||||
}
|
||||
return Completion{}, true, errors.New(msg)
|
||||
httpErr := errors.New(msg)
|
||||
stopWait(httpErr)
|
||||
return Completion{}, true, httpErr
|
||||
}
|
||||
if res.StatusCode >= 400 {
|
||||
msg := fmt.Sprintf("openai http %d", res.StatusCode)
|
||||
if parsed.Error != nil && parsed.Error.Message != "" {
|
||||
msg = TruncateError(errors.New(parsed.Error.Message))
|
||||
}
|
||||
return Completion{}, false, errors.New(msg)
|
||||
httpErr := errors.New(msg)
|
||||
stopWait(httpErr)
|
||||
return Completion{}, false, httpErr
|
||||
}
|
||||
text := ""
|
||||
finishReason := ""
|
||||
@@ -523,18 +536,44 @@ func (c *OpenAIClient) doComplete(ctx context.Context, system, user string, temp
|
||||
finishReason = strings.TrimSpace(parsed.Choices[0].FinishReason)
|
||||
text = SanitizeOutput(choiceMessageText(parsed.Choices[0].Message.Content, parsed.Choices[0].Message.ReasoningContent, parsed.Choices[0].Message.Reasoning))
|
||||
}
|
||||
usagePrompt := parsed.Usage.PromptTokens
|
||||
usageOut := parsed.Usage.CompletionTokens
|
||||
usageTotal := parsed.Usage.TotalTokens
|
||||
canBump := maxTokens <= 0 || maxTokens < MaxTokensEnhanceRetry
|
||||
lengthCapped := strings.EqualFold(finishReason, "length")
|
||||
if text == "" {
|
||||
// Reasoning models often return empty content when max_tokens cuts mid-thought.
|
||||
// Only retry when CompleteWithOptions can still raise max_tokens.
|
||||
canBump := maxTokens <= 0 || maxTokens < maxTokensReasoningBudget
|
||||
retryable := strings.EqualFold(finishReason, "length") && canBump
|
||||
// Never log HTTP-ok as ok=1 when content is empty (prod looked "successful" at 30–272ms).
|
||||
retryable := lengthCapped && canBump
|
||||
emptyErr := fmt.Errorf("%w (finish_reason=%s max_tokens=%d prompt_tokens=%d completion_tokens=%d)",
|
||||
errEmptyModelResponse, finishReason, maxTokens, usagePrompt, usageOut)
|
||||
stopWait(emptyErr)
|
||||
log.Printf("openai: chat content empty model=%s finish_reason=%s max_tokens=%d prompt_tokens=%d completion_tokens=%d total_tokens=%d retryable=%t",
|
||||
c.Model, finishReason, maxTokens, usagePrompt, usageOut, usageTotal, retryable)
|
||||
return Completion{}, retryable, errEmptyModelResponse
|
||||
}
|
||||
if lengthCapped {
|
||||
// Truncated JSON was previously treated as success → parse_failed → synthesize.
|
||||
// Accept only when the truncated body still parses as a JSON object.
|
||||
if _, err := ParseJSONObject(text); err != nil {
|
||||
retryable := canBump
|
||||
truncErr := fmt.Errorf("%w (finish_reason=length max_tokens=%d prompt_tokens=%d completion_tokens=%d content_runes=%d)",
|
||||
errLengthCappedResponse, maxTokens, usagePrompt, usageOut, len([]rune(text)))
|
||||
stopWait(truncErr)
|
||||
log.Printf("openai: chat content truncated model=%s finish_reason=length max_tokens=%d prompt_tokens=%d completion_tokens=%d total_tokens=%d content_runes=%d retryable=%t",
|
||||
c.Model, maxTokens, usagePrompt, usageOut, usageTotal, len([]rune(text)), retryable)
|
||||
return Completion{}, retryable, errLengthCappedResponse
|
||||
}
|
||||
}
|
||||
stopWait(nil)
|
||||
log.Printf("openai: chat content ok model=%s finish_reason=%s content_runes=%d prompt_tokens=%d completion_tokens=%d total_tokens=%d",
|
||||
c.Model, finishReason, len([]rune(text)), usagePrompt, usageOut, usageTotal)
|
||||
return Completion{
|
||||
Text: text,
|
||||
PromptTokens: parsed.Usage.PromptTokens,
|
||||
OutputTokens: parsed.Usage.CompletionTokens,
|
||||
TotalTokens: parsed.Usage.TotalTokens,
|
||||
PromptTokens: usagePrompt,
|
||||
OutputTokens: usageOut,
|
||||
TotalTokens: usageTotal,
|
||||
Model: parsed.Model,
|
||||
Raw: map[string]any{
|
||||
"model": parsed.Model,
|
||||
@@ -678,7 +717,7 @@ func inventHeuristicDescription(system, user, name string) string {
|
||||
name = "Product"
|
||||
}
|
||||
cat := labeledPromptValue(user, "category:")
|
||||
if isPromptLabelTitle(cat) {
|
||||
if isUnusableCategoryValue(cat, name) {
|
||||
cat = ""
|
||||
}
|
||||
lang := languageCodeFromEnhancePrompt(system, user)
|
||||
|
||||
@@ -201,8 +201,8 @@ func TestOpenAIClient_doComplete_emptyContentLengthRetriesWithBudget(t *testing.
|
||||
})
|
||||
return
|
||||
}
|
||||
if maxTok != float64(maxTokensReasoningBudget) {
|
||||
t.Errorf("retry max_tokens=%v want %d", maxTok, maxTokensReasoningBudget)
|
||||
if maxTok != float64(MaxTokensEnhanceRetry) {
|
||||
t.Errorf("retry max_tokens=%v want %d", maxTok, MaxTokensEnhanceRetry)
|
||||
}
|
||||
_ = json.NewEncoder(w).Encode(map[string]any{
|
||||
"model": "code-fast",
|
||||
@@ -228,6 +228,61 @@ func TestOpenAIClient_doComplete_emptyContentLengthRetriesWithBudget(t *testing.
|
||||
}
|
||||
}
|
||||
|
||||
func TestOpenAIClient_doComplete_truncatedJSONLengthRetriesFromEnhanceBudget(t *testing.T) {
|
||||
t.Parallel()
|
||||
calls := 0
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
calls++
|
||||
var req map[string]any
|
||||
_ = json.NewDecoder(r.Body).Decode(&req)
|
||||
maxTok, _ := req["max_tokens"].(float64)
|
||||
if calls == 1 {
|
||||
if maxTok != float64(MaxTokensEnhance) {
|
||||
t.Errorf("first max_tokens=%v want %d", maxTok, MaxTokensEnhance)
|
||||
}
|
||||
_ = json.NewEncoder(w).Encode(map[string]any{
|
||||
"model": "code-fast",
|
||||
"choices": []map[string]any{{
|
||||
"finish_reason": "length",
|
||||
"message": map[string]any{
|
||||
"role": "assistant",
|
||||
"content": `{"name":"Lenovo","description":"<h1>Lenovo G27-20: Igralski monitor za vrhunsko vizualno izkušn`,
|
||||
},
|
||||
}},
|
||||
"usage": map[string]int{"prompt_tokens": 200, "completion_tokens": MaxTokensEnhance, "total_tokens": 200 + MaxTokensEnhance},
|
||||
})
|
||||
return
|
||||
}
|
||||
if maxTok != float64(MaxTokensEnhanceRetry) {
|
||||
t.Errorf("retry max_tokens=%v want %d", maxTok, MaxTokensEnhanceRetry)
|
||||
}
|
||||
_ = json.NewEncoder(w).Encode(map[string]any{
|
||||
"model": "code-fast",
|
||||
"choices": []map[string]any{{
|
||||
"finish_reason": "stop",
|
||||
"message": map[string]any{
|
||||
"role": "assistant",
|
||||
"content": `{"name":"Lenovo G27-20","description":"<h1>Lenovo G27-20</h1><p>Monitor.</p>"}`,
|
||||
},
|
||||
}},
|
||||
"usage": map[string]int{"prompt_tokens": 200, "completion_tokens": 80, "total_tokens": 280},
|
||||
})
|
||||
}))
|
||||
defer srv.Close()
|
||||
c := NewOpenAIClient("test-key", srv.URL, "code-fast", 0, 2)
|
||||
c.HTTPClient = srv.Client()
|
||||
comp, err := c.CompleteWithOptions(context.Background(), "sys", "user", CompleteOptions{MaxTokens: MaxTokensEnhance, Temperature: 0.2})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if calls != 2 {
|
||||
t.Fatalf("calls=%d want 2 (length truncated must not succeed)", calls)
|
||||
}
|
||||
if !strings.Contains(comp.Text, "</h1>") {
|
||||
t.Fatalf("text=%q", comp.Text)
|
||||
}
|
||||
}
|
||||
|
||||
func TestOpenAIClient_Complete_timeoutNotRetried(t *testing.T) {
|
||||
calls := 0
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
@@ -269,7 +324,7 @@ func TestOpenAIClient_Complete_timeoutNotRetried(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestOpenAIClient_Complete_emptyAtReasoningBudgetNotRetried(t *testing.T) {
|
||||
func TestOpenAIClient_Complete_emptyAtEnhanceRetryBudgetNotRetried(t *testing.T) {
|
||||
t.Parallel()
|
||||
calls := 0
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
@@ -280,21 +335,53 @@ func TestOpenAIClient_Complete_emptyAtReasoningBudgetNotRetried(t *testing.T) {
|
||||
"finish_reason": "length",
|
||||
"message": map[string]any{"role": "assistant", "content": "", "reasoning_content": "still thinking"},
|
||||
}},
|
||||
"usage": map[string]int{"prompt_tokens": 1, "completion_tokens": 4096, "total_tokens": 4097},
|
||||
"usage": map[string]int{"prompt_tokens": 1, "completion_tokens": MaxTokensEnhanceRetry, "total_tokens": 1 + MaxTokensEnhanceRetry},
|
||||
})
|
||||
}))
|
||||
defer srv.Close()
|
||||
c := NewOpenAIClient("test-key", srv.URL, "code-fast", 0, 3)
|
||||
c.HTTPClient = srv.Client()
|
||||
_, err := c.CompleteWithOptions(context.Background(), "sys", "user", CompleteOptions{MaxTokens: maxTokensReasoningBudget})
|
||||
_, err := c.CompleteWithOptions(context.Background(), "sys", "user", CompleteOptions{MaxTokens: MaxTokensEnhanceRetry})
|
||||
if err == nil {
|
||||
t.Fatal("expected empty response error")
|
||||
t.Fatal("expected length-capped error")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "empty response") {
|
||||
if !strings.Contains(err.Error(), "length-capped") {
|
||||
t.Fatalf("err=%v", err)
|
||||
}
|
||||
if calls != 1 {
|
||||
t.Fatalf("calls=%d want 1 (no retry at reasoning budget)", calls)
|
||||
t.Fatalf("calls=%d want 1 (no retry at enhance retry ceiling)", calls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestOpenAIClient_Complete_truncatedAtEnhanceRetryBudgetNotSuccess(t *testing.T) {
|
||||
t.Parallel()
|
||||
calls := 0
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
calls++
|
||||
_ = json.NewEncoder(w).Encode(map[string]any{
|
||||
"model": "code-fast",
|
||||
"choices": []map[string]any{{
|
||||
"finish_reason": "length",
|
||||
"message": map[string]any{
|
||||
"role": "assistant",
|
||||
"content": `{"name":"X","description":"<h1>cut off`,
|
||||
},
|
||||
}},
|
||||
"usage": map[string]int{"prompt_tokens": 10, "completion_tokens": MaxTokensEnhanceRetry, "total_tokens": 10 + MaxTokensEnhanceRetry},
|
||||
})
|
||||
}))
|
||||
defer srv.Close()
|
||||
c := NewOpenAIClient("test-key", srv.URL, "code-fast", 0, 3)
|
||||
c.HTTPClient = srv.Client()
|
||||
_, err := c.CompleteWithOptions(context.Background(), "sys", "user", CompleteOptions{MaxTokens: MaxTokensEnhanceRetry})
|
||||
if err == nil {
|
||||
t.Fatal("truncated JSON at ceiling must not succeed")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "length-capped") {
|
||||
t.Fatalf("err=%v", err)
|
||||
}
|
||||
if calls != 1 {
|
||||
t.Fatalf("calls=%d want 1", calls)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -1522,6 +1522,9 @@ func (p *Pipeline) processOne(ctx context.Context, companyID, jobID uuid.UUID, i
|
||||
CategoryNamesByUID: categoryNamesByUID,
|
||||
AllowedAttrKeys: allowedAttrKeysForCategory(cache, stringFromAny(enriched["category"])),
|
||||
CategoryAttrKeys: categoryAttrKeysFromCache(cache),
|
||||
JobID: jobID.String(),
|
||||
CompanyID: companyID.String(),
|
||||
RawProductID: it.RawID.String(),
|
||||
}
|
||||
if it.hydrated {
|
||||
if it.hasPrior {
|
||||
@@ -1577,6 +1580,8 @@ func (p *Pipeline) processOne(ctx context.Context, companyID, jobID uuid.UUID, i
|
||||
jobID, it.RawID, catPreFilter)
|
||||
}
|
||||
syncCategoryName(&result, cache.categoryNamesByUID)
|
||||
scrubCategoryPollution(&result, cache.categoryNamesByUID, cache.categoryUniqueIDs)
|
||||
syncCategoryName(&result, cache.categoryNamesByUID)
|
||||
}
|
||||
// Re-apply after category validation so allowlist matches the persisted category.
|
||||
allowed := allowedAttrKeysForCategory(cache, result.Category)
|
||||
@@ -1676,7 +1681,10 @@ const upsertProcessedProductSQL = `
|
||||
ON CONFLICT (company_id, raw_product_id) DO UPDATE SET
|
||||
product_id = EXCLUDED.product_id,
|
||||
name = EXCLUDED.name,
|
||||
category = COALESCE(NULLIF(BTRIM(EXCLUDED.category), ''), processed_products.category),
|
||||
category = CASE
|
||||
WHEN COALESCE(EXCLUDED.field_sources->>'category', '') = 'cleared_invalid' THEN ''
|
||||
ELSE COALESCE(NULLIF(BTRIM(EXCLUDED.category), ''), processed_products.category)
|
||||
END,
|
||||
description = EXCLUDED.description,
|
||||
processed_name = EXCLUDED.processed_name,
|
||||
processed_description = EXCLUDED.processed_description,
|
||||
|
||||
@@ -4,8 +4,10 @@ import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"log"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/descrybe/descrybe-v2/apps/api/internal/company"
|
||||
"github.com/descrybe/descrybe-v2/apps/api/internal/eprel"
|
||||
@@ -215,6 +217,7 @@ func (e *Engine) RunSteps(ctx context.Context, companyID string, in ProductInput
|
||||
out.ProcessedName = out.Name
|
||||
out.ProcessedDescription = out.Description
|
||||
out.Notes = append(out.Notes, "ai_enhance: skipped (Free plan — upgrade for AI titles/descriptions)")
|
||||
log.Printf("processing: ai_enhance skip reason=entitlement_can_use_ai")
|
||||
preservePriorEnhanceHash()
|
||||
appendStepLog(out.GPTResponse, StepAIEnhance, map[string]any{
|
||||
"status": "skipped",
|
||||
@@ -226,6 +229,7 @@ func (e *Engine) RunSteps(ctx context.Context, companyID string, in ProductInput
|
||||
out.ProcessedName = out.Name
|
||||
out.ProcessedDescription = out.Description
|
||||
out.Notes = append(out.Notes, "ai_enhance: skipped (platform OpenAI unset; configure admin settings or company BYOK)")
|
||||
log.Printf("processing: ai_enhance skip reason=openai_not_configured")
|
||||
preservePriorEnhanceHash()
|
||||
appendStepLog(out.GPTResponse, StepAIEnhance, map[string]any{
|
||||
"status": "skipped",
|
||||
@@ -298,6 +302,10 @@ func (e *Engine) RunSteps(ctx context.Context, companyID string, in ProductInput
|
||||
PriorEnhanceHash: priorHash,
|
||||
PriorProcessedName: priorName,
|
||||
PriorProcessedDescription: priorDesc,
|
||||
JobID: in.JobID,
|
||||
CompanyID: in.CompanyID,
|
||||
RawProductID: in.RawProductID,
|
||||
CategoryUniqueID: out.Category,
|
||||
}, displayCat, enhanceAttrs)
|
||||
out.TotalTokens += tokens
|
||||
status := enhanceStatusFromMeta(raw)
|
||||
@@ -318,6 +326,15 @@ func (e *Engine) RunSteps(ctx context.Context, companyID string, in ProductInput
|
||||
}
|
||||
} else if status == "unchanged" {
|
||||
meta["status"] = "unchanged"
|
||||
} else if status == "refused" || status == "synthesized" || status == "parse_failed" {
|
||||
// Soft quality failure: keep usable copy / synth, never count as real enhance ok.
|
||||
allUnchanged = false
|
||||
meta["status"] = status
|
||||
if reason := enhanceReasonFromMeta(raw); reason != "" {
|
||||
out.Notes = append(out.Notes, "ai_enhance: "+status+" ("+reason+")")
|
||||
} else {
|
||||
out.Notes = append(out.Notes, "ai_enhance: "+status)
|
||||
}
|
||||
} else {
|
||||
allUnchanged = false
|
||||
anyOK = true
|
||||
@@ -327,13 +344,13 @@ func (e *Engine) RunSteps(ctx context.Context, companyID string, in ProductInput
|
||||
name = preferredProductTitle(in.GTIN, name, out.Name, priorName, in.Name)
|
||||
desc = preferredProductDescription(name, desc, out.Description, priorDesc)
|
||||
outerSynth := false
|
||||
if isWeakPriorEnhanceDescription(desc, name) {
|
||||
if descriptionNeedsEnhanceRepair(desc, descTpl, name) {
|
||||
if synth := synthesizeProductDescription(name, displayCat, lang, enhanceAttrs, descTpl); synth != "" {
|
||||
desc = synth
|
||||
outerSynth = true
|
||||
}
|
||||
}
|
||||
weakDesc := isWeakPriorEnhanceDescription(desc, name)
|
||||
weakDesc := descriptionNeedsEnhanceRepair(desc, descTpl, name)
|
||||
// Only persist enhance_input_hash for quality ok / hash-skip unchanged.
|
||||
// Never copy input_hash from error/passthrough/thin/synthesized meta.
|
||||
persistHash := ""
|
||||
@@ -341,7 +358,7 @@ func (e *Engine) RunSteps(ctx context.Context, companyID string, in ProductInput
|
||||
switch {
|
||||
case err != nil:
|
||||
persistHash = ""
|
||||
case status == "synthesized" || outerSynth:
|
||||
case status == "synthesized" || status == "refused" || outerSynth:
|
||||
persistHash = ""
|
||||
allUnchanged = false
|
||||
case status == "unchanged" && !weakDesc:
|
||||
@@ -349,7 +366,7 @@ func (e *Engine) RunSteps(ctx context.Context, companyID string, in ProductInput
|
||||
case status == "ok" && !weakDesc:
|
||||
persistHash = rawHash
|
||||
default:
|
||||
// thin ok / parse_failed / skipped — do not poison reprocess skip
|
||||
// thin ok / parse_failed / formula mismatch / skipped — do not poison reprocess skip
|
||||
persistHash = ""
|
||||
if status == "ok" && weakDesc {
|
||||
allUnchanged = false
|
||||
@@ -404,13 +421,12 @@ func (e *Engine) RunSteps(ctx context.Context, companyID string, in ProductInput
|
||||
if out.ProcessedDescription == "" {
|
||||
out.ProcessedDescription = out.Description
|
||||
}
|
||||
// Timeout/error/empty: always synthesize a factual fallback when a title exists.
|
||||
if out.ProcessedName != "" && (out.ProcessedDescription == "" ||
|
||||
isWeakPriorEnhanceDescription(out.ProcessedDescription, out.ProcessedName, out.Name)) {
|
||||
_, failDescTpl := categoryFormulasFor(in, out.Category)
|
||||
// Timeout/error/empty/formula-miss: always synthesize a factual fallback when a title exists.
|
||||
_, failDescTpl := categoryFormulasFor(in, out.Category)
|
||||
if out.ProcessedName != "" && descriptionNeedsEnhanceRepair(out.ProcessedDescription, failDescTpl, out.ProcessedName, out.Name) {
|
||||
if synth := synthesizeProductDescription(out.ProcessedName, displayCat, primary, enhanceAttrs, failDescTpl); synth != "" {
|
||||
out.ProcessedDescription = synth
|
||||
if out.Description == "" || isWeakPriorEnhanceDescription(out.Description, out.Name, out.ProcessedName) {
|
||||
if out.Description == "" || descriptionNeedsEnhanceRepair(out.Description, failDescTpl, out.Name, out.ProcessedName) {
|
||||
out.Description = synth
|
||||
}
|
||||
if lf, ok := localized[primary]; ok {
|
||||
@@ -453,11 +469,27 @@ func (e *Engine) RunSteps(ctx context.Context, companyID string, in ProductInput
|
||||
out.SkipCreditDebit = true
|
||||
out.FieldSources["name"] = "ai_enhance_unchanged"
|
||||
out.FieldSources["description"] = "ai_enhance_unchanged"
|
||||
out.Notes = append(out.Notes, "ai_enhance: skipped (inputs unchanged)")
|
||||
out.Notes = append(out.Notes, "ai_enhance_unchanged")
|
||||
log.Printf("processing: ai_enhance_unchanged")
|
||||
} else {
|
||||
out.AIProviderMode = e.EngineProviderMode()
|
||||
out.FieldSources["name"] = "ai_enhance"
|
||||
out.FieldSources["description"] = "ai_enhance"
|
||||
for _, m := range langMetas {
|
||||
if mm, ok := m.(map[string]any); ok {
|
||||
reason, _ := mm["reason"].(string)
|
||||
if reason == "" {
|
||||
if raw, ok := mm["raw"].(map[string]any); ok {
|
||||
reason, _ = raw["reason"].(string)
|
||||
}
|
||||
}
|
||||
if reason != "" {
|
||||
out.Notes = append(out.Notes, "ai_enhance: forced re-enhance ("+reason+")")
|
||||
log.Printf("processing: ai_enhance forced re-enhance reason=%s", reason)
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
appendStepLog(out.GPTResponse, StepAIEnhance, map[string]any{
|
||||
"status": map[string]any{"unchanged": allUnchanged, "ok": anyOK, "failed": anyFailed},
|
||||
@@ -475,7 +507,6 @@ func (e *Engine) RunSteps(ctx context.Context, companyID string, in ProductInput
|
||||
preserveCategoryIfEmpty(&out, in.PriorCategory)
|
||||
noteMissingCategory(&out, policy, e != nil && e.Vector != nil && e.Vector.Enabled())
|
||||
syncCategoryName(&out, in.CategoryNamesByUID)
|
||||
displayCat := categoryDisplayLabel(out)
|
||||
out.Name = preferredProductTitle(in.GTIN, out.Name, out.ProcessedName, in.Name, in.PriorProcessedName)
|
||||
out.ProcessedName = preferredProductTitle(in.GTIN, out.ProcessedName, out.Name, in.PriorProcessedName, in.Name)
|
||||
out.Description = preferredProductDescription(out.Name, out.Description, out.ProcessedDescription, in.Description)
|
||||
@@ -483,22 +514,52 @@ func (e *Engine) RunSteps(ctx context.Context, companyID string, in ProductInput
|
||||
if out.ProcessedName == "" {
|
||||
out.ProcessedName = out.Name
|
||||
}
|
||||
// Titles are finalized first so name-as-category and formula leakage can be cleared.
|
||||
scrubCategoryPollution(&out, in.CategoryNamesByUID, nil)
|
||||
syncCategoryName(&out, in.CategoryNamesByUID)
|
||||
displayCat := categoryDisplayLabel(out)
|
||||
// After vector/mapped category resolution, formulas may key for the first time —
|
||||
// repair short prose / title echo that ignored A1 description_template.
|
||||
_, finalDescTpl := categoryFormulasFor(in, out.Category)
|
||||
if out.ProcessedDescription == "" || isWeakPriorEnhanceDescription(out.ProcessedDescription, out.ProcessedName, out.Name) {
|
||||
finalRepaired := false
|
||||
if descriptionNeedsEnhanceRepair(out.ProcessedDescription, finalDescTpl, out.ProcessedName, out.Name) {
|
||||
if synth := synthesizeProductDescription(out.ProcessedName, displayCat, in.Language, out.Attributes, finalDescTpl); synth != "" {
|
||||
out.ProcessedDescription = synth
|
||||
finalRepaired = true
|
||||
}
|
||||
}
|
||||
if out.Description == "" || isWeakPriorEnhanceDescription(out.Description, out.Name, out.ProcessedName) {
|
||||
if out.ProcessedDescription != "" && !isWeakPriorEnhanceDescription(out.ProcessedDescription, out.Name, out.ProcessedName) {
|
||||
if descriptionNeedsEnhanceRepair(out.Description, finalDescTpl, out.Name, out.ProcessedName) {
|
||||
if out.ProcessedDescription != "" && !descriptionNeedsEnhanceRepair(out.ProcessedDescription, finalDescTpl, out.Name, out.ProcessedName) {
|
||||
out.Description = out.ProcessedDescription
|
||||
finalRepaired = true
|
||||
} else if synth := synthesizeProductDescription(out.Name, displayCat, in.Language, out.Attributes, finalDescTpl); synth != "" {
|
||||
out.Description = synth
|
||||
if out.ProcessedDescription == "" || isWeakPriorEnhanceDescription(out.ProcessedDescription, out.ProcessedName, out.Name) {
|
||||
finalRepaired = true
|
||||
if descriptionNeedsEnhanceRepair(out.ProcessedDescription, finalDescTpl, out.ProcessedName, out.Name) {
|
||||
out.ProcessedDescription = synth
|
||||
}
|
||||
}
|
||||
}
|
||||
if finalRepaired {
|
||||
delete(out.FieldSources, FieldEnhanceInputHash)
|
||||
primary := strings.TrimSpace(in.Language)
|
||||
if primary == "" && len(in.ContentLanguages) > 0 {
|
||||
primary = strings.TrimSpace(in.ContentLanguages[0])
|
||||
}
|
||||
if primary == "" {
|
||||
primary = company.DefaultLanguage
|
||||
}
|
||||
for lang, lf := range out.LocalizedContent {
|
||||
lf.EnhanceInputHash = ""
|
||||
if lang == primary {
|
||||
lf.ProcessedDescription = out.ProcessedDescription
|
||||
if lf.ProcessedName == "" {
|
||||
lf.ProcessedName = out.ProcessedName
|
||||
}
|
||||
}
|
||||
out.LocalizedContent[lang] = lf
|
||||
}
|
||||
}
|
||||
if out.Attributes == nil {
|
||||
out.Attributes = map[string]any{}
|
||||
}
|
||||
@@ -528,7 +589,7 @@ func preserveCategoryIfEmpty(out *StepResult, prior string) {
|
||||
return
|
||||
}
|
||||
prior = strings.TrimSpace(prior)
|
||||
if prior == "" {
|
||||
if prior == "" || isUnusableCategoryValue(prior, out.ProcessedName, out.Name) {
|
||||
return
|
||||
}
|
||||
out.Category = SanitizeText(prior)
|
||||
@@ -563,7 +624,7 @@ func tryVectorCategorize(ctx context.Context, e *Engine, companyID string, out *
|
||||
})
|
||||
return
|
||||
}
|
||||
if strings.TrimSpace(cat) == "" {
|
||||
if strings.TrimSpace(cat) == "" || isUnusableCategoryValue(cat, out.ProcessedName, out.Name) {
|
||||
return
|
||||
}
|
||||
out.Category = SanitizeOutput(cat)
|
||||
@@ -632,78 +693,252 @@ func appendStepLog(gpt map[string]any, name string, raw any) {
|
||||
gpt["steps"] = append(steps, map[string]any{"step": name, "raw": raw})
|
||||
}
|
||||
|
||||
// descriptionFormulaRetrySuffix is appended once when LLM JSON ignored the
|
||||
// category description_template (short prose / title echo instead of multi-section HTML).
|
||||
const descriptionFormulaRetrySuffix = "\n\nINVALID DESCRIPTION. Your JSON ignored the Description formula. Reply with ONLY one JSON object; \"description\" must be ONE HTML string covering each formula section in order with matching tags (h1/h2/h3/h4, p, ul)."
|
||||
|
||||
// descriptionNeedsEnhanceRepair is true when desc is weak/empty/title-echo or
|
||||
// fails an active category description_template (A1 multi-section HTML).
|
||||
// Heuristic invent/synth is intentionally excluded here — enhanceHashForceReason
|
||||
// blocks hash-skip for synth so reprocess can still upgrade to real LLM copy.
|
||||
func descriptionNeedsEnhanceRepair(desc string, template any, titles ...string) bool {
|
||||
if isWeakPriorEnhanceDescription(desc, titles...) {
|
||||
return true
|
||||
}
|
||||
return !descriptionSatisfiesFormula(desc, template)
|
||||
}
|
||||
|
||||
// enhanceHashForceReason returns why a matching prior enhance_input_hash must not
|
||||
// skip the LLM (empty = safe to reuse as ai_enhance_unchanged).
|
||||
func enhanceHashForceReason(in ProductInput) string {
|
||||
if isPromptLabelTitle(in.PriorProcessedName) {
|
||||
return "prompt_label_title"
|
||||
}
|
||||
if reason := company.EnhanceHashSkipBlockReason(in.PriorProcessedDescription, in.PriorProcessedName, in.Name); reason != "" {
|
||||
return reason
|
||||
}
|
||||
if !descriptionSatisfiesFormula(in.PriorProcessedDescription, in.DescriptionTemplate) {
|
||||
return "formula-mismatch"
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func (e *Engine) enhance(ctx context.Context, in ProductInput, category string, attrs map[string]any) (string, string, int, any, error) {
|
||||
catUID := strings.TrimSpace(in.CategoryUniqueID)
|
||||
catName := strings.TrimSpace(category)
|
||||
sysTpl, userTpl := resolveProductPromptTemplates(in)
|
||||
hash := HashEnhanceInput(category, in.Name, in.Description, in.BrandPrompt, in.Language, sysTpl, userTpl, attrs)
|
||||
if e == nil || e.Completer == nil {
|
||||
name := preferredProductTitle(in.GTIN, in.Name, in.PriorProcessedName)
|
||||
logEnhanceOutcome(in, catUID, catName, "skip", "completer_nil", Completion{}, 0, "")
|
||||
return name,
|
||||
preferredProductDescription(name, in.Description, in.PriorProcessedDescription),
|
||||
0, map[string]any{"status": "skipped", "input_hash": hash}, nil
|
||||
0, map[string]any{"status": "skipped", "reason": "completer_nil"}, nil
|
||||
}
|
||||
// Skip LLM when inputs match the last successful enhance (before any credit debit),
|
||||
// but never reuse a thin / title-echo prior description, or a prompt-leakage title.
|
||||
// but never reuse thin / title-echo / invent-synth / formula-mismatch priors,
|
||||
// or a prompt-leakage title.
|
||||
var forceReason string
|
||||
if in.PriorEnhanceHash != "" && in.PriorEnhanceHash == hash &&
|
||||
(in.PriorProcessedName != "" || in.PriorProcessedDescription != "") &&
|
||||
!isPromptLabelTitle(in.PriorProcessedName) &&
|
||||
!isWeakPriorEnhanceDescription(in.PriorProcessedDescription, in.PriorProcessedName, in.Name) {
|
||||
name := preferredProductTitle(in.GTIN, in.PriorProcessedName, in.Name)
|
||||
desc := preferredProductDescription(name, in.PriorProcessedDescription, in.Description)
|
||||
return name, desc, 0, map[string]any{
|
||||
"status": "unchanged",
|
||||
"input_hash": hash,
|
||||
}, nil
|
||||
(in.PriorProcessedName != "" || in.PriorProcessedDescription != "") {
|
||||
forceReason = enhanceHashForceReason(in)
|
||||
if forceReason == "" {
|
||||
name := preferredProductTitle(in.GTIN, in.PriorProcessedName, in.Name)
|
||||
desc := preferredProductDescription(name, in.PriorProcessedDescription, in.Description)
|
||||
logEnhanceOutcome(in, catUID, catName, "skip", "unchanged_hash", Completion{}, 0, "")
|
||||
return name, desc, 0, map[string]any{
|
||||
"status": "unchanged",
|
||||
"input_hash": hash,
|
||||
}, nil
|
||||
}
|
||||
log.Printf("processing: ai_enhance skip_blocked reason=%s category=%q name=%q",
|
||||
forceReason, truncateRunes(category, 80), truncateRunes(in.Name, 80))
|
||||
}
|
||||
system, user := RenderProductEnhancePrompts(sysTpl, userTpl, category, in.Name, in.Description, in.GTIN, in.BrandPrompt, in.Language, attrs)
|
||||
logEnhancePrompt(in, catUID, catName, system, user)
|
||||
started := time.Now()
|
||||
comp, obj, err := CompleteJSON(ctx, e.Completer, system, user, CompleteOptions{
|
||||
MaxTokens: MaxTokensEnhance,
|
||||
Temperature: DefaultStructuredTemp,
|
||||
})
|
||||
elapsed := time.Since(started)
|
||||
if err != nil {
|
||||
// Network/provider failure vs parse failure after retry
|
||||
name := preferredProductTitle(in.GTIN, in.Name, in.PriorProcessedName)
|
||||
desc := preferredProductDescription(name, in.Description, in.PriorProcessedDescription)
|
||||
if isWeakPriorEnhanceDescription(desc, name) {
|
||||
outcome := "refuse"
|
||||
reason := "empty_or_provider_error"
|
||||
if descriptionNeedsEnhanceRepair(desc, in.DescriptionTemplate, name) {
|
||||
if synth := synthesizeProductDescription(name, category, in.Language, attrs, in.DescriptionTemplate); synth != "" {
|
||||
desc = synth
|
||||
outcome = "synthesized"
|
||||
}
|
||||
}
|
||||
if obj == nil && comp.Text == "" {
|
||||
logEnhanceOutcome(in, catUID, catName, outcome, reason+":"+TruncateError(err), comp, elapsed, forceReason)
|
||||
return name, desc, 0, map[string]any{
|
||||
"provider": "passthrough",
|
||||
"error": TruncateError(err),
|
||||
}, err
|
||||
}
|
||||
// Parse failed after retry — keep usable copy; synthesize when empty/weak.
|
||||
// Parse failed after retry — keep usable copy; synthesize when empty/weak/formula-miss.
|
||||
if outcome != "synthesized" {
|
||||
outcome = "refuse"
|
||||
reason = "parse_failed"
|
||||
} else {
|
||||
reason = "parse_failed"
|
||||
}
|
||||
logEnhanceOutcome(in, catUID, catName, outcome, reason, comp, elapsed, forceReason)
|
||||
return name, desc, comp.TotalTokens, map[string]any{
|
||||
"status": "parse_failed",
|
||||
"reason": "invalid_json",
|
||||
"error": "AI returned invalid JSON; kept original title/description",
|
||||
"raw": truncateRunes(comp.Text, 200),
|
||||
}, nil
|
||||
}
|
||||
name := preferredProductTitle(in.GTIN, SanitizeOutput(fmt.Sprint(obj["name"])), in.Name, in.PriorProcessedName)
|
||||
desc := preferredProductDescription(name, SanitizeOutput(fmt.Sprint(obj["description"])), in.Description, in.PriorProcessedDescription)
|
||||
llmName := SanitizeOutput(fmt.Sprint(obj["name"]))
|
||||
llmDesc := SanitizeOutput(fmt.Sprint(obj["description"]))
|
||||
// Never prefer-fallback to originals for empty/leakage LLM fields — that stamped
|
||||
// status=ok + input_hash on garbage enhance output (prod: 30–272ms "ok", wrong copy).
|
||||
if reason := llmEnhanceHardRefuseReason(llmName, llmDesc); reason != "" {
|
||||
name := preferredProductTitle(in.GTIN, llmName, in.Name, in.PriorProcessedName)
|
||||
desc := preferredProductDescription(name, llmDesc)
|
||||
synthesized := false
|
||||
if descriptionNeedsEnhanceRepair(desc, in.DescriptionTemplate, name) {
|
||||
if desc == "" {
|
||||
desc = preferredProductDescription(name, in.Description, in.PriorProcessedDescription)
|
||||
}
|
||||
if descriptionNeedsEnhanceRepair(desc, in.DescriptionTemplate, name) {
|
||||
if synth := synthesizeProductDescription(name, category, in.Language, attrs, in.DescriptionTemplate); synth != "" {
|
||||
desc = synth
|
||||
synthesized = true
|
||||
}
|
||||
}
|
||||
}
|
||||
status := "refused"
|
||||
outcome := "refuse"
|
||||
if synthesized {
|
||||
status = "synthesized"
|
||||
outcome = "synthesized"
|
||||
}
|
||||
meta := map[string]any{
|
||||
"status": status,
|
||||
"reason": reason,
|
||||
"raw": comp.Raw,
|
||||
}
|
||||
if forceReason != "" {
|
||||
meta["forced_reenhance"] = true
|
||||
meta["force_reason"] = forceReason
|
||||
}
|
||||
logEnhanceOutcome(in, catUID, catName, outcome, reason, comp, elapsed, forceReason)
|
||||
return name, desc, comp.TotalTokens, meta, nil
|
||||
}
|
||||
|
||||
name := preferredProductTitle(in.GTIN, llmName, in.Name, in.PriorProcessedName)
|
||||
// Quality-gate on LLM description alone (do not absorb originals yet).
|
||||
desc := preferredProductDescription(name, llmDesc)
|
||||
didFormulaRetry := false
|
||||
// Fast garbage that ignores A1 description_template: one formula-aware retry, then synthesize.
|
||||
if !descriptionSatisfiesFormula(desc, in.DescriptionTemplate) &&
|
||||
FormatDescriptionFormulaConstraint(in.DescriptionTemplate) != "" {
|
||||
didFormulaRetry = true
|
||||
retryUser := user + descriptionFormulaRetrySuffix
|
||||
retryStarted := time.Now()
|
||||
comp2, obj2, err2 := CompleteJSON(ctx, e.Completer, system, retryUser, CompleteOptions{
|
||||
MaxTokens: MaxTokensEnhance,
|
||||
Temperature: DefaultStructuredTemp,
|
||||
})
|
||||
elapsed += time.Since(retryStarted)
|
||||
comp.PromptTokens += comp2.PromptTokens
|
||||
comp.OutputTokens += comp2.OutputTokens
|
||||
comp.TotalTokens += comp2.TotalTokens
|
||||
if err2 == nil && obj2 != nil {
|
||||
n2 := SanitizeOutput(fmt.Sprint(obj2["name"]))
|
||||
d2 := SanitizeOutput(fmt.Sprint(obj2["description"]))
|
||||
if llmEnhanceHardRefuseReason(n2, d2) == "" {
|
||||
name = preferredProductTitle(in.GTIN, n2, name, in.Name, in.PriorProcessedName)
|
||||
desc = preferredProductDescription(name, d2)
|
||||
if comp2.Raw != nil {
|
||||
comp.Raw = comp2.Raw
|
||||
}
|
||||
if comp2.Text != "" {
|
||||
comp.Text = comp2.Text
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
synthesized := false
|
||||
if isWeakPriorEnhanceDescription(desc, name) {
|
||||
if synth := synthesizeProductDescription(name, category, in.Language, attrs, in.DescriptionTemplate); synth != "" {
|
||||
desc = synth
|
||||
synthesized = true
|
||||
if descriptionNeedsEnhanceRepair(desc, in.DescriptionTemplate, name) {
|
||||
if desc == "" {
|
||||
desc = preferredProductDescription(name, in.Description, in.PriorProcessedDescription)
|
||||
}
|
||||
if descriptionNeedsEnhanceRepair(desc, in.DescriptionTemplate, name) {
|
||||
if synth := synthesizeProductDescription(name, category, in.Language, attrs, in.DescriptionTemplate); synth != "" {
|
||||
desc = synth
|
||||
synthesized = true
|
||||
}
|
||||
}
|
||||
}
|
||||
meta := map[string]any{
|
||||
"status": "ok",
|
||||
"raw": comp.Raw,
|
||||
}
|
||||
if forceReason != "" {
|
||||
meta["reason"] = forceReason
|
||||
meta["forced_reenhance"] = true
|
||||
}
|
||||
outcome := "ok"
|
||||
reason := forceReason
|
||||
// Heuristic synthesize must not poison enhance_input_hash (would hash-skip
|
||||
// formula-aware LLM copy on reprocess). Only persist hash for real LLM quality.
|
||||
if synthesized {
|
||||
meta["status"] = "synthesized"
|
||||
} else if !isWeakPriorEnhanceDescription(desc, name) {
|
||||
outcome = "synthesized"
|
||||
if didFormulaRetry {
|
||||
reason = "formula_retry"
|
||||
} else {
|
||||
reason = "weak_or_formula_mismatch"
|
||||
}
|
||||
meta["reason"] = reason
|
||||
} else if descriptionNeedsEnhanceRepair(desc, in.DescriptionTemplate, name) {
|
||||
meta["status"] = "refused"
|
||||
outcome = "refuse"
|
||||
if didFormulaRetry {
|
||||
reason = "formula_retry"
|
||||
} else {
|
||||
reason = "weak_or_formula_mismatch"
|
||||
}
|
||||
meta["reason"] = reason
|
||||
} else {
|
||||
meta["input_hash"] = hash
|
||||
if didFormulaRetry {
|
||||
reason = "formula_retry"
|
||||
if meta["reason"] == nil {
|
||||
meta["reason"] = reason
|
||||
}
|
||||
}
|
||||
}
|
||||
logEnhanceOutcome(in, catUID, catName, outcome, reason, comp, elapsed, forceReason)
|
||||
return name, desc, comp.TotalTokens, meta, nil
|
||||
}
|
||||
|
||||
// llmEnhanceHardRefuseReason reports empty/leakage LLM fields that must never be
|
||||
// accepted as quality enhance (even when originals could fill the gap).
|
||||
func llmEnhanceHardRefuseReason(name, desc string) string {
|
||||
name = strings.TrimSpace(name)
|
||||
desc = strings.TrimSpace(desc)
|
||||
if name == "" || name == "<nil>" {
|
||||
return "empty_name"
|
||||
}
|
||||
if isPromptLabelTitle(name) {
|
||||
return "prompt_leakage_name"
|
||||
}
|
||||
if desc == "" || desc == "<nil>" {
|
||||
return "empty_description"
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func sanitizeJSON(v any) string {
|
||||
if v == nil {
|
||||
return "{}"
|
||||
@@ -841,6 +1076,24 @@ var promptLeakagePhrases = []string{
|
||||
"prefer attrs values",
|
||||
"order matters; join with",
|
||||
"do not hardcode a language",
|
||||
// Description / title formula scaffolding (Format*FormulaConstraint + enhance templates).
|
||||
"emit description as one html",
|
||||
"emit one html string",
|
||||
"covering each section",
|
||||
"tags matching section",
|
||||
"overrides any shorter",
|
||||
"competing full html",
|
||||
"metadescription",
|
||||
"never ignore the description",
|
||||
"follow any description formula",
|
||||
"when a description formula",
|
||||
"structured html sections",
|
||||
"never copy name as description",
|
||||
"do not invent specs",
|
||||
"prefer 1-3 factual",
|
||||
"1-2 sentences",
|
||||
"keep literal text as written",
|
||||
"build name from attrs using this structure",
|
||||
}
|
||||
|
||||
var promptLeakageLongKeywords = []string{
|
||||
@@ -850,6 +1103,97 @@ var promptLeakageLongKeywords = []string{
|
||||
"json",
|
||||
"attrs",
|
||||
"retail title",
|
||||
"html string",
|
||||
"section type",
|
||||
}
|
||||
|
||||
// isUnusableCategoryValue rejects tokens that must never become Category /
|
||||
// CategoryName: prompt labels / title-formula scaffolding, or the product
|
||||
// title itself (name-as-category). Callers with taxonomy should prefer
|
||||
// scrubCategoryPollution so a title that legitimately equals a category name
|
||||
// can still resolve to unique_id.
|
||||
func isUnusableCategoryValue(s string, productTitles ...string) bool {
|
||||
s = strings.TrimSpace(s)
|
||||
if s == "" || s == "<nil>" {
|
||||
return false
|
||||
}
|
||||
if isPromptLabelTitle(s) {
|
||||
return true
|
||||
}
|
||||
return categoryTokenEqualsProductTitle(s, productTitles...)
|
||||
}
|
||||
|
||||
func categoryTokenEqualsProductTitle(s string, productTitles ...string) bool {
|
||||
s = strings.TrimSpace(s)
|
||||
if s == "" || s == "<nil>" {
|
||||
return false
|
||||
}
|
||||
for _, t := range productTitles {
|
||||
t = strings.TrimSpace(t)
|
||||
if t == "" || t == "<nil>" {
|
||||
continue
|
||||
}
|
||||
if strings.EqualFold(s, t) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func taxonomyDisplayNameSet(namesByUID map[string]string) map[string]struct{} {
|
||||
if len(namesByUID) == 0 {
|
||||
return nil
|
||||
}
|
||||
out := make(map[string]struct{}, len(namesByUID))
|
||||
for _, name := range namesByUID {
|
||||
n := strings.ToLower(strings.TrimSpace(name))
|
||||
if n != "" {
|
||||
out[n] = struct{}{}
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// scrubCategoryPollution clears Category / CategoryName when they hold prompt
|
||||
// leakage/formula text, or equal the product title without resolving to a
|
||||
// company taxonomy unique_id / display name.
|
||||
func scrubCategoryPollution(out *StepResult, namesByUID map[string]string, valid map[string]struct{}) {
|
||||
if out == nil {
|
||||
return
|
||||
}
|
||||
titles := []string{out.ProcessedName, out.Name}
|
||||
clearCategory := func(note string) {
|
||||
out.Category = ""
|
||||
out.CategoryName = ""
|
||||
if out.FieldSources == nil {
|
||||
out.FieldSources = map[string]any{}
|
||||
}
|
||||
out.FieldSources["category"] = "cleared_invalid"
|
||||
out.Notes = append(out.Notes, note)
|
||||
}
|
||||
cat := strings.TrimSpace(out.Category)
|
||||
if cat != "" {
|
||||
switch {
|
||||
case isPromptLabelTitle(cat):
|
||||
clearCategory("category: ignored (prompt leakage)")
|
||||
case categoryTokenEqualsProductTitle(cat, titles...) &&
|
||||
resolveCompanyCategoryUniqueID(cat, namesByUID, valid) == "":
|
||||
clearCategory("category: ignored (product title)")
|
||||
}
|
||||
}
|
||||
name := strings.TrimSpace(out.CategoryName)
|
||||
if name == "" {
|
||||
return
|
||||
}
|
||||
if isPromptLabelTitle(name) {
|
||||
out.CategoryName = ""
|
||||
return
|
||||
}
|
||||
if categoryTokenEqualsProductTitle(name, titles...) {
|
||||
if _, ok := taxonomyDisplayNameSet(namesByUID)[strings.ToLower(name)]; !ok {
|
||||
out.CategoryName = ""
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// preferredProductTitle picks the first usable title, skipping empty values,
|
||||
|
||||
@@ -223,8 +223,12 @@ func TestIsPromptLeakageTitle(t *testing.T) {
|
||||
"write name in {{language}}": true,
|
||||
"Schema: {\"name\":\"string\",\"description\":\"string\"}": true,
|
||||
"Reply with ONLY JSON (no markdown)": true,
|
||||
"Vox SWA-8000W dishwasher": false,
|
||||
"Acme Widget Pro": false,
|
||||
"Emit description as ONE HTML string covering each section in order": true,
|
||||
"Description formula (REQUIRED — overrides any shorter \"1-2 sentences\" rule)": true,
|
||||
"when a Description formula follows, emit ONE HTML string": true,
|
||||
"never copy name as description": true,
|
||||
"Vox SWA-8000W dishwasher": false,
|
||||
"Acme Widget Pro": false,
|
||||
"": false,
|
||||
}
|
||||
for in, want := range cases {
|
||||
@@ -242,6 +246,119 @@ func TestIsPromptLeakageTitle(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestIsUnusableCategoryValue_rejectsNameAndFormula(t *testing.T) {
|
||||
t.Parallel()
|
||||
title := "NOSILEC W53070 81-140CM 180ST VOGELS"
|
||||
if !isUnusableCategoryValue(title, title) {
|
||||
t.Fatal("product title must not be usable as category")
|
||||
}
|
||||
if !isUnusableCategoryValue("Title formula (order matters; join with \" \")", title) {
|
||||
t.Fatal("title formula must not be usable as category")
|
||||
}
|
||||
if !isUnusableCategoryValue("Emit description as ONE HTML string covering each section", title) {
|
||||
t.Fatal("description formula must not be usable as category")
|
||||
}
|
||||
if !isUnusableCategoryValue("Category:", title) {
|
||||
t.Fatal("prompt label must not be usable as category")
|
||||
}
|
||||
if isUnusableCategoryValue("28", title) {
|
||||
t.Fatal("taxonomy unique_id must remain usable")
|
||||
}
|
||||
if isUnusableCategoryValue("TV mounts", title) {
|
||||
t.Fatal("real category display name must remain usable")
|
||||
}
|
||||
}
|
||||
|
||||
func TestScrubCategoryPollution_clearsNameAsCategory(t *testing.T) {
|
||||
t.Parallel()
|
||||
title := "Vogel's WALL 3245 TV Wall Mount"
|
||||
out := StepResult{
|
||||
Name: title,
|
||||
ProcessedName: title,
|
||||
Category: title,
|
||||
CategoryName: title,
|
||||
FieldSources: map[string]any{"category": "vector"},
|
||||
}
|
||||
scrubCategoryPollution(&out, nil, nil)
|
||||
if out.Category != "" || out.CategoryName != "" {
|
||||
t.Fatalf("expected category cleared, got Category=%q CategoryName=%q", out.Category, out.CategoryName)
|
||||
}
|
||||
if src, _ := out.FieldSources["category"].(string); src != "cleared_invalid" {
|
||||
t.Fatalf("field_sources.category=%v want cleared_invalid", out.FieldSources["category"])
|
||||
}
|
||||
}
|
||||
|
||||
func TestEnhance_rejectsDescriptionFormulaLeakageTitle(t *testing.T) {
|
||||
leak := "Emit description as ONE HTML string covering each section in order in Slovenian"
|
||||
e := &Engine{
|
||||
Completer: stubCompleter{fn: func(system, user string) (Completion, error) {
|
||||
return Completion{Text: `{"name":"` + leak + `","description":"A solid washer."}`, TotalTokens: 2}, nil
|
||||
}},
|
||||
Vector: NoopVectorCategorizer{},
|
||||
}
|
||||
out, err := e.RunSteps(context.Background(), "co", ProductInput{
|
||||
GTIN: "8712285326882",
|
||||
Name: "Vox SWA-8000W",
|
||||
Mapped: map[string]any{"name": "Vox SWA-8000W", "description": "Washer", "category": "demo-electronics"},
|
||||
}, "enhance_only", nil, StepPolicy{AllowAI: true})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if out.ProcessedName != "Vox SWA-8000W" {
|
||||
t.Fatalf("ProcessedName=%q want Vox SWA-8000W (reject description formula leakage)", out.ProcessedName)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunSteps_rejectsProductNameAsCategory(t *testing.T) {
|
||||
title := "NOSILEC W53070 81-140CM 180ST VOGELS"
|
||||
e := &Engine{
|
||||
Completer: stubCompleter{fn: func(system, user string) (Completion, error) {
|
||||
return Completion{Text: `{"name":"` + title + `","description":"TV wall mount for large screens."}`, TotalTokens: 2}, nil
|
||||
}},
|
||||
Vector: NoopVectorCategorizer{},
|
||||
}
|
||||
out, err := e.RunSteps(context.Background(), "co", ProductInput{
|
||||
GTIN: "8712285326882",
|
||||
Name: title,
|
||||
PriorCategory: title,
|
||||
Mapped: map[string]any{"name": title, "description": title, "category": title},
|
||||
}, "enhance_only", nil, StepPolicy{AllowAI: true})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if out.Category == title || strings.EqualFold(out.Category, out.ProcessedName) {
|
||||
t.Fatalf("Category=%q must not equal product title", out.Category)
|
||||
}
|
||||
if categoryDisplayLabel(out) == title {
|
||||
t.Fatalf("display category leaked product title: %q", categoryDisplayLabel(out))
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunSteps_rejectsFormulaAsCategory(t *testing.T) {
|
||||
leak := "Title formula (order matters; join with \" \"). Build name from Attrs"
|
||||
e := &Engine{
|
||||
Completer: stubCompleter{fn: func(system, user string) (Completion, error) {
|
||||
return Completion{Text: `{"name":"Vox SWA-8000W","description":"Washer."}`, TotalTokens: 2}, nil
|
||||
}},
|
||||
Vector: NoopVectorCategorizer{},
|
||||
}
|
||||
out, err := e.RunSteps(context.Background(), "co", ProductInput{
|
||||
GTIN: "8712285326882",
|
||||
Name: "Vox SWA-8000W",
|
||||
PriorCategory: leak,
|
||||
Mapped: map[string]any{"name": "Vox SWA-8000W", "description": "Washer", "category": leak},
|
||||
}, "enhance_only", nil, StepPolicy{AllowAI: true})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if out.Category != "" {
|
||||
t.Fatalf("Category=%q want empty (formula rejected)", out.Category)
|
||||
}
|
||||
if categoryDisplayLabel(out) != "" {
|
||||
t.Fatalf("display=%q want empty", categoryDisplayLabel(out))
|
||||
}
|
||||
}
|
||||
|
||||
func TestEnhance_rejectsFormulaLeakageTitle(t *testing.T) {
|
||||
leak := "short retail title; follow any Title formula constraints that follow; use Attrs"
|
||||
e := &Engine{
|
||||
|
||||
@@ -28,6 +28,10 @@ func TestUpsertProcessedProductSQL_usesOnConflict(t *testing.T) {
|
||||
if !strings.Contains(upsertProcessedProductSQL, "COALESCE(NULLIF(BTRIM(EXCLUDED.category), ''), processed_products.category)") {
|
||||
t.Fatalf("expected category preserve on conflict: got SQL without COALESCE preserve")
|
||||
}
|
||||
// Intentional scrub/filter clears must wipe a previously persisted product-title category.
|
||||
if !strings.Contains(upsertProcessedProductSQL, "cleared_invalid") {
|
||||
t.Fatalf("expected cleared_invalid category wipe path in upsert SQL")
|
||||
}
|
||||
// Free template meta must persist; existing non-empty meta must be preserved on reprocess.
|
||||
if !strings.Contains(upsertProcessedProductSQL, "meta_title") || !strings.Contains(upsertProcessedProductSQL, "meta_description") {
|
||||
t.Fatalf("expected meta_title/meta_description columns in upsert")
|
||||
|
||||
@@ -326,11 +326,23 @@ func (p *Pipeline) LoadV1ProcessJobItems(ctx context.Context, companyID, jobID u
|
||||
if name, ok := categoryNames[catStr]; ok && name != "" {
|
||||
catNameStr = name
|
||||
} else if nested, ok := mapped["category"].(map[string]any); ok {
|
||||
for _, k := range []string{"name", "category_name", "title"} {
|
||||
if s := strings.TrimSpace(stringFromAny(nested[k])); s != "" {
|
||||
catNameStr = s
|
||||
break
|
||||
// Only accept nested display names that resolve in company taxonomy —
|
||||
// never product title / title-formula leftovers from feed objects.
|
||||
for _, k := range []string{"name", "category_name"} {
|
||||
s := strings.TrimSpace(stringFromAny(nested[k]))
|
||||
if s == "" || isUnusableCategoryValue(s, titleStr) {
|
||||
continue
|
||||
}
|
||||
uid := resolveCompanyCategoryUniqueID(s, categoryNames, nil)
|
||||
if uid == "" {
|
||||
continue
|
||||
}
|
||||
if resolvedName := strings.TrimSpace(categoryNames[uid]); resolvedName != "" {
|
||||
catNameStr = resolvedName
|
||||
} else {
|
||||
catNameStr = s
|
||||
}
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user