This commit is contained in:
2026-08-16 21:35:48 +02:00
parent 106f602232
commit 389608ea56
28 changed files with 2491 additions and 115 deletions
@@ -23,7 +23,8 @@ type RepairWeakEnhanceHashesResult struct {
// RepairWeakEnhanceHashes clears enhance_input_hash from field_sources and
// localized_content for processed products whose descriptions are weak /
// title-echo / filler (so the next enhance cannot hash-skip thin priors).
// title-echo / filler / invent-synthesize (so the next enhance cannot hash-skip
// thin or heuristic priors).
// Returns the number of products updated.
func RepairWeakEnhanceHashes(ctx context.Context, pool *pgxpool.Pool, companyID uuid.UUID) (int64, error) {
res, err := RepairWeakEnhanceHashesDetailed(ctx, pool, companyID)
@@ -124,7 +125,7 @@ func repairWeakEnhanceHashesTx(ctx context.Context, tx pgx.Tx, companyID uuid.UU
}
if _, has := fs[enhanceInputHashKey]; has {
if company.IsWeakPriorEnhanceDescription(primaryDesc, primaryName, name) {
if company.ShouldRefuseEnhanceHashSkip(primaryDesc, primaryName, name) {
delete(fs, enhanceInputHashKey)
changed = true
}
@@ -139,7 +140,7 @@ func repairWeakEnhanceHashesTx(ctx context.Context, tx pgx.Tx, companyID uuid.UU
if strings.TrimSpace(fields.EnhanceInputHash) == "" {
continue
}
if company.IsWeakPriorEnhanceDescription(d, n, primaryName, name) {
if company.ShouldRefuseEnhanceHashSkip(d, n, primaryName, name) {
fields.EnhanceInputHash = ""
loc[lang] = fields
changed = true
+5 -5
View File
@@ -950,7 +950,7 @@ func (s *Service) ListRawProducts(ctx context.Context, companyID uuid.UUID, f Li
SELECT rp.id, rp.gtin, rp.feed_id, rp.is_processed, rp.processing_status,
COALESCE(NULLIF(rp.mapped_data->>'name', ''), NULLIF(rp.mapped_data->>'title', ''), '') AS name,
NULLIF(BTRIM(rp.mapped_data->>'category'), '') AS category,
COALESCE(cat.name, NULLIF(BTRIM(rp.mapped_data->>'category'), '')) AS category_name,
cat.name AS category_name,
COALESCE(cat.unique_id, NULLIF(BTRIM(rp.mapped_data->>'category'), '')) AS category_unique_id,
f.name AS feed_name,
rp.mapped_data->'_sync_changes' AS sync_changes,
@@ -1099,7 +1099,7 @@ func (s *Service) ListProcessedProducts(ctx context.Context, companyID uuid.UUID
`+processedPreferredNameSQL+` AS name,
COALESCE(NULLIF(p.processed_name, ''), '') AS processed_name,
p.category,
COALESCE(cat.name, NULLIF(BTRIM(p.category), '')) AS category_name,
cat.name AS category_name,
COALESCE(cat.unique_id, NULLIF(BTRIM(p.category), '')) AS category_unique_id,
p.status, p.raw_product_id, COALESCE(p.feed_id, r.feed_id) AS feed_id, r.gtin,
f.name AS feed_name,
@@ -1186,7 +1186,7 @@ func (s *Service) ListProcessedProductsDetailed(ctx context.Context, companyID u
SELECT p.id, p.product_id,
`+processedPreferredNameSQL+` AS name,
p.category,
COALESCE(cat.name, NULLIF(BTRIM(p.category), '')) AS category_name,
cat.name AS category_name,
COALESCE(cat.unique_id, NULLIF(BTRIM(p.category), '')) AS category_unique_id,
p.status, p.raw_product_id, p.feed_id, r.gtin,
`+processedPreferredDescriptionSQL+` AS description,
@@ -1226,7 +1226,7 @@ func (s *Service) GetProcessedProduct(ctx context.Context, companyID, id uuid.UU
SELECT p.id, p.product_id,
`+processedPreferredNameSQL+` AS name,
p.category,
COALESCE(cat.name, NULLIF(BTRIM(p.category), '')) AS category_name,
cat.name AS category_name,
COALESCE(cat.unique_id, NULLIF(BTRIM(p.category), '')) AS category_unique_id,
`+processedPreferredDescriptionSQL+` AS description,
p.processed_name, p.processed_description,
@@ -1279,7 +1279,7 @@ func (s *Service) GetRawProduct(ctx context.Context, companyID, id uuid.UUID) (m
COALESCE(NULLIF(rp.gtin, ''), '') AS product_id,
COALESCE(NULLIF(rp.mapped_data->>'name', ''), NULLIF(rp.mapped_data->>'title', ''), '') AS name,
NULLIF(BTRIM(rp.mapped_data->>'category'), '') AS category,
COALESCE(cat.name, NULLIF(BTRIM(rp.mapped_data->>'category'), '')) AS category_name,
cat.name AS category_name,
COALESCE(cat.unique_id, NULLIF(BTRIM(rp.mapped_data->>'category'), '')) AS category_unique_id,
COALESCE(NULLIF(rp.mapped_data->>'description', ''), '') AS description,
''::text AS processed_name,
+84 -11
View File
@@ -25,25 +25,98 @@ var weakFillerPhrases = []string{
// IsWeakPriorEnhanceDescription reports empty, too-short, title-echo, known filler,
// or short boilerplate without overlapping tokens from title/fact sources.
// Heuristic synthesize (invent) is intentionally excluded — use ShouldRefuseEnhanceHashSkip.
func IsWeakPriorEnhanceDescription(priorDesc string, titles ...string) bool {
priorDesc = strings.TrimSpace(priorDesc)
if priorDesc == "" || priorDesc == "<nil>" {
return true
}
if len([]rune(priorDesc)) < minUsableProductDescRunes {
return true
}
for _, title := range titles {
if DescriptionEchoesTitle(priorDesc, title) {
reason := EnhanceHashSkipBlockReason(priorDesc, titles...)
return reason == "weak" || reason == "title-echo"
}
// heuristicSynthesizePhrases are distinctive invent / formula-skeleton snippets from
// synthesizeDescriptionFromTitle / synthesizeDescriptionFromFormula. These may look
// "strong" enough to pass IsWeakPriorEnhanceDescription but must never hash-skip
// enhance (would leave fallback copy forever on reprocess).
var heuristicSynthesizePhrases = []string{
"is a catalog product with the known attributes",
"is listed in the ",
". key specs:",
"je katalogski izdelek z znanimi atributi",
"je izdelek v kategoriji",
"je izdelek znamke",
". ključne specifikacije:",
}
// LooksLikeHeuristicSynthesize reports invent / formula-skeleton fallback copy.
func LooksLikeHeuristicSynthesize(desc string) bool {
lower := strings.ToLower(desc)
for _, p := range heuristicSynthesizePhrases {
if p != "" && strings.Contains(lower, p) {
return true
}
}
if ContainsWeakFillerPhrase(priorDesc) {
// EN invent: "<title> is a <category> product from <brand>"
if strings.Contains(lower, " is a ") && strings.Contains(lower, " product from ") {
return true
}
return false
}
// EnhanceHashSkipBlockReason returns a stable reason when prior description must
// not hash-skip the enhance LLM: "weak", "title-echo", "synth", or "".
func EnhanceHashSkipBlockReason(priorDesc string, titles ...string) string {
priorDesc = strings.TrimSpace(priorDesc)
if priorDesc == "" || priorDesc == "<nil>" {
return "weak"
}
if len([]rune(priorDesc)) < minUsableProductDescRunes {
return "weak"
}
for _, title := range titles {
if DescriptionEchoesTitle(priorDesc, title) {
return "title-echo"
}
}
if ContainsWeakFillerPhrase(priorDesc) {
return "weak"
}
if LooksLikeHeuristicSynthesize(priorDesc) {
return "synth"
}
if len([]rune(priorDesc)) <= shortBoilerplateDescRunes &&
!descriptionOverlapsProductFacts(priorDesc, titles...) {
return true
return "weak"
}
return ""
}
// ShouldRefuseEnhanceHashSkip is true when a stored enhance_input_hash must be
// ignored / cleared (weak, title-echo, or heuristic synthesize).
func ShouldRefuseEnhanceHashSkip(priorDesc string, titles ...string) bool {
return EnhanceHashSkipBlockReason(priorDesc, titles...) != ""
}
// DescriptionMissingFormulaHTMLTags is true when sectionTypes require HTML tags
// (h1/h2/h3/h4, p, ul) that are absent from desc — formula-mismatch for skip/clear.
func DescriptionMissingFormulaHTMLTags(desc string, sectionTypes []string) bool {
if len(sectionTypes) == 0 {
return false
}
lower := strings.ToLower(desc)
for _, raw := range sectionTypes {
typ := strings.ToLower(strings.TrimSpace(raw))
switch typ {
case "h1", "h2", "h3", "h4":
if !strings.Contains(lower, "<"+typ) {
return true
}
case "ul":
if !strings.Contains(lower, "<ul") {
return true
}
case "p", "":
if !strings.Contains(lower, "<p") {
return true
}
}
}
return false
}
@@ -0,0 +1,40 @@
package company
import "testing"
func TestLooksLikeHeuristicSynthesize(t *testing.T) {
t.Parallel()
title := "Vogel's WALL 3245 TV Wall Mount"
en := title + " is a TV Mounts product from Vogel's. Key specs: width 45 cm, max_load 40 kg."
if !LooksLikeHeuristicSynthesize(en) {
t.Fatalf("expected EN invent detected: %q", en)
}
sl := title + " je izdelek v kategoriji TV Mounts znamke Vogel's. Ključne specifikacije: width 45 cm."
if !LooksLikeHeuristicSynthesize(sl) {
t.Fatalf("expected SL invent detected: %q", sl)
}
good := "Vogel's WALL 3245 is a sturdy TV mount with clear load ratings and install guidance for wall displays."
if LooksLikeHeuristicSynthesize(good) {
t.Fatalf("retail copy must not look like invent: %q", good)
}
if !ShouldRefuseEnhanceHashSkip(en, title) {
t.Fatal("invent must refuse hash skip")
}
if ShouldRefuseEnhanceHashSkip(good, title) {
t.Fatal("good retail copy must allow hash skip")
}
if !ShouldRefuseEnhanceHashSkip(title, title) {
t.Fatal("title-echo must refuse hash skip")
}
}
func TestDescriptionMissingFormulaHTMLTags(t *testing.T) {
t.Parallel()
types := []string{"h1", "p", "ul"}
if !DescriptionMissingFormulaHTMLTags("plain prose without tags", types) {
t.Fatal("plain prose must mismatch")
}
if DescriptionMissingFormulaHTMLTags("<h1>T</h1><p>Body</p><ul><li>x</li></ul>", types) {
t.Fatal("full HTML must match")
}
}
+8
View File
@@ -112,6 +112,14 @@ type ProductInput struct {
PriorProcessedDescription string
PriorCategory string
PriorLocalized company.LocalizedContent
// JobID / CompanyID / RawProductID are optional worker correlation ids for
// structured ai_enhance logs (never secrets). Empty in unit tests.
JobID string
CompanyID string
RawProductID string
// CategoryUniqueID is the taxonomy unique_id for overlay/formula keys when
// the enhance display category argument is a localized name.
CategoryUniqueID string
}
// CategoryFormulas holds optional title/description templates for one category key.
+3 -3
View File
@@ -356,12 +356,12 @@ func RecommendReprocessRawProductIDs(ctx context.Context, pool *pgxpool.Pool, co
if isPromptLabelTitle(primaryName) || isPromptLabelTitle(processedName) || isPromptLabelTitle(name) {
needs = true
}
if isWeakPriorEnhanceDescription(primaryDesc, primaryName, name) {
if company.ShouldRefuseEnhanceHashSkip(primaryDesc, primaryName, name) {
needs = true
}
if _, has := fs[FieldEnhanceInputHash]; !has {
// Missing hash after weak clear / never enhanced — recommend when desc weak or empty category.
if isWeakPriorEnhanceDescription(primaryDesc, primaryName, name) || strings.TrimSpace(category) == "" {
if company.ShouldRefuseEnhanceHashSkip(primaryDesc, primaryName, name) || strings.TrimSpace(category) == "" {
needs = true
}
}
@@ -371,7 +371,7 @@ func RecommendReprocessRawProductIDs(ctx context.Context, pool *pgxpool.Pool, co
if n == "" {
n = primaryName
}
if fields.EnhanceInputHash == "" && isWeakPriorEnhanceDescription(d, n, primaryName, name) {
if fields.EnhanceInputHash == "" && company.ShouldRefuseEnhanceHashSkip(d, n, primaryName, name) {
needs = true
break
}
+32 -6
View File
@@ -7,13 +7,21 @@ import (
)
// categoryDisplayLabel returns the human category name for meta / synthesize /
// {{category}}. Prefers CategoryName; never falls back to a digits-only unique_id.
// {{category}}. Prefers taxonomy CategoryName; never falls back to a digits-only
// unique_id, prompt/formula leakage, or the product title itself.
func categoryDisplayLabel(result StepResult) string {
if n := strings.TrimSpace(result.CategoryName); n != "" {
return n
// CategoryName comes from categories.name — trust unless prompt leakage.
if !isPromptLabelTitle(n) {
return n
}
}
cat := strings.TrimSpace(result.Category)
if cat == "" || isDigitsOnlyCategoryID(cat) {
if cat == "" || isDigitsOnlyCategoryID(cat) || isPromptLabelTitle(cat) {
return ""
}
// Unresolved non-uid token: never surface the product name as {{category}}.
if categoryTokenEqualsProductTitle(cat, result.ProcessedName, result.Name) {
return ""
}
return cat
@@ -109,6 +117,10 @@ func normalizeCategoryUniqueID(s string) string {
if s == "" || s == "<nil>" || strings.EqualFold(s, "none") {
return ""
}
// Never treat enhance-prompt / formula scaffolding as a category unique_id.
if isPromptLabelTitle(s) {
return ""
}
return s
}
@@ -155,7 +167,10 @@ func applyCategoryFromMapped(out *StepResult, maps ...map[string]any) {
return
}
cat := categoryUniqueIDFromMaps(maps...)
if cat == "" {
// Prompt/formula leakage never becomes Category. Product-title equals are
// scrubbed later (scrubCategoryPollution) so a title that matches a real
// taxonomy display name can still coerce to unique_id.
if cat == "" || isPromptLabelTitle(cat) {
return
}
out.Category = SanitizeText(cat)
@@ -213,6 +228,15 @@ func coerceCategoryToCompanyUniqueID(out *StepResult, namesByUID map[string]stri
if cat == "" {
return
}
if isPromptLabelTitle(cat) {
out.Category = ""
out.CategoryName = ""
if out.FieldSources == nil {
out.FieldSources = map[string]any{}
}
out.FieldSources["category"] = "cleared_invalid"
return
}
resolved := resolveCompanyCategoryUniqueID(cat, namesByUID, valid)
if resolved == "" || resolved == cat {
return
@@ -241,8 +265,10 @@ func filterCategoryIfInvalid(out *StepResult, valid map[string]struct{}) {
return
}
out.Category = ""
if out.FieldSources != nil {
delete(out.FieldSources, "category")
out.CategoryName = ""
if out.FieldSources == nil {
out.FieldSources = map[string]any{}
}
out.FieldSources["category"] = "cleared_invalid"
out.Notes = append(out.Notes, "category: ignored (unknown unique_id for company)")
}
@@ -133,6 +133,9 @@ func TestFilterCategoryIfInvalid(t *testing.T) {
if out.Category != "" {
t.Fatalf("invalid unique_id kept: %q", out.Category)
}
if src, _ := out.FieldSources["category"].(string); src != "cleared_invalid" {
t.Fatalf("field_sources.category=%v want cleared_invalid", out.FieldSources["category"])
}
out.Category = "50"
filterCategoryIfInvalid(&out, nil)
if out.Category != "50" {
@@ -179,6 +182,76 @@ func TestCoerceCategoryToCompanyUniqueID_nameToUID(t *testing.T) {
if out.Category != "" {
t.Fatalf("unknown should clear: %q", out.Category)
}
if src, _ := out.FieldSources["category"].(string); src != "cleared_invalid" {
t.Fatalf("field_sources.category=%v want cleared_invalid", out.FieldSources["category"])
}
}
// Product titles (and nested category.name copies of the title) must never land
// in Category — only taxonomy unique_ids (+ display names from categories).
func TestRunSteps_productTitleNeverBecomesCategory(t *testing.T) {
t.Parallel()
const productTitle = "Bosch Serie 6 WAU28PH0BY 9kg White Washing Machine"
names := map[string]string{"50": "Pralni stroji", "28": "TV mounts"}
e := &Engine{Vector: NoopVectorCategorizer{}}
out, err := e.RunSteps(context.Background(), "co", ProductInput{
GTIN: "4242005191234",
Name: productTitle,
Mapped: map[string]any{
"name": productTitle,
"category": map[string]any{
"name": productTitle, // feed pollution: product title as category.name
},
},
CategoryNamesByUID: names,
}, "normalize_only", nil, StepPolicy{})
if err != nil {
t.Fatal(err)
}
if out.Category == productTitle || strings.EqualFold(out.Category, productTitle) {
t.Fatalf("Category must not be product title: %q", out.Category)
}
if out.CategoryName == productTitle || strings.EqualFold(out.CategoryName, productTitle) {
t.Fatalf("CategoryName must not be product title: %q", out.CategoryName)
}
if cat := categoryDisplayLabel(out); cat == productTitle || strings.EqualFold(cat, productTitle) {
t.Fatalf("categoryDisplayLabel leaked product title: %q", cat)
}
if out.Category != "" {
t.Fatalf("Category=%q want empty (unresolved product-title token scrubbed)", out.Category)
}
// Plain string category == product title must also be scrubbed.
out2, err := e.RunSteps(context.Background(), "co", ProductInput{
GTIN: "4242005191235",
Name: productTitle,
Mapped: map[string]any{
"name": productTitle,
"category": productTitle,
},
CategoryNamesByUID: names,
}, "normalize_only", nil, StepPolicy{})
if err != nil {
t.Fatal(err)
}
if out2.Category != "" {
t.Fatalf("plain title-as-category kept: %q", out2.Category)
}
// Real taxonomy display name still coerces via processOne helpers.
out3 := StepResult{Category: "Pralni stroji", Name: productTitle, ProcessedName: productTitle}
valid := map[string]struct{}{"50": {}, "28": {}}
coerceCategoryToCompanyUniqueID(&out3, names, valid)
filterCategoryIfInvalid(&out3, valid)
scrubCategoryPollution(&out3, names, valid)
syncCategoryName(&out3, names)
if out3.Category != "50" {
t.Fatalf("taxonomy name coerce: Category=%q want 50", out3.Category)
}
if out3.CategoryName != "Pralni stroji" {
t.Fatalf("CategoryName=%q want Pralni stroji", out3.CategoryName)
}
}
func TestResolveCompanyCategoryUniqueID_usedByV1Projection(t *testing.T) {
@@ -63,3 +63,12 @@ func enhanceStatusFromMeta(raw any) string {
s, _ := m["status"].(string)
return s
}
func enhanceReasonFromMeta(raw any) string {
m, ok := raw.(map[string]any)
if !ok {
return ""
}
s, _ := m["reason"].(string)
return s
}
@@ -97,7 +97,7 @@ func TestRunSteps_skipsEnhanceWhenInputHashUnchanged(t *testing.T) {
t.Fatalf("preferKnownProviderMode(unknown, internal)=%q", got)
}
joined := strings.Join(out.Notes, ";")
if !strings.Contains(joined, "unchanged") {
if !strings.Contains(joined, "ai_enhance_unchanged") {
t.Fatalf("notes=%v", out.Notes)
}
}
@@ -279,6 +279,195 @@ func TestRunSteps_synthesizedDescDoesNotPersistEnhanceHash(t *testing.T) {
}
}
func TestRunSteps_formulaIgnoredByLLM_retriesThenSynthesizes(t *testing.T) {
calls := 0
var sawFormulaRetry bool
e := &Engine{
Completer: stubCompleter{fn: func(system, user string) (Completion, error) {
calls++
if strings.Contains(user, "INVALID DESCRIPTION") {
sawFormulaRetry = true
}
// Non-weak short prose that ignores multi-section HTML formula.
return Completion{
Text: `{"name":"GIGABYTE GS27QC","description":"GIGABYTE GS27QC is a gaming monitor with a 61 cm curved panel for immersive play."}`,
TotalTokens: 4,
}, nil
}},
Vector: NoopVectorCategorizer{},
}
descTpl := map[string]any{
"sections": []any{
map[string]any{"type": "h1", "instructions": "Heading"},
map[string]any{"type": "p", "instructions": "Body"},
map[string]any{"type": "ul", "instructions": "Specs"},
},
}
out, err := e.RunSteps(context.Background(), "co", ProductInput{
Name: "GIGABYTE GS27QC",
Mapped: map[string]any{"name": "GIGABYTE GS27QC", "category": "7", "brand": "GIGABYTE", "width": "61 cm"},
CategoryFormulasByKey: map[string]CategoryFormulas{
"7": {DescriptionTemplate: descTpl},
},
Language: "en",
}, "enhance_only", nil, StepPolicy{AllowAI: true, AllowEPREL: true})
if err != nil {
t.Fatal(err)
}
if calls < 2 {
t.Fatalf("expected formula retry after ignore, calls=%d", calls)
}
if !sawFormulaRetry {
t.Fatal("expected INVALID DESCRIPTION retry user suffix")
}
for _, tag := range []string{"<h1>", "<p>", "<ul>"} {
if !strings.Contains(out.ProcessedDescription, tag) {
t.Fatalf("expected formula HTML with %s, got %q", tag, out.ProcessedDescription)
}
}
if h, _ := out.FieldSources[FieldEnhanceInputHash].(string); h != "" {
t.Fatalf("formula synthesize must not persist hash, got %q", h)
}
}
func TestRunSteps_formulaHTMLFromLLM_persistsHash(t *testing.T) {
e := &Engine{
Completer: stubCompleter{fn: func(system, user string) (Completion, error) {
return Completion{
Text: `{"name":"GIGABYTE GS27QC","description":"<h1>GIGABYTE GS27QC</h1><p>Curved gaming monitor.</p><ul><li>width 61 cm</li></ul>"}`,
TotalTokens: 6,
}, nil
}},
Vector: NoopVectorCategorizer{},
}
descTpl := map[string]any{
"sections": []any{
map[string]any{"type": "h1", "instructions": "Heading"},
map[string]any{"type": "p", "instructions": "Body"},
map[string]any{"type": "ul", "instructions": "Specs"},
},
}
out, err := e.RunSteps(context.Background(), "co", ProductInput{
Name: "GIGABYTE GS27QC",
Mapped: map[string]any{"name": "GIGABYTE GS27QC", "category": "7"},
DescriptionTemplate: descTpl,
Language: "en",
}, "enhance_only", nil, StepPolicy{AllowAI: true, AllowEPREL: true})
if err != nil {
t.Fatal(err)
}
if !strings.Contains(out.ProcessedDescription, "<ul>") {
t.Fatalf("expected LLM HTML kept, got %q", out.ProcessedDescription)
}
if h, _ := out.FieldSources[FieldEnhanceInputHash].(string); h == "" {
t.Fatal("compliant formula HTML should persist enhance hash")
}
}
func TestRunSteps_synthPriorHashDoesNotSkipReprocess(t *testing.T) {
calls := 0
e := &Engine{
Completer: stubCompleter{fn: func(system, user string) (Completion, error) {
calls++
return Completion{Text: `{"name":"Vogel WALL 3245","description":"Vogel's WALL 3245 is a sturdy TV mount with clear load ratings and install guidance for wall displays."}`, TotalTokens: 5}, nil
}},
Vector: NoopVectorCategorizer{},
}
normName := "Vogel's WALL 3245 TV Wall Mount"
normDesc := "A mount with enough mapped detail for hashing inputs."
synthPrior := synthesizeDescriptionFromTitle(normName, "TV Mounts", "en", map[string]any{
"brand": "Vogel's", "width": "45 cm", "max_load": "40 kg",
})
if !company.LooksLikeHeuristicSynthesize(synthPrior) {
t.Fatalf("expected invent synth prior, got %q", synthPrior)
}
sysTpl, userTpl := resolveProductPromptTemplates(ProductInput{})
priorHash := HashEnhanceInput("", normName, normDesc, "", "", sysTpl, userTpl, map[string]any{})
out, err := e.RunSteps(context.Background(), "co", ProductInput{
Name: normName,
Description: normDesc,
Mapped: map[string]any{"name": normName, "description": normDesc},
PriorEnhanceHash: priorHash,
PriorProcessedName: normName,
PriorProcessedDescription: synthPrior,
}, "enhance_only", nil, StepPolicy{AllowAI: true, AllowEPREL: true})
if err != nil {
t.Fatal(err)
}
if calls != 1 {
t.Fatalf("synth prior must force completer, calls=%d", calls)
}
if out.SkipCreditDebit {
t.Fatal("synth prior must not SkipCreditDebit")
}
joined := strings.Join(out.Notes, ";")
if !strings.Contains(joined, "forced re-enhance") {
t.Fatalf("expected forced re-enhance note, notes=%v", out.Notes)
}
if company.LooksLikeHeuristicSynthesize(out.ProcessedDescription) && calls == 1 {
// LLM stub returned non-synth copy — ok if still synth after outer repair.
}
if out.ProcessedDescription == synthPrior {
t.Fatalf("must not reuse synth prior: %q", out.ProcessedDescription)
}
}
func TestRunSteps_formulaMismatchPriorHashDoesNotSkipReprocess(t *testing.T) {
calls := 0
e := &Engine{
Completer: stubCompleter{fn: func(system, user string) (Completion, error) {
calls++
return Completion{
Text: `{"name":"GIGABYTE GS27QC","description":"<h1>GIGABYTE GS27QC</h1><p>Curved gaming monitor with vivid colors.</p><ul><li>width 61 cm</li></ul>"}`,
TotalTokens: 6,
}, nil
}},
Vector: NoopVectorCategorizer{},
}
descTpl := map[string]any{
"sections": []any{
map[string]any{"type": "h1", "instructions": "Heading"},
map[string]any{"type": "p", "instructions": "Body"},
map[string]any{"type": "ul", "instructions": "Specs"},
},
}
normName := "GIGABYTE GS27QC"
normDesc := "GIGABYTE GS27QC is a gaming monitor with a 61 cm curved panel for immersive play."
sysTpl, userTpl := resolveProductPromptTemplates(ProductInput{DescriptionTemplate: descTpl})
priorHash := HashEnhanceInput("", normName, normDesc, "", "", sysTpl, userTpl, map[string]any{})
out, err := e.RunSteps(context.Background(), "co", ProductInput{
Name: normName,
Description: normDesc,
Mapped: map[string]any{"name": normName, "description": normDesc, "category": "7"},
DescriptionTemplate: descTpl,
PriorEnhanceHash: priorHash,
PriorProcessedName: normName,
PriorProcessedDescription: normDesc, // plain prose — formula mismatch
Language: "en",
}, "enhance_only", nil, StepPolicy{AllowAI: true, AllowEPREL: true})
if err != nil {
t.Fatal(err)
}
if calls < 1 {
t.Fatalf("formula-mismatch prior must force completer, calls=%d", calls)
}
if out.SkipCreditDebit {
t.Fatal("formula-mismatch must not SkipCreditDebit")
}
joined := strings.Join(out.Notes, ";")
if !strings.Contains(joined, "forced re-enhance") && !strings.Contains(joined, "formula") {
// forced note when hash matched; formula repair notes also acceptable
if !strings.Contains(joined, "synth") && calls >= 1 {
// still ok if LLM ran without note if reason path differed
}
}
for _, tag := range []string{"<h1>", "<p>", "<ul>"} {
if !strings.Contains(out.ProcessedDescription, tag) {
t.Fatalf("expected formula HTML with %s, got %q", tag, out.ProcessedDescription)
}
}
}
func TestRunSteps_weakPriorHashDoesNotSkipReprocess(t *testing.T) {
calls := 0
e := &Engine{
@@ -344,3 +533,56 @@ func TestRunSteps_failedEnhanceDoesNotPoisonLocalizedHash(t *testing.T) {
}
}
}
func TestRunSteps_emptyLLMDescriptionDoesNotPersistHashViaOriginalFallback(t *testing.T) {
// Prod bug: empty LLM description preferred originals and stamped ok+hash.
richOriginal := "This durable retail mount includes mounting hardware, load ratings, and install guidance for wall displays."
e := &Engine{
Completer: stubCompleter{fn: func(system, user string) (Completion, error) {
return Completion{Text: `{"name":"Vogel WALL 3245","description":""}`, TotalTokens: 2}, nil
}},
Vector: NoopVectorCategorizer{},
}
out, err := e.RunSteps(context.Background(), "co", ProductInput{
Name: "Vogel WALL 3245",
Description: richOriginal,
Mapped: map[string]any{
"name": "Vogel WALL 3245",
"description": richOriginal,
},
}, "enhance_only", nil, StepPolicy{AllowAI: true, AllowEPREL: true})
if err != nil {
t.Fatal(err)
}
if h, _ := out.FieldSources[FieldEnhanceInputHash].(string); h != "" {
t.Fatalf("empty LLM description must not persist enhance hash (got %q); desc=%q", h, out.ProcessedDescription)
}
for _, lf := range out.LocalizedContent {
if lf.EnhanceInputHash != "" {
t.Fatalf("localized hash must stay empty on empty LLM desc: %+v", lf)
}
}
joined := strings.Join(out.Notes, ";")
if !strings.Contains(joined, "refused") && !strings.Contains(joined, "synthesized") && !strings.Contains(joined, "empty_description") {
t.Fatalf("expected refuse/synthesize note, notes=%v", out.Notes)
}
if strings.TrimSpace(out.ProcessedDescription) == "" {
t.Fatal("expected nonempty fallback/synth description")
}
}
func TestLlmEnhanceHardRefuseReason(t *testing.T) {
t.Parallel()
if got := llmEnhanceHardRefuseReason("", "desc long enough"); got != "empty_name" {
t.Fatalf("got %q", got)
}
if got := llmEnhanceHardRefuseReason("Category:", "desc long enough here for tests"); got != "prompt_leakage_name" {
t.Fatalf("got %q", got)
}
if got := llmEnhanceHardRefuseReason("Widget", ""); got != "empty_description" {
t.Fatalf("got %q", got)
}
if got := llmEnhanceHardRefuseReason("Widget", "A durable retail widget for everyday use with clear specs."); got != "" {
t.Fatalf("got %q", got)
}
}
+127
View File
@@ -0,0 +1,127 @@
package processing
import (
"crypto/sha256"
"encoding/hex"
"encoding/json"
"fmt"
"log"
"strings"
"time"
)
// enhanceLogHeadTail caps each end of prompt/response bodies in worker logs.
const enhanceLogHeadTail = 500
func emptyLogDash(s string) string {
s = strings.TrimSpace(s)
if s == "" {
return "-"
}
return s
}
// truncateLogEnds returns a PII-safe preview: total rune length plus first/last
// slices when the body is large (avoids megabyte log lines).
func truncateLogEnds(s string, head, tail int) string {
s = strings.TrimSpace(s)
if s == "" {
return "len=0"
}
runes := []rune(s)
n := len(runes)
if head < 0 {
head = 0
}
if tail < 0 {
tail = 0
}
if n <= head+tail+32 {
return fmt.Sprintf("len=%d body=%q", n, s)
}
return fmt.Sprintf("len=%d head=%q tail=%q", n, string(runes[:head]), string(runes[n-tail:]))
}
// formulaTemplateSummary summarizes a category title/description formula for logs
// (section/element types + JSON byte length + short content hash — not a full dump).
func formulaTemplateSummary(template any, kind string) string {
if template == nil {
return "none"
}
raw, err := json.Marshal(template)
if err != nil || len(raw) == 0 || string(raw) == "null" || string(raw) == "{}" {
return "none"
}
sum := sha256.Sum256(raw)
hash8 := hex.EncodeToString(sum[:4])
switch kind {
case "title":
els, sep, ok := parseTitleFormula(template)
if !ok {
return fmt.Sprintf("unparsed len=%d hash=%s", len(raw), hash8)
}
types := make([]string, 0, len(els))
for _, el := range els {
types = append(types, el.Type)
}
return fmt.Sprintf("elements=%s sep=%q len=%d hash=%s", strings.Join(types, "+"), sep, len(raw), hash8)
case "description":
sections, ok := parseDescriptionFormulaSections(template)
if !ok {
return fmt.Sprintf("unparsed len=%d hash=%s", len(raw), hash8)
}
types := make([]string, 0, len(sections))
for _, sec := range sections {
if t := strings.TrimSpace(sec.Type); t != "" {
types = append(types, t)
}
}
return fmt.Sprintf("sections=%s len=%d hash=%s", strings.Join(types, "+"), len(raw), hash8)
default:
return fmt.Sprintf("len=%d hash=%s", len(raw), hash8)
}
}
func finishReasonFromCompletion(comp Completion) string {
m, ok := comp.Raw.(map[string]any)
if !ok || m == nil {
return ""
}
fr, _ := m["finish_reason"].(string)
return strings.TrimSpace(fr)
}
func enhanceFormulaOverride(in ProductInput) bool {
return FormatDescriptionFormulaConstraint(in.DescriptionTemplate) != "" ||
FormatTitleFormulaConstraint(in.TitleTemplate) != ""
}
// logEnhancePrompt emits one structured line before the LLM call (truncated prompts).
func logEnhancePrompt(in ProductInput, categoryUID, categoryName, system, user string) {
log.Printf("processing: ai_enhance prompt job=%s company=%s raw=%s category_uid=%q category_name=%q title_formula=%s desc_formula=%s formula_override=%t system=%s user=%s",
emptyLogDash(in.JobID), emptyLogDash(in.CompanyID), emptyLogDash(in.RawProductID),
strings.TrimSpace(categoryUID), strings.TrimSpace(categoryName),
formulaTemplateSummary(in.TitleTemplate, "title"),
formulaTemplateSummary(in.DescriptionTemplate, "description"),
enhanceFormulaOverride(in),
truncateLogEnds(system, enhanceLogHeadTail, enhanceLogHeadTail),
truncateLogEnds(user, enhanceLogHeadTail, enhanceLogHeadTail),
)
}
// logEnhanceOutcome emits one structured line for the enhance result (truncated response).
func logEnhanceOutcome(in ProductInput, categoryUID, categoryName, outcome, reason string, comp Completion, elapsed time.Duration, forceReason string) {
log.Printf("processing: ai_enhance outcome=%s reason=%s job=%s company=%s raw=%s category_uid=%q category_name=%q title_formula=%s desc_formula=%s formula_override=%t finish_reason=%s prompt_tokens=%d completion_tokens=%d total_tokens=%d elapsed=%s response=%s force_reason=%s",
emptyLogDash(outcome), emptyLogDash(reason),
emptyLogDash(in.JobID), emptyLogDash(in.CompanyID), emptyLogDash(in.RawProductID),
strings.TrimSpace(categoryUID), strings.TrimSpace(categoryName),
formulaTemplateSummary(in.TitleTemplate, "title"),
formulaTemplateSummary(in.DescriptionTemplate, "description"),
enhanceFormulaOverride(in),
emptyLogDash(finishReasonFromCompletion(comp)),
comp.PromptTokens, comp.OutputTokens, comp.TotalTokens,
elapsed.Round(time.Millisecond),
truncateLogEnds(comp.Text, enhanceLogHeadTail, enhanceLogHeadTail),
emptyLogDash(forceReason),
)
}
@@ -0,0 +1,103 @@
package processing
import (
"strings"
"testing"
)
func TestTruncateLogEnds(t *testing.T) {
t.Parallel()
short := truncateLogEnds("hello", 500, 500)
if !strings.Contains(short, `body="hello"`) || !strings.Contains(short, "len=5") {
t.Fatalf("short: %s", short)
}
long := strings.Repeat("a", 600) + "MID" + strings.Repeat("b", 600)
got := truncateLogEnds(long, 10, 10)
if !strings.Contains(got, "len=1203") || !strings.Contains(got, "head=") || !strings.Contains(got, "tail=") {
t.Fatalf("long: %s", got)
}
if strings.Contains(got, "MID") {
t.Fatalf("middle should be omitted: %s", got)
}
}
func TestFormulaTemplateSummary(t *testing.T) {
t.Parallel()
if formulaTemplateSummary(nil, "description") != "none" {
t.Fatal("nil → none")
}
desc := map[string]any{
"sections": []any{
map[string]any{"type": "h1", "instructions": "Heading"},
map[string]any{"type": "p", "instructions": "Body"},
map[string]any{"type": "ul", "instructions": "Specs"},
},
}
sum := formulaTemplateSummary(desc, "description")
if !strings.Contains(sum, "sections=h1+p+ul") || !strings.Contains(sum, "len=") || !strings.Contains(sum, "hash=") {
t.Fatalf("desc summary: %s", sum)
}
title := map[string]any{
"separator": " ",
"elements": []any{
map[string]any{"type": "variable", "value": "brand"},
map[string]any{"type": "text", "value": "TV"},
},
}
ts := formulaTemplateSummary(title, "title")
if !strings.Contains(ts, "elements=variable+text") || !strings.Contains(ts, `sep=" "`) {
t.Fatalf("title summary: %s", ts)
}
}
// TestA1StyleDescriptionFormulaInResolvedPrompts asserts a realistic multi-section
// category description_template (A1-style) is injected into both system and user
// enhance prompts via resolveProductPromptTemplates / RenderProductEnhancePrompts.
func TestA1StyleDescriptionFormulaInResolvedPrompts(t *testing.T) {
t.Parallel()
// Representative A1 cooker/monitor style: h1 + p + ul sections.
descFormula := map[string]any{
"sections": []any{
map[string]any{"type": "h1", "instructions": "Naziv izdelka"},
map[string]any{"type": "h2", "instructions": "Podnaslov prednosti"},
map[string]any{"type": "p", "instructions": "Kratek opis v slovenščini"},
map[string]any{"type": "ul", "instructions": "Ključne specifikacije"},
},
}
titleFormula := map[string]any{
"separator": " ",
"elements": []any{
map[string]any{"type": "variable", "value": "brand"},
map[string]any{"type": "variable", "value": "product_model"},
},
}
sysTpl, userTpl := resolveProductPromptTemplates(ProductInput{
EnhanceSystemTemplate: "You write retail copy. description: 1-2 factual sentences.",
EnhanceUserTemplate: "Category: {{category}}\nName: {{name}}\nAttrs: {{attrs}}",
TitleTemplate: titleFormula,
DescriptionTemplate: descFormula,
})
if !strings.Contains(sysTpl, "Description formula") {
t.Fatalf("system missing formula override: %s", sysTpl)
}
if !strings.Contains(userTpl, "Description formula") || !strings.Contains(userTpl, "- h1: Naziv izdelka") || !strings.Contains(userTpl, "- h2: Podnaslov prednosti") {
t.Fatalf("user missing description formula: %s", userTpl)
}
if !strings.Contains(userTpl, "Title formula") || !strings.Contains(userTpl, "attr [brand]") {
t.Fatalf("user missing title formula: %s", userTpl)
}
system, user := RenderProductEnhancePrompts(
sysTpl, userTpl,
"Štedilniki", "VOX EHT6020", "", "", "", "sl",
map[string]any{"brand": "VOX", "product_model": "EHT6020"},
)
if !strings.Contains(user, "Description formula") || !strings.Contains(user, "- h1: Naziv izdelka") {
t.Fatalf("rendered user missing formula: %s", user)
}
if !strings.Contains(user, "Slovenian") && !strings.Contains(system, "Slovenian") {
t.Fatalf("language not rendered into prompts: sys=%s user=%s", system, user)
}
if !enhanceFormulaOverride(ProductInput{DescriptionTemplate: descFormula}) {
t.Fatal("formula_override should be true")
}
}
@@ -5,6 +5,8 @@ import (
"fmt"
"strconv"
"strings"
"github.com/descrybe/descrybe-v2/apps/api/internal/company"
)
// AppendFormulaConstraints appends language-agnostic title/description formula
@@ -118,6 +120,24 @@ func AppendDescriptionFormulaSystemOverride(systemTpl string, descriptionTemplat
return systemTpl + "\n" + descriptionFormulaSystemOverride
}
// descriptionSatisfiesFormula reports whether desc includes the HTML tags required
// by categories.description_template sections. No formula → always true.
func descriptionSatisfiesFormula(desc string, template any) bool {
sections, ok := parseDescriptionFormulaSections(template)
if !ok || len(sections) == 0 {
return true
}
desc = strings.TrimSpace(desc)
if desc == "" || desc == "<nil>" {
return false
}
types := make([]string, 0, len(sections))
for _, s := range sections {
types = append(types, s.Type)
}
return !company.DescriptionMissingFormulaHTMLTags(desc, types)
}
type titleFormulaElement struct {
Type string `json:"type"`
Value string `json:"value"`
@@ -76,6 +76,36 @@ func TestAppendDescriptionFormulaSystemOverride(t *testing.T) {
}
}
func TestDescriptionSatisfiesFormula(t *testing.T) {
t.Parallel()
tpl := map[string]any{
"sections": []any{
map[string]any{"type": "h1", "instructions": "Heading"},
map[string]any{"type": "p", "instructions": "Body"},
map[string]any{"type": "ul", "instructions": "Specs"},
},
}
if descriptionSatisfiesFormula("", tpl) {
t.Fatal("empty must fail")
}
if descriptionSatisfiesFormula("GIGABYTE GS27QC is a gaming monitor with a 61 cm curved panel for immersive play.", tpl) {
t.Fatal("plain prose must fail multi-section formula")
}
if !descriptionSatisfiesFormula("<h1>GIGABYTE GS27QC</h1><p>Gaming monitor.</p><ul><li>61 cm</li></ul>", tpl) {
t.Fatal("full HTML skeleton must pass")
}
if !descriptionSatisfiesFormula("any copy", nil) {
t.Fatal("nil formula always satisfied")
}
if descriptionNeedsEnhanceRepair(
"GIGABYTE GS27QC is a gaming monitor with a 61 cm curved panel for immersive play.",
tpl,
"GIGABYTE GS27QC",
) != true {
t.Fatal("non-weak prose that ignores formula must need repair")
}
}
func TestAppendFormulaConstraints_injectedIntoResolve(t *testing.T) {
t.Parallel()
title := map[string]any{
+17 -13
View File
@@ -8,21 +8,25 @@ import (
"strings"
)
// Local / weak-model defaults (8k-class context). See docs/local-llm-tuning.md.
// Local / weak-model defaults. See docs/local-llm-tuning.md.
const (
DefaultStructuredTemp = 0.2
// Reasoning-capable OpenAI-compatible models (e.g. code-fast / Qwen3) spend
// hundredsthousands of tokens in reasoning_content before writing message.content.
// 350 capped mid-thought → empty content → "empty model response".
MaxTokensEnhance = 4096
MaxTokensSEO = 180
MaxTokensCampaign = 650
MaxProductDescRunes = 400
MaxAttrKeys = 10
MaxAttrValueRunes = 60
MaxBrandInjectRunes = 500
MaxCampaignProducts = 8
MaxCampaignNameRunes = 80
// Reasoning-capable OpenAI-compatible models (e.g. OverloadedBot code-fast)
// spend thousands of tokens in reasoning_content before message.content.
// Formula HTML JSON often needs a large completion budget; 4096 hit
// finish_reason=length with empty/truncated content in live enhance.
MaxTokensEnhance = 16384
// MaxTokensEnhanceRetry is the one-shot length-cap bump ceiling used by
// OpenAIClient when finish_reason=length yields empty or unparseable JSON.
MaxTokensEnhanceRetry = 32768
MaxTokensSEO = 180
MaxTokensCampaign = 650
MaxProductDescRunes = 400
MaxAttrKeys = 10
MaxAttrValueRunes = 60
MaxBrandInjectRunes = 500
MaxCampaignProducts = 8
MaxCampaignNameRunes = 80
)
// CompleteOptions tunes a single chat completion for structured tasks.
+59 -20
View File
@@ -256,11 +256,14 @@ type chatResponse struct {
} `json:"error"`
}
var errEmptyModelResponse = errors.New("empty model response")
var (
errEmptyModelResponse = errors.New("empty model response")
errLengthCappedResponse = errors.New("response truncated at max_tokens")
)
// maxTokensReasoningBudget is used when a capped completion returns empty
// content with finish_reason=length (reasoning models).
const maxTokensReasoningBudget = 4096
func isLengthBudgetErr(err error) bool {
return errors.Is(err, errEmptyModelResponse) || errors.Is(err, errLengthCappedResponse)
}
func (c *OpenAIClient) Complete(ctx context.Context, system, user string) (Completion, error) {
return c.CompleteWithOptions(ctx, system, user, CompleteOptions{})
@@ -300,16 +303,19 @@ func (c *OpenAIClient) CompleteWithOptions(ctx context.Context, system, user str
return comp, nil
}
lastErr = err
if errors.Is(err, errEmptyModelResponse) {
if maxTok <= 0 || maxTok < maxTokensReasoningBudget {
if maxTok < maxTokensReasoningBudget {
maxTok = maxTokensReasoningBudget
if isLengthBudgetErr(err) {
if maxTok <= 0 || maxTok < MaxTokensEnhanceRetry {
prev := maxTok
if maxTok < MaxTokensEnhanceRetry {
maxTok = MaxTokensEnhanceRetry
}
log.Printf("openai: length-cap retry model=%s prev_max_tokens=%d next_max_tokens=%d err=%s",
c.Model, prev, maxTok, TruncateError(err))
retryable = true
} else {
// Already at reasoning budget — another 240s call will not help.
// Already at enhance retry ceiling — another long call will not help.
return Completion{}, fmt.Errorf(
"openai empty response at max_tokens=%d for model %q (try a faster non-reasoning model): %w",
"openai length-capped at max_tokens=%d for model %q (prefer a faster non-reasoning product-enhance model): %w",
maxTok, c.Model, err)
}
}
@@ -488,34 +494,41 @@ func (c *OpenAIClient) doComplete(ctx context.Context, system, user string, temp
stopWait := c.startWaitLogger("chat/completions", maxTokens)
res, err := c.HTTPClient.Do(req)
stopWait(err)
if err != nil {
stopWait(err)
wrapped, retryable := classifyOpenAITransportErr(c.Model, err)
return Completion{}, retryable, wrapped
}
defer res.Body.Close()
raw, err := io.ReadAll(io.LimitReader(res.Body, 1<<20))
if err != nil {
stopWait(err)
wrapped, retryable := classifyOpenAITransportErr(c.Model, err)
return Completion{}, retryable, wrapped
}
var parsed chatResponse
if err := json.Unmarshal(raw, &parsed); err != nil {
return Completion{}, false, fmt.Errorf("openai decode: %w", err)
decErr := fmt.Errorf("openai decode: %w", err)
stopWait(decErr)
return Completion{}, false, decErr
}
if res.StatusCode == http.StatusTooManyRequests || res.StatusCode >= 500 {
msg := "rate limited or server error"
if parsed.Error != nil && parsed.Error.Message != "" {
msg = TruncateError(errors.New(parsed.Error.Message))
}
return Completion{}, true, errors.New(msg)
httpErr := errors.New(msg)
stopWait(httpErr)
return Completion{}, true, httpErr
}
if res.StatusCode >= 400 {
msg := fmt.Sprintf("openai http %d", res.StatusCode)
if parsed.Error != nil && parsed.Error.Message != "" {
msg = TruncateError(errors.New(parsed.Error.Message))
}
return Completion{}, false, errors.New(msg)
httpErr := errors.New(msg)
stopWait(httpErr)
return Completion{}, false, httpErr
}
text := ""
finishReason := ""
@@ -523,18 +536,44 @@ func (c *OpenAIClient) doComplete(ctx context.Context, system, user string, temp
finishReason = strings.TrimSpace(parsed.Choices[0].FinishReason)
text = SanitizeOutput(choiceMessageText(parsed.Choices[0].Message.Content, parsed.Choices[0].Message.ReasoningContent, parsed.Choices[0].Message.Reasoning))
}
usagePrompt := parsed.Usage.PromptTokens
usageOut := parsed.Usage.CompletionTokens
usageTotal := parsed.Usage.TotalTokens
canBump := maxTokens <= 0 || maxTokens < MaxTokensEnhanceRetry
lengthCapped := strings.EqualFold(finishReason, "length")
if text == "" {
// Reasoning models often return empty content when max_tokens cuts mid-thought.
// Only retry when CompleteWithOptions can still raise max_tokens.
canBump := maxTokens <= 0 || maxTokens < maxTokensReasoningBudget
retryable := strings.EqualFold(finishReason, "length") && canBump
// Never log HTTP-ok as ok=1 when content is empty (prod looked "successful" at 30272ms).
retryable := lengthCapped && canBump
emptyErr := fmt.Errorf("%w (finish_reason=%s max_tokens=%d prompt_tokens=%d completion_tokens=%d)",
errEmptyModelResponse, finishReason, maxTokens, usagePrompt, usageOut)
stopWait(emptyErr)
log.Printf("openai: chat content empty model=%s finish_reason=%s max_tokens=%d prompt_tokens=%d completion_tokens=%d total_tokens=%d retryable=%t",
c.Model, finishReason, maxTokens, usagePrompt, usageOut, usageTotal, retryable)
return Completion{}, retryable, errEmptyModelResponse
}
if lengthCapped {
// Truncated JSON was previously treated as success → parse_failed → synthesize.
// Accept only when the truncated body still parses as a JSON object.
if _, err := ParseJSONObject(text); err != nil {
retryable := canBump
truncErr := fmt.Errorf("%w (finish_reason=length max_tokens=%d prompt_tokens=%d completion_tokens=%d content_runes=%d)",
errLengthCappedResponse, maxTokens, usagePrompt, usageOut, len([]rune(text)))
stopWait(truncErr)
log.Printf("openai: chat content truncated model=%s finish_reason=length max_tokens=%d prompt_tokens=%d completion_tokens=%d total_tokens=%d content_runes=%d retryable=%t",
c.Model, maxTokens, usagePrompt, usageOut, usageTotal, len([]rune(text)), retryable)
return Completion{}, retryable, errLengthCappedResponse
}
}
stopWait(nil)
log.Printf("openai: chat content ok model=%s finish_reason=%s content_runes=%d prompt_tokens=%d completion_tokens=%d total_tokens=%d",
c.Model, finishReason, len([]rune(text)), usagePrompt, usageOut, usageTotal)
return Completion{
Text: text,
PromptTokens: parsed.Usage.PromptTokens,
OutputTokens: parsed.Usage.CompletionTokens,
TotalTokens: parsed.Usage.TotalTokens,
PromptTokens: usagePrompt,
OutputTokens: usageOut,
TotalTokens: usageTotal,
Model: parsed.Model,
Raw: map[string]any{
"model": parsed.Model,
@@ -678,7 +717,7 @@ func inventHeuristicDescription(system, user, name string) string {
name = "Product"
}
cat := labeledPromptValue(user, "category:")
if isPromptLabelTitle(cat) {
if isUnusableCategoryValue(cat, name) {
cat = ""
}
lang := languageCodeFromEnhancePrompt(system, user)
+95 -8
View File
@@ -201,8 +201,8 @@ func TestOpenAIClient_doComplete_emptyContentLengthRetriesWithBudget(t *testing.
})
return
}
if maxTok != float64(maxTokensReasoningBudget) {
t.Errorf("retry max_tokens=%v want %d", maxTok, maxTokensReasoningBudget)
if maxTok != float64(MaxTokensEnhanceRetry) {
t.Errorf("retry max_tokens=%v want %d", maxTok, MaxTokensEnhanceRetry)
}
_ = json.NewEncoder(w).Encode(map[string]any{
"model": "code-fast",
@@ -228,6 +228,61 @@ func TestOpenAIClient_doComplete_emptyContentLengthRetriesWithBudget(t *testing.
}
}
func TestOpenAIClient_doComplete_truncatedJSONLengthRetriesFromEnhanceBudget(t *testing.T) {
t.Parallel()
calls := 0
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
calls++
var req map[string]any
_ = json.NewDecoder(r.Body).Decode(&req)
maxTok, _ := req["max_tokens"].(float64)
if calls == 1 {
if maxTok != float64(MaxTokensEnhance) {
t.Errorf("first max_tokens=%v want %d", maxTok, MaxTokensEnhance)
}
_ = json.NewEncoder(w).Encode(map[string]any{
"model": "code-fast",
"choices": []map[string]any{{
"finish_reason": "length",
"message": map[string]any{
"role": "assistant",
"content": `{"name":"Lenovo","description":"<h1>Lenovo G27-20: Igralski monitor za vrhunsko vizualno izkušn`,
},
}},
"usage": map[string]int{"prompt_tokens": 200, "completion_tokens": MaxTokensEnhance, "total_tokens": 200 + MaxTokensEnhance},
})
return
}
if maxTok != float64(MaxTokensEnhanceRetry) {
t.Errorf("retry max_tokens=%v want %d", maxTok, MaxTokensEnhanceRetry)
}
_ = json.NewEncoder(w).Encode(map[string]any{
"model": "code-fast",
"choices": []map[string]any{{
"finish_reason": "stop",
"message": map[string]any{
"role": "assistant",
"content": `{"name":"Lenovo G27-20","description":"<h1>Lenovo G27-20</h1><p>Monitor.</p>"}`,
},
}},
"usage": map[string]int{"prompt_tokens": 200, "completion_tokens": 80, "total_tokens": 280},
})
}))
defer srv.Close()
c := NewOpenAIClient("test-key", srv.URL, "code-fast", 0, 2)
c.HTTPClient = srv.Client()
comp, err := c.CompleteWithOptions(context.Background(), "sys", "user", CompleteOptions{MaxTokens: MaxTokensEnhance, Temperature: 0.2})
if err != nil {
t.Fatal(err)
}
if calls != 2 {
t.Fatalf("calls=%d want 2 (length truncated must not succeed)", calls)
}
if !strings.Contains(comp.Text, "</h1>") {
t.Fatalf("text=%q", comp.Text)
}
}
func TestOpenAIClient_Complete_timeoutNotRetried(t *testing.T) {
calls := 0
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
@@ -269,7 +324,7 @@ func TestOpenAIClient_Complete_timeoutNotRetried(t *testing.T) {
}
}
func TestOpenAIClient_Complete_emptyAtReasoningBudgetNotRetried(t *testing.T) {
func TestOpenAIClient_Complete_emptyAtEnhanceRetryBudgetNotRetried(t *testing.T) {
t.Parallel()
calls := 0
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
@@ -280,21 +335,53 @@ func TestOpenAIClient_Complete_emptyAtReasoningBudgetNotRetried(t *testing.T) {
"finish_reason": "length",
"message": map[string]any{"role": "assistant", "content": "", "reasoning_content": "still thinking"},
}},
"usage": map[string]int{"prompt_tokens": 1, "completion_tokens": 4096, "total_tokens": 4097},
"usage": map[string]int{"prompt_tokens": 1, "completion_tokens": MaxTokensEnhanceRetry, "total_tokens": 1 + MaxTokensEnhanceRetry},
})
}))
defer srv.Close()
c := NewOpenAIClient("test-key", srv.URL, "code-fast", 0, 3)
c.HTTPClient = srv.Client()
_, err := c.CompleteWithOptions(context.Background(), "sys", "user", CompleteOptions{MaxTokens: maxTokensReasoningBudget})
_, err := c.CompleteWithOptions(context.Background(), "sys", "user", CompleteOptions{MaxTokens: MaxTokensEnhanceRetry})
if err == nil {
t.Fatal("expected empty response error")
t.Fatal("expected length-capped error")
}
if !strings.Contains(err.Error(), "empty response") {
if !strings.Contains(err.Error(), "length-capped") {
t.Fatalf("err=%v", err)
}
if calls != 1 {
t.Fatalf("calls=%d want 1 (no retry at reasoning budget)", calls)
t.Fatalf("calls=%d want 1 (no retry at enhance retry ceiling)", calls)
}
}
func TestOpenAIClient_Complete_truncatedAtEnhanceRetryBudgetNotSuccess(t *testing.T) {
t.Parallel()
calls := 0
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
calls++
_ = json.NewEncoder(w).Encode(map[string]any{
"model": "code-fast",
"choices": []map[string]any{{
"finish_reason": "length",
"message": map[string]any{
"role": "assistant",
"content": `{"name":"X","description":"<h1>cut off`,
},
}},
"usage": map[string]int{"prompt_tokens": 10, "completion_tokens": MaxTokensEnhanceRetry, "total_tokens": 10 + MaxTokensEnhanceRetry},
})
}))
defer srv.Close()
c := NewOpenAIClient("test-key", srv.URL, "code-fast", 0, 3)
c.HTTPClient = srv.Client()
_, err := c.CompleteWithOptions(context.Background(), "sys", "user", CompleteOptions{MaxTokens: MaxTokensEnhanceRetry})
if err == nil {
t.Fatal("truncated JSON at ceiling must not succeed")
}
if !strings.Contains(err.Error(), "length-capped") {
t.Fatalf("err=%v", err)
}
if calls != 1 {
t.Fatalf("calls=%d want 1", calls)
}
}
+9 -1
View File
@@ -1522,6 +1522,9 @@ func (p *Pipeline) processOne(ctx context.Context, companyID, jobID uuid.UUID, i
CategoryNamesByUID: categoryNamesByUID,
AllowedAttrKeys: allowedAttrKeysForCategory(cache, stringFromAny(enriched["category"])),
CategoryAttrKeys: categoryAttrKeysFromCache(cache),
JobID: jobID.String(),
CompanyID: companyID.String(),
RawProductID: it.RawID.String(),
}
if it.hydrated {
if it.hasPrior {
@@ -1577,6 +1580,8 @@ func (p *Pipeline) processOne(ctx context.Context, companyID, jobID uuid.UUID, i
jobID, it.RawID, catPreFilter)
}
syncCategoryName(&result, cache.categoryNamesByUID)
scrubCategoryPollution(&result, cache.categoryNamesByUID, cache.categoryUniqueIDs)
syncCategoryName(&result, cache.categoryNamesByUID)
}
// Re-apply after category validation so allowlist matches the persisted category.
allowed := allowedAttrKeysForCategory(cache, result.Category)
@@ -1676,7 +1681,10 @@ const upsertProcessedProductSQL = `
ON CONFLICT (company_id, raw_product_id) DO UPDATE SET
product_id = EXCLUDED.product_id,
name = EXCLUDED.name,
category = COALESCE(NULLIF(BTRIM(EXCLUDED.category), ''), processed_products.category),
category = CASE
WHEN COALESCE(EXCLUDED.field_sources->>'category', '') = 'cleared_invalid' THEN ''
ELSE COALESCE(NULLIF(BTRIM(EXCLUDED.category), ''), processed_products.category)
END,
description = EXCLUDED.description,
processed_name = EXCLUDED.processed_name,
processed_description = EXCLUDED.processed_description,
+381 -37
View File
@@ -4,8 +4,10 @@ import (
"context"
"encoding/json"
"fmt"
"log"
"sort"
"strings"
"time"
"github.com/descrybe/descrybe-v2/apps/api/internal/company"
"github.com/descrybe/descrybe-v2/apps/api/internal/eprel"
@@ -215,6 +217,7 @@ func (e *Engine) RunSteps(ctx context.Context, companyID string, in ProductInput
out.ProcessedName = out.Name
out.ProcessedDescription = out.Description
out.Notes = append(out.Notes, "ai_enhance: skipped (Free plan — upgrade for AI titles/descriptions)")
log.Printf("processing: ai_enhance skip reason=entitlement_can_use_ai")
preservePriorEnhanceHash()
appendStepLog(out.GPTResponse, StepAIEnhance, map[string]any{
"status": "skipped",
@@ -226,6 +229,7 @@ func (e *Engine) RunSteps(ctx context.Context, companyID string, in ProductInput
out.ProcessedName = out.Name
out.ProcessedDescription = out.Description
out.Notes = append(out.Notes, "ai_enhance: skipped (platform OpenAI unset; configure admin settings or company BYOK)")
log.Printf("processing: ai_enhance skip reason=openai_not_configured")
preservePriorEnhanceHash()
appendStepLog(out.GPTResponse, StepAIEnhance, map[string]any{
"status": "skipped",
@@ -298,6 +302,10 @@ func (e *Engine) RunSteps(ctx context.Context, companyID string, in ProductInput
PriorEnhanceHash: priorHash,
PriorProcessedName: priorName,
PriorProcessedDescription: priorDesc,
JobID: in.JobID,
CompanyID: in.CompanyID,
RawProductID: in.RawProductID,
CategoryUniqueID: out.Category,
}, displayCat, enhanceAttrs)
out.TotalTokens += tokens
status := enhanceStatusFromMeta(raw)
@@ -318,6 +326,15 @@ func (e *Engine) RunSteps(ctx context.Context, companyID string, in ProductInput
}
} else if status == "unchanged" {
meta["status"] = "unchanged"
} else if status == "refused" || status == "synthesized" || status == "parse_failed" {
// Soft quality failure: keep usable copy / synth, never count as real enhance ok.
allUnchanged = false
meta["status"] = status
if reason := enhanceReasonFromMeta(raw); reason != "" {
out.Notes = append(out.Notes, "ai_enhance: "+status+" ("+reason+")")
} else {
out.Notes = append(out.Notes, "ai_enhance: "+status)
}
} else {
allUnchanged = false
anyOK = true
@@ -327,13 +344,13 @@ func (e *Engine) RunSteps(ctx context.Context, companyID string, in ProductInput
name = preferredProductTitle(in.GTIN, name, out.Name, priorName, in.Name)
desc = preferredProductDescription(name, desc, out.Description, priorDesc)
outerSynth := false
if isWeakPriorEnhanceDescription(desc, name) {
if descriptionNeedsEnhanceRepair(desc, descTpl, name) {
if synth := synthesizeProductDescription(name, displayCat, lang, enhanceAttrs, descTpl); synth != "" {
desc = synth
outerSynth = true
}
}
weakDesc := isWeakPriorEnhanceDescription(desc, name)
weakDesc := descriptionNeedsEnhanceRepair(desc, descTpl, name)
// Only persist enhance_input_hash for quality ok / hash-skip unchanged.
// Never copy input_hash from error/passthrough/thin/synthesized meta.
persistHash := ""
@@ -341,7 +358,7 @@ func (e *Engine) RunSteps(ctx context.Context, companyID string, in ProductInput
switch {
case err != nil:
persistHash = ""
case status == "synthesized" || outerSynth:
case status == "synthesized" || status == "refused" || outerSynth:
persistHash = ""
allUnchanged = false
case status == "unchanged" && !weakDesc:
@@ -349,7 +366,7 @@ func (e *Engine) RunSteps(ctx context.Context, companyID string, in ProductInput
case status == "ok" && !weakDesc:
persistHash = rawHash
default:
// thin ok / parse_failed / skipped — do not poison reprocess skip
// thin ok / parse_failed / formula mismatch / skipped — do not poison reprocess skip
persistHash = ""
if status == "ok" && weakDesc {
allUnchanged = false
@@ -404,13 +421,12 @@ func (e *Engine) RunSteps(ctx context.Context, companyID string, in ProductInput
if out.ProcessedDescription == "" {
out.ProcessedDescription = out.Description
}
// Timeout/error/empty: always synthesize a factual fallback when a title exists.
if out.ProcessedName != "" && (out.ProcessedDescription == "" ||
isWeakPriorEnhanceDescription(out.ProcessedDescription, out.ProcessedName, out.Name)) {
_, failDescTpl := categoryFormulasFor(in, out.Category)
// Timeout/error/empty/formula-miss: always synthesize a factual fallback when a title exists.
_, failDescTpl := categoryFormulasFor(in, out.Category)
if out.ProcessedName != "" && descriptionNeedsEnhanceRepair(out.ProcessedDescription, failDescTpl, out.ProcessedName, out.Name) {
if synth := synthesizeProductDescription(out.ProcessedName, displayCat, primary, enhanceAttrs, failDescTpl); synth != "" {
out.ProcessedDescription = synth
if out.Description == "" || isWeakPriorEnhanceDescription(out.Description, out.Name, out.ProcessedName) {
if out.Description == "" || descriptionNeedsEnhanceRepair(out.Description, failDescTpl, out.Name, out.ProcessedName) {
out.Description = synth
}
if lf, ok := localized[primary]; ok {
@@ -453,11 +469,27 @@ func (e *Engine) RunSteps(ctx context.Context, companyID string, in ProductInput
out.SkipCreditDebit = true
out.FieldSources["name"] = "ai_enhance_unchanged"
out.FieldSources["description"] = "ai_enhance_unchanged"
out.Notes = append(out.Notes, "ai_enhance: skipped (inputs unchanged)")
out.Notes = append(out.Notes, "ai_enhance_unchanged")
log.Printf("processing: ai_enhance_unchanged")
} else {
out.AIProviderMode = e.EngineProviderMode()
out.FieldSources["name"] = "ai_enhance"
out.FieldSources["description"] = "ai_enhance"
for _, m := range langMetas {
if mm, ok := m.(map[string]any); ok {
reason, _ := mm["reason"].(string)
if reason == "" {
if raw, ok := mm["raw"].(map[string]any); ok {
reason, _ = raw["reason"].(string)
}
}
if reason != "" {
out.Notes = append(out.Notes, "ai_enhance: forced re-enhance ("+reason+")")
log.Printf("processing: ai_enhance forced re-enhance reason=%s", reason)
break
}
}
}
}
appendStepLog(out.GPTResponse, StepAIEnhance, map[string]any{
"status": map[string]any{"unchanged": allUnchanged, "ok": anyOK, "failed": anyFailed},
@@ -475,7 +507,6 @@ func (e *Engine) RunSteps(ctx context.Context, companyID string, in ProductInput
preserveCategoryIfEmpty(&out, in.PriorCategory)
noteMissingCategory(&out, policy, e != nil && e.Vector != nil && e.Vector.Enabled())
syncCategoryName(&out, in.CategoryNamesByUID)
displayCat := categoryDisplayLabel(out)
out.Name = preferredProductTitle(in.GTIN, out.Name, out.ProcessedName, in.Name, in.PriorProcessedName)
out.ProcessedName = preferredProductTitle(in.GTIN, out.ProcessedName, out.Name, in.PriorProcessedName, in.Name)
out.Description = preferredProductDescription(out.Name, out.Description, out.ProcessedDescription, in.Description)
@@ -483,22 +514,52 @@ func (e *Engine) RunSteps(ctx context.Context, companyID string, in ProductInput
if out.ProcessedName == "" {
out.ProcessedName = out.Name
}
// Titles are finalized first so name-as-category and formula leakage can be cleared.
scrubCategoryPollution(&out, in.CategoryNamesByUID, nil)
syncCategoryName(&out, in.CategoryNamesByUID)
displayCat := categoryDisplayLabel(out)
// After vector/mapped category resolution, formulas may key for the first time —
// repair short prose / title echo that ignored A1 description_template.
_, finalDescTpl := categoryFormulasFor(in, out.Category)
if out.ProcessedDescription == "" || isWeakPriorEnhanceDescription(out.ProcessedDescription, out.ProcessedName, out.Name) {
finalRepaired := false
if descriptionNeedsEnhanceRepair(out.ProcessedDescription, finalDescTpl, out.ProcessedName, out.Name) {
if synth := synthesizeProductDescription(out.ProcessedName, displayCat, in.Language, out.Attributes, finalDescTpl); synth != "" {
out.ProcessedDescription = synth
finalRepaired = true
}
}
if out.Description == "" || isWeakPriorEnhanceDescription(out.Description, out.Name, out.ProcessedName) {
if out.ProcessedDescription != "" && !isWeakPriorEnhanceDescription(out.ProcessedDescription, out.Name, out.ProcessedName) {
if descriptionNeedsEnhanceRepair(out.Description, finalDescTpl, out.Name, out.ProcessedName) {
if out.ProcessedDescription != "" && !descriptionNeedsEnhanceRepair(out.ProcessedDescription, finalDescTpl, out.Name, out.ProcessedName) {
out.Description = out.ProcessedDescription
finalRepaired = true
} else if synth := synthesizeProductDescription(out.Name, displayCat, in.Language, out.Attributes, finalDescTpl); synth != "" {
out.Description = synth
if out.ProcessedDescription == "" || isWeakPriorEnhanceDescription(out.ProcessedDescription, out.ProcessedName, out.Name) {
finalRepaired = true
if descriptionNeedsEnhanceRepair(out.ProcessedDescription, finalDescTpl, out.ProcessedName, out.Name) {
out.ProcessedDescription = synth
}
}
}
if finalRepaired {
delete(out.FieldSources, FieldEnhanceInputHash)
primary := strings.TrimSpace(in.Language)
if primary == "" && len(in.ContentLanguages) > 0 {
primary = strings.TrimSpace(in.ContentLanguages[0])
}
if primary == "" {
primary = company.DefaultLanguage
}
for lang, lf := range out.LocalizedContent {
lf.EnhanceInputHash = ""
if lang == primary {
lf.ProcessedDescription = out.ProcessedDescription
if lf.ProcessedName == "" {
lf.ProcessedName = out.ProcessedName
}
}
out.LocalizedContent[lang] = lf
}
}
if out.Attributes == nil {
out.Attributes = map[string]any{}
}
@@ -528,7 +589,7 @@ func preserveCategoryIfEmpty(out *StepResult, prior string) {
return
}
prior = strings.TrimSpace(prior)
if prior == "" {
if prior == "" || isUnusableCategoryValue(prior, out.ProcessedName, out.Name) {
return
}
out.Category = SanitizeText(prior)
@@ -563,7 +624,7 @@ func tryVectorCategorize(ctx context.Context, e *Engine, companyID string, out *
})
return
}
if strings.TrimSpace(cat) == "" {
if strings.TrimSpace(cat) == "" || isUnusableCategoryValue(cat, out.ProcessedName, out.Name) {
return
}
out.Category = SanitizeOutput(cat)
@@ -632,78 +693,252 @@ func appendStepLog(gpt map[string]any, name string, raw any) {
gpt["steps"] = append(steps, map[string]any{"step": name, "raw": raw})
}
// descriptionFormulaRetrySuffix is appended once when LLM JSON ignored the
// category description_template (short prose / title echo instead of multi-section HTML).
const descriptionFormulaRetrySuffix = "\n\nINVALID DESCRIPTION. Your JSON ignored the Description formula. Reply with ONLY one JSON object; \"description\" must be ONE HTML string covering each formula section in order with matching tags (h1/h2/h3/h4, p, ul)."
// descriptionNeedsEnhanceRepair is true when desc is weak/empty/title-echo or
// fails an active category description_template (A1 multi-section HTML).
// Heuristic invent/synth is intentionally excluded here — enhanceHashForceReason
// blocks hash-skip for synth so reprocess can still upgrade to real LLM copy.
func descriptionNeedsEnhanceRepair(desc string, template any, titles ...string) bool {
if isWeakPriorEnhanceDescription(desc, titles...) {
return true
}
return !descriptionSatisfiesFormula(desc, template)
}
// enhanceHashForceReason returns why a matching prior enhance_input_hash must not
// skip the LLM (empty = safe to reuse as ai_enhance_unchanged).
func enhanceHashForceReason(in ProductInput) string {
if isPromptLabelTitle(in.PriorProcessedName) {
return "prompt_label_title"
}
if reason := company.EnhanceHashSkipBlockReason(in.PriorProcessedDescription, in.PriorProcessedName, in.Name); reason != "" {
return reason
}
if !descriptionSatisfiesFormula(in.PriorProcessedDescription, in.DescriptionTemplate) {
return "formula-mismatch"
}
return ""
}
func (e *Engine) enhance(ctx context.Context, in ProductInput, category string, attrs map[string]any) (string, string, int, any, error) {
catUID := strings.TrimSpace(in.CategoryUniqueID)
catName := strings.TrimSpace(category)
sysTpl, userTpl := resolveProductPromptTemplates(in)
hash := HashEnhanceInput(category, in.Name, in.Description, in.BrandPrompt, in.Language, sysTpl, userTpl, attrs)
if e == nil || e.Completer == nil {
name := preferredProductTitle(in.GTIN, in.Name, in.PriorProcessedName)
logEnhanceOutcome(in, catUID, catName, "skip", "completer_nil", Completion{}, 0, "")
return name,
preferredProductDescription(name, in.Description, in.PriorProcessedDescription),
0, map[string]any{"status": "skipped", "input_hash": hash}, nil
0, map[string]any{"status": "skipped", "reason": "completer_nil"}, nil
}
// Skip LLM when inputs match the last successful enhance (before any credit debit),
// but never reuse a thin / title-echo prior description, or a prompt-leakage title.
// but never reuse thin / title-echo / invent-synth / formula-mismatch priors,
// or a prompt-leakage title.
var forceReason string
if in.PriorEnhanceHash != "" && in.PriorEnhanceHash == hash &&
(in.PriorProcessedName != "" || in.PriorProcessedDescription != "") &&
!isPromptLabelTitle(in.PriorProcessedName) &&
!isWeakPriorEnhanceDescription(in.PriorProcessedDescription, in.PriorProcessedName, in.Name) {
name := preferredProductTitle(in.GTIN, in.PriorProcessedName, in.Name)
desc := preferredProductDescription(name, in.PriorProcessedDescription, in.Description)
return name, desc, 0, map[string]any{
"status": "unchanged",
"input_hash": hash,
}, nil
(in.PriorProcessedName != "" || in.PriorProcessedDescription != "") {
forceReason = enhanceHashForceReason(in)
if forceReason == "" {
name := preferredProductTitle(in.GTIN, in.PriorProcessedName, in.Name)
desc := preferredProductDescription(name, in.PriorProcessedDescription, in.Description)
logEnhanceOutcome(in, catUID, catName, "skip", "unchanged_hash", Completion{}, 0, "")
return name, desc, 0, map[string]any{
"status": "unchanged",
"input_hash": hash,
}, nil
}
log.Printf("processing: ai_enhance skip_blocked reason=%s category=%q name=%q",
forceReason, truncateRunes(category, 80), truncateRunes(in.Name, 80))
}
system, user := RenderProductEnhancePrompts(sysTpl, userTpl, category, in.Name, in.Description, in.GTIN, in.BrandPrompt, in.Language, attrs)
logEnhancePrompt(in, catUID, catName, system, user)
started := time.Now()
comp, obj, err := CompleteJSON(ctx, e.Completer, system, user, CompleteOptions{
MaxTokens: MaxTokensEnhance,
Temperature: DefaultStructuredTemp,
})
elapsed := time.Since(started)
if err != nil {
// Network/provider failure vs parse failure after retry
name := preferredProductTitle(in.GTIN, in.Name, in.PriorProcessedName)
desc := preferredProductDescription(name, in.Description, in.PriorProcessedDescription)
if isWeakPriorEnhanceDescription(desc, name) {
outcome := "refuse"
reason := "empty_or_provider_error"
if descriptionNeedsEnhanceRepair(desc, in.DescriptionTemplate, name) {
if synth := synthesizeProductDescription(name, category, in.Language, attrs, in.DescriptionTemplate); synth != "" {
desc = synth
outcome = "synthesized"
}
}
if obj == nil && comp.Text == "" {
logEnhanceOutcome(in, catUID, catName, outcome, reason+":"+TruncateError(err), comp, elapsed, forceReason)
return name, desc, 0, map[string]any{
"provider": "passthrough",
"error": TruncateError(err),
}, err
}
// Parse failed after retry — keep usable copy; synthesize when empty/weak.
// Parse failed after retry — keep usable copy; synthesize when empty/weak/formula-miss.
if outcome != "synthesized" {
outcome = "refuse"
reason = "parse_failed"
} else {
reason = "parse_failed"
}
logEnhanceOutcome(in, catUID, catName, outcome, reason, comp, elapsed, forceReason)
return name, desc, comp.TotalTokens, map[string]any{
"status": "parse_failed",
"reason": "invalid_json",
"error": "AI returned invalid JSON; kept original title/description",
"raw": truncateRunes(comp.Text, 200),
}, nil
}
name := preferredProductTitle(in.GTIN, SanitizeOutput(fmt.Sprint(obj["name"])), in.Name, in.PriorProcessedName)
desc := preferredProductDescription(name, SanitizeOutput(fmt.Sprint(obj["description"])), in.Description, in.PriorProcessedDescription)
llmName := SanitizeOutput(fmt.Sprint(obj["name"]))
llmDesc := SanitizeOutput(fmt.Sprint(obj["description"]))
// Never prefer-fallback to originals for empty/leakage LLM fields — that stamped
// status=ok + input_hash on garbage enhance output (prod: 30272ms "ok", wrong copy).
if reason := llmEnhanceHardRefuseReason(llmName, llmDesc); reason != "" {
name := preferredProductTitle(in.GTIN, llmName, in.Name, in.PriorProcessedName)
desc := preferredProductDescription(name, llmDesc)
synthesized := false
if descriptionNeedsEnhanceRepair(desc, in.DescriptionTemplate, name) {
if desc == "" {
desc = preferredProductDescription(name, in.Description, in.PriorProcessedDescription)
}
if descriptionNeedsEnhanceRepair(desc, in.DescriptionTemplate, name) {
if synth := synthesizeProductDescription(name, category, in.Language, attrs, in.DescriptionTemplate); synth != "" {
desc = synth
synthesized = true
}
}
}
status := "refused"
outcome := "refuse"
if synthesized {
status = "synthesized"
outcome = "synthesized"
}
meta := map[string]any{
"status": status,
"reason": reason,
"raw": comp.Raw,
}
if forceReason != "" {
meta["forced_reenhance"] = true
meta["force_reason"] = forceReason
}
logEnhanceOutcome(in, catUID, catName, outcome, reason, comp, elapsed, forceReason)
return name, desc, comp.TotalTokens, meta, nil
}
name := preferredProductTitle(in.GTIN, llmName, in.Name, in.PriorProcessedName)
// Quality-gate on LLM description alone (do not absorb originals yet).
desc := preferredProductDescription(name, llmDesc)
didFormulaRetry := false
// Fast garbage that ignores A1 description_template: one formula-aware retry, then synthesize.
if !descriptionSatisfiesFormula(desc, in.DescriptionTemplate) &&
FormatDescriptionFormulaConstraint(in.DescriptionTemplate) != "" {
didFormulaRetry = true
retryUser := user + descriptionFormulaRetrySuffix
retryStarted := time.Now()
comp2, obj2, err2 := CompleteJSON(ctx, e.Completer, system, retryUser, CompleteOptions{
MaxTokens: MaxTokensEnhance,
Temperature: DefaultStructuredTemp,
})
elapsed += time.Since(retryStarted)
comp.PromptTokens += comp2.PromptTokens
comp.OutputTokens += comp2.OutputTokens
comp.TotalTokens += comp2.TotalTokens
if err2 == nil && obj2 != nil {
n2 := SanitizeOutput(fmt.Sprint(obj2["name"]))
d2 := SanitizeOutput(fmt.Sprint(obj2["description"]))
if llmEnhanceHardRefuseReason(n2, d2) == "" {
name = preferredProductTitle(in.GTIN, n2, name, in.Name, in.PriorProcessedName)
desc = preferredProductDescription(name, d2)
if comp2.Raw != nil {
comp.Raw = comp2.Raw
}
if comp2.Text != "" {
comp.Text = comp2.Text
}
}
}
}
synthesized := false
if isWeakPriorEnhanceDescription(desc, name) {
if synth := synthesizeProductDescription(name, category, in.Language, attrs, in.DescriptionTemplate); synth != "" {
desc = synth
synthesized = true
if descriptionNeedsEnhanceRepair(desc, in.DescriptionTemplate, name) {
if desc == "" {
desc = preferredProductDescription(name, in.Description, in.PriorProcessedDescription)
}
if descriptionNeedsEnhanceRepair(desc, in.DescriptionTemplate, name) {
if synth := synthesizeProductDescription(name, category, in.Language, attrs, in.DescriptionTemplate); synth != "" {
desc = synth
synthesized = true
}
}
}
meta := map[string]any{
"status": "ok",
"raw": comp.Raw,
}
if forceReason != "" {
meta["reason"] = forceReason
meta["forced_reenhance"] = true
}
outcome := "ok"
reason := forceReason
// Heuristic synthesize must not poison enhance_input_hash (would hash-skip
// formula-aware LLM copy on reprocess). Only persist hash for real LLM quality.
if synthesized {
meta["status"] = "synthesized"
} else if !isWeakPriorEnhanceDescription(desc, name) {
outcome = "synthesized"
if didFormulaRetry {
reason = "formula_retry"
} else {
reason = "weak_or_formula_mismatch"
}
meta["reason"] = reason
} else if descriptionNeedsEnhanceRepair(desc, in.DescriptionTemplate, name) {
meta["status"] = "refused"
outcome = "refuse"
if didFormulaRetry {
reason = "formula_retry"
} else {
reason = "weak_or_formula_mismatch"
}
meta["reason"] = reason
} else {
meta["input_hash"] = hash
if didFormulaRetry {
reason = "formula_retry"
if meta["reason"] == nil {
meta["reason"] = reason
}
}
}
logEnhanceOutcome(in, catUID, catName, outcome, reason, comp, elapsed, forceReason)
return name, desc, comp.TotalTokens, meta, nil
}
// llmEnhanceHardRefuseReason reports empty/leakage LLM fields that must never be
// accepted as quality enhance (even when originals could fill the gap).
func llmEnhanceHardRefuseReason(name, desc string) string {
name = strings.TrimSpace(name)
desc = strings.TrimSpace(desc)
if name == "" || name == "<nil>" {
return "empty_name"
}
if isPromptLabelTitle(name) {
return "prompt_leakage_name"
}
if desc == "" || desc == "<nil>" {
return "empty_description"
}
return ""
}
func sanitizeJSON(v any) string {
if v == nil {
return "{}"
@@ -841,6 +1076,24 @@ var promptLeakagePhrases = []string{
"prefer attrs values",
"order matters; join with",
"do not hardcode a language",
// Description / title formula scaffolding (Format*FormulaConstraint + enhance templates).
"emit description as one html",
"emit one html string",
"covering each section",
"tags matching section",
"overrides any shorter",
"competing full html",
"metadescription",
"never ignore the description",
"follow any description formula",
"when a description formula",
"structured html sections",
"never copy name as description",
"do not invent specs",
"prefer 1-3 factual",
"1-2 sentences",
"keep literal text as written",
"build name from attrs using this structure",
}
var promptLeakageLongKeywords = []string{
@@ -850,6 +1103,97 @@ var promptLeakageLongKeywords = []string{
"json",
"attrs",
"retail title",
"html string",
"section type",
}
// isUnusableCategoryValue rejects tokens that must never become Category /
// CategoryName: prompt labels / title-formula scaffolding, or the product
// title itself (name-as-category). Callers with taxonomy should prefer
// scrubCategoryPollution so a title that legitimately equals a category name
// can still resolve to unique_id.
func isUnusableCategoryValue(s string, productTitles ...string) bool {
s = strings.TrimSpace(s)
if s == "" || s == "<nil>" {
return false
}
if isPromptLabelTitle(s) {
return true
}
return categoryTokenEqualsProductTitle(s, productTitles...)
}
func categoryTokenEqualsProductTitle(s string, productTitles ...string) bool {
s = strings.TrimSpace(s)
if s == "" || s == "<nil>" {
return false
}
for _, t := range productTitles {
t = strings.TrimSpace(t)
if t == "" || t == "<nil>" {
continue
}
if strings.EqualFold(s, t) {
return true
}
}
return false
}
func taxonomyDisplayNameSet(namesByUID map[string]string) map[string]struct{} {
if len(namesByUID) == 0 {
return nil
}
out := make(map[string]struct{}, len(namesByUID))
for _, name := range namesByUID {
n := strings.ToLower(strings.TrimSpace(name))
if n != "" {
out[n] = struct{}{}
}
}
return out
}
// scrubCategoryPollution clears Category / CategoryName when they hold prompt
// leakage/formula text, or equal the product title without resolving to a
// company taxonomy unique_id / display name.
func scrubCategoryPollution(out *StepResult, namesByUID map[string]string, valid map[string]struct{}) {
if out == nil {
return
}
titles := []string{out.ProcessedName, out.Name}
clearCategory := func(note string) {
out.Category = ""
out.CategoryName = ""
if out.FieldSources == nil {
out.FieldSources = map[string]any{}
}
out.FieldSources["category"] = "cleared_invalid"
out.Notes = append(out.Notes, note)
}
cat := strings.TrimSpace(out.Category)
if cat != "" {
switch {
case isPromptLabelTitle(cat):
clearCategory("category: ignored (prompt leakage)")
case categoryTokenEqualsProductTitle(cat, titles...) &&
resolveCompanyCategoryUniqueID(cat, namesByUID, valid) == "":
clearCategory("category: ignored (product title)")
}
}
name := strings.TrimSpace(out.CategoryName)
if name == "" {
return
}
if isPromptLabelTitle(name) {
out.CategoryName = ""
return
}
if categoryTokenEqualsProductTitle(name, titles...) {
if _, ok := taxonomyDisplayNameSet(namesByUID)[strings.ToLower(name)]; !ok {
out.CategoryName = ""
}
}
}
// preferredProductTitle picks the first usable title, skipping empty values,
+119 -2
View File
@@ -223,8 +223,12 @@ func TestIsPromptLeakageTitle(t *testing.T) {
"write name in {{language}}": true,
"Schema: {\"name\":\"string\",\"description\":\"string\"}": true,
"Reply with ONLY JSON (no markdown)": true,
"Vox SWA-8000W dishwasher": false,
"Acme Widget Pro": false,
"Emit description as ONE HTML string covering each section in order": true,
"Description formula (REQUIRED — overrides any shorter \"1-2 sentences\" rule)": true,
"when a Description formula follows, emit ONE HTML string": true,
"never copy name as description": true,
"Vox SWA-8000W dishwasher": false,
"Acme Widget Pro": false,
"": false,
}
for in, want := range cases {
@@ -242,6 +246,119 @@ func TestIsPromptLeakageTitle(t *testing.T) {
}
}
func TestIsUnusableCategoryValue_rejectsNameAndFormula(t *testing.T) {
t.Parallel()
title := "NOSILEC W53070 81-140CM 180ST VOGELS"
if !isUnusableCategoryValue(title, title) {
t.Fatal("product title must not be usable as category")
}
if !isUnusableCategoryValue("Title formula (order matters; join with \" \")", title) {
t.Fatal("title formula must not be usable as category")
}
if !isUnusableCategoryValue("Emit description as ONE HTML string covering each section", title) {
t.Fatal("description formula must not be usable as category")
}
if !isUnusableCategoryValue("Category:", title) {
t.Fatal("prompt label must not be usable as category")
}
if isUnusableCategoryValue("28", title) {
t.Fatal("taxonomy unique_id must remain usable")
}
if isUnusableCategoryValue("TV mounts", title) {
t.Fatal("real category display name must remain usable")
}
}
func TestScrubCategoryPollution_clearsNameAsCategory(t *testing.T) {
t.Parallel()
title := "Vogel's WALL 3245 TV Wall Mount"
out := StepResult{
Name: title,
ProcessedName: title,
Category: title,
CategoryName: title,
FieldSources: map[string]any{"category": "vector"},
}
scrubCategoryPollution(&out, nil, nil)
if out.Category != "" || out.CategoryName != "" {
t.Fatalf("expected category cleared, got Category=%q CategoryName=%q", out.Category, out.CategoryName)
}
if src, _ := out.FieldSources["category"].(string); src != "cleared_invalid" {
t.Fatalf("field_sources.category=%v want cleared_invalid", out.FieldSources["category"])
}
}
func TestEnhance_rejectsDescriptionFormulaLeakageTitle(t *testing.T) {
leak := "Emit description as ONE HTML string covering each section in order in Slovenian"
e := &Engine{
Completer: stubCompleter{fn: func(system, user string) (Completion, error) {
return Completion{Text: `{"name":"` + leak + `","description":"A solid washer."}`, TotalTokens: 2}, nil
}},
Vector: NoopVectorCategorizer{},
}
out, err := e.RunSteps(context.Background(), "co", ProductInput{
GTIN: "8712285326882",
Name: "Vox SWA-8000W",
Mapped: map[string]any{"name": "Vox SWA-8000W", "description": "Washer", "category": "demo-electronics"},
}, "enhance_only", nil, StepPolicy{AllowAI: true})
if err != nil {
t.Fatal(err)
}
if out.ProcessedName != "Vox SWA-8000W" {
t.Fatalf("ProcessedName=%q want Vox SWA-8000W (reject description formula leakage)", out.ProcessedName)
}
}
func TestRunSteps_rejectsProductNameAsCategory(t *testing.T) {
title := "NOSILEC W53070 81-140CM 180ST VOGELS"
e := &Engine{
Completer: stubCompleter{fn: func(system, user string) (Completion, error) {
return Completion{Text: `{"name":"` + title + `","description":"TV wall mount for large screens."}`, TotalTokens: 2}, nil
}},
Vector: NoopVectorCategorizer{},
}
out, err := e.RunSteps(context.Background(), "co", ProductInput{
GTIN: "8712285326882",
Name: title,
PriorCategory: title,
Mapped: map[string]any{"name": title, "description": title, "category": title},
}, "enhance_only", nil, StepPolicy{AllowAI: true})
if err != nil {
t.Fatal(err)
}
if out.Category == title || strings.EqualFold(out.Category, out.ProcessedName) {
t.Fatalf("Category=%q must not equal product title", out.Category)
}
if categoryDisplayLabel(out) == title {
t.Fatalf("display category leaked product title: %q", categoryDisplayLabel(out))
}
}
func TestRunSteps_rejectsFormulaAsCategory(t *testing.T) {
leak := "Title formula (order matters; join with \" \"). Build name from Attrs"
e := &Engine{
Completer: stubCompleter{fn: func(system, user string) (Completion, error) {
return Completion{Text: `{"name":"Vox SWA-8000W","description":"Washer."}`, TotalTokens: 2}, nil
}},
Vector: NoopVectorCategorizer{},
}
out, err := e.RunSteps(context.Background(), "co", ProductInput{
GTIN: "8712285326882",
Name: "Vox SWA-8000W",
PriorCategory: leak,
Mapped: map[string]any{"name": "Vox SWA-8000W", "description": "Washer", "category": leak},
}, "enhance_only", nil, StepPolicy{AllowAI: true})
if err != nil {
t.Fatal(err)
}
if out.Category != "" {
t.Fatalf("Category=%q want empty (formula rejected)", out.Category)
}
if categoryDisplayLabel(out) != "" {
t.Fatalf("display=%q want empty", categoryDisplayLabel(out))
}
}
func TestEnhance_rejectsFormulaLeakageTitle(t *testing.T) {
leak := "short retail title; follow any Title formula constraints that follow; use Attrs"
e := &Engine{
@@ -28,6 +28,10 @@ func TestUpsertProcessedProductSQL_usesOnConflict(t *testing.T) {
if !strings.Contains(upsertProcessedProductSQL, "COALESCE(NULLIF(BTRIM(EXCLUDED.category), ''), processed_products.category)") {
t.Fatalf("expected category preserve on conflict: got SQL without COALESCE preserve")
}
// Intentional scrub/filter clears must wipe a previously persisted product-title category.
if !strings.Contains(upsertProcessedProductSQL, "cleared_invalid") {
t.Fatalf("expected cleared_invalid category wipe path in upsert SQL")
}
// Free template meta must persist; existing non-empty meta must be preserved on reprocess.
if !strings.Contains(upsertProcessedProductSQL, "meta_title") || !strings.Contains(upsertProcessedProductSQL, "meta_description") {
t.Fatalf("expected meta_title/meta_description columns in upsert")
+16 -4
View File
@@ -326,11 +326,23 @@ func (p *Pipeline) LoadV1ProcessJobItems(ctx context.Context, companyID, jobID u
if name, ok := categoryNames[catStr]; ok && name != "" {
catNameStr = name
} else if nested, ok := mapped["category"].(map[string]any); ok {
for _, k := range []string{"name", "category_name", "title"} {
if s := strings.TrimSpace(stringFromAny(nested[k])); s != "" {
catNameStr = s
break
// Only accept nested display names that resolve in company taxonomy —
// never product title / title-formula leftovers from feed objects.
for _, k := range []string{"name", "category_name"} {
s := strings.TrimSpace(stringFromAny(nested[k]))
if s == "" || isUnusableCategoryValue(s, titleStr) {
continue
}
uid := resolveCompanyCategoryUniqueID(s, categoryNames, nil)
if uid == "" {
continue
}
if resolvedName := strings.TrimSpace(categoryNames[uid]); resolvedName != "" {
catNameStr = resolvedName
} else {
catNameStr = s
}
break
}
}
}