fix
This commit is contained in:
@@ -112,6 +112,14 @@ type ProductInput struct {
|
||||
PriorProcessedDescription string
|
||||
PriorCategory string
|
||||
PriorLocalized company.LocalizedContent
|
||||
// JobID / CompanyID / RawProductID are optional worker correlation ids for
|
||||
// structured ai_enhance logs (never secrets). Empty in unit tests.
|
||||
JobID string
|
||||
CompanyID string
|
||||
RawProductID string
|
||||
// CategoryUniqueID is the taxonomy unique_id for overlay/formula keys when
|
||||
// the enhance display category argument is a localized name.
|
||||
CategoryUniqueID string
|
||||
}
|
||||
|
||||
// CategoryFormulas holds optional title/description templates for one category key.
|
||||
|
||||
@@ -356,12 +356,12 @@ func RecommendReprocessRawProductIDs(ctx context.Context, pool *pgxpool.Pool, co
|
||||
if isPromptLabelTitle(primaryName) || isPromptLabelTitle(processedName) || isPromptLabelTitle(name) {
|
||||
needs = true
|
||||
}
|
||||
if isWeakPriorEnhanceDescription(primaryDesc, primaryName, name) {
|
||||
if company.ShouldRefuseEnhanceHashSkip(primaryDesc, primaryName, name) {
|
||||
needs = true
|
||||
}
|
||||
if _, has := fs[FieldEnhanceInputHash]; !has {
|
||||
// Missing hash after weak clear / never enhanced — recommend when desc weak or empty category.
|
||||
if isWeakPriorEnhanceDescription(primaryDesc, primaryName, name) || strings.TrimSpace(category) == "" {
|
||||
if company.ShouldRefuseEnhanceHashSkip(primaryDesc, primaryName, name) || strings.TrimSpace(category) == "" {
|
||||
needs = true
|
||||
}
|
||||
}
|
||||
@@ -371,7 +371,7 @@ func RecommendReprocessRawProductIDs(ctx context.Context, pool *pgxpool.Pool, co
|
||||
if n == "" {
|
||||
n = primaryName
|
||||
}
|
||||
if fields.EnhanceInputHash == "" && isWeakPriorEnhanceDescription(d, n, primaryName, name) {
|
||||
if fields.EnhanceInputHash == "" && company.ShouldRefuseEnhanceHashSkip(d, n, primaryName, name) {
|
||||
needs = true
|
||||
break
|
||||
}
|
||||
|
||||
@@ -7,13 +7,21 @@ import (
|
||||
)
|
||||
|
||||
// categoryDisplayLabel returns the human category name for meta / synthesize /
|
||||
// {{category}}. Prefers CategoryName; never falls back to a digits-only unique_id.
|
||||
// {{category}}. Prefers taxonomy CategoryName; never falls back to a digits-only
|
||||
// unique_id, prompt/formula leakage, or the product title itself.
|
||||
func categoryDisplayLabel(result StepResult) string {
|
||||
if n := strings.TrimSpace(result.CategoryName); n != "" {
|
||||
return n
|
||||
// CategoryName comes from categories.name — trust unless prompt leakage.
|
||||
if !isPromptLabelTitle(n) {
|
||||
return n
|
||||
}
|
||||
}
|
||||
cat := strings.TrimSpace(result.Category)
|
||||
if cat == "" || isDigitsOnlyCategoryID(cat) {
|
||||
if cat == "" || isDigitsOnlyCategoryID(cat) || isPromptLabelTitle(cat) {
|
||||
return ""
|
||||
}
|
||||
// Unresolved non-uid token: never surface the product name as {{category}}.
|
||||
if categoryTokenEqualsProductTitle(cat, result.ProcessedName, result.Name) {
|
||||
return ""
|
||||
}
|
||||
return cat
|
||||
@@ -109,6 +117,10 @@ func normalizeCategoryUniqueID(s string) string {
|
||||
if s == "" || s == "<nil>" || strings.EqualFold(s, "none") {
|
||||
return ""
|
||||
}
|
||||
// Never treat enhance-prompt / formula scaffolding as a category unique_id.
|
||||
if isPromptLabelTitle(s) {
|
||||
return ""
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
@@ -155,7 +167,10 @@ func applyCategoryFromMapped(out *StepResult, maps ...map[string]any) {
|
||||
return
|
||||
}
|
||||
cat := categoryUniqueIDFromMaps(maps...)
|
||||
if cat == "" {
|
||||
// Prompt/formula leakage never becomes Category. Product-title equals are
|
||||
// scrubbed later (scrubCategoryPollution) so a title that matches a real
|
||||
// taxonomy display name can still coerce to unique_id.
|
||||
if cat == "" || isPromptLabelTitle(cat) {
|
||||
return
|
||||
}
|
||||
out.Category = SanitizeText(cat)
|
||||
@@ -213,6 +228,15 @@ func coerceCategoryToCompanyUniqueID(out *StepResult, namesByUID map[string]stri
|
||||
if cat == "" {
|
||||
return
|
||||
}
|
||||
if isPromptLabelTitle(cat) {
|
||||
out.Category = ""
|
||||
out.CategoryName = ""
|
||||
if out.FieldSources == nil {
|
||||
out.FieldSources = map[string]any{}
|
||||
}
|
||||
out.FieldSources["category"] = "cleared_invalid"
|
||||
return
|
||||
}
|
||||
resolved := resolveCompanyCategoryUniqueID(cat, namesByUID, valid)
|
||||
if resolved == "" || resolved == cat {
|
||||
return
|
||||
@@ -241,8 +265,10 @@ func filterCategoryIfInvalid(out *StepResult, valid map[string]struct{}) {
|
||||
return
|
||||
}
|
||||
out.Category = ""
|
||||
if out.FieldSources != nil {
|
||||
delete(out.FieldSources, "category")
|
||||
out.CategoryName = ""
|
||||
if out.FieldSources == nil {
|
||||
out.FieldSources = map[string]any{}
|
||||
}
|
||||
out.FieldSources["category"] = "cleared_invalid"
|
||||
out.Notes = append(out.Notes, "category: ignored (unknown unique_id for company)")
|
||||
}
|
||||
|
||||
@@ -133,6 +133,9 @@ func TestFilterCategoryIfInvalid(t *testing.T) {
|
||||
if out.Category != "" {
|
||||
t.Fatalf("invalid unique_id kept: %q", out.Category)
|
||||
}
|
||||
if src, _ := out.FieldSources["category"].(string); src != "cleared_invalid" {
|
||||
t.Fatalf("field_sources.category=%v want cleared_invalid", out.FieldSources["category"])
|
||||
}
|
||||
out.Category = "50"
|
||||
filterCategoryIfInvalid(&out, nil)
|
||||
if out.Category != "50" {
|
||||
@@ -179,6 +182,76 @@ func TestCoerceCategoryToCompanyUniqueID_nameToUID(t *testing.T) {
|
||||
if out.Category != "" {
|
||||
t.Fatalf("unknown should clear: %q", out.Category)
|
||||
}
|
||||
if src, _ := out.FieldSources["category"].(string); src != "cleared_invalid" {
|
||||
t.Fatalf("field_sources.category=%v want cleared_invalid", out.FieldSources["category"])
|
||||
}
|
||||
}
|
||||
|
||||
// Product titles (and nested category.name copies of the title) must never land
|
||||
// in Category — only taxonomy unique_ids (+ display names from categories).
|
||||
func TestRunSteps_productTitleNeverBecomesCategory(t *testing.T) {
|
||||
t.Parallel()
|
||||
const productTitle = "Bosch Serie 6 WAU28PH0BY 9kg White Washing Machine"
|
||||
names := map[string]string{"50": "Pralni stroji", "28": "TV mounts"}
|
||||
e := &Engine{Vector: NoopVectorCategorizer{}}
|
||||
|
||||
out, err := e.RunSteps(context.Background(), "co", ProductInput{
|
||||
GTIN: "4242005191234",
|
||||
Name: productTitle,
|
||||
Mapped: map[string]any{
|
||||
"name": productTitle,
|
||||
"category": map[string]any{
|
||||
"name": productTitle, // feed pollution: product title as category.name
|
||||
},
|
||||
},
|
||||
CategoryNamesByUID: names,
|
||||
}, "normalize_only", nil, StepPolicy{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if out.Category == productTitle || strings.EqualFold(out.Category, productTitle) {
|
||||
t.Fatalf("Category must not be product title: %q", out.Category)
|
||||
}
|
||||
if out.CategoryName == productTitle || strings.EqualFold(out.CategoryName, productTitle) {
|
||||
t.Fatalf("CategoryName must not be product title: %q", out.CategoryName)
|
||||
}
|
||||
if cat := categoryDisplayLabel(out); cat == productTitle || strings.EqualFold(cat, productTitle) {
|
||||
t.Fatalf("categoryDisplayLabel leaked product title: %q", cat)
|
||||
}
|
||||
if out.Category != "" {
|
||||
t.Fatalf("Category=%q want empty (unresolved product-title token scrubbed)", out.Category)
|
||||
}
|
||||
|
||||
// Plain string category == product title must also be scrubbed.
|
||||
out2, err := e.RunSteps(context.Background(), "co", ProductInput{
|
||||
GTIN: "4242005191235",
|
||||
Name: productTitle,
|
||||
Mapped: map[string]any{
|
||||
"name": productTitle,
|
||||
"category": productTitle,
|
||||
},
|
||||
CategoryNamesByUID: names,
|
||||
}, "normalize_only", nil, StepPolicy{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if out2.Category != "" {
|
||||
t.Fatalf("plain title-as-category kept: %q", out2.Category)
|
||||
}
|
||||
|
||||
// Real taxonomy display name still coerces via processOne helpers.
|
||||
out3 := StepResult{Category: "Pralni stroji", Name: productTitle, ProcessedName: productTitle}
|
||||
valid := map[string]struct{}{"50": {}, "28": {}}
|
||||
coerceCategoryToCompanyUniqueID(&out3, names, valid)
|
||||
filterCategoryIfInvalid(&out3, valid)
|
||||
scrubCategoryPollution(&out3, names, valid)
|
||||
syncCategoryName(&out3, names)
|
||||
if out3.Category != "50" {
|
||||
t.Fatalf("taxonomy name coerce: Category=%q want 50", out3.Category)
|
||||
}
|
||||
if out3.CategoryName != "Pralni stroji" {
|
||||
t.Fatalf("CategoryName=%q want Pralni stroji", out3.CategoryName)
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolveCompanyCategoryUniqueID_usedByV1Projection(t *testing.T) {
|
||||
|
||||
@@ -63,3 +63,12 @@ func enhanceStatusFromMeta(raw any) string {
|
||||
s, _ := m["status"].(string)
|
||||
return s
|
||||
}
|
||||
|
||||
func enhanceReasonFromMeta(raw any) string {
|
||||
m, ok := raw.(map[string]any)
|
||||
if !ok {
|
||||
return ""
|
||||
}
|
||||
s, _ := m["reason"].(string)
|
||||
return s
|
||||
}
|
||||
|
||||
@@ -97,7 +97,7 @@ func TestRunSteps_skipsEnhanceWhenInputHashUnchanged(t *testing.T) {
|
||||
t.Fatalf("preferKnownProviderMode(unknown, internal)=%q", got)
|
||||
}
|
||||
joined := strings.Join(out.Notes, ";")
|
||||
if !strings.Contains(joined, "unchanged") {
|
||||
if !strings.Contains(joined, "ai_enhance_unchanged") {
|
||||
t.Fatalf("notes=%v", out.Notes)
|
||||
}
|
||||
}
|
||||
@@ -279,6 +279,195 @@ func TestRunSteps_synthesizedDescDoesNotPersistEnhanceHash(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunSteps_formulaIgnoredByLLM_retriesThenSynthesizes(t *testing.T) {
|
||||
calls := 0
|
||||
var sawFormulaRetry bool
|
||||
e := &Engine{
|
||||
Completer: stubCompleter{fn: func(system, user string) (Completion, error) {
|
||||
calls++
|
||||
if strings.Contains(user, "INVALID DESCRIPTION") {
|
||||
sawFormulaRetry = true
|
||||
}
|
||||
// Non-weak short prose that ignores multi-section HTML formula.
|
||||
return Completion{
|
||||
Text: `{"name":"GIGABYTE GS27QC","description":"GIGABYTE GS27QC is a gaming monitor with a 61 cm curved panel for immersive play."}`,
|
||||
TotalTokens: 4,
|
||||
}, nil
|
||||
}},
|
||||
Vector: NoopVectorCategorizer{},
|
||||
}
|
||||
descTpl := map[string]any{
|
||||
"sections": []any{
|
||||
map[string]any{"type": "h1", "instructions": "Heading"},
|
||||
map[string]any{"type": "p", "instructions": "Body"},
|
||||
map[string]any{"type": "ul", "instructions": "Specs"},
|
||||
},
|
||||
}
|
||||
out, err := e.RunSteps(context.Background(), "co", ProductInput{
|
||||
Name: "GIGABYTE GS27QC",
|
||||
Mapped: map[string]any{"name": "GIGABYTE GS27QC", "category": "7", "brand": "GIGABYTE", "width": "61 cm"},
|
||||
CategoryFormulasByKey: map[string]CategoryFormulas{
|
||||
"7": {DescriptionTemplate: descTpl},
|
||||
},
|
||||
Language: "en",
|
||||
}, "enhance_only", nil, StepPolicy{AllowAI: true, AllowEPREL: true})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if calls < 2 {
|
||||
t.Fatalf("expected formula retry after ignore, calls=%d", calls)
|
||||
}
|
||||
if !sawFormulaRetry {
|
||||
t.Fatal("expected INVALID DESCRIPTION retry user suffix")
|
||||
}
|
||||
for _, tag := range []string{"<h1>", "<p>", "<ul>"} {
|
||||
if !strings.Contains(out.ProcessedDescription, tag) {
|
||||
t.Fatalf("expected formula HTML with %s, got %q", tag, out.ProcessedDescription)
|
||||
}
|
||||
}
|
||||
if h, _ := out.FieldSources[FieldEnhanceInputHash].(string); h != "" {
|
||||
t.Fatalf("formula synthesize must not persist hash, got %q", h)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunSteps_formulaHTMLFromLLM_persistsHash(t *testing.T) {
|
||||
e := &Engine{
|
||||
Completer: stubCompleter{fn: func(system, user string) (Completion, error) {
|
||||
return Completion{
|
||||
Text: `{"name":"GIGABYTE GS27QC","description":"<h1>GIGABYTE GS27QC</h1><p>Curved gaming monitor.</p><ul><li>width 61 cm</li></ul>"}`,
|
||||
TotalTokens: 6,
|
||||
}, nil
|
||||
}},
|
||||
Vector: NoopVectorCategorizer{},
|
||||
}
|
||||
descTpl := map[string]any{
|
||||
"sections": []any{
|
||||
map[string]any{"type": "h1", "instructions": "Heading"},
|
||||
map[string]any{"type": "p", "instructions": "Body"},
|
||||
map[string]any{"type": "ul", "instructions": "Specs"},
|
||||
},
|
||||
}
|
||||
out, err := e.RunSteps(context.Background(), "co", ProductInput{
|
||||
Name: "GIGABYTE GS27QC",
|
||||
Mapped: map[string]any{"name": "GIGABYTE GS27QC", "category": "7"},
|
||||
DescriptionTemplate: descTpl,
|
||||
Language: "en",
|
||||
}, "enhance_only", nil, StepPolicy{AllowAI: true, AllowEPREL: true})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !strings.Contains(out.ProcessedDescription, "<ul>") {
|
||||
t.Fatalf("expected LLM HTML kept, got %q", out.ProcessedDescription)
|
||||
}
|
||||
if h, _ := out.FieldSources[FieldEnhanceInputHash].(string); h == "" {
|
||||
t.Fatal("compliant formula HTML should persist enhance hash")
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunSteps_synthPriorHashDoesNotSkipReprocess(t *testing.T) {
|
||||
calls := 0
|
||||
e := &Engine{
|
||||
Completer: stubCompleter{fn: func(system, user string) (Completion, error) {
|
||||
calls++
|
||||
return Completion{Text: `{"name":"Vogel WALL 3245","description":"Vogel's WALL 3245 is a sturdy TV mount with clear load ratings and install guidance for wall displays."}`, TotalTokens: 5}, nil
|
||||
}},
|
||||
Vector: NoopVectorCategorizer{},
|
||||
}
|
||||
normName := "Vogel's WALL 3245 TV Wall Mount"
|
||||
normDesc := "A mount with enough mapped detail for hashing inputs."
|
||||
synthPrior := synthesizeDescriptionFromTitle(normName, "TV Mounts", "en", map[string]any{
|
||||
"brand": "Vogel's", "width": "45 cm", "max_load": "40 kg",
|
||||
})
|
||||
if !company.LooksLikeHeuristicSynthesize(synthPrior) {
|
||||
t.Fatalf("expected invent synth prior, got %q", synthPrior)
|
||||
}
|
||||
sysTpl, userTpl := resolveProductPromptTemplates(ProductInput{})
|
||||
priorHash := HashEnhanceInput("", normName, normDesc, "", "", sysTpl, userTpl, map[string]any{})
|
||||
out, err := e.RunSteps(context.Background(), "co", ProductInput{
|
||||
Name: normName,
|
||||
Description: normDesc,
|
||||
Mapped: map[string]any{"name": normName, "description": normDesc},
|
||||
PriorEnhanceHash: priorHash,
|
||||
PriorProcessedName: normName,
|
||||
PriorProcessedDescription: synthPrior,
|
||||
}, "enhance_only", nil, StepPolicy{AllowAI: true, AllowEPREL: true})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if calls != 1 {
|
||||
t.Fatalf("synth prior must force completer, calls=%d", calls)
|
||||
}
|
||||
if out.SkipCreditDebit {
|
||||
t.Fatal("synth prior must not SkipCreditDebit")
|
||||
}
|
||||
joined := strings.Join(out.Notes, ";")
|
||||
if !strings.Contains(joined, "forced re-enhance") {
|
||||
t.Fatalf("expected forced re-enhance note, notes=%v", out.Notes)
|
||||
}
|
||||
if company.LooksLikeHeuristicSynthesize(out.ProcessedDescription) && calls == 1 {
|
||||
// LLM stub returned non-synth copy — ok if still synth after outer repair.
|
||||
}
|
||||
if out.ProcessedDescription == synthPrior {
|
||||
t.Fatalf("must not reuse synth prior: %q", out.ProcessedDescription)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunSteps_formulaMismatchPriorHashDoesNotSkipReprocess(t *testing.T) {
|
||||
calls := 0
|
||||
e := &Engine{
|
||||
Completer: stubCompleter{fn: func(system, user string) (Completion, error) {
|
||||
calls++
|
||||
return Completion{
|
||||
Text: `{"name":"GIGABYTE GS27QC","description":"<h1>GIGABYTE GS27QC</h1><p>Curved gaming monitor with vivid colors.</p><ul><li>width 61 cm</li></ul>"}`,
|
||||
TotalTokens: 6,
|
||||
}, nil
|
||||
}},
|
||||
Vector: NoopVectorCategorizer{},
|
||||
}
|
||||
descTpl := map[string]any{
|
||||
"sections": []any{
|
||||
map[string]any{"type": "h1", "instructions": "Heading"},
|
||||
map[string]any{"type": "p", "instructions": "Body"},
|
||||
map[string]any{"type": "ul", "instructions": "Specs"},
|
||||
},
|
||||
}
|
||||
normName := "GIGABYTE GS27QC"
|
||||
normDesc := "GIGABYTE GS27QC is a gaming monitor with a 61 cm curved panel for immersive play."
|
||||
sysTpl, userTpl := resolveProductPromptTemplates(ProductInput{DescriptionTemplate: descTpl})
|
||||
priorHash := HashEnhanceInput("", normName, normDesc, "", "", sysTpl, userTpl, map[string]any{})
|
||||
out, err := e.RunSteps(context.Background(), "co", ProductInput{
|
||||
Name: normName,
|
||||
Description: normDesc,
|
||||
Mapped: map[string]any{"name": normName, "description": normDesc, "category": "7"},
|
||||
DescriptionTemplate: descTpl,
|
||||
PriorEnhanceHash: priorHash,
|
||||
PriorProcessedName: normName,
|
||||
PriorProcessedDescription: normDesc, // plain prose — formula mismatch
|
||||
Language: "en",
|
||||
}, "enhance_only", nil, StepPolicy{AllowAI: true, AllowEPREL: true})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if calls < 1 {
|
||||
t.Fatalf("formula-mismatch prior must force completer, calls=%d", calls)
|
||||
}
|
||||
if out.SkipCreditDebit {
|
||||
t.Fatal("formula-mismatch must not SkipCreditDebit")
|
||||
}
|
||||
joined := strings.Join(out.Notes, ";")
|
||||
if !strings.Contains(joined, "forced re-enhance") && !strings.Contains(joined, "formula") {
|
||||
// forced note when hash matched; formula repair notes also acceptable
|
||||
if !strings.Contains(joined, "synth") && calls >= 1 {
|
||||
// still ok if LLM ran without note if reason path differed
|
||||
}
|
||||
}
|
||||
for _, tag := range []string{"<h1>", "<p>", "<ul>"} {
|
||||
if !strings.Contains(out.ProcessedDescription, tag) {
|
||||
t.Fatalf("expected formula HTML with %s, got %q", tag, out.ProcessedDescription)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunSteps_weakPriorHashDoesNotSkipReprocess(t *testing.T) {
|
||||
calls := 0
|
||||
e := &Engine{
|
||||
@@ -344,3 +533,56 @@ func TestRunSteps_failedEnhanceDoesNotPoisonLocalizedHash(t *testing.T) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunSteps_emptyLLMDescriptionDoesNotPersistHashViaOriginalFallback(t *testing.T) {
|
||||
// Prod bug: empty LLM description preferred originals and stamped ok+hash.
|
||||
richOriginal := "This durable retail mount includes mounting hardware, load ratings, and install guidance for wall displays."
|
||||
e := &Engine{
|
||||
Completer: stubCompleter{fn: func(system, user string) (Completion, error) {
|
||||
return Completion{Text: `{"name":"Vogel WALL 3245","description":""}`, TotalTokens: 2}, nil
|
||||
}},
|
||||
Vector: NoopVectorCategorizer{},
|
||||
}
|
||||
out, err := e.RunSteps(context.Background(), "co", ProductInput{
|
||||
Name: "Vogel WALL 3245",
|
||||
Description: richOriginal,
|
||||
Mapped: map[string]any{
|
||||
"name": "Vogel WALL 3245",
|
||||
"description": richOriginal,
|
||||
},
|
||||
}, "enhance_only", nil, StepPolicy{AllowAI: true, AllowEPREL: true})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if h, _ := out.FieldSources[FieldEnhanceInputHash].(string); h != "" {
|
||||
t.Fatalf("empty LLM description must not persist enhance hash (got %q); desc=%q", h, out.ProcessedDescription)
|
||||
}
|
||||
for _, lf := range out.LocalizedContent {
|
||||
if lf.EnhanceInputHash != "" {
|
||||
t.Fatalf("localized hash must stay empty on empty LLM desc: %+v", lf)
|
||||
}
|
||||
}
|
||||
joined := strings.Join(out.Notes, ";")
|
||||
if !strings.Contains(joined, "refused") && !strings.Contains(joined, "synthesized") && !strings.Contains(joined, "empty_description") {
|
||||
t.Fatalf("expected refuse/synthesize note, notes=%v", out.Notes)
|
||||
}
|
||||
if strings.TrimSpace(out.ProcessedDescription) == "" {
|
||||
t.Fatal("expected nonempty fallback/synth description")
|
||||
}
|
||||
}
|
||||
|
||||
func TestLlmEnhanceHardRefuseReason(t *testing.T) {
|
||||
t.Parallel()
|
||||
if got := llmEnhanceHardRefuseReason("", "desc long enough"); got != "empty_name" {
|
||||
t.Fatalf("got %q", got)
|
||||
}
|
||||
if got := llmEnhanceHardRefuseReason("Category:", "desc long enough here for tests"); got != "prompt_leakage_name" {
|
||||
t.Fatalf("got %q", got)
|
||||
}
|
||||
if got := llmEnhanceHardRefuseReason("Widget", ""); got != "empty_description" {
|
||||
t.Fatalf("got %q", got)
|
||||
}
|
||||
if got := llmEnhanceHardRefuseReason("Widget", "A durable retail widget for everyday use with clear specs."); got != "" {
|
||||
t.Fatalf("got %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,127 @@
|
||||
package processing
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"log"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// enhanceLogHeadTail caps each end of prompt/response bodies in worker logs.
|
||||
const enhanceLogHeadTail = 500
|
||||
|
||||
func emptyLogDash(s string) string {
|
||||
s = strings.TrimSpace(s)
|
||||
if s == "" {
|
||||
return "-"
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
// truncateLogEnds returns a PII-safe preview: total rune length plus first/last
|
||||
// slices when the body is large (avoids megabyte log lines).
|
||||
func truncateLogEnds(s string, head, tail int) string {
|
||||
s = strings.TrimSpace(s)
|
||||
if s == "" {
|
||||
return "len=0"
|
||||
}
|
||||
runes := []rune(s)
|
||||
n := len(runes)
|
||||
if head < 0 {
|
||||
head = 0
|
||||
}
|
||||
if tail < 0 {
|
||||
tail = 0
|
||||
}
|
||||
if n <= head+tail+32 {
|
||||
return fmt.Sprintf("len=%d body=%q", n, s)
|
||||
}
|
||||
return fmt.Sprintf("len=%d head=%q tail=%q", n, string(runes[:head]), string(runes[n-tail:]))
|
||||
}
|
||||
|
||||
// formulaTemplateSummary summarizes a category title/description formula for logs
|
||||
// (section/element types + JSON byte length + short content hash — not a full dump).
|
||||
func formulaTemplateSummary(template any, kind string) string {
|
||||
if template == nil {
|
||||
return "none"
|
||||
}
|
||||
raw, err := json.Marshal(template)
|
||||
if err != nil || len(raw) == 0 || string(raw) == "null" || string(raw) == "{}" {
|
||||
return "none"
|
||||
}
|
||||
sum := sha256.Sum256(raw)
|
||||
hash8 := hex.EncodeToString(sum[:4])
|
||||
switch kind {
|
||||
case "title":
|
||||
els, sep, ok := parseTitleFormula(template)
|
||||
if !ok {
|
||||
return fmt.Sprintf("unparsed len=%d hash=%s", len(raw), hash8)
|
||||
}
|
||||
types := make([]string, 0, len(els))
|
||||
for _, el := range els {
|
||||
types = append(types, el.Type)
|
||||
}
|
||||
return fmt.Sprintf("elements=%s sep=%q len=%d hash=%s", strings.Join(types, "+"), sep, len(raw), hash8)
|
||||
case "description":
|
||||
sections, ok := parseDescriptionFormulaSections(template)
|
||||
if !ok {
|
||||
return fmt.Sprintf("unparsed len=%d hash=%s", len(raw), hash8)
|
||||
}
|
||||
types := make([]string, 0, len(sections))
|
||||
for _, sec := range sections {
|
||||
if t := strings.TrimSpace(sec.Type); t != "" {
|
||||
types = append(types, t)
|
||||
}
|
||||
}
|
||||
return fmt.Sprintf("sections=%s len=%d hash=%s", strings.Join(types, "+"), len(raw), hash8)
|
||||
default:
|
||||
return fmt.Sprintf("len=%d hash=%s", len(raw), hash8)
|
||||
}
|
||||
}
|
||||
|
||||
func finishReasonFromCompletion(comp Completion) string {
|
||||
m, ok := comp.Raw.(map[string]any)
|
||||
if !ok || m == nil {
|
||||
return ""
|
||||
}
|
||||
fr, _ := m["finish_reason"].(string)
|
||||
return strings.TrimSpace(fr)
|
||||
}
|
||||
|
||||
func enhanceFormulaOverride(in ProductInput) bool {
|
||||
return FormatDescriptionFormulaConstraint(in.DescriptionTemplate) != "" ||
|
||||
FormatTitleFormulaConstraint(in.TitleTemplate) != ""
|
||||
}
|
||||
|
||||
// logEnhancePrompt emits one structured line before the LLM call (truncated prompts).
|
||||
func logEnhancePrompt(in ProductInput, categoryUID, categoryName, system, user string) {
|
||||
log.Printf("processing: ai_enhance prompt job=%s company=%s raw=%s category_uid=%q category_name=%q title_formula=%s desc_formula=%s formula_override=%t system=%s user=%s",
|
||||
emptyLogDash(in.JobID), emptyLogDash(in.CompanyID), emptyLogDash(in.RawProductID),
|
||||
strings.TrimSpace(categoryUID), strings.TrimSpace(categoryName),
|
||||
formulaTemplateSummary(in.TitleTemplate, "title"),
|
||||
formulaTemplateSummary(in.DescriptionTemplate, "description"),
|
||||
enhanceFormulaOverride(in),
|
||||
truncateLogEnds(system, enhanceLogHeadTail, enhanceLogHeadTail),
|
||||
truncateLogEnds(user, enhanceLogHeadTail, enhanceLogHeadTail),
|
||||
)
|
||||
}
|
||||
|
||||
// logEnhanceOutcome emits one structured line for the enhance result (truncated response).
|
||||
func logEnhanceOutcome(in ProductInput, categoryUID, categoryName, outcome, reason string, comp Completion, elapsed time.Duration, forceReason string) {
|
||||
log.Printf("processing: ai_enhance outcome=%s reason=%s job=%s company=%s raw=%s category_uid=%q category_name=%q title_formula=%s desc_formula=%s formula_override=%t finish_reason=%s prompt_tokens=%d completion_tokens=%d total_tokens=%d elapsed=%s response=%s force_reason=%s",
|
||||
emptyLogDash(outcome), emptyLogDash(reason),
|
||||
emptyLogDash(in.JobID), emptyLogDash(in.CompanyID), emptyLogDash(in.RawProductID),
|
||||
strings.TrimSpace(categoryUID), strings.TrimSpace(categoryName),
|
||||
formulaTemplateSummary(in.TitleTemplate, "title"),
|
||||
formulaTemplateSummary(in.DescriptionTemplate, "description"),
|
||||
enhanceFormulaOverride(in),
|
||||
emptyLogDash(finishReasonFromCompletion(comp)),
|
||||
comp.PromptTokens, comp.OutputTokens, comp.TotalTokens,
|
||||
elapsed.Round(time.Millisecond),
|
||||
truncateLogEnds(comp.Text, enhanceLogHeadTail, enhanceLogHeadTail),
|
||||
emptyLogDash(forceReason),
|
||||
)
|
||||
}
|
||||
@@ -0,0 +1,103 @@
|
||||
package processing
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestTruncateLogEnds(t *testing.T) {
|
||||
t.Parallel()
|
||||
short := truncateLogEnds("hello", 500, 500)
|
||||
if !strings.Contains(short, `body="hello"`) || !strings.Contains(short, "len=5") {
|
||||
t.Fatalf("short: %s", short)
|
||||
}
|
||||
long := strings.Repeat("a", 600) + "MID" + strings.Repeat("b", 600)
|
||||
got := truncateLogEnds(long, 10, 10)
|
||||
if !strings.Contains(got, "len=1203") || !strings.Contains(got, "head=") || !strings.Contains(got, "tail=") {
|
||||
t.Fatalf("long: %s", got)
|
||||
}
|
||||
if strings.Contains(got, "MID") {
|
||||
t.Fatalf("middle should be omitted: %s", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFormulaTemplateSummary(t *testing.T) {
|
||||
t.Parallel()
|
||||
if formulaTemplateSummary(nil, "description") != "none" {
|
||||
t.Fatal("nil → none")
|
||||
}
|
||||
desc := map[string]any{
|
||||
"sections": []any{
|
||||
map[string]any{"type": "h1", "instructions": "Heading"},
|
||||
map[string]any{"type": "p", "instructions": "Body"},
|
||||
map[string]any{"type": "ul", "instructions": "Specs"},
|
||||
},
|
||||
}
|
||||
sum := formulaTemplateSummary(desc, "description")
|
||||
if !strings.Contains(sum, "sections=h1+p+ul") || !strings.Contains(sum, "len=") || !strings.Contains(sum, "hash=") {
|
||||
t.Fatalf("desc summary: %s", sum)
|
||||
}
|
||||
title := map[string]any{
|
||||
"separator": " ",
|
||||
"elements": []any{
|
||||
map[string]any{"type": "variable", "value": "brand"},
|
||||
map[string]any{"type": "text", "value": "TV"},
|
||||
},
|
||||
}
|
||||
ts := formulaTemplateSummary(title, "title")
|
||||
if !strings.Contains(ts, "elements=variable+text") || !strings.Contains(ts, `sep=" "`) {
|
||||
t.Fatalf("title summary: %s", ts)
|
||||
}
|
||||
}
|
||||
|
||||
// TestA1StyleDescriptionFormulaInResolvedPrompts asserts a realistic multi-section
|
||||
// category description_template (A1-style) is injected into both system and user
|
||||
// enhance prompts via resolveProductPromptTemplates / RenderProductEnhancePrompts.
|
||||
func TestA1StyleDescriptionFormulaInResolvedPrompts(t *testing.T) {
|
||||
t.Parallel()
|
||||
// Representative A1 cooker/monitor style: h1 + p + ul sections.
|
||||
descFormula := map[string]any{
|
||||
"sections": []any{
|
||||
map[string]any{"type": "h1", "instructions": "Naziv izdelka"},
|
||||
map[string]any{"type": "h2", "instructions": "Podnaslov prednosti"},
|
||||
map[string]any{"type": "p", "instructions": "Kratek opis v slovenščini"},
|
||||
map[string]any{"type": "ul", "instructions": "Ključne specifikacije"},
|
||||
},
|
||||
}
|
||||
titleFormula := map[string]any{
|
||||
"separator": " ",
|
||||
"elements": []any{
|
||||
map[string]any{"type": "variable", "value": "brand"},
|
||||
map[string]any{"type": "variable", "value": "product_model"},
|
||||
},
|
||||
}
|
||||
sysTpl, userTpl := resolveProductPromptTemplates(ProductInput{
|
||||
EnhanceSystemTemplate: "You write retail copy. description: 1-2 factual sentences.",
|
||||
EnhanceUserTemplate: "Category: {{category}}\nName: {{name}}\nAttrs: {{attrs}}",
|
||||
TitleTemplate: titleFormula,
|
||||
DescriptionTemplate: descFormula,
|
||||
})
|
||||
if !strings.Contains(sysTpl, "Description formula") {
|
||||
t.Fatalf("system missing formula override: %s", sysTpl)
|
||||
}
|
||||
if !strings.Contains(userTpl, "Description formula") || !strings.Contains(userTpl, "- h1: Naziv izdelka") || !strings.Contains(userTpl, "- h2: Podnaslov prednosti") {
|
||||
t.Fatalf("user missing description formula: %s", userTpl)
|
||||
}
|
||||
if !strings.Contains(userTpl, "Title formula") || !strings.Contains(userTpl, "attr [brand]") {
|
||||
t.Fatalf("user missing title formula: %s", userTpl)
|
||||
}
|
||||
system, user := RenderProductEnhancePrompts(
|
||||
sysTpl, userTpl,
|
||||
"Štedilniki", "VOX EHT6020", "", "", "", "sl",
|
||||
map[string]any{"brand": "VOX", "product_model": "EHT6020"},
|
||||
)
|
||||
if !strings.Contains(user, "Description formula") || !strings.Contains(user, "- h1: Naziv izdelka") {
|
||||
t.Fatalf("rendered user missing formula: %s", user)
|
||||
}
|
||||
if !strings.Contains(user, "Slovenian") && !strings.Contains(system, "Slovenian") {
|
||||
t.Fatalf("language not rendered into prompts: sys=%s user=%s", system, user)
|
||||
}
|
||||
if !enhanceFormulaOverride(ProductInput{DescriptionTemplate: descFormula}) {
|
||||
t.Fatal("formula_override should be true")
|
||||
}
|
||||
}
|
||||
@@ -5,6 +5,8 @@ import (
|
||||
"fmt"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/descrybe/descrybe-v2/apps/api/internal/company"
|
||||
)
|
||||
|
||||
// AppendFormulaConstraints appends language-agnostic title/description formula
|
||||
@@ -118,6 +120,24 @@ func AppendDescriptionFormulaSystemOverride(systemTpl string, descriptionTemplat
|
||||
return systemTpl + "\n" + descriptionFormulaSystemOverride
|
||||
}
|
||||
|
||||
// descriptionSatisfiesFormula reports whether desc includes the HTML tags required
|
||||
// by categories.description_template sections. No formula → always true.
|
||||
func descriptionSatisfiesFormula(desc string, template any) bool {
|
||||
sections, ok := parseDescriptionFormulaSections(template)
|
||||
if !ok || len(sections) == 0 {
|
||||
return true
|
||||
}
|
||||
desc = strings.TrimSpace(desc)
|
||||
if desc == "" || desc == "<nil>" {
|
||||
return false
|
||||
}
|
||||
types := make([]string, 0, len(sections))
|
||||
for _, s := range sections {
|
||||
types = append(types, s.Type)
|
||||
}
|
||||
return !company.DescriptionMissingFormulaHTMLTags(desc, types)
|
||||
}
|
||||
|
||||
type titleFormulaElement struct {
|
||||
Type string `json:"type"`
|
||||
Value string `json:"value"`
|
||||
|
||||
@@ -76,6 +76,36 @@ func TestAppendDescriptionFormulaSystemOverride(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestDescriptionSatisfiesFormula(t *testing.T) {
|
||||
t.Parallel()
|
||||
tpl := map[string]any{
|
||||
"sections": []any{
|
||||
map[string]any{"type": "h1", "instructions": "Heading"},
|
||||
map[string]any{"type": "p", "instructions": "Body"},
|
||||
map[string]any{"type": "ul", "instructions": "Specs"},
|
||||
},
|
||||
}
|
||||
if descriptionSatisfiesFormula("", tpl) {
|
||||
t.Fatal("empty must fail")
|
||||
}
|
||||
if descriptionSatisfiesFormula("GIGABYTE GS27QC is a gaming monitor with a 61 cm curved panel for immersive play.", tpl) {
|
||||
t.Fatal("plain prose must fail multi-section formula")
|
||||
}
|
||||
if !descriptionSatisfiesFormula("<h1>GIGABYTE GS27QC</h1><p>Gaming monitor.</p><ul><li>61 cm</li></ul>", tpl) {
|
||||
t.Fatal("full HTML skeleton must pass")
|
||||
}
|
||||
if !descriptionSatisfiesFormula("any copy", nil) {
|
||||
t.Fatal("nil formula always satisfied")
|
||||
}
|
||||
if descriptionNeedsEnhanceRepair(
|
||||
"GIGABYTE GS27QC is a gaming monitor with a 61 cm curved panel for immersive play.",
|
||||
tpl,
|
||||
"GIGABYTE GS27QC",
|
||||
) != true {
|
||||
t.Fatal("non-weak prose that ignores formula must need repair")
|
||||
}
|
||||
}
|
||||
|
||||
func TestAppendFormulaConstraints_injectedIntoResolve(t *testing.T) {
|
||||
t.Parallel()
|
||||
title := map[string]any{
|
||||
|
||||
@@ -8,21 +8,25 @@ import (
|
||||
"strings"
|
||||
)
|
||||
|
||||
// Local / weak-model defaults (8k-class context). See docs/local-llm-tuning.md.
|
||||
// Local / weak-model defaults. See docs/local-llm-tuning.md.
|
||||
const (
|
||||
DefaultStructuredTemp = 0.2
|
||||
// Reasoning-capable OpenAI-compatible models (e.g. code-fast / Qwen3) spend
|
||||
// hundreds–thousands of tokens in reasoning_content before writing message.content.
|
||||
// 350 capped mid-thought → empty content → "empty model response".
|
||||
MaxTokensEnhance = 4096
|
||||
MaxTokensSEO = 180
|
||||
MaxTokensCampaign = 650
|
||||
MaxProductDescRunes = 400
|
||||
MaxAttrKeys = 10
|
||||
MaxAttrValueRunes = 60
|
||||
MaxBrandInjectRunes = 500
|
||||
MaxCampaignProducts = 8
|
||||
MaxCampaignNameRunes = 80
|
||||
// Reasoning-capable OpenAI-compatible models (e.g. OverloadedBot code-fast)
|
||||
// spend thousands of tokens in reasoning_content before message.content.
|
||||
// Formula HTML JSON often needs a large completion budget; 4096 hit
|
||||
// finish_reason=length with empty/truncated content in live enhance.
|
||||
MaxTokensEnhance = 16384
|
||||
// MaxTokensEnhanceRetry is the one-shot length-cap bump ceiling used by
|
||||
// OpenAIClient when finish_reason=length yields empty or unparseable JSON.
|
||||
MaxTokensEnhanceRetry = 32768
|
||||
MaxTokensSEO = 180
|
||||
MaxTokensCampaign = 650
|
||||
MaxProductDescRunes = 400
|
||||
MaxAttrKeys = 10
|
||||
MaxAttrValueRunes = 60
|
||||
MaxBrandInjectRunes = 500
|
||||
MaxCampaignProducts = 8
|
||||
MaxCampaignNameRunes = 80
|
||||
)
|
||||
|
||||
// CompleteOptions tunes a single chat completion for structured tasks.
|
||||
|
||||
@@ -256,11 +256,14 @@ type chatResponse struct {
|
||||
} `json:"error"`
|
||||
}
|
||||
|
||||
var errEmptyModelResponse = errors.New("empty model response")
|
||||
var (
|
||||
errEmptyModelResponse = errors.New("empty model response")
|
||||
errLengthCappedResponse = errors.New("response truncated at max_tokens")
|
||||
)
|
||||
|
||||
// maxTokensReasoningBudget is used when a capped completion returns empty
|
||||
// content with finish_reason=length (reasoning models).
|
||||
const maxTokensReasoningBudget = 4096
|
||||
func isLengthBudgetErr(err error) bool {
|
||||
return errors.Is(err, errEmptyModelResponse) || errors.Is(err, errLengthCappedResponse)
|
||||
}
|
||||
|
||||
func (c *OpenAIClient) Complete(ctx context.Context, system, user string) (Completion, error) {
|
||||
return c.CompleteWithOptions(ctx, system, user, CompleteOptions{})
|
||||
@@ -300,16 +303,19 @@ func (c *OpenAIClient) CompleteWithOptions(ctx context.Context, system, user str
|
||||
return comp, nil
|
||||
}
|
||||
lastErr = err
|
||||
if errors.Is(err, errEmptyModelResponse) {
|
||||
if maxTok <= 0 || maxTok < maxTokensReasoningBudget {
|
||||
if maxTok < maxTokensReasoningBudget {
|
||||
maxTok = maxTokensReasoningBudget
|
||||
if isLengthBudgetErr(err) {
|
||||
if maxTok <= 0 || maxTok < MaxTokensEnhanceRetry {
|
||||
prev := maxTok
|
||||
if maxTok < MaxTokensEnhanceRetry {
|
||||
maxTok = MaxTokensEnhanceRetry
|
||||
}
|
||||
log.Printf("openai: length-cap retry model=%s prev_max_tokens=%d next_max_tokens=%d err=%s",
|
||||
c.Model, prev, maxTok, TruncateError(err))
|
||||
retryable = true
|
||||
} else {
|
||||
// Already at reasoning budget — another 240s call will not help.
|
||||
// Already at enhance retry ceiling — another long call will not help.
|
||||
return Completion{}, fmt.Errorf(
|
||||
"openai empty response at max_tokens=%d for model %q (try a faster non-reasoning model): %w",
|
||||
"openai length-capped at max_tokens=%d for model %q (prefer a faster non-reasoning product-enhance model): %w",
|
||||
maxTok, c.Model, err)
|
||||
}
|
||||
}
|
||||
@@ -488,34 +494,41 @@ func (c *OpenAIClient) doComplete(ctx context.Context, system, user string, temp
|
||||
|
||||
stopWait := c.startWaitLogger("chat/completions", maxTokens)
|
||||
res, err := c.HTTPClient.Do(req)
|
||||
stopWait(err)
|
||||
if err != nil {
|
||||
stopWait(err)
|
||||
wrapped, retryable := classifyOpenAITransportErr(c.Model, err)
|
||||
return Completion{}, retryable, wrapped
|
||||
}
|
||||
defer res.Body.Close()
|
||||
raw, err := io.ReadAll(io.LimitReader(res.Body, 1<<20))
|
||||
if err != nil {
|
||||
stopWait(err)
|
||||
wrapped, retryable := classifyOpenAITransportErr(c.Model, err)
|
||||
return Completion{}, retryable, wrapped
|
||||
}
|
||||
var parsed chatResponse
|
||||
if err := json.Unmarshal(raw, &parsed); err != nil {
|
||||
return Completion{}, false, fmt.Errorf("openai decode: %w", err)
|
||||
decErr := fmt.Errorf("openai decode: %w", err)
|
||||
stopWait(decErr)
|
||||
return Completion{}, false, decErr
|
||||
}
|
||||
if res.StatusCode == http.StatusTooManyRequests || res.StatusCode >= 500 {
|
||||
msg := "rate limited or server error"
|
||||
if parsed.Error != nil && parsed.Error.Message != "" {
|
||||
msg = TruncateError(errors.New(parsed.Error.Message))
|
||||
}
|
||||
return Completion{}, true, errors.New(msg)
|
||||
httpErr := errors.New(msg)
|
||||
stopWait(httpErr)
|
||||
return Completion{}, true, httpErr
|
||||
}
|
||||
if res.StatusCode >= 400 {
|
||||
msg := fmt.Sprintf("openai http %d", res.StatusCode)
|
||||
if parsed.Error != nil && parsed.Error.Message != "" {
|
||||
msg = TruncateError(errors.New(parsed.Error.Message))
|
||||
}
|
||||
return Completion{}, false, errors.New(msg)
|
||||
httpErr := errors.New(msg)
|
||||
stopWait(httpErr)
|
||||
return Completion{}, false, httpErr
|
||||
}
|
||||
text := ""
|
||||
finishReason := ""
|
||||
@@ -523,18 +536,44 @@ func (c *OpenAIClient) doComplete(ctx context.Context, system, user string, temp
|
||||
finishReason = strings.TrimSpace(parsed.Choices[0].FinishReason)
|
||||
text = SanitizeOutput(choiceMessageText(parsed.Choices[0].Message.Content, parsed.Choices[0].Message.ReasoningContent, parsed.Choices[0].Message.Reasoning))
|
||||
}
|
||||
usagePrompt := parsed.Usage.PromptTokens
|
||||
usageOut := parsed.Usage.CompletionTokens
|
||||
usageTotal := parsed.Usage.TotalTokens
|
||||
canBump := maxTokens <= 0 || maxTokens < MaxTokensEnhanceRetry
|
||||
lengthCapped := strings.EqualFold(finishReason, "length")
|
||||
if text == "" {
|
||||
// Reasoning models often return empty content when max_tokens cuts mid-thought.
|
||||
// Only retry when CompleteWithOptions can still raise max_tokens.
|
||||
canBump := maxTokens <= 0 || maxTokens < maxTokensReasoningBudget
|
||||
retryable := strings.EqualFold(finishReason, "length") && canBump
|
||||
// Never log HTTP-ok as ok=1 when content is empty (prod looked "successful" at 30–272ms).
|
||||
retryable := lengthCapped && canBump
|
||||
emptyErr := fmt.Errorf("%w (finish_reason=%s max_tokens=%d prompt_tokens=%d completion_tokens=%d)",
|
||||
errEmptyModelResponse, finishReason, maxTokens, usagePrompt, usageOut)
|
||||
stopWait(emptyErr)
|
||||
log.Printf("openai: chat content empty model=%s finish_reason=%s max_tokens=%d prompt_tokens=%d completion_tokens=%d total_tokens=%d retryable=%t",
|
||||
c.Model, finishReason, maxTokens, usagePrompt, usageOut, usageTotal, retryable)
|
||||
return Completion{}, retryable, errEmptyModelResponse
|
||||
}
|
||||
if lengthCapped {
|
||||
// Truncated JSON was previously treated as success → parse_failed → synthesize.
|
||||
// Accept only when the truncated body still parses as a JSON object.
|
||||
if _, err := ParseJSONObject(text); err != nil {
|
||||
retryable := canBump
|
||||
truncErr := fmt.Errorf("%w (finish_reason=length max_tokens=%d prompt_tokens=%d completion_tokens=%d content_runes=%d)",
|
||||
errLengthCappedResponse, maxTokens, usagePrompt, usageOut, len([]rune(text)))
|
||||
stopWait(truncErr)
|
||||
log.Printf("openai: chat content truncated model=%s finish_reason=length max_tokens=%d prompt_tokens=%d completion_tokens=%d total_tokens=%d content_runes=%d retryable=%t",
|
||||
c.Model, maxTokens, usagePrompt, usageOut, usageTotal, len([]rune(text)), retryable)
|
||||
return Completion{}, retryable, errLengthCappedResponse
|
||||
}
|
||||
}
|
||||
stopWait(nil)
|
||||
log.Printf("openai: chat content ok model=%s finish_reason=%s content_runes=%d prompt_tokens=%d completion_tokens=%d total_tokens=%d",
|
||||
c.Model, finishReason, len([]rune(text)), usagePrompt, usageOut, usageTotal)
|
||||
return Completion{
|
||||
Text: text,
|
||||
PromptTokens: parsed.Usage.PromptTokens,
|
||||
OutputTokens: parsed.Usage.CompletionTokens,
|
||||
TotalTokens: parsed.Usage.TotalTokens,
|
||||
PromptTokens: usagePrompt,
|
||||
OutputTokens: usageOut,
|
||||
TotalTokens: usageTotal,
|
||||
Model: parsed.Model,
|
||||
Raw: map[string]any{
|
||||
"model": parsed.Model,
|
||||
@@ -678,7 +717,7 @@ func inventHeuristicDescription(system, user, name string) string {
|
||||
name = "Product"
|
||||
}
|
||||
cat := labeledPromptValue(user, "category:")
|
||||
if isPromptLabelTitle(cat) {
|
||||
if isUnusableCategoryValue(cat, name) {
|
||||
cat = ""
|
||||
}
|
||||
lang := languageCodeFromEnhancePrompt(system, user)
|
||||
|
||||
@@ -201,8 +201,8 @@ func TestOpenAIClient_doComplete_emptyContentLengthRetriesWithBudget(t *testing.
|
||||
})
|
||||
return
|
||||
}
|
||||
if maxTok != float64(maxTokensReasoningBudget) {
|
||||
t.Errorf("retry max_tokens=%v want %d", maxTok, maxTokensReasoningBudget)
|
||||
if maxTok != float64(MaxTokensEnhanceRetry) {
|
||||
t.Errorf("retry max_tokens=%v want %d", maxTok, MaxTokensEnhanceRetry)
|
||||
}
|
||||
_ = json.NewEncoder(w).Encode(map[string]any{
|
||||
"model": "code-fast",
|
||||
@@ -228,6 +228,61 @@ func TestOpenAIClient_doComplete_emptyContentLengthRetriesWithBudget(t *testing.
|
||||
}
|
||||
}
|
||||
|
||||
func TestOpenAIClient_doComplete_truncatedJSONLengthRetriesFromEnhanceBudget(t *testing.T) {
|
||||
t.Parallel()
|
||||
calls := 0
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
calls++
|
||||
var req map[string]any
|
||||
_ = json.NewDecoder(r.Body).Decode(&req)
|
||||
maxTok, _ := req["max_tokens"].(float64)
|
||||
if calls == 1 {
|
||||
if maxTok != float64(MaxTokensEnhance) {
|
||||
t.Errorf("first max_tokens=%v want %d", maxTok, MaxTokensEnhance)
|
||||
}
|
||||
_ = json.NewEncoder(w).Encode(map[string]any{
|
||||
"model": "code-fast",
|
||||
"choices": []map[string]any{{
|
||||
"finish_reason": "length",
|
||||
"message": map[string]any{
|
||||
"role": "assistant",
|
||||
"content": `{"name":"Lenovo","description":"<h1>Lenovo G27-20: Igralski monitor za vrhunsko vizualno izkušn`,
|
||||
},
|
||||
}},
|
||||
"usage": map[string]int{"prompt_tokens": 200, "completion_tokens": MaxTokensEnhance, "total_tokens": 200 + MaxTokensEnhance},
|
||||
})
|
||||
return
|
||||
}
|
||||
if maxTok != float64(MaxTokensEnhanceRetry) {
|
||||
t.Errorf("retry max_tokens=%v want %d", maxTok, MaxTokensEnhanceRetry)
|
||||
}
|
||||
_ = json.NewEncoder(w).Encode(map[string]any{
|
||||
"model": "code-fast",
|
||||
"choices": []map[string]any{{
|
||||
"finish_reason": "stop",
|
||||
"message": map[string]any{
|
||||
"role": "assistant",
|
||||
"content": `{"name":"Lenovo G27-20","description":"<h1>Lenovo G27-20</h1><p>Monitor.</p>"}`,
|
||||
},
|
||||
}},
|
||||
"usage": map[string]int{"prompt_tokens": 200, "completion_tokens": 80, "total_tokens": 280},
|
||||
})
|
||||
}))
|
||||
defer srv.Close()
|
||||
c := NewOpenAIClient("test-key", srv.URL, "code-fast", 0, 2)
|
||||
c.HTTPClient = srv.Client()
|
||||
comp, err := c.CompleteWithOptions(context.Background(), "sys", "user", CompleteOptions{MaxTokens: MaxTokensEnhance, Temperature: 0.2})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if calls != 2 {
|
||||
t.Fatalf("calls=%d want 2 (length truncated must not succeed)", calls)
|
||||
}
|
||||
if !strings.Contains(comp.Text, "</h1>") {
|
||||
t.Fatalf("text=%q", comp.Text)
|
||||
}
|
||||
}
|
||||
|
||||
func TestOpenAIClient_Complete_timeoutNotRetried(t *testing.T) {
|
||||
calls := 0
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
@@ -269,7 +324,7 @@ func TestOpenAIClient_Complete_timeoutNotRetried(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestOpenAIClient_Complete_emptyAtReasoningBudgetNotRetried(t *testing.T) {
|
||||
func TestOpenAIClient_Complete_emptyAtEnhanceRetryBudgetNotRetried(t *testing.T) {
|
||||
t.Parallel()
|
||||
calls := 0
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
@@ -280,21 +335,53 @@ func TestOpenAIClient_Complete_emptyAtReasoningBudgetNotRetried(t *testing.T) {
|
||||
"finish_reason": "length",
|
||||
"message": map[string]any{"role": "assistant", "content": "", "reasoning_content": "still thinking"},
|
||||
}},
|
||||
"usage": map[string]int{"prompt_tokens": 1, "completion_tokens": 4096, "total_tokens": 4097},
|
||||
"usage": map[string]int{"prompt_tokens": 1, "completion_tokens": MaxTokensEnhanceRetry, "total_tokens": 1 + MaxTokensEnhanceRetry},
|
||||
})
|
||||
}))
|
||||
defer srv.Close()
|
||||
c := NewOpenAIClient("test-key", srv.URL, "code-fast", 0, 3)
|
||||
c.HTTPClient = srv.Client()
|
||||
_, err := c.CompleteWithOptions(context.Background(), "sys", "user", CompleteOptions{MaxTokens: maxTokensReasoningBudget})
|
||||
_, err := c.CompleteWithOptions(context.Background(), "sys", "user", CompleteOptions{MaxTokens: MaxTokensEnhanceRetry})
|
||||
if err == nil {
|
||||
t.Fatal("expected empty response error")
|
||||
t.Fatal("expected length-capped error")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "empty response") {
|
||||
if !strings.Contains(err.Error(), "length-capped") {
|
||||
t.Fatalf("err=%v", err)
|
||||
}
|
||||
if calls != 1 {
|
||||
t.Fatalf("calls=%d want 1 (no retry at reasoning budget)", calls)
|
||||
t.Fatalf("calls=%d want 1 (no retry at enhance retry ceiling)", calls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestOpenAIClient_Complete_truncatedAtEnhanceRetryBudgetNotSuccess(t *testing.T) {
|
||||
t.Parallel()
|
||||
calls := 0
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
calls++
|
||||
_ = json.NewEncoder(w).Encode(map[string]any{
|
||||
"model": "code-fast",
|
||||
"choices": []map[string]any{{
|
||||
"finish_reason": "length",
|
||||
"message": map[string]any{
|
||||
"role": "assistant",
|
||||
"content": `{"name":"X","description":"<h1>cut off`,
|
||||
},
|
||||
}},
|
||||
"usage": map[string]int{"prompt_tokens": 10, "completion_tokens": MaxTokensEnhanceRetry, "total_tokens": 10 + MaxTokensEnhanceRetry},
|
||||
})
|
||||
}))
|
||||
defer srv.Close()
|
||||
c := NewOpenAIClient("test-key", srv.URL, "code-fast", 0, 3)
|
||||
c.HTTPClient = srv.Client()
|
||||
_, err := c.CompleteWithOptions(context.Background(), "sys", "user", CompleteOptions{MaxTokens: MaxTokensEnhanceRetry})
|
||||
if err == nil {
|
||||
t.Fatal("truncated JSON at ceiling must not succeed")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "length-capped") {
|
||||
t.Fatalf("err=%v", err)
|
||||
}
|
||||
if calls != 1 {
|
||||
t.Fatalf("calls=%d want 1", calls)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -1522,6 +1522,9 @@ func (p *Pipeline) processOne(ctx context.Context, companyID, jobID uuid.UUID, i
|
||||
CategoryNamesByUID: categoryNamesByUID,
|
||||
AllowedAttrKeys: allowedAttrKeysForCategory(cache, stringFromAny(enriched["category"])),
|
||||
CategoryAttrKeys: categoryAttrKeysFromCache(cache),
|
||||
JobID: jobID.String(),
|
||||
CompanyID: companyID.String(),
|
||||
RawProductID: it.RawID.String(),
|
||||
}
|
||||
if it.hydrated {
|
||||
if it.hasPrior {
|
||||
@@ -1577,6 +1580,8 @@ func (p *Pipeline) processOne(ctx context.Context, companyID, jobID uuid.UUID, i
|
||||
jobID, it.RawID, catPreFilter)
|
||||
}
|
||||
syncCategoryName(&result, cache.categoryNamesByUID)
|
||||
scrubCategoryPollution(&result, cache.categoryNamesByUID, cache.categoryUniqueIDs)
|
||||
syncCategoryName(&result, cache.categoryNamesByUID)
|
||||
}
|
||||
// Re-apply after category validation so allowlist matches the persisted category.
|
||||
allowed := allowedAttrKeysForCategory(cache, result.Category)
|
||||
@@ -1676,7 +1681,10 @@ const upsertProcessedProductSQL = `
|
||||
ON CONFLICT (company_id, raw_product_id) DO UPDATE SET
|
||||
product_id = EXCLUDED.product_id,
|
||||
name = EXCLUDED.name,
|
||||
category = COALESCE(NULLIF(BTRIM(EXCLUDED.category), ''), processed_products.category),
|
||||
category = CASE
|
||||
WHEN COALESCE(EXCLUDED.field_sources->>'category', '') = 'cleared_invalid' THEN ''
|
||||
ELSE COALESCE(NULLIF(BTRIM(EXCLUDED.category), ''), processed_products.category)
|
||||
END,
|
||||
description = EXCLUDED.description,
|
||||
processed_name = EXCLUDED.processed_name,
|
||||
processed_description = EXCLUDED.processed_description,
|
||||
|
||||
@@ -4,8 +4,10 @@ import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"log"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/descrybe/descrybe-v2/apps/api/internal/company"
|
||||
"github.com/descrybe/descrybe-v2/apps/api/internal/eprel"
|
||||
@@ -215,6 +217,7 @@ func (e *Engine) RunSteps(ctx context.Context, companyID string, in ProductInput
|
||||
out.ProcessedName = out.Name
|
||||
out.ProcessedDescription = out.Description
|
||||
out.Notes = append(out.Notes, "ai_enhance: skipped (Free plan — upgrade for AI titles/descriptions)")
|
||||
log.Printf("processing: ai_enhance skip reason=entitlement_can_use_ai")
|
||||
preservePriorEnhanceHash()
|
||||
appendStepLog(out.GPTResponse, StepAIEnhance, map[string]any{
|
||||
"status": "skipped",
|
||||
@@ -226,6 +229,7 @@ func (e *Engine) RunSteps(ctx context.Context, companyID string, in ProductInput
|
||||
out.ProcessedName = out.Name
|
||||
out.ProcessedDescription = out.Description
|
||||
out.Notes = append(out.Notes, "ai_enhance: skipped (platform OpenAI unset; configure admin settings or company BYOK)")
|
||||
log.Printf("processing: ai_enhance skip reason=openai_not_configured")
|
||||
preservePriorEnhanceHash()
|
||||
appendStepLog(out.GPTResponse, StepAIEnhance, map[string]any{
|
||||
"status": "skipped",
|
||||
@@ -298,6 +302,10 @@ func (e *Engine) RunSteps(ctx context.Context, companyID string, in ProductInput
|
||||
PriorEnhanceHash: priorHash,
|
||||
PriorProcessedName: priorName,
|
||||
PriorProcessedDescription: priorDesc,
|
||||
JobID: in.JobID,
|
||||
CompanyID: in.CompanyID,
|
||||
RawProductID: in.RawProductID,
|
||||
CategoryUniqueID: out.Category,
|
||||
}, displayCat, enhanceAttrs)
|
||||
out.TotalTokens += tokens
|
||||
status := enhanceStatusFromMeta(raw)
|
||||
@@ -318,6 +326,15 @@ func (e *Engine) RunSteps(ctx context.Context, companyID string, in ProductInput
|
||||
}
|
||||
} else if status == "unchanged" {
|
||||
meta["status"] = "unchanged"
|
||||
} else if status == "refused" || status == "synthesized" || status == "parse_failed" {
|
||||
// Soft quality failure: keep usable copy / synth, never count as real enhance ok.
|
||||
allUnchanged = false
|
||||
meta["status"] = status
|
||||
if reason := enhanceReasonFromMeta(raw); reason != "" {
|
||||
out.Notes = append(out.Notes, "ai_enhance: "+status+" ("+reason+")")
|
||||
} else {
|
||||
out.Notes = append(out.Notes, "ai_enhance: "+status)
|
||||
}
|
||||
} else {
|
||||
allUnchanged = false
|
||||
anyOK = true
|
||||
@@ -327,13 +344,13 @@ func (e *Engine) RunSteps(ctx context.Context, companyID string, in ProductInput
|
||||
name = preferredProductTitle(in.GTIN, name, out.Name, priorName, in.Name)
|
||||
desc = preferredProductDescription(name, desc, out.Description, priorDesc)
|
||||
outerSynth := false
|
||||
if isWeakPriorEnhanceDescription(desc, name) {
|
||||
if descriptionNeedsEnhanceRepair(desc, descTpl, name) {
|
||||
if synth := synthesizeProductDescription(name, displayCat, lang, enhanceAttrs, descTpl); synth != "" {
|
||||
desc = synth
|
||||
outerSynth = true
|
||||
}
|
||||
}
|
||||
weakDesc := isWeakPriorEnhanceDescription(desc, name)
|
||||
weakDesc := descriptionNeedsEnhanceRepair(desc, descTpl, name)
|
||||
// Only persist enhance_input_hash for quality ok / hash-skip unchanged.
|
||||
// Never copy input_hash from error/passthrough/thin/synthesized meta.
|
||||
persistHash := ""
|
||||
@@ -341,7 +358,7 @@ func (e *Engine) RunSteps(ctx context.Context, companyID string, in ProductInput
|
||||
switch {
|
||||
case err != nil:
|
||||
persistHash = ""
|
||||
case status == "synthesized" || outerSynth:
|
||||
case status == "synthesized" || status == "refused" || outerSynth:
|
||||
persistHash = ""
|
||||
allUnchanged = false
|
||||
case status == "unchanged" && !weakDesc:
|
||||
@@ -349,7 +366,7 @@ func (e *Engine) RunSteps(ctx context.Context, companyID string, in ProductInput
|
||||
case status == "ok" && !weakDesc:
|
||||
persistHash = rawHash
|
||||
default:
|
||||
// thin ok / parse_failed / skipped — do not poison reprocess skip
|
||||
// thin ok / parse_failed / formula mismatch / skipped — do not poison reprocess skip
|
||||
persistHash = ""
|
||||
if status == "ok" && weakDesc {
|
||||
allUnchanged = false
|
||||
@@ -404,13 +421,12 @@ func (e *Engine) RunSteps(ctx context.Context, companyID string, in ProductInput
|
||||
if out.ProcessedDescription == "" {
|
||||
out.ProcessedDescription = out.Description
|
||||
}
|
||||
// Timeout/error/empty: always synthesize a factual fallback when a title exists.
|
||||
if out.ProcessedName != "" && (out.ProcessedDescription == "" ||
|
||||
isWeakPriorEnhanceDescription(out.ProcessedDescription, out.ProcessedName, out.Name)) {
|
||||
_, failDescTpl := categoryFormulasFor(in, out.Category)
|
||||
// Timeout/error/empty/formula-miss: always synthesize a factual fallback when a title exists.
|
||||
_, failDescTpl := categoryFormulasFor(in, out.Category)
|
||||
if out.ProcessedName != "" && descriptionNeedsEnhanceRepair(out.ProcessedDescription, failDescTpl, out.ProcessedName, out.Name) {
|
||||
if synth := synthesizeProductDescription(out.ProcessedName, displayCat, primary, enhanceAttrs, failDescTpl); synth != "" {
|
||||
out.ProcessedDescription = synth
|
||||
if out.Description == "" || isWeakPriorEnhanceDescription(out.Description, out.Name, out.ProcessedName) {
|
||||
if out.Description == "" || descriptionNeedsEnhanceRepair(out.Description, failDescTpl, out.Name, out.ProcessedName) {
|
||||
out.Description = synth
|
||||
}
|
||||
if lf, ok := localized[primary]; ok {
|
||||
@@ -453,11 +469,27 @@ func (e *Engine) RunSteps(ctx context.Context, companyID string, in ProductInput
|
||||
out.SkipCreditDebit = true
|
||||
out.FieldSources["name"] = "ai_enhance_unchanged"
|
||||
out.FieldSources["description"] = "ai_enhance_unchanged"
|
||||
out.Notes = append(out.Notes, "ai_enhance: skipped (inputs unchanged)")
|
||||
out.Notes = append(out.Notes, "ai_enhance_unchanged")
|
||||
log.Printf("processing: ai_enhance_unchanged")
|
||||
} else {
|
||||
out.AIProviderMode = e.EngineProviderMode()
|
||||
out.FieldSources["name"] = "ai_enhance"
|
||||
out.FieldSources["description"] = "ai_enhance"
|
||||
for _, m := range langMetas {
|
||||
if mm, ok := m.(map[string]any); ok {
|
||||
reason, _ := mm["reason"].(string)
|
||||
if reason == "" {
|
||||
if raw, ok := mm["raw"].(map[string]any); ok {
|
||||
reason, _ = raw["reason"].(string)
|
||||
}
|
||||
}
|
||||
if reason != "" {
|
||||
out.Notes = append(out.Notes, "ai_enhance: forced re-enhance ("+reason+")")
|
||||
log.Printf("processing: ai_enhance forced re-enhance reason=%s", reason)
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
appendStepLog(out.GPTResponse, StepAIEnhance, map[string]any{
|
||||
"status": map[string]any{"unchanged": allUnchanged, "ok": anyOK, "failed": anyFailed},
|
||||
@@ -475,7 +507,6 @@ func (e *Engine) RunSteps(ctx context.Context, companyID string, in ProductInput
|
||||
preserveCategoryIfEmpty(&out, in.PriorCategory)
|
||||
noteMissingCategory(&out, policy, e != nil && e.Vector != nil && e.Vector.Enabled())
|
||||
syncCategoryName(&out, in.CategoryNamesByUID)
|
||||
displayCat := categoryDisplayLabel(out)
|
||||
out.Name = preferredProductTitle(in.GTIN, out.Name, out.ProcessedName, in.Name, in.PriorProcessedName)
|
||||
out.ProcessedName = preferredProductTitle(in.GTIN, out.ProcessedName, out.Name, in.PriorProcessedName, in.Name)
|
||||
out.Description = preferredProductDescription(out.Name, out.Description, out.ProcessedDescription, in.Description)
|
||||
@@ -483,22 +514,52 @@ func (e *Engine) RunSteps(ctx context.Context, companyID string, in ProductInput
|
||||
if out.ProcessedName == "" {
|
||||
out.ProcessedName = out.Name
|
||||
}
|
||||
// Titles are finalized first so name-as-category and formula leakage can be cleared.
|
||||
scrubCategoryPollution(&out, in.CategoryNamesByUID, nil)
|
||||
syncCategoryName(&out, in.CategoryNamesByUID)
|
||||
displayCat := categoryDisplayLabel(out)
|
||||
// After vector/mapped category resolution, formulas may key for the first time —
|
||||
// repair short prose / title echo that ignored A1 description_template.
|
||||
_, finalDescTpl := categoryFormulasFor(in, out.Category)
|
||||
if out.ProcessedDescription == "" || isWeakPriorEnhanceDescription(out.ProcessedDescription, out.ProcessedName, out.Name) {
|
||||
finalRepaired := false
|
||||
if descriptionNeedsEnhanceRepair(out.ProcessedDescription, finalDescTpl, out.ProcessedName, out.Name) {
|
||||
if synth := synthesizeProductDescription(out.ProcessedName, displayCat, in.Language, out.Attributes, finalDescTpl); synth != "" {
|
||||
out.ProcessedDescription = synth
|
||||
finalRepaired = true
|
||||
}
|
||||
}
|
||||
if out.Description == "" || isWeakPriorEnhanceDescription(out.Description, out.Name, out.ProcessedName) {
|
||||
if out.ProcessedDescription != "" && !isWeakPriorEnhanceDescription(out.ProcessedDescription, out.Name, out.ProcessedName) {
|
||||
if descriptionNeedsEnhanceRepair(out.Description, finalDescTpl, out.Name, out.ProcessedName) {
|
||||
if out.ProcessedDescription != "" && !descriptionNeedsEnhanceRepair(out.ProcessedDescription, finalDescTpl, out.Name, out.ProcessedName) {
|
||||
out.Description = out.ProcessedDescription
|
||||
finalRepaired = true
|
||||
} else if synth := synthesizeProductDescription(out.Name, displayCat, in.Language, out.Attributes, finalDescTpl); synth != "" {
|
||||
out.Description = synth
|
||||
if out.ProcessedDescription == "" || isWeakPriorEnhanceDescription(out.ProcessedDescription, out.ProcessedName, out.Name) {
|
||||
finalRepaired = true
|
||||
if descriptionNeedsEnhanceRepair(out.ProcessedDescription, finalDescTpl, out.ProcessedName, out.Name) {
|
||||
out.ProcessedDescription = synth
|
||||
}
|
||||
}
|
||||
}
|
||||
if finalRepaired {
|
||||
delete(out.FieldSources, FieldEnhanceInputHash)
|
||||
primary := strings.TrimSpace(in.Language)
|
||||
if primary == "" && len(in.ContentLanguages) > 0 {
|
||||
primary = strings.TrimSpace(in.ContentLanguages[0])
|
||||
}
|
||||
if primary == "" {
|
||||
primary = company.DefaultLanguage
|
||||
}
|
||||
for lang, lf := range out.LocalizedContent {
|
||||
lf.EnhanceInputHash = ""
|
||||
if lang == primary {
|
||||
lf.ProcessedDescription = out.ProcessedDescription
|
||||
if lf.ProcessedName == "" {
|
||||
lf.ProcessedName = out.ProcessedName
|
||||
}
|
||||
}
|
||||
out.LocalizedContent[lang] = lf
|
||||
}
|
||||
}
|
||||
if out.Attributes == nil {
|
||||
out.Attributes = map[string]any{}
|
||||
}
|
||||
@@ -528,7 +589,7 @@ func preserveCategoryIfEmpty(out *StepResult, prior string) {
|
||||
return
|
||||
}
|
||||
prior = strings.TrimSpace(prior)
|
||||
if prior == "" {
|
||||
if prior == "" || isUnusableCategoryValue(prior, out.ProcessedName, out.Name) {
|
||||
return
|
||||
}
|
||||
out.Category = SanitizeText(prior)
|
||||
@@ -563,7 +624,7 @@ func tryVectorCategorize(ctx context.Context, e *Engine, companyID string, out *
|
||||
})
|
||||
return
|
||||
}
|
||||
if strings.TrimSpace(cat) == "" {
|
||||
if strings.TrimSpace(cat) == "" || isUnusableCategoryValue(cat, out.ProcessedName, out.Name) {
|
||||
return
|
||||
}
|
||||
out.Category = SanitizeOutput(cat)
|
||||
@@ -632,78 +693,252 @@ func appendStepLog(gpt map[string]any, name string, raw any) {
|
||||
gpt["steps"] = append(steps, map[string]any{"step": name, "raw": raw})
|
||||
}
|
||||
|
||||
// descriptionFormulaRetrySuffix is appended once when LLM JSON ignored the
|
||||
// category description_template (short prose / title echo instead of multi-section HTML).
|
||||
const descriptionFormulaRetrySuffix = "\n\nINVALID DESCRIPTION. Your JSON ignored the Description formula. Reply with ONLY one JSON object; \"description\" must be ONE HTML string covering each formula section in order with matching tags (h1/h2/h3/h4, p, ul)."
|
||||
|
||||
// descriptionNeedsEnhanceRepair is true when desc is weak/empty/title-echo or
|
||||
// fails an active category description_template (A1 multi-section HTML).
|
||||
// Heuristic invent/synth is intentionally excluded here — enhanceHashForceReason
|
||||
// blocks hash-skip for synth so reprocess can still upgrade to real LLM copy.
|
||||
func descriptionNeedsEnhanceRepair(desc string, template any, titles ...string) bool {
|
||||
if isWeakPriorEnhanceDescription(desc, titles...) {
|
||||
return true
|
||||
}
|
||||
return !descriptionSatisfiesFormula(desc, template)
|
||||
}
|
||||
|
||||
// enhanceHashForceReason returns why a matching prior enhance_input_hash must not
|
||||
// skip the LLM (empty = safe to reuse as ai_enhance_unchanged).
|
||||
func enhanceHashForceReason(in ProductInput) string {
|
||||
if isPromptLabelTitle(in.PriorProcessedName) {
|
||||
return "prompt_label_title"
|
||||
}
|
||||
if reason := company.EnhanceHashSkipBlockReason(in.PriorProcessedDescription, in.PriorProcessedName, in.Name); reason != "" {
|
||||
return reason
|
||||
}
|
||||
if !descriptionSatisfiesFormula(in.PriorProcessedDescription, in.DescriptionTemplate) {
|
||||
return "formula-mismatch"
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func (e *Engine) enhance(ctx context.Context, in ProductInput, category string, attrs map[string]any) (string, string, int, any, error) {
|
||||
catUID := strings.TrimSpace(in.CategoryUniqueID)
|
||||
catName := strings.TrimSpace(category)
|
||||
sysTpl, userTpl := resolveProductPromptTemplates(in)
|
||||
hash := HashEnhanceInput(category, in.Name, in.Description, in.BrandPrompt, in.Language, sysTpl, userTpl, attrs)
|
||||
if e == nil || e.Completer == nil {
|
||||
name := preferredProductTitle(in.GTIN, in.Name, in.PriorProcessedName)
|
||||
logEnhanceOutcome(in, catUID, catName, "skip", "completer_nil", Completion{}, 0, "")
|
||||
return name,
|
||||
preferredProductDescription(name, in.Description, in.PriorProcessedDescription),
|
||||
0, map[string]any{"status": "skipped", "input_hash": hash}, nil
|
||||
0, map[string]any{"status": "skipped", "reason": "completer_nil"}, nil
|
||||
}
|
||||
// Skip LLM when inputs match the last successful enhance (before any credit debit),
|
||||
// but never reuse a thin / title-echo prior description, or a prompt-leakage title.
|
||||
// but never reuse thin / title-echo / invent-synth / formula-mismatch priors,
|
||||
// or a prompt-leakage title.
|
||||
var forceReason string
|
||||
if in.PriorEnhanceHash != "" && in.PriorEnhanceHash == hash &&
|
||||
(in.PriorProcessedName != "" || in.PriorProcessedDescription != "") &&
|
||||
!isPromptLabelTitle(in.PriorProcessedName) &&
|
||||
!isWeakPriorEnhanceDescription(in.PriorProcessedDescription, in.PriorProcessedName, in.Name) {
|
||||
name := preferredProductTitle(in.GTIN, in.PriorProcessedName, in.Name)
|
||||
desc := preferredProductDescription(name, in.PriorProcessedDescription, in.Description)
|
||||
return name, desc, 0, map[string]any{
|
||||
"status": "unchanged",
|
||||
"input_hash": hash,
|
||||
}, nil
|
||||
(in.PriorProcessedName != "" || in.PriorProcessedDescription != "") {
|
||||
forceReason = enhanceHashForceReason(in)
|
||||
if forceReason == "" {
|
||||
name := preferredProductTitle(in.GTIN, in.PriorProcessedName, in.Name)
|
||||
desc := preferredProductDescription(name, in.PriorProcessedDescription, in.Description)
|
||||
logEnhanceOutcome(in, catUID, catName, "skip", "unchanged_hash", Completion{}, 0, "")
|
||||
return name, desc, 0, map[string]any{
|
||||
"status": "unchanged",
|
||||
"input_hash": hash,
|
||||
}, nil
|
||||
}
|
||||
log.Printf("processing: ai_enhance skip_blocked reason=%s category=%q name=%q",
|
||||
forceReason, truncateRunes(category, 80), truncateRunes(in.Name, 80))
|
||||
}
|
||||
system, user := RenderProductEnhancePrompts(sysTpl, userTpl, category, in.Name, in.Description, in.GTIN, in.BrandPrompt, in.Language, attrs)
|
||||
logEnhancePrompt(in, catUID, catName, system, user)
|
||||
started := time.Now()
|
||||
comp, obj, err := CompleteJSON(ctx, e.Completer, system, user, CompleteOptions{
|
||||
MaxTokens: MaxTokensEnhance,
|
||||
Temperature: DefaultStructuredTemp,
|
||||
})
|
||||
elapsed := time.Since(started)
|
||||
if err != nil {
|
||||
// Network/provider failure vs parse failure after retry
|
||||
name := preferredProductTitle(in.GTIN, in.Name, in.PriorProcessedName)
|
||||
desc := preferredProductDescription(name, in.Description, in.PriorProcessedDescription)
|
||||
if isWeakPriorEnhanceDescription(desc, name) {
|
||||
outcome := "refuse"
|
||||
reason := "empty_or_provider_error"
|
||||
if descriptionNeedsEnhanceRepair(desc, in.DescriptionTemplate, name) {
|
||||
if synth := synthesizeProductDescription(name, category, in.Language, attrs, in.DescriptionTemplate); synth != "" {
|
||||
desc = synth
|
||||
outcome = "synthesized"
|
||||
}
|
||||
}
|
||||
if obj == nil && comp.Text == "" {
|
||||
logEnhanceOutcome(in, catUID, catName, outcome, reason+":"+TruncateError(err), comp, elapsed, forceReason)
|
||||
return name, desc, 0, map[string]any{
|
||||
"provider": "passthrough",
|
||||
"error": TruncateError(err),
|
||||
}, err
|
||||
}
|
||||
// Parse failed after retry — keep usable copy; synthesize when empty/weak.
|
||||
// Parse failed after retry — keep usable copy; synthesize when empty/weak/formula-miss.
|
||||
if outcome != "synthesized" {
|
||||
outcome = "refuse"
|
||||
reason = "parse_failed"
|
||||
} else {
|
||||
reason = "parse_failed"
|
||||
}
|
||||
logEnhanceOutcome(in, catUID, catName, outcome, reason, comp, elapsed, forceReason)
|
||||
return name, desc, comp.TotalTokens, map[string]any{
|
||||
"status": "parse_failed",
|
||||
"reason": "invalid_json",
|
||||
"error": "AI returned invalid JSON; kept original title/description",
|
||||
"raw": truncateRunes(comp.Text, 200),
|
||||
}, nil
|
||||
}
|
||||
name := preferredProductTitle(in.GTIN, SanitizeOutput(fmt.Sprint(obj["name"])), in.Name, in.PriorProcessedName)
|
||||
desc := preferredProductDescription(name, SanitizeOutput(fmt.Sprint(obj["description"])), in.Description, in.PriorProcessedDescription)
|
||||
llmName := SanitizeOutput(fmt.Sprint(obj["name"]))
|
||||
llmDesc := SanitizeOutput(fmt.Sprint(obj["description"]))
|
||||
// Never prefer-fallback to originals for empty/leakage LLM fields — that stamped
|
||||
// status=ok + input_hash on garbage enhance output (prod: 30–272ms "ok", wrong copy).
|
||||
if reason := llmEnhanceHardRefuseReason(llmName, llmDesc); reason != "" {
|
||||
name := preferredProductTitle(in.GTIN, llmName, in.Name, in.PriorProcessedName)
|
||||
desc := preferredProductDescription(name, llmDesc)
|
||||
synthesized := false
|
||||
if descriptionNeedsEnhanceRepair(desc, in.DescriptionTemplate, name) {
|
||||
if desc == "" {
|
||||
desc = preferredProductDescription(name, in.Description, in.PriorProcessedDescription)
|
||||
}
|
||||
if descriptionNeedsEnhanceRepair(desc, in.DescriptionTemplate, name) {
|
||||
if synth := synthesizeProductDescription(name, category, in.Language, attrs, in.DescriptionTemplate); synth != "" {
|
||||
desc = synth
|
||||
synthesized = true
|
||||
}
|
||||
}
|
||||
}
|
||||
status := "refused"
|
||||
outcome := "refuse"
|
||||
if synthesized {
|
||||
status = "synthesized"
|
||||
outcome = "synthesized"
|
||||
}
|
||||
meta := map[string]any{
|
||||
"status": status,
|
||||
"reason": reason,
|
||||
"raw": comp.Raw,
|
||||
}
|
||||
if forceReason != "" {
|
||||
meta["forced_reenhance"] = true
|
||||
meta["force_reason"] = forceReason
|
||||
}
|
||||
logEnhanceOutcome(in, catUID, catName, outcome, reason, comp, elapsed, forceReason)
|
||||
return name, desc, comp.TotalTokens, meta, nil
|
||||
}
|
||||
|
||||
name := preferredProductTitle(in.GTIN, llmName, in.Name, in.PriorProcessedName)
|
||||
// Quality-gate on LLM description alone (do not absorb originals yet).
|
||||
desc := preferredProductDescription(name, llmDesc)
|
||||
didFormulaRetry := false
|
||||
// Fast garbage that ignores A1 description_template: one formula-aware retry, then synthesize.
|
||||
if !descriptionSatisfiesFormula(desc, in.DescriptionTemplate) &&
|
||||
FormatDescriptionFormulaConstraint(in.DescriptionTemplate) != "" {
|
||||
didFormulaRetry = true
|
||||
retryUser := user + descriptionFormulaRetrySuffix
|
||||
retryStarted := time.Now()
|
||||
comp2, obj2, err2 := CompleteJSON(ctx, e.Completer, system, retryUser, CompleteOptions{
|
||||
MaxTokens: MaxTokensEnhance,
|
||||
Temperature: DefaultStructuredTemp,
|
||||
})
|
||||
elapsed += time.Since(retryStarted)
|
||||
comp.PromptTokens += comp2.PromptTokens
|
||||
comp.OutputTokens += comp2.OutputTokens
|
||||
comp.TotalTokens += comp2.TotalTokens
|
||||
if err2 == nil && obj2 != nil {
|
||||
n2 := SanitizeOutput(fmt.Sprint(obj2["name"]))
|
||||
d2 := SanitizeOutput(fmt.Sprint(obj2["description"]))
|
||||
if llmEnhanceHardRefuseReason(n2, d2) == "" {
|
||||
name = preferredProductTitle(in.GTIN, n2, name, in.Name, in.PriorProcessedName)
|
||||
desc = preferredProductDescription(name, d2)
|
||||
if comp2.Raw != nil {
|
||||
comp.Raw = comp2.Raw
|
||||
}
|
||||
if comp2.Text != "" {
|
||||
comp.Text = comp2.Text
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
synthesized := false
|
||||
if isWeakPriorEnhanceDescription(desc, name) {
|
||||
if synth := synthesizeProductDescription(name, category, in.Language, attrs, in.DescriptionTemplate); synth != "" {
|
||||
desc = synth
|
||||
synthesized = true
|
||||
if descriptionNeedsEnhanceRepair(desc, in.DescriptionTemplate, name) {
|
||||
if desc == "" {
|
||||
desc = preferredProductDescription(name, in.Description, in.PriorProcessedDescription)
|
||||
}
|
||||
if descriptionNeedsEnhanceRepair(desc, in.DescriptionTemplate, name) {
|
||||
if synth := synthesizeProductDescription(name, category, in.Language, attrs, in.DescriptionTemplate); synth != "" {
|
||||
desc = synth
|
||||
synthesized = true
|
||||
}
|
||||
}
|
||||
}
|
||||
meta := map[string]any{
|
||||
"status": "ok",
|
||||
"raw": comp.Raw,
|
||||
}
|
||||
if forceReason != "" {
|
||||
meta["reason"] = forceReason
|
||||
meta["forced_reenhance"] = true
|
||||
}
|
||||
outcome := "ok"
|
||||
reason := forceReason
|
||||
// Heuristic synthesize must not poison enhance_input_hash (would hash-skip
|
||||
// formula-aware LLM copy on reprocess). Only persist hash for real LLM quality.
|
||||
if synthesized {
|
||||
meta["status"] = "synthesized"
|
||||
} else if !isWeakPriorEnhanceDescription(desc, name) {
|
||||
outcome = "synthesized"
|
||||
if didFormulaRetry {
|
||||
reason = "formula_retry"
|
||||
} else {
|
||||
reason = "weak_or_formula_mismatch"
|
||||
}
|
||||
meta["reason"] = reason
|
||||
} else if descriptionNeedsEnhanceRepair(desc, in.DescriptionTemplate, name) {
|
||||
meta["status"] = "refused"
|
||||
outcome = "refuse"
|
||||
if didFormulaRetry {
|
||||
reason = "formula_retry"
|
||||
} else {
|
||||
reason = "weak_or_formula_mismatch"
|
||||
}
|
||||
meta["reason"] = reason
|
||||
} else {
|
||||
meta["input_hash"] = hash
|
||||
if didFormulaRetry {
|
||||
reason = "formula_retry"
|
||||
if meta["reason"] == nil {
|
||||
meta["reason"] = reason
|
||||
}
|
||||
}
|
||||
}
|
||||
logEnhanceOutcome(in, catUID, catName, outcome, reason, comp, elapsed, forceReason)
|
||||
return name, desc, comp.TotalTokens, meta, nil
|
||||
}
|
||||
|
||||
// llmEnhanceHardRefuseReason reports empty/leakage LLM fields that must never be
|
||||
// accepted as quality enhance (even when originals could fill the gap).
|
||||
func llmEnhanceHardRefuseReason(name, desc string) string {
|
||||
name = strings.TrimSpace(name)
|
||||
desc = strings.TrimSpace(desc)
|
||||
if name == "" || name == "<nil>" {
|
||||
return "empty_name"
|
||||
}
|
||||
if isPromptLabelTitle(name) {
|
||||
return "prompt_leakage_name"
|
||||
}
|
||||
if desc == "" || desc == "<nil>" {
|
||||
return "empty_description"
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func sanitizeJSON(v any) string {
|
||||
if v == nil {
|
||||
return "{}"
|
||||
@@ -841,6 +1076,24 @@ var promptLeakagePhrases = []string{
|
||||
"prefer attrs values",
|
||||
"order matters; join with",
|
||||
"do not hardcode a language",
|
||||
// Description / title formula scaffolding (Format*FormulaConstraint + enhance templates).
|
||||
"emit description as one html",
|
||||
"emit one html string",
|
||||
"covering each section",
|
||||
"tags matching section",
|
||||
"overrides any shorter",
|
||||
"competing full html",
|
||||
"metadescription",
|
||||
"never ignore the description",
|
||||
"follow any description formula",
|
||||
"when a description formula",
|
||||
"structured html sections",
|
||||
"never copy name as description",
|
||||
"do not invent specs",
|
||||
"prefer 1-3 factual",
|
||||
"1-2 sentences",
|
||||
"keep literal text as written",
|
||||
"build name from attrs using this structure",
|
||||
}
|
||||
|
||||
var promptLeakageLongKeywords = []string{
|
||||
@@ -850,6 +1103,97 @@ var promptLeakageLongKeywords = []string{
|
||||
"json",
|
||||
"attrs",
|
||||
"retail title",
|
||||
"html string",
|
||||
"section type",
|
||||
}
|
||||
|
||||
// isUnusableCategoryValue rejects tokens that must never become Category /
|
||||
// CategoryName: prompt labels / title-formula scaffolding, or the product
|
||||
// title itself (name-as-category). Callers with taxonomy should prefer
|
||||
// scrubCategoryPollution so a title that legitimately equals a category name
|
||||
// can still resolve to unique_id.
|
||||
func isUnusableCategoryValue(s string, productTitles ...string) bool {
|
||||
s = strings.TrimSpace(s)
|
||||
if s == "" || s == "<nil>" {
|
||||
return false
|
||||
}
|
||||
if isPromptLabelTitle(s) {
|
||||
return true
|
||||
}
|
||||
return categoryTokenEqualsProductTitle(s, productTitles...)
|
||||
}
|
||||
|
||||
func categoryTokenEqualsProductTitle(s string, productTitles ...string) bool {
|
||||
s = strings.TrimSpace(s)
|
||||
if s == "" || s == "<nil>" {
|
||||
return false
|
||||
}
|
||||
for _, t := range productTitles {
|
||||
t = strings.TrimSpace(t)
|
||||
if t == "" || t == "<nil>" {
|
||||
continue
|
||||
}
|
||||
if strings.EqualFold(s, t) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func taxonomyDisplayNameSet(namesByUID map[string]string) map[string]struct{} {
|
||||
if len(namesByUID) == 0 {
|
||||
return nil
|
||||
}
|
||||
out := make(map[string]struct{}, len(namesByUID))
|
||||
for _, name := range namesByUID {
|
||||
n := strings.ToLower(strings.TrimSpace(name))
|
||||
if n != "" {
|
||||
out[n] = struct{}{}
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// scrubCategoryPollution clears Category / CategoryName when they hold prompt
|
||||
// leakage/formula text, or equal the product title without resolving to a
|
||||
// company taxonomy unique_id / display name.
|
||||
func scrubCategoryPollution(out *StepResult, namesByUID map[string]string, valid map[string]struct{}) {
|
||||
if out == nil {
|
||||
return
|
||||
}
|
||||
titles := []string{out.ProcessedName, out.Name}
|
||||
clearCategory := func(note string) {
|
||||
out.Category = ""
|
||||
out.CategoryName = ""
|
||||
if out.FieldSources == nil {
|
||||
out.FieldSources = map[string]any{}
|
||||
}
|
||||
out.FieldSources["category"] = "cleared_invalid"
|
||||
out.Notes = append(out.Notes, note)
|
||||
}
|
||||
cat := strings.TrimSpace(out.Category)
|
||||
if cat != "" {
|
||||
switch {
|
||||
case isPromptLabelTitle(cat):
|
||||
clearCategory("category: ignored (prompt leakage)")
|
||||
case categoryTokenEqualsProductTitle(cat, titles...) &&
|
||||
resolveCompanyCategoryUniqueID(cat, namesByUID, valid) == "":
|
||||
clearCategory("category: ignored (product title)")
|
||||
}
|
||||
}
|
||||
name := strings.TrimSpace(out.CategoryName)
|
||||
if name == "" {
|
||||
return
|
||||
}
|
||||
if isPromptLabelTitle(name) {
|
||||
out.CategoryName = ""
|
||||
return
|
||||
}
|
||||
if categoryTokenEqualsProductTitle(name, titles...) {
|
||||
if _, ok := taxonomyDisplayNameSet(namesByUID)[strings.ToLower(name)]; !ok {
|
||||
out.CategoryName = ""
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// preferredProductTitle picks the first usable title, skipping empty values,
|
||||
|
||||
@@ -223,8 +223,12 @@ func TestIsPromptLeakageTitle(t *testing.T) {
|
||||
"write name in {{language}}": true,
|
||||
"Schema: {\"name\":\"string\",\"description\":\"string\"}": true,
|
||||
"Reply with ONLY JSON (no markdown)": true,
|
||||
"Vox SWA-8000W dishwasher": false,
|
||||
"Acme Widget Pro": false,
|
||||
"Emit description as ONE HTML string covering each section in order": true,
|
||||
"Description formula (REQUIRED — overrides any shorter \"1-2 sentences\" rule)": true,
|
||||
"when a Description formula follows, emit ONE HTML string": true,
|
||||
"never copy name as description": true,
|
||||
"Vox SWA-8000W dishwasher": false,
|
||||
"Acme Widget Pro": false,
|
||||
"": false,
|
||||
}
|
||||
for in, want := range cases {
|
||||
@@ -242,6 +246,119 @@ func TestIsPromptLeakageTitle(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestIsUnusableCategoryValue_rejectsNameAndFormula(t *testing.T) {
|
||||
t.Parallel()
|
||||
title := "NOSILEC W53070 81-140CM 180ST VOGELS"
|
||||
if !isUnusableCategoryValue(title, title) {
|
||||
t.Fatal("product title must not be usable as category")
|
||||
}
|
||||
if !isUnusableCategoryValue("Title formula (order matters; join with \" \")", title) {
|
||||
t.Fatal("title formula must not be usable as category")
|
||||
}
|
||||
if !isUnusableCategoryValue("Emit description as ONE HTML string covering each section", title) {
|
||||
t.Fatal("description formula must not be usable as category")
|
||||
}
|
||||
if !isUnusableCategoryValue("Category:", title) {
|
||||
t.Fatal("prompt label must not be usable as category")
|
||||
}
|
||||
if isUnusableCategoryValue("28", title) {
|
||||
t.Fatal("taxonomy unique_id must remain usable")
|
||||
}
|
||||
if isUnusableCategoryValue("TV mounts", title) {
|
||||
t.Fatal("real category display name must remain usable")
|
||||
}
|
||||
}
|
||||
|
||||
func TestScrubCategoryPollution_clearsNameAsCategory(t *testing.T) {
|
||||
t.Parallel()
|
||||
title := "Vogel's WALL 3245 TV Wall Mount"
|
||||
out := StepResult{
|
||||
Name: title,
|
||||
ProcessedName: title,
|
||||
Category: title,
|
||||
CategoryName: title,
|
||||
FieldSources: map[string]any{"category": "vector"},
|
||||
}
|
||||
scrubCategoryPollution(&out, nil, nil)
|
||||
if out.Category != "" || out.CategoryName != "" {
|
||||
t.Fatalf("expected category cleared, got Category=%q CategoryName=%q", out.Category, out.CategoryName)
|
||||
}
|
||||
if src, _ := out.FieldSources["category"].(string); src != "cleared_invalid" {
|
||||
t.Fatalf("field_sources.category=%v want cleared_invalid", out.FieldSources["category"])
|
||||
}
|
||||
}
|
||||
|
||||
func TestEnhance_rejectsDescriptionFormulaLeakageTitle(t *testing.T) {
|
||||
leak := "Emit description as ONE HTML string covering each section in order in Slovenian"
|
||||
e := &Engine{
|
||||
Completer: stubCompleter{fn: func(system, user string) (Completion, error) {
|
||||
return Completion{Text: `{"name":"` + leak + `","description":"A solid washer."}`, TotalTokens: 2}, nil
|
||||
}},
|
||||
Vector: NoopVectorCategorizer{},
|
||||
}
|
||||
out, err := e.RunSteps(context.Background(), "co", ProductInput{
|
||||
GTIN: "8712285326882",
|
||||
Name: "Vox SWA-8000W",
|
||||
Mapped: map[string]any{"name": "Vox SWA-8000W", "description": "Washer", "category": "demo-electronics"},
|
||||
}, "enhance_only", nil, StepPolicy{AllowAI: true})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if out.ProcessedName != "Vox SWA-8000W" {
|
||||
t.Fatalf("ProcessedName=%q want Vox SWA-8000W (reject description formula leakage)", out.ProcessedName)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunSteps_rejectsProductNameAsCategory(t *testing.T) {
|
||||
title := "NOSILEC W53070 81-140CM 180ST VOGELS"
|
||||
e := &Engine{
|
||||
Completer: stubCompleter{fn: func(system, user string) (Completion, error) {
|
||||
return Completion{Text: `{"name":"` + title + `","description":"TV wall mount for large screens."}`, TotalTokens: 2}, nil
|
||||
}},
|
||||
Vector: NoopVectorCategorizer{},
|
||||
}
|
||||
out, err := e.RunSteps(context.Background(), "co", ProductInput{
|
||||
GTIN: "8712285326882",
|
||||
Name: title,
|
||||
PriorCategory: title,
|
||||
Mapped: map[string]any{"name": title, "description": title, "category": title},
|
||||
}, "enhance_only", nil, StepPolicy{AllowAI: true})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if out.Category == title || strings.EqualFold(out.Category, out.ProcessedName) {
|
||||
t.Fatalf("Category=%q must not equal product title", out.Category)
|
||||
}
|
||||
if categoryDisplayLabel(out) == title {
|
||||
t.Fatalf("display category leaked product title: %q", categoryDisplayLabel(out))
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunSteps_rejectsFormulaAsCategory(t *testing.T) {
|
||||
leak := "Title formula (order matters; join with \" \"). Build name from Attrs"
|
||||
e := &Engine{
|
||||
Completer: stubCompleter{fn: func(system, user string) (Completion, error) {
|
||||
return Completion{Text: `{"name":"Vox SWA-8000W","description":"Washer."}`, TotalTokens: 2}, nil
|
||||
}},
|
||||
Vector: NoopVectorCategorizer{},
|
||||
}
|
||||
out, err := e.RunSteps(context.Background(), "co", ProductInput{
|
||||
GTIN: "8712285326882",
|
||||
Name: "Vox SWA-8000W",
|
||||
PriorCategory: leak,
|
||||
Mapped: map[string]any{"name": "Vox SWA-8000W", "description": "Washer", "category": leak},
|
||||
}, "enhance_only", nil, StepPolicy{AllowAI: true})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if out.Category != "" {
|
||||
t.Fatalf("Category=%q want empty (formula rejected)", out.Category)
|
||||
}
|
||||
if categoryDisplayLabel(out) != "" {
|
||||
t.Fatalf("display=%q want empty", categoryDisplayLabel(out))
|
||||
}
|
||||
}
|
||||
|
||||
func TestEnhance_rejectsFormulaLeakageTitle(t *testing.T) {
|
||||
leak := "short retail title; follow any Title formula constraints that follow; use Attrs"
|
||||
e := &Engine{
|
||||
|
||||
@@ -28,6 +28,10 @@ func TestUpsertProcessedProductSQL_usesOnConflict(t *testing.T) {
|
||||
if !strings.Contains(upsertProcessedProductSQL, "COALESCE(NULLIF(BTRIM(EXCLUDED.category), ''), processed_products.category)") {
|
||||
t.Fatalf("expected category preserve on conflict: got SQL without COALESCE preserve")
|
||||
}
|
||||
// Intentional scrub/filter clears must wipe a previously persisted product-title category.
|
||||
if !strings.Contains(upsertProcessedProductSQL, "cleared_invalid") {
|
||||
t.Fatalf("expected cleared_invalid category wipe path in upsert SQL")
|
||||
}
|
||||
// Free template meta must persist; existing non-empty meta must be preserved on reprocess.
|
||||
if !strings.Contains(upsertProcessedProductSQL, "meta_title") || !strings.Contains(upsertProcessedProductSQL, "meta_description") {
|
||||
t.Fatalf("expected meta_title/meta_description columns in upsert")
|
||||
|
||||
@@ -326,11 +326,23 @@ func (p *Pipeline) LoadV1ProcessJobItems(ctx context.Context, companyID, jobID u
|
||||
if name, ok := categoryNames[catStr]; ok && name != "" {
|
||||
catNameStr = name
|
||||
} else if nested, ok := mapped["category"].(map[string]any); ok {
|
||||
for _, k := range []string{"name", "category_name", "title"} {
|
||||
if s := strings.TrimSpace(stringFromAny(nested[k])); s != "" {
|
||||
catNameStr = s
|
||||
break
|
||||
// Only accept nested display names that resolve in company taxonomy —
|
||||
// never product title / title-formula leftovers from feed objects.
|
||||
for _, k := range []string{"name", "category_name"} {
|
||||
s := strings.TrimSpace(stringFromAny(nested[k]))
|
||||
if s == "" || isUnusableCategoryValue(s, titleStr) {
|
||||
continue
|
||||
}
|
||||
uid := resolveCompanyCategoryUniqueID(s, categoryNames, nil)
|
||||
if uid == "" {
|
||||
continue
|
||||
}
|
||||
if resolvedName := strings.TrimSpace(categoryNames[uid]); resolvedName != "" {
|
||||
catNameStr = resolvedName
|
||||
} else {
|
||||
catNameStr = s
|
||||
}
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user