greeneclipse/descrybe
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fix

Browse Source
This commit is contained in:
greeneclipse
2026-08-16 21:35:48 +02:00
parent 106f602232
commit 389608ea56
28 changed files with 2491 additions and 115 deletions
Download Patch File Download Diff File Expand all files Collapse all files
apps/api/internal/processing/steps.go
+381 -37
View File
@@ -4,8 +4,10 @@ import (
"context"
"encoding/json"
"fmt"
"log"
"sort"
"strings"
"time"
"github.com/descrybe/descrybe-v2/apps/api/internal/company"
"github.com/descrybe/descrybe-v2/apps/api/internal/eprel"
@@ -215,6 +217,7 @@ func (e *Engine) RunSteps(ctx context.Context, companyID string, in ProductInput
out.ProcessedName = out.Name
out.ProcessedDescription = out.Description
out.Notes = append(out.Notes, "ai_enhance: skipped (Free plan — upgrade for AI titles/descriptions)")
log.Printf("processing: ai_enhance skip reason=entitlement_can_use_ai")
preservePriorEnhanceHash()
appendStepLog(out.GPTResponse, StepAIEnhance, map[string]any{
"status": "skipped",
@@ -226,6 +229,7 @@ func (e *Engine) RunSteps(ctx context.Context, companyID string, in ProductInput
out.ProcessedName = out.Name
out.ProcessedDescription = out.Description
out.Notes = append(out.Notes, "ai_enhance: skipped (platform OpenAI unset; configure admin settings or company BYOK)")
log.Printf("processing: ai_enhance skip reason=openai_not_configured")
preservePriorEnhanceHash()
appendStepLog(out.GPTResponse, StepAIEnhance, map[string]any{
"status": "skipped",
@@ -298,6 +302,10 @@ func (e *Engine) RunSteps(ctx context.Context, companyID string, in ProductInput
PriorEnhanceHash: priorHash,
PriorProcessedName: priorName,
PriorProcessedDescription: priorDesc,
JobID: in.JobID,
CompanyID: in.CompanyID,
RawProductID: in.RawProductID,
CategoryUniqueID: out.Category,
}, displayCat, enhanceAttrs)
out.TotalTokens += tokens
status := enhanceStatusFromMeta(raw)
@@ -318,6 +326,15 @@ func (e *Engine) RunSteps(ctx context.Context, companyID string, in ProductInput
}
} else if status == "unchanged" {
meta["status"] = "unchanged"
} else if status == "refused" || status == "synthesized" || status == "parse_failed" {
// Soft quality failure: keep usable copy / synth, never count as real enhance ok.
allUnchanged = false
meta["status"] = status
if reason := enhanceReasonFromMeta(raw); reason != "" {
out.Notes = append(out.Notes, "ai_enhance: "+status+" ("+reason+")")
} else {
out.Notes = append(out.Notes, "ai_enhance: "+status)
}
} else {
allUnchanged = false
anyOK = true
@@ -327,13 +344,13 @@ func (e *Engine) RunSteps(ctx context.Context, companyID string, in ProductInput
name = preferredProductTitle(in.GTIN, name, out.Name, priorName, in.Name)
desc = preferredProductDescription(name, desc, out.Description, priorDesc)
outerSynth := false
if isWeakPriorEnhanceDescription(desc, name) {
if descriptionNeedsEnhanceRepair(desc, descTpl, name) {
if synth := synthesizeProductDescription(name, displayCat, lang, enhanceAttrs, descTpl); synth != "" {
desc = synth
outerSynth = true
}
}
weakDesc := isWeakPriorEnhanceDescription(desc, name)
weakDesc := descriptionNeedsEnhanceRepair(desc, descTpl, name)
// Only persist enhance_input_hash for quality ok / hash-skip unchanged.
// Never copy input_hash from error/passthrough/thin/synthesized meta.
persistHash := ""
@@ -341,7 +358,7 @@ func (e *Engine) RunSteps(ctx context.Context, companyID string, in ProductInput
switch {
case err != nil:
persistHash = ""
case status == "synthesized" || outerSynth:
case status == "synthesized" || status == "refused" || outerSynth:
persistHash = ""
allUnchanged = false
case status == "unchanged" && !weakDesc:
@@ -349,7 +366,7 @@ func (e *Engine) RunSteps(ctx context.Context, companyID string, in ProductInput
case status == "ok" && !weakDesc:
persistHash = rawHash
default:
// thin ok / parse_failed / skipped — do not poison reprocess skip
// thin ok / parse_failed / formula mismatch / skipped — do not poison reprocess skip
persistHash = ""
if status == "ok" && weakDesc {
allUnchanged = false
@@ -404,13 +421,12 @@ func (e *Engine) RunSteps(ctx context.Context, companyID string, in ProductInput
if out.ProcessedDescription == "" {
out.ProcessedDescription = out.Description
}
// Timeout/error/empty: always synthesize a factual fallback when a title exists.
if out.ProcessedName != "" && (out.ProcessedDescription == "" ||
isWeakPriorEnhanceDescription(out.ProcessedDescription, out.ProcessedName, out.Name)) {
_, failDescTpl := categoryFormulasFor(in, out.Category)
// Timeout/error/empty/formula-miss: always synthesize a factual fallback when a title exists.
_, failDescTpl := categoryFormulasFor(in, out.Category)
if out.ProcessedName != "" && descriptionNeedsEnhanceRepair(out.ProcessedDescription, failDescTpl, out.ProcessedName, out.Name) {
if synth := synthesizeProductDescription(out.ProcessedName, displayCat, primary, enhanceAttrs, failDescTpl); synth != "" {
out.ProcessedDescription = synth
if out.Description == "" || isWeakPriorEnhanceDescription(out.Description, out.Name, out.ProcessedName) {
if out.Description == "" || descriptionNeedsEnhanceRepair(out.Description, failDescTpl, out.Name, out.ProcessedName) {
out.Description = synth
}
if lf, ok := localized[primary]; ok {
@@ -453,11 +469,27 @@ func (e *Engine) RunSteps(ctx context.Context, companyID string, in ProductInput
out.SkipCreditDebit = true
out.FieldSources["name"] = "ai_enhance_unchanged"
out.FieldSources["description"] = "ai_enhance_unchanged"
out.Notes = append(out.Notes, "ai_enhance: skipped (inputs unchanged)")
out.Notes = append(out.Notes, "ai_enhance_unchanged")
log.Printf("processing: ai_enhance_unchanged")
} else {
out.AIProviderMode = e.EngineProviderMode()
out.FieldSources["name"] = "ai_enhance"
out.FieldSources["description"] = "ai_enhance"
for _, m := range langMetas {
if mm, ok := m.(map[string]any); ok {
reason, _ := mm["reason"].(string)
if reason == "" {
if raw, ok := mm["raw"].(map[string]any); ok {
reason, _ = raw["reason"].(string)
}
}
if reason != "" {
out.Notes = append(out.Notes, "ai_enhance: forced re-enhance ("+reason+")")
log.Printf("processing: ai_enhance forced re-enhance reason=%s", reason)
break
}
}
}
}
appendStepLog(out.GPTResponse, StepAIEnhance, map[string]any{
"status": map[string]any{"unchanged": allUnchanged, "ok": anyOK, "failed": anyFailed},
@@ -475,7 +507,6 @@ func (e *Engine) RunSteps(ctx context.Context, companyID string, in ProductInput
preserveCategoryIfEmpty(&out, in.PriorCategory)
noteMissingCategory(&out, policy, e != nil && e.Vector != nil && e.Vector.Enabled())
syncCategoryName(&out, in.CategoryNamesByUID)
displayCat := categoryDisplayLabel(out)
out.Name = preferredProductTitle(in.GTIN, out.Name, out.ProcessedName, in.Name, in.PriorProcessedName)
out.ProcessedName = preferredProductTitle(in.GTIN, out.ProcessedName, out.Name, in.PriorProcessedName, in.Name)
out.Description = preferredProductDescription(out.Name, out.Description, out.ProcessedDescription, in.Description)
@@ -483,22 +514,52 @@ func (e *Engine) RunSteps(ctx context.Context, companyID string, in ProductInput
if out.ProcessedName == "" {
out.ProcessedName = out.Name
}
// Titles are finalized first so name-as-category and formula leakage can be cleared.
scrubCategoryPollution(&out, in.CategoryNamesByUID, nil)
syncCategoryName(&out, in.CategoryNamesByUID)
displayCat := categoryDisplayLabel(out)
// After vector/mapped category resolution, formulas may key for the first time —
// repair short prose / title echo that ignored A1 description_template.
_, finalDescTpl := categoryFormulasFor(in, out.Category)
if out.ProcessedDescription == "" || isWeakPriorEnhanceDescription(out.ProcessedDescription, out.ProcessedName, out.Name) {
finalRepaired := false
if descriptionNeedsEnhanceRepair(out.ProcessedDescription, finalDescTpl, out.ProcessedName, out.Name) {
if synth := synthesizeProductDescription(out.ProcessedName, displayCat, in.Language, out.Attributes, finalDescTpl); synth != "" {
out.ProcessedDescription = synth
finalRepaired = true
}
}
if out.Description == "" || isWeakPriorEnhanceDescription(out.Description, out.Name, out.ProcessedName) {
if out.ProcessedDescription != "" && !isWeakPriorEnhanceDescription(out.ProcessedDescription, out.Name, out.ProcessedName) {
if descriptionNeedsEnhanceRepair(out.Description, finalDescTpl, out.Name, out.ProcessedName) {
if out.ProcessedDescription != "" && !descriptionNeedsEnhanceRepair(out.ProcessedDescription, finalDescTpl, out.Name, out.ProcessedName) {
out.Description = out.ProcessedDescription
finalRepaired = true
} else if synth := synthesizeProductDescription(out.Name, displayCat, in.Language, out.Attributes, finalDescTpl); synth != "" {
out.Description = synth
if out.ProcessedDescription == "" || isWeakPriorEnhanceDescription(out.ProcessedDescription, out.ProcessedName, out.Name) {
finalRepaired = true
if descriptionNeedsEnhanceRepair(out.ProcessedDescription, finalDescTpl, out.ProcessedName, out.Name) {
out.ProcessedDescription = synth
}
}
}
if finalRepaired {
delete(out.FieldSources, FieldEnhanceInputHash)
primary := strings.TrimSpace(in.Language)
if primary == "" && len(in.ContentLanguages) > 0 {
primary = strings.TrimSpace(in.ContentLanguages[0])
}
if primary == "" {
primary = company.DefaultLanguage
}
for lang, lf := range out.LocalizedContent {
lf.EnhanceInputHash = ""
if lang == primary {
lf.ProcessedDescription = out.ProcessedDescription
if lf.ProcessedName == "" {
lf.ProcessedName = out.ProcessedName
}
}
out.LocalizedContent[lang] = lf
}
}
if out.Attributes == nil {
out.Attributes = map[string]any{}
}
@@ -528,7 +589,7 @@ func preserveCategoryIfEmpty(out *StepResult, prior string) {
return
}
prior = strings.TrimSpace(prior)
if prior == "" {
if prior == "" || isUnusableCategoryValue(prior, out.ProcessedName, out.Name) {
return
}
out.Category = SanitizeText(prior)
@@ -563,7 +624,7 @@ func tryVectorCategorize(ctx context.Context, e *Engine, companyID string, out *
})
return
}
if strings.TrimSpace(cat) == "" {
if strings.TrimSpace(cat) == "" || isUnusableCategoryValue(cat, out.ProcessedName, out.Name) {
return
}
out.Category = SanitizeOutput(cat)
@@ -632,78 +693,252 @@ func appendStepLog(gpt map[string]any, name string, raw any) {
gpt["steps"] = append(steps, map[string]any{"step": name, "raw": raw})
}
// descriptionFormulaRetrySuffix is appended once when LLM JSON ignored the
// category description_template (short prose / title echo instead of multi-section HTML).
const descriptionFormulaRetrySuffix = "\n\nINVALID DESCRIPTION. Your JSON ignored the Description formula. Reply with ONLY one JSON object; \"description\" must be ONE HTML string covering each formula section in order with matching tags (h1/h2/h3/h4, p, ul)."
// descriptionNeedsEnhanceRepair is true when desc is weak/empty/title-echo or
// fails an active category description_template (A1 multi-section HTML).
// Heuristic invent/synth is intentionally excluded here — enhanceHashForceReason
// blocks hash-skip for synth so reprocess can still upgrade to real LLM copy.
func descriptionNeedsEnhanceRepair(desc string, template any, titles ...string) bool {
if isWeakPriorEnhanceDescription(desc, titles...) {
return true
}
return !descriptionSatisfiesFormula(desc, template)
}
// enhanceHashForceReason returns why a matching prior enhance_input_hash must not
// skip the LLM (empty = safe to reuse as ai_enhance_unchanged).
func enhanceHashForceReason(in ProductInput) string {
if isPromptLabelTitle(in.PriorProcessedName) {
return "prompt_label_title"
}
if reason := company.EnhanceHashSkipBlockReason(in.PriorProcessedDescription, in.PriorProcessedName, in.Name); reason != "" {
return reason
}
if !descriptionSatisfiesFormula(in.PriorProcessedDescription, in.DescriptionTemplate) {
return "formula-mismatch"
}
return ""
}
func (e *Engine) enhance(ctx context.Context, in ProductInput, category string, attrs map[string]any) (string, string, int, any, error) {
catUID := strings.TrimSpace(in.CategoryUniqueID)
catName := strings.TrimSpace(category)
sysTpl, userTpl := resolveProductPromptTemplates(in)
hash := HashEnhanceInput(category, in.Name, in.Description, in.BrandPrompt, in.Language, sysTpl, userTpl, attrs)
if e == nil || e.Completer == nil {
name := preferredProductTitle(in.GTIN, in.Name, in.PriorProcessedName)
logEnhanceOutcome(in, catUID, catName, "skip", "completer_nil", Completion{}, 0, "")
return name,
preferredProductDescription(name, in.Description, in.PriorProcessedDescription),
0, map[string]any{"status": "skipped", "input_hash": hash}, nil
0, map[string]any{"status": "skipped", "reason": "completer_nil"}, nil
}
// Skip LLM when inputs match the last successful enhance (before any credit debit),
// but never reuse a thin / title-echo prior description, or a prompt-leakage title.
// but never reuse thin / title-echo / invent-synth / formula-mismatch priors,
// or a prompt-leakage title.
var forceReason string
if in.PriorEnhanceHash != "" && in.PriorEnhanceHash == hash &&
(in.PriorProcessedName != "" || in.PriorProcessedDescription != "") &&
!isPromptLabelTitle(in.PriorProcessedName) &&
!isWeakPriorEnhanceDescription(in.PriorProcessedDescription, in.PriorProcessedName, in.Name) {
name := preferredProductTitle(in.GTIN, in.PriorProcessedName, in.Name)
desc := preferredProductDescription(name, in.PriorProcessedDescription, in.Description)
return name, desc, 0, map[string]any{
"status": "unchanged",
"input_hash": hash,
}, nil
(in.PriorProcessedName != "" || in.PriorProcessedDescription != "") {
forceReason = enhanceHashForceReason(in)
if forceReason == "" {
name := preferredProductTitle(in.GTIN, in.PriorProcessedName, in.Name)
desc := preferredProductDescription(name, in.PriorProcessedDescription, in.Description)
logEnhanceOutcome(in, catUID, catName, "skip", "unchanged_hash", Completion{}, 0, "")
return name, desc, 0, map[string]any{
"status": "unchanged",
"input_hash": hash,
}, nil
}
log.Printf("processing: ai_enhance skip_blocked reason=%s category=%q name=%q",
forceReason, truncateRunes(category, 80), truncateRunes(in.Name, 80))
}
system, user := RenderProductEnhancePrompts(sysTpl, userTpl, category, in.Name, in.Description, in.GTIN, in.BrandPrompt, in.Language, attrs)
logEnhancePrompt(in, catUID, catName, system, user)
started := time.Now()
comp, obj, err := CompleteJSON(ctx, e.Completer, system, user, CompleteOptions{
MaxTokens: MaxTokensEnhance,
Temperature: DefaultStructuredTemp,
})
elapsed := time.Since(started)
if err != nil {
// Network/provider failure vs parse failure after retry
name := preferredProductTitle(in.GTIN, in.Name, in.PriorProcessedName)
desc := preferredProductDescription(name, in.Description, in.PriorProcessedDescription)
if isWeakPriorEnhanceDescription(desc, name) {
outcome := "refuse"
reason := "empty_or_provider_error"
if descriptionNeedsEnhanceRepair(desc, in.DescriptionTemplate, name) {
if synth := synthesizeProductDescription(name, category, in.Language, attrs, in.DescriptionTemplate); synth != "" {
desc = synth
outcome = "synthesized"
}
}
if obj == nil && comp.Text == "" {
logEnhanceOutcome(in, catUID, catName, outcome, reason+":"+TruncateError(err), comp, elapsed, forceReason)
return name, desc, 0, map[string]any{
"provider": "passthrough",
"error": TruncateError(err),
}, err
}
// Parse failed after retry — keep usable copy; synthesize when empty/weak.
// Parse failed after retry — keep usable copy; synthesize when empty/weak/formula-miss.
if outcome != "synthesized" {
outcome = "refuse"
reason = "parse_failed"
} else {
reason = "parse_failed"
}
logEnhanceOutcome(in, catUID, catName, outcome, reason, comp, elapsed, forceReason)
return name, desc, comp.TotalTokens, map[string]any{
"status": "parse_failed",
"reason": "invalid_json",
"error": "AI returned invalid JSON; kept original title/description",
"raw": truncateRunes(comp.Text, 200),
}, nil
}
name := preferredProductTitle(in.GTIN, SanitizeOutput(fmt.Sprint(obj["name"])), in.Name, in.PriorProcessedName)
desc := preferredProductDescription(name, SanitizeOutput(fmt.Sprint(obj["description"])), in.Description, in.PriorProcessedDescription)
llmName := SanitizeOutput(fmt.Sprint(obj["name"]))
llmDesc := SanitizeOutput(fmt.Sprint(obj["description"]))
// Never prefer-fallback to originals for empty/leakage LLM fields — that stamped
// status=ok + input_hash on garbage enhance output (prod: 30–272ms "ok", wrong copy).
if reason := llmEnhanceHardRefuseReason(llmName, llmDesc); reason != "" {
name := preferredProductTitle(in.GTIN, llmName, in.Name, in.PriorProcessedName)
desc := preferredProductDescription(name, llmDesc)
synthesized := false
if descriptionNeedsEnhanceRepair(desc, in.DescriptionTemplate, name) {
if desc == "" {
desc = preferredProductDescription(name, in.Description, in.PriorProcessedDescription)
}
if descriptionNeedsEnhanceRepair(desc, in.DescriptionTemplate, name) {
if synth := synthesizeProductDescription(name, category, in.Language, attrs, in.DescriptionTemplate); synth != "" {
desc = synth
synthesized = true
}
}
}
status := "refused"
outcome := "refuse"
if synthesized {
status = "synthesized"
outcome = "synthesized"
}
meta := map[string]any{
"status": status,
"reason": reason,
"raw": comp.Raw,
}
if forceReason != "" {
meta["forced_reenhance"] = true
meta["force_reason"] = forceReason
}
logEnhanceOutcome(in, catUID, catName, outcome, reason, comp, elapsed, forceReason)
return name, desc, comp.TotalTokens, meta, nil
}
name := preferredProductTitle(in.GTIN, llmName, in.Name, in.PriorProcessedName)
// Quality-gate on LLM description alone (do not absorb originals yet).
desc := preferredProductDescription(name, llmDesc)
didFormulaRetry := false
// Fast garbage that ignores A1 description_template: one formula-aware retry, then synthesize.
if !descriptionSatisfiesFormula(desc, in.DescriptionTemplate) &&
FormatDescriptionFormulaConstraint(in.DescriptionTemplate) != "" {
didFormulaRetry = true
retryUser := user + descriptionFormulaRetrySuffix
retryStarted := time.Now()
comp2, obj2, err2 := CompleteJSON(ctx, e.Completer, system, retryUser, CompleteOptions{
MaxTokens: MaxTokensEnhance,
Temperature: DefaultStructuredTemp,
})
elapsed += time.Since(retryStarted)
comp.PromptTokens += comp2.PromptTokens
comp.OutputTokens += comp2.OutputTokens
comp.TotalTokens += comp2.TotalTokens
if err2 == nil && obj2 != nil {
n2 := SanitizeOutput(fmt.Sprint(obj2["name"]))
d2 := SanitizeOutput(fmt.Sprint(obj2["description"]))
if llmEnhanceHardRefuseReason(n2, d2) == "" {
name = preferredProductTitle(in.GTIN, n2, name, in.Name, in.PriorProcessedName)
desc = preferredProductDescription(name, d2)
if comp2.Raw != nil {
comp.Raw = comp2.Raw
}
if comp2.Text != "" {
comp.Text = comp2.Text
}
}
}
}
synthesized := false
if isWeakPriorEnhanceDescription(desc, name) {
if synth := synthesizeProductDescription(name, category, in.Language, attrs, in.DescriptionTemplate); synth != "" {
desc = synth
synthesized = true
if descriptionNeedsEnhanceRepair(desc, in.DescriptionTemplate, name) {
if desc == "" {
desc = preferredProductDescription(name, in.Description, in.PriorProcessedDescription)
}
if descriptionNeedsEnhanceRepair(desc, in.DescriptionTemplate, name) {
if synth := synthesizeProductDescription(name, category, in.Language, attrs, in.DescriptionTemplate); synth != "" {
desc = synth
synthesized = true
}
}
}
meta := map[string]any{
"status": "ok",
"raw": comp.Raw,
}
if forceReason != "" {
meta["reason"] = forceReason
meta["forced_reenhance"] = true
}
outcome := "ok"
reason := forceReason
// Heuristic synthesize must not poison enhance_input_hash (would hash-skip
// formula-aware LLM copy on reprocess). Only persist hash for real LLM quality.
if synthesized {
meta["status"] = "synthesized"
} else if !isWeakPriorEnhanceDescription(desc, name) {
outcome = "synthesized"
if didFormulaRetry {
reason = "formula_retry"
} else {
reason = "weak_or_formula_mismatch"
}
meta["reason"] = reason
} else if descriptionNeedsEnhanceRepair(desc, in.DescriptionTemplate, name) {
meta["status"] = "refused"
outcome = "refuse"
if didFormulaRetry {
reason = "formula_retry"
} else {
reason = "weak_or_formula_mismatch"
}
meta["reason"] = reason
} else {
meta["input_hash"] = hash
if didFormulaRetry {
reason = "formula_retry"
if meta["reason"] == nil {
meta["reason"] = reason
}
}
}
logEnhanceOutcome(in, catUID, catName, outcome, reason, comp, elapsed, forceReason)
return name, desc, comp.TotalTokens, meta, nil
}
// llmEnhanceHardRefuseReason reports empty/leakage LLM fields that must never be
// accepted as quality enhance (even when originals could fill the gap).
func llmEnhanceHardRefuseReason(name, desc string) string {
name = strings.TrimSpace(name)
desc = strings.TrimSpace(desc)
if name == "" || name == "<nil>" {
return "empty_name"
}
if isPromptLabelTitle(name) {
return "prompt_leakage_name"
}
if desc == "" || desc == "<nil>" {
return "empty_description"
}
return ""
}
func sanitizeJSON(v any) string {
if v == nil {
return "{}"
@@ -841,6 +1076,24 @@ var promptLeakagePhrases = []string{
"prefer attrs values",
"order matters; join with",
"do not hardcode a language",
// Description / title formula scaffolding (Format*FormulaConstraint + enhance templates).
"emit description as one html",
"emit one html string",
"covering each section",
"tags matching section",
"overrides any shorter",
"competing full html",
"metadescription",
"never ignore the description",
"follow any description formula",
"when a description formula",
"structured html sections",
"never copy name as description",
"do not invent specs",
"prefer 1-3 factual",
"1-2 sentences",
"keep literal text as written",
"build name from attrs using this structure",
}
var promptLeakageLongKeywords = []string{
@@ -850,6 +1103,97 @@ var promptLeakageLongKeywords = []string{
"json",
"attrs",
"retail title",
"html string",
"section type",
}
// isUnusableCategoryValue rejects tokens that must never become Category /
// CategoryName: prompt labels / title-formula scaffolding, or the product
// title itself (name-as-category). Callers with taxonomy should prefer
// scrubCategoryPollution so a title that legitimately equals a category name
// can still resolve to unique_id.
func isUnusableCategoryValue(s string, productTitles ...string) bool {
s = strings.TrimSpace(s)
if s == "" || s == "<nil>" {
return false
}
if isPromptLabelTitle(s) {
return true
}
return categoryTokenEqualsProductTitle(s, productTitles...)
}
func categoryTokenEqualsProductTitle(s string, productTitles ...string) bool {
s = strings.TrimSpace(s)
if s == "" || s == "<nil>" {
return false
}
for _, t := range productTitles {
t = strings.TrimSpace(t)
if t == "" || t == "<nil>" {
continue
}
if strings.EqualFold(s, t) {
return true
}
}
return false
}
func taxonomyDisplayNameSet(namesByUID map[string]string) map[string]struct{} {
if len(namesByUID) == 0 {
return nil
}
out := make(map[string]struct{}, len(namesByUID))
for _, name := range namesByUID {
n := strings.ToLower(strings.TrimSpace(name))
if n != "" {
out[n] = struct{}{}
}
}
return out
}
// scrubCategoryPollution clears Category / CategoryName when they hold prompt
// leakage/formula text, or equal the product title without resolving to a
// company taxonomy unique_id / display name.
func scrubCategoryPollution(out *StepResult, namesByUID map[string]string, valid map[string]struct{}) {
if out == nil {
return
}
titles := []string{out.ProcessedName, out.Name}
clearCategory := func(note string) {
out.Category = ""
out.CategoryName = ""
if out.FieldSources == nil {
out.FieldSources = map[string]any{}
}
out.FieldSources["category"] = "cleared_invalid"
out.Notes = append(out.Notes, note)
}
cat := strings.TrimSpace(out.Category)
if cat != "" {
switch {
case isPromptLabelTitle(cat):
clearCategory("category: ignored (prompt leakage)")
case categoryTokenEqualsProductTitle(cat, titles...) &&
resolveCompanyCategoryUniqueID(cat, namesByUID, valid) == "":
clearCategory("category: ignored (product title)")
}
}
name := strings.TrimSpace(out.CategoryName)
if name == "" {
return
}
if isPromptLabelTitle(name) {
out.CategoryName = ""
return
}
if categoryTokenEqualsProductTitle(name, titles...) {
if _, ok := taxonomyDisplayNameSet(namesByUID)[strings.ToLower(name)]; !ok {
out.CategoryName = ""
}
}
}
// preferredProductTitle picks the first usable title, skipping empty values,