This commit is contained in:
2026-08-16 16:57:36 +02:00
parent 96f8c1115c
commit 532d439c41
106 changed files with 10147 additions and 494 deletions
+499 -110
View File
@@ -4,6 +4,7 @@ import (
"context"
"encoding/json"
"fmt"
"sort"
"strings"
"github.com/descrybe/descrybe-v2/apps/api/internal/company"
@@ -43,14 +44,15 @@ func (e *Engine) RunSteps(ctx context.Context, companyID string, in ProductInput
in.Name,
in.PriorProcessedName,
)
out.Description = preferredProductDescription(
out.Description = preferredProductDescription(out.Name,
stringFromAny(normalized["description"]),
in.Description,
in.PriorProcessedDescription,
)
out.Category = stringFromAny(normalized["category"])
// mapped_data.category wins; otherwise keep existing processed category
// so enhance_only / reprocess cannot blank A1 legacy categories.
// mapped_data.category / category_unique_id (unique_id codes) win.
applyCategoryFromMapped(&out, normalized, in.Mapped, in.Raw)
// otherwise keep existing processed category so enhance_only / reprocess
// cannot blank A1 legacy categories.
preserveCategoryIfEmpty(&out, in.PriorCategory)
out.FieldSources["normalize"] = "mapped+raw"
appendStepLog(out.GPTResponse, StepNormalize, map[string]any{
@@ -74,10 +76,13 @@ func (e *Engine) RunSteps(ctx context.Context, companyID string, in ProductInput
}
}
}
attrs = parsed
out.Attributes = SanitizeProductAttributes(attrs)
out.ProcessedAttributes = out.Attributes
attrs = out.Attributes
attrs = SanitizeProductAttributes(parsed)
// Map feed/spec labels onto category_attributes (formula) keys early.
if allowed := enhanceAllowedAttrKeys(in, out.Category); allowed != nil {
attrs = MapAttrsOntoAllowedKeys(attrs, allowed)
}
out.Attributes = attrs
out.ProcessedAttributes = attrs
out.FieldSources["attributes"] = "specifications"
appendStepLog(out.GPTResponse, StepParseSpecs, map[string]any{
"count": len(attrs),
@@ -95,12 +100,12 @@ func (e *Engine) RunSteps(ctx context.Context, companyID string, in ProductInput
in.Name,
in.PriorProcessedName,
)
out.Description = preferredProductDescription(
out.Description = preferredProductDescription(out.Name,
stringFromAny(normalized["description"]),
out.Description,
in.Description,
)
out.Category = stringFromAny(normalized["category"])
applyCategoryFromMapped(&out, normalized, in.Mapped, in.Raw)
// Promote characteristic fields only — never core product identity/content
// (those belong on the V1 item root: title, description, ean, images, …).
promote := []string{"brand", "width", "height", "depth", "weight", "product_model", "warranty"}
@@ -125,31 +130,15 @@ func (e *Engine) RunSteps(ctx context.Context, companyID string, in ProductInput
}
}
attrs = SanitizeProductAttributes(attrs)
// Remap again after category may have resolved in applyCategoryFromMapped.
if allowed := enhanceAllowedAttrKeys(in, out.Category); allowed != nil {
attrs = MapAttrsOntoAllowedKeys(attrs, allowed)
}
out.Attributes = attrs
out.ProcessedAttributes = attrs
appendStepLog(out.GPTResponse, StepFillFields, map[string]any{
"brand": stringFromAny(normalized["brand"]),
})
if out.Category == "" && e != nil && e.Vector != nil && e.Vector.Enabled() {
text := strings.TrimSpace(out.Name + " " + out.Description)
if text != "" {
if cat, err := e.Vector.SuggestCategory(ctx, companyID, text, categoryNames); err == nil && strings.TrimSpace(cat) != "" {
out.Category = SanitizeOutput(cat)
out.FieldSources["category"] = "vector"
out.Notes = append(out.Notes, "category: vector")
appendStepLog(out.GPTResponse, "vector_categorize", map[string]any{
"status": "ok",
"category": out.Category,
})
} else if err != nil {
out.Notes = append(out.Notes, "vector_categorize: "+TruncateError(err))
appendStepLog(out.GPTResponse, "vector_categorize", map[string]any{
"status": "failed",
"error": TruncateError(err),
})
}
}
}
preserveCategoryIfEmpty(&out, in.PriorCategory)
case StepEPREL:
@@ -161,7 +150,8 @@ func (e *Engine) RunSteps(ctx context.Context, companyID string, in ProductInput
})
break
}
id := eprel.ExtractID(normalized, in.Mapped, in.Raw)
// Also scan parsed attrs — eprel_id may only appear after parse_specs.
id := eprel.ExtractID(normalized, in.Mapped, in.Raw, attrs)
if id == "" {
out.Notes = append(out.Notes, "eprel: no id")
appendStepLog(out.GPTResponse, StepEPREL, map[string]any{"status": "skipped", "reason": "no_id"})
@@ -173,7 +163,7 @@ func (e *Engine) RunSteps(ctx context.Context, companyID string, in ProductInput
}
if !enricher.Enabled() {
out.Notes = append(out.Notes, "eprel: enricher disabled")
out.EPREL = map[string]any{"eprel_id": id, "status": "skipped"}
out.EPREL = map[string]any{"id": id, "status": "skipped"}
appendStepLog(out.GPTResponse, StepEPREL, map[string]any{"status": "skipped", "eprel_id": id, "reason": "disabled"})
break
}
@@ -191,15 +181,22 @@ func (e *Engine) RunSteps(ctx context.Context, companyID string, in ProductInput
attrs["eprel_id"] = id
out.Attributes = attrs
out.ProcessedAttributes = attrs
out.EPREL = map[string]any{"eprel_id": id, "status": "empty"}
out.EPREL = map[string]any{"id": id, "status": "empty"}
appendStepLog(out.GPTResponse, StepEPREL, map[string]any{"status": "empty", "eprel_id": id})
break
}
attrs = eprel.MergeInto(attrs, data)
// Promote energy class onto the characteristic attr key when missing
// (feeds often omit it; EPREL API is the source of truth).
if data.EnergyClass != "" {
if cur := strings.TrimSpace(fmt.Sprint(attrs["energy_class"])); cur == "" || cur == "<nil>" {
attrs["energy_class"] = data.EnergyClass
}
}
out.Attributes = attrs
out.ProcessedAttributes = attrs
out.EPREL = map[string]any{
"eprel_id": data.ID,
"id": data.ID,
"label": data.Label,
"pdf": data.PDF,
"energy_class": data.EnergyClass,
@@ -247,6 +244,10 @@ func (e *Engine) RunSteps(ctx context.Context, companyID string, in ProductInput
if primary == "" {
primary = langs[0]
}
syncCategoryName(&out, in.CategoryNamesByUID)
displayCat := categoryDisplayLabel(out)
ensureEnergyClassFromEPREL(attrs)
enhanceAttrs := AttrsForEnhance(attrs, enhanceAllowedAttrKeys(in, out.Category))
localized := company.LocalizedContent{}
if in.PriorLocalized != nil {
for k, v := range in.PriorLocalized {
@@ -264,8 +265,9 @@ func (e *Engine) RunSteps(ctx context.Context, companyID string, in ProductInput
}
catPrompt := in.CategoryEnhancePrompt
if lang != primary || catPrompt == "" {
catPrompt = categoryEnhancePromptFor(in.CategoryPromptsByLang, out.Category, lang)
catPrompt = categoryEnhancePromptFor(in.CategoryPromptsByLang, out.Category, lang, primary)
}
titleTpl, descTpl := categoryFormulasFor(in, out.Category)
priorFields := company.FieldsForLanguage(in.PriorLocalized, lang)
priorHash := priorFields.EnhanceInputHash
priorName := priorFields.ProcessedName
@@ -291,10 +293,12 @@ func (e *Engine) RunSteps(ctx context.Context, companyID string, in ProductInput
EnhanceSystemTemplate: tpl.System,
EnhanceUserTemplate: tpl.User,
CategoryEnhancePrompt: catPrompt,
TitleTemplate: titleTpl,
DescriptionTemplate: descTpl,
PriorEnhanceHash: priorHash,
PriorProcessedName: priorName,
PriorProcessedDescription: priorDesc,
}, out.Category, attrs)
}, displayCat, enhanceAttrs)
out.TotalTokens += tokens
status := enhanceStatusFromMeta(raw)
meta := map[string]any{"language": lang, "raw": raw}
@@ -319,29 +323,59 @@ func (e *Engine) RunSteps(ctx context.Context, companyID string, in ProductInput
anyOK = true
meta["status"] = status
}
hash := enhanceHashFromMeta(raw)
// Prefer title-aware selection + synthesize before deciding hash persistence.
name = preferredProductTitle(in.GTIN, name, out.Name, priorName, in.Name)
desc = preferredProductDescription(name, desc, out.Description, priorDesc)
if isWeakPriorEnhanceDescription(desc, name) {
if synth := synthesizeDescriptionFromTitle(name, displayCat, lang, enhanceAttrs); synth != "" {
desc = synth
}
}
weakDesc := isWeakPriorEnhanceDescription(desc, name)
// Only persist enhance_input_hash for quality ok / hash-skip unchanged.
// Never copy input_hash from error/passthrough/thin meta into localized.
persistHash := ""
rawHash := enhanceHashFromMeta(raw)
switch {
case err != nil:
persistHash = ""
case status == "unchanged" && !weakDesc:
persistHash = rawHash
case status == "ok" && !weakDesc:
persistHash = rawHash
default:
// thin ok / parse_failed / skipped — do not poison reprocess skip
persistHash = ""
if status == "ok" && weakDesc {
allUnchanged = false
}
}
localized[lang] = company.LocalizedFields{
ProcessedName: name,
ProcessedDescription: desc,
EnhanceInputHash: hash,
EnhanceInputHash: persistHash,
MetaTitle: company.FieldsForLanguage(localized, lang).MetaTitle,
MetaDescription: company.FieldsForLanguage(localized, lang).MetaDescription,
}
// Preserve existing meta when re-enhancing titles only.
// Never keep a bare "| <unique_id>" poisoned meta_title.
// When dropping poisoned title, also drop empty/weak stub meta_description.
if prev := company.FieldsForLanguage(in.PriorLocalized, lang); prev.MetaTitle != "" || prev.MetaDescription != "" {
f := localized[lang]
if f.MetaTitle == "" {
poisonedTitle := isPoisonedMetaTitle(prev.MetaTitle)
if f.MetaTitle == "" && !poisonedTitle {
f.MetaTitle = prev.MetaTitle
}
if f.MetaDescription == "" {
f.MetaDescription = prev.MetaDescription
weakStub := poisonedTitle && isWeakPriorEnhanceDescription(prev.MetaDescription, name, priorName)
if !weakStub {
f.MetaDescription = prev.MetaDescription
}
}
localized[lang] = f
}
langMetas = append(langMetas, meta)
if lang == primary {
name = preferredProductTitle(in.GTIN, name, out.Name, in.PriorProcessedName)
desc = preferredProductDescription(desc, out.Description, in.PriorProcessedDescription)
out.ProcessedName = name
out.ProcessedDescription = desc
if name != "" {
@@ -350,10 +384,10 @@ func (e *Engine) RunSteps(ctx context.Context, companyID string, in ProductInput
if desc != "" {
out.Description = desc
}
if hash != "" && (status == "ok" || status == "unchanged") {
out.FieldSources[FieldEnhanceInputHash] = hash
} else if err != nil {
preservePriorEnhanceHash()
if persistHash != "" {
out.FieldSources[FieldEnhanceInputHash] = persistHash
} else {
delete(out.FieldSources, FieldEnhanceInputHash)
}
}
}
@@ -365,7 +399,34 @@ func (e *Engine) RunSteps(ctx context.Context, companyID string, in ProductInput
if out.ProcessedDescription == "" {
out.ProcessedDescription = out.Description
}
preservePriorEnhanceHash()
// Timeout/error/empty: always synthesize a factual fallback when a title exists.
if out.ProcessedName != "" && (out.ProcessedDescription == "" ||
isWeakPriorEnhanceDescription(out.ProcessedDescription, out.ProcessedName, out.Name)) {
if synth := synthesizeDescriptionFromTitle(out.ProcessedName, displayCat, primary, enhanceAttrs); synth != "" {
out.ProcessedDescription = synth
if out.Description == "" || isWeakPriorEnhanceDescription(out.Description, out.Name, out.ProcessedName) {
out.Description = synth
}
if lf, ok := localized[primary]; ok {
lf.ProcessedDescription = synth
if lf.ProcessedName == "" {
lf.ProcessedName = out.ProcessedName
}
localized[primary] = lf
}
}
}
// Attempted enhance (even on timeout) — record provider mode, not misleading unknown.
out.AIProviderMode = e.EngineProviderMode()
out.FieldSources["name"] = "ai_enhance_failed"
out.FieldSources["description"] = "ai_enhance_failed"
// Failed enhance must not leave a skippable hash behind.
delete(out.FieldSources, FieldEnhanceInputHash)
for lang, lf := range localized {
lf.EnhanceInputHash = ""
localized[lang] = lf
}
out.LocalizedContent = localized
errNote := ""
for _, m := range langMetas {
if mm, ok := m.(map[string]any); ok {
@@ -402,31 +463,50 @@ func (e *Engine) RunSteps(ctx context.Context, companyID string, in ProductInput
}
}
// Paths without fill_fields (enhance_only / normalize_only) still get vector
// categorize when AllowAI + embeddings are available.
tryVectorCategorize(ctx, e, companyID, &out, categoryNames, policy)
preserveCategoryIfEmpty(&out, in.PriorCategory)
noteMissingCategory(&out, policy, e != nil && e.Vector != nil && e.Vector.Enabled())
syncCategoryName(&out, in.CategoryNamesByUID)
displayCat := categoryDisplayLabel(out)
out.Name = preferredProductTitle(in.GTIN, out.Name, out.ProcessedName, in.Name, in.PriorProcessedName)
out.ProcessedName = preferredProductTitle(in.GTIN, out.ProcessedName, out.Name, in.PriorProcessedName, in.Name)
out.Description = preferredProductDescription(out.Description, out.ProcessedDescription, in.Description)
out.ProcessedDescription = preferredProductDescription(out.ProcessedDescription, out.Description, in.PriorProcessedDescription)
out.Description = preferredProductDescription(out.Name, out.Description, out.ProcessedDescription, in.Description)
out.ProcessedDescription = preferredProductDescription(out.ProcessedName, out.ProcessedDescription, out.Description, in.PriorProcessedDescription)
if out.ProcessedName == "" {
out.ProcessedName = out.Name
}
if out.ProcessedDescription == "" {
out.ProcessedDescription = out.Description
if out.ProcessedDescription == "" || isWeakPriorEnhanceDescription(out.ProcessedDescription, out.ProcessedName, out.Name) {
if synth := synthesizeDescriptionFromTitle(out.ProcessedName, displayCat, in.Language, out.Attributes); synth != "" {
out.ProcessedDescription = synth
}
}
if out.Description == "" || isWeakPriorEnhanceDescription(out.Description, out.Name, out.ProcessedName) {
if out.ProcessedDescription != "" && !isWeakPriorEnhanceDescription(out.ProcessedDescription, out.Name, out.ProcessedName) {
out.Description = out.ProcessedDescription
} else if synth := synthesizeDescriptionFromTitle(out.Name, displayCat, in.Language, out.Attributes); synth != "" {
out.Description = synth
if out.ProcessedDescription == "" || isWeakPriorEnhanceDescription(out.ProcessedDescription, out.ProcessedName, out.Name) {
out.ProcessedDescription = synth
}
}
}
if out.Attributes == nil {
out.Attributes = map[string]any{}
}
out.Attributes = SanitizeProductAttributes(out.Attributes)
out.ProcessedAttributes = SanitizeProductAttributes(out.ProcessedAttributes)
// Persist the same sanitize + category allowlist used by enhance (poll already
// projects clean attrs; DB must not keep feed junk like zavora/vzmetenje).
allowed := enhanceAllowedAttrKeys(in, out.Category)
out.Attributes = AttrsForPersist(out.Attributes, allowed)
out.ProcessedAttributes = AttrsForPersist(out.ProcessedAttributes, allowed)
if len(out.ProcessedAttributes) == 0 {
out.ProcessedAttributes = out.Attributes
}
if out.AIProviderMode == "" {
if out.TotalTokens > 0 {
out.AIProviderMode = e.EngineProviderMode()
} else {
out.AIProviderMode = AIProviderUnknown
}
// Prefer EngineProviderMode (job/completer label) over stamping "unknown" on
// 0-token paths (hash-skip / AI skipped). processOne also treats unknown as empty.
if isUnknownProviderMode(out.AIProviderMode) {
out.AIProviderMode = preferKnownProviderMode(e.EngineProviderMode(), out.AIProviderMode)
}
if len(out.Notes) > 0 {
out.GPTResponse["notes"] = out.Notes
@@ -451,6 +531,66 @@ func preserveCategoryIfEmpty(out *StepResult, prior string) {
out.FieldSources["category"] = "prior_processed"
}
// tryVectorCategorize sets Category from embeddings when mapped unique_id is absent.
// Requires policy.AllowAI and a configured/enabled VectorCategorizer.
func tryVectorCategorize(ctx context.Context, e *Engine, companyID string, out *StepResult, categoryNames []string, policy StepPolicy) {
if out == nil || strings.TrimSpace(out.Category) != "" {
return
}
if !policy.AllowAI {
return
}
if e == nil || e.Vector == nil || !e.Vector.Enabled() {
return
}
text := strings.TrimSpace(out.Name + " " + out.Description)
if text == "" {
return
}
cat, err := e.Vector.SuggestCategory(ctx, companyID, text, categoryNames)
if err != nil {
out.Notes = append(out.Notes, "vector_categorize: "+TruncateError(err))
appendStepLog(out.GPTResponse, "vector_categorize", map[string]any{
"status": "failed",
"error": TruncateError(err),
})
return
}
if strings.TrimSpace(cat) == "" {
return
}
out.Category = SanitizeOutput(cat)
if out.FieldSources == nil {
out.FieldSources = map[string]any{}
}
out.FieldSources["category"] = "vector"
out.Notes = append(out.Notes, "category: vector")
appendStepLog(out.GPTResponse, "vector_categorize", map[string]any{
"status": "ok",
"category": out.Category,
})
}
// noteMissingCategory records why Category stayed empty (mapped absent; vector skipped or failed).
func noteMissingCategory(out *StepResult, policy StepPolicy, vectorEnabled bool) {
if out == nil || strings.TrimSpace(out.Category) != "" {
return
}
for _, n := range out.Notes {
if strings.HasPrefix(n, "category: unset") || strings.HasPrefix(n, "category: vector") {
return
}
}
switch {
case !policy.AllowAI:
out.Notes = append(out.Notes, "category: unset (no mapped unique_id; AI/vector not allowed)")
case !vectorEnabled:
out.Notes = append(out.Notes, "category: unset (no mapped unique_id; vector embeddings unavailable)")
default:
out.Notes = append(out.Notes, "category: unset (no mapped unique_id; vector did not match)")
}
}
func resolveSteps(processingType string) []string {
switch strings.ToLower(strings.TrimSpace(processingType)) {
case "enhance", "enhance_only", "enhance-only", "title", "description":
@@ -458,7 +598,8 @@ func resolveSteps(processingType string) []string {
case "attributes", "attributes_only", "specs", "specifications":
return []string{StepNormalize, StepParseSpecs, StepFillFields}
case "eprel", "eprel_only":
return []string{StepNormalize, StepEPREL}
// parse_specs first so eprel_id buried in specifications/attributes is visible.
return []string{StepNormalize, StepParseSpecs, StepEPREL}
case "normalize_only":
return []string{StepNormalize}
case "categorize", "categorize_only", "categorize_enhance":
@@ -488,15 +629,19 @@ func (e *Engine) enhance(ctx context.Context, in ProductInput, category string,
sysTpl, userTpl := resolveProductPromptTemplates(in)
hash := HashEnhanceInput(category, in.Name, in.Description, in.BrandPrompt, in.Language, sysTpl, userTpl, attrs)
if e == nil || e.Completer == nil {
return preferredProductTitle(in.GTIN, in.Name, in.PriorProcessedName),
preferredProductDescription(in.Description, in.PriorProcessedDescription),
name := preferredProductTitle(in.GTIN, in.Name, in.PriorProcessedName)
return name,
preferredProductDescription(name, in.Description, in.PriorProcessedDescription),
0, map[string]any{"status": "skipped", "input_hash": hash}, nil
}
// Skip LLM when inputs match the last successful enhance (before any credit debit).
// Skip LLM when inputs match the last successful enhance (before any credit debit),
// but never reuse a thin / title-echo prior description, or a prompt-leakage title.
if in.PriorEnhanceHash != "" && in.PriorEnhanceHash == hash &&
(in.PriorProcessedName != "" || in.PriorProcessedDescription != "") {
(in.PriorProcessedName != "" || in.PriorProcessedDescription != "") &&
!isPromptLabelTitle(in.PriorProcessedName) &&
!isWeakPriorEnhanceDescription(in.PriorProcessedDescription, in.PriorProcessedName, in.Name) {
name := preferredProductTitle(in.GTIN, in.PriorProcessedName, in.Name)
desc := preferredProductDescription(in.PriorProcessedDescription, in.Description)
desc := preferredProductDescription(name, in.PriorProcessedDescription, in.Description)
return name, desc, 0, map[string]any{
"status": "unchanged",
"input_hash": hash,
@@ -509,32 +654,43 @@ func (e *Engine) enhance(ctx context.Context, in ProductInput, category string,
})
if err != nil {
// Network/provider failure vs parse failure after retry
if obj == nil && comp.Text == "" {
return preferredProductTitle(in.GTIN, in.Name, in.PriorProcessedName),
preferredProductDescription(in.Description, in.PriorProcessedDescription),
0, map[string]any{
"provider": "passthrough",
"error": TruncateError(err),
"input_hash": hash,
}, err
name := preferredProductTitle(in.GTIN, in.Name, in.PriorProcessedName)
desc := preferredProductDescription(name, in.Description, in.PriorProcessedDescription)
if isWeakPriorEnhanceDescription(desc, name) {
if synth := synthesizeDescriptionFromTitle(name, category, in.Language, attrs); synth != "" {
desc = synth
}
}
// Parse failed after retry — keep original copy (avoid garbage titles)
return preferredProductTitle(in.GTIN, in.Name, in.PriorProcessedName),
preferredProductDescription(in.Description, in.PriorProcessedDescription),
comp.TotalTokens, map[string]any{
"status": "parse_failed",
"error": "AI returned invalid JSON; kept original title/description",
"raw": truncateRunes(comp.Text, 200),
"input_hash": hash,
}, nil
if obj == nil && comp.Text == "" {
return name, desc, 0, map[string]any{
"provider": "passthrough",
"error": TruncateError(err),
}, err
}
// Parse failed after retry — keep usable copy; synthesize when empty/weak.
return name, desc, comp.TotalTokens, map[string]any{
"status": "parse_failed",
"error": "AI returned invalid JSON; kept original title/description",
"raw": truncateRunes(comp.Text, 200),
}, nil
}
name := preferredProductTitle(in.GTIN, SanitizeOutput(fmt.Sprint(obj["name"])), in.Name, in.PriorProcessedName)
desc := preferredProductDescription(SanitizeOutput(fmt.Sprint(obj["description"])), in.Description, in.PriorProcessedDescription)
return name, desc, comp.TotalTokens, map[string]any{
"status": "ok",
"input_hash": hash,
"raw": comp.Raw,
}, nil
desc := preferredProductDescription(name, SanitizeOutput(fmt.Sprint(obj["description"])), in.Description, in.PriorProcessedDescription)
if isWeakPriorEnhanceDescription(desc, name) {
if synth := synthesizeDescriptionFromTitle(name, category, in.Language, attrs); synth != "" {
desc = synth
}
}
meta := map[string]any{
"status": "ok",
"raw": comp.Raw,
}
// Only attach input_hash when output description is quality-worthy; thin
// title-echo / heuristic filler must not poison field_sources / localized skip hashes.
if !isWeakPriorEnhanceDescription(desc, name) {
meta["input_hash"] = hash
}
return name, desc, comp.TotalTokens, meta, nil
}
func sanitizeJSON(v any) string {
@@ -556,46 +712,67 @@ func firstLine(s string) string {
return SanitizeOutput(strings.Trim(s, "\"'` "))
}
// labeledPromptValue returns the first line after any of the given labels
// labeledPromptValue returns the first usable line after any of the given labels
// (case-insensitive), e.g. "Name:" / "Desc:" from ProductEnhanceUser.
// Skips matches whose value is prompt-label / formula-leakage text so instruction
// bullets like "- name: short retail title; follow any Title formula…" do not
// win over the later "Name: <product>" line in CategoryEnhanceUserTemplate.
func labeledPromptValue(user string, labels ...string) string {
lower := strings.ToLower(user)
bestAt := -1
bestLabel := ""
type hit struct {
at int
label string
}
var hits []hit
for _, label := range labels {
label = strings.ToLower(strings.TrimSpace(label))
if label == "" {
continue
}
at := strings.Index(lower, label)
if at < 0 {
continue
}
if bestAt < 0 || at < bestAt {
bestAt = at
bestLabel = label
searchFrom := 0
for {
rel := strings.Index(lower[searchFrom:], label)
if rel < 0 {
break
}
at := searchFrom + rel
hits = append(hits, hit{at: at, label: label})
searchFrom = at + len(label)
}
}
if bestAt < 0 {
if len(hits) == 0 {
return ""
}
rest := user[bestAt+len(bestLabel):]
if j := strings.Index(strings.ToLower(rest), "attrs:"); j >= 0 {
rest = rest[:j]
sort.Slice(hits, func(i, j int) bool { return hits[i].at < hits[j].at })
for _, h := range hits {
rest := user[h.at+len(h.label):]
if j := strings.Index(strings.ToLower(rest), "attrs:"); j >= 0 {
rest = rest[:j]
}
if j := strings.Index(strings.ToLower(rest), "attributes:"); j >= 0 {
rest = rest[:j]
}
val := firstLine(rest)
if val == "" || isPromptLabelTitle(val) {
continue
}
return val
}
if j := strings.Index(strings.ToLower(rest), "attributes:"); j >= 0 {
rest = rest[:j]
}
return firstLine(rest)
return ""
}
// isPromptLabelTitle detects enhance pollution where the model echoed a
// prompt header ("Category:" / "Category: 120") as the product title.
// prompt header ("Category:" / "Category: 120") as the product title, or
// leaked instruction scaffolding from CategoryEnhanceUserTemplate /
// AppendFormulaConstraints into name.
func isPromptLabelTitle(s string) bool {
s = strings.TrimSpace(s)
if s == "" || s == "<nil>" {
return false
}
if isPromptLeakageTitle(s) {
return true
}
lower := strings.ToLower(s)
for _, label := range []string{
"category", "name", "desc", "description",
@@ -611,8 +788,61 @@ func isPromptLabelTitle(s string) bool {
return false
}
// preferredProductTitle picks the first usable title, skipping empty values and
// prompt-label echoes like "Category:" (seen on A1 Elkotex reprocess).
// isPromptLeakageTitle detects when an LLM echoed enhance-prompt instructions
// (Title formula / short retail title / Schema / Reply with ONLY JSON / …)
// as the product name instead of a real title.
func isPromptLeakageTitle(s string) bool {
s = strings.TrimSpace(s)
if s == "" || s == "<nil>" {
return false
}
lower := strings.ToLower(s)
for _, phrase := range promptLeakagePhrases {
if strings.Contains(lower, phrase) {
return true
}
}
// Long dumps of the enhance template: any formula/schema keyword is enough.
if len([]rune(s)) > 120 {
for _, kw := range promptLeakageLongKeywords {
if strings.Contains(lower, kw) {
return true
}
}
}
return false
}
// Phrases copied from aiprompts.CategoryEnhanceUserTemplate, BuiltInDefaults,
// and processing.AppendFormulaConstraints / FormatTitleFormulaConstraint.
var promptLeakagePhrases = []string{
"title formula",
"follow any",
"constraints that follow",
"short retail title",
"use attrs",
"write name in",
"schema:",
"reply with only json",
"your reply is parsed as json",
"description formula",
"build name from attrs",
"prefer attrs values",
"order matters; join with",
"do not hardcode a language",
}
var promptLeakageLongKeywords = []string{
"formula",
"constraints",
"schema",
"json",
"attrs",
"retail title",
}
// preferredProductTitle picks the first usable title, skipping empty values,
// prompt-label echoes like "Category:", and instruction-text leakage.
func preferredProductTitle(gtin string, candidates ...string) string {
for _, c := range candidates {
c = strings.TrimSpace(c)
@@ -627,13 +857,172 @@ func preferredProductTitle(gtin string, candidates ...string) string {
return "Product"
}
func preferredProductDescription(candidates ...string) string {
// Thin wrappers keep processing call sites stable; logic lives in company so
// catalog.RepairWeakEnhanceHashes can reuse it without an import cycle.
func isWeakPriorEnhanceDescription(priorDesc string, titles ...string) bool {
return company.IsWeakPriorEnhanceDescription(priorDesc, titles...)
}
func containsWeakFillerPhrase(desc string) bool {
return company.ContainsWeakFillerPhrase(desc)
}
func descriptionEchoesTitle(desc, title string) bool {
return company.DescriptionEchoesTitle(desc, title)
}
// preferredProductDescription picks the first usable description, skipping empty
// values, prompt-label echoes, weak filler phrases, and copy that merely repeats
// the product title.
func preferredProductDescription(title string, candidates ...string) string {
for _, c := range candidates {
c = strings.TrimSpace(c)
if c == "" || c == "<nil>" || isPromptLabelTitle(c) {
continue
}
if descriptionEchoesTitle(c, title) {
continue
}
if containsWeakFillerPhrase(c) {
continue
}
return SanitizeOutput(c)
}
return ""
}
func attrLookupCI(attrs map[string]any, keys ...string) string {
if attrs == nil {
return ""
}
for _, want := range keys {
want = strings.TrimSpace(want)
if want == "" {
continue
}
if v := strings.TrimSpace(stringFromAny(attrs[want])); v != "" {
return v
}
for k, raw := range attrs {
if strings.EqualFold(strings.TrimSpace(k), want) {
if v := strings.TrimSpace(stringFromAny(raw)); v != "" {
return v
}
}
}
}
return ""
}
func formatAttrDimParts(attrs map[string]any, maxParts int) []string {
if attrs == nil || maxParts <= 0 {
return nil
}
prefer := []string{
"width", "height", "depth", "weight", "max_load", "max load", "load_capacity",
"vesa", "screen_size", "diagonal", "color", "material", "size",
}
parts := make([]string, 0, maxParts)
seen := map[string]struct{}{}
add := func(k, v string) {
k = strings.TrimSpace(k)
v = strings.TrimSpace(v)
if k == "" || v == "" {
return
}
lk := strings.ToLower(k)
if _, ok := seen[lk]; ok {
return
}
seen[lk] = struct{}{}
parts = append(parts, fmt.Sprintf("%s %s", k, v))
}
for _, k := range prefer {
if len(parts) >= maxParts {
break
}
if v := attrLookupCI(attrs, k); v != "" {
add(k, v)
}
}
return parts
}
// synthesizeDescriptionFromTitle builds a short factual fallback when enhance
// returns empty/title-echo/filler copy. Uses title, category, brand, model, and
// key dims. language is a content-language code (en/sl/…) or English label.
func synthesizeDescriptionFromTitle(title, category, language string, attrs map[string]any) string {
title = strings.TrimSpace(title)
if title == "" || title == "<nil>" || isPromptLabelTitle(title) {
return ""
}
cat := strings.TrimSpace(category)
if strings.EqualFold(cat, "general") {
cat = ""
}
brand := attrLookupCI(attrs, "brand")
model := attrLookupCI(attrs, "product_model", "model", "sku")
dims := formatAttrDimParts(attrs, 3)
sl := isSlovenianContentLanguage(language)
var b strings.Builder
if sl {
b.WriteString(title)
switch {
case cat != "" && brand != "":
fmt.Fprintf(&b, " je izdelek v kategoriji %s znamke %s", cat, brand)
case cat != "":
fmt.Fprintf(&b, " je izdelek v kategoriji %s", cat)
case brand != "":
fmt.Fprintf(&b, " je izdelek znamke %s", brand)
default:
b.WriteString(" je katalogski izdelek z znanimi atributi")
}
if model != "" && !strings.Contains(strings.ToLower(title), strings.ToLower(model)) {
fmt.Fprintf(&b, " (model %s)", model)
}
if len(dims) > 0 {
fmt.Fprintf(&b, ". Ključne specifikacije: %s", strings.Join(dims, ", "))
}
b.WriteByte('.')
} else {
b.WriteString(title)
switch {
case cat != "" && brand != "":
fmt.Fprintf(&b, " is a %s product from %s", cat, brand)
case cat != "":
fmt.Fprintf(&b, " is listed in the %s category", cat)
case brand != "":
fmt.Fprintf(&b, " is a product from %s", brand)
default:
b.WriteString(" is a catalog product with the known attributes")
}
if model != "" && !strings.Contains(strings.ToLower(title), strings.ToLower(model)) {
fmt.Fprintf(&b, " (model %s)", model)
}
if len(dims) > 0 {
fmt.Fprintf(&b, ". Key specs: %s", strings.Join(dims, ", "))
}
b.WriteByte('.')
}
out := SanitizeOutput(b.String())
// Never emit sole retail-filler when title/attrs exist — strip legacy phrase if any helper reintroduces it.
if containsWeakFillerPhrase(out) {
out = strings.TrimSpace(strings.ReplaceAll(out, "Ready for retail listing.", ""))
out = strings.TrimSpace(strings.ReplaceAll(out, "ready for retail listing.", ""))
out = strings.TrimSpace(strings.Trim(out, ".")) + "."
}
return out
}
func isSlovenianContentLanguage(raw string) bool {
raw = strings.TrimSpace(raw)
if raw == "" {
return false
}
if code, err := company.ParseLanguage(raw, false); err == nil && code == "sl" {
return true
}
lower := strings.ToLower(raw)
return lower == "slovenian" || strings.Contains(lower, "slovenian")
}