fixes
This commit is contained in:
@@ -4,6 +4,7 @@ import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"github.com/descrybe/descrybe-v2/apps/api/internal/company"
|
||||
@@ -43,14 +44,15 @@ func (e *Engine) RunSteps(ctx context.Context, companyID string, in ProductInput
|
||||
in.Name,
|
||||
in.PriorProcessedName,
|
||||
)
|
||||
out.Description = preferredProductDescription(
|
||||
out.Description = preferredProductDescription(out.Name,
|
||||
stringFromAny(normalized["description"]),
|
||||
in.Description,
|
||||
in.PriorProcessedDescription,
|
||||
)
|
||||
out.Category = stringFromAny(normalized["category"])
|
||||
// mapped_data.category wins; otherwise keep existing processed category
|
||||
// so enhance_only / reprocess cannot blank A1 legacy categories.
|
||||
// mapped_data.category / category_unique_id (unique_id codes) win.
|
||||
applyCategoryFromMapped(&out, normalized, in.Mapped, in.Raw)
|
||||
// otherwise keep existing processed category so enhance_only / reprocess
|
||||
// cannot blank A1 legacy categories.
|
||||
preserveCategoryIfEmpty(&out, in.PriorCategory)
|
||||
out.FieldSources["normalize"] = "mapped+raw"
|
||||
appendStepLog(out.GPTResponse, StepNormalize, map[string]any{
|
||||
@@ -74,10 +76,13 @@ func (e *Engine) RunSteps(ctx context.Context, companyID string, in ProductInput
|
||||
}
|
||||
}
|
||||
}
|
||||
attrs = parsed
|
||||
out.Attributes = SanitizeProductAttributes(attrs)
|
||||
out.ProcessedAttributes = out.Attributes
|
||||
attrs = out.Attributes
|
||||
attrs = SanitizeProductAttributes(parsed)
|
||||
// Map feed/spec labels onto category_attributes (formula) keys early.
|
||||
if allowed := enhanceAllowedAttrKeys(in, out.Category); allowed != nil {
|
||||
attrs = MapAttrsOntoAllowedKeys(attrs, allowed)
|
||||
}
|
||||
out.Attributes = attrs
|
||||
out.ProcessedAttributes = attrs
|
||||
out.FieldSources["attributes"] = "specifications"
|
||||
appendStepLog(out.GPTResponse, StepParseSpecs, map[string]any{
|
||||
"count": len(attrs),
|
||||
@@ -95,12 +100,12 @@ func (e *Engine) RunSteps(ctx context.Context, companyID string, in ProductInput
|
||||
in.Name,
|
||||
in.PriorProcessedName,
|
||||
)
|
||||
out.Description = preferredProductDescription(
|
||||
out.Description = preferredProductDescription(out.Name,
|
||||
stringFromAny(normalized["description"]),
|
||||
out.Description,
|
||||
in.Description,
|
||||
)
|
||||
out.Category = stringFromAny(normalized["category"])
|
||||
applyCategoryFromMapped(&out, normalized, in.Mapped, in.Raw)
|
||||
// Promote characteristic fields only — never core product identity/content
|
||||
// (those belong on the V1 item root: title, description, ean, images, …).
|
||||
promote := []string{"brand", "width", "height", "depth", "weight", "product_model", "warranty"}
|
||||
@@ -125,31 +130,15 @@ func (e *Engine) RunSteps(ctx context.Context, companyID string, in ProductInput
|
||||
}
|
||||
}
|
||||
attrs = SanitizeProductAttributes(attrs)
|
||||
// Remap again after category may have resolved in applyCategoryFromMapped.
|
||||
if allowed := enhanceAllowedAttrKeys(in, out.Category); allowed != nil {
|
||||
attrs = MapAttrsOntoAllowedKeys(attrs, allowed)
|
||||
}
|
||||
out.Attributes = attrs
|
||||
out.ProcessedAttributes = attrs
|
||||
appendStepLog(out.GPTResponse, StepFillFields, map[string]any{
|
||||
"brand": stringFromAny(normalized["brand"]),
|
||||
})
|
||||
if out.Category == "" && e != nil && e.Vector != nil && e.Vector.Enabled() {
|
||||
text := strings.TrimSpace(out.Name + " " + out.Description)
|
||||
if text != "" {
|
||||
if cat, err := e.Vector.SuggestCategory(ctx, companyID, text, categoryNames); err == nil && strings.TrimSpace(cat) != "" {
|
||||
out.Category = SanitizeOutput(cat)
|
||||
out.FieldSources["category"] = "vector"
|
||||
out.Notes = append(out.Notes, "category: vector")
|
||||
appendStepLog(out.GPTResponse, "vector_categorize", map[string]any{
|
||||
"status": "ok",
|
||||
"category": out.Category,
|
||||
})
|
||||
} else if err != nil {
|
||||
out.Notes = append(out.Notes, "vector_categorize: "+TruncateError(err))
|
||||
appendStepLog(out.GPTResponse, "vector_categorize", map[string]any{
|
||||
"status": "failed",
|
||||
"error": TruncateError(err),
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
preserveCategoryIfEmpty(&out, in.PriorCategory)
|
||||
|
||||
case StepEPREL:
|
||||
@@ -161,7 +150,8 @@ func (e *Engine) RunSteps(ctx context.Context, companyID string, in ProductInput
|
||||
})
|
||||
break
|
||||
}
|
||||
id := eprel.ExtractID(normalized, in.Mapped, in.Raw)
|
||||
// Also scan parsed attrs — eprel_id may only appear after parse_specs.
|
||||
id := eprel.ExtractID(normalized, in.Mapped, in.Raw, attrs)
|
||||
if id == "" {
|
||||
out.Notes = append(out.Notes, "eprel: no id")
|
||||
appendStepLog(out.GPTResponse, StepEPREL, map[string]any{"status": "skipped", "reason": "no_id"})
|
||||
@@ -173,7 +163,7 @@ func (e *Engine) RunSteps(ctx context.Context, companyID string, in ProductInput
|
||||
}
|
||||
if !enricher.Enabled() {
|
||||
out.Notes = append(out.Notes, "eprel: enricher disabled")
|
||||
out.EPREL = map[string]any{"eprel_id": id, "status": "skipped"}
|
||||
out.EPREL = map[string]any{"id": id, "status": "skipped"}
|
||||
appendStepLog(out.GPTResponse, StepEPREL, map[string]any{"status": "skipped", "eprel_id": id, "reason": "disabled"})
|
||||
break
|
||||
}
|
||||
@@ -191,15 +181,22 @@ func (e *Engine) RunSteps(ctx context.Context, companyID string, in ProductInput
|
||||
attrs["eprel_id"] = id
|
||||
out.Attributes = attrs
|
||||
out.ProcessedAttributes = attrs
|
||||
out.EPREL = map[string]any{"eprel_id": id, "status": "empty"}
|
||||
out.EPREL = map[string]any{"id": id, "status": "empty"}
|
||||
appendStepLog(out.GPTResponse, StepEPREL, map[string]any{"status": "empty", "eprel_id": id})
|
||||
break
|
||||
}
|
||||
attrs = eprel.MergeInto(attrs, data)
|
||||
// Promote energy class onto the characteristic attr key when missing
|
||||
// (feeds often omit it; EPREL API is the source of truth).
|
||||
if data.EnergyClass != "" {
|
||||
if cur := strings.TrimSpace(fmt.Sprint(attrs["energy_class"])); cur == "" || cur == "<nil>" {
|
||||
attrs["energy_class"] = data.EnergyClass
|
||||
}
|
||||
}
|
||||
out.Attributes = attrs
|
||||
out.ProcessedAttributes = attrs
|
||||
out.EPREL = map[string]any{
|
||||
"eprel_id": data.ID,
|
||||
"id": data.ID,
|
||||
"label": data.Label,
|
||||
"pdf": data.PDF,
|
||||
"energy_class": data.EnergyClass,
|
||||
@@ -247,6 +244,10 @@ func (e *Engine) RunSteps(ctx context.Context, companyID string, in ProductInput
|
||||
if primary == "" {
|
||||
primary = langs[0]
|
||||
}
|
||||
syncCategoryName(&out, in.CategoryNamesByUID)
|
||||
displayCat := categoryDisplayLabel(out)
|
||||
ensureEnergyClassFromEPREL(attrs)
|
||||
enhanceAttrs := AttrsForEnhance(attrs, enhanceAllowedAttrKeys(in, out.Category))
|
||||
localized := company.LocalizedContent{}
|
||||
if in.PriorLocalized != nil {
|
||||
for k, v := range in.PriorLocalized {
|
||||
@@ -264,8 +265,9 @@ func (e *Engine) RunSteps(ctx context.Context, companyID string, in ProductInput
|
||||
}
|
||||
catPrompt := in.CategoryEnhancePrompt
|
||||
if lang != primary || catPrompt == "" {
|
||||
catPrompt = categoryEnhancePromptFor(in.CategoryPromptsByLang, out.Category, lang)
|
||||
catPrompt = categoryEnhancePromptFor(in.CategoryPromptsByLang, out.Category, lang, primary)
|
||||
}
|
||||
titleTpl, descTpl := categoryFormulasFor(in, out.Category)
|
||||
priorFields := company.FieldsForLanguage(in.PriorLocalized, lang)
|
||||
priorHash := priorFields.EnhanceInputHash
|
||||
priorName := priorFields.ProcessedName
|
||||
@@ -291,10 +293,12 @@ func (e *Engine) RunSteps(ctx context.Context, companyID string, in ProductInput
|
||||
EnhanceSystemTemplate: tpl.System,
|
||||
EnhanceUserTemplate: tpl.User,
|
||||
CategoryEnhancePrompt: catPrompt,
|
||||
TitleTemplate: titleTpl,
|
||||
DescriptionTemplate: descTpl,
|
||||
PriorEnhanceHash: priorHash,
|
||||
PriorProcessedName: priorName,
|
||||
PriorProcessedDescription: priorDesc,
|
||||
}, out.Category, attrs)
|
||||
}, displayCat, enhanceAttrs)
|
||||
out.TotalTokens += tokens
|
||||
status := enhanceStatusFromMeta(raw)
|
||||
meta := map[string]any{"language": lang, "raw": raw}
|
||||
@@ -319,29 +323,59 @@ func (e *Engine) RunSteps(ctx context.Context, companyID string, in ProductInput
|
||||
anyOK = true
|
||||
meta["status"] = status
|
||||
}
|
||||
hash := enhanceHashFromMeta(raw)
|
||||
// Prefer title-aware selection + synthesize before deciding hash persistence.
|
||||
name = preferredProductTitle(in.GTIN, name, out.Name, priorName, in.Name)
|
||||
desc = preferredProductDescription(name, desc, out.Description, priorDesc)
|
||||
if isWeakPriorEnhanceDescription(desc, name) {
|
||||
if synth := synthesizeDescriptionFromTitle(name, displayCat, lang, enhanceAttrs); synth != "" {
|
||||
desc = synth
|
||||
}
|
||||
}
|
||||
weakDesc := isWeakPriorEnhanceDescription(desc, name)
|
||||
// Only persist enhance_input_hash for quality ok / hash-skip unchanged.
|
||||
// Never copy input_hash from error/passthrough/thin meta into localized.
|
||||
persistHash := ""
|
||||
rawHash := enhanceHashFromMeta(raw)
|
||||
switch {
|
||||
case err != nil:
|
||||
persistHash = ""
|
||||
case status == "unchanged" && !weakDesc:
|
||||
persistHash = rawHash
|
||||
case status == "ok" && !weakDesc:
|
||||
persistHash = rawHash
|
||||
default:
|
||||
// thin ok / parse_failed / skipped — do not poison reprocess skip
|
||||
persistHash = ""
|
||||
if status == "ok" && weakDesc {
|
||||
allUnchanged = false
|
||||
}
|
||||
}
|
||||
localized[lang] = company.LocalizedFields{
|
||||
ProcessedName: name,
|
||||
ProcessedDescription: desc,
|
||||
EnhanceInputHash: hash,
|
||||
EnhanceInputHash: persistHash,
|
||||
MetaTitle: company.FieldsForLanguage(localized, lang).MetaTitle,
|
||||
MetaDescription: company.FieldsForLanguage(localized, lang).MetaDescription,
|
||||
}
|
||||
// Preserve existing meta when re-enhancing titles only.
|
||||
// Never keep a bare "| <unique_id>" poisoned meta_title.
|
||||
// When dropping poisoned title, also drop empty/weak stub meta_description.
|
||||
if prev := company.FieldsForLanguage(in.PriorLocalized, lang); prev.MetaTitle != "" || prev.MetaDescription != "" {
|
||||
f := localized[lang]
|
||||
if f.MetaTitle == "" {
|
||||
poisonedTitle := isPoisonedMetaTitle(prev.MetaTitle)
|
||||
if f.MetaTitle == "" && !poisonedTitle {
|
||||
f.MetaTitle = prev.MetaTitle
|
||||
}
|
||||
if f.MetaDescription == "" {
|
||||
f.MetaDescription = prev.MetaDescription
|
||||
weakStub := poisonedTitle && isWeakPriorEnhanceDescription(prev.MetaDescription, name, priorName)
|
||||
if !weakStub {
|
||||
f.MetaDescription = prev.MetaDescription
|
||||
}
|
||||
}
|
||||
localized[lang] = f
|
||||
}
|
||||
langMetas = append(langMetas, meta)
|
||||
if lang == primary {
|
||||
name = preferredProductTitle(in.GTIN, name, out.Name, in.PriorProcessedName)
|
||||
desc = preferredProductDescription(desc, out.Description, in.PriorProcessedDescription)
|
||||
out.ProcessedName = name
|
||||
out.ProcessedDescription = desc
|
||||
if name != "" {
|
||||
@@ -350,10 +384,10 @@ func (e *Engine) RunSteps(ctx context.Context, companyID string, in ProductInput
|
||||
if desc != "" {
|
||||
out.Description = desc
|
||||
}
|
||||
if hash != "" && (status == "ok" || status == "unchanged") {
|
||||
out.FieldSources[FieldEnhanceInputHash] = hash
|
||||
} else if err != nil {
|
||||
preservePriorEnhanceHash()
|
||||
if persistHash != "" {
|
||||
out.FieldSources[FieldEnhanceInputHash] = persistHash
|
||||
} else {
|
||||
delete(out.FieldSources, FieldEnhanceInputHash)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -365,7 +399,34 @@ func (e *Engine) RunSteps(ctx context.Context, companyID string, in ProductInput
|
||||
if out.ProcessedDescription == "" {
|
||||
out.ProcessedDescription = out.Description
|
||||
}
|
||||
preservePriorEnhanceHash()
|
||||
// Timeout/error/empty: always synthesize a factual fallback when a title exists.
|
||||
if out.ProcessedName != "" && (out.ProcessedDescription == "" ||
|
||||
isWeakPriorEnhanceDescription(out.ProcessedDescription, out.ProcessedName, out.Name)) {
|
||||
if synth := synthesizeDescriptionFromTitle(out.ProcessedName, displayCat, primary, enhanceAttrs); synth != "" {
|
||||
out.ProcessedDescription = synth
|
||||
if out.Description == "" || isWeakPriorEnhanceDescription(out.Description, out.Name, out.ProcessedName) {
|
||||
out.Description = synth
|
||||
}
|
||||
if lf, ok := localized[primary]; ok {
|
||||
lf.ProcessedDescription = synth
|
||||
if lf.ProcessedName == "" {
|
||||
lf.ProcessedName = out.ProcessedName
|
||||
}
|
||||
localized[primary] = lf
|
||||
}
|
||||
}
|
||||
}
|
||||
// Attempted enhance (even on timeout) — record provider mode, not misleading unknown.
|
||||
out.AIProviderMode = e.EngineProviderMode()
|
||||
out.FieldSources["name"] = "ai_enhance_failed"
|
||||
out.FieldSources["description"] = "ai_enhance_failed"
|
||||
// Failed enhance must not leave a skippable hash behind.
|
||||
delete(out.FieldSources, FieldEnhanceInputHash)
|
||||
for lang, lf := range localized {
|
||||
lf.EnhanceInputHash = ""
|
||||
localized[lang] = lf
|
||||
}
|
||||
out.LocalizedContent = localized
|
||||
errNote := ""
|
||||
for _, m := range langMetas {
|
||||
if mm, ok := m.(map[string]any); ok {
|
||||
@@ -402,31 +463,50 @@ func (e *Engine) RunSteps(ctx context.Context, companyID string, in ProductInput
|
||||
}
|
||||
}
|
||||
|
||||
// Paths without fill_fields (enhance_only / normalize_only) still get vector
|
||||
// categorize when AllowAI + embeddings are available.
|
||||
tryVectorCategorize(ctx, e, companyID, &out, categoryNames, policy)
|
||||
preserveCategoryIfEmpty(&out, in.PriorCategory)
|
||||
noteMissingCategory(&out, policy, e != nil && e.Vector != nil && e.Vector.Enabled())
|
||||
syncCategoryName(&out, in.CategoryNamesByUID)
|
||||
displayCat := categoryDisplayLabel(out)
|
||||
out.Name = preferredProductTitle(in.GTIN, out.Name, out.ProcessedName, in.Name, in.PriorProcessedName)
|
||||
out.ProcessedName = preferredProductTitle(in.GTIN, out.ProcessedName, out.Name, in.PriorProcessedName, in.Name)
|
||||
out.Description = preferredProductDescription(out.Description, out.ProcessedDescription, in.Description)
|
||||
out.ProcessedDescription = preferredProductDescription(out.ProcessedDescription, out.Description, in.PriorProcessedDescription)
|
||||
out.Description = preferredProductDescription(out.Name, out.Description, out.ProcessedDescription, in.Description)
|
||||
out.ProcessedDescription = preferredProductDescription(out.ProcessedName, out.ProcessedDescription, out.Description, in.PriorProcessedDescription)
|
||||
if out.ProcessedName == "" {
|
||||
out.ProcessedName = out.Name
|
||||
}
|
||||
if out.ProcessedDescription == "" {
|
||||
out.ProcessedDescription = out.Description
|
||||
if out.ProcessedDescription == "" || isWeakPriorEnhanceDescription(out.ProcessedDescription, out.ProcessedName, out.Name) {
|
||||
if synth := synthesizeDescriptionFromTitle(out.ProcessedName, displayCat, in.Language, out.Attributes); synth != "" {
|
||||
out.ProcessedDescription = synth
|
||||
}
|
||||
}
|
||||
if out.Description == "" || isWeakPriorEnhanceDescription(out.Description, out.Name, out.ProcessedName) {
|
||||
if out.ProcessedDescription != "" && !isWeakPriorEnhanceDescription(out.ProcessedDescription, out.Name, out.ProcessedName) {
|
||||
out.Description = out.ProcessedDescription
|
||||
} else if synth := synthesizeDescriptionFromTitle(out.Name, displayCat, in.Language, out.Attributes); synth != "" {
|
||||
out.Description = synth
|
||||
if out.ProcessedDescription == "" || isWeakPriorEnhanceDescription(out.ProcessedDescription, out.ProcessedName, out.Name) {
|
||||
out.ProcessedDescription = synth
|
||||
}
|
||||
}
|
||||
}
|
||||
if out.Attributes == nil {
|
||||
out.Attributes = map[string]any{}
|
||||
}
|
||||
out.Attributes = SanitizeProductAttributes(out.Attributes)
|
||||
out.ProcessedAttributes = SanitizeProductAttributes(out.ProcessedAttributes)
|
||||
// Persist the same sanitize + category allowlist used by enhance (poll already
|
||||
// projects clean attrs; DB must not keep feed junk like zavora/vzmetenje).
|
||||
allowed := enhanceAllowedAttrKeys(in, out.Category)
|
||||
out.Attributes = AttrsForPersist(out.Attributes, allowed)
|
||||
out.ProcessedAttributes = AttrsForPersist(out.ProcessedAttributes, allowed)
|
||||
if len(out.ProcessedAttributes) == 0 {
|
||||
out.ProcessedAttributes = out.Attributes
|
||||
}
|
||||
if out.AIProviderMode == "" {
|
||||
if out.TotalTokens > 0 {
|
||||
out.AIProviderMode = e.EngineProviderMode()
|
||||
} else {
|
||||
out.AIProviderMode = AIProviderUnknown
|
||||
}
|
||||
// Prefer EngineProviderMode (job/completer label) over stamping "unknown" on
|
||||
// 0-token paths (hash-skip / AI skipped). processOne also treats unknown as empty.
|
||||
if isUnknownProviderMode(out.AIProviderMode) {
|
||||
out.AIProviderMode = preferKnownProviderMode(e.EngineProviderMode(), out.AIProviderMode)
|
||||
}
|
||||
if len(out.Notes) > 0 {
|
||||
out.GPTResponse["notes"] = out.Notes
|
||||
@@ -451,6 +531,66 @@ func preserveCategoryIfEmpty(out *StepResult, prior string) {
|
||||
out.FieldSources["category"] = "prior_processed"
|
||||
}
|
||||
|
||||
// tryVectorCategorize sets Category from embeddings when mapped unique_id is absent.
|
||||
// Requires policy.AllowAI and a configured/enabled VectorCategorizer.
|
||||
func tryVectorCategorize(ctx context.Context, e *Engine, companyID string, out *StepResult, categoryNames []string, policy StepPolicy) {
|
||||
if out == nil || strings.TrimSpace(out.Category) != "" {
|
||||
return
|
||||
}
|
||||
if !policy.AllowAI {
|
||||
return
|
||||
}
|
||||
if e == nil || e.Vector == nil || !e.Vector.Enabled() {
|
||||
return
|
||||
}
|
||||
text := strings.TrimSpace(out.Name + " " + out.Description)
|
||||
if text == "" {
|
||||
return
|
||||
}
|
||||
cat, err := e.Vector.SuggestCategory(ctx, companyID, text, categoryNames)
|
||||
if err != nil {
|
||||
out.Notes = append(out.Notes, "vector_categorize: "+TruncateError(err))
|
||||
appendStepLog(out.GPTResponse, "vector_categorize", map[string]any{
|
||||
"status": "failed",
|
||||
"error": TruncateError(err),
|
||||
})
|
||||
return
|
||||
}
|
||||
if strings.TrimSpace(cat) == "" {
|
||||
return
|
||||
}
|
||||
out.Category = SanitizeOutput(cat)
|
||||
if out.FieldSources == nil {
|
||||
out.FieldSources = map[string]any{}
|
||||
}
|
||||
out.FieldSources["category"] = "vector"
|
||||
out.Notes = append(out.Notes, "category: vector")
|
||||
appendStepLog(out.GPTResponse, "vector_categorize", map[string]any{
|
||||
"status": "ok",
|
||||
"category": out.Category,
|
||||
})
|
||||
}
|
||||
|
||||
// noteMissingCategory records why Category stayed empty (mapped absent; vector skipped or failed).
|
||||
func noteMissingCategory(out *StepResult, policy StepPolicy, vectorEnabled bool) {
|
||||
if out == nil || strings.TrimSpace(out.Category) != "" {
|
||||
return
|
||||
}
|
||||
for _, n := range out.Notes {
|
||||
if strings.HasPrefix(n, "category: unset") || strings.HasPrefix(n, "category: vector") {
|
||||
return
|
||||
}
|
||||
}
|
||||
switch {
|
||||
case !policy.AllowAI:
|
||||
out.Notes = append(out.Notes, "category: unset (no mapped unique_id; AI/vector not allowed)")
|
||||
case !vectorEnabled:
|
||||
out.Notes = append(out.Notes, "category: unset (no mapped unique_id; vector embeddings unavailable)")
|
||||
default:
|
||||
out.Notes = append(out.Notes, "category: unset (no mapped unique_id; vector did not match)")
|
||||
}
|
||||
}
|
||||
|
||||
func resolveSteps(processingType string) []string {
|
||||
switch strings.ToLower(strings.TrimSpace(processingType)) {
|
||||
case "enhance", "enhance_only", "enhance-only", "title", "description":
|
||||
@@ -458,7 +598,8 @@ func resolveSteps(processingType string) []string {
|
||||
case "attributes", "attributes_only", "specs", "specifications":
|
||||
return []string{StepNormalize, StepParseSpecs, StepFillFields}
|
||||
case "eprel", "eprel_only":
|
||||
return []string{StepNormalize, StepEPREL}
|
||||
// parse_specs first so eprel_id buried in specifications/attributes is visible.
|
||||
return []string{StepNormalize, StepParseSpecs, StepEPREL}
|
||||
case "normalize_only":
|
||||
return []string{StepNormalize}
|
||||
case "categorize", "categorize_only", "categorize_enhance":
|
||||
@@ -488,15 +629,19 @@ func (e *Engine) enhance(ctx context.Context, in ProductInput, category string,
|
||||
sysTpl, userTpl := resolveProductPromptTemplates(in)
|
||||
hash := HashEnhanceInput(category, in.Name, in.Description, in.BrandPrompt, in.Language, sysTpl, userTpl, attrs)
|
||||
if e == nil || e.Completer == nil {
|
||||
return preferredProductTitle(in.GTIN, in.Name, in.PriorProcessedName),
|
||||
preferredProductDescription(in.Description, in.PriorProcessedDescription),
|
||||
name := preferredProductTitle(in.GTIN, in.Name, in.PriorProcessedName)
|
||||
return name,
|
||||
preferredProductDescription(name, in.Description, in.PriorProcessedDescription),
|
||||
0, map[string]any{"status": "skipped", "input_hash": hash}, nil
|
||||
}
|
||||
// Skip LLM when inputs match the last successful enhance (before any credit debit).
|
||||
// Skip LLM when inputs match the last successful enhance (before any credit debit),
|
||||
// but never reuse a thin / title-echo prior description, or a prompt-leakage title.
|
||||
if in.PriorEnhanceHash != "" && in.PriorEnhanceHash == hash &&
|
||||
(in.PriorProcessedName != "" || in.PriorProcessedDescription != "") {
|
||||
(in.PriorProcessedName != "" || in.PriorProcessedDescription != "") &&
|
||||
!isPromptLabelTitle(in.PriorProcessedName) &&
|
||||
!isWeakPriorEnhanceDescription(in.PriorProcessedDescription, in.PriorProcessedName, in.Name) {
|
||||
name := preferredProductTitle(in.GTIN, in.PriorProcessedName, in.Name)
|
||||
desc := preferredProductDescription(in.PriorProcessedDescription, in.Description)
|
||||
desc := preferredProductDescription(name, in.PriorProcessedDescription, in.Description)
|
||||
return name, desc, 0, map[string]any{
|
||||
"status": "unchanged",
|
||||
"input_hash": hash,
|
||||
@@ -509,32 +654,43 @@ func (e *Engine) enhance(ctx context.Context, in ProductInput, category string,
|
||||
})
|
||||
if err != nil {
|
||||
// Network/provider failure vs parse failure after retry
|
||||
if obj == nil && comp.Text == "" {
|
||||
return preferredProductTitle(in.GTIN, in.Name, in.PriorProcessedName),
|
||||
preferredProductDescription(in.Description, in.PriorProcessedDescription),
|
||||
0, map[string]any{
|
||||
"provider": "passthrough",
|
||||
"error": TruncateError(err),
|
||||
"input_hash": hash,
|
||||
}, err
|
||||
name := preferredProductTitle(in.GTIN, in.Name, in.PriorProcessedName)
|
||||
desc := preferredProductDescription(name, in.Description, in.PriorProcessedDescription)
|
||||
if isWeakPriorEnhanceDescription(desc, name) {
|
||||
if synth := synthesizeDescriptionFromTitle(name, category, in.Language, attrs); synth != "" {
|
||||
desc = synth
|
||||
}
|
||||
}
|
||||
// Parse failed after retry — keep original copy (avoid garbage titles)
|
||||
return preferredProductTitle(in.GTIN, in.Name, in.PriorProcessedName),
|
||||
preferredProductDescription(in.Description, in.PriorProcessedDescription),
|
||||
comp.TotalTokens, map[string]any{
|
||||
"status": "parse_failed",
|
||||
"error": "AI returned invalid JSON; kept original title/description",
|
||||
"raw": truncateRunes(comp.Text, 200),
|
||||
"input_hash": hash,
|
||||
}, nil
|
||||
if obj == nil && comp.Text == "" {
|
||||
return name, desc, 0, map[string]any{
|
||||
"provider": "passthrough",
|
||||
"error": TruncateError(err),
|
||||
}, err
|
||||
}
|
||||
// Parse failed after retry — keep usable copy; synthesize when empty/weak.
|
||||
return name, desc, comp.TotalTokens, map[string]any{
|
||||
"status": "parse_failed",
|
||||
"error": "AI returned invalid JSON; kept original title/description",
|
||||
"raw": truncateRunes(comp.Text, 200),
|
||||
}, nil
|
||||
}
|
||||
name := preferredProductTitle(in.GTIN, SanitizeOutput(fmt.Sprint(obj["name"])), in.Name, in.PriorProcessedName)
|
||||
desc := preferredProductDescription(SanitizeOutput(fmt.Sprint(obj["description"])), in.Description, in.PriorProcessedDescription)
|
||||
return name, desc, comp.TotalTokens, map[string]any{
|
||||
"status": "ok",
|
||||
"input_hash": hash,
|
||||
"raw": comp.Raw,
|
||||
}, nil
|
||||
desc := preferredProductDescription(name, SanitizeOutput(fmt.Sprint(obj["description"])), in.Description, in.PriorProcessedDescription)
|
||||
if isWeakPriorEnhanceDescription(desc, name) {
|
||||
if synth := synthesizeDescriptionFromTitle(name, category, in.Language, attrs); synth != "" {
|
||||
desc = synth
|
||||
}
|
||||
}
|
||||
meta := map[string]any{
|
||||
"status": "ok",
|
||||
"raw": comp.Raw,
|
||||
}
|
||||
// Only attach input_hash when output description is quality-worthy; thin
|
||||
// title-echo / heuristic filler must not poison field_sources / localized skip hashes.
|
||||
if !isWeakPriorEnhanceDescription(desc, name) {
|
||||
meta["input_hash"] = hash
|
||||
}
|
||||
return name, desc, comp.TotalTokens, meta, nil
|
||||
}
|
||||
|
||||
func sanitizeJSON(v any) string {
|
||||
@@ -556,46 +712,67 @@ func firstLine(s string) string {
|
||||
return SanitizeOutput(strings.Trim(s, "\"'` "))
|
||||
}
|
||||
|
||||
// labeledPromptValue returns the first line after any of the given labels
|
||||
// labeledPromptValue returns the first usable line after any of the given labels
|
||||
// (case-insensitive), e.g. "Name:" / "Desc:" from ProductEnhanceUser.
|
||||
// Skips matches whose value is prompt-label / formula-leakage text so instruction
|
||||
// bullets like "- name: short retail title; follow any Title formula…" do not
|
||||
// win over the later "Name: <product>" line in CategoryEnhanceUserTemplate.
|
||||
func labeledPromptValue(user string, labels ...string) string {
|
||||
lower := strings.ToLower(user)
|
||||
bestAt := -1
|
||||
bestLabel := ""
|
||||
type hit struct {
|
||||
at int
|
||||
label string
|
||||
}
|
||||
var hits []hit
|
||||
for _, label := range labels {
|
||||
label = strings.ToLower(strings.TrimSpace(label))
|
||||
if label == "" {
|
||||
continue
|
||||
}
|
||||
at := strings.Index(lower, label)
|
||||
if at < 0 {
|
||||
continue
|
||||
}
|
||||
if bestAt < 0 || at < bestAt {
|
||||
bestAt = at
|
||||
bestLabel = label
|
||||
searchFrom := 0
|
||||
for {
|
||||
rel := strings.Index(lower[searchFrom:], label)
|
||||
if rel < 0 {
|
||||
break
|
||||
}
|
||||
at := searchFrom + rel
|
||||
hits = append(hits, hit{at: at, label: label})
|
||||
searchFrom = at + len(label)
|
||||
}
|
||||
}
|
||||
if bestAt < 0 {
|
||||
if len(hits) == 0 {
|
||||
return ""
|
||||
}
|
||||
rest := user[bestAt+len(bestLabel):]
|
||||
if j := strings.Index(strings.ToLower(rest), "attrs:"); j >= 0 {
|
||||
rest = rest[:j]
|
||||
sort.Slice(hits, func(i, j int) bool { return hits[i].at < hits[j].at })
|
||||
for _, h := range hits {
|
||||
rest := user[h.at+len(h.label):]
|
||||
if j := strings.Index(strings.ToLower(rest), "attrs:"); j >= 0 {
|
||||
rest = rest[:j]
|
||||
}
|
||||
if j := strings.Index(strings.ToLower(rest), "attributes:"); j >= 0 {
|
||||
rest = rest[:j]
|
||||
}
|
||||
val := firstLine(rest)
|
||||
if val == "" || isPromptLabelTitle(val) {
|
||||
continue
|
||||
}
|
||||
return val
|
||||
}
|
||||
if j := strings.Index(strings.ToLower(rest), "attributes:"); j >= 0 {
|
||||
rest = rest[:j]
|
||||
}
|
||||
return firstLine(rest)
|
||||
return ""
|
||||
}
|
||||
|
||||
// isPromptLabelTitle detects enhance pollution where the model echoed a
|
||||
// prompt header ("Category:" / "Category: 120") as the product title.
|
||||
// prompt header ("Category:" / "Category: 120") as the product title, or
|
||||
// leaked instruction scaffolding from CategoryEnhanceUserTemplate /
|
||||
// AppendFormulaConstraints into name.
|
||||
func isPromptLabelTitle(s string) bool {
|
||||
s = strings.TrimSpace(s)
|
||||
if s == "" || s == "<nil>" {
|
||||
return false
|
||||
}
|
||||
if isPromptLeakageTitle(s) {
|
||||
return true
|
||||
}
|
||||
lower := strings.ToLower(s)
|
||||
for _, label := range []string{
|
||||
"category", "name", "desc", "description",
|
||||
@@ -611,8 +788,61 @@ func isPromptLabelTitle(s string) bool {
|
||||
return false
|
||||
}
|
||||
|
||||
// preferredProductTitle picks the first usable title, skipping empty values and
|
||||
// prompt-label echoes like "Category:" (seen on A1 Elkotex reprocess).
|
||||
// isPromptLeakageTitle detects when an LLM echoed enhance-prompt instructions
|
||||
// (Title formula / short retail title / Schema / Reply with ONLY JSON / …)
|
||||
// as the product name instead of a real title.
|
||||
func isPromptLeakageTitle(s string) bool {
|
||||
s = strings.TrimSpace(s)
|
||||
if s == "" || s == "<nil>" {
|
||||
return false
|
||||
}
|
||||
lower := strings.ToLower(s)
|
||||
for _, phrase := range promptLeakagePhrases {
|
||||
if strings.Contains(lower, phrase) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
// Long dumps of the enhance template: any formula/schema keyword is enough.
|
||||
if len([]rune(s)) > 120 {
|
||||
for _, kw := range promptLeakageLongKeywords {
|
||||
if strings.Contains(lower, kw) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// Phrases copied from aiprompts.CategoryEnhanceUserTemplate, BuiltInDefaults,
|
||||
// and processing.AppendFormulaConstraints / FormatTitleFormulaConstraint.
|
||||
var promptLeakagePhrases = []string{
|
||||
"title formula",
|
||||
"follow any",
|
||||
"constraints that follow",
|
||||
"short retail title",
|
||||
"use attrs",
|
||||
"write name in",
|
||||
"schema:",
|
||||
"reply with only json",
|
||||
"your reply is parsed as json",
|
||||
"description formula",
|
||||
"build name from attrs",
|
||||
"prefer attrs values",
|
||||
"order matters; join with",
|
||||
"do not hardcode a language",
|
||||
}
|
||||
|
||||
var promptLeakageLongKeywords = []string{
|
||||
"formula",
|
||||
"constraints",
|
||||
"schema",
|
||||
"json",
|
||||
"attrs",
|
||||
"retail title",
|
||||
}
|
||||
|
||||
// preferredProductTitle picks the first usable title, skipping empty values,
|
||||
// prompt-label echoes like "Category:", and instruction-text leakage.
|
||||
func preferredProductTitle(gtin string, candidates ...string) string {
|
||||
for _, c := range candidates {
|
||||
c = strings.TrimSpace(c)
|
||||
@@ -627,13 +857,172 @@ func preferredProductTitle(gtin string, candidates ...string) string {
|
||||
return "Product"
|
||||
}
|
||||
|
||||
func preferredProductDescription(candidates ...string) string {
|
||||
// Thin wrappers keep processing call sites stable; logic lives in company so
|
||||
// catalog.RepairWeakEnhanceHashes can reuse it without an import cycle.
|
||||
func isWeakPriorEnhanceDescription(priorDesc string, titles ...string) bool {
|
||||
return company.IsWeakPriorEnhanceDescription(priorDesc, titles...)
|
||||
}
|
||||
|
||||
func containsWeakFillerPhrase(desc string) bool {
|
||||
return company.ContainsWeakFillerPhrase(desc)
|
||||
}
|
||||
|
||||
func descriptionEchoesTitle(desc, title string) bool {
|
||||
return company.DescriptionEchoesTitle(desc, title)
|
||||
}
|
||||
|
||||
// preferredProductDescription picks the first usable description, skipping empty
|
||||
// values, prompt-label echoes, weak filler phrases, and copy that merely repeats
|
||||
// the product title.
|
||||
func preferredProductDescription(title string, candidates ...string) string {
|
||||
for _, c := range candidates {
|
||||
c = strings.TrimSpace(c)
|
||||
if c == "" || c == "<nil>" || isPromptLabelTitle(c) {
|
||||
continue
|
||||
}
|
||||
if descriptionEchoesTitle(c, title) {
|
||||
continue
|
||||
}
|
||||
if containsWeakFillerPhrase(c) {
|
||||
continue
|
||||
}
|
||||
return SanitizeOutput(c)
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func attrLookupCI(attrs map[string]any, keys ...string) string {
|
||||
if attrs == nil {
|
||||
return ""
|
||||
}
|
||||
for _, want := range keys {
|
||||
want = strings.TrimSpace(want)
|
||||
if want == "" {
|
||||
continue
|
||||
}
|
||||
if v := strings.TrimSpace(stringFromAny(attrs[want])); v != "" {
|
||||
return v
|
||||
}
|
||||
for k, raw := range attrs {
|
||||
if strings.EqualFold(strings.TrimSpace(k), want) {
|
||||
if v := strings.TrimSpace(stringFromAny(raw)); v != "" {
|
||||
return v
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func formatAttrDimParts(attrs map[string]any, maxParts int) []string {
|
||||
if attrs == nil || maxParts <= 0 {
|
||||
return nil
|
||||
}
|
||||
prefer := []string{
|
||||
"width", "height", "depth", "weight", "max_load", "max load", "load_capacity",
|
||||
"vesa", "screen_size", "diagonal", "color", "material", "size",
|
||||
}
|
||||
parts := make([]string, 0, maxParts)
|
||||
seen := map[string]struct{}{}
|
||||
add := func(k, v string) {
|
||||
k = strings.TrimSpace(k)
|
||||
v = strings.TrimSpace(v)
|
||||
if k == "" || v == "" {
|
||||
return
|
||||
}
|
||||
lk := strings.ToLower(k)
|
||||
if _, ok := seen[lk]; ok {
|
||||
return
|
||||
}
|
||||
seen[lk] = struct{}{}
|
||||
parts = append(parts, fmt.Sprintf("%s %s", k, v))
|
||||
}
|
||||
for _, k := range prefer {
|
||||
if len(parts) >= maxParts {
|
||||
break
|
||||
}
|
||||
if v := attrLookupCI(attrs, k); v != "" {
|
||||
add(k, v)
|
||||
}
|
||||
}
|
||||
return parts
|
||||
}
|
||||
|
||||
// synthesizeDescriptionFromTitle builds a short factual fallback when enhance
|
||||
// returns empty/title-echo/filler copy. Uses title, category, brand, model, and
|
||||
// key dims. language is a content-language code (en/sl/…) or English label.
|
||||
func synthesizeDescriptionFromTitle(title, category, language string, attrs map[string]any) string {
|
||||
title = strings.TrimSpace(title)
|
||||
if title == "" || title == "<nil>" || isPromptLabelTitle(title) {
|
||||
return ""
|
||||
}
|
||||
cat := strings.TrimSpace(category)
|
||||
if strings.EqualFold(cat, "general") {
|
||||
cat = ""
|
||||
}
|
||||
brand := attrLookupCI(attrs, "brand")
|
||||
model := attrLookupCI(attrs, "product_model", "model", "sku")
|
||||
dims := formatAttrDimParts(attrs, 3)
|
||||
sl := isSlovenianContentLanguage(language)
|
||||
|
||||
var b strings.Builder
|
||||
if sl {
|
||||
b.WriteString(title)
|
||||
switch {
|
||||
case cat != "" && brand != "":
|
||||
fmt.Fprintf(&b, " je izdelek v kategoriji %s znamke %s", cat, brand)
|
||||
case cat != "":
|
||||
fmt.Fprintf(&b, " je izdelek v kategoriji %s", cat)
|
||||
case brand != "":
|
||||
fmt.Fprintf(&b, " je izdelek znamke %s", brand)
|
||||
default:
|
||||
b.WriteString(" je katalogski izdelek z znanimi atributi")
|
||||
}
|
||||
if model != "" && !strings.Contains(strings.ToLower(title), strings.ToLower(model)) {
|
||||
fmt.Fprintf(&b, " (model %s)", model)
|
||||
}
|
||||
if len(dims) > 0 {
|
||||
fmt.Fprintf(&b, ". Ključne specifikacije: %s", strings.Join(dims, ", "))
|
||||
}
|
||||
b.WriteByte('.')
|
||||
} else {
|
||||
b.WriteString(title)
|
||||
switch {
|
||||
case cat != "" && brand != "":
|
||||
fmt.Fprintf(&b, " is a %s product from %s", cat, brand)
|
||||
case cat != "":
|
||||
fmt.Fprintf(&b, " is listed in the %s category", cat)
|
||||
case brand != "":
|
||||
fmt.Fprintf(&b, " is a product from %s", brand)
|
||||
default:
|
||||
b.WriteString(" is a catalog product with the known attributes")
|
||||
}
|
||||
if model != "" && !strings.Contains(strings.ToLower(title), strings.ToLower(model)) {
|
||||
fmt.Fprintf(&b, " (model %s)", model)
|
||||
}
|
||||
if len(dims) > 0 {
|
||||
fmt.Fprintf(&b, ". Key specs: %s", strings.Join(dims, ", "))
|
||||
}
|
||||
b.WriteByte('.')
|
||||
}
|
||||
out := SanitizeOutput(b.String())
|
||||
// Never emit sole retail-filler when title/attrs exist — strip legacy phrase if any helper reintroduces it.
|
||||
if containsWeakFillerPhrase(out) {
|
||||
out = strings.TrimSpace(strings.ReplaceAll(out, "Ready for retail listing.", ""))
|
||||
out = strings.TrimSpace(strings.ReplaceAll(out, "ready for retail listing.", ""))
|
||||
out = strings.TrimSpace(strings.Trim(out, ".")) + "."
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func isSlovenianContentLanguage(raw string) bool {
|
||||
raw = strings.TrimSpace(raw)
|
||||
if raw == "" {
|
||||
return false
|
||||
}
|
||||
if code, err := company.ParseLanguage(raw, false); err == nil && code == "sl" {
|
||||
return true
|
||||
}
|
||||
lower := strings.ToLower(raw)
|
||||
return lower == "slovenian" || strings.Contains(lower, "slovenian")
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user