1853 lines
60 KiB
Go
1853 lines
60 KiB
Go
package processing
|
||
|
||
import (
|
||
"context"
|
||
"encoding/json"
|
||
"fmt"
|
||
"log"
|
||
"sort"
|
||
"strings"
|
||
"time"
|
||
"unicode"
|
||
|
||
"github.com/descrybe/descrybe-v2/apps/api/internal/aiprompts"
|
||
"github.com/descrybe/descrybe-v2/apps/api/internal/company"
|
||
"github.com/descrybe/descrybe-v2/apps/api/internal/eprel"
|
||
)
|
||
|
||
// StepPolicy controls entitlement-gated steps (AI / EPREL).
|
||
type StepPolicy struct {
|
||
AllowAI bool
|
||
AllowEPREL bool
|
||
}
|
||
|
||
// RunSteps executes the multi-step product pipeline.
|
||
// OpenAI enhance runs only when Completer is configured, Enabled(), and policy.AllowAI.
|
||
// EPREL runs only when enricher enabled and policy.AllowEPREL.
|
||
func (e *Engine) RunSteps(ctx context.Context, companyID string, in ProductInput, processingType string, categoryNames []string, policy StepPolicy) (StepResult, error) {
|
||
steps := resolveSteps(processingType)
|
||
out := StepResult{
|
||
Attributes: map[string]any{},
|
||
ProcessedAttributes: map[string]any{},
|
||
FieldSources: map[string]any{},
|
||
EPREL: map[string]any{},
|
||
GPTResponse: map[string]any{"steps": []any{}},
|
||
Notes: []string{},
|
||
}
|
||
|
||
normalized := map[string]any{}
|
||
attrs := map[string]any{}
|
||
|
||
for _, step := range steps {
|
||
switch step {
|
||
case StepNormalize:
|
||
normalized = NormalizeMapped(in.Mapped, in.Raw)
|
||
out.Name = preferredProductTitle(in.GTIN,
|
||
stringFromAny(normalized["name"]),
|
||
stringFromAny(normalized["title"]),
|
||
in.Name,
|
||
in.PriorProcessedName,
|
||
)
|
||
out.Description = preferredProductDescription(out.Name,
|
||
stringFromAny(normalized["description"]),
|
||
in.Description,
|
||
in.PriorProcessedDescription,
|
||
)
|
||
// mapped_data.category / category_unique_id (unique_id codes) win.
|
||
applyCategoryFromMapped(&out, normalized, in.Mapped, in.Raw)
|
||
// otherwise keep existing processed category so enhance_only / reprocess
|
||
// cannot blank A1 legacy categories.
|
||
preserveCategoryIfEmpty(&out, in.PriorCategory)
|
||
out.FieldSources["normalize"] = "mapped+raw"
|
||
appendStepLog(out.GPTResponse, StepNormalize, map[string]any{
|
||
"keys": len(normalized),
|
||
})
|
||
|
||
case StepParseSpecs:
|
||
specVal := normalized["specifications"]
|
||
if specVal == nil {
|
||
specVal = in.Mapped["specifications"]
|
||
}
|
||
if specVal == nil {
|
||
specVal = in.Raw["specifications"]
|
||
}
|
||
parsed := ParseSpecifications(specVal)
|
||
// Also accept pre-mapped attributes map
|
||
if am, ok := normalized["attributes"].(map[string]any); ok {
|
||
for k, v := range ParseSpecifications(am) {
|
||
if _, exists := parsed[k]; !exists {
|
||
parsed[k] = v
|
||
}
|
||
}
|
||
}
|
||
attrs = SanitizeProductAttributes(parsed)
|
||
// Map feed/spec labels onto category_attributes (formula) keys early.
|
||
if allowed := enhanceAllowedAttrKeys(in, out.Category); allowed != nil {
|
||
attrs = MapAttrsOntoAllowedKeys(attrs, allowed)
|
||
}
|
||
out.Attributes = attrs
|
||
out.ProcessedAttributes = attrs
|
||
out.FieldSources["attributes"] = "specifications"
|
||
appendStepLog(out.GPTResponse, StepParseSpecs, map[string]any{
|
||
"count": len(attrs),
|
||
})
|
||
|
||
case StepFillFields:
|
||
normalized = FillMissingFields(normalized, attrs)
|
||
if len(in.StandardFields) > 0 {
|
||
normalized = FillMissingStandardFields(normalized, in.Raw, in.StandardFields)
|
||
}
|
||
out.Name = preferredProductTitle(in.GTIN,
|
||
stringFromAny(normalized["name"]),
|
||
stringFromAny(normalized["title"]),
|
||
out.Name,
|
||
in.Name,
|
||
in.PriorProcessedName,
|
||
)
|
||
out.Description = preferredProductDescription(out.Name,
|
||
stringFromAny(normalized["description"]),
|
||
out.Description,
|
||
in.Description,
|
||
)
|
||
applyCategoryFromMapped(&out, normalized, in.Mapped, in.Raw)
|
||
// Promote characteristic fields only — never core product identity/content
|
||
// (those belong on the V1 item root: title, description, ean, images, …).
|
||
promote := []string{"brand", "width", "height", "depth", "weight", "product_model", "warranty"}
|
||
for _, f := range in.StandardFields {
|
||
k := strings.TrimSpace(f.Key)
|
||
if k == "" || isReservedProductKey(k) || isInvalidAttributeKey(k) {
|
||
continue
|
||
}
|
||
promote = append(promote, k)
|
||
}
|
||
seen := map[string]bool{}
|
||
for _, k := range promote {
|
||
if seen[k] {
|
||
continue
|
||
}
|
||
seen[k] = true
|
||
if v := stringFromAny(normalized[k]); v != "" {
|
||
if _, exists := attrs[k]; !exists {
|
||
attrs[k] = v
|
||
}
|
||
out.FieldSources[k] = "fill_fields"
|
||
}
|
||
}
|
||
attrs = SanitizeProductAttributes(attrs)
|
||
// Remap again after category may have resolved in applyCategoryFromMapped.
|
||
if allowed := enhanceAllowedAttrKeys(in, out.Category); allowed != nil {
|
||
attrs = MapAttrsOntoAllowedKeys(attrs, allowed)
|
||
}
|
||
out.Attributes = attrs
|
||
out.ProcessedAttributes = attrs
|
||
appendStepLog(out.GPTResponse, StepFillFields, map[string]any{
|
||
"brand": stringFromAny(normalized["brand"]),
|
||
})
|
||
preserveCategoryIfEmpty(&out, in.PriorCategory)
|
||
|
||
case StepEPREL:
|
||
if !policy.AllowEPREL {
|
||
out.Notes = append(out.Notes, "eprel: skipped (not allowed for this job)")
|
||
appendStepLog(out.GPTResponse, StepEPREL, map[string]any{
|
||
"status": "skipped",
|
||
"reason": "entitlement_can_use_eprel",
|
||
})
|
||
break
|
||
}
|
||
// Also scan parsed attrs — eprel_id may only appear after parse_specs.
|
||
id := eprel.ExtractID(normalized, in.Mapped, in.Raw, attrs)
|
||
if id == "" {
|
||
out.Notes = append(out.Notes, "eprel: no id")
|
||
appendStepLog(out.GPTResponse, StepEPREL, map[string]any{"status": "skipped", "reason": "no_id"})
|
||
break
|
||
}
|
||
enricher := e.EPREL
|
||
if enricher == nil {
|
||
enricher = eprel.Disabled{}
|
||
}
|
||
if !enricher.Enabled() {
|
||
out.Notes = append(out.Notes, "eprel: enricher disabled")
|
||
out.EPREL = map[string]any{"id": id, "status": "skipped"}
|
||
appendStepLog(out.GPTResponse, StepEPREL, map[string]any{"status": "skipped", "eprel_id": id, "reason": "disabled"})
|
||
break
|
||
}
|
||
data, err := enricher.Fetch(ctx, id)
|
||
if err != nil {
|
||
out.Notes = append(out.Notes, "eprel: "+TruncateError(err))
|
||
attrs["eprel_id"] = id
|
||
out.Attributes = attrs
|
||
out.ProcessedAttributes = attrs
|
||
appendStepLog(out.GPTResponse, StepEPREL, map[string]any{"status": "failed", "error": TruncateError(err)})
|
||
// Non-fatal: continue pipeline
|
||
break
|
||
}
|
||
if data == nil {
|
||
attrs["eprel_id"] = id
|
||
out.Attributes = attrs
|
||
out.ProcessedAttributes = attrs
|
||
out.EPREL = map[string]any{"id": id, "status": "empty"}
|
||
appendStepLog(out.GPTResponse, StepEPREL, map[string]any{"status": "empty", "eprel_id": id})
|
||
break
|
||
}
|
||
attrs = eprel.MergeInto(attrs, data)
|
||
// Promote energy class onto the characteristic attr key when missing
|
||
// (feeds often omit it; EPREL API is the source of truth).
|
||
if data.EnergyClass != "" {
|
||
if cur := strings.TrimSpace(fmt.Sprint(attrs["energy_class"])); cur == "" || cur == "<nil>" {
|
||
attrs["energy_class"] = data.EnergyClass
|
||
}
|
||
}
|
||
out.Attributes = attrs
|
||
out.ProcessedAttributes = attrs
|
||
out.EPREL = map[string]any{
|
||
"id": data.ID,
|
||
"label": data.Label,
|
||
"pdf": data.PDF,
|
||
"energy_class": data.EnergyClass,
|
||
"energy_scale": data.EnergyScale,
|
||
}
|
||
out.FieldSources["eprel"] = "eprel_api"
|
||
appendStepLog(out.GPTResponse, StepEPREL, map[string]any{"status": "ok", "eprel_id": data.ID})
|
||
|
||
case StepCategorize:
|
||
// Vector (if enabled) then LLM taxonomy pick when category still empty.
|
||
runCategorizeStep(ctx, e, companyID, &out, in, categoryNames, policy)
|
||
|
||
case StepAIEnhance:
|
||
preservePriorEnhanceHash := func() {
|
||
if in.PriorEnhanceHash != "" {
|
||
out.FieldSources[FieldEnhanceInputHash] = in.PriorEnhanceHash
|
||
}
|
||
}
|
||
if !policy.AllowAI {
|
||
out.ProcessedName = out.Name
|
||
out.ProcessedDescription = out.Description
|
||
out.Notes = append(out.Notes, "ai_enhance: skipped (Free plan — upgrade for AI titles/descriptions)")
|
||
log.Printf("processing: ai_enhance skip reason=entitlement_can_use_ai")
|
||
preservePriorEnhanceHash()
|
||
appendStepLog(out.GPTResponse, StepAIEnhance, map[string]any{
|
||
"status": "skipped",
|
||
"reason": "entitlement_can_use_ai",
|
||
})
|
||
break
|
||
}
|
||
if !e.CompleterEnabled() {
|
||
out.ProcessedName = out.Name
|
||
out.ProcessedDescription = out.Description
|
||
out.Notes = append(out.Notes, "ai_enhance: skipped (platform OpenAI unset; configure admin settings or company BYOK)")
|
||
log.Printf("processing: ai_enhance skip reason=openai_not_configured")
|
||
preservePriorEnhanceHash()
|
||
appendStepLog(out.GPTResponse, StepAIEnhance, map[string]any{
|
||
"status": "skipped",
|
||
"reason": "openai_not_configured",
|
||
})
|
||
break
|
||
}
|
||
langs := in.ContentLanguages
|
||
if len(langs) == 0 {
|
||
langs = []string{in.Language}
|
||
}
|
||
if len(langs) == 0 {
|
||
langs = []string{company.DefaultLanguage}
|
||
}
|
||
primary := in.Language
|
||
if primary == "" {
|
||
primary = langs[0]
|
||
}
|
||
syncCategoryName(&out, in.CategoryNamesByUID)
|
||
displayCat := categoryDisplayLabel(out)
|
||
ensureEnergyClassFromEPREL(attrs)
|
||
enhanceAttrs := AttrsForEnhance(attrs, enhanceAllowedAttrKeys(in, out.Category))
|
||
localized := company.LocalizedContent{}
|
||
if in.PriorLocalized != nil {
|
||
for k, v := range in.PriorLocalized {
|
||
localized[k] = v
|
||
}
|
||
}
|
||
anyFailed := false
|
||
anyOK := false
|
||
allUnchanged := true
|
||
langMetas := make([]any, 0, len(langs))
|
||
for _, lang := range langs {
|
||
tpl := in.EnhanceByLang[lang]
|
||
if tpl.System == "" && tpl.User == "" && lang == primary {
|
||
tpl = PromptTemplates{System: in.EnhanceSystemTemplate, User: in.EnhanceUserTemplate}
|
||
}
|
||
catPrompt := in.CategoryEnhancePrompt
|
||
if lang != primary || catPrompt == "" {
|
||
catPrompt = categoryEnhancePromptFor(in.CategoryPromptsByLang, out.Category, lang, primary)
|
||
}
|
||
titleTpl, descTpl := categoryFormulasFor(in, out.Category)
|
||
if effective := aiprompts.EffectiveDescriptionTemplateAny(descTpl, catPrompt); effective != nil {
|
||
descTpl = effective
|
||
}
|
||
priorFields := company.FieldsForLanguage(in.PriorLocalized, lang)
|
||
priorHash := priorFields.EnhanceInputHash
|
||
priorName := priorFields.ProcessedName
|
||
priorDesc := priorFields.ProcessedDescription
|
||
if lang == primary {
|
||
if priorHash == "" {
|
||
priorHash = in.PriorEnhanceHash
|
||
}
|
||
if priorName == "" {
|
||
priorName = in.PriorProcessedName
|
||
}
|
||
if priorDesc == "" {
|
||
priorDesc = in.PriorProcessedDescription
|
||
}
|
||
}
|
||
name, desc, tokens, raw, err := e.enhance(ctx, ProductInput{
|
||
GTIN: in.GTIN,
|
||
Name: out.Name,
|
||
Description: out.Description,
|
||
Mapped: normalized,
|
||
BrandPrompt: in.BrandPrompt,
|
||
Language: lang,
|
||
EnhanceSystemTemplate: tpl.System,
|
||
EnhanceUserTemplate: tpl.User,
|
||
CategoryEnhancePrompt: catPrompt,
|
||
TitleTemplate: titleTpl,
|
||
DescriptionTemplate: descTpl,
|
||
AllowedAttrKeys: in.AllowedAttrKeys,
|
||
CategoryAttrKeys: in.CategoryAttrKeys,
|
||
PriorEnhanceHash: priorHash,
|
||
PriorProcessedName: priorName,
|
||
PriorProcessedDescription: priorDesc,
|
||
JobID: in.JobID,
|
||
CompanyID: in.CompanyID,
|
||
RawProductID: in.RawProductID,
|
||
CategoryUniqueID: out.Category,
|
||
}, displayCat, enhanceAttrs)
|
||
out.TotalTokens += tokens
|
||
status := enhanceStatusFromMeta(raw)
|
||
meta := map[string]any{"language": lang, "raw": raw}
|
||
if err != nil {
|
||
anyFailed = true
|
||
allUnchanged = false
|
||
meta["status"] = "failed"
|
||
meta["error"] = TruncateError(err)
|
||
out.Notes = append(out.Notes, "ai_enhance: "+TruncateError(err))
|
||
if lang == primary {
|
||
name, desc = out.Name, out.Description
|
||
} else if priorName != "" || priorDesc != "" {
|
||
name, desc = priorName, priorDesc
|
||
} else {
|
||
langMetas = append(langMetas, meta)
|
||
continue
|
||
}
|
||
} else if status == "unchanged" {
|
||
meta["status"] = "unchanged"
|
||
} else if status == "refused" || status == "synthesized" || status == "parse_failed" {
|
||
// Soft quality failure: keep usable copy / synth, never count as real enhance ok.
|
||
allUnchanged = false
|
||
meta["status"] = status
|
||
if reason := enhanceReasonFromMeta(raw); reason != "" {
|
||
out.Notes = append(out.Notes, "ai_enhance: "+status+" ("+reason+")")
|
||
} else {
|
||
out.Notes = append(out.Notes, "ai_enhance: "+status)
|
||
}
|
||
} else {
|
||
allUnchanged = false
|
||
anyOK = true
|
||
meta["status"] = status
|
||
}
|
||
// Prefer title-aware selection + synthesize before deciding hash persistence.
|
||
name = preferredProductTitle(in.GTIN, name, out.Name, priorName, in.Name)
|
||
desc = preferredProductDescription(name, desc, out.Description, priorDesc)
|
||
outerSynth := false
|
||
if descriptionNeedsEnhanceRepair(desc, descTpl, name) {
|
||
if synth := synthesizeProductDescription(name, displayCat, lang, enhanceAttrs, descTpl); synth != "" {
|
||
desc = synth
|
||
outerSynth = true
|
||
}
|
||
}
|
||
weakDesc := descriptionNeedsEnhanceRepair(desc, descTpl, name)
|
||
enhanceMetaTitle, enhanceMetaDesc := enhanceMetaFromRaw(raw)
|
||
if in.OmitSEOMeta {
|
||
enhanceMetaTitle, enhanceMetaDesc = "", ""
|
||
}
|
||
// Only persist enhance_input_hash for quality ok / hash-skip unchanged.
|
||
// Never copy input_hash from error/passthrough/thin/synthesized meta.
|
||
persistHash := ""
|
||
rawHash := enhanceHashFromMeta(raw)
|
||
switch {
|
||
case err != nil:
|
||
persistHash = ""
|
||
case status == "synthesized" || status == "refused" || outerSynth:
|
||
persistHash = ""
|
||
allUnchanged = false
|
||
case status == "unchanged" && !weakDesc:
|
||
persistHash = rawHash
|
||
case status == "ok" && !weakDesc:
|
||
persistHash = rawHash
|
||
default:
|
||
// thin ok / parse_failed / formula mismatch / skipped — do not poison reprocess skip
|
||
persistHash = ""
|
||
if status == "ok" && weakDesc {
|
||
allUnchanged = false
|
||
}
|
||
}
|
||
localized[lang] = company.LocalizedFields{
|
||
ProcessedName: name,
|
||
ProcessedDescription: desc,
|
||
EnhanceInputHash: persistHash,
|
||
MetaTitle: enhanceMetaTitle,
|
||
MetaDescription: enhanceMetaDesc,
|
||
}
|
||
// Preserve existing meta when re-enhancing titles only.
|
||
// Never keep a bare "| <unique_id>" poisoned meta_title.
|
||
// When dropping poisoned title, also drop empty/weak stub meta_description.
|
||
if prev := company.FieldsForLanguage(in.PriorLocalized, lang); prev.MetaTitle != "" || prev.MetaDescription != "" {
|
||
f := localized[lang]
|
||
poisonedTitle := isPoisonedMetaTitle(prev.MetaTitle)
|
||
if f.MetaTitle == "" && !poisonedTitle {
|
||
f.MetaTitle = prev.MetaTitle
|
||
}
|
||
if f.MetaDescription == "" {
|
||
weakStub := poisonedTitle && isWeakPriorEnhanceDescription(prev.MetaDescription, name, priorName)
|
||
if !weakStub {
|
||
f.MetaDescription = prev.MetaDescription
|
||
}
|
||
}
|
||
localized[lang] = f
|
||
}
|
||
langMetas = append(langMetas, meta)
|
||
if lang == primary {
|
||
out.ProcessedName = name
|
||
out.ProcessedDescription = desc
|
||
if name != "" {
|
||
out.Name = name
|
||
}
|
||
if desc != "" {
|
||
out.Description = desc
|
||
}
|
||
if lf := localized[lang]; lf.MetaTitle != "" {
|
||
out.MetaTitle = lf.MetaTitle
|
||
}
|
||
if lf := localized[lang]; lf.MetaDescription != "" {
|
||
out.MetaDescription = lf.MetaDescription
|
||
}
|
||
// Merge LLM attrs onto pipeline attrs (validated against category keys).
|
||
// Attrs are independent of title/description soft-fail (synthesized/refused).
|
||
if err == nil && status != "failed" && status != "parse_failed" {
|
||
if merged := mergeEnhanceAttrsInto(attrs, enhanceAttrsFromRaw(raw), enhanceAllowedAttrKeys(in, out.Category)); len(merged) > 0 {
|
||
attrs = merged
|
||
enhanceAttrs = AttrsForEnhance(attrs, enhanceAllowedAttrKeys(in, out.Category))
|
||
out.Attributes = attrs
|
||
out.ProcessedAttributes = attrs
|
||
out.FieldSources["attributes"] = "ai_enhance"
|
||
}
|
||
}
|
||
if persistHash != "" {
|
||
out.FieldSources[FieldEnhanceInputHash] = persistHash
|
||
} else {
|
||
delete(out.FieldSources, FieldEnhanceInputHash)
|
||
}
|
||
}
|
||
}
|
||
out.LocalizedContent = localized
|
||
if anyFailed && !anyOK {
|
||
if out.ProcessedName == "" {
|
||
out.ProcessedName = out.Name
|
||
}
|
||
if out.ProcessedDescription == "" {
|
||
out.ProcessedDescription = out.Description
|
||
}
|
||
// Timeout/error/empty/formula-miss: always synthesize a factual fallback when a title exists.
|
||
_, failDescTpl := categoryFormulasFor(in, out.Category)
|
||
if out.ProcessedName != "" && descriptionNeedsEnhanceRepair(out.ProcessedDescription, failDescTpl, out.ProcessedName, out.Name) {
|
||
if synth := synthesizeProductDescription(out.ProcessedName, displayCat, primary, enhanceAttrs, failDescTpl); synth != "" {
|
||
out.ProcessedDescription = synth
|
||
if out.Description == "" || descriptionNeedsEnhanceRepair(out.Description, failDescTpl, out.Name, out.ProcessedName) {
|
||
out.Description = synth
|
||
}
|
||
if lf, ok := localized[primary]; ok {
|
||
lf.ProcessedDescription = synth
|
||
if lf.ProcessedName == "" {
|
||
lf.ProcessedName = out.ProcessedName
|
||
}
|
||
localized[primary] = lf
|
||
}
|
||
}
|
||
}
|
||
// Attempted enhance (even on timeout) — record provider mode, not misleading unknown.
|
||
out.AIProviderMode = e.EngineProviderMode()
|
||
out.FieldSources["name"] = "ai_enhance_failed"
|
||
out.FieldSources["description"] = "ai_enhance_failed"
|
||
// Failed enhance must not leave a skippable hash behind.
|
||
delete(out.FieldSources, FieldEnhanceInputHash)
|
||
for lang, lf := range localized {
|
||
lf.EnhanceInputHash = ""
|
||
localized[lang] = lf
|
||
}
|
||
out.LocalizedContent = localized
|
||
errNote := ""
|
||
for _, m := range langMetas {
|
||
if mm, ok := m.(map[string]any); ok {
|
||
if e, ok := mm["error"].(string); ok && e != "" {
|
||
errNote = e
|
||
break
|
||
}
|
||
}
|
||
}
|
||
appendStepLog(out.GPTResponse, StepAIEnhance, map[string]any{
|
||
"status": "failed",
|
||
"error": errNote,
|
||
"languages": langMetas,
|
||
})
|
||
break
|
||
}
|
||
if allUnchanged {
|
||
out.SkipCreditDebit = true
|
||
out.FieldSources["name"] = "ai_enhance_unchanged"
|
||
out.FieldSources["description"] = "ai_enhance_unchanged"
|
||
out.Notes = append(out.Notes, "ai_enhance_unchanged")
|
||
log.Printf("processing: ai_enhance_unchanged")
|
||
} else {
|
||
out.AIProviderMode = e.EngineProviderMode()
|
||
out.FieldSources["name"] = "ai_enhance"
|
||
out.FieldSources["description"] = "ai_enhance"
|
||
for _, m := range langMetas {
|
||
if mm, ok := m.(map[string]any); ok {
|
||
reason, _ := mm["reason"].(string)
|
||
if reason == "" {
|
||
if raw, ok := mm["raw"].(map[string]any); ok {
|
||
reason, _ = raw["reason"].(string)
|
||
}
|
||
}
|
||
if reason != "" {
|
||
out.Notes = append(out.Notes, "ai_enhance: forced re-enhance ("+reason+")")
|
||
log.Printf("processing: ai_enhance forced re-enhance reason=%s", reason)
|
||
break
|
||
}
|
||
}
|
||
}
|
||
}
|
||
appendStepLog(out.GPTResponse, StepAIEnhance, map[string]any{
|
||
"status": map[string]any{"unchanged": allUnchanged, "ok": anyOK, "failed": anyFailed},
|
||
"languages": langMetas,
|
||
})
|
||
|
||
default:
|
||
appendStepLog(out.GPTResponse, step, map[string]any{"status": "unknown"})
|
||
}
|
||
}
|
||
|
||
// Pipelines without StepCategorize (enhance_only / normalize_only) still run
|
||
// vector + LLM categorize when category is empty so enhance is not fed a blank.
|
||
if !stepsContain(steps, StepCategorize) {
|
||
runCategorizeStep(ctx, e, companyID, &out, in, categoryNames, policy)
|
||
noteMissingCategory(&out, policy, e != nil && e.Vector != nil && e.Vector.Enabled())
|
||
}
|
||
preserveCategoryIfEmpty(&out, in.PriorCategory)
|
||
syncCategoryName(&out, in.CategoryNamesByUID)
|
||
out.Name = preferredProductTitle(in.GTIN, out.Name, out.ProcessedName, in.Name, in.PriorProcessedName)
|
||
out.ProcessedName = preferredProductTitle(in.GTIN, out.ProcessedName, out.Name, in.PriorProcessedName, in.Name)
|
||
out.Description = preferredProductDescription(out.Name, out.Description, out.ProcessedDescription, in.Description)
|
||
out.ProcessedDescription = preferredProductDescription(out.ProcessedName, out.ProcessedDescription, out.Description, in.PriorProcessedDescription)
|
||
if out.ProcessedName == "" {
|
||
out.ProcessedName = out.Name
|
||
}
|
||
// Titles are finalized first so name-as-category and formula leakage can be cleared.
|
||
scrubCategoryPollution(&out, in.CategoryNamesByUID, nil)
|
||
syncCategoryName(&out, in.CategoryNamesByUID)
|
||
displayCat := categoryDisplayLabel(out)
|
||
// After vector/mapped category resolution, formulas may key for the first time —
|
||
// repair short prose / title echo that ignored A1 description_template.
|
||
_, finalDescTpl := categoryFormulasFor(in, out.Category)
|
||
finalRepaired := false
|
||
if descriptionNeedsEnhanceRepair(out.ProcessedDescription, finalDescTpl, out.ProcessedName, out.Name) {
|
||
if synth := synthesizeProductDescription(out.ProcessedName, displayCat, in.Language, out.Attributes, finalDescTpl); synth != "" {
|
||
out.ProcessedDescription = synth
|
||
finalRepaired = true
|
||
}
|
||
}
|
||
if descriptionNeedsEnhanceRepair(out.Description, finalDescTpl, out.Name, out.ProcessedName) {
|
||
if out.ProcessedDescription != "" && !descriptionNeedsEnhanceRepair(out.ProcessedDescription, finalDescTpl, out.Name, out.ProcessedName) {
|
||
out.Description = out.ProcessedDescription
|
||
finalRepaired = true
|
||
} else if synth := synthesizeProductDescription(out.Name, displayCat, in.Language, out.Attributes, finalDescTpl); synth != "" {
|
||
out.Description = synth
|
||
finalRepaired = true
|
||
if descriptionNeedsEnhanceRepair(out.ProcessedDescription, finalDescTpl, out.ProcessedName, out.Name) {
|
||
out.ProcessedDescription = synth
|
||
}
|
||
}
|
||
}
|
||
if finalRepaired {
|
||
delete(out.FieldSources, FieldEnhanceInputHash)
|
||
primary := strings.TrimSpace(in.Language)
|
||
if primary == "" && len(in.ContentLanguages) > 0 {
|
||
primary = strings.TrimSpace(in.ContentLanguages[0])
|
||
}
|
||
if primary == "" {
|
||
primary = company.DefaultLanguage
|
||
}
|
||
for lang, lf := range out.LocalizedContent {
|
||
lf.EnhanceInputHash = ""
|
||
if lang == primary {
|
||
lf.ProcessedDescription = out.ProcessedDescription
|
||
if lf.ProcessedName == "" {
|
||
lf.ProcessedName = out.ProcessedName
|
||
}
|
||
}
|
||
out.LocalizedContent[lang] = lf
|
||
}
|
||
}
|
||
if out.Attributes == nil {
|
||
out.Attributes = map[string]any{}
|
||
}
|
||
// Persist the same sanitize + category allowlist used by enhance (poll already
|
||
// projects clean attrs; DB must not keep feed junk like zavora/vzmetenje).
|
||
allowed := enhanceAllowedAttrKeys(in, out.Category)
|
||
out.Attributes = AttrsForPersist(out.Attributes, allowed)
|
||
out.ProcessedAttributes = AttrsForPersist(out.ProcessedAttributes, allowed)
|
||
if len(out.ProcessedAttributes) == 0 {
|
||
out.ProcessedAttributes = out.Attributes
|
||
}
|
||
// Prefer EngineProviderMode (job/completer label) over stamping "unknown" on
|
||
// 0-token paths (hash-skip / AI skipped). processOne also treats unknown as empty.
|
||
if isUnknownProviderMode(out.AIProviderMode) {
|
||
out.AIProviderMode = preferKnownProviderMode(e.EngineProviderMode(), out.AIProviderMode)
|
||
}
|
||
if len(out.Notes) > 0 {
|
||
out.GPTResponse["notes"] = out.Notes
|
||
}
|
||
return out, nil
|
||
}
|
||
|
||
// preserveCategoryIfEmpty keeps an existing processed category when normalize/AI
|
||
// left Category empty (common for A1 feeds where category lives only on processed).
|
||
func preserveCategoryIfEmpty(out *StepResult, prior string) {
|
||
if out == nil || strings.TrimSpace(out.Category) != "" {
|
||
return
|
||
}
|
||
prior = strings.TrimSpace(prior)
|
||
if prior == "" || isUnusableCategoryValue(prior, out.ProcessedName, out.Name) {
|
||
return
|
||
}
|
||
out.Category = SanitizeText(prior)
|
||
if out.FieldSources == nil {
|
||
out.FieldSources = map[string]any{}
|
||
}
|
||
out.FieldSources["category"] = "prior_processed"
|
||
}
|
||
|
||
// tryVectorCategorize sets Category from embeddings when mapped unique_id is absent.
|
||
// Requires policy.AllowAI and a configured/enabled VectorCategorizer.
|
||
func tryVectorCategorize(ctx context.Context, e *Engine, companyID string, out *StepResult, categoryNames []string, policy StepPolicy) {
|
||
if out == nil || strings.TrimSpace(out.Category) != "" {
|
||
return
|
||
}
|
||
if !policy.AllowAI {
|
||
return
|
||
}
|
||
if e == nil || e.Vector == nil || !e.Vector.Enabled() {
|
||
return
|
||
}
|
||
text := strings.TrimSpace(out.Name + " " + out.Description)
|
||
if text == "" {
|
||
return
|
||
}
|
||
cat, err := e.Vector.SuggestCategory(ctx, companyID, text, categoryNames)
|
||
if err != nil {
|
||
out.Notes = append(out.Notes, "vector_categorize: "+TruncateError(err))
|
||
appendStepLog(out.GPTResponse, "vector_categorize", map[string]any{
|
||
"status": "failed",
|
||
"error": TruncateError(err),
|
||
})
|
||
return
|
||
}
|
||
if strings.TrimSpace(cat) == "" || isUnusableCategoryValue(cat, out.ProcessedName, out.Name) {
|
||
return
|
||
}
|
||
out.Category = SanitizeOutput(cat)
|
||
if out.FieldSources == nil {
|
||
out.FieldSources = map[string]any{}
|
||
}
|
||
out.FieldSources["category"] = "vector"
|
||
out.Notes = append(out.Notes, "category: vector")
|
||
appendStepLog(out.GPTResponse, "vector_categorize", map[string]any{
|
||
"status": "ok",
|
||
"category": out.Category,
|
||
})
|
||
}
|
||
|
||
// noteMissingCategory records why Category stayed empty (mapped absent; vector skipped or failed).
|
||
// Used for pipelines that skip StepCategorize (vector-only post-pass).
|
||
func noteMissingCategory(out *StepResult, policy StepPolicy, vectorEnabled bool) {
|
||
if out == nil || strings.TrimSpace(out.Category) != "" {
|
||
return
|
||
}
|
||
for _, n := range out.Notes {
|
||
if strings.HasPrefix(n, "category: unset") || strings.HasPrefix(n, "category: vector") ||
|
||
strings.HasPrefix(n, "category: llm") || strings.HasPrefix(n, "ai_categorize:") {
|
||
return
|
||
}
|
||
}
|
||
switch {
|
||
case !policy.AllowAI:
|
||
out.Notes = append(out.Notes, "category: unset (no mapped unique_id; AI/vector not allowed)")
|
||
case !vectorEnabled:
|
||
out.Notes = append(out.Notes, "category: unset (no mapped unique_id; vector embeddings unavailable)")
|
||
default:
|
||
out.Notes = append(out.Notes, "category: unset (no mapped unique_id; vector did not match)")
|
||
}
|
||
}
|
||
|
||
func resolveSteps(processingType string) []string {
|
||
switch strings.ToLower(strings.TrimSpace(processingType)) {
|
||
case "enhance", "enhance_only", "enhance-only", "title", "description":
|
||
return []string{StepNormalize, StepAIEnhance}
|
||
case "attributes", "attributes_only", "specs", "specifications":
|
||
return []string{StepNormalize, StepParseSpecs, StepFillFields}
|
||
case "eprel", "eprel_only":
|
||
// parse_specs first so eprel_id buried in specifications/attributes is visible.
|
||
return []string{StepNormalize, StepParseSpecs, StepEPREL}
|
||
case "normalize_only":
|
||
return []string{StepNormalize}
|
||
case "categorize", "categorize_only":
|
||
// Taxonomy assign only (vector then LLM) — no title/description rewrite.
|
||
return []string{StepNormalize, StepParseSpecs, StepFillFields, StepEPREL, StepCategorize}
|
||
case "categorize_enhance":
|
||
return append([]string{}, CanonicalSteps...)
|
||
default: // full
|
||
return append([]string{}, CanonicalSteps...)
|
||
}
|
||
}
|
||
|
||
func stepsContain(steps []string, want string) bool {
|
||
for _, s := range steps {
|
||
if s == want {
|
||
return true
|
||
}
|
||
}
|
||
return false
|
||
}
|
||
|
||
// InitialStepProgress builds pending step_progress rows for a job.
|
||
func InitialStepProgress(processingType string) []StepProgress {
|
||
steps := resolveSteps(processingType)
|
||
out := make([]StepProgress, 0, len(steps))
|
||
for _, s := range steps {
|
||
out = append(out, StepProgress{Step: s, Status: "pending"})
|
||
}
|
||
return out
|
||
}
|
||
|
||
func appendStepLog(gpt map[string]any, name string, raw any) {
|
||
steps, _ := gpt["steps"].([]any)
|
||
gpt["steps"] = append(steps, map[string]any{"step": name, "raw": raw})
|
||
}
|
||
|
||
// descriptionFormulaRetrySuffix is appended once when LLM JSON ignored the
|
||
// category description_template (short prose / title echo instead of multi-section HTML).
|
||
const descriptionFormulaRetrySuffix = "\n\nINVALID DESCRIPTION. Your JSON ignored the Description formula. Reply with ONLY one JSON object; \"description\" must be ONE HTML string covering each formula section in order with matching tags (h1/h2/h3/h4, p, ul)."
|
||
|
||
// titleRewriteRetrySuffix is appended when the Title role requires a new retail
|
||
// name but the model echoed the supplier Name unchanged.
|
||
const titleRewriteRetrySuffix = "\n\nINVALID NAME. Category Title instructions require a NEW product name (formula / rewrite). \"name\" must NOT equal the supplier Name. Reply with ONLY one JSON object; rebuild \"name\" from type + brand + model + color (or the Title formula) in {{language}}."
|
||
|
||
// descriptionNeedsEnhanceRepair is true when desc is weak/empty/title-echo or
|
||
// fails an active category description_template (A1 multi-section HTML).
|
||
// Heuristic invent/synth is intentionally excluded here — enhanceHashForceReason
|
||
// blocks hash-skip for synth so reprocess can still upgrade to real LLM copy.
|
||
func descriptionNeedsEnhanceRepair(desc string, template any, titles ...string) bool {
|
||
if isWeakPriorEnhanceDescription(desc, titles...) {
|
||
return true
|
||
}
|
||
return !descriptionSatisfiesFormula(desc, template)
|
||
}
|
||
|
||
// titlesAreEquivalent treats supplier vs enriched names as the same when only
|
||
// case/spacing/punctuation differ — used to detect Title formula copy-paste.
|
||
func titlesAreEquivalent(a, b string) bool {
|
||
na := normalizeTitleCompare(a)
|
||
nb := normalizeTitleCompare(b)
|
||
if na == "" || nb == "" {
|
||
return false
|
||
}
|
||
return na == nb
|
||
}
|
||
|
||
func normalizeTitleCompare(s string) string {
|
||
s = strings.TrimSpace(strings.ToLower(s))
|
||
if s == "" || s == "<nil>" {
|
||
return ""
|
||
}
|
||
var b strings.Builder
|
||
b.Grow(len(s))
|
||
prevSpace := false
|
||
for _, r := range s {
|
||
switch {
|
||
case unicode.IsLetter(r) || unicode.IsDigit(r):
|
||
b.WriteRune(r)
|
||
prevSpace = false
|
||
case unicode.IsSpace(r) || r == '-' || r == '_' || r == '/' || r == ',':
|
||
if !prevSpace && b.Len() > 0 {
|
||
b.WriteByte(' ')
|
||
prevSpace = true
|
||
}
|
||
}
|
||
}
|
||
return strings.TrimSpace(b.String())
|
||
}
|
||
|
||
func titleNeedsRewriteRetry(in ProductInput, name string) bool {
|
||
if !aiprompts.CategoryTitlePromptRequiresRewrite(in.CategoryEnhancePrompt) {
|
||
return false
|
||
}
|
||
return titlesAreEquivalent(name, in.Name)
|
||
}
|
||
|
||
// enhanceHashForceReason returns why a matching prior enhance_input_hash must not
|
||
// skip the LLM (empty = safe to reuse as ai_enhance_unchanged).
|
||
func enhanceHashForceReason(in ProductInput) string {
|
||
if isPromptLabelTitle(in.PriorProcessedName) {
|
||
return "prompt_label_title"
|
||
}
|
||
if reason := company.EnhanceHashSkipBlockReason(in.PriorProcessedDescription, in.PriorProcessedName, in.Name); reason != "" {
|
||
return reason
|
||
}
|
||
if !descriptionSatisfiesFormula(in.PriorProcessedDescription, in.DescriptionTemplate) {
|
||
return "formula-mismatch"
|
||
}
|
||
if titleNeedsRewriteRetry(in, in.PriorProcessedName) {
|
||
return "title-copy-paste"
|
||
}
|
||
return ""
|
||
}
|
||
|
||
func (e *Engine) enhance(ctx context.Context, in ProductInput, category string, attrs map[string]any) (string, string, int, any, error) {
|
||
// Prompt Description HTML overlays count as the formula when description_template
|
||
// is empty — otherwise enhance accepts supplier-like prose and skips formula retry.
|
||
if effective := aiprompts.EffectiveDescriptionTemplateAny(in.DescriptionTemplate, in.CategoryEnhancePrompt); effective != nil {
|
||
in.DescriptionTemplate = effective
|
||
}
|
||
catUID := strings.TrimSpace(in.CategoryUniqueID)
|
||
catName := strings.TrimSpace(category)
|
||
sysTpl, userTpl := resolveProductPromptTemplates(in)
|
||
hash := HashEnhanceInput(category, in.Name, in.Description, in.BrandPrompt, in.Language, sysTpl, userTpl, attrs)
|
||
if e == nil || e.Completer == nil {
|
||
name := preferredProductTitle(in.GTIN, in.Name, in.PriorProcessedName)
|
||
logEnhanceOutcome(in, catUID, catName, "skip", "completer_nil", Completion{}, 0, "")
|
||
return name,
|
||
preferredProductDescription(name, in.Description, in.PriorProcessedDescription),
|
||
0, map[string]any{"status": "skipped", "reason": "completer_nil"}, nil
|
||
}
|
||
// Skip LLM when inputs match the last successful enhance (before any credit debit),
|
||
// but never reuse thin / title-echo / invent-synth / formula-mismatch priors,
|
||
// or a prompt-leakage title.
|
||
var forceReason string
|
||
if in.PriorEnhanceHash != "" && in.PriorEnhanceHash == hash &&
|
||
(in.PriorProcessedName != "" || in.PriorProcessedDescription != "") {
|
||
forceReason = enhanceHashForceReason(in)
|
||
if forceReason == "" {
|
||
name := preferredProductTitle(in.GTIN, in.PriorProcessedName, in.Name)
|
||
desc := preferredProductDescription(name, in.PriorProcessedDescription, in.Description)
|
||
logEnhanceOutcome(in, catUID, catName, "skip", "unchanged_hash", Completion{}, 0, "")
|
||
return name, desc, 0, map[string]any{
|
||
"status": "unchanged",
|
||
"input_hash": hash,
|
||
}, nil
|
||
}
|
||
log.Printf("processing: ai_enhance skip_blocked reason=%s category=%q name=%q",
|
||
forceReason, truncateRunes(category, 80), truncateRunes(in.Name, 80))
|
||
}
|
||
system, user := RenderProductEnhancePrompts(sysTpl, userTpl, category, in.Name, in.Description, in.GTIN, in.BrandPrompt, in.Language, attrs)
|
||
logEnhancePrompt(in, catUID, catName, system, user)
|
||
started := time.Now()
|
||
comp, obj, err := CompleteJSON(ctx, e.Completer, system, user, CompleteOptions{
|
||
MaxTokens: MaxTokensEnhance,
|
||
Temperature: DefaultStructuredTemp,
|
||
ReasoningEffort: "low",
|
||
})
|
||
elapsed := time.Since(started)
|
||
if err != nil {
|
||
// Network/provider failure vs parse failure after retry
|
||
name := preferredProductTitle(in.GTIN, in.Name, in.PriorProcessedName)
|
||
desc := preferredProductDescription(name, in.Description, in.PriorProcessedDescription)
|
||
outcome := "refuse"
|
||
reason := "empty_or_provider_error"
|
||
if descriptionNeedsEnhanceRepair(desc, in.DescriptionTemplate, name) {
|
||
if synth := synthesizeProductDescription(name, category, in.Language, attrs, in.DescriptionTemplate); synth != "" {
|
||
desc = synth
|
||
outcome = "synthesized"
|
||
}
|
||
}
|
||
if obj == nil && comp.Text == "" {
|
||
logEnhanceOutcome(in, catUID, catName, outcome, reason+":"+TruncateError(err), comp, elapsed, forceReason)
|
||
return name, desc, 0, map[string]any{
|
||
"provider": "passthrough",
|
||
"error": TruncateError(err),
|
||
}, err
|
||
}
|
||
// Parse failed after retry — keep usable copy; synthesize when empty/weak/formula-miss.
|
||
if outcome != "synthesized" {
|
||
outcome = "refuse"
|
||
reason = "parse_failed"
|
||
} else {
|
||
reason = "parse_failed"
|
||
}
|
||
logEnhanceOutcome(in, catUID, catName, outcome, reason, comp, elapsed, forceReason)
|
||
return name, desc, comp.TotalTokens, map[string]any{
|
||
"status": "parse_failed",
|
||
"reason": "invalid_json",
|
||
"error": "AI returned invalid JSON; kept original title/description",
|
||
"raw": truncateRunes(comp.Text, 200),
|
||
}, nil
|
||
}
|
||
llmName := SanitizeOutput(fmt.Sprint(obj["name"]))
|
||
llmDesc := SanitizeOutput(fmt.Sprint(obj["description"]))
|
||
llmMetaTitle, llmMetaDesc := metaFieldsFromEnhanceObj(obj)
|
||
llmAttrs := attrsFromEnhanceObj(obj)
|
||
// Never prefer-fallback to originals for empty/leakage LLM fields — that stamped
|
||
// status=ok + input_hash on garbage enhance output (prod: 30–272ms "ok", wrong copy).
|
||
if reason := llmEnhanceHardRefuseReason(llmName, llmDesc); reason != "" {
|
||
name := preferredProductTitle(in.GTIN, llmName, in.Name, in.PriorProcessedName)
|
||
desc := preferredProductDescription(name, llmDesc)
|
||
synthesized := false
|
||
if descriptionNeedsEnhanceRepair(desc, in.DescriptionTemplate, name) {
|
||
if desc == "" {
|
||
desc = preferredProductDescription(name, in.Description, in.PriorProcessedDescription)
|
||
}
|
||
if descriptionNeedsEnhanceRepair(desc, in.DescriptionTemplate, name) {
|
||
if synth := synthesizeProductDescription(name, category, in.Language, attrs, in.DescriptionTemplate); synth != "" {
|
||
desc = synth
|
||
synthesized = true
|
||
}
|
||
}
|
||
}
|
||
status := "refused"
|
||
outcome := "refuse"
|
||
if synthesized {
|
||
status = "synthesized"
|
||
outcome = "synthesized"
|
||
}
|
||
meta := map[string]any{
|
||
"status": status,
|
||
"reason": reason,
|
||
"raw": comp.Raw,
|
||
}
|
||
attachEnhanceMeta(meta, llmMetaTitle, llmMetaDesc)
|
||
attachEnhanceAttrs(meta, llmAttrs)
|
||
if forceReason != "" {
|
||
meta["forced_reenhance"] = true
|
||
meta["force_reason"] = forceReason
|
||
}
|
||
logEnhanceOutcome(in, catUID, catName, outcome, reason, comp, elapsed, forceReason)
|
||
return name, desc, comp.TotalTokens, meta, nil
|
||
}
|
||
|
||
name := preferredProductTitle(in.GTIN, llmName, in.Name, in.PriorProcessedName)
|
||
// Quality-gate on LLM description alone (do not absorb originals yet).
|
||
desc := preferredProductDescription(name, llmDesc)
|
||
didFormulaRetry := false
|
||
needsDescRetry := !descriptionSatisfiesFormula(desc, in.DescriptionTemplate) &&
|
||
FormatDescriptionFormulaConstraint(in.DescriptionTemplate) != ""
|
||
needsTitleRetry := titleNeedsRewriteRetry(in, name)
|
||
// One retry when Description HTML formula and/or Title rewrite were ignored.
|
||
if needsDescRetry || needsTitleRetry {
|
||
didFormulaRetry = true
|
||
retryUser := user
|
||
if needsDescRetry {
|
||
retryUser += descriptionFormulaRetrySuffix
|
||
}
|
||
if needsTitleRetry {
|
||
retryUser += titleRewriteRetrySuffix
|
||
}
|
||
retryStarted := time.Now()
|
||
comp2, obj2, err2 := CompleteJSON(ctx, e.Completer, system, retryUser, CompleteOptions{
|
||
MaxTokens: MaxTokensEnhance,
|
||
Temperature: DefaultStructuredTemp,
|
||
ReasoningEffort: "low",
|
||
})
|
||
elapsed += time.Since(retryStarted)
|
||
comp.PromptTokens += comp2.PromptTokens
|
||
comp.OutputTokens += comp2.OutputTokens
|
||
comp.TotalTokens += comp2.TotalTokens
|
||
if err2 == nil && obj2 != nil {
|
||
n2 := SanitizeOutput(fmt.Sprint(obj2["name"]))
|
||
d2 := SanitizeOutput(fmt.Sprint(obj2["description"]))
|
||
if llmEnhanceHardRefuseReason(n2, d2) == "" {
|
||
// Prefer rewritten title over supplier echo when Title role requires it.
|
||
if needsTitleRetry && n2 != "" && !titlesAreEquivalent(n2, in.Name) {
|
||
name = preferredProductTitle(in.GTIN, n2)
|
||
} else {
|
||
name = preferredProductTitle(in.GTIN, n2, name, in.Name, in.PriorProcessedName)
|
||
}
|
||
desc = preferredProductDescription(name, d2)
|
||
mt2, md2 := metaFieldsFromEnhanceObj(obj2)
|
||
if mt2 != "" {
|
||
llmMetaTitle = mt2
|
||
}
|
||
if md2 != "" {
|
||
llmMetaDesc = md2
|
||
}
|
||
if a2 := attrsFromEnhanceObj(obj2); len(a2) > 0 {
|
||
llmAttrs = a2
|
||
}
|
||
if comp2.Raw != nil {
|
||
comp.Raw = comp2.Raw
|
||
}
|
||
if comp2.Text != "" {
|
||
comp.Text = comp2.Text
|
||
}
|
||
}
|
||
}
|
||
}
|
||
synthesized := false
|
||
if descriptionNeedsEnhanceRepair(desc, in.DescriptionTemplate, name) {
|
||
if desc == "" {
|
||
desc = preferredProductDescription(name, in.Description, in.PriorProcessedDescription)
|
||
}
|
||
if descriptionNeedsEnhanceRepair(desc, in.DescriptionTemplate, name) {
|
||
if synth := synthesizeProductDescription(name, category, in.Language, attrs, in.DescriptionTemplate); synth != "" {
|
||
desc = synth
|
||
synthesized = true
|
||
}
|
||
}
|
||
}
|
||
meta := map[string]any{
|
||
"status": "ok",
|
||
"raw": comp.Raw,
|
||
}
|
||
attachEnhanceMeta(meta, llmMetaTitle, llmMetaDesc)
|
||
attachEnhanceAttrs(meta, llmAttrs)
|
||
if forceReason != "" {
|
||
meta["reason"] = forceReason
|
||
meta["forced_reenhance"] = true
|
||
}
|
||
outcome := "ok"
|
||
reason := forceReason
|
||
// Heuristic synthesize must not poison enhance_input_hash (would hash-skip
|
||
// formula-aware LLM copy on reprocess). Only persist hash for real LLM quality.
|
||
if synthesized {
|
||
meta["status"] = "synthesized"
|
||
outcome = "synthesized"
|
||
if didFormulaRetry {
|
||
reason = "formula_retry"
|
||
} else {
|
||
reason = "weak_or_formula_mismatch"
|
||
}
|
||
meta["reason"] = reason
|
||
} else if descriptionNeedsEnhanceRepair(desc, in.DescriptionTemplate, name) {
|
||
meta["status"] = "refused"
|
||
outcome = "refuse"
|
||
if didFormulaRetry {
|
||
reason = "formula_retry"
|
||
} else {
|
||
reason = "weak_or_formula_mismatch"
|
||
}
|
||
meta["reason"] = reason
|
||
} else {
|
||
meta["input_hash"] = hash
|
||
if didFormulaRetry {
|
||
reason = "formula_retry"
|
||
if meta["reason"] == nil {
|
||
meta["reason"] = reason
|
||
}
|
||
}
|
||
}
|
||
logEnhanceOutcome(in, catUID, catName, outcome, reason, comp, elapsed, forceReason)
|
||
return name, desc, comp.TotalTokens, meta, nil
|
||
}
|
||
|
||
// attachEnhanceMeta stores parsed SEO fields on enhance raw meta for RunSteps.
|
||
func attachEnhanceMeta(meta map[string]any, title, description string) {
|
||
if meta == nil {
|
||
return
|
||
}
|
||
if title != "" {
|
||
meta["meta_title"] = title
|
||
}
|
||
if description != "" {
|
||
meta["meta_description"] = description
|
||
}
|
||
}
|
||
|
||
// attachEnhanceAttrs stores parsed attrs on enhance raw meta for RunSteps merge.
|
||
func attachEnhanceAttrs(meta map[string]any, attrs map[string]any) {
|
||
if meta == nil || len(attrs) == 0 {
|
||
return
|
||
}
|
||
meta["attrs"] = attrs
|
||
}
|
||
|
||
// attrsFromEnhanceObj extracts an attrs/attributes object from enhance JSON.
|
||
func attrsFromEnhanceObj(obj map[string]any) map[string]any {
|
||
if obj == nil {
|
||
return nil
|
||
}
|
||
raw := obj["attrs"]
|
||
if raw == nil {
|
||
raw = obj["attributes"]
|
||
}
|
||
return coerceAttrMap(raw)
|
||
}
|
||
|
||
// enhanceAttrsFromRaw reads attrs stashed on enhance raw meta by attachEnhanceAttrs.
|
||
func enhanceAttrsFromRaw(raw any) map[string]any {
|
||
m, ok := raw.(map[string]any)
|
||
if !ok || m == nil {
|
||
return nil
|
||
}
|
||
if v, ok := m["attrs"]; ok {
|
||
return coerceAttrMap(v)
|
||
}
|
||
return nil
|
||
}
|
||
|
||
// coerceAttrMap normalizes JSON object / map[string]string into map[string]any.
|
||
func coerceAttrMap(raw any) map[string]any {
|
||
switch v := raw.(type) {
|
||
case map[string]any:
|
||
if len(v) == 0 {
|
||
return nil
|
||
}
|
||
out := make(map[string]any, len(v))
|
||
for k, val := range v {
|
||
k = strings.TrimSpace(k)
|
||
if k == "" || val == nil {
|
||
continue
|
||
}
|
||
out[k] = val
|
||
}
|
||
if len(out) == 0 {
|
||
return nil
|
||
}
|
||
return out
|
||
case map[string]string:
|
||
if len(v) == 0 {
|
||
return nil
|
||
}
|
||
out := make(map[string]any, len(v))
|
||
for k, val := range v {
|
||
k = strings.TrimSpace(k)
|
||
val = strings.TrimSpace(val)
|
||
if k == "" || val == "" {
|
||
continue
|
||
}
|
||
out[k] = val
|
||
}
|
||
if len(out) == 0 {
|
||
return nil
|
||
}
|
||
return out
|
||
default:
|
||
return nil
|
||
}
|
||
}
|
||
|
||
// mergeEnhanceAttrsInto merges LLM attrs onto base after AttrsForEnhance validation
|
||
// (MapAttrsOntoAllowedKeys + category allowlist). Empty LLM attrs → nil (no change).
|
||
func mergeEnhanceAttrsInto(base, llmAttrs map[string]any, allowed map[string]struct{}) map[string]any {
|
||
if len(llmAttrs) == 0 {
|
||
return nil
|
||
}
|
||
validated := AttrsForEnhance(llmAttrs, allowed)
|
||
if len(validated) == 0 {
|
||
return nil
|
||
}
|
||
out := make(map[string]any, len(base)+len(validated))
|
||
for k, v := range base {
|
||
out[k] = v
|
||
}
|
||
for k, v := range validated {
|
||
if !attrValuePresent(v) {
|
||
continue
|
||
}
|
||
out[k] = v
|
||
}
|
||
return out
|
||
}
|
||
|
||
// llmEnhanceHardRefuseReason reports empty/leakage LLM fields that must never be
|
||
// accepted as quality enhance (even when originals could fill the gap).
|
||
func llmEnhanceHardRefuseReason(name, desc string) string {
|
||
name = strings.TrimSpace(name)
|
||
desc = strings.TrimSpace(desc)
|
||
if name == "" || name == "<nil>" {
|
||
return "empty_name"
|
||
}
|
||
if isPromptLabelTitle(name) {
|
||
return "prompt_leakage_name"
|
||
}
|
||
if desc == "" || desc == "<nil>" {
|
||
return "empty_description"
|
||
}
|
||
return ""
|
||
}
|
||
|
||
func sanitizeJSON(v any) string {
|
||
if v == nil {
|
||
return "{}"
|
||
}
|
||
b, err := json.Marshal(v)
|
||
if err != nil {
|
||
return "{}"
|
||
}
|
||
return SanitizeText(string(b))
|
||
}
|
||
|
||
func firstLine(s string) string {
|
||
s = strings.TrimSpace(s)
|
||
if i := strings.IndexByte(s, '\n'); i >= 0 {
|
||
s = s[:i]
|
||
}
|
||
return SanitizeOutput(strings.Trim(s, "\"'` "))
|
||
}
|
||
|
||
// labeledPromptValue returns the first usable line after any of the given labels
|
||
// (case-insensitive), e.g. "Name:" / "Desc:" from ProductEnhanceUser.
|
||
// Skips matches whose value is prompt-label / formula-leakage text so instruction
|
||
// bullets like "- name: short retail title; follow any Title formula…" do not
|
||
// win over the later "Name: <product>" line in CategoryEnhanceUserTemplate.
|
||
func labeledPromptValue(user string, labels ...string) string {
|
||
lower := strings.ToLower(user)
|
||
type hit struct {
|
||
at int
|
||
label string
|
||
}
|
||
var hits []hit
|
||
for _, label := range labels {
|
||
label = strings.ToLower(strings.TrimSpace(label))
|
||
if label == "" {
|
||
continue
|
||
}
|
||
searchFrom := 0
|
||
for {
|
||
rel := strings.Index(lower[searchFrom:], label)
|
||
if rel < 0 {
|
||
break
|
||
}
|
||
at := searchFrom + rel
|
||
hits = append(hits, hit{at: at, label: label})
|
||
searchFrom = at + len(label)
|
||
}
|
||
}
|
||
if len(hits) == 0 {
|
||
return ""
|
||
}
|
||
sort.Slice(hits, func(i, j int) bool { return hits[i].at < hits[j].at })
|
||
for _, h := range hits {
|
||
rest := user[h.at+len(h.label):]
|
||
if j := strings.Index(strings.ToLower(rest), "attrs:"); j >= 0 {
|
||
rest = rest[:j]
|
||
}
|
||
if j := strings.Index(strings.ToLower(rest), "attributes:"); j >= 0 {
|
||
rest = rest[:j]
|
||
}
|
||
val := firstLine(rest)
|
||
if val == "" || isPromptLabelTitle(val) {
|
||
continue
|
||
}
|
||
return val
|
||
}
|
||
return ""
|
||
}
|
||
|
||
// isPromptLabelTitle detects enhance pollution where the model echoed a
|
||
// prompt header ("Category:" / "Category: 120") as the product title, or
|
||
// leaked instruction scaffolding from CategoryEnhanceUserTemplate /
|
||
// AppendFormulaConstraints into name.
|
||
func isPromptLabelTitle(s string) bool {
|
||
s = strings.TrimSpace(s)
|
||
if s == "" || s == "<nil>" {
|
||
return false
|
||
}
|
||
if isPromptLeakageTitle(s) {
|
||
return true
|
||
}
|
||
lower := strings.ToLower(s)
|
||
for _, label := range []string{
|
||
"category", "name", "desc", "description",
|
||
"meta", "meta_title", "meta_description", "metadescription",
|
||
"attrs", "attributes", "current name", "current description", "current meta",
|
||
} {
|
||
if lower == label || lower == label+":" {
|
||
return true
|
||
}
|
||
if strings.HasPrefix(lower, label+":") || strings.HasPrefix(lower, label+" :") {
|
||
return true
|
||
}
|
||
}
|
||
return false
|
||
}
|
||
|
||
// isPromptLeakageTitle detects when an LLM echoed enhance-prompt instructions
|
||
// (Title formula / short retail title / Schema / Reply with ONLY JSON / …)
|
||
// as the product name instead of a real title.
|
||
func isPromptLeakageTitle(s string) bool {
|
||
s = strings.TrimSpace(s)
|
||
if s == "" || s == "<nil>" {
|
||
return false
|
||
}
|
||
lower := strings.ToLower(s)
|
||
for _, phrase := range promptLeakagePhrases {
|
||
if strings.Contains(lower, phrase) {
|
||
return true
|
||
}
|
||
}
|
||
// Long dumps of the enhance template: any formula/schema keyword is enough.
|
||
if len([]rune(s)) > 120 {
|
||
for _, kw := range promptLeakageLongKeywords {
|
||
if strings.Contains(lower, kw) {
|
||
return true
|
||
}
|
||
}
|
||
}
|
||
return false
|
||
}
|
||
|
||
// Phrases copied from aiprompts.CategoryEnhanceUserTemplate, BuiltInDefaults,
|
||
// and processing.AppendFormulaConstraints / FormatTitleFormulaConstraint.
|
||
var promptLeakagePhrases = []string{
|
||
"title formula",
|
||
"follow any",
|
||
"constraints that follow",
|
||
"short retail title",
|
||
"use attrs",
|
||
"write name in",
|
||
"schema:",
|
||
"reply with only json",
|
||
"your reply is parsed as json",
|
||
"description formula",
|
||
"seo meta formula",
|
||
"build name from attrs",
|
||
"prefer attrs values",
|
||
"order matters; join with",
|
||
"do not hardcode a language",
|
||
// Description / title formula scaffolding (Format*FormulaConstraint + enhance templates).
|
||
"emit description as one html",
|
||
"emit one html string",
|
||
"covering each section",
|
||
"tags matching section",
|
||
"overrides any shorter",
|
||
"competing full html",
|
||
"metadescription",
|
||
"never ignore the description",
|
||
"follow any description formula",
|
||
"when a description formula",
|
||
"structured html sections",
|
||
"never copy name as description",
|
||
"do not invent specs",
|
||
"prefer 1-3 factual",
|
||
"1-2 sentences",
|
||
"keep literal text as written",
|
||
"build name from attrs using this structure",
|
||
"category guidance",
|
||
"applies to name and description",
|
||
"apply it to both",
|
||
"never ignore the title formula",
|
||
// Role-sectioned CategoryEnhanceUserTemplate / KeySEOMeta markers.
|
||
"--- title ---",
|
||
"--- end title ---",
|
||
"--- description ---",
|
||
"--- end description ---",
|
||
"--- meta ---",
|
||
"--- end meta ---",
|
||
"--- attributes ---",
|
||
"--- end attributes ---",
|
||
"role: title",
|
||
"role: description",
|
||
"role: meta",
|
||
"role: attributes",
|
||
}
|
||
|
||
var promptLeakageLongKeywords = []string{
|
||
"formula",
|
||
"constraints",
|
||
"schema",
|
||
"json",
|
||
"attrs",
|
||
"retail title",
|
||
"html string",
|
||
"section type",
|
||
}
|
||
|
||
// isUnusableCategoryValue rejects tokens that must never become Category /
|
||
// CategoryName: prompt labels / title-formula scaffolding, or the product
|
||
// title itself (name-as-category). Callers with taxonomy should prefer
|
||
// scrubCategoryPollution so a title that legitimately equals a category name
|
||
// can still resolve to unique_id.
|
||
func isUnusableCategoryValue(s string, productTitles ...string) bool {
|
||
s = strings.TrimSpace(s)
|
||
if s == "" || s == "<nil>" {
|
||
return false
|
||
}
|
||
if isPromptLabelTitle(s) {
|
||
return true
|
||
}
|
||
return categoryTokenEqualsProductTitle(s, productTitles...)
|
||
}
|
||
|
||
func categoryTokenEqualsProductTitle(s string, productTitles ...string) bool {
|
||
s = strings.TrimSpace(s)
|
||
if s == "" || s == "<nil>" {
|
||
return false
|
||
}
|
||
for _, t := range productTitles {
|
||
t = strings.TrimSpace(t)
|
||
if t == "" || t == "<nil>" {
|
||
continue
|
||
}
|
||
if strings.EqualFold(s, t) {
|
||
return true
|
||
}
|
||
}
|
||
return false
|
||
}
|
||
|
||
func taxonomyDisplayNameSet(namesByUID map[string]string) map[string]struct{} {
|
||
if len(namesByUID) == 0 {
|
||
return nil
|
||
}
|
||
out := make(map[string]struct{}, len(namesByUID))
|
||
for _, name := range namesByUID {
|
||
n := strings.ToLower(strings.TrimSpace(name))
|
||
if n != "" {
|
||
out[n] = struct{}{}
|
||
}
|
||
}
|
||
return out
|
||
}
|
||
|
||
// scrubCategoryPollution clears Category / CategoryName when they hold prompt
|
||
// leakage/formula text, or equal the product title without resolving to a
|
||
// company taxonomy unique_id / display name.
|
||
func scrubCategoryPollution(out *StepResult, namesByUID map[string]string, valid map[string]struct{}) {
|
||
if out == nil {
|
||
return
|
||
}
|
||
titles := []string{out.ProcessedName, out.Name}
|
||
clearCategory := func(note string) {
|
||
out.Category = ""
|
||
out.CategoryName = ""
|
||
if out.FieldSources == nil {
|
||
out.FieldSources = map[string]any{}
|
||
}
|
||
out.FieldSources["category"] = "cleared_invalid"
|
||
out.Notes = append(out.Notes, note)
|
||
}
|
||
cat := strings.TrimSpace(out.Category)
|
||
if cat != "" {
|
||
switch {
|
||
case isPromptLabelTitle(cat):
|
||
clearCategory("category: ignored (prompt leakage)")
|
||
case categoryTokenEqualsProductTitle(cat, titles...) &&
|
||
resolveCompanyCategoryUniqueID(cat, namesByUID, valid) == "":
|
||
clearCategory("category: ignored (product title)")
|
||
}
|
||
}
|
||
name := strings.TrimSpace(out.CategoryName)
|
||
if name == "" {
|
||
return
|
||
}
|
||
if isPromptLabelTitle(name) {
|
||
out.CategoryName = ""
|
||
return
|
||
}
|
||
if categoryTokenEqualsProductTitle(name, titles...) {
|
||
if _, ok := taxonomyDisplayNameSet(namesByUID)[strings.ToLower(name)]; !ok {
|
||
out.CategoryName = ""
|
||
}
|
||
}
|
||
}
|
||
|
||
// preferredProductTitle picks the first usable title, skipping empty values,
|
||
// prompt-label echoes like "Category:", instruction-text leakage, and brand-only
|
||
// stubs when a longer product name exists among the candidates (e.g. "ANKER"
|
||
// vs "Anker Soundcore Space One Pro").
|
||
func preferredProductTitle(gtin string, candidates ...string) string {
|
||
usable := make([]string, 0, len(candidates))
|
||
for _, c := range candidates {
|
||
c = strings.TrimSpace(c)
|
||
if c == "" || c == "<nil>" || isPromptLabelTitle(c) {
|
||
continue
|
||
}
|
||
usable = append(usable, ensureReadableTitleSpacing(c))
|
||
}
|
||
for _, c := range usable {
|
||
if isBrandOnlyTitleAmong(c, usable) {
|
||
continue
|
||
}
|
||
return SanitizeOutput(c)
|
||
}
|
||
// All remaining candidates are brand-only relative to each other — pick longest.
|
||
best := ""
|
||
for _, c := range usable {
|
||
if len([]rune(c)) > len([]rune(best)) {
|
||
best = c
|
||
}
|
||
}
|
||
if best != "" {
|
||
return SanitizeOutput(best)
|
||
}
|
||
if strings.TrimSpace(gtin) != "" {
|
||
return SanitizeText("Product " + strings.TrimSpace(gtin))
|
||
}
|
||
return "Product"
|
||
}
|
||
|
||
// isBrandOnlyTitleAmong is true when candidate looks like a brand stub that a
|
||
// longer candidate expands (prefix / first-token match).
|
||
func isBrandOnlyTitleAmong(candidate string, all []string) bool {
|
||
c := strings.TrimSpace(candidate)
|
||
if c == "" {
|
||
return false
|
||
}
|
||
words := strings.Fields(c)
|
||
if len(words) > 2 {
|
||
return false
|
||
}
|
||
if len(words) == 1 && len([]rune(c)) > 40 {
|
||
return false
|
||
}
|
||
clower := strings.ToLower(c)
|
||
for _, other := range all {
|
||
o := strings.TrimSpace(other)
|
||
if o == "" || strings.EqualFold(o, c) {
|
||
continue
|
||
}
|
||
if len([]rune(o)) <= len([]rune(c)) {
|
||
continue
|
||
}
|
||
olower := strings.ToLower(o)
|
||
if strings.HasPrefix(olower, clower+" ") || strings.HasPrefix(olower, clower+"-") {
|
||
return true
|
||
}
|
||
oWords := strings.Fields(o)
|
||
if len(words) == 1 && len(oWords) >= 2 && strings.EqualFold(oWords[0], words[0]) {
|
||
return true
|
||
}
|
||
}
|
||
return false
|
||
}
|
||
|
||
// Thin wrappers keep processing call sites stable; logic lives in company so
|
||
// catalog.RepairWeakEnhanceHashes can reuse it without an import cycle.
|
||
func isWeakPriorEnhanceDescription(priorDesc string, titles ...string) bool {
|
||
return company.IsWeakPriorEnhanceDescription(priorDesc, titles...)
|
||
}
|
||
|
||
func containsWeakFillerPhrase(desc string) bool {
|
||
return company.ContainsWeakFillerPhrase(desc)
|
||
}
|
||
|
||
func descriptionEchoesTitle(desc, title string) bool {
|
||
return company.DescriptionEchoesTitle(desc, title)
|
||
}
|
||
|
||
// preferredProductDescription picks the first usable description, skipping empty
|
||
// values, prompt-label echoes, weak filler phrases, and copy that merely repeats
|
||
// the product title.
|
||
func preferredProductDescription(title string, candidates ...string) string {
|
||
for _, c := range candidates {
|
||
c = strings.TrimSpace(c)
|
||
if c == "" || c == "<nil>" || isPromptLabelTitle(c) {
|
||
continue
|
||
}
|
||
if descriptionEchoesTitle(c, title) {
|
||
continue
|
||
}
|
||
if containsWeakFillerPhrase(c) {
|
||
continue
|
||
}
|
||
if company.LooksLikeHeuristicSynthesize(c) {
|
||
continue
|
||
}
|
||
return SanitizeOutput(c)
|
||
}
|
||
return ""
|
||
}
|
||
|
||
func attrLookupCI(attrs map[string]any, keys ...string) string {
|
||
if attrs == nil {
|
||
return ""
|
||
}
|
||
for _, want := range keys {
|
||
want = strings.TrimSpace(want)
|
||
if want == "" {
|
||
continue
|
||
}
|
||
if v := strings.TrimSpace(stringFromAny(attrs[want])); v != "" {
|
||
return v
|
||
}
|
||
for k, raw := range attrs {
|
||
if strings.EqualFold(strings.TrimSpace(k), want) {
|
||
if v := strings.TrimSpace(stringFromAny(raw)); v != "" {
|
||
return v
|
||
}
|
||
}
|
||
}
|
||
}
|
||
return ""
|
||
}
|
||
|
||
func formatAttrDimParts(attrs map[string]any, maxParts int) []string {
|
||
return formatAttrDimPartsLang(attrs, maxParts, "")
|
||
}
|
||
|
||
func formatAttrDimPartsLang(attrs map[string]any, maxParts int, language string) []string {
|
||
if attrs == nil || maxParts <= 0 {
|
||
return nil
|
||
}
|
||
prefer := []string{
|
||
"width", "height", "depth", "weight", "max_load", "max load", "load_capacity",
|
||
"vesa", "screen_size", "diagonal", "color", "material", "size", "energy_class", "warranty",
|
||
}
|
||
parts := make([]string, 0, maxParts)
|
||
seen := map[string]struct{}{}
|
||
sl := isSlovenianContentLanguage(language)
|
||
add := func(k, v string) {
|
||
k = strings.TrimSpace(k)
|
||
v = strings.TrimSpace(v)
|
||
if k == "" || v == "" {
|
||
return
|
||
}
|
||
lk := strings.ToLower(k)
|
||
if _, ok := seen[lk]; ok {
|
||
return
|
||
}
|
||
if isDimensionKey(canonicalizeAttrKey(k)) && isZeroishString(v) {
|
||
return
|
||
}
|
||
seen[lk] = struct{}{}
|
||
parts = append(parts, fmt.Sprintf("%s: %s", attrDimLabel(k, sl), v))
|
||
}
|
||
for _, k := range prefer {
|
||
if len(parts) >= maxParts {
|
||
break
|
||
}
|
||
if v := attrLookupCI(attrs, k); v != "" {
|
||
add(k, v)
|
||
}
|
||
}
|
||
return parts
|
||
}
|
||
|
||
func attrDimLabel(key string, slovenian bool) string {
|
||
canon := canonicalizeAttrKey(key)
|
||
if slovenian {
|
||
switch canon {
|
||
case "width":
|
||
return "Širina"
|
||
case "height":
|
||
return "Višina"
|
||
case "depth":
|
||
return "Globina"
|
||
case "weight":
|
||
return "Teža"
|
||
case "energy_class":
|
||
return "Energijski razred"
|
||
case "warranty":
|
||
return "Garancija"
|
||
case "color":
|
||
return "Barva"
|
||
case "material":
|
||
return "Material"
|
||
case "size", "screen_size", "diagonal":
|
||
return "Velikost"
|
||
}
|
||
}
|
||
switch canon {
|
||
case "width":
|
||
return "Width"
|
||
case "height":
|
||
return "Height"
|
||
case "depth":
|
||
return "Depth"
|
||
case "weight":
|
||
return "Weight"
|
||
case "energy_class":
|
||
return "Energy class"
|
||
case "warranty":
|
||
return "Warranty"
|
||
case "max_load", "load_capacity":
|
||
return "Max load"
|
||
case "screen_size", "diagonal":
|
||
return "Screen size"
|
||
case "product_model":
|
||
return "Model"
|
||
default:
|
||
if canon == "" {
|
||
return key
|
||
}
|
||
return strings.ReplaceAll(canon, "_", " ")
|
||
}
|
||
}
|
||
|
||
// synthesizeProductDescription prefers a category description_template skeleton
|
||
// (HTML section types) when present; otherwise falls back to plain title synthesize.
|
||
func synthesizeProductDescription(title, category, language string, attrs map[string]any, descriptionTemplate any) string {
|
||
title = ensureReadableTitleSpacing(title)
|
||
if sections, ok := parseDescriptionFormulaSections(descriptionTemplate); ok && len(sections) > 0 {
|
||
if out := synthesizeDescriptionFromFormula(title, category, language, attrs, sections); out != "" {
|
||
return out
|
||
}
|
||
}
|
||
return synthesizeDescriptionFromTitle(title, category, language, attrs)
|
||
}
|
||
|
||
// synthesizeDescriptionFromFormula builds a minimal HTML description matching
|
||
// category description_template section types so timeout/fallback still respects
|
||
// A1 category structure (unlike plain title synthesize).
|
||
// Duplicate paragraph/list section types are emitted once to avoid invent-boilerplate
|
||
// repetition (legacy bug: every <p> repeated the same synthesize sentence).
|
||
func synthesizeDescriptionFromFormula(title, category, language string, attrs map[string]any, sections []descriptionFormulaSection) string {
|
||
title = ensureReadableTitleSpacing(strings.TrimSpace(title))
|
||
if title == "" || title == "<nil>" || isPromptLabelTitle(title) {
|
||
return ""
|
||
}
|
||
dims := formatAttrDimPartsLang(attrs, 6, language)
|
||
intro := factualDescriptionIntro(title, category, language, attrs)
|
||
if intro == "" {
|
||
return ""
|
||
}
|
||
var b strings.Builder
|
||
paraUsed := false
|
||
listUsed := false
|
||
headingLevels := map[string]bool{}
|
||
for _, s := range sections {
|
||
typ := strings.ToLower(strings.TrimSpace(s.Type))
|
||
switch typ {
|
||
case "h1", "h2", "h3", "h4":
|
||
if headingLevels[typ] {
|
||
continue
|
||
}
|
||
headingLevels[typ] = true
|
||
heading := title
|
||
if typ != "h1" {
|
||
if isSlovenianContentLanguage(language) {
|
||
heading = "Ključne lastnosti"
|
||
} else {
|
||
heading = "Key features"
|
||
}
|
||
}
|
||
fmt.Fprintf(&b, "<%s>%s</%s>", typ, SanitizeOutput(heading), typ)
|
||
case "ul":
|
||
if listUsed {
|
||
continue
|
||
}
|
||
items := dims
|
||
if len(items) == 0 {
|
||
if secondary := factualSecondaryFacts(language, attrs); len(secondary) > 0 {
|
||
items = secondary
|
||
} else {
|
||
items = []string{intro}
|
||
}
|
||
}
|
||
b.WriteString("<ul>")
|
||
for _, it := range items {
|
||
fmt.Fprintf(&b, "<li>%s</li>", SanitizeOutput(it))
|
||
}
|
||
b.WriteString("</ul>")
|
||
listUsed = true
|
||
default: // p and unknown → paragraph (once)
|
||
if paraUsed {
|
||
continue
|
||
}
|
||
fmt.Fprintf(&b, "<p>%s</p>", SanitizeOutput(intro))
|
||
paraUsed = true
|
||
}
|
||
}
|
||
out := strings.TrimSpace(b.String())
|
||
if out == "" {
|
||
return ""
|
||
}
|
||
return SanitizeOutput(out)
|
||
}
|
||
|
||
// factualDescriptionIntro builds a short non-invent paragraph for formula fallback.
|
||
// Intentionally avoids heuristicSynthesizePhrases ("je izdelek v kategoriji", …).
|
||
func factualDescriptionIntro(title, category, language string, attrs map[string]any) string {
|
||
title = ensureReadableTitleSpacing(strings.TrimSpace(title))
|
||
if title == "" || title == "<nil>" || isPromptLabelTitle(title) {
|
||
return ""
|
||
}
|
||
cat := strings.TrimSpace(category)
|
||
if strings.EqualFold(cat, "general") {
|
||
cat = ""
|
||
}
|
||
brand := attrLookupCI(attrs, "brand")
|
||
model := attrLookupCI(attrs, "product_model", "model", "sku")
|
||
dims := formatAttrDimPartsLang(attrs, 3, language)
|
||
sl := isSlovenianContentLanguage(language)
|
||
|
||
var b strings.Builder
|
||
b.WriteString(title)
|
||
if sl {
|
||
switch {
|
||
case brand != "" && cat != "":
|
||
fmt.Fprintf(&b, " — %s, znamka %s", cat, brand)
|
||
case brand != "":
|
||
fmt.Fprintf(&b, " — znamka %s", brand)
|
||
case cat != "":
|
||
fmt.Fprintf(&b, " — %s", cat)
|
||
default:
|
||
b.WriteString(" — katalogski izdelek")
|
||
}
|
||
if model != "" && !strings.Contains(strings.ToLower(title), strings.ToLower(model)) {
|
||
fmt.Fprintf(&b, " (model %s)", model)
|
||
}
|
||
if len(dims) > 0 {
|
||
fmt.Fprintf(&b, ". %s", strings.Join(dims, ", "))
|
||
}
|
||
b.WriteByte('.')
|
||
} else {
|
||
switch {
|
||
case brand != "" && cat != "":
|
||
fmt.Fprintf(&b, " — %s from %s", cat, brand)
|
||
case brand != "":
|
||
fmt.Fprintf(&b, " — from %s", brand)
|
||
case cat != "":
|
||
fmt.Fprintf(&b, " — %s", cat)
|
||
default:
|
||
b.WriteString(" — catalog product")
|
||
}
|
||
if model != "" && !strings.Contains(strings.ToLower(title), strings.ToLower(model)) {
|
||
fmt.Fprintf(&b, " (model %s)", model)
|
||
}
|
||
if len(dims) > 0 {
|
||
fmt.Fprintf(&b, ". %s", strings.Join(dims, ", "))
|
||
}
|
||
b.WriteByte('.')
|
||
}
|
||
return SanitizeOutput(b.String())
|
||
}
|
||
|
||
func factualSecondaryFacts(language string, attrs map[string]any) []string {
|
||
prefer := []string{"energy_class", "warranty", "color", "material"}
|
||
sl := isSlovenianContentLanguage(language)
|
||
var out []string
|
||
for _, k := range prefer {
|
||
v := attrLookupCI(attrs, k)
|
||
if v == "" || isZeroishString(v) {
|
||
continue
|
||
}
|
||
out = append(out, fmt.Sprintf("%s: %s", attrDimLabel(k, sl), v))
|
||
}
|
||
return out
|
||
}
|
||
|
||
// synthesizeDescriptionFromTitle builds a short factual fallback when enhance
|
||
// returns empty/title-echo/filler copy. Uses title, category, brand, model, and
|
||
// key dims. language is a content-language code (en/sl/…) or English label.
|
||
// Prefer synthesizeProductDescription when a description_template is available.
|
||
func synthesizeDescriptionFromTitle(title, category, language string, attrs map[string]any) string {
|
||
return factualDescriptionIntro(title, category, language, attrs)
|
||
}
|
||
|
||
func isSlovenianContentLanguage(raw string) bool {
|
||
raw = strings.TrimSpace(raw)
|
||
if raw == "" {
|
||
return false
|
||
}
|
||
if code, err := company.ParseLanguage(raw, false); err == nil && code == "sl" {
|
||
return true
|
||
}
|
||
lower := strings.ToLower(raw)
|
||
return lower == "slovenian" || strings.Contains(lower, "slovenian")
|
||
}
|