package processing import ( "context" "encoding/json" "fmt" "strings" "github.com/descrybe/descrybe-v2/apps/api/internal/company" "github.com/descrybe/descrybe-v2/apps/api/internal/eprel" ) // StepPolicy controls entitlement-gated steps (AI / EPREL). type StepPolicy struct { AllowAI bool AllowEPREL bool } // RunSteps executes the multi-step product pipeline. // OpenAI enhance runs only when Completer is configured, Enabled(), and policy.AllowAI. // EPREL runs only when enricher enabled and policy.AllowEPREL. func (e *Engine) RunSteps(ctx context.Context, companyID string, in ProductInput, processingType string, categoryNames []string, policy StepPolicy) (StepResult, error) { steps := resolveSteps(processingType) out := StepResult{ Attributes: map[string]any{}, ProcessedAttributes: map[string]any{}, FieldSources: map[string]any{}, EPREL: map[string]any{}, GPTResponse: map[string]any{"steps": []any{}}, Notes: []string{}, } normalized := map[string]any{} attrs := map[string]any{} for _, step := range steps { switch step { case StepNormalize: normalized = NormalizeMapped(in.Mapped, in.Raw) out.Name = preferredProductTitle(in.GTIN, stringFromAny(normalized["name"]), stringFromAny(normalized["title"]), in.Name, in.PriorProcessedName, ) out.Description = preferredProductDescription( stringFromAny(normalized["description"]), in.Description, in.PriorProcessedDescription, ) out.Category = stringFromAny(normalized["category"]) // mapped_data.category wins; otherwise keep existing processed category // so enhance_only / reprocess cannot blank A1 legacy categories. preserveCategoryIfEmpty(&out, in.PriorCategory) out.FieldSources["normalize"] = "mapped+raw" appendStepLog(out.GPTResponse, StepNormalize, map[string]any{ "keys": len(normalized), }) case StepParseSpecs: specVal := normalized["specifications"] if specVal == nil { specVal = in.Mapped["specifications"] } if specVal == nil { specVal = in.Raw["specifications"] } parsed := ParseSpecifications(specVal) // Also accept pre-mapped attributes map if am, ok := normalized["attributes"].(map[string]any); ok { for k, v := range ParseSpecifications(am) { if _, exists := parsed[k]; !exists { parsed[k] = v } } } attrs = parsed out.Attributes = attrs out.ProcessedAttributes = attrs out.FieldSources["attributes"] = "specifications" appendStepLog(out.GPTResponse, StepParseSpecs, map[string]any{ "count": len(attrs), }) case StepFillFields: normalized = FillMissingFields(normalized, attrs) if len(in.StandardFields) > 0 { normalized = FillMissingStandardFields(normalized, in.Raw, in.StandardFields) } out.Name = preferredProductTitle(in.GTIN, stringFromAny(normalized["name"]), stringFromAny(normalized["title"]), out.Name, in.Name, in.PriorProcessedName, ) out.Description = preferredProductDescription( stringFromAny(normalized["description"]), out.Description, in.Description, ) out.Category = stringFromAny(normalized["category"]) // Promote filled scalar fields into attributes when useful promote := []string{"brand", "width", "height", "depth", "weight", "gtin", "stock_status"} for _, f := range in.StandardFields { if f.Key != "" { promote = append(promote, f.Key) } } seen := map[string]bool{} for _, k := range promote { if seen[k] { continue } seen[k] = true if v := stringFromAny(normalized[k]); v != "" { if _, exists := attrs[k]; !exists { attrs[k] = v } out.FieldSources[k] = "fill_fields" } } out.Attributes = attrs out.ProcessedAttributes = attrs appendStepLog(out.GPTResponse, StepFillFields, map[string]any{ "brand": stringFromAny(normalized["brand"]), }) if out.Category == "" && e != nil && e.Vector != nil && e.Vector.Enabled() { text := strings.TrimSpace(out.Name + " " + out.Description) if text != "" { if cat, err := e.Vector.SuggestCategory(ctx, companyID, text, categoryNames); err == nil && strings.TrimSpace(cat) != "" { out.Category = SanitizeOutput(cat) out.FieldSources["category"] = "vector" out.Notes = append(out.Notes, "category: vector") appendStepLog(out.GPTResponse, "vector_categorize", map[string]any{ "status": "ok", "category": out.Category, }) } else if err != nil { out.Notes = append(out.Notes, "vector_categorize: "+TruncateError(err)) appendStepLog(out.GPTResponse, "vector_categorize", map[string]any{ "status": "failed", "error": TruncateError(err), }) } } } preserveCategoryIfEmpty(&out, in.PriorCategory) case StepEPREL: if !policy.AllowEPREL { out.Notes = append(out.Notes, "eprel: skipped (not allowed for this job)") appendStepLog(out.GPTResponse, StepEPREL, map[string]any{ "status": "skipped", "reason": "entitlement_can_use_eprel", }) break } id := eprel.ExtractID(normalized, in.Mapped, in.Raw) if id == "" { out.Notes = append(out.Notes, "eprel: no id") appendStepLog(out.GPTResponse, StepEPREL, map[string]any{"status": "skipped", "reason": "no_id"}) break } enricher := e.EPREL if enricher == nil { enricher = eprel.Disabled{} } if !enricher.Enabled() { out.Notes = append(out.Notes, "eprel: enricher disabled") out.EPREL = map[string]any{"eprel_id": id, "status": "skipped"} appendStepLog(out.GPTResponse, StepEPREL, map[string]any{"status": "skipped", "eprel_id": id, "reason": "disabled"}) break } data, err := enricher.Fetch(ctx, id) if err != nil { out.Notes = append(out.Notes, "eprel: "+TruncateError(err)) attrs["eprel_id"] = id out.Attributes = attrs out.ProcessedAttributes = attrs appendStepLog(out.GPTResponse, StepEPREL, map[string]any{"status": "failed", "error": TruncateError(err)}) // Non-fatal: continue pipeline break } if data == nil { attrs["eprel_id"] = id out.Attributes = attrs out.ProcessedAttributes = attrs out.EPREL = map[string]any{"eprel_id": id, "status": "empty"} appendStepLog(out.GPTResponse, StepEPREL, map[string]any{"status": "empty", "eprel_id": id}) break } attrs = eprel.MergeInto(attrs, data) out.Attributes = attrs out.ProcessedAttributes = attrs out.EPREL = map[string]any{ "eprel_id": data.ID, "label": data.Label, "pdf": data.PDF, "energy_class": data.EnergyClass, "energy_scale": data.EnergyScale, } out.FieldSources["eprel"] = "eprel_api" appendStepLog(out.GPTResponse, StepEPREL, map[string]any{"status": "ok", "eprel_id": data.ID}) case StepAIEnhance: preservePriorEnhanceHash := func() { if in.PriorEnhanceHash != "" { out.FieldSources[FieldEnhanceInputHash] = in.PriorEnhanceHash } } if !policy.AllowAI { out.ProcessedName = out.Name out.ProcessedDescription = out.Description out.Notes = append(out.Notes, "ai_enhance: skipped (Free plan — upgrade for AI titles/descriptions)") preservePriorEnhanceHash() appendStepLog(out.GPTResponse, StepAIEnhance, map[string]any{ "status": "skipped", "reason": "entitlement_can_use_ai", }) break } if !e.CompleterEnabled() { out.ProcessedName = out.Name out.ProcessedDescription = out.Description out.Notes = append(out.Notes, "ai_enhance: skipped (platform OpenAI unset; configure admin settings or company BYOK)") preservePriorEnhanceHash() appendStepLog(out.GPTResponse, StepAIEnhance, map[string]any{ "status": "skipped", "reason": "openai_not_configured", }) break } langs := in.ContentLanguages if len(langs) == 0 { langs = []string{in.Language} } if len(langs) == 0 { langs = []string{company.DefaultLanguage} } primary := in.Language if primary == "" { primary = langs[0] } localized := company.LocalizedContent{} if in.PriorLocalized != nil { for k, v := range in.PriorLocalized { localized[k] = v } } anyFailed := false anyOK := false allUnchanged := true langMetas := make([]any, 0, len(langs)) for _, lang := range langs { tpl := in.EnhanceByLang[lang] if tpl.System == "" && tpl.User == "" && lang == primary { tpl = PromptTemplates{System: in.EnhanceSystemTemplate, User: in.EnhanceUserTemplate} } catPrompt := in.CategoryEnhancePrompt if lang != primary || catPrompt == "" { catPrompt = categoryEnhancePromptFor(in.CategoryPromptsByLang, out.Category, lang) } priorFields := company.FieldsForLanguage(in.PriorLocalized, lang) priorHash := priorFields.EnhanceInputHash priorName := priorFields.ProcessedName priorDesc := priorFields.ProcessedDescription if lang == primary { if priorHash == "" { priorHash = in.PriorEnhanceHash } if priorName == "" { priorName = in.PriorProcessedName } if priorDesc == "" { priorDesc = in.PriorProcessedDescription } } name, desc, tokens, raw, err := e.enhance(ctx, ProductInput{ GTIN: in.GTIN, Name: out.Name, Description: out.Description, Mapped: normalized, BrandPrompt: in.BrandPrompt, Language: lang, EnhanceSystemTemplate: tpl.System, EnhanceUserTemplate: tpl.User, CategoryEnhancePrompt: catPrompt, PriorEnhanceHash: priorHash, PriorProcessedName: priorName, PriorProcessedDescription: priorDesc, }, out.Category, attrs) out.TotalTokens += tokens status := enhanceStatusFromMeta(raw) meta := map[string]any{"language": lang, "raw": raw} if err != nil { anyFailed = true allUnchanged = false meta["status"] = "failed" meta["error"] = TruncateError(err) out.Notes = append(out.Notes, "ai_enhance: "+TruncateError(err)) if lang == primary { name, desc = out.Name, out.Description } else if priorName != "" || priorDesc != "" { name, desc = priorName, priorDesc } else { langMetas = append(langMetas, meta) continue } } else if status == "unchanged" { meta["status"] = "unchanged" } else { allUnchanged = false anyOK = true meta["status"] = status } hash := enhanceHashFromMeta(raw) localized[lang] = company.LocalizedFields{ ProcessedName: name, ProcessedDescription: desc, EnhanceInputHash: hash, MetaTitle: company.FieldsForLanguage(localized, lang).MetaTitle, MetaDescription: company.FieldsForLanguage(localized, lang).MetaDescription, } // Preserve existing meta when re-enhancing titles only. if prev := company.FieldsForLanguage(in.PriorLocalized, lang); prev.MetaTitle != "" || prev.MetaDescription != "" { f := localized[lang] if f.MetaTitle == "" { f.MetaTitle = prev.MetaTitle } if f.MetaDescription == "" { f.MetaDescription = prev.MetaDescription } localized[lang] = f } langMetas = append(langMetas, meta) if lang == primary { name = preferredProductTitle(in.GTIN, name, out.Name, in.PriorProcessedName) desc = preferredProductDescription(desc, out.Description, in.PriorProcessedDescription) out.ProcessedName = name out.ProcessedDescription = desc if name != "" { out.Name = name } if desc != "" { out.Description = desc } if hash != "" && (status == "ok" || status == "unchanged") { out.FieldSources[FieldEnhanceInputHash] = hash } else if err != nil { preservePriorEnhanceHash() } } } out.LocalizedContent = localized if anyFailed && !anyOK { if out.ProcessedName == "" { out.ProcessedName = out.Name } if out.ProcessedDescription == "" { out.ProcessedDescription = out.Description } preservePriorEnhanceHash() errNote := "" for _, m := range langMetas { if mm, ok := m.(map[string]any); ok { if e, ok := mm["error"].(string); ok && e != "" { errNote = e break } } } appendStepLog(out.GPTResponse, StepAIEnhance, map[string]any{ "status": "failed", "error": errNote, "languages": langMetas, }) break } if allUnchanged { out.SkipCreditDebit = true out.FieldSources["name"] = "ai_enhance_unchanged" out.FieldSources["description"] = "ai_enhance_unchanged" out.Notes = append(out.Notes, "ai_enhance: skipped (inputs unchanged)") } else { out.AIProviderMode = e.EngineProviderMode() out.FieldSources["name"] = "ai_enhance" out.FieldSources["description"] = "ai_enhance" } appendStepLog(out.GPTResponse, StepAIEnhance, map[string]any{ "status": map[string]any{"unchanged": allUnchanged, "ok": anyOK, "failed": anyFailed}, "languages": langMetas, }) default: appendStepLog(out.GPTResponse, step, map[string]any{"status": "unknown"}) } } preserveCategoryIfEmpty(&out, in.PriorCategory) out.Name = preferredProductTitle(in.GTIN, out.Name, out.ProcessedName, in.Name, in.PriorProcessedName) out.ProcessedName = preferredProductTitle(in.GTIN, out.ProcessedName, out.Name, in.PriorProcessedName, in.Name) out.Description = preferredProductDescription(out.Description, out.ProcessedDescription, in.Description) out.ProcessedDescription = preferredProductDescription(out.ProcessedDescription, out.Description, in.PriorProcessedDescription) if out.ProcessedName == "" { out.ProcessedName = out.Name } if out.ProcessedDescription == "" { out.ProcessedDescription = out.Description } if out.Attributes == nil { out.Attributes = map[string]any{} } if out.ProcessedAttributes == nil { out.ProcessedAttributes = out.Attributes } if out.AIProviderMode == "" { if out.TotalTokens > 0 { out.AIProviderMode = e.EngineProviderMode() } else { out.AIProviderMode = AIProviderUnknown } } if len(out.Notes) > 0 { out.GPTResponse["notes"] = out.Notes } return out, nil } // preserveCategoryIfEmpty keeps an existing processed category when normalize/AI // left Category empty (common for A1 feeds where category lives only on processed). func preserveCategoryIfEmpty(out *StepResult, prior string) { if out == nil || strings.TrimSpace(out.Category) != "" { return } prior = strings.TrimSpace(prior) if prior == "" { return } out.Category = SanitizeText(prior) if out.FieldSources == nil { out.FieldSources = map[string]any{} } out.FieldSources["category"] = "prior_processed" } func resolveSteps(processingType string) []string { switch strings.ToLower(strings.TrimSpace(processingType)) { case "enhance", "enhance_only", "enhance-only", "title", "description": return []string{StepNormalize, StepAIEnhance} case "attributes", "attributes_only", "specs", "specifications": return []string{StepNormalize, StepParseSpecs, StepFillFields} case "eprel", "eprel_only": return []string{StepNormalize, StepEPREL} case "normalize_only": return []string{StepNormalize} case "categorize", "categorize_only", "categorize_enhance": // Legacy aliases → full deterministic + optional AI return append([]string{}, CanonicalSteps...) default: // full return append([]string{}, CanonicalSteps...) } } // InitialStepProgress builds pending step_progress rows for a job. func InitialStepProgress(processingType string) []StepProgress { steps := resolveSteps(processingType) out := make([]StepProgress, 0, len(steps)) for _, s := range steps { out = append(out, StepProgress{Step: s, Status: "pending"}) } return out } func appendStepLog(gpt map[string]any, name string, raw any) { steps, _ := gpt["steps"].([]any) gpt["steps"] = append(steps, map[string]any{"step": name, "raw": raw}) } func (e *Engine) enhance(ctx context.Context, in ProductInput, category string, attrs map[string]any) (string, string, int, any, error) { sysTpl, userTpl := resolveProductPromptTemplates(in) hash := HashEnhanceInput(category, in.Name, in.Description, in.BrandPrompt, in.Language, sysTpl, userTpl, attrs) if e == nil || e.Completer == nil { return preferredProductTitle(in.GTIN, in.Name, in.PriorProcessedName), preferredProductDescription(in.Description, in.PriorProcessedDescription), 0, map[string]any{"status": "skipped", "input_hash": hash}, nil } // Skip LLM when inputs match the last successful enhance (before any credit debit). if in.PriorEnhanceHash != "" && in.PriorEnhanceHash == hash && (in.PriorProcessedName != "" || in.PriorProcessedDescription != "") { name := preferredProductTitle(in.GTIN, in.PriorProcessedName, in.Name) desc := preferredProductDescription(in.PriorProcessedDescription, in.Description) return name, desc, 0, map[string]any{ "status": "unchanged", "input_hash": hash, }, nil } system, user := RenderProductEnhancePrompts(sysTpl, userTpl, category, in.Name, in.Description, in.GTIN, in.BrandPrompt, in.Language, attrs) comp, obj, err := CompleteJSON(ctx, e.Completer, system, user, CompleteOptions{ MaxTokens: MaxTokensEnhance, Temperature: DefaultStructuredTemp, }) if err != nil { // Network/provider failure vs parse failure after retry if obj == nil && comp.Text == "" { return preferredProductTitle(in.GTIN, in.Name, in.PriorProcessedName), preferredProductDescription(in.Description, in.PriorProcessedDescription), 0, map[string]any{ "provider": "passthrough", "error": TruncateError(err), "input_hash": hash, }, err } // Parse failed after retry — keep original copy (avoid garbage titles) return preferredProductTitle(in.GTIN, in.Name, in.PriorProcessedName), preferredProductDescription(in.Description, in.PriorProcessedDescription), comp.TotalTokens, map[string]any{ "status": "parse_failed", "error": "AI returned invalid JSON; kept original title/description", "raw": truncateRunes(comp.Text, 200), "input_hash": hash, }, nil } name := preferredProductTitle(in.GTIN, SanitizeOutput(fmt.Sprint(obj["name"])), in.Name, in.PriorProcessedName) desc := preferredProductDescription(SanitizeOutput(fmt.Sprint(obj["description"])), in.Description, in.PriorProcessedDescription) return name, desc, comp.TotalTokens, map[string]any{ "status": "ok", "input_hash": hash, "raw": comp.Raw, }, nil } func sanitizeJSON(v any) string { if v == nil { return "{}" } b, err := json.Marshal(v) if err != nil { return "{}" } return SanitizeText(string(b)) } func firstLine(s string) string { s = strings.TrimSpace(s) if i := strings.IndexByte(s, '\n'); i >= 0 { s = s[:i] } return SanitizeOutput(strings.Trim(s, "\"'` ")) } // labeledPromptValue returns the first line after any of the given labels // (case-insensitive), e.g. "Name:" / "Desc:" from ProductEnhanceUser. func labeledPromptValue(user string, labels ...string) string { lower := strings.ToLower(user) bestAt := -1 bestLabel := "" for _, label := range labels { label = strings.ToLower(strings.TrimSpace(label)) if label == "" { continue } at := strings.Index(lower, label) if at < 0 { continue } if bestAt < 0 || at < bestAt { bestAt = at bestLabel = label } } if bestAt < 0 { return "" } rest := user[bestAt+len(bestLabel):] if j := strings.Index(strings.ToLower(rest), "attrs:"); j >= 0 { rest = rest[:j] } if j := strings.Index(strings.ToLower(rest), "attributes:"); j >= 0 { rest = rest[:j] } return firstLine(rest) } // isPromptLabelTitle detects enhance pollution where the model echoed a // prompt header ("Category:" / "Category: 120") as the product title. func isPromptLabelTitle(s string) bool { s = strings.TrimSpace(s) if s == "" || s == "" { return false } lower := strings.ToLower(s) for _, label := range []string{ "category", "name", "desc", "description", "attrs", "attributes", "current name", "current description", } { if lower == label || lower == label+":" { return true } if strings.HasPrefix(lower, label+":") || strings.HasPrefix(lower, label+" :") { return true } } return false } // preferredProductTitle picks the first usable title, skipping empty values and // prompt-label echoes like "Category:" (seen on A1 Elkotex reprocess). func preferredProductTitle(gtin string, candidates ...string) string { for _, c := range candidates { c = strings.TrimSpace(c) if c == "" || c == "" || isPromptLabelTitle(c) { continue } return SanitizeOutput(c) } if strings.TrimSpace(gtin) != "" { return SanitizeText("Product " + strings.TrimSpace(gtin)) } return "Product" } func preferredProductDescription(candidates ...string) string { for _, c := range candidates { c = strings.TrimSpace(c) if c == "" || c == "" || isPromptLabelTitle(c) { continue } return SanitizeOutput(c) } return "" }