descrybe/apps/api/internal/processing/steps.go

package processing

import (
	"context"
	"encoding/json"
	"fmt"
	"log"
	"sort"
	"strings"
	"time"

	"github.com/descrybe/descrybe-v2/apps/api/internal/company"
	"github.com/descrybe/descrybe-v2/apps/api/internal/eprel"
)

// StepPolicy controls entitlement-gated steps (AI / EPREL).
type StepPolicy struct {
	AllowAI    bool
	AllowEPREL bool
}

// RunSteps executes the multi-step product pipeline.
// OpenAI enhance runs only when Completer is configured, Enabled(), and policy.AllowAI.
// EPREL runs only when enricher enabled and policy.AllowEPREL.
func (e *Engine) RunSteps(ctx context.Context, companyID string, in ProductInput, processingType string, categoryNames []string, policy StepPolicy) (StepResult, error) {
	steps := resolveSteps(processingType)
	out := StepResult{
		Attributes:          map[string]any{},
		ProcessedAttributes: map[string]any{},
		FieldSources:        map[string]any{},
		EPREL:               map[string]any{},
		GPTResponse:         map[string]any{"steps": []any{}},
		Notes:               []string{},
	}

	normalized := map[string]any{}
	attrs := map[string]any{}

	for _, step := range steps {
		switch step {
		case StepNormalize:
			normalized = NormalizeMapped(in.Mapped, in.Raw)
			out.Name = preferredProductTitle(in.GTIN,
				stringFromAny(normalized["name"]),
				stringFromAny(normalized["title"]),
				in.Name,
				in.PriorProcessedName,
			)
			out.Description = preferredProductDescription(out.Name,
				stringFromAny(normalized["description"]),
				in.Description,
				in.PriorProcessedDescription,
			)
			// mapped_data.category / category_unique_id (unique_id codes) win.
			applyCategoryFromMapped(&out, normalized, in.Mapped, in.Raw)
			// otherwise keep existing processed category so enhance_only / reprocess
			// cannot blank A1 legacy categories.
			preserveCategoryIfEmpty(&out, in.PriorCategory)
			out.FieldSources["normalize"] = "mapped+raw"
			appendStepLog(out.GPTResponse, StepNormalize, map[string]any{
				"keys": len(normalized),
			})

		case StepParseSpecs:
			specVal := normalized["specifications"]
			if specVal == nil {
				specVal = in.Mapped["specifications"]
			}
			if specVal == nil {
				specVal = in.Raw["specifications"]
			}
			parsed := ParseSpecifications(specVal)
			// Also accept pre-mapped attributes map
			if am, ok := normalized["attributes"].(map[string]any); ok {
				for k, v := range ParseSpecifications(am) {
					if _, exists := parsed[k]; !exists {
						parsed[k] = v
					}
				}
			}
			attrs = SanitizeProductAttributes(parsed)
			// Map feed/spec labels onto category_attributes (formula) keys early.
			if allowed := enhanceAllowedAttrKeys(in, out.Category); allowed != nil {
				attrs = MapAttrsOntoAllowedKeys(attrs, allowed)
			}
			out.Attributes = attrs
			out.ProcessedAttributes = attrs
			out.FieldSources["attributes"] = "specifications"
			appendStepLog(out.GPTResponse, StepParseSpecs, map[string]any{
				"count": len(attrs),
			})

		case StepFillFields:
			normalized = FillMissingFields(normalized, attrs)
			if len(in.StandardFields) > 0 {
				normalized = FillMissingStandardFields(normalized, in.Raw, in.StandardFields)
			}
			out.Name = preferredProductTitle(in.GTIN,
				stringFromAny(normalized["name"]),
				stringFromAny(normalized["title"]),
				out.Name,
				in.Name,
				in.PriorProcessedName,
			)
			out.Description = preferredProductDescription(out.Name,
				stringFromAny(normalized["description"]),
				out.Description,
				in.Description,
			)
			applyCategoryFromMapped(&out, normalized, in.Mapped, in.Raw)
			// Promote characteristic fields only — never core product identity/content
			// (those belong on the V1 item root: title, description, ean, images, …).
			promote := []string{"brand", "width", "height", "depth", "weight", "product_model", "warranty"}
			for _, f := range in.StandardFields {
				k := strings.TrimSpace(f.Key)
				if k == "" || isReservedProductKey(k) || isInvalidAttributeKey(k) {
					continue
				}
				promote = append(promote, k)
			}
			seen := map[string]bool{}
			for _, k := range promote {
				if seen[k] {
					continue
				}
				seen[k] = true
				if v := stringFromAny(normalized[k]); v != "" {
					if _, exists := attrs[k]; !exists {
						attrs[k] = v
					}
					out.FieldSources[k] = "fill_fields"
				}
			}
			attrs = SanitizeProductAttributes(attrs)
			// Remap again after category may have resolved in applyCategoryFromMapped.
			if allowed := enhanceAllowedAttrKeys(in, out.Category); allowed != nil {
				attrs = MapAttrsOntoAllowedKeys(attrs, allowed)
			}
			out.Attributes = attrs
			out.ProcessedAttributes = attrs
			appendStepLog(out.GPTResponse, StepFillFields, map[string]any{
				"brand": stringFromAny(normalized["brand"]),
			})
			preserveCategoryIfEmpty(&out, in.PriorCategory)

		case StepEPREL:
			if !policy.AllowEPREL {
				out.Notes = append(out.Notes, "eprel: skipped (not allowed for this job)")
				appendStepLog(out.GPTResponse, StepEPREL, map[string]any{
					"status": "skipped",
					"reason": "entitlement_can_use_eprel",
				})
				break
			}
			// Also scan parsed attrs — eprel_id may only appear after parse_specs.
			id := eprel.ExtractID(normalized, in.Mapped, in.Raw, attrs)
			if id == "" {
				out.Notes = append(out.Notes, "eprel: no id")
				appendStepLog(out.GPTResponse, StepEPREL, map[string]any{"status": "skipped", "reason": "no_id"})
				break
			}
			enricher := e.EPREL
			if enricher == nil {
				enricher = eprel.Disabled{}
			}
			if !enricher.Enabled() {
				out.Notes = append(out.Notes, "eprel: enricher disabled")
				out.EPREL = map[string]any{"id": id, "status": "skipped"}
				appendStepLog(out.GPTResponse, StepEPREL, map[string]any{"status": "skipped", "eprel_id": id, "reason": "disabled"})
				break
			}
			data, err := enricher.Fetch(ctx, id)
			if err != nil {
				out.Notes = append(out.Notes, "eprel: "+TruncateError(err))
				attrs["eprel_id"] = id
				out.Attributes = attrs
				out.ProcessedAttributes = attrs
				appendStepLog(out.GPTResponse, StepEPREL, map[string]any{"status": "failed", "error": TruncateError(err)})
				// Non-fatal: continue pipeline
				break
			}
			if data == nil {
				attrs["eprel_id"] = id
				out.Attributes = attrs
				out.ProcessedAttributes = attrs
				out.EPREL = map[string]any{"id": id, "status": "empty"}
				appendStepLog(out.GPTResponse, StepEPREL, map[string]any{"status": "empty", "eprel_id": id})
				break
			}
			attrs = eprel.MergeInto(attrs, data)
			// Promote energy class onto the characteristic attr key when missing
			// (feeds often omit it; EPREL API is the source of truth).
			if data.EnergyClass != "" {
				if cur := strings.TrimSpace(fmt.Sprint(attrs["energy_class"])); cur == "" || cur == "<nil>" {
					attrs["energy_class"] = data.EnergyClass
				}
			}
			out.Attributes = attrs
			out.ProcessedAttributes = attrs
			out.EPREL = map[string]any{
				"id":           data.ID,
				"label":        data.Label,
				"pdf":          data.PDF,
				"energy_class": data.EnergyClass,
				"energy_scale": data.EnergyScale,
			}
			out.FieldSources["eprel"] = "eprel_api"
			appendStepLog(out.GPTResponse, StepEPREL, map[string]any{"status": "ok", "eprel_id": data.ID})

		case StepCategorize:
			// Vector (if enabled) then LLM taxonomy pick when category still empty.
			runCategorizeStep(ctx, e, companyID, &out, in, categoryNames, policy)

		case StepAIEnhance:
			preservePriorEnhanceHash := func() {
				if in.PriorEnhanceHash != "" {
					out.FieldSources[FieldEnhanceInputHash] = in.PriorEnhanceHash
				}
			}
			if !policy.AllowAI {
				out.ProcessedName = out.Name
				out.ProcessedDescription = out.Description
				out.Notes = append(out.Notes, "ai_enhance: skipped (Free plan — upgrade for AI titles/descriptions)")
				log.Printf("processing: ai_enhance skip reason=entitlement_can_use_ai")
				preservePriorEnhanceHash()
				appendStepLog(out.GPTResponse, StepAIEnhance, map[string]any{
					"status": "skipped",
					"reason": "entitlement_can_use_ai",
				})
				break
			}
			if !e.CompleterEnabled() {
				out.ProcessedName = out.Name
				out.ProcessedDescription = out.Description
				out.Notes = append(out.Notes, "ai_enhance: skipped (platform OpenAI unset; configure admin settings or company BYOK)")
				log.Printf("processing: ai_enhance skip reason=openai_not_configured")
				preservePriorEnhanceHash()
				appendStepLog(out.GPTResponse, StepAIEnhance, map[string]any{
					"status": "skipped",
					"reason": "openai_not_configured",
				})
				break
			}
			langs := in.ContentLanguages
			if len(langs) == 0 {
				langs = []string{in.Language}
			}
			if len(langs) == 0 {
				langs = []string{company.DefaultLanguage}
			}
			primary := in.Language
			if primary == "" {
				primary = langs[0]
			}
			syncCategoryName(&out, in.CategoryNamesByUID)
			displayCat := categoryDisplayLabel(out)
			ensureEnergyClassFromEPREL(attrs)
			enhanceAttrs := AttrsForEnhance(attrs, enhanceAllowedAttrKeys(in, out.Category))
			localized := company.LocalizedContent{}
			if in.PriorLocalized != nil {
				for k, v := range in.PriorLocalized {
					localized[k] = v
				}
			}
			anyFailed := false
			anyOK := false
			allUnchanged := true
			langMetas := make([]any, 0, len(langs))
			for _, lang := range langs {
				tpl := in.EnhanceByLang[lang]
				if tpl.System == "" && tpl.User == "" && lang == primary {
					tpl = PromptTemplates{System: in.EnhanceSystemTemplate, User: in.EnhanceUserTemplate}
				}
				catPrompt := in.CategoryEnhancePrompt
				if lang != primary || catPrompt == "" {
					catPrompt = categoryEnhancePromptFor(in.CategoryPromptsByLang, out.Category, lang, primary)
				}
				titleTpl, descTpl := categoryFormulasFor(in, out.Category)
				priorFields := company.FieldsForLanguage(in.PriorLocalized, lang)
				priorHash := priorFields.EnhanceInputHash
				priorName := priorFields.ProcessedName
				priorDesc := priorFields.ProcessedDescription
				if lang == primary {
					if priorHash == "" {
						priorHash = in.PriorEnhanceHash
					}
					if priorName == "" {
						priorName = in.PriorProcessedName
					}
					if priorDesc == "" {
						priorDesc = in.PriorProcessedDescription
					}
				}
				name, desc, tokens, raw, err := e.enhance(ctx, ProductInput{
					GTIN:                      in.GTIN,
					Name:                      out.Name,
					Description:               out.Description,
					Mapped:                    normalized,
					BrandPrompt:               in.BrandPrompt,
					Language:                  lang,
					EnhanceSystemTemplate:     tpl.System,
					EnhanceUserTemplate:       tpl.User,
					CategoryEnhancePrompt:     catPrompt,
					TitleTemplate:             titleTpl,
					DescriptionTemplate:       descTpl,
					AllowedAttrKeys:           in.AllowedAttrKeys,
					CategoryAttrKeys:          in.CategoryAttrKeys,
					PriorEnhanceHash:          priorHash,
					PriorProcessedName:        priorName,
					PriorProcessedDescription: priorDesc,
					JobID:                     in.JobID,
					CompanyID:                 in.CompanyID,
					RawProductID:              in.RawProductID,
					CategoryUniqueID:          out.Category,
				}, displayCat, enhanceAttrs)
				out.TotalTokens += tokens
				status := enhanceStatusFromMeta(raw)
				meta := map[string]any{"language": lang, "raw": raw}
				if err != nil {
					anyFailed = true
					allUnchanged = false
					meta["status"] = "failed"
					meta["error"] = TruncateError(err)
					out.Notes = append(out.Notes, "ai_enhance: "+TruncateError(err))
					if lang == primary {
						name, desc = out.Name, out.Description
					} else if priorName != "" || priorDesc != "" {
						name, desc = priorName, priorDesc
					} else {
						langMetas = append(langMetas, meta)
						continue
					}
				} else if status == "unchanged" {
					meta["status"] = "unchanged"
				} else if status == "refused" || status == "synthesized" || status == "parse_failed" {
					// Soft quality failure: keep usable copy / synth, never count as real enhance ok.
					allUnchanged = false
					meta["status"] = status
					if reason := enhanceReasonFromMeta(raw); reason != "" {
						out.Notes = append(out.Notes, "ai_enhance: "+status+" ("+reason+")")
					} else {
						out.Notes = append(out.Notes, "ai_enhance: "+status)
					}
				} else {
					allUnchanged = false
					anyOK = true
					meta["status"] = status
				}
				// Prefer title-aware selection + synthesize before deciding hash persistence.
				name = preferredProductTitle(in.GTIN, name, out.Name, priorName, in.Name)
				desc = preferredProductDescription(name, desc, out.Description, priorDesc)
				outerSynth := false
				if descriptionNeedsEnhanceRepair(desc, descTpl, name) {
					if synth := synthesizeProductDescription(name, displayCat, lang, enhanceAttrs, descTpl); synth != "" {
						desc = synth
						outerSynth = true
					}
				}
				weakDesc := descriptionNeedsEnhanceRepair(desc, descTpl, name)
				enhanceMetaTitle, enhanceMetaDesc := enhanceMetaFromRaw(raw)
				if in.OmitSEOMeta {
					enhanceMetaTitle, enhanceMetaDesc = "", ""
				}
				// Only persist enhance_input_hash for quality ok / hash-skip unchanged.
				// Never copy input_hash from error/passthrough/thin/synthesized meta.
				persistHash := ""
				rawHash := enhanceHashFromMeta(raw)
				switch {
				case err != nil:
					persistHash = ""
				case status == "synthesized" || status == "refused" || outerSynth:
					persistHash = ""
					allUnchanged = false
				case status == "unchanged" && !weakDesc:
					persistHash = rawHash
				case status == "ok" && !weakDesc:
					persistHash = rawHash
				default:
					// thin ok / parse_failed / formula mismatch / skipped — do not poison reprocess skip
					persistHash = ""
					if status == "ok" && weakDesc {
						allUnchanged = false
					}
				}
				localized[lang] = company.LocalizedFields{
					ProcessedName:        name,
					ProcessedDescription: desc,
					EnhanceInputHash:     persistHash,
					MetaTitle:            enhanceMetaTitle,
					MetaDescription:      enhanceMetaDesc,
				}
				// Preserve existing meta when re-enhancing titles only.
				// Never keep a bare "| <unique_id>" poisoned meta_title.
				// When dropping poisoned title, also drop empty/weak stub meta_description.
				if prev := company.FieldsForLanguage(in.PriorLocalized, lang); prev.MetaTitle != "" || prev.MetaDescription != "" {
					f := localized[lang]
					poisonedTitle := isPoisonedMetaTitle(prev.MetaTitle)
					if f.MetaTitle == "" && !poisonedTitle {
						f.MetaTitle = prev.MetaTitle
					}
					if f.MetaDescription == "" {
						weakStub := poisonedTitle && isWeakPriorEnhanceDescription(prev.MetaDescription, name, priorName)
						if !weakStub {
							f.MetaDescription = prev.MetaDescription
						}
					}
					localized[lang] = f
				}
				langMetas = append(langMetas, meta)
				if lang == primary {
					out.ProcessedName = name
					out.ProcessedDescription = desc
					if name != "" {
						out.Name = name
					}
					if desc != "" {
						out.Description = desc
					}
					if lf := localized[lang]; lf.MetaTitle != "" {
						out.MetaTitle = lf.MetaTitle
					}
					if lf := localized[lang]; lf.MetaDescription != "" {
						out.MetaDescription = lf.MetaDescription
					}
					// Merge LLM attrs onto pipeline attrs (validated against category keys).
					// Attrs are independent of title/description soft-fail (synthesized/refused).
					if err == nil && status != "failed" && status != "parse_failed" {
						if merged := mergeEnhanceAttrsInto(attrs, enhanceAttrsFromRaw(raw), enhanceAllowedAttrKeys(in, out.Category)); len(merged) > 0 {
							attrs = merged
							enhanceAttrs = AttrsForEnhance(attrs, enhanceAllowedAttrKeys(in, out.Category))
							out.Attributes = attrs
							out.ProcessedAttributes = attrs
							out.FieldSources["attributes"] = "ai_enhance"
						}
					}
					if persistHash != "" {
						out.FieldSources[FieldEnhanceInputHash] = persistHash
					} else {
						delete(out.FieldSources, FieldEnhanceInputHash)
					}
				}
			}
			out.LocalizedContent = localized
			if anyFailed && !anyOK {
				if out.ProcessedName == "" {
					out.ProcessedName = out.Name
				}
				if out.ProcessedDescription == "" {
					out.ProcessedDescription = out.Description
				}
				// Timeout/error/empty/formula-miss: always synthesize a factual fallback when a title exists.
				_, failDescTpl := categoryFormulasFor(in, out.Category)
				if out.ProcessedName != "" && descriptionNeedsEnhanceRepair(out.ProcessedDescription, failDescTpl, out.ProcessedName, out.Name) {
					if synth := synthesizeProductDescription(out.ProcessedName, displayCat, primary, enhanceAttrs, failDescTpl); synth != "" {
						out.ProcessedDescription = synth
						if out.Description == "" || descriptionNeedsEnhanceRepair(out.Description, failDescTpl, out.Name, out.ProcessedName) {
							out.Description = synth
						}
						if lf, ok := localized[primary]; ok {
							lf.ProcessedDescription = synth
							if lf.ProcessedName == "" {
								lf.ProcessedName = out.ProcessedName
							}
							localized[primary] = lf
						}
					}
				}
				// Attempted enhance (even on timeout) — record provider mode, not misleading unknown.
				out.AIProviderMode = e.EngineProviderMode()
				out.FieldSources["name"] = "ai_enhance_failed"
				out.FieldSources["description"] = "ai_enhance_failed"
				// Failed enhance must not leave a skippable hash behind.
				delete(out.FieldSources, FieldEnhanceInputHash)
				for lang, lf := range localized {
					lf.EnhanceInputHash = ""
					localized[lang] = lf
				}
				out.LocalizedContent = localized
				errNote := ""
				for _, m := range langMetas {
					if mm, ok := m.(map[string]any); ok {
						if e, ok := mm["error"].(string); ok && e != "" {
							errNote = e
							break
						}
					}
				}
				appendStepLog(out.GPTResponse, StepAIEnhance, map[string]any{
					"status":    "failed",
					"error":     errNote,
					"languages": langMetas,
				})
				break
			}
			if allUnchanged {
				out.SkipCreditDebit = true
				out.FieldSources["name"] = "ai_enhance_unchanged"
				out.FieldSources["description"] = "ai_enhance_unchanged"
				out.Notes = append(out.Notes, "ai_enhance_unchanged")
				log.Printf("processing: ai_enhance_unchanged")
			} else {
				out.AIProviderMode = e.EngineProviderMode()
				out.FieldSources["name"] = "ai_enhance"
				out.FieldSources["description"] = "ai_enhance"
				for _, m := range langMetas {
					if mm, ok := m.(map[string]any); ok {
						reason, _ := mm["reason"].(string)
						if reason == "" {
							if raw, ok := mm["raw"].(map[string]any); ok {
								reason, _ = raw["reason"].(string)
							}
						}
						if reason != "" {
							out.Notes = append(out.Notes, "ai_enhance: forced re-enhance ("+reason+")")
							log.Printf("processing: ai_enhance forced re-enhance reason=%s", reason)
							break
						}
					}
				}
			}
			appendStepLog(out.GPTResponse, StepAIEnhance, map[string]any{
				"status":    map[string]any{"unchanged": allUnchanged, "ok": anyOK, "failed": anyFailed},
				"languages": langMetas,
			})

		default:
			appendStepLog(out.GPTResponse, step, map[string]any{"status": "unknown"})
		}
	}

	// Pipelines without StepCategorize (enhance_only / normalize_only) still try
	// vector categorize when AllowAI + embeddings are available. Full/categorize
	// already ran runCategorizeStep (vector then LLM) inside the loop.
	if !stepsContain(steps, StepCategorize) {
		tryVectorCategorize(ctx, e, companyID, &out, categoryNames, policy)
		noteMissingCategory(&out, policy, e != nil && e.Vector != nil && e.Vector.Enabled())
	}
	preserveCategoryIfEmpty(&out, in.PriorCategory)
	syncCategoryName(&out, in.CategoryNamesByUID)
	out.Name = preferredProductTitle(in.GTIN, out.Name, out.ProcessedName, in.Name, in.PriorProcessedName)
	out.ProcessedName = preferredProductTitle(in.GTIN, out.ProcessedName, out.Name, in.PriorProcessedName, in.Name)
	out.Description = preferredProductDescription(out.Name, out.Description, out.ProcessedDescription, in.Description)
	out.ProcessedDescription = preferredProductDescription(out.ProcessedName, out.ProcessedDescription, out.Description, in.PriorProcessedDescription)
	if out.ProcessedName == "" {
		out.ProcessedName = out.Name
	}
	// Titles are finalized first so name-as-category and formula leakage can be cleared.
	scrubCategoryPollution(&out, in.CategoryNamesByUID, nil)
	syncCategoryName(&out, in.CategoryNamesByUID)
	displayCat := categoryDisplayLabel(out)
	// After vector/mapped category resolution, formulas may key for the first time —
	// repair short prose / title echo that ignored A1 description_template.
	_, finalDescTpl := categoryFormulasFor(in, out.Category)
	finalRepaired := false
	if descriptionNeedsEnhanceRepair(out.ProcessedDescription, finalDescTpl, out.ProcessedName, out.Name) {
		if synth := synthesizeProductDescription(out.ProcessedName, displayCat, in.Language, out.Attributes, finalDescTpl); synth != "" {
			out.ProcessedDescription = synth
			finalRepaired = true
		}
	}
	if descriptionNeedsEnhanceRepair(out.Description, finalDescTpl, out.Name, out.ProcessedName) {
		if out.ProcessedDescription != "" && !descriptionNeedsEnhanceRepair(out.ProcessedDescription, finalDescTpl, out.Name, out.ProcessedName) {
			out.Description = out.ProcessedDescription
			finalRepaired = true
		} else if synth := synthesizeProductDescription(out.Name, displayCat, in.Language, out.Attributes, finalDescTpl); synth != "" {
			out.Description = synth
			finalRepaired = true
			if descriptionNeedsEnhanceRepair(out.ProcessedDescription, finalDescTpl, out.ProcessedName, out.Name) {
				out.ProcessedDescription = synth
			}
		}
	}
	if finalRepaired {
		delete(out.FieldSources, FieldEnhanceInputHash)
		primary := strings.TrimSpace(in.Language)
		if primary == "" && len(in.ContentLanguages) > 0 {
			primary = strings.TrimSpace(in.ContentLanguages[0])
		}
		if primary == "" {
			primary = company.DefaultLanguage
		}
		for lang, lf := range out.LocalizedContent {
			lf.EnhanceInputHash = ""
			if lang == primary {
				lf.ProcessedDescription = out.ProcessedDescription
				if lf.ProcessedName == "" {
					lf.ProcessedName = out.ProcessedName
				}
			}
			out.LocalizedContent[lang] = lf
		}
	}
	if out.Attributes == nil {
		out.Attributes = map[string]any{}
	}
	// Persist the same sanitize + category allowlist used by enhance (poll already
	// projects clean attrs; DB must not keep feed junk like zavora/vzmetenje).
	allowed := enhanceAllowedAttrKeys(in, out.Category)
	out.Attributes = AttrsForPersist(out.Attributes, allowed)
	out.ProcessedAttributes = AttrsForPersist(out.ProcessedAttributes, allowed)
	if len(out.ProcessedAttributes) == 0 {
		out.ProcessedAttributes = out.Attributes
	}
	// Prefer EngineProviderMode (job/completer label) over stamping "unknown" on
	// 0-token paths (hash-skip / AI skipped). processOne also treats unknown as empty.
	if isUnknownProviderMode(out.AIProviderMode) {
		out.AIProviderMode = preferKnownProviderMode(e.EngineProviderMode(), out.AIProviderMode)
	}
	if len(out.Notes) > 0 {
		out.GPTResponse["notes"] = out.Notes
	}
	return out, nil
}

// preserveCategoryIfEmpty keeps an existing processed category when normalize/AI
// left Category empty (common for A1 feeds where category lives only on processed).
func preserveCategoryIfEmpty(out *StepResult, prior string) {
	if out == nil || strings.TrimSpace(out.Category) != "" {
		return
	}
	prior = strings.TrimSpace(prior)
	if prior == "" || isUnusableCategoryValue(prior, out.ProcessedName, out.Name) {
		return
	}
	out.Category = SanitizeText(prior)
	if out.FieldSources == nil {
		out.FieldSources = map[string]any{}
	}
	out.FieldSources["category"] = "prior_processed"
}

// tryVectorCategorize sets Category from embeddings when mapped unique_id is absent.
// Requires policy.AllowAI and a configured/enabled VectorCategorizer.
func tryVectorCategorize(ctx context.Context, e *Engine, companyID string, out *StepResult, categoryNames []string, policy StepPolicy) {
	if out == nil || strings.TrimSpace(out.Category) != "" {
		return
	}
	if !policy.AllowAI {
		return
	}
	if e == nil || e.Vector == nil || !e.Vector.Enabled() {
		return
	}
	text := strings.TrimSpace(out.Name + " " + out.Description)
	if text == "" {
		return
	}
	cat, err := e.Vector.SuggestCategory(ctx, companyID, text, categoryNames)
	if err != nil {
		out.Notes = append(out.Notes, "vector_categorize: "+TruncateError(err))
		appendStepLog(out.GPTResponse, "vector_categorize", map[string]any{
			"status": "failed",
			"error":  TruncateError(err),
		})
		return
	}
	if strings.TrimSpace(cat) == "" || isUnusableCategoryValue(cat, out.ProcessedName, out.Name) {
		return
	}
	out.Category = SanitizeOutput(cat)
	if out.FieldSources == nil {
		out.FieldSources = map[string]any{}
	}
	out.FieldSources["category"] = "vector"
	out.Notes = append(out.Notes, "category: vector")
	appendStepLog(out.GPTResponse, "vector_categorize", map[string]any{
		"status":   "ok",
		"category": out.Category,
	})
}

// noteMissingCategory records why Category stayed empty (mapped absent; vector skipped or failed).
// Used for pipelines that skip StepCategorize (vector-only post-pass).
func noteMissingCategory(out *StepResult, policy StepPolicy, vectorEnabled bool) {
	if out == nil || strings.TrimSpace(out.Category) != "" {
		return
	}
	for _, n := range out.Notes {
		if strings.HasPrefix(n, "category: unset") || strings.HasPrefix(n, "category: vector") ||
			strings.HasPrefix(n, "category: llm") || strings.HasPrefix(n, "ai_categorize:") {
			return
		}
	}
	switch {
	case !policy.AllowAI:
		out.Notes = append(out.Notes, "category: unset (no mapped unique_id; AI/vector not allowed)")
	case !vectorEnabled:
		out.Notes = append(out.Notes, "category: unset (no mapped unique_id; vector embeddings unavailable)")
	default:
		out.Notes = append(out.Notes, "category: unset (no mapped unique_id; vector did not match)")
	}
}

func resolveSteps(processingType string) []string {
	switch strings.ToLower(strings.TrimSpace(processingType)) {
	case "enhance", "enhance_only", "enhance-only", "title", "description":
		return []string{StepNormalize, StepAIEnhance}
	case "attributes", "attributes_only", "specs", "specifications":
		return []string{StepNormalize, StepParseSpecs, StepFillFields}
	case "eprel", "eprel_only":
		// parse_specs first so eprel_id buried in specifications/attributes is visible.
		return []string{StepNormalize, StepParseSpecs, StepEPREL}
	case "normalize_only":
		return []string{StepNormalize}
	case "categorize", "categorize_only":
		// Taxonomy assign only (vector then LLM) — no title/description rewrite.
		return []string{StepNormalize, StepParseSpecs, StepFillFields, StepEPREL, StepCategorize}
	case "categorize_enhance":
		return append([]string{}, CanonicalSteps...)
	default: // full
		return append([]string{}, CanonicalSteps...)
	}
}

func stepsContain(steps []string, want string) bool {
	for _, s := range steps {
		if s == want {
			return true
		}
	}
	return false
}

// InitialStepProgress builds pending step_progress rows for a job.
func InitialStepProgress(processingType string) []StepProgress {
	steps := resolveSteps(processingType)
	out := make([]StepProgress, 0, len(steps))
	for _, s := range steps {
		out = append(out, StepProgress{Step: s, Status: "pending"})
	}
	return out
}

func appendStepLog(gpt map[string]any, name string, raw any) {
	steps, _ := gpt["steps"].([]any)
	gpt["steps"] = append(steps, map[string]any{"step": name, "raw": raw})
}

// descriptionFormulaRetrySuffix is appended once when LLM JSON ignored the
// category description_template (short prose / title echo instead of multi-section HTML).
const descriptionFormulaRetrySuffix = "\n\nINVALID DESCRIPTION. Your JSON ignored the Description formula. Reply with ONLY one JSON object; \"description\" must be ONE HTML string covering each formula section in order with matching tags (h1/h2/h3/h4, p, ul)."

// descriptionNeedsEnhanceRepair is true when desc is weak/empty/title-echo or
// fails an active category description_template (A1 multi-section HTML).
// Heuristic invent/synth is intentionally excluded here — enhanceHashForceReason
// blocks hash-skip for synth so reprocess can still upgrade to real LLM copy.
func descriptionNeedsEnhanceRepair(desc string, template any, titles ...string) bool {
	if isWeakPriorEnhanceDescription(desc, titles...) {
		return true
	}
	return !descriptionSatisfiesFormula(desc, template)
}

// enhanceHashForceReason returns why a matching prior enhance_input_hash must not
// skip the LLM (empty = safe to reuse as ai_enhance_unchanged).
func enhanceHashForceReason(in ProductInput) string {
	if isPromptLabelTitle(in.PriorProcessedName) {
		return "prompt_label_title"
	}
	if reason := company.EnhanceHashSkipBlockReason(in.PriorProcessedDescription, in.PriorProcessedName, in.Name); reason != "" {
		return reason
	}
	if !descriptionSatisfiesFormula(in.PriorProcessedDescription, in.DescriptionTemplate) {
		return "formula-mismatch"
	}
	return ""
}

func (e *Engine) enhance(ctx context.Context, in ProductInput, category string, attrs map[string]any) (string, string, int, any, error) {
	catUID := strings.TrimSpace(in.CategoryUniqueID)
	catName := strings.TrimSpace(category)
	sysTpl, userTpl := resolveProductPromptTemplates(in)
	hash := HashEnhanceInput(category, in.Name, in.Description, in.BrandPrompt, in.Language, sysTpl, userTpl, attrs)
	if e == nil || e.Completer == nil {
		name := preferredProductTitle(in.GTIN, in.Name, in.PriorProcessedName)
		logEnhanceOutcome(in, catUID, catName, "skip", "completer_nil", Completion{}, 0, "")
		return name,
			preferredProductDescription(name, in.Description, in.PriorProcessedDescription),
			0, map[string]any{"status": "skipped", "reason": "completer_nil"}, nil
	}
	// Skip LLM when inputs match the last successful enhance (before any credit debit),
	// but never reuse thin / title-echo / invent-synth / formula-mismatch priors,
	// or a prompt-leakage title.
	var forceReason string
	if in.PriorEnhanceHash != "" && in.PriorEnhanceHash == hash &&
		(in.PriorProcessedName != "" || in.PriorProcessedDescription != "") {
		forceReason = enhanceHashForceReason(in)
		if forceReason == "" {
			name := preferredProductTitle(in.GTIN, in.PriorProcessedName, in.Name)
			desc := preferredProductDescription(name, in.PriorProcessedDescription, in.Description)
			logEnhanceOutcome(in, catUID, catName, "skip", "unchanged_hash", Completion{}, 0, "")
			return name, desc, 0, map[string]any{
				"status":     "unchanged",
				"input_hash": hash,
			}, nil
		}
		log.Printf("processing: ai_enhance skip_blocked reason=%s category=%q name=%q",
			forceReason, truncateRunes(category, 80), truncateRunes(in.Name, 80))
	}
	system, user := RenderProductEnhancePrompts(sysTpl, userTpl, category, in.Name, in.Description, in.GTIN, in.BrandPrompt, in.Language, attrs)
	logEnhancePrompt(in, catUID, catName, system, user)
	started := time.Now()
	comp, obj, err := CompleteJSON(ctx, e.Completer, system, user, CompleteOptions{
		MaxTokens:   MaxTokensEnhance,
		Temperature: DefaultStructuredTemp,
	})
	elapsed := time.Since(started)
	if err != nil {
		// Network/provider failure vs parse failure after retry
		name := preferredProductTitle(in.GTIN, in.Name, in.PriorProcessedName)
		desc := preferredProductDescription(name, in.Description, in.PriorProcessedDescription)
		outcome := "refuse"
		reason := "empty_or_provider_error"
		if descriptionNeedsEnhanceRepair(desc, in.DescriptionTemplate, name) {
			if synth := synthesizeProductDescription(name, category, in.Language, attrs, in.DescriptionTemplate); synth != "" {
				desc = synth
				outcome = "synthesized"
			}
		}
		if obj == nil && comp.Text == "" {
			logEnhanceOutcome(in, catUID, catName, outcome, reason+":"+TruncateError(err), comp, elapsed, forceReason)
			return name, desc, 0, map[string]any{
				"provider": "passthrough",
				"error":    TruncateError(err),
			}, err
		}
		// Parse failed after retry — keep usable copy; synthesize when empty/weak/formula-miss.
		if outcome != "synthesized" {
			outcome = "refuse"
			reason = "parse_failed"
		} else {
			reason = "parse_failed"
		}
		logEnhanceOutcome(in, catUID, catName, outcome, reason, comp, elapsed, forceReason)
		return name, desc, comp.TotalTokens, map[string]any{
			"status": "parse_failed",
			"reason": "invalid_json",
			"error":  "AI returned invalid JSON; kept original title/description",
			"raw":    truncateRunes(comp.Text, 200),
		}, nil
	}
	llmName := SanitizeOutput(fmt.Sprint(obj["name"]))
	llmDesc := SanitizeOutput(fmt.Sprint(obj["description"]))
	llmMetaTitle, llmMetaDesc := metaFieldsFromEnhanceObj(obj)
	llmAttrs := attrsFromEnhanceObj(obj)
	// Never prefer-fallback to originals for empty/leakage LLM fields — that stamped
	// status=ok + input_hash on garbage enhance output (prod: 30–272ms "ok", wrong copy).
	if reason := llmEnhanceHardRefuseReason(llmName, llmDesc); reason != "" {
		name := preferredProductTitle(in.GTIN, llmName, in.Name, in.PriorProcessedName)
		desc := preferredProductDescription(name, llmDesc)
		synthesized := false
		if descriptionNeedsEnhanceRepair(desc, in.DescriptionTemplate, name) {
			if desc == "" {
				desc = preferredProductDescription(name, in.Description, in.PriorProcessedDescription)
			}
			if descriptionNeedsEnhanceRepair(desc, in.DescriptionTemplate, name) {
				if synth := synthesizeProductDescription(name, category, in.Language, attrs, in.DescriptionTemplate); synth != "" {
					desc = synth
					synthesized = true
				}
			}
		}
		status := "refused"
		outcome := "refuse"
		if synthesized {
			status = "synthesized"
			outcome = "synthesized"
		}
		meta := map[string]any{
			"status": status,
			"reason": reason,
			"raw":    comp.Raw,
		}
		attachEnhanceMeta(meta, llmMetaTitle, llmMetaDesc)
		attachEnhanceAttrs(meta, llmAttrs)
		if forceReason != "" {
			meta["forced_reenhance"] = true
			meta["force_reason"] = forceReason
		}
		logEnhanceOutcome(in, catUID, catName, outcome, reason, comp, elapsed, forceReason)
		return name, desc, comp.TotalTokens, meta, nil
	}

	name := preferredProductTitle(in.GTIN, llmName, in.Name, in.PriorProcessedName)
	// Quality-gate on LLM description alone (do not absorb originals yet).
	desc := preferredProductDescription(name, llmDesc)
	didFormulaRetry := false
	// Fast garbage that ignores A1 description_template: one formula-aware retry, then synthesize.
	if !descriptionSatisfiesFormula(desc, in.DescriptionTemplate) &&
		FormatDescriptionFormulaConstraint(in.DescriptionTemplate) != "" {
		didFormulaRetry = true
		retryUser := user + descriptionFormulaRetrySuffix
		retryStarted := time.Now()
		comp2, obj2, err2 := CompleteJSON(ctx, e.Completer, system, retryUser, CompleteOptions{
			MaxTokens:   MaxTokensEnhance,
			Temperature: DefaultStructuredTemp,
		})
		elapsed += time.Since(retryStarted)
		comp.PromptTokens += comp2.PromptTokens
		comp.OutputTokens += comp2.OutputTokens
		comp.TotalTokens += comp2.TotalTokens
		if err2 == nil && obj2 != nil {
			n2 := SanitizeOutput(fmt.Sprint(obj2["name"]))
			d2 := SanitizeOutput(fmt.Sprint(obj2["description"]))
			if llmEnhanceHardRefuseReason(n2, d2) == "" {
				name = preferredProductTitle(in.GTIN, n2, name, in.Name, in.PriorProcessedName)
				desc = preferredProductDescription(name, d2)
				mt2, md2 := metaFieldsFromEnhanceObj(obj2)
				if mt2 != "" {
					llmMetaTitle = mt2
				}
				if md2 != "" {
					llmMetaDesc = md2
				}
				if a2 := attrsFromEnhanceObj(obj2); len(a2) > 0 {
					llmAttrs = a2
				}
				if comp2.Raw != nil {
					comp.Raw = comp2.Raw
				}
				if comp2.Text != "" {
					comp.Text = comp2.Text
				}
			}
		}
	}
	synthesized := false
	if descriptionNeedsEnhanceRepair(desc, in.DescriptionTemplate, name) {
		if desc == "" {
			desc = preferredProductDescription(name, in.Description, in.PriorProcessedDescription)
		}
		if descriptionNeedsEnhanceRepair(desc, in.DescriptionTemplate, name) {
			if synth := synthesizeProductDescription(name, category, in.Language, attrs, in.DescriptionTemplate); synth != "" {
				desc = synth
				synthesized = true
			}
		}
	}
	meta := map[string]any{
		"status": "ok",
		"raw":    comp.Raw,
	}
	attachEnhanceMeta(meta, llmMetaTitle, llmMetaDesc)
	attachEnhanceAttrs(meta, llmAttrs)
	if forceReason != "" {
		meta["reason"] = forceReason
		meta["forced_reenhance"] = true
	}
	outcome := "ok"
	reason := forceReason
	// Heuristic synthesize must not poison enhance_input_hash (would hash-skip
	// formula-aware LLM copy on reprocess). Only persist hash for real LLM quality.
	if synthesized {
		meta["status"] = "synthesized"
		outcome = "synthesized"
		if didFormulaRetry {
			reason = "formula_retry"
		} else {
			reason = "weak_or_formula_mismatch"
		}
		meta["reason"] = reason
	} else if descriptionNeedsEnhanceRepair(desc, in.DescriptionTemplate, name) {
		meta["status"] = "refused"
		outcome = "refuse"
		if didFormulaRetry {
			reason = "formula_retry"
		} else {
			reason = "weak_or_formula_mismatch"
		}
		meta["reason"] = reason
	} else {
		meta["input_hash"] = hash
		if didFormulaRetry {
			reason = "formula_retry"
			if meta["reason"] == nil {
				meta["reason"] = reason
			}
		}
	}
	logEnhanceOutcome(in, catUID, catName, outcome, reason, comp, elapsed, forceReason)
	return name, desc, comp.TotalTokens, meta, nil
}

// attachEnhanceMeta stores parsed SEO fields on enhance raw meta for RunSteps.
func attachEnhanceMeta(meta map[string]any, title, description string) {
	if meta == nil {
		return
	}
	if title != "" {
		meta["meta_title"] = title
	}
	if description != "" {
		meta["meta_description"] = description
	}
}

// attachEnhanceAttrs stores parsed attrs on enhance raw meta for RunSteps merge.
func attachEnhanceAttrs(meta map[string]any, attrs map[string]any) {
	if meta == nil || len(attrs) == 0 {
		return
	}
	meta["attrs"] = attrs
}

// attrsFromEnhanceObj extracts an attrs/attributes object from enhance JSON.
func attrsFromEnhanceObj(obj map[string]any) map[string]any {
	if obj == nil {
		return nil
	}
	raw := obj["attrs"]
	if raw == nil {
		raw = obj["attributes"]
	}
	return coerceAttrMap(raw)
}

// enhanceAttrsFromRaw reads attrs stashed on enhance raw meta by attachEnhanceAttrs.
func enhanceAttrsFromRaw(raw any) map[string]any {
	m, ok := raw.(map[string]any)
	if !ok || m == nil {
		return nil
	}
	if v, ok := m["attrs"]; ok {
		return coerceAttrMap(v)
	}
	return nil
}

// coerceAttrMap normalizes JSON object / map[string]string into map[string]any.
func coerceAttrMap(raw any) map[string]any {
	switch v := raw.(type) {
	case map[string]any:
		if len(v) == 0 {
			return nil
		}
		out := make(map[string]any, len(v))
		for k, val := range v {
			k = strings.TrimSpace(k)
			if k == "" || val == nil {
				continue
			}
			out[k] = val
		}
		if len(out) == 0 {
			return nil
		}
		return out
	case map[string]string:
		if len(v) == 0 {
			return nil
		}
		out := make(map[string]any, len(v))
		for k, val := range v {
			k = strings.TrimSpace(k)
			val = strings.TrimSpace(val)
			if k == "" || val == "" {
				continue
			}
			out[k] = val
		}
		if len(out) == 0 {
			return nil
		}
		return out
	default:
		return nil
	}
}

// mergeEnhanceAttrsInto merges LLM attrs onto base after AttrsForEnhance validation
// (MapAttrsOntoAllowedKeys + category allowlist). Empty LLM attrs → nil (no change).
func mergeEnhanceAttrsInto(base, llmAttrs map[string]any, allowed map[string]struct{}) map[string]any {
	if len(llmAttrs) == 0 {
		return nil
	}
	validated := AttrsForEnhance(llmAttrs, allowed)
	if len(validated) == 0 {
		return nil
	}
	out := make(map[string]any, len(base)+len(validated))
	for k, v := range base {
		out[k] = v
	}
	for k, v := range validated {
		if !attrValuePresent(v) {
			continue
		}
		out[k] = v
	}
	return out
}

// llmEnhanceHardRefuseReason reports empty/leakage LLM fields that must never be
// accepted as quality enhance (even when originals could fill the gap).
func llmEnhanceHardRefuseReason(name, desc string) string {
	name = strings.TrimSpace(name)
	desc = strings.TrimSpace(desc)
	if name == "" || name == "<nil>" {
		return "empty_name"
	}
	if isPromptLabelTitle(name) {
		return "prompt_leakage_name"
	}
	if desc == "" || desc == "<nil>" {
		return "empty_description"
	}
	return ""
}

func sanitizeJSON(v any) string {
	if v == nil {
		return "{}"
	}
	b, err := json.Marshal(v)
	if err != nil {
		return "{}"
	}
	return SanitizeText(string(b))
}

func firstLine(s string) string {
	s = strings.TrimSpace(s)
	if i := strings.IndexByte(s, '\n'); i >= 0 {
		s = s[:i]
	}
	return SanitizeOutput(strings.Trim(s, "\"'` "))
}

// labeledPromptValue returns the first usable line after any of the given labels
// (case-insensitive), e.g. "Name:" / "Desc:" from ProductEnhanceUser.
// Skips matches whose value is prompt-label / formula-leakage text so instruction
// bullets like "- name: short retail title; follow any Title formula…" do not
// win over the later "Name: <product>" line in CategoryEnhanceUserTemplate.
func labeledPromptValue(user string, labels ...string) string {
	lower := strings.ToLower(user)
	type hit struct {
		at    int
		label string
	}
	var hits []hit
	for _, label := range labels {
		label = strings.ToLower(strings.TrimSpace(label))
		if label == "" {
			continue
		}
		searchFrom := 0
		for {
			rel := strings.Index(lower[searchFrom:], label)
			if rel < 0 {
				break
			}
			at := searchFrom + rel
			hits = append(hits, hit{at: at, label: label})
			searchFrom = at + len(label)
		}
	}
	if len(hits) == 0 {
		return ""
	}
	sort.Slice(hits, func(i, j int) bool { return hits[i].at < hits[j].at })
	for _, h := range hits {
		rest := user[h.at+len(h.label):]
		if j := strings.Index(strings.ToLower(rest), "attrs:"); j >= 0 {
			rest = rest[:j]
		}
		if j := strings.Index(strings.ToLower(rest), "attributes:"); j >= 0 {
			rest = rest[:j]
		}
		val := firstLine(rest)
		if val == "" || isPromptLabelTitle(val) {
			continue
		}
		return val
	}
	return ""
}

// isPromptLabelTitle detects enhance pollution where the model echoed a
// prompt header ("Category:" / "Category: 120") as the product title, or
// leaked instruction scaffolding from CategoryEnhanceUserTemplate /
// AppendFormulaConstraints into name.
func isPromptLabelTitle(s string) bool {
	s = strings.TrimSpace(s)
	if s == "" || s == "<nil>" {
		return false
	}
	if isPromptLeakageTitle(s) {
		return true
	}
	lower := strings.ToLower(s)
	for _, label := range []string{
		"category", "name", "desc", "description",
		"meta", "meta_title", "meta_description", "metadescription",
		"attrs", "attributes", "current name", "current description", "current meta",
	} {
		if lower == label || lower == label+":" {
			return true
		}
		if strings.HasPrefix(lower, label+":") || strings.HasPrefix(lower, label+" :") {
			return true
		}
	}
	return false
}

// isPromptLeakageTitle detects when an LLM echoed enhance-prompt instructions
// (Title formula / short retail title / Schema / Reply with ONLY JSON / …)
// as the product name instead of a real title.
func isPromptLeakageTitle(s string) bool {
	s = strings.TrimSpace(s)
	if s == "" || s == "<nil>" {
		return false
	}
	lower := strings.ToLower(s)
	for _, phrase := range promptLeakagePhrases {
		if strings.Contains(lower, phrase) {
			return true
		}
	}
	// Long dumps of the enhance template: any formula/schema keyword is enough.
	if len([]rune(s)) > 120 {
		for _, kw := range promptLeakageLongKeywords {
			if strings.Contains(lower, kw) {
				return true
			}
		}
	}
	return false
}

// Phrases copied from aiprompts.CategoryEnhanceUserTemplate, BuiltInDefaults,
// and processing.AppendFormulaConstraints / FormatTitleFormulaConstraint.
var promptLeakagePhrases = []string{
	"title formula",
	"follow any",
	"constraints that follow",
	"short retail title",
	"use attrs",
	"write name in",
	"schema:",
	"reply with only json",
	"your reply is parsed as json",
	"description formula",
	"seo meta formula",
	"build name from attrs",
	"prefer attrs values",
	"order matters; join with",
	"do not hardcode a language",
	// Description / title formula scaffolding (Format*FormulaConstraint + enhance templates).
	"emit description as one html",
	"emit one html string",
	"covering each section",
	"tags matching section",
	"overrides any shorter",
	"competing full html",
	"metadescription",
	"never ignore the description",
	"follow any description formula",
	"when a description formula",
	"structured html sections",
	"never copy name as description",
	"do not invent specs",
	"prefer 1-3 factual",
	"1-2 sentences",
	"keep literal text as written",
	"build name from attrs using this structure",
	"category guidance",
	"applies to name and description",
	"apply it to both",
	"never ignore the title formula",
	// Role-sectioned CategoryEnhanceUserTemplate / KeySEOMeta markers.
	"--- title ---",
	"--- end title ---",
	"--- description ---",
	"--- end description ---",
	"--- meta ---",
	"--- end meta ---",
	"--- attributes ---",
	"--- end attributes ---",
	"role: title",
	"role: description",
	"role: meta",
	"role: attributes",
}

var promptLeakageLongKeywords = []string{
	"formula",
	"constraints",
	"schema",
	"json",
	"attrs",
	"retail title",
	"html string",
	"section type",
}

// isUnusableCategoryValue rejects tokens that must never become Category /
// CategoryName: prompt labels / title-formula scaffolding, or the product
// title itself (name-as-category). Callers with taxonomy should prefer
// scrubCategoryPollution so a title that legitimately equals a category name
// can still resolve to unique_id.
func isUnusableCategoryValue(s string, productTitles ...string) bool {
	s = strings.TrimSpace(s)
	if s == "" || s == "<nil>" {
		return false
	}
	if isPromptLabelTitle(s) {
		return true
	}
	return categoryTokenEqualsProductTitle(s, productTitles...)
}

func categoryTokenEqualsProductTitle(s string, productTitles ...string) bool {
	s = strings.TrimSpace(s)
	if s == "" || s == "<nil>" {
		return false
	}
	for _, t := range productTitles {
		t = strings.TrimSpace(t)
		if t == "" || t == "<nil>" {
			continue
		}
		if strings.EqualFold(s, t) {
			return true
		}
	}
	return false
}

func taxonomyDisplayNameSet(namesByUID map[string]string) map[string]struct{} {
	if len(namesByUID) == 0 {
		return nil
	}
	out := make(map[string]struct{}, len(namesByUID))
	for _, name := range namesByUID {
		n := strings.ToLower(strings.TrimSpace(name))
		if n != "" {
			out[n] = struct{}{}
		}
	}
	return out
}

// scrubCategoryPollution clears Category / CategoryName when they hold prompt
// leakage/formula text, or equal the product title without resolving to a
// company taxonomy unique_id / display name.
func scrubCategoryPollution(out *StepResult, namesByUID map[string]string, valid map[string]struct{}) {
	if out == nil {
		return
	}
	titles := []string{out.ProcessedName, out.Name}
	clearCategory := func(note string) {
		out.Category = ""
		out.CategoryName = ""
		if out.FieldSources == nil {
			out.FieldSources = map[string]any{}
		}
		out.FieldSources["category"] = "cleared_invalid"
		out.Notes = append(out.Notes, note)
	}
	cat := strings.TrimSpace(out.Category)
	if cat != "" {
		switch {
		case isPromptLabelTitle(cat):
			clearCategory("category: ignored (prompt leakage)")
		case categoryTokenEqualsProductTitle(cat, titles...) &&
			resolveCompanyCategoryUniqueID(cat, namesByUID, valid) == "":
			clearCategory("category: ignored (product title)")
		}
	}
	name := strings.TrimSpace(out.CategoryName)
	if name == "" {
		return
	}
	if isPromptLabelTitle(name) {
		out.CategoryName = ""
		return
	}
	if categoryTokenEqualsProductTitle(name, titles...) {
		if _, ok := taxonomyDisplayNameSet(namesByUID)[strings.ToLower(name)]; !ok {
			out.CategoryName = ""
		}
	}
}

// preferredProductTitle picks the first usable title, skipping empty values,
// prompt-label echoes like "Category:", instruction-text leakage, and brand-only
// stubs when a longer product name exists among the candidates (e.g. "ANKER"
// vs "Anker Soundcore Space One Pro").
func preferredProductTitle(gtin string, candidates ...string) string {
	usable := make([]string, 0, len(candidates))
	for _, c := range candidates {
		c = strings.TrimSpace(c)
		if c == "" || c == "<nil>" || isPromptLabelTitle(c) {
			continue
		}
		usable = append(usable, ensureReadableTitleSpacing(c))
	}
	for _, c := range usable {
		if isBrandOnlyTitleAmong(c, usable) {
			continue
		}
		return SanitizeOutput(c)
	}
	// All remaining candidates are brand-only relative to each other — pick longest.
	best := ""
	for _, c := range usable {
		if len([]rune(c)) > len([]rune(best)) {
			best = c
		}
	}
	if best != "" {
		return SanitizeOutput(best)
	}
	if strings.TrimSpace(gtin) != "" {
		return SanitizeText("Product " + strings.TrimSpace(gtin))
	}
	return "Product"
}

// isBrandOnlyTitleAmong is true when candidate looks like a brand stub that a
// longer candidate expands (prefix / first-token match).
func isBrandOnlyTitleAmong(candidate string, all []string) bool {
	c := strings.TrimSpace(candidate)
	if c == "" {
		return false
	}
	words := strings.Fields(c)
	if len(words) > 2 {
		return false
	}
	if len(words) == 1 && len([]rune(c)) > 40 {
		return false
	}
	clower := strings.ToLower(c)
	for _, other := range all {
		o := strings.TrimSpace(other)
		if o == "" || strings.EqualFold(o, c) {
			continue
		}
		if len([]rune(o)) <= len([]rune(c)) {
			continue
		}
		olower := strings.ToLower(o)
		if strings.HasPrefix(olower, clower+" ") || strings.HasPrefix(olower, clower+"-") {
			return true
		}
		oWords := strings.Fields(o)
		if len(words) == 1 && len(oWords) >= 2 && strings.EqualFold(oWords[0], words[0]) {
			return true
		}
	}
	return false
}

// Thin wrappers keep processing call sites stable; logic lives in company so
// catalog.RepairWeakEnhanceHashes can reuse it without an import cycle.
func isWeakPriorEnhanceDescription(priorDesc string, titles ...string) bool {
	return company.IsWeakPriorEnhanceDescription(priorDesc, titles...)
}

func containsWeakFillerPhrase(desc string) bool {
	return company.ContainsWeakFillerPhrase(desc)
}

func descriptionEchoesTitle(desc, title string) bool {
	return company.DescriptionEchoesTitle(desc, title)
}

// preferredProductDescription picks the first usable description, skipping empty
// values, prompt-label echoes, weak filler phrases, and copy that merely repeats
// the product title.
func preferredProductDescription(title string, candidates ...string) string {
	for _, c := range candidates {
		c = strings.TrimSpace(c)
		if c == "" || c == "<nil>" || isPromptLabelTitle(c) {
			continue
		}
		if descriptionEchoesTitle(c, title) {
			continue
		}
		if containsWeakFillerPhrase(c) {
			continue
		}
		if company.LooksLikeHeuristicSynthesize(c) {
			continue
		}
		return SanitizeOutput(c)
	}
	return ""
}

func attrLookupCI(attrs map[string]any, keys ...string) string {
	if attrs == nil {
		return ""
	}
	for _, want := range keys {
		want = strings.TrimSpace(want)
		if want == "" {
			continue
		}
		if v := strings.TrimSpace(stringFromAny(attrs[want])); v != "" {
			return v
		}
		for k, raw := range attrs {
			if strings.EqualFold(strings.TrimSpace(k), want) {
				if v := strings.TrimSpace(stringFromAny(raw)); v != "" {
					return v
				}
			}
		}
	}
	return ""
}

func formatAttrDimParts(attrs map[string]any, maxParts int) []string {
	return formatAttrDimPartsLang(attrs, maxParts, "")
}

func formatAttrDimPartsLang(attrs map[string]any, maxParts int, language string) []string {
	if attrs == nil || maxParts <= 0 {
		return nil
	}
	prefer := []string{
		"width", "height", "depth", "weight", "max_load", "max load", "load_capacity",
		"vesa", "screen_size", "diagonal", "color", "material", "size", "energy_class", "warranty",
	}
	parts := make([]string, 0, maxParts)
	seen := map[string]struct{}{}
	sl := isSlovenianContentLanguage(language)
	add := func(k, v string) {
		k = strings.TrimSpace(k)
		v = strings.TrimSpace(v)
		if k == "" || v == "" {
			return
		}
		lk := strings.ToLower(k)
		if _, ok := seen[lk]; ok {
			return
		}
		if isDimensionKey(canonicalizeAttrKey(k)) && isZeroishString(v) {
			return
		}
		seen[lk] = struct{}{}
		parts = append(parts, fmt.Sprintf("%s: %s", attrDimLabel(k, sl), v))
	}
	for _, k := range prefer {
		if len(parts) >= maxParts {
			break
		}
		if v := attrLookupCI(attrs, k); v != "" {
			add(k, v)
		}
	}
	return parts
}

func attrDimLabel(key string, slovenian bool) string {
	canon := canonicalizeAttrKey(key)
	if slovenian {
		switch canon {
		case "width":
			return "Širina"
		case "height":
			return "Višina"
		case "depth":
			return "Globina"
		case "weight":
			return "Teža"
		case "energy_class":
			return "Energijski razred"
		case "warranty":
			return "Garancija"
		case "color":
			return "Barva"
		case "material":
			return "Material"
		case "size", "screen_size", "diagonal":
			return "Velikost"
		}
	}
	switch canon {
	case "width":
		return "Width"
	case "height":
		return "Height"
	case "depth":
		return "Depth"
	case "weight":
		return "Weight"
	case "energy_class":
		return "Energy class"
	case "warranty":
		return "Warranty"
	case "max_load", "load_capacity":
		return "Max load"
	case "screen_size", "diagonal":
		return "Screen size"
	case "product_model":
		return "Model"
	default:
		if canon == "" {
			return key
		}
		return strings.ReplaceAll(canon, "_", " ")
	}
}

// synthesizeProductDescription prefers a category description_template skeleton
// (HTML section types) when present; otherwise falls back to plain title synthesize.
func synthesizeProductDescription(title, category, language string, attrs map[string]any, descriptionTemplate any) string {
	title = ensureReadableTitleSpacing(title)
	if sections, ok := parseDescriptionFormulaSections(descriptionTemplate); ok && len(sections) > 0 {
		if out := synthesizeDescriptionFromFormula(title, category, language, attrs, sections); out != "" {
			return out
		}
	}
	return synthesizeDescriptionFromTitle(title, category, language, attrs)
}

// synthesizeDescriptionFromFormula builds a minimal HTML description matching
// category description_template section types so timeout/fallback still respects
// A1 category structure (unlike plain title synthesize).
// Duplicate paragraph/list section types are emitted once to avoid invent-boilerplate
// repetition (legacy bug: every <p> repeated the same synthesize sentence).
func synthesizeDescriptionFromFormula(title, category, language string, attrs map[string]any, sections []descriptionFormulaSection) string {
	title = ensureReadableTitleSpacing(strings.TrimSpace(title))
	if title == "" || title == "<nil>" || isPromptLabelTitle(title) {
		return ""
	}
	dims := formatAttrDimPartsLang(attrs, 6, language)
	intro := factualDescriptionIntro(title, category, language, attrs)
	if intro == "" {
		return ""
	}
	var b strings.Builder
	paraUsed := false
	listUsed := false
	headingLevels := map[string]bool{}
	for _, s := range sections {
		typ := strings.ToLower(strings.TrimSpace(s.Type))
		switch typ {
		case "h1", "h2", "h3", "h4":
			if headingLevels[typ] {
				continue
			}
			headingLevels[typ] = true
			heading := title
			if typ != "h1" {
				if isSlovenianContentLanguage(language) {
					heading = "Ključne lastnosti"
				} else {
					heading = "Key features"
				}
			}
			fmt.Fprintf(&b, "<%s>%s</%s>", typ, SanitizeOutput(heading), typ)
		case "ul":
			if listUsed {
				continue
			}
			items := dims
			if len(items) == 0 {
				if secondary := factualSecondaryFacts(language, attrs); len(secondary) > 0 {
					items = secondary
				} else {
					items = []string{intro}
				}
			}
			b.WriteString("<ul>")
			for _, it := range items {
				fmt.Fprintf(&b, "<li>%s</li>", SanitizeOutput(it))
			}
			b.WriteString("</ul>")
			listUsed = true
		default: // p and unknown → paragraph (once)
			if paraUsed {
				continue
			}
			fmt.Fprintf(&b, "<p>%s</p>", SanitizeOutput(intro))
			paraUsed = true
		}
	}
	out := strings.TrimSpace(b.String())
	if out == "" {
		return ""
	}
	return SanitizeOutput(out)
}

// factualDescriptionIntro builds a short non-invent paragraph for formula fallback.
// Intentionally avoids heuristicSynthesizePhrases ("je izdelek v kategoriji", …).
func factualDescriptionIntro(title, category, language string, attrs map[string]any) string {
	title = ensureReadableTitleSpacing(strings.TrimSpace(title))
	if title == "" || title == "<nil>" || isPromptLabelTitle(title) {
		return ""
	}
	cat := strings.TrimSpace(category)
	if strings.EqualFold(cat, "general") {
		cat = ""
	}
	brand := attrLookupCI(attrs, "brand")
	model := attrLookupCI(attrs, "product_model", "model", "sku")
	dims := formatAttrDimPartsLang(attrs, 3, language)
	sl := isSlovenianContentLanguage(language)

	var b strings.Builder
	b.WriteString(title)
	if sl {
		switch {
		case brand != "" && cat != "":
			fmt.Fprintf(&b, " — %s, znamka %s", cat, brand)
		case brand != "":
			fmt.Fprintf(&b, " — znamka %s", brand)
		case cat != "":
			fmt.Fprintf(&b, " — %s", cat)
		default:
			b.WriteString(" — katalogski izdelek")
		}
		if model != "" && !strings.Contains(strings.ToLower(title), strings.ToLower(model)) {
			fmt.Fprintf(&b, " (model %s)", model)
		}
		if len(dims) > 0 {
			fmt.Fprintf(&b, ". %s", strings.Join(dims, ", "))
		}
		b.WriteByte('.')
	} else {
		switch {
		case brand != "" && cat != "":
			fmt.Fprintf(&b, " — %s from %s", cat, brand)
		case brand != "":
			fmt.Fprintf(&b, " — from %s", brand)
		case cat != "":
			fmt.Fprintf(&b, " — %s", cat)
		default:
			b.WriteString(" — catalog product")
		}
		if model != "" && !strings.Contains(strings.ToLower(title), strings.ToLower(model)) {
			fmt.Fprintf(&b, " (model %s)", model)
		}
		if len(dims) > 0 {
			fmt.Fprintf(&b, ". %s", strings.Join(dims, ", "))
		}
		b.WriteByte('.')
	}
	return SanitizeOutput(b.String())
}

func factualSecondaryFacts(language string, attrs map[string]any) []string {
	prefer := []string{"energy_class", "warranty", "color", "material"}
	sl := isSlovenianContentLanguage(language)
	var out []string
	for _, k := range prefer {
		v := attrLookupCI(attrs, k)
		if v == "" || isZeroishString(v) {
			continue
		}
		out = append(out, fmt.Sprintf("%s: %s", attrDimLabel(k, sl), v))
	}
	return out
}

// synthesizeDescriptionFromTitle builds a short factual fallback when enhance
// returns empty/title-echo/filler copy. Uses title, category, brand, model, and
// key dims. language is a content-language code (en/sl/…) or English label.
// Prefer synthesizeProductDescription when a description_template is available.
func synthesizeDescriptionFromTitle(title, category, language string, attrs map[string]any) string {
	return factualDescriptionIntro(title, category, language, attrs)
}

func isSlovenianContentLanguage(raw string) bool {
	raw = strings.TrimSpace(raw)
	if raw == "" {
		return false
	}
	if code, err := company.ParseLanguage(raw, false); err == nil && code == "sl" {
		return true
	}
	lower := strings.ToLower(raw)
	return lower == "slovenian" || strings.Contains(lower, "slovenian")
}