package processing import ( "context" "github.com/descrybe/descrybe-v2/apps/api/internal/company" "github.com/descrybe/descrybe-v2/apps/api/internal/eprel" ) // Pipeline step names (canonical order for "full"). const ( StepNormalize = "normalize" StepParseSpecs = "parse_specs" StepFillFields = "fill_fields" StepEPREL = "eprel" StepCategorize = "categorize" StepAIEnhance = "ai_enhance" ) // CanonicalSteps is the default full pipeline order. // categorize runs before ai_enhance so category formulas / overlays key correctly // (legacy Descrybe: GPT picks a taxonomy unique_id when mapped category is empty). var CanonicalSteps = []string{ StepNormalize, StepParseSpecs, StepFillFields, StepEPREL, StepCategorize, StepAIEnhance, } // Completer is the LLM chat boundary (OpenAI-compatible HTTP API). type Completer interface { Complete(ctx context.Context, system, user string) (Completion, error) } // EnableChecker optionally reports whether a Completer should run. type EnableChecker interface { Enabled() bool } // Completion is a single model response with usage for cost recording. type Completion struct { Text string PromptTokens int // CachedPromptTokens is the cached subset of PromptTokens (OpenAI // usage.prompt_tokens_details.cached_tokens). Billed far cheaper than fresh // input, so cost accounting must subtract it. 0 when the provider omits it. CachedPromptTokens int OutputTokens int TotalTokens int Model string Raw any } // Embedder turns text into vectors (OpenAI-compatible /v1/embeddings). // Used by vectorization / Pinecone paths; admin role: platformsettings.AIRoleVectorization. type Embedder interface { Embed(ctx context.Context, texts []string) ([][]float32, error) } // VectorCategorizer optionally ranks categories by embedding similarity (Pinecone). type VectorCategorizer interface { SuggestCategory(ctx context.Context, companyID, productText string, candidates []string) (string, error) Enabled() bool } // EPRELEnricher fetches EU energy-label data when an EPREL ID is present. type EPRELEnricher interface { Enabled() bool Fetch(ctx context.Context, eprelID string) (*eprel.Data, error) } // ProductInput is sanitized product payload for pipeline steps. type ProductInput struct { GTIN string Name string Description string Mapped map[string]any Raw map[string]any StandardFields []StandardFieldDef // BrandPrompt is brand-kit guidance injected into AI enhance when non-empty // (paid plans only; Free may edit the kit but AI apply is gated). BrandPrompt string // Language is the primary content language (companies.language). Language string // ContentLanguages is the ordered list of languages to enhance (primary first). ContentLanguages []string // EnhanceByLang maps language → company/built-in system+user templates. EnhanceByLang map[string]PromptTemplates // EnhanceSystemTemplate / EnhanceUserTemplate are primary-language prompts // (kept for tests / single-lang callers). EnhanceSystemTemplate string EnhanceUserTemplate string // CategoryEnhancePrompt overrides EnhanceUserTemplate when non-empty // (resolved for the active language before enhance). Applied to BOTH name // and description via user framing + system overlay; formulas still win // for structure. CategoryEnhancePrompt string // CategoryPromptsByLang maps lower(name|unique_id) → lang → category override prompt // (JSON/overlay text with {{attrs}}/{{category}}; works with repaired A1 overlays). CategoryPromptsByLang map[string]company.LangPromptMap // TitleTemplate / DescriptionTemplate are language-agnostic category formulas // (categories.title_template / description_template). Injected into the enhance // user prompt as structure constraints; not per-lang text maps. TitleTemplate any DescriptionTemplate any // CategoryFormulasByKey maps lower(name|unique_id) → title/description templates. CategoryFormulasByKey map[string]CategoryFormulas // CategoryNamesByUID maps category unique_id → display name (e.g. "50"→"Štedilniki"). CategoryNamesByUID map[string]string // AllowedAttrKeys are company category_attributes keys for this product's // category unique_id (plus coreCharacteristicAttrKeys via AttrsForEnhance). // nil = sanitize only (unit tests); empty non-nil = core keys only. AllowedAttrKeys map[string]struct{} // CategoryAttrKeys maps category unique_id → attribute keys (job cache). // When set, enhance resolves allowlist from the step result category. CategoryAttrKeys map[string]map[string]struct{} // Prior* are loaded from the last processed_products row for this raw product. PriorEnhanceHash string PriorProcessedName string PriorProcessedDescription string PriorCategory string PriorLocalized company.LocalizedContent // JobID / CompanyID / RawProductID are optional worker correlation ids for // structured ai_enhance logs (never secrets). Empty in unit tests. JobID string CompanyID string RawProductID string // CategoryUniqueID is the taxonomy unique_id for overlay/formula keys when // the enhance display category argument is a localized name. CategoryUniqueID string // CategoryDisplayName is the resolved category label. Together with // CategoryUniqueID it selects the platform-default formula for categories that // carry no prompt/formula of their own (aiprompts.DefaultCategoryFormula). CategoryDisplayName string // OmitSEOMeta skips meta_title / meta_description enhance + free-template fill // (A1 cohort does not use SEO meta fields). OmitSEOMeta bool } // CategoryFormulas holds optional title/description templates for one category key. type CategoryFormulas struct { TitleTemplate any DescriptionTemplate any } // PromptTemplates is a system+user pair for one language. type PromptTemplates struct { System string User string } // StepResult is the cumulative output for one product. type StepResult struct { // Category is the company taxonomy unique_id (DB/API). Prefer CategoryName // for human-facing copy (meta, synthesize, {{category}}). Category string // CategoryName is the display label resolved from categories.name. CategoryName string Name string Description string ProcessedName string ProcessedDescription string LocalizedContent company.LocalizedContent Attributes map[string]any ProcessedAttributes map[string]any FieldSources map[string]any EPREL map[string]any GPTResponse map[string]any TotalTokens int // AIProviderMode is written to processed_products.ai_provider_mode / // processing_jobs.ai_provider_mode for analytics // ("internal" | "popular:" | "custom" | "unknown"). AIProviderMode string Notes []string // SkipCreditDebit is set when AI enhance reused prior output because the // enhance input hash matched (ai_enhance_unchanged). processOne must not // ConsumeCredits in that case — no LLM and no meaningful rework. SkipCreditDebit bool // MetaTitle / MetaDescription are free template SEO fields written by // processOne via fillMetaFromResult (never AI-enhanced by default). MetaTitle string MetaDescription string } // StepProgress is a job-level snapshot of pipeline step status. type StepProgress struct { Step string `json:"step"` Status string `json:"status"` // pending|running|done|skipped|failed Note string `json:"note,omitempty"` } // Engine runs ordered processing steps behind interfaces. type Engine struct { Completer Completer Vector VectorCategorizer EPREL EPRELEnricher // ProviderMode is the analytics label for the active Completer: // "internal" | "popular:" | "custom". Empty falls back to CompleterProviderMode. ProviderMode string } // CompleterEnabled reports whether AI enhance should run. func (e *Engine) CompleterEnabled() bool { if e == nil || e.Completer == nil { return false } if c, ok := e.Completer.(EnableChecker); ok { return c.Enabled() } // HeuristicCompleter (local/dev fallback) and other Completers without // EnableChecker are treated as enabled when explicitly wired on the Engine. // Worker still sets Completer=nil when no provider is configured. return true }