descrybe/apps/api/internal/processing/ai.go

package processing

import (
	"context"

	"github.com/descrybe/descrybe-v2/apps/api/internal/company"
	"github.com/descrybe/descrybe-v2/apps/api/internal/eprel"
)

// Pipeline step names (canonical order for "full").
const (
	StepNormalize   = "normalize"
	StepParseSpecs  = "parse_specs"
	StepFillFields  = "fill_fields"
	StepEPREL       = "eprel"
	StepCategorize  = "categorize"
	StepAIEnhance   = "ai_enhance"
)

// CanonicalSteps is the default full pipeline order.
// categorize runs before ai_enhance so category formulas / overlays key correctly
// (legacy Descrybe: GPT picks a taxonomy unique_id when mapped category is empty).
var CanonicalSteps = []string{
	StepNormalize,
	StepParseSpecs,
	StepFillFields,
	StepEPREL,
	StepCategorize,
	StepAIEnhance,
}

// Completer is the LLM chat boundary (OpenAI-compatible HTTP API).
type Completer interface {
	Complete(ctx context.Context, system, user string) (Completion, error)
}

// EnableChecker optionally reports whether a Completer should run.
type EnableChecker interface {
	Enabled() bool
}

// Completion is a single model response with usage for cost recording.
type Completion struct {
	Text         string
	PromptTokens int
	// CachedPromptTokens is the cached subset of PromptTokens (OpenAI
	// usage.prompt_tokens_details.cached_tokens). Billed far cheaper than fresh
	// input, so cost accounting must subtract it. 0 when the provider omits it.
	CachedPromptTokens int
	OutputTokens       int
	TotalTokens        int
	Model              string
	Raw                any
}

// Embedder turns text into vectors (OpenAI-compatible /v1/embeddings).
// Used by vectorization / Pinecone paths; admin role: platformsettings.AIRoleVectorization.
type Embedder interface {
	Embed(ctx context.Context, texts []string) ([][]float32, error)
}

// VectorCategorizer optionally ranks categories by embedding similarity (Pinecone).
type VectorCategorizer interface {
	SuggestCategory(ctx context.Context, companyID, productText string, candidates []string) (string, error)
	Enabled() bool
}

// EPRELEnricher fetches EU energy-label data when an EPREL ID is present.
type EPRELEnricher interface {
	Enabled() bool
	Fetch(ctx context.Context, eprelID string) (*eprel.Data, error)
}

// ProductInput is sanitized product payload for pipeline steps.
type ProductInput struct {
	GTIN           string
	Name           string
	Description    string
	Mapped         map[string]any
	Raw            map[string]any
	StandardFields []StandardFieldDef
	// BrandPrompt is brand-kit guidance injected into AI enhance when non-empty
	// (paid plans only; Free may edit the kit but AI apply is gated).
	BrandPrompt string
	// Language is the primary content language (companies.language).
	Language string
	// ContentLanguages is the ordered list of languages to enhance (primary first).
	ContentLanguages []string
	// EnhanceByLang maps language → company/built-in system+user templates.
	EnhanceByLang map[string]PromptTemplates
	// EnhanceSystemTemplate / EnhanceUserTemplate are primary-language prompts
	// (kept for tests / single-lang callers).
	EnhanceSystemTemplate string
	EnhanceUserTemplate   string
	// CategoryEnhancePrompt overrides EnhanceUserTemplate when non-empty
	// (resolved for the active language before enhance). Applied to BOTH name
	// and description via user framing + system overlay; formulas still win
	// for structure.
	CategoryEnhancePrompt string
	// CategoryPromptsByLang maps lower(name|unique_id) → lang → category override prompt
	// (JSON/overlay text with {{attrs}}/{{category}}; works with repaired A1 overlays).
	CategoryPromptsByLang map[string]company.LangPromptMap
	// TitleTemplate / DescriptionTemplate are language-agnostic category formulas
	// (categories.title_template / description_template). Injected into the enhance
	// user prompt as structure constraints; not per-lang text maps.
	TitleTemplate       any
	DescriptionTemplate any
	// CategoryFormulasByKey maps lower(name|unique_id) → title/description templates.
	CategoryFormulasByKey map[string]CategoryFormulas
	// CategoryNamesByUID maps category unique_id → display name (e.g. "50"→"Štedilniki").
	CategoryNamesByUID map[string]string
	// AllowedAttrKeys are company category_attributes keys for this product's
	// category unique_id (plus coreCharacteristicAttrKeys via AttrsForEnhance).
	// nil = sanitize only (unit tests); empty non-nil = core keys only.
	AllowedAttrKeys map[string]struct{}
	// CategoryAttrKeys maps category unique_id → attribute keys (job cache).
	// When set, enhance resolves allowlist from the step result category.
	CategoryAttrKeys map[string]map[string]struct{}
	// Prior* are loaded from the last processed_products row for this raw product.
	PriorEnhanceHash          string
	PriorProcessedName        string
	PriorProcessedDescription string
	PriorCategory             string
	PriorLocalized            company.LocalizedContent
	// JobID / CompanyID / RawProductID are optional worker correlation ids for
	// structured ai_enhance logs (never secrets). Empty in unit tests.
	JobID         string
	CompanyID     string
	RawProductID  string
	// CategoryUniqueID is the taxonomy unique_id for overlay/formula keys when
	// the enhance display category argument is a localized name.
	CategoryUniqueID string
	// CategoryDisplayName is the resolved category label. Together with
	// CategoryUniqueID it selects the platform-default formula for categories that
	// carry no prompt/formula of their own (aiprompts.DefaultCategoryFormula).
	CategoryDisplayName string
	// OmitSEOMeta skips meta_title / meta_description enhance + free-template fill
	// (A1 cohort does not use SEO meta fields).
	OmitSEOMeta bool
}

// CategoryFormulas holds optional title/description templates for one category key.
type CategoryFormulas struct {
	TitleTemplate       any
	DescriptionTemplate any
}

// PromptTemplates is a system+user pair for one language.
type PromptTemplates struct {
	System string
	User   string
}

// StepResult is the cumulative output for one product.
type StepResult struct {
	// Category is the company taxonomy unique_id (DB/API). Prefer CategoryName
	// for human-facing copy (meta, synthesize, {{category}}).
	Category string
	// CategoryName is the display label resolved from categories.name.
	CategoryName         string
	Name                 string
	Description          string
	ProcessedName        string
	ProcessedDescription string
	LocalizedContent     company.LocalizedContent
	Attributes           map[string]any
	ProcessedAttributes  map[string]any
	FieldSources         map[string]any
	EPREL                map[string]any
	GPTResponse          map[string]any
	TotalTokens          int
	// AIProviderMode is written to processed_products.ai_provider_mode /
	// processing_jobs.ai_provider_mode for analytics
	// ("internal" | "popular:<name>" | "custom" | "unknown").
	AIProviderMode string
	Notes          []string
	// SkipCreditDebit is set when AI enhance reused prior output because the
	// enhance input hash matched (ai_enhance_unchanged). processOne must not
	// ConsumeCredits in that case — no LLM and no meaningful rework.
	SkipCreditDebit bool
	// MetaTitle / MetaDescription are free template SEO fields written by
	// processOne via fillMetaFromResult (never AI-enhanced by default).
	MetaTitle       string
	MetaDescription string
}

// StepProgress is a job-level snapshot of pipeline step status.
type StepProgress struct {
	Step   string `json:"step"`
	Status string `json:"status"` // pending|running|done|skipped|failed
	Note   string `json:"note,omitempty"`
}

// Engine runs ordered processing steps behind interfaces.
type Engine struct {
	Completer Completer
	Vector    VectorCategorizer
	EPREL     EPRELEnricher
	// ProviderMode is the analytics label for the active Completer:
	// "internal" | "popular:<name>" | "custom". Empty falls back to CompleterProviderMode.
	ProviderMode string
}

// CompleterEnabled reports whether AI enhance should run.
func (e *Engine) CompleterEnabled() bool {
	if e == nil || e.Completer == nil {
		return false
	}
	if c, ok := e.Completer.(EnableChecker); ok {
		return c.Enabled()
	}
	// HeuristicCompleter (local/dev fallback) and other Completers without
	// EnableChecker are treated as enabled when explicitly wired on the Engine.
	// Worker still sets Completer=nil when no provider is configured.
	return true
}