Initial commit of Descrybe v2 without local scratch artifacts.

Drop one-shot tmp/axe scripts and agent i18n scratch so the Gitea tree is deployable.
2026-08-09 22:47:43 +02:00
commit 8580c996c3
1285 changed files with 325780 additions and 0 deletions
@@ -0,0 +1,160 @@
+package processing
+
+import (
+	"context"
+
+	"github.com/descrybe/descrybe-v2/apps/api/internal/company"
+	"github.com/descrybe/descrybe-v2/apps/api/internal/eprel"
+)
+
+// Pipeline step names (canonical order for "full").
+const (
+	StepNormalize  = "normalize"
+	StepParseSpecs = "parse_specs"
+	StepFillFields = "fill_fields"
+	StepEPREL      = "eprel"
+	StepAIEnhance  = "ai_enhance"
+)
+
+// CanonicalSteps is the default full pipeline order.
+var CanonicalSteps = []string{
+	StepNormalize,
+	StepParseSpecs,
+	StepFillFields,
+	StepEPREL,
+	StepAIEnhance,
+}
+
+// Completer is the LLM chat boundary (OpenAI-compatible HTTP API).
+type Completer interface {
+	Complete(ctx context.Context, system, user string) (Completion, error)
+}
+
+// EnableChecker optionally reports whether a Completer should run.
+type EnableChecker interface {
+	Enabled() bool
+}
+
+// Completion is a single model response with usage for cost recording.
+type Completion struct {
+	Text         string
+	PromptTokens int
+	OutputTokens int
+	TotalTokens  int
+	Model        string
+	Raw          any
+}
+
+// Embedder turns text into vectors (OpenAI-compatible /v1/embeddings).
+// Used by vectorization / Pinecone paths; admin role: platformsettings.AIRoleVectorization.
+type Embedder interface {
+	Embed(ctx context.Context, texts []string) ([][]float32, error)
+}
+
+// VectorCategorizer optionally ranks categories by embedding similarity (Pinecone).
+type VectorCategorizer interface {
+	SuggestCategory(ctx context.Context, companyID, productText string, candidates []string) (string, error)
+	Enabled() bool
+}
+
+// EPRELEnricher fetches EU energy-label data when an EPREL ID is present.
+type EPRELEnricher interface {
+	Enabled() bool
+	Fetch(ctx context.Context, eprelID string) (*eprel.Data, error)
+}
+
+// ProductInput is sanitized product payload for pipeline steps.
+type ProductInput struct {
+	GTIN           string
+	Name           string
+	Description    string
+	Mapped         map[string]any
+	Raw            map[string]any
+	StandardFields []StandardFieldDef
+	// BrandPrompt is brand-kit guidance injected into AI enhance when non-empty
+	// (paid plans only; Free may edit the kit but AI apply is gated).
+	BrandPrompt string
+	// Language is the primary content language (companies.language).
+	Language string
+	// ContentLanguages is the ordered list of languages to enhance (primary first).
+	ContentLanguages []string
+	// EnhanceByLang maps language → company/built-in system+user templates.
+	EnhanceByLang map[string]PromptTemplates
+	// EnhanceSystemTemplate / EnhanceUserTemplate are primary-language prompts
+	// (kept for tests / single-lang callers).
+	EnhanceSystemTemplate string
+	EnhanceUserTemplate   string
+	// CategoryEnhancePrompt overrides EnhanceUserTemplate when non-empty
+	// (resolved for the active language before enhance).
+	CategoryEnhancePrompt string
+	// CategoryPromptsByLang maps lower(name) → lang → category override prompt.
+	CategoryPromptsByLang map[string]company.LangPromptMap
+	// Prior* are loaded from the last processed_products row for this raw product.
+	PriorEnhanceHash          string
+	PriorProcessedName        string
+	PriorProcessedDescription string
+	PriorCategory             string
+	PriorLocalized            company.LocalizedContent
+}
+
+// PromptTemplates is a system+user pair for one language.
+type PromptTemplates struct {
+	System string
+	User   string
+}
+
+// StepResult is the cumulative output for one product.
+type StepResult struct {
+	Category             string
+	Name                 string
+	Description          string
+	ProcessedName        string
+	ProcessedDescription string
+	LocalizedContent     company.LocalizedContent
+	Attributes           map[string]any
+	ProcessedAttributes  map[string]any
+	FieldSources         map[string]any
+	EPREL                map[string]any
+	GPTResponse          map[string]any
+	TotalTokens          int
+	// AIProviderMode is written to processed_products.ai_provider_mode /
+	// processing_jobs.ai_provider_mode for analytics
+	// ("internal" | "popular:<name>" | "custom" | "unknown").
+	AIProviderMode string
+	Notes          []string
+	// SkipCreditDebit is set when AI enhance reused prior output because the
+	// enhance input hash matched (ai_enhance_unchanged). processOne must not
+	// ConsumeCredits in that case — no LLM and no meaningful rework.
+	SkipCreditDebit bool
+}
+
+// StepProgress is a job-level snapshot of pipeline step status.
+type StepProgress struct {
+	Step   string `json:"step"`
+	Status string `json:"status"` // pending|running|done|skipped|failed
+	Note   string `json:"note,omitempty"`
+}
+
+// Engine runs ordered processing steps behind interfaces.
+type Engine struct {
+	Completer Completer
+	Vector    VectorCategorizer
+	EPREL     EPRELEnricher
+	// ProviderMode is the analytics label for the active Completer:
+	// "internal" | "popular:<name>" | "custom". Empty falls back to CompleterProviderMode.
+	ProviderMode string
+}
+
+// CompleterEnabled reports whether AI enhance should run.
+func (e *Engine) CompleterEnabled() bool {
+	if e == nil || e.Completer == nil {
+		return false
+	}
+	if c, ok := e.Completer.(EnableChecker); ok {
+		return c.Enabled()
+	}
+	// HeuristicCompleter has no Enabled — treat as enabled only if explicitly set.
+	// Worker sets Completer=nil when platform OpenAI (admin settings / env) is unset.
+	_, isHeuristic := e.Completer.(HeuristicCompleter)
+	return !isHeuristic
+}