fix

2026-08-16 23:07:32 +02:00
parent 389608ea56
commit 6e77154228
14 changed files with 1189 additions and 21 deletions
@@ -0,0 +1,280 @@
+package processing
+
+import (
+	"context"
+	"log"
+	"sort"
+	"strings"
+)
+
+// MaxCategorizeOptions caps the taxonomy list injected into the categorize LLM
+// prompt (token/cost bound). Prefer stable sort by display name then unique_id.
+const MaxCategorizeOptions = 200
+
+// MaxTokensCategorize budgets a short JSON reply {"categoryId","confidence"}.
+// code-fast / reasoning-style models often burn internal tokens before JSON;
+// 256 frequently finishes with empty content and forces a length-cap retry.
+const MaxTokensCategorize = 4096
+
+// categoryOption is one company taxonomy row for the categorize prompt.
+type categoryOption struct {
+	UniqueID string
+	Name     string
+}
+
+// categoryOptionsFromNames builds a stable, capped list from unique_id → name.
+// Empty map → no AI categorize (nothing valid to choose).
+func categoryOptionsFromNames(namesByUID map[string]string) []categoryOption {
+	if len(namesByUID) == 0 {
+		return nil
+	}
+	out := make([]categoryOption, 0, len(namesByUID))
+	for uid, name := range namesByUID {
+		uid = strings.TrimSpace(uid)
+		if uid == "" || strings.EqualFold(uid, "none") {
+			continue
+		}
+		name = strings.TrimSpace(name)
+		if name == "" {
+			name = uid
+		}
+		out = append(out, categoryOption{UniqueID: uid, Name: name})
+	}
+	sort.Slice(out, func(i, j int) bool {
+		if out[i].Name != out[j].Name {
+			return strings.ToLower(out[i].Name) < strings.ToLower(out[j].Name)
+		}
+		return out[i].UniqueID < out[j].UniqueID
+	})
+	if len(out) > MaxCategorizeOptions {
+		out = out[:MaxCategorizeOptions]
+	}
+	return out
+}
+
+// categoryUniqueIDsList returns sorted unique_ids for vector SuggestCategory candidates.
+func categoryUniqueIDsList(namesByUID map[string]string) []string {
+	opts := categoryOptionsFromNames(namesByUID)
+	if len(opts) == 0 {
+		return nil
+	}
+	out := make([]string, len(opts))
+	for i, o := range opts {
+		out[i] = o.UniqueID
+	}
+	return out
+}
+
+// formatAvailableCategoriesList mirrors legacy Descrybe "Available Categories:"
+// bullets: "Name (ID: unique_id)".
+func formatAvailableCategoriesList(opts []categoryOption) string {
+	if len(opts) == 0 {
+		return ""
+	}
+	var b strings.Builder
+	for _, o := range opts {
+		b.WriteString("- ")
+		b.WriteString(SanitizeText(o.Name))
+		b.WriteString(" (ID: ")
+		b.WriteString(SanitizeText(o.UniqueID))
+		b.WriteString(")\n")
+	}
+	return strings.TrimSpace(b.String())
+}
+
+// ProductCategorizeSystem is the built-in system prompt for taxonomy selection.
+const ProductCategorizeSystem = `Product categorization expert.
+Rules:
+- Reply with ONLY a single JSON object (no markdown, no prose, no reasoning)
+- Schema: {"categoryId":"string","confidence":0.0}
+- categoryId MUST be one of the Available Categories IDs exactly (the value in parentheses after ID:)
+- Never invent IDs or names; inventing IDs fails categorization
+- Prefer the most specific category that matches the product
+Example:
+{"categoryId":"50","confidence":0.9}`
+
+// ProductCategorizeUser builds the user prompt with product context + taxonomy list.
+func ProductCategorizeUser(name, description string, opts []categoryOption) string {
+	var b strings.Builder
+	b.WriteString("Select the best category for this product from Available Categories only.\n\n")
+	b.WriteString("Product Information:\n")
+	b.WriteString("Name: ")
+	b.WriteString(SanitizeText(truncateRunes(name, 200)))
+	b.WriteString("\nDesc: ")
+	b.WriteString(SanitizeText(truncateRunes(description, MaxProductDescRunes)))
+	b.WriteString("\n\nAvailable Categories:\n")
+	b.WriteString(formatAvailableCategoriesList(opts))
+	b.WriteString("\n\nImportant:\n")
+	b.WriteString("- Return categoryId as the exact ID from the list\n")
+	b.WriteString("- Do not invent categories outside the list\n")
+	return b.String()
+}
+
+// tryAICategorize asks the Completer to pick a company taxonomy unique_id when
+// Category is still empty after mapped/prior/vector. Never invents outside taxonomy:
+// responses are coerced then filtered to namesByUID / valid unique_ids.
+func tryAICategorize(ctx context.Context, e *Engine, out *StepResult, in ProductInput, policy StepPolicy) {
+	if out == nil || strings.TrimSpace(out.Category) != "" {
+		return
+	}
+	if !policy.AllowAI {
+		return
+	}
+	if e == nil || !e.CompleterEnabled() {
+		return
+	}
+	opts := categoryOptionsFromNames(in.CategoryNamesByUID)
+	if len(opts) == 0 {
+		out.Notes = append(out.Notes, "ai_categorize: skipped (no company taxonomy)")
+		appendStepLog(out.GPTResponse, StepCategorize, map[string]any{
+			"status": "skipped",
+			"reason": "no_taxonomy",
+		})
+		return
+	}
+	name := strings.TrimSpace(out.Name)
+	if name == "" {
+		name = strings.TrimSpace(out.ProcessedName)
+	}
+	desc := strings.TrimSpace(out.Description)
+	if desc == "" {
+		desc = strings.TrimSpace(out.ProcessedDescription)
+	}
+	if name == "" && desc == "" {
+		out.Notes = append(out.Notes, "ai_categorize: skipped (empty product text)")
+		appendStepLog(out.GPTResponse, StepCategorize, map[string]any{
+			"status": "skipped",
+			"reason": "empty_product",
+		})
+		return
+	}
+
+	system := ProductCategorizeSystem
+	user := ProductCategorizeUser(name, desc, opts)
+	comp, obj, err := CompleteJSON(ctx, e.Completer, system, user, CompleteOptions{
+		MaxTokens:   MaxTokensCategorize,
+		Temperature: DefaultStructuredTemp,
+	})
+	out.TotalTokens += comp.TotalTokens
+	if err != nil {
+		out.Notes = append(out.Notes, "ai_categorize: "+TruncateError(err))
+		appendStepLog(out.GPTResponse, StepCategorize, map[string]any{
+			"status": "failed",
+			"error":  TruncateError(err),
+			"tokens": comp.TotalTokens,
+		})
+		return
+	}
+
+	raw := categoryIDFromCategorizeJSON(obj)
+	valid := make(map[string]struct{}, len(opts))
+	for _, o := range opts {
+		valid[o.UniqueID] = struct{}{}
+	}
+	resolved := resolveCompanyCategoryUniqueID(raw, in.CategoryNamesByUID, valid)
+	if resolved == "" || isUnusableCategoryValue(resolved, out.ProcessedName, out.Name) {
+		out.Notes = append(out.Notes, "ai_categorize: ignored (not in company taxonomy)")
+		appendStepLog(out.GPTResponse, StepCategorize, map[string]any{
+			"status":   "rejected",
+			"returned": raw,
+			"tokens":   comp.TotalTokens,
+		})
+		return
+	}
+
+	out.Category = SanitizeText(resolved)
+	if out.FieldSources == nil {
+		out.FieldSources = map[string]any{}
+	}
+	out.FieldSources["category"] = "llm"
+	syncCategoryName(out, in.CategoryNamesByUID)
+	out.Notes = append(out.Notes, "category: llm")
+	log.Printf("processing: category choice uid=%s name=%s source=llm", out.Category, out.CategoryName)
+	appendStepLog(out.GPTResponse, StepCategorize, map[string]any{
+		"status":   "ok",
+		"category": out.Category,
+		"name":     out.CategoryName,
+		"source":   "llm",
+		"tokens":   comp.TotalTokens,
+		"options":  len(opts),
+	})
+}
+
+func categoryIDFromCategorizeJSON(obj map[string]any) string {
+	if obj == nil {
+		return ""
+	}
+	for _, k := range []string{"categoryId", "category_id", "categoryUniqueId", "category_unique_id", "unique_id", "id", "category"} {
+		if s := categoryUniqueIDFromAny(obj[k]); s != "" {
+			return s
+		}
+	}
+	return ""
+}
+
+// firstAvailableCategoryIDFromPrompt extracts the first "(ID: …)" token from a
+// categorize user prompt (HeuristicCompleter / weak-model fallback).
+func firstAvailableCategoryIDFromPrompt(user string) string {
+	const marker = "(id:"
+	lower := strings.ToLower(user)
+	at := strings.Index(lower, marker)
+	if at < 0 {
+		return ""
+	}
+	rest := strings.TrimSpace(user[at+len(marker):])
+	end := strings.Index(rest, ")")
+	if end <= 0 {
+		return ""
+	}
+	return strings.TrimSpace(rest[:end])
+}
+
+// runCategorizeStep applies vector then LLM taxonomy selection when Category empty.
+func runCategorizeStep(ctx context.Context, e *Engine, companyID string, out *StepResult, in ProductInput, categoryNames []string, policy StepPolicy) {
+	if out == nil {
+		return
+	}
+	if strings.TrimSpace(out.Category) != "" {
+		appendStepLog(out.GPTResponse, StepCategorize, map[string]any{
+			"status":   "skipped",
+			"reason":   "already_set",
+			"category": out.Category,
+		})
+		return
+	}
+	if !policy.AllowAI {
+		out.Notes = append(out.Notes, "categorize: skipped (AI not allowed)")
+		appendStepLog(out.GPTResponse, StepCategorize, map[string]any{
+			"status": "skipped",
+			"reason": "entitlement_can_use_ai",
+		})
+		return
+	}
+	tryVectorCategorize(ctx, e, companyID, out, categoryNames, policy)
+	if strings.TrimSpace(out.Category) != "" {
+		return
+	}
+	tryAICategorize(ctx, e, out, in, policy)
+	if strings.TrimSpace(out.Category) == "" {
+		appendStepLog(out.GPTResponse, StepCategorize, map[string]any{
+			"status": "unset",
+			"reason": "no_vector_or_ai_match",
+		})
+	}
+}
+
+// persistMappedCategorySQL writes a taxonomy unique_id onto mapped_data.category
+// when the mapped value is still empty (so Products UI + reprocess stick).
+const persistMappedCategorySQL = `
+	UPDATE raw_products
+	SET mapped_data = jsonb_set(
+		COALESCE(mapped_data, '{}'::jsonb),
+		'{category}',
+		to_jsonb($3::text),
+		true
+	),
+	updated_at = now()
+	WHERE id = $1 AND company_id = $2
+	  AND COALESCE(NULLIF(trim(mapped_data->>'category'), ''), '') = ''
+	  AND COALESCE(NULLIF(trim($3), ''), '') <> ''
+	  AND lower(trim($3)) <> 'none'`