descrybe/apps/api/internal/processing/categorize_ai.go

package processing

import (
	"context"
	"log"
	"sort"
	"strings"
)

// MaxCategorizeOptions caps the taxonomy list injected into the categorize LLM
// prompt (token/cost bound). Prefer stable sort by display name then unique_id.
const MaxCategorizeOptions = 200

// MaxTokensCategorize budgets a short JSON reply {"categoryId","confidence"}.
// code-fast / reasoning-style models often burn internal tokens before JSON;
// 256 frequently finishes with empty content and forces a length-cap retry.
const MaxTokensCategorize = 4096

// categoryOption is one company taxonomy row for the categorize prompt.
type categoryOption struct {
	UniqueID string
	Name     string
}

// categoryOptionsFromNames builds a stable, capped list from unique_id → name.
// Empty map → no AI categorize (nothing valid to choose).
func categoryOptionsFromNames(namesByUID map[string]string) []categoryOption {
	if len(namesByUID) == 0 {
		return nil
	}
	out := make([]categoryOption, 0, len(namesByUID))
	for uid, name := range namesByUID {
		uid = strings.TrimSpace(uid)
		if uid == "" || strings.EqualFold(uid, "none") {
			continue
		}
		name = strings.TrimSpace(name)
		if name == "" {
			name = uid
		}
		out = append(out, categoryOption{UniqueID: uid, Name: name})
	}
	sort.Slice(out, func(i, j int) bool {
		if out[i].Name != out[j].Name {
			return strings.ToLower(out[i].Name) < strings.ToLower(out[j].Name)
		}
		return out[i].UniqueID < out[j].UniqueID
	})
	if len(out) > MaxCategorizeOptions {
		out = out[:MaxCategorizeOptions]
	}
	return out
}

// categoryUniqueIDsList returns sorted unique_ids for vector SuggestCategory candidates.
func categoryUniqueIDsList(namesByUID map[string]string) []string {
	opts := categoryOptionsFromNames(namesByUID)
	if len(opts) == 0 {
		return nil
	}
	out := make([]string, len(opts))
	for i, o := range opts {
		out[i] = o.UniqueID
	}
	return out
}

// formatAvailableCategoriesList mirrors legacy Descrybe "Available Categories:"
// bullets: "Name (ID: unique_id)".
func formatAvailableCategoriesList(opts []categoryOption) string {
	if len(opts) == 0 {
		return ""
	}
	var b strings.Builder
	for _, o := range opts {
		b.WriteString("- ")
		b.WriteString(SanitizeText(o.Name))
		b.WriteString(" (ID: ")
		b.WriteString(SanitizeText(o.UniqueID))
		b.WriteString(")\n")
	}
	return strings.TrimSpace(b.String())
}

// ProductCategorizeSystem is the built-in system prompt for taxonomy selection.
const ProductCategorizeSystem = `Product categorization expert.
Rules:
- Reply with ONLY a single JSON object (no markdown, no prose, no reasoning)
- Schema: {"categoryId":"string","confidence":0.0}
- categoryId MUST be one of the Available Categories IDs exactly (the value in parentheses after ID:)
- Never invent IDs or names; inventing IDs fails categorization
- Prefer the most specific category that matches the product
Example:
{"categoryId":"50","confidence":0.9}`

// ProductCategorizeUser builds the user prompt with product context + taxonomy list.
func ProductCategorizeUser(name, description string, opts []categoryOption) string {
	var b strings.Builder
	b.WriteString("Select the best category for this product from Available Categories only.\n\n")
	b.WriteString("Product Information:\n")
	b.WriteString("Name: ")
	b.WriteString(SanitizeText(truncateRunes(name, 200)))
	b.WriteString("\nDesc: ")
	b.WriteString(SanitizeText(truncateRunes(description, MaxProductDescRunes)))
	b.WriteString("\n\nAvailable Categories:\n")
	b.WriteString(formatAvailableCategoriesList(opts))
	b.WriteString("\n\nImportant:\n")
	b.WriteString("- Return categoryId as the exact ID from the list\n")
	b.WriteString("- Do not invent categories outside the list\n")
	return b.String()
}

// tryAICategorize asks the Completer to pick a company taxonomy unique_id when
// Category is still empty after mapped/prior/vector. Never invents outside taxonomy:
// responses are coerced then filtered to namesByUID / valid unique_ids.
func tryAICategorize(ctx context.Context, e *Engine, out *StepResult, in ProductInput, policy StepPolicy) {
	if out == nil || strings.TrimSpace(out.Category) != "" {
		return
	}
	if !policy.AllowAI {
		return
	}
	if e == nil || !e.CompleterEnabled() {
		return
	}
	opts := categoryOptionsFromNames(in.CategoryNamesByUID)
	if len(opts) == 0 {
		out.Notes = append(out.Notes, "ai_categorize: skipped (no company taxonomy)")
		appendStepLog(out.GPTResponse, StepCategorize, map[string]any{
			"status": "skipped",
			"reason": "no_taxonomy",
		})
		return
	}
	name := strings.TrimSpace(out.Name)
	if name == "" {
		name = strings.TrimSpace(out.ProcessedName)
	}
	desc := strings.TrimSpace(out.Description)
	if desc == "" {
		desc = strings.TrimSpace(out.ProcessedDescription)
	}
	if name == "" && desc == "" {
		out.Notes = append(out.Notes, "ai_categorize: skipped (empty product text)")
		appendStepLog(out.GPTResponse, StepCategorize, map[string]any{
			"status": "skipped",
			"reason": "empty_product",
		})
		return
	}

	system := ProductCategorizeSystem
	user := ProductCategorizeUser(name, desc, opts)
	comp, obj, err := CompleteJSON(ctx, e.Completer, system, user, CompleteOptions{
		MaxTokens:   MaxTokensCategorize,
		Temperature: DefaultStructuredTemp,
	})
	out.TotalTokens += comp.TotalTokens
	if err != nil {
		out.Notes = append(out.Notes, "ai_categorize: "+TruncateError(err))
		appendStepLog(out.GPTResponse, StepCategorize, map[string]any{
			"status": "failed",
			"error":  TruncateError(err),
			"tokens": comp.TotalTokens,
		})
		return
	}

	raw := categoryIDFromCategorizeJSON(obj)
	valid := make(map[string]struct{}, len(opts))
	for _, o := range opts {
		valid[o.UniqueID] = struct{}{}
	}
	resolved := resolveCompanyCategoryUniqueID(raw, in.CategoryNamesByUID, valid)
	if resolved == "" || isUnusableCategoryValue(resolved, out.ProcessedName, out.Name) {
		out.Notes = append(out.Notes, "ai_categorize: ignored (not in company taxonomy)")
		appendStepLog(out.GPTResponse, StepCategorize, map[string]any{
			"status":   "rejected",
			"returned": raw,
			"tokens":   comp.TotalTokens,
		})
		return
	}

	out.Category = SanitizeText(resolved)
	if out.FieldSources == nil {
		out.FieldSources = map[string]any{}
	}
	out.FieldSources["category"] = "llm"
	syncCategoryName(out, in.CategoryNamesByUID)
	out.Notes = append(out.Notes, "category: llm")
	log.Printf("processing: category choice uid=%s name=%s source=llm", out.Category, out.CategoryName)
	appendStepLog(out.GPTResponse, StepCategorize, map[string]any{
		"status":   "ok",
		"category": out.Category,
		"name":     out.CategoryName,
		"source":   "llm",
		"tokens":   comp.TotalTokens,
		"options":  len(opts),
	})
}

func categoryIDFromCategorizeJSON(obj map[string]any) string {
	if obj == nil {
		return ""
	}
	for _, k := range []string{"categoryId", "category_id", "categoryUniqueId", "category_unique_id", "unique_id", "id", "category"} {
		if s := categoryUniqueIDFromAny(obj[k]); s != "" {
			return s
		}
	}
	return ""
}

// firstAvailableCategoryIDFromPrompt extracts the first "(ID: …)" token from a
// categorize user prompt (HeuristicCompleter / weak-model fallback).
func firstAvailableCategoryIDFromPrompt(user string) string {
	const marker = "(id:"
	lower := strings.ToLower(user)
	at := strings.Index(lower, marker)
	if at < 0 {
		return ""
	}
	rest := strings.TrimSpace(user[at+len(marker):])
	end := strings.Index(rest, ")")
	if end <= 0 {
		return ""
	}
	return strings.TrimSpace(rest[:end])
}

// runCategorizeStep applies vector then LLM taxonomy selection when Category empty.
func runCategorizeStep(ctx context.Context, e *Engine, companyID string, out *StepResult, in ProductInput, categoryNames []string, policy StepPolicy) {
	if out == nil {
		return
	}
	if strings.TrimSpace(out.Category) != "" {
		appendStepLog(out.GPTResponse, StepCategorize, map[string]any{
			"status":   "skipped",
			"reason":   "already_set",
			"category": out.Category,
		})
		return
	}
	if !policy.AllowAI {
		out.Notes = append(out.Notes, "categorize: skipped (AI not allowed)")
		appendStepLog(out.GPTResponse, StepCategorize, map[string]any{
			"status": "skipped",
			"reason": "entitlement_can_use_ai",
		})
		return
	}
	tryVectorCategorize(ctx, e, companyID, out, categoryNames, policy)
	if strings.TrimSpace(out.Category) != "" {
		return
	}
	tryAICategorize(ctx, e, out, in, policy)
	if strings.TrimSpace(out.Category) == "" {
		appendStepLog(out.GPTResponse, StepCategorize, map[string]any{
			"status": "unset",
			"reason": "no_vector_or_ai_match",
		})
	}
}

// persistMappedCategorySQL writes a taxonomy unique_id onto mapped_data.category
// when the mapped value is still empty (so Products UI + reprocess stick).
const persistMappedCategorySQL = `
	UPDATE raw_products
	SET mapped_data = jsonb_set(
		COALESCE(mapped_data, '{}'::jsonb),
		'{category}',
		to_jsonb($3::text),
		true
	),
	updated_at = now()
	WHERE id = $1 AND company_id = $2
	  AND COALESCE(NULLIF(trim(mapped_data->>'category'), ''), '') = ''
	  AND COALESCE(NULLIF(trim($3), ''), '') <> ''
	  AND lower(trim($3)) <> 'none'`