descrybe/apps/api/internal/processing/categorize_ai.go

package processing

import (
	"context"
	"encoding/json"
	"errors"
	"fmt"
	"log"
	"sort"
	"strconv"
	"strings"

	"github.com/descrybe/descrybe-v2/apps/api/internal/aiaudit"
)

// ErrCategoryNotFound fails a product whose category could not be determined:
// the model's pick was outside the taxonomy, or it scored below the confidence
// floor. Processing stops for that product instead of enhancing it under a
// category nobody believes in.
var ErrCategoryNotFound = errors.New("product category could not be determined")

// MaxCategorizeOptions caps the taxonomy list injected into the categorize LLM
// prompt (token/cost bound). Prefer stable sort by display name then unique_id.
const MaxCategorizeOptions = 200

// DefaultMinCategorizeConfidence is the floor for accepting an LLM category pick.
//
// The model always answers with SOME id from the list, so a low-confidence reply
// like {"categoryId":"1","confidence":0.08} is the model saying "I do not know" —
// taking it at face value files a camping chair under Generators and then drives
// that category's title/description formula, producing confidently wrong copy.
// Below this floor the product fails with ErrCategoryNotFound instead.
const DefaultMinCategorizeConfidence = 0.75

// MaxTokensCategorize budgets a short JSON reply {"categoryId","confidence"}.
// code-fast / reasoning-style models often burn internal tokens before JSON;
// 256 frequently finishes with empty content and forces a length-cap retry.
const MaxTokensCategorize = 4096

// categoryOption is one company taxonomy row for the categorize prompt.
type categoryOption struct {
	UniqueID string
	Name     string
}

// categoryOptionsFromNames builds a stable, capped list from unique_id → name.
// Empty map → no AI categorize (nothing valid to choose).
func categoryOptionsFromNames(namesByUID map[string]string) []categoryOption {
	if len(namesByUID) == 0 {
		return nil
	}
	out := make([]categoryOption, 0, len(namesByUID))
	for uid, name := range namesByUID {
		uid = strings.TrimSpace(uid)
		if uid == "" || strings.EqualFold(uid, "none") {
			continue
		}
		name = strings.TrimSpace(name)
		if name == "" {
			name = uid
		}
		out = append(out, categoryOption{UniqueID: uid, Name: name})
	}
	sort.Slice(out, func(i, j int) bool {
		if out[i].Name != out[j].Name {
			return strings.ToLower(out[i].Name) < strings.ToLower(out[j].Name)
		}
		return out[i].UniqueID < out[j].UniqueID
	})
	if len(out) > MaxCategorizeOptions {
		out = out[:MaxCategorizeOptions]
	}
	return out
}

// categoryUniqueIDsList returns sorted unique_ids for vector SuggestCategory candidates.
func categoryUniqueIDsList(namesByUID map[string]string) []string {
	opts := categoryOptionsFromNames(namesByUID)
	if len(opts) == 0 {
		return nil
	}
	out := make([]string, len(opts))
	for i, o := range opts {
		out[i] = o.UniqueID
	}
	return out
}

// formatAvailableCategoriesList mirrors legacy Descrybe "Available Categories:"
// bullets: "Name (ID: unique_id)".
func formatAvailableCategoriesList(opts []categoryOption) string {
	if len(opts) == 0 {
		return ""
	}
	var b strings.Builder
	for _, o := range opts {
		b.WriteString("- ")
		b.WriteString(SanitizeText(o.Name))
		b.WriteString(" (ID: ")
		b.WriteString(SanitizeText(o.UniqueID))
		b.WriteString(")\n")
	}
	return strings.TrimSpace(b.String())
}

// ProductCategorizeSystem is the built-in system prompt for taxonomy selection
// (aiprompts.RoleCategorize — pick company unique_id; separate from enhance overlay).
const ProductCategorizeSystem = `Product categorization expert.
Rules:
- Reply with ONLY a single JSON object (no markdown, no prose, no reasoning)
- Schema: {"categoryId":"string","confidence":0.0}
- categoryId MUST be one of the Available Categories IDs exactly (the value in parentheses after ID:)
- Never invent IDs or names; inventing IDs fails categorization
- Prefer the most specific category that matches the product
Example:
{"categoryId":"50","confidence":0.9}`

// ProductCategorizeUser builds the user prompt with product context + taxonomy list.
func ProductCategorizeUser(name, description string, opts []categoryOption) string {
	var b strings.Builder
	b.WriteString("Select the best category for this product from Available Categories only.\n\n")
	b.WriteString("Product Information:\n")
	b.WriteString("Name: ")
	b.WriteString(SanitizeText(truncateRunes(name, 200)))
	b.WriteString("\nDesc: ")
	b.WriteString(SanitizeText(truncateRunes(description, MaxProductDescRunes)))
	b.WriteString("\n\nAvailable Categories:\n")
	b.WriteString(formatAvailableCategoriesList(opts))
	b.WriteString("\n\nImportant:\n")
	b.WriteString("- Return categoryId as the exact ID from the list\n")
	b.WriteString("- Do not invent categories outside the list\n")
	return b.String()
}

// tryAICategorize asks the Completer to pick a company taxonomy unique_id when
// Category is still empty after mapped/prior/vector. Never invents outside taxonomy:
// responses are coerced then filtered to namesByUID / valid unique_ids.
func tryAICategorize(ctx context.Context, e *Engine, out *StepResult, in ProductInput, policy StepPolicy) error {
	if out == nil || strings.TrimSpace(out.Category) != "" {
		return nil
	}
	if !policy.AllowAI {
		return nil
	}
	if e == nil || !e.CompleterEnabled() {
		return nil
	}
	opts := categoryOptionsFromNames(in.CategoryNamesByUID)
	if len(opts) == 0 {
		out.Notes = append(out.Notes, "ai_categorize: skipped (no company taxonomy)")
		appendStepLog(out.GPTResponse, StepCategorize, map[string]any{
			"status": "skipped",
			"reason": "no_taxonomy",
		})
		return nil
	}
	name := strings.TrimSpace(out.Name)
	if name == "" {
		name = strings.TrimSpace(out.ProcessedName)
	}
	desc := strings.TrimSpace(out.Description)
	if desc == "" {
		desc = strings.TrimSpace(out.ProcessedDescription)
	}
	if name == "" && desc == "" {
		out.Notes = append(out.Notes, "ai_categorize: skipped (empty product text)")
		appendStepLog(out.GPTResponse, StepCategorize, map[string]any{
			"status": "skipped",
			"reason": "empty_product",
		})
		return nil
	}

	system := ProductCategorizeSystem
	user := ProductCategorizeUser(name, desc, opts)
	comp, obj, err := CompleteJSON(ctx, e.Completer, system, user, CompleteOptions{
		MaxTokens:   MaxTokensCategorize,
		Temperature: DefaultStructuredTemp,
	})
	out.TotalTokens += comp.TotalTokens
	if err != nil {
		out.Notes = append(out.Notes, "ai_categorize: "+TruncateError(err))
		appendStepLog(out.GPTResponse, StepCategorize, map[string]any{
			"status": "failed",
			"error":  TruncateError(err),
			"tokens": comp.TotalTokens,
		})
		return nil
	}

	raw := categoryIDFromCategorizeJSON(obj)
	confidence, hasConfidence := categorizeConfidenceFromJSON(obj)
	valid := make(map[string]struct{}, len(opts))
	for _, o := range opts {
		valid[o.UniqueID] = struct{}{}
	}
	resolved := resolveCompanyCategoryUniqueID(raw, in.CategoryNamesByUID, valid)
	if resolved == "" || isUnusableCategoryValue(resolved, out.ProcessedName, out.Name) {
		out.Notes = append(out.Notes, "ai_categorize: no category found (not in company taxonomy)")
		appendStepLog(out.GPTResponse, StepCategorize, map[string]any{
			"status":   "rejected",
			"returned": raw,
			"tokens":   comp.TotalTokens,
		})
		return fmt.Errorf("%w: model returned %q, which is not in the company taxonomy",
			ErrCategoryNotFound, truncateRunes(raw, 60))
	}

	// A confident-looking id with a low score is the model guessing. Refuse it
	// rather than driving the wrong category's formula with it.
	if min := policy.CategorizeConfidence(); hasConfidence && confidence < min {
		out.Notes = append(out.Notes, fmt.Sprintf(
			"ai_categorize: no category found (confidence %.2f below %.2f)", confidence, min))
		appendStepLog(out.GPTResponse, StepCategorize, map[string]any{
			"status":     "low_confidence",
			"returned":   raw,
			"resolved":   resolved,
			"confidence": confidence,
			"minimum":    min,
			"tokens":     comp.TotalTokens,
		})
		log.Printf("processing: category rejected uid=%s confidence=%.2f min=%.2f reason=low_confidence",
			resolved, confidence, min)
		return fmt.Errorf("%w: best match %q scored %.2f, below the %.2f minimum",
			ErrCategoryNotFound, truncateRunes(resolved, 60), confidence, min)
	}

	out.Category = SanitizeText(resolved)
	if out.FieldSources == nil {
		out.FieldSources = map[string]any{}
	}
	out.FieldSources["category"] = "llm"
	syncCategoryName(out, in.CategoryNamesByUID)
	out.Notes = append(out.Notes, "category: llm")
	log.Printf("processing: category choice uid=%s name=%s source=llm", out.Category, out.CategoryName)
	appendStepLog(out.GPTResponse, StepCategorize, map[string]any{
		"status":   "ok",
		"category": out.Category,
		"name":     out.CategoryName,
		"source":   "llm",
		"tokens":   comp.TotalTokens,
		"options":  len(opts),
	})
	if hasConfidence {
		out.Notes = append(out.Notes, fmt.Sprintf("ai_categorize: confidence %.2f", confidence))
	}
	return nil
}

// categorizeConfidenceFromJSON reads the model's self-reported score. ok=false when
// the field is missing or unparseable — a model that reports nothing is not
// penalised, only one that reports a low score.
func categorizeConfidenceFromJSON(obj map[string]any) (float64, bool) {
	if obj == nil {
		return 0, false
	}
	for _, k := range []string{"confidence", "score", "certainty"} {
		switch v := obj[k].(type) {
		case float64:
			return clampConfidence(v), true
		case int:
			return clampConfidence(float64(v)), true
		case json.Number:
			if f, err := v.Float64(); err == nil {
				return clampConfidence(f), true
			}
		case string:
			if f, err := strconv.ParseFloat(strings.TrimSpace(v), 64); err == nil {
				return clampConfidence(f), true
			}
		}
	}
	return 0, false
}

// clampConfidence folds a 0-100 style score onto 0-1 and bounds it.
func clampConfidence(v float64) float64 {
	if v > 1 && v <= 100 {
		v /= 100
	}
	if v < 0 {
		return 0
	}
	if v > 1 {
		return 1
	}
	return v
}

func categoryIDFromCategorizeJSON(obj map[string]any) string {
	if obj == nil {
		return ""
	}
	for _, k := range []string{"categoryId", "category_id", "categoryUniqueId", "category_unique_id", "unique_id", "id", "category"} {
		if s := categoryUniqueIDFromAny(obj[k]); s != "" {
			return s
		}
	}
	return ""
}

// firstAvailableCategoryIDFromPrompt extracts the first "(ID: …)" token from a
// categorize user prompt (HeuristicCompleter / weak-model fallback).
func firstAvailableCategoryIDFromPrompt(user string) string {
	const marker = "(id:"
	lower := strings.ToLower(user)
	at := strings.Index(lower, marker)
	if at < 0 {
		return ""
	}
	rest := strings.TrimSpace(user[at+len(marker):])
	end := strings.Index(rest, ")")
	if end <= 0 {
		return ""
	}
	return strings.TrimSpace(rest[:end])
}

// runCategorizeStep applies vector then LLM taxonomy selection when Category empty.
// runCategorizeStep returns ErrCategoryNotFound when the model was asked for a
// category and could not give a usable one — the caller fails the product rather
// than enhancing it under a guess.
func runCategorizeStep(ctx context.Context, e *Engine, companyID string, out *StepResult, in ProductInput, categoryNames []string, policy StepPolicy) error {
	if out == nil {
		return nil
	}
	if strings.TrimSpace(out.Category) != "" {
		appendStepLog(out.GPTResponse, StepCategorize, map[string]any{
			"status":   "skipped",
			"reason":   "already_set",
			"category": out.Category,
		})
		return nil
	}
	if !policy.AllowAI {
		out.Notes = append(out.Notes, "categorize: skipped (AI not allowed)")
		appendStepLog(out.GPTResponse, StepCategorize, map[string]any{
			"status": "skipped",
			"reason": "entitlement_can_use_ai",
		})
		return nil
	}
	tryVectorCategorize(ctx, e, companyID, out, categoryNames, policy)
	if strings.TrimSpace(out.Category) != "" {
		return nil
	}
	// Taxonomy picks and copy generation share one Completer; tag the role so the
	// admin inspector can tell them apart.
	if err := tryAICategorize(aiaudit.WithCall(ctx, aiaudit.CallContext{Role: aiaudit.RoleCategorize}), e, out, in, policy); err != nil {
		return err
	}
	if strings.TrimSpace(out.Category) == "" {
		appendStepLog(out.GPTResponse, StepCategorize, map[string]any{
			"status": "unset",
			"reason": "no_vector_or_ai_match",
		})
	}
	return nil
}

// persistMappedCategorySQL writes a taxonomy unique_id onto mapped_data.category
// when the mapped value is still empty (so Products UI + reprocess stick).
const persistMappedCategorySQL = `
	UPDATE raw_products
	SET mapped_data = jsonb_set(
		COALESCE(mapped_data, '{}'::jsonb),
		'{category}',
		to_jsonb($3::text),
		true
	),
	updated_at = now()
	WHERE id = $1 AND company_id = $2
	  AND COALESCE(NULLIF(trim(mapped_data->>'category'), ''), '') = ''
	  AND COALESCE(NULLIF(trim($3), ''), '') <> ''
	  AND lower(trim($3)) <> 'none'`