descrybe/apps/api/internal/processing/categorize_confidence_test.go

package processing

import (
	"context"
	"errors"
	"fmt"
	"strings"
	"testing"
)

func categorizeEngine(reply string) (*Engine, *int) {
	enhanceCalls := 0
	e := &Engine{
		Completer: stubCompleter{fn: func(system, user string) (Completion, error) {
			if strings.Contains(strings.ToLower(system), "categoryid") {
				return Completion{Text: reply, TotalTokens: 3}, nil
			}
			enhanceCalls++
			return Completion{Text: `{"name":"Enhanced","description":"Body"}`, TotalTokens: 5}, nil
		}},
		Vector: NoopVectorCategorizer{},
	}
	return e, &enhanceCalls
}

func categorizeInput() ProductInput {
	return ProductInput{
		Name:        "BRUNNER folding camping chair ONE SHOT",
		Description: "A light folding aluminium chair for outdoor use.",
		Mapped: map[string]any{
			"name":        "BRUNNER folding camping chair ONE SHOT",
			"description": "A light folding aluminium chair for outdoor use.",
		},
		CategoryNamesByUID: map[string]string{"1": "Generators", "34": "Smartwatches"},
	}
}

// The reported case: the model always answers with SOME id, so a low score is it
// saying "I do not know". Taking it at face value files a camping chair under
// Generators and then drives that category's formula.
func TestRunSteps_lowConfidenceCategoryFailsTheProduct(t *testing.T) {
	t.Parallel()
	e, enhanceCalls := categorizeEngine(`{"categoryId":"1","confidence":0.08}`)

	out, err := e.RunSteps(context.Background(), "co", categorizeInput(), "full", nil,
		StepPolicy{AllowAI: true})
	if !errors.Is(err, ErrCategoryNotFound) {
		t.Fatalf("err=%v want ErrCategoryNotFound", err)
	}
	for _, want := range []string{"0.08", "0.75"} {
		if !strings.Contains(err.Error(), want) {
			t.Fatalf("error should report the score and the minimum, got %v", err)
		}
	}
	if out.Category != "" {
		t.Fatalf("a rejected pick must not be persisted, got %q", out.Category)
	}
	// The whole point: stop before spending an enhance call on the wrong category.
	if *enhanceCalls != 0 {
		t.Fatalf("enhance ran %d time(s) after the category was rejected", *enhanceCalls)
	}
}

func TestRunSteps_confidentCategoryProceeds(t *testing.T) {
	t.Parallel()
	e, enhanceCalls := categorizeEngine(`{"categoryId":"1","confidence":0.92}`)

	out, err := e.RunSteps(context.Background(), "co", categorizeInput(), "full", nil,
		StepPolicy{AllowAI: true})
	if err != nil {
		t.Fatal(err)
	}
	if out.Category != "1" {
		t.Fatalf("Category=%q want 1", out.Category)
	}
	if *enhanceCalls == 0 {
		t.Fatal("enhance should run for a confident category")
	}
}

// Exactly at the floor is acceptable — the threshold is a minimum, not a bar to clear.
func TestRunSteps_confidenceAtThresholdIsAccepted(t *testing.T) {
	t.Parallel()
	e, _ := categorizeEngine(fmt.Sprintf(`{"categoryId":"34","confidence":%v}`, DefaultMinCategorizeConfidence))
	out, err := e.RunSteps(context.Background(), "co", categorizeInput(), "full", nil,
		StepPolicy{AllowAI: true})
	if err != nil {
		t.Fatalf("confidence == minimum must pass, got %v", err)
	}
	if out.Category != "34" {
		t.Fatalf("Category=%q want 34", out.Category)
	}
}

// A model that reports no score is not penalised — only one that reports a low one.
func TestRunSteps_missingConfidenceIsAccepted(t *testing.T) {
	t.Parallel()
	e, _ := categorizeEngine(`{"categoryId":"34"}`)
	out, err := e.RunSteps(context.Background(), "co", categorizeInput(), "full", nil,
		StepPolicy{AllowAI: true})
	if err != nil {
		t.Fatalf("missing confidence must not fail the product, got %v", err)
	}
	if out.Category != "34" {
		t.Fatalf("Category=%q want 34", out.Category)
	}
}

func TestStepPolicy_confidenceOverride(t *testing.T) {
	t.Parallel()
	if got := (StepPolicy{}).CategorizeConfidence(); got != DefaultMinCategorizeConfidence {
		t.Fatalf("default = %v want %v", got, DefaultMinCategorizeConfidence)
	}
	if got := (StepPolicy{MinCategorizeConfidence: 0.4}).CategorizeConfidence(); got != 0.4 {
		t.Fatalf("override = %v want 0.4", got)
	}
	// A tenant that lowers the bar accepts what the default would reject.
	e, _ := categorizeEngine(`{"categoryId":"1","confidence":0.5}`)
	if _, err := e.RunSteps(context.Background(), "co", categorizeInput(), "full", nil,
		StepPolicy{AllowAI: true, MinCategorizeConfidence: 0.4}); err != nil {
		t.Fatalf("0.5 should pass a 0.4 floor, got %v", err)
	}
}

// A category already resolved from the feed is never second-guessed: categorize
// does not run, so its confidence cannot fail the product.
func TestRunSteps_mappedCategorySkipsConfidenceGate(t *testing.T) {
	t.Parallel()
	e, _ := categorizeEngine(`{"categoryId":"1","confidence":0.01}`)
	in := categorizeInput()
	in.Mapped["category"] = "34"

	out, err := e.RunSteps(context.Background(), "co", in, "full", nil, StepPolicy{AllowAI: true})
	if err != nil {
		t.Fatalf("a feed-supplied category must not be gated, got %v", err)
	}
	if out.Category != "34" {
		t.Fatalf("Category=%q want 34", out.Category)
	}
}

// Free plan / AI disabled must keep working — nothing was asked of a model, so
// there is no failed categorisation to report.
func TestRunSteps_noAIDoesNotFailOnMissingCategory(t *testing.T) {
	t.Parallel()
	e, _ := categorizeEngine(`{"categoryId":"1","confidence":0.01}`)
	if _, err := e.RunSteps(context.Background(), "co", categorizeInput(), "full", nil,
		StepPolicy{AllowAI: false}); err != nil {
		t.Fatalf("AI-disabled processing must not fail on category, got %v", err)
	}
}

func TestCategorizeConfidenceFromJSON(t *testing.T) {
	t.Parallel()
	cases := []struct {
		obj  map[string]any
		want float64
		ok   bool
	}{
		{map[string]any{"confidence": 0.08}, 0.08, true},
		{map[string]any{"confidence": "0.42"}, 0.42, true},
		{map[string]any{"score": 0.9}, 0.9, true},
		// Some models answer on a 0-100 scale.
		{map[string]any{"confidence": 85.0}, 0.85, true},
		{map[string]any{"confidence": 1}, 1, true},
		{map[string]any{"categoryId": "1"}, 0, false},
		{map[string]any{"confidence": "high"}, 0, false},
		{nil, 0, false},
	}
	for _, c := range cases {
		got, ok := categorizeConfidenceFromJSON(c.obj)
		if ok != c.ok || (ok && got != c.want) {
			t.Fatalf("%v → (%v, %v) want (%v, %v)", c.obj, got, ok, c.want, c.ok)
		}
	}
}