This commit is contained in:
2026-08-23 23:14:57 +02:00
parent 296dc3c841
commit a246ed8fe5
6 changed files with 399 additions and 38 deletions
+64 -9
View File
@@ -31,14 +31,15 @@ import (
)
type server struct {
apiKey string
model string
logReq atomic.Int64
apiKey string
model string
categorizeConfidence float64
logReq atomic.Int64
}
type chatRequest struct {
Model string `json:"model"`
Messages []struct {
Model string `json:"model"`
Messages []struct {
Role string `json:"role"`
Content string `json:"content"`
} `json:"messages"`
@@ -50,11 +51,16 @@ func main() {
addr := flag.String("addr", "127.0.0.1:18767", "listen address")
key := flag.String("key", envOr("MOCK_LLM_API_KEY", "local-test"), "Bearer API key (non-empty placeholder)")
model := flag.String("model", envOr("MOCK_LLM_MODEL", "mock-llm"), "model id returned by /v1/models")
// Lets a local run reproduce a low-confidence taxonomy pick (see
// processing.DefaultMinCategorizeConfidence).
categorizeConfidence := flag.Float64("categorize-confidence", 0,
"when > 0, answer categorize calls with this confidence score")
flag.Parse()
s := &server{
apiKey: strings.TrimSpace(*key),
model: strings.TrimSpace(*model),
apiKey: strings.TrimSpace(*key),
model: strings.TrimSpace(*model),
categorizeConfidence: *categorizeConfidence,
}
if s.apiKey == "" {
log.Fatal("mock-llm: API key must be non-empty (Descrybe Completer.Enabled requires it)")
@@ -77,6 +83,13 @@ func main() {
}
}
func truncTrace(s string, n int) string {
if len(s) <= n {
return s
}
return s[:n]
}
func envOr(k, def string) string {
if v := strings.TrimSpace(os.Getenv(k)); v != "" {
return v
@@ -140,11 +153,17 @@ func (s *server) handleChatCompletions(w http.ResponseWriter, r *http.Request) {
return
}
system, user := splitMessages(req.Messages)
if os.Getenv("MOCK_LLM_TRACE") != "" {
log.Printf("mock-llm: roles=%d system_len=%d user_len=%d system_head=%q",
len(req.Messages), len(system), len(user), truncTrace(system, 60))
}
var comp processing.Completion
// A category formula in the prompt is answered in that formula's shape, so a
// local run proves the formula reached the model and the reply passes the
// pipeline's formula gate. Everything else keeps the heuristic fallback.
if text, ok := buildFormulaReply(system, user); ok {
if text, ok := s.categorizeReply(system, user); ok {
comp = processing.Completion{Text: text}
} else if text, ok := buildFormulaReply(system, user); ok {
comp = processing.Completion{Text: text}
} else {
var err error
@@ -204,6 +223,39 @@ func (s *server) handleEmbeddings(w http.ResponseWriter, r *http.Request) {
})
}
// categorizeReply overrides the heuristic taxonomy pick so a local run can
// reproduce a low-confidence answer.
func (s *server) categorizeReply(system, user string) (string, bool) {
if s.categorizeConfidence <= 0 || !strings.Contains(strings.ToLower(system), "categoryid") {
return "", false
}
id := firstCategoryIDFromPrompt(user)
if id == "" {
return "", false
}
b, err := json.Marshal(map[string]any{"categoryId": id, "confidence": s.categorizeConfidence})
if err != nil {
return "", false
}
return string(b), true
}
// firstCategoryIDFromPrompt mirrors the "(ID: …)" format of the categorize prompt.
func firstCategoryIDFromPrompt(user string) string {
const marker = "(id:"
lower := strings.ToLower(user)
at := strings.Index(lower, marker)
if at < 0 {
return ""
}
rest := strings.TrimSpace(user[at+len(marker):])
end := strings.Index(rest, ")")
if end <= 0 {
return ""
}
return strings.TrimSpace(rest[:end])
}
func splitMessages(msgs []struct {
Role string `json:"role"`
Content string `json:"content"`
@@ -211,7 +263,10 @@ func splitMessages(msgs []struct {
var users []string
for _, m := range msgs {
switch strings.ToLower(strings.TrimSpace(m.Role)) {
case "system":
// Reasoning-model requests send the system message as "developer"
// (see processing.OpenAIClient sysRole). Treating it as a user turn made the
// mock mis-route every call whose branch depends on the system prompt.
case "system", "developer":
if system == "" {
system = m.Content
} else {
+105 -15
View File
@@ -2,17 +2,36 @@ package processing
import (
"context"
"encoding/json"
"errors"
"fmt"
"log"
"sort"
"strconv"
"strings"
"github.com/descrybe/descrybe-v2/apps/api/internal/aiaudit"
)
// ErrCategoryNotFound fails a product whose category could not be determined:
// the model's pick was outside the taxonomy, or it scored below the confidence
// floor. Processing stops for that product instead of enhancing it under a
// category nobody believes in.
var ErrCategoryNotFound = errors.New("product category could not be determined")
// MaxCategorizeOptions caps the taxonomy list injected into the categorize LLM
// prompt (token/cost bound). Prefer stable sort by display name then unique_id.
const MaxCategorizeOptions = 200
// DefaultMinCategorizeConfidence is the floor for accepting an LLM category pick.
//
// The model always answers with SOME id from the list, so a low-confidence reply
// like {"categoryId":"1","confidence":0.08} is the model saying "I do not know" —
// taking it at face value files a camping chair under Generators and then drives
// that category's title/description formula, producing confidently wrong copy.
// Below this floor the product fails with ErrCategoryNotFound instead.
const DefaultMinCategorizeConfidence = 0.75
// MaxTokensCategorize budgets a short JSON reply {"categoryId","confidence"}.
// code-fast / reasoning-style models often burn internal tokens before JSON;
// 256 frequently finishes with empty content and forces a length-cap retry.
@@ -116,15 +135,15 @@ func ProductCategorizeUser(name, description string, opts []categoryOption) stri
// tryAICategorize asks the Completer to pick a company taxonomy unique_id when
// Category is still empty after mapped/prior/vector. Never invents outside taxonomy:
// responses are coerced then filtered to namesByUID / valid unique_ids.
func tryAICategorize(ctx context.Context, e *Engine, out *StepResult, in ProductInput, policy StepPolicy) {
func tryAICategorize(ctx context.Context, e *Engine, out *StepResult, in ProductInput, policy StepPolicy) error {
if out == nil || strings.TrimSpace(out.Category) != "" {
return
return nil
}
if !policy.AllowAI {
return
return nil
}
if e == nil || !e.CompleterEnabled() {
return
return nil
}
opts := categoryOptionsFromNames(in.CategoryNamesByUID)
if len(opts) == 0 {
@@ -133,7 +152,7 @@ func tryAICategorize(ctx context.Context, e *Engine, out *StepResult, in Product
"status": "skipped",
"reason": "no_taxonomy",
})
return
return nil
}
name := strings.TrimSpace(out.Name)
if name == "" {
@@ -149,7 +168,7 @@ func tryAICategorize(ctx context.Context, e *Engine, out *StepResult, in Product
"status": "skipped",
"reason": "empty_product",
})
return
return nil
}
system := ProductCategorizeSystem
@@ -166,23 +185,44 @@ func tryAICategorize(ctx context.Context, e *Engine, out *StepResult, in Product
"error": TruncateError(err),
"tokens": comp.TotalTokens,
})
return
return nil
}
raw := categoryIDFromCategorizeJSON(obj)
confidence, hasConfidence := categorizeConfidenceFromJSON(obj)
valid := make(map[string]struct{}, len(opts))
for _, o := range opts {
valid[o.UniqueID] = struct{}{}
}
resolved := resolveCompanyCategoryUniqueID(raw, in.CategoryNamesByUID, valid)
if resolved == "" || isUnusableCategoryValue(resolved, out.ProcessedName, out.Name) {
out.Notes = append(out.Notes, "ai_categorize: ignored (not in company taxonomy)")
out.Notes = append(out.Notes, "ai_categorize: no category found (not in company taxonomy)")
appendStepLog(out.GPTResponse, StepCategorize, map[string]any{
"status": "rejected",
"returned": raw,
"tokens": comp.TotalTokens,
})
return
return fmt.Errorf("%w: model returned %q, which is not in the company taxonomy",
ErrCategoryNotFound, truncateRunes(raw, 60))
}
// A confident-looking id with a low score is the model guessing. Refuse it
// rather than driving the wrong category's formula with it.
if min := policy.CategorizeConfidence(); hasConfidence && confidence < min {
out.Notes = append(out.Notes, fmt.Sprintf(
"ai_categorize: no category found (confidence %.2f below %.2f)", confidence, min))
appendStepLog(out.GPTResponse, StepCategorize, map[string]any{
"status": "low_confidence",
"returned": raw,
"resolved": resolved,
"confidence": confidence,
"minimum": min,
"tokens": comp.TotalTokens,
})
log.Printf("processing: category rejected uid=%s confidence=%.2f min=%.2f reason=low_confidence",
resolved, confidence, min)
return fmt.Errorf("%w: best match %q scored %.2f, below the %.2f minimum",
ErrCategoryNotFound, truncateRunes(resolved, 60), confidence, min)
}
out.Category = SanitizeText(resolved)
@@ -201,6 +241,50 @@ func tryAICategorize(ctx context.Context, e *Engine, out *StepResult, in Product
"tokens": comp.TotalTokens,
"options": len(opts),
})
if hasConfidence {
out.Notes = append(out.Notes, fmt.Sprintf("ai_categorize: confidence %.2f", confidence))
}
return nil
}
// categorizeConfidenceFromJSON reads the model's self-reported score. ok=false when
// the field is missing or unparseable — a model that reports nothing is not
// penalised, only one that reports a low score.
func categorizeConfidenceFromJSON(obj map[string]any) (float64, bool) {
if obj == nil {
return 0, false
}
for _, k := range []string{"confidence", "score", "certainty"} {
switch v := obj[k].(type) {
case float64:
return clampConfidence(v), true
case int:
return clampConfidence(float64(v)), true
case json.Number:
if f, err := v.Float64(); err == nil {
return clampConfidence(f), true
}
case string:
if f, err := strconv.ParseFloat(strings.TrimSpace(v), 64); err == nil {
return clampConfidence(f), true
}
}
}
return 0, false
}
// clampConfidence folds a 0-100 style score onto 0-1 and bounds it.
func clampConfidence(v float64) float64 {
if v > 1 && v <= 100 {
v /= 100
}
if v < 0 {
return 0
}
if v > 1 {
return 1
}
return v
}
func categoryIDFromCategorizeJSON(obj map[string]any) string {
@@ -233,9 +317,12 @@ func firstAvailableCategoryIDFromPrompt(user string) string {
}
// runCategorizeStep applies vector then LLM taxonomy selection when Category empty.
func runCategorizeStep(ctx context.Context, e *Engine, companyID string, out *StepResult, in ProductInput, categoryNames []string, policy StepPolicy) {
// runCategorizeStep returns ErrCategoryNotFound when the model was asked for a
// category and could not give a usable one — the caller fails the product rather
// than enhancing it under a guess.
func runCategorizeStep(ctx context.Context, e *Engine, companyID string, out *StepResult, in ProductInput, categoryNames []string, policy StepPolicy) error {
if out == nil {
return
return nil
}
if strings.TrimSpace(out.Category) != "" {
appendStepLog(out.GPTResponse, StepCategorize, map[string]any{
@@ -243,7 +330,7 @@ func runCategorizeStep(ctx context.Context, e *Engine, companyID string, out *St
"reason": "already_set",
"category": out.Category,
})
return
return nil
}
if !policy.AllowAI {
out.Notes = append(out.Notes, "categorize: skipped (AI not allowed)")
@@ -251,21 +338,24 @@ func runCategorizeStep(ctx context.Context, e *Engine, companyID string, out *St
"status": "skipped",
"reason": "entitlement_can_use_ai",
})
return
return nil
}
tryVectorCategorize(ctx, e, companyID, out, categoryNames, policy)
if strings.TrimSpace(out.Category) != "" {
return
return nil
}
// Taxonomy picks and copy generation share one Completer; tag the role so the
// admin inspector can tell them apart.
tryAICategorize(aiaudit.WithCall(ctx, aiaudit.CallContext{Role: aiaudit.RoleCategorize}), e, out, in, policy)
if err := tryAICategorize(aiaudit.WithCall(ctx, aiaudit.CallContext{Role: aiaudit.RoleCategorize}), e, out, in, policy); err != nil {
return err
}
if strings.TrimSpace(out.Category) == "" {
appendStepLog(out.GPTResponse, StepCategorize, map[string]any{
"status": "unset",
"reason": "no_vector_or_ai_match",
})
}
return nil
}
// persistMappedCategorySQL writes a taxonomy unique_id onto mapped_data.category
@@ -2,6 +2,7 @@ package processing
import (
"context"
"errors"
"strings"
"testing"
)
@@ -62,28 +63,24 @@ func TestTryAICategorize_rejectsInventedID(t *testing.T) {
}},
Vector: NoopVectorCategorizer{},
}
// An id outside the taxonomy means the category could not be determined. The
// product must fail rather than continue uncategorised into enhance, which
// would generate copy with no formula behind it.
out, err := e.RunSteps(context.Background(), "co", ProductInput{
Name: "Widget",
CategoryNamesByUID: map[string]string{
"50": "Štedilniki",
},
}, "full", nil, StepPolicy{AllowAI: true})
if err != nil {
t.Fatal(err)
if !errors.Is(err, ErrCategoryNotFound) {
t.Fatalf("err=%v want ErrCategoryNotFound", err)
}
if !strings.Contains(err.Error(), "99999") {
t.Fatalf("error should name the rejected id, got %v", err)
}
if out.Category != "" {
t.Fatalf("Category=%q want empty (invented id rejected)", out.Category)
}
found := false
for _, n := range out.Notes {
if strings.Contains(n, "ai_categorize: ignored") {
found = true
break
}
}
if !found {
t.Fatalf("expected reject note, got %v", out.Notes)
}
}
func TestTryAICategorize_resolvesDisplayName(t *testing.T) {
@@ -0,0 +1,176 @@
package processing
import (
"context"
"errors"
"fmt"
"strings"
"testing"
)
func categorizeEngine(reply string) (*Engine, *int) {
enhanceCalls := 0
e := &Engine{
Completer: stubCompleter{fn: func(system, user string) (Completion, error) {
if strings.Contains(strings.ToLower(system), "categoryid") {
return Completion{Text: reply, TotalTokens: 3}, nil
}
enhanceCalls++
return Completion{Text: `{"name":"Enhanced","description":"Body"}`, TotalTokens: 5}, nil
}},
Vector: NoopVectorCategorizer{},
}
return e, &enhanceCalls
}
func categorizeInput() ProductInput {
return ProductInput{
Name: "BRUNNER folding camping chair ONE SHOT",
Description: "A light folding aluminium chair for outdoor use.",
Mapped: map[string]any{
"name": "BRUNNER folding camping chair ONE SHOT",
"description": "A light folding aluminium chair for outdoor use.",
},
CategoryNamesByUID: map[string]string{"1": "Generators", "34": "Smartwatches"},
}
}
// The reported case: the model always answers with SOME id, so a low score is it
// saying "I do not know". Taking it at face value files a camping chair under
// Generators and then drives that category's formula.
func TestRunSteps_lowConfidenceCategoryFailsTheProduct(t *testing.T) {
t.Parallel()
e, enhanceCalls := categorizeEngine(`{"categoryId":"1","confidence":0.08}`)
out, err := e.RunSteps(context.Background(), "co", categorizeInput(), "full", nil,
StepPolicy{AllowAI: true})
if !errors.Is(err, ErrCategoryNotFound) {
t.Fatalf("err=%v want ErrCategoryNotFound", err)
}
for _, want := range []string{"0.08", "0.75"} {
if !strings.Contains(err.Error(), want) {
t.Fatalf("error should report the score and the minimum, got %v", err)
}
}
if out.Category != "" {
t.Fatalf("a rejected pick must not be persisted, got %q", out.Category)
}
// The whole point: stop before spending an enhance call on the wrong category.
if *enhanceCalls != 0 {
t.Fatalf("enhance ran %d time(s) after the category was rejected", *enhanceCalls)
}
}
func TestRunSteps_confidentCategoryProceeds(t *testing.T) {
t.Parallel()
e, enhanceCalls := categorizeEngine(`{"categoryId":"1","confidence":0.92}`)
out, err := e.RunSteps(context.Background(), "co", categorizeInput(), "full", nil,
StepPolicy{AllowAI: true})
if err != nil {
t.Fatal(err)
}
if out.Category != "1" {
t.Fatalf("Category=%q want 1", out.Category)
}
if *enhanceCalls == 0 {
t.Fatal("enhance should run for a confident category")
}
}
// Exactly at the floor is acceptable — the threshold is a minimum, not a bar to clear.
func TestRunSteps_confidenceAtThresholdIsAccepted(t *testing.T) {
t.Parallel()
e, _ := categorizeEngine(fmt.Sprintf(`{"categoryId":"34","confidence":%v}`, DefaultMinCategorizeConfidence))
out, err := e.RunSteps(context.Background(), "co", categorizeInput(), "full", nil,
StepPolicy{AllowAI: true})
if err != nil {
t.Fatalf("confidence == minimum must pass, got %v", err)
}
if out.Category != "34" {
t.Fatalf("Category=%q want 34", out.Category)
}
}
// A model that reports no score is not penalised — only one that reports a low one.
func TestRunSteps_missingConfidenceIsAccepted(t *testing.T) {
t.Parallel()
e, _ := categorizeEngine(`{"categoryId":"34"}`)
out, err := e.RunSteps(context.Background(), "co", categorizeInput(), "full", nil,
StepPolicy{AllowAI: true})
if err != nil {
t.Fatalf("missing confidence must not fail the product, got %v", err)
}
if out.Category != "34" {
t.Fatalf("Category=%q want 34", out.Category)
}
}
func TestStepPolicy_confidenceOverride(t *testing.T) {
t.Parallel()
if got := (StepPolicy{}).CategorizeConfidence(); got != DefaultMinCategorizeConfidence {
t.Fatalf("default = %v want %v", got, DefaultMinCategorizeConfidence)
}
if got := (StepPolicy{MinCategorizeConfidence: 0.4}).CategorizeConfidence(); got != 0.4 {
t.Fatalf("override = %v want 0.4", got)
}
// A tenant that lowers the bar accepts what the default would reject.
e, _ := categorizeEngine(`{"categoryId":"1","confidence":0.5}`)
if _, err := e.RunSteps(context.Background(), "co", categorizeInput(), "full", nil,
StepPolicy{AllowAI: true, MinCategorizeConfidence: 0.4}); err != nil {
t.Fatalf("0.5 should pass a 0.4 floor, got %v", err)
}
}
// A category already resolved from the feed is never second-guessed: categorize
// does not run, so its confidence cannot fail the product.
func TestRunSteps_mappedCategorySkipsConfidenceGate(t *testing.T) {
t.Parallel()
e, _ := categorizeEngine(`{"categoryId":"1","confidence":0.01}`)
in := categorizeInput()
in.Mapped["category"] = "34"
out, err := e.RunSteps(context.Background(), "co", in, "full", nil, StepPolicy{AllowAI: true})
if err != nil {
t.Fatalf("a feed-supplied category must not be gated, got %v", err)
}
if out.Category != "34" {
t.Fatalf("Category=%q want 34", out.Category)
}
}
// Free plan / AI disabled must keep working — nothing was asked of a model, so
// there is no failed categorisation to report.
func TestRunSteps_noAIDoesNotFailOnMissingCategory(t *testing.T) {
t.Parallel()
e, _ := categorizeEngine(`{"categoryId":"1","confidence":0.01}`)
if _, err := e.RunSteps(context.Background(), "co", categorizeInput(), "full", nil,
StepPolicy{AllowAI: false}); err != nil {
t.Fatalf("AI-disabled processing must not fail on category, got %v", err)
}
}
func TestCategorizeConfidenceFromJSON(t *testing.T) {
t.Parallel()
cases := []struct {
obj map[string]any
want float64
ok bool
}{
{map[string]any{"confidence": 0.08}, 0.08, true},
{map[string]any{"confidence": "0.42"}, 0.42, true},
{map[string]any{"score": 0.9}, 0.9, true},
// Some models answer on a 0-100 scale.
{map[string]any{"confidence": 85.0}, 0.85, true},
{map[string]any{"confidence": 1}, 1, true},
{map[string]any{"categoryId": "1"}, 0, false},
{map[string]any{"confidence": "high"}, 0, false},
{nil, 0, false},
}
for _, c := range cases {
got, ok := categorizeConfidenceFromJSON(c.obj)
if ok != c.ok || (ok && got != c.want) {
t.Fatalf("%v → (%v, %v) want (%v, %v)", c.obj, got, ok, c.want, c.ok)
}
}
}
+20 -2
View File
@@ -19,6 +19,18 @@ import (
type StepPolicy struct {
AllowAI bool
AllowEPREL bool
// MinCategorizeConfidence overrides DefaultMinCategorizeConfidence. A categorize
// reply below it is treated as "no category found" and fails the product rather
// than letting a wrong guess drive the formula. 0 uses the default.
MinCategorizeConfidence float64
}
// CategorizeConfidence returns the effective low-confidence floor for this policy.
func (p StepPolicy) CategorizeConfidence() float64 {
if p.MinCategorizeConfidence > 0 {
return p.MinCategorizeConfidence
}
return DefaultMinCategorizeConfidence
}
// RunSteps executes the multi-step product pipeline.
@@ -213,7 +225,11 @@ func (e *Engine) RunSteps(ctx context.Context, companyID string, in ProductInput
case StepCategorize:
// Vector (if enabled) then LLM taxonomy pick when category still empty.
runCategorizeStep(ctx, e, companyID, &out, in, categoryNames, policy)
// A category the model could not determine fails the product here, before
// enhance spends a call writing copy for the wrong category.
if err := runCategorizeStep(ctx, e, companyID, &out, in, categoryNames, policy); err != nil {
return out, err
}
case StepAIEnhance:
preservePriorEnhanceHash := func() {
@@ -533,7 +549,9 @@ func (e *Engine) RunSteps(ctx context.Context, companyID string, in ProductInput
// Pipelines without StepCategorize (normalize_only) still run vector + LLM
// categorize when category is empty so downstream steps are not fed a blank.
if !stepsContain(steps, StepCategorize) {
runCategorizeStep(ctx, e, companyID, &out, in, categoryNames, policy)
if err := runCategorizeStep(ctx, e, companyID, &out, in, categoryNames, policy); err != nil {
return out, err
}
}
// Record why category stayed empty for every pipeline, including the ones that
// now categorize inside the loop (enhance / title / description).