Files
descrybe/apps/api/internal/processing/pipeline_llm_mock_test.go
T

637 lines
20 KiB
Go
Raw Normal View History

package processing
import (
"context"
"encoding/json"
"errors"
"fmt"
"net/http"
"net/http/httptest"
"os"
"strings"
"sync/atomic"
"testing"
"time"
"github.com/google/uuid"
"github.com/jackc/pgx/v5/pgxpool"
)
// mockChatCompletionsServer is an OpenAI-compatible stand-in for the small/test LLM.
func mockChatCompletionsServer(t *testing.T, handler http.HandlerFunc) *httptest.Server {
t.Helper()
mux := http.NewServeMux()
mux.HandleFunc("/v1/chat/completions", handler)
srv := httptest.NewServer(mux)
t.Cleanup(srv.Close)
return srv
}
func openAIClientForMock(t *testing.T, baseURL string, maxRetries int) *OpenAIClient {
t.Helper()
t.Setenv("APP_ENV", "local")
// NewOpenAIClient coerces maxRetries<=0 to 3; set the field after for single-shot tests.
c := NewOpenAIClient("sk-test-pipeline-llm", strings.TrimRight(baseURL, "/")+"/v1", "test-small-model", 0, 1)
if maxRetries < 0 {
maxRetries = 0
}
c.MaxRetries = maxRetries
if !c.Enabled() {
t.Fatal("expected mock OpenAI client enabled")
}
return c
}
func TestRunSteps_enhanceMockHappyPath(t *testing.T) {
t.Parallel()
var gotSystem string
e := &Engine{
Completer: stubCompleter{fn: func(system, _ string) (Completion, error) {
gotSystem = system
return Completion{
2026-08-23 19:30:05 +02:00
Text: `{"name":"Mock Shoe","description":"Mock Shoe is a light runner built for daily miles, with cushioned foam midsole and a grippy outsole that stays stable on wet pavement."}`,
TotalTokens: 11,
Model: "mock",
}, nil
}},
Vector: NoopVectorCategorizer{},
}
out, err := e.RunSteps(context.Background(), "co-llm-mock", ProductInput{
GTIN: "8712345678901",
Name: "Shoe", Description: "runner",
Mapped: map[string]any{"name": "Shoe", "description": "runner", "brand": "Acme"},
Language: "de",
}, "enhance_only", nil, StepPolicy{AllowAI: true})
if err != nil {
t.Fatal(err)
}
if out.ProcessedName != "Mock Shoe" {
t.Fatalf("ProcessedName=%q", out.ProcessedName)
}
2026-08-23 19:30:05 +02:00
if out.Name != "Shoe" {
t.Fatalf("Original Name must stay supplier, got %q", out.Name)
}
if out.Description != "runner" {
t.Fatalf("Original Description must stay supplier, got %q", out.Description)
}
if out.ProcessedDescription != "Mock Shoe is a light runner built for daily miles, with cushioned foam midsole and a grippy outsole that stays stable on wet pavement." {
t.Fatalf("ProcessedDescription=%q", out.ProcessedDescription)
}
if out.TotalTokens != 11 {
t.Fatalf("TotalTokens=%d", out.TotalTokens)
}
if !strings.Contains(gotSystem, "German") {
t.Fatalf("expected {{language}}→German in system prompt, got %q", gotSystem)
}
prog := progressFromResult("enhance_only", &out)
if len(prog) < 2 || prog[len(prog)-1].Status != "done" {
t.Fatalf("step progress=%v", prog)
}
}
func TestRunSteps_enhanceMockProviderFailure(t *testing.T) {
t.Parallel()
var gotSystem string
e := &Engine{
Completer: stubCompleter{fn: func(system, _ string) (Completion, error) {
gotSystem = system
return Completion{}, errors.New("upstream 503: model overloaded")
}},
}
out, err := e.RunSteps(context.Background(), "co-llm-mock", ProductInput{
Mapped: map[string]any{"name": "Widget", "description": "plain"},
Language: "fr",
}, "enhance_only", nil, StepPolicy{AllowAI: true})
if err != nil {
t.Fatalf("RunSteps should not fail the call on AI error: %v", err)
}
if out.ProcessedName != "Widget" {
t.Fatalf("expected passthrough name, got %q", out.ProcessedName)
}
if !strings.Contains(gotSystem, "French") {
t.Fatalf("failure path must still inject language before error: %q", gotSystem)
}
joined := strings.Join(out.Notes, ";")
if !strings.Contains(joined, "ai_enhance:") || !strings.Contains(joined, "503") {
t.Fatalf("expected failure note, got %v", out.Notes)
}
prog := progressFromResult("enhance_only", &out)
foundFailed := false
for _, s := range prog {
if s.Step == StepAIEnhance && s.Status == "failed" {
foundFailed = true
if !strings.Contains(s.Note, "503") {
t.Fatalf("failed note=%q", s.Note)
}
}
}
if !foundFailed {
t.Fatalf("expected ai_enhance failed in progress=%v", prog)
}
2026-08-16 16:57:36 +02:00
if strings.TrimSpace(out.ProcessedDescription) == "" {
t.Fatal("provider failure must leave nonempty ProcessedDescription when title exists")
}
if descriptionEchoesTitle(out.ProcessedDescription, out.ProcessedName) {
t.Fatalf("failure synth must not echo title: name=%q desc=%q", out.ProcessedName, out.ProcessedDescription)
}
}
func TestRunSteps_enhanceMockTimeout(t *testing.T) {
t.Parallel()
e := &Engine{
Completer: stubCompleter{fn: func(_, _ string) (Completion, error) {
return Completion{}, context.DeadlineExceeded
}},
}
out, err := e.RunSteps(context.Background(), "co-llm-mock", ProductInput{
Mapped: map[string]any{"name": "Timeout Widget", "description": "slow"},
Language: "nl",
}, "enhance_only", nil, StepPolicy{AllowAI: true})
if err != nil {
t.Fatalf("RunSteps should swallow provider timeout: %v", err)
}
joined := strings.Join(out.Notes, ";")
lowerJoined := strings.ToLower(joined)
if !strings.Contains(lowerJoined, "timed out") && !strings.Contains(lowerJoined, "deadline") {
t.Fatalf("expected timeout note, got %v", out.Notes)
}
prog := progressFromResult("enhance_only", &out)
ok := false
for _, s := range prog {
if s.Step == StepAIEnhance && s.Status == "failed" {
ok = true
}
}
if !ok {
t.Fatalf("expected ai_enhance failed, progress=%v", prog)
}
2026-08-16 16:57:36 +02:00
if strings.TrimSpace(out.ProcessedDescription) == "" {
t.Fatal("timeout must leave nonempty ProcessedDescription when title exists")
}
if descriptionEchoesTitle(out.ProcessedDescription, out.ProcessedName) {
t.Fatalf("timeout synth must not echo title: name=%q desc=%q", out.ProcessedName, out.ProcessedDescription)
}
}
func TestRunSteps_enhanceTimeoutSynthesizesDescriptionAndInternalMode(t *testing.T) {
t.Parallel()
e := &Engine{
Completer: stubCompleter{fn: func(_, _ string) (Completion, error) {
return Completion{}, context.DeadlineExceeded
}},
ProviderMode: AIProviderInternal,
Vector: NoopVectorCategorizer{},
}
out, err := e.RunSteps(context.Background(), "co-timeout-synth", ProductInput{
Mapped: map[string]any{
"name": "VOX EBR700",
"description": "",
"brand": "VOX",
"category": "50",
},
Language: "sl",
CategoryNamesByUID: map[string]string{
"50": "Štedilniki",
},
}, "enhance_only", nil, StepPolicy{AllowAI: true})
if err != nil {
t.Fatalf("RunSteps should swallow timeout: %v", err)
}
if strings.TrimSpace(out.ProcessedName) == "" {
t.Fatal("expected nonempty ProcessedName")
}
if strings.TrimSpace(out.ProcessedDescription) == "" {
t.Fatal("timeout must synthesize nonempty ProcessedDescription when title exists")
}
if descriptionEchoesTitle(out.ProcessedDescription, out.ProcessedName) {
t.Fatalf("synth must not echo title: name=%q desc=%q", out.ProcessedName, out.ProcessedDescription)
}
if out.AIProviderMode != AIProviderInternal {
t.Fatalf("ai_provider_mode=%q want %q (attempted enhance, not unknown)", out.AIProviderMode, AIProviderInternal)
}
if _, ok := out.FieldSources[FieldEnhanceInputHash]; ok {
t.Fatalf("timeout must clear enhance hash, got %v", out.FieldSources[FieldEnhanceInputHash])
}
}
func TestOpenAIClient_Complete_httptestHappyPath(t *testing.T) {
srv := mockChatCompletionsServer(t, func(w http.ResponseWriter, r *http.Request) {
if r.Method != http.MethodPost {
t.Fatalf("method=%s", r.Method)
}
auth := r.Header.Get("Authorization")
if !strings.HasPrefix(auth, "Bearer sk-test-") {
http.Error(w, "unauthorized", http.StatusUnauthorized)
return
}
_ = json.NewEncoder(w).Encode(map[string]any{
"model": "test-small-model",
"choices": []map[string]any{
{"message": map[string]any{"content": `{"name":"HTTP Shoe","description":"From mock LLM."}`}},
},
"usage": map[string]any{
"prompt_tokens": 3, "completion_tokens": 5, "total_tokens": 8,
},
})
})
c := openAIClientForMock(t, srv.URL, 0)
comp, err := c.Complete(context.Background(), "sys", "user")
if err != nil {
t.Fatal(err)
}
if !strings.Contains(comp.Text, "HTTP Shoe") {
t.Fatalf("text=%q", comp.Text)
}
if comp.TotalTokens != 8 {
t.Fatalf("tokens=%d", comp.TotalTokens)
}
}
func TestOpenAIClient_Complete_httptestTimeout(t *testing.T) {
srv := mockChatCompletionsServer(t, func(w http.ResponseWriter, r *http.Request) {
deadline := time.After(300 * time.Millisecond)
for {
select {
case <-r.Context().Done():
return
case <-deadline:
_ = json.NewEncoder(w).Encode(map[string]any{
"choices": []map[string]any{
{"message": map[string]any{"content": "late"}},
},
})
return
case <-time.After(10 * time.Millisecond):
}
}
})
c := openAIClientForMock(t, srv.URL, 0)
c.HTTPClient.Timeout = 60 * time.Millisecond
start := time.Now()
_, err := c.Complete(context.Background(), "sys", "user")
elapsed := time.Since(start)
if err == nil {
t.Fatal("expected timeout error")
}
if elapsed > time.Second {
t.Fatalf("timeout too slow: %s err=%v", elapsed, err)
}
}
func TestRunSteps_enhanceViaHTTPLLMMock(t *testing.T) {
var calls atomic.Int32
srv := mockChatCompletionsServer(t, func(w http.ResponseWriter, r *http.Request) {
calls.Add(1)
_ = json.NewEncoder(w).Encode(map[string]any{
"model": "test-small-model",
"choices": []map[string]any{
{"message": map[string]any{"content": `{"name":"Pipeline Mock","description":"Happy path via httptest."}`}},
},
"usage": map[string]any{"total_tokens": 9},
})
})
e := &Engine{
Completer: openAIClientForMock(t, srv.URL, 0),
Vector: NoopVectorCategorizer{},
}
out, err := e.RunSteps(context.Background(), "co-llm-http", ProductInput{
Mapped: map[string]any{"name": "Raw", "description": "desc"},
Language: "it",
}, "enhance_only", nil, StepPolicy{AllowAI: true})
if err != nil {
t.Fatal(err)
}
if calls.Load() < 1 {
t.Fatal("expected chat completions call")
}
if out.ProcessedName != "Pipeline Mock" {
t.Fatalf("name=%q", out.ProcessedName)
}
if out.TotalTokens < 1 {
t.Fatalf("tokens=%d", out.TotalTokens)
}
}
func TestRunSteps_enhanceViaHTTPLLMTimeout(t *testing.T) {
srv := mockChatCompletionsServer(t, func(w http.ResponseWriter, r *http.Request) {
deadline := time.After(300 * time.Millisecond)
for {
select {
case <-r.Context().Done():
return
case <-deadline:
_ = json.NewEncoder(w).Encode(map[string]any{
"choices": []map[string]any{
{"message": map[string]any{"content": `{"name":"Late","description":"x"}`}},
},
})
return
case <-time.After(10 * time.Millisecond):
}
}
})
client := openAIClientForMock(t, srv.URL, 0)
client.HTTPClient.Timeout = 60 * time.Millisecond
e := &Engine{Completer: client}
out, err := e.RunSteps(context.Background(), "co-llm-http", ProductInput{
Mapped: map[string]any{"name": "Raw", "description": "desc"},
}, "enhance_only", nil, StepPolicy{AllowAI: true})
if err != nil {
t.Fatalf("RunSteps err=%v", err)
}
joined := strings.Join(out.Notes, ";")
if !strings.Contains(joined, "ai_enhance:") {
t.Fatalf("expected ai_enhance note, got %v", out.Notes)
}
prog := progressFromResult("enhance_only", &out)
failed := false
for _, s := range prog {
if s.Step == StepAIEnhance && s.Status == "failed" {
failed = true
}
}
if !failed {
t.Fatalf("expected failed ai_enhance, progress=%v notes=%v", prog, out.Notes)
}
}
func TestRunSteps_enhanceViaHTTPLLMServerError(t *testing.T) {
srv := mockChatCompletionsServer(t, func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusBadGateway)
_ = json.NewEncoder(w).Encode(map[string]any{
"error": map[string]any{"message": "green-chat unavailable"},
})
})
e := &Engine{Completer: openAIClientForMock(t, srv.URL, 0)}
out, err := e.RunSteps(context.Background(), "co-llm-http", ProductInput{
Mapped: map[string]any{"name": "Raw", "description": "desc"},
}, "enhance_only", nil, StepPolicy{AllowAI: true})
if err != nil {
t.Fatal(err)
}
joined := strings.Join(out.Notes, ";")
if !strings.Contains(joined, "ai_enhance:") {
t.Fatalf("notes=%v", out.Notes)
}
if out.ProcessedName != "Raw" {
t.Fatalf("expected original name on failure, got %q", out.ProcessedName)
}
}
func pipelineLLMMockFixtures(t *testing.T, pg *pgxpool.Pool, ctx context.Context, language string) (companyID, userID, rawID uuid.UUID, cleanup func()) {
t.Helper()
if language == "" {
language = "de"
}
// Prefer demo sandbox users only — never a1-primary / A1 cohort emails.
// Exact emails only (no LIKE '%a1%' — false positives and random-user fallback).
err := pg.QueryRow(ctx, `
SELECT id FROM users
WHERE LOWER(COALESCE(email, '')) = 'demo@descrybe.local'
LIMIT 1`).Scan(&userID)
if err != nil {
err = pg.QueryRow(ctx, `
SELECT id FROM users
WHERE LOWER(COALESCE(email, '')) = 'demo@descrybe.test'
LIMIT 1`).Scan(&userID)
if err != nil {
t.Skip("need demo@descrybe.local or demo@descrybe.test (run seed-demo); refusing random/a1 users")
}
}
companyID = uuid.New()
rawID = uuid.New()
if _, err := pg.Exec(ctx, `
INSERT INTO companies (id, name, language) VALUES ($1, $2, $3)`,
companyID, "pipeline-llm-mock-co", language); err != nil {
t.Fatal(err)
}
gtin := fmt.Sprintf("llm-mock-%s", companyID.String()[:8])
if _, err := pg.Exec(ctx, `
INSERT INTO raw_products (id, company_id, gtin, raw_data, mapped_data, is_processed, processing_status)
VALUES ($1, $2, $3, '{}'::jsonb, '{"name":"Raw Widget","description":"original desc"}'::jsonb, false, 'unprocessed')`,
rawID, companyID, gtin); err != nil {
t.Fatal(err)
}
cleanup = func() {
_, _ = pg.Exec(context.Background(), `DELETE FROM processed_products WHERE company_id = $1`, companyID)
_, _ = pg.Exec(context.Background(), `DELETE FROM processing_job_products WHERE job_id IN (SELECT id FROM processing_jobs WHERE company_id = $1)`, companyID)
_, _ = pg.Exec(context.Background(), `DELETE FROM processing_jobs WHERE company_id = $1`, companyID)
_, _ = pg.Exec(context.Background(), `DELETE FROM raw_products WHERE company_id = $1`, companyID)
_, _ = pg.Exec(context.Background(), `DELETE FROM companies WHERE id = $1`, companyID)
}
return companyID, userID, rawID, cleanup
}
// TestProcessJob_mockLLMHappyPath runs ProcessJob with a stub Completer (no live LLM, no a1 tenant).
// Asserts companies.language is loaded and injected into the enhance system prompt.
func TestProcessJob_mockLLMHappyPath(t *testing.T) {
dsn := os.Getenv("DATABASE_URL")
if dsn == "" {
t.Skip("DATABASE_URL not set")
}
ctx, cancel := context.WithTimeout(context.Background(), 45*time.Second)
defer cancel()
pg, err := pgxpool.New(ctx, dsn)
if err != nil {
t.Fatal(err)
}
defer pg.Close()
companyID, userID, rawID, cleanup := pipelineLLMMockFixtures(t, pg, ctx, "de")
defer cleanup()
var gotSystem string
p := NewPipeline(pg)
p.Billing = nil
p.Limiter = nil
p.AI = nil
p.Engine = &Engine{
Completer: stubCompleter{fn: func(system, _ string) (Completion, error) {
gotSystem = system
return Completion{
Text: `{"name":"Happy Mock","description":"Processed by mock LLM."}`,
TotalTokens: 6,
Model: "mock",
}, nil
}},
Vector: NoopVectorCategorizer{},
}
jobs, err := p.StartJob(ctx, companyID, userID, []uuid.UUID{rawID}, "enhance_only")
if err != nil {
t.Fatal(err)
}
if err := p.ProcessJob(ctx, jobs[0].ID); err != nil {
t.Fatal(err)
}
job, err := p.GetJob(ctx, companyID, jobs[0].ID)
if err != nil {
t.Fatal(err)
}
if job.Status != "completed" || job.ProcessedProducts != 1 {
t.Fatalf("status=%s processed=%d", job.Status, job.ProcessedProducts)
}
if !strings.Contains(gotSystem, "German") {
t.Fatalf("ProcessJob must inject companies.language into enhance prompt, got %q", gotSystem)
}
var processedName string
if err := pg.QueryRow(ctx, `
SELECT processed_name FROM processed_products
WHERE company_id = $1 AND raw_product_id = $2`, companyID, rawID).Scan(&processedName); err != nil {
t.Fatal(err)
}
if processedName != "Happy Mock" {
t.Fatalf("processed_name=%q", processedName)
}
foundDone := false
for _, s := range job.StepProgress {
if s.Step == StepAIEnhance && s.Status == "done" {
foundDone = true
}
}
if !foundDone {
t.Fatalf("expected ai_enhance done, progress=%v", job.StepProgress)
}
// Second ProcessJob on a completed job must be a no-op (idempotent / safe with test LLM).
if err := p.ProcessJob(ctx, jobs[0].ID); err != nil {
t.Fatal(err)
}
job2, err := p.GetJob(ctx, companyID, jobs[0].ID)
if err != nil {
t.Fatal(err)
}
if job2.Status != "completed" || job2.ProcessedProducts != 1 {
t.Fatalf("re-run mutated job status=%s processed=%d", job2.Status, job2.ProcessedProducts)
}
}
// TestProcessJob_mockLLMTimeoutSurfacesFailedStep: provider timeout keeps the item
// deliverable (passthrough title) but marks ai_enhance failed in step_progress.
// Language is still loaded from the company before the provider error.
func TestProcessJob_mockLLMTimeoutSurfacesFailedStep(t *testing.T) {
dsn := os.Getenv("DATABASE_URL")
if dsn == "" {
t.Skip("DATABASE_URL not set")
}
ctx, cancel := context.WithTimeout(context.Background(), 45*time.Second)
defer cancel()
pg, err := pgxpool.New(ctx, dsn)
if err != nil {
t.Fatal(err)
}
defer pg.Close()
companyID, userID, rawID, cleanup := pipelineLLMMockFixtures(t, pg, ctx, "fr")
defer cleanup()
var gotSystem string
p := NewPipeline(pg)
p.Billing = nil
p.Limiter = nil
p.AI = nil
p.Engine = &Engine{
Completer: stubCompleter{fn: func(system, _ string) (Completion, error) {
gotSystem = system
return Completion{}, context.DeadlineExceeded
}},
Vector: NoopVectorCategorizer{},
}
jobs, err := p.StartJob(ctx, companyID, userID, []uuid.UUID{rawID}, "enhance_only")
if err != nil {
t.Fatal(err)
}
if err := p.ProcessJob(ctx, jobs[0].ID); err != nil {
t.Fatal(err)
}
job, err := p.GetJob(ctx, companyID, jobs[0].ID)
if err != nil {
t.Fatal(err)
}
if job.Status != "completed" {
t.Fatalf("status=%s (AI timeout is non-fatal for the job item)", job.Status)
}
if !strings.Contains(gotSystem, "French") {
t.Fatalf("timeout path must still inject language before error: %q", gotSystem)
}
var processedName string
if err := pg.QueryRow(ctx, `
SELECT processed_name FROM processed_products
WHERE company_id = $1 AND raw_product_id = $2`, companyID, rawID).Scan(&processedName); err != nil {
t.Fatal(err)
}
if processedName != "Raw Widget" {
t.Fatalf("expected passthrough name, got %q", processedName)
}
foundFailed := false
for _, s := range job.StepProgress {
if s.Step == StepAIEnhance && s.Status == "failed" {
foundFailed = true
note := strings.ToLower(s.Note)
if !strings.Contains(note, "timed out") && !strings.Contains(note, "deadline") {
t.Fatalf("failed note=%q", s.Note)
}
}
}
if !foundFailed {
t.Fatalf("expected ai_enhance failed in step_progress=%v", job.StepProgress)
}
}
// TestRunSteps_liveSmallLLMIfConfigured optionally hits OPENAI_BASE_URL (Green Chat / local).
// Skips when unset or unreachable — CI uses the httptest mocks above.
func TestRunSteps_liveSmallLLMIfConfigured(t *testing.T) {
base := strings.TrimSpace(os.Getenv("OPENAI_BASE_URL"))
key := strings.TrimSpace(os.Getenv("OPENAI_API_KEY"))
model := strings.TrimSpace(os.Getenv("OPENAI_MODEL"))
if base == "" || key == "" {
t.Skip("OPENAI_BASE_URL / OPENAI_API_KEY not set")
}
if model == "" {
model = "gpt-4o-mini"
}
t.Setenv("APP_ENV", "local")
c := NewOpenAIClient(key, base, model, 0, 0)
if !c.Enabled() {
t.Skip("OpenAI client not enabled")
}
probeCtx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
defer cancel()
req, err := http.NewRequestWithContext(probeCtx, http.MethodGet, strings.TrimRight(base, "/")+"/models", nil)
if err != nil {
t.Fatal(err)
}
req.Header.Set("Authorization", "Bearer "+key)
res, err := c.HTTPClient.Do(req)
if err != nil {
t.Skipf("LLM unreachable: %v", err)
}
_ = res.Body.Close()
if res.StatusCode >= 500 {
t.Skipf("LLM models probe HTTP %d", res.StatusCode)
}
e := &Engine{Completer: c, Vector: NoopVectorCategorizer{}}
runCtx, runCancel := context.WithTimeout(context.Background(), 45*time.Second)
defer runCancel()
out, err := e.RunSteps(runCtx, "co-live-llm", ProductInput{
Mapped: map[string]any{
"name": "Live LLM Test Widget",
"description": "Short product used only in automated pipeline tests.",
"brand": "DescrybeTest",
},
Language: "en",
}, "enhance_only", nil, StepPolicy{AllowAI: true})
if err != nil {
t.Fatal(err)
}
if strings.TrimSpace(out.ProcessedName) == "" {
t.Fatalf("empty ProcessedName notes=%v", out.Notes)
}
joined := strings.Join(out.Notes, ";")
if strings.Contains(joined, "ai_enhance: skipped") {
t.Fatalf("AI skipped unexpectedly: %v", out.Notes)
}
}