Initial commit of Descrybe v2 without local scratch artifacts.

Drop one-shot tmp/axe scripts and agent i18n scratch so the Gitea tree is deployable.
2026-08-09 22:47:43 +02:00
commit 8580c996c3
1285 changed files with 325780 additions and 0 deletions
@@ -0,0 +1,574 @@
+package processing
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"net/http"
+	"net/http/httptest"
+	"os"
+	"strings"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	"github.com/google/uuid"
+	"github.com/jackc/pgx/v5/pgxpool"
+)
+
+// mockChatCompletionsServer is an OpenAI-compatible stand-in for the small/test LLM.
+func mockChatCompletionsServer(t *testing.T, handler http.HandlerFunc) *httptest.Server {
+	t.Helper()
+	mux := http.NewServeMux()
+	mux.HandleFunc("/v1/chat/completions", handler)
+	srv := httptest.NewServer(mux)
+	t.Cleanup(srv.Close)
+	return srv
+}
+
+func openAIClientForMock(t *testing.T, baseURL string, maxRetries int) *OpenAIClient {
+	t.Helper()
+	t.Setenv("APP_ENV", "local")
+	// NewOpenAIClient coerces maxRetries<=0 to 3; set the field after for single-shot tests.
+	c := NewOpenAIClient("sk-test-pipeline-llm", strings.TrimRight(baseURL, "/")+"/v1", "test-small-model", 0, 1)
+	if maxRetries < 0 {
+		maxRetries = 0
+	}
+	c.MaxRetries = maxRetries
+	if !c.Enabled() {
+		t.Fatal("expected mock OpenAI client enabled")
+	}
+	return c
+}
+
+func TestRunSteps_enhanceMockHappyPath(t *testing.T) {
+	t.Parallel()
+	var gotSystem string
+	e := &Engine{
+		Completer: stubCompleter{fn: func(system, _ string) (Completion, error) {
+			gotSystem = system
+			return Completion{
+				Text:        `{"name":"Mock Shoe","description":"Light runner for tests."}`,
+				TotalTokens: 11,
+				Model:       "mock",
+			}, nil
+		}},
+		Vector: NoopVectorCategorizer{},
+	}
+	out, err := e.RunSteps(context.Background(), "co-llm-mock", ProductInput{
+		GTIN: "8712345678901",
+		Name: "Shoe", Description: "runner",
+		Mapped:   map[string]any{"name": "Shoe", "description": "runner", "brand": "Acme"},
+		Language: "de",
+	}, "enhance_only", nil, StepPolicy{AllowAI: true})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if out.ProcessedName != "Mock Shoe" {
+		t.Fatalf("ProcessedName=%q", out.ProcessedName)
+	}
+	if out.TotalTokens != 11 {
+		t.Fatalf("TotalTokens=%d", out.TotalTokens)
+	}
+	if !strings.Contains(gotSystem, "German") {
+		t.Fatalf("expected {{language}}→German in system prompt, got %q", gotSystem)
+	}
+	prog := progressFromResult("enhance_only", &out)
+	if len(prog) < 2 || prog[len(prog)-1].Status != "done" {
+		t.Fatalf("step progress=%v", prog)
+	}
+}
+
+func TestRunSteps_enhanceMockProviderFailure(t *testing.T) {
+	t.Parallel()
+	var gotSystem string
+	e := &Engine{
+		Completer: stubCompleter{fn: func(system, _ string) (Completion, error) {
+			gotSystem = system
+			return Completion{}, errors.New("upstream 503: model overloaded")
+		}},
+	}
+	out, err := e.RunSteps(context.Background(), "co-llm-mock", ProductInput{
+		Mapped:   map[string]any{"name": "Widget", "description": "plain"},
+		Language: "fr",
+	}, "enhance_only", nil, StepPolicy{AllowAI: true})
+	if err != nil {
+		t.Fatalf("RunSteps should not fail the call on AI error: %v", err)
+	}
+	if out.ProcessedName != "Widget" {
+		t.Fatalf("expected passthrough name, got %q", out.ProcessedName)
+	}
+	if !strings.Contains(gotSystem, "French") {
+		t.Fatalf("failure path must still inject language before error: %q", gotSystem)
+	}
+	joined := strings.Join(out.Notes, ";")
+	if !strings.Contains(joined, "ai_enhance:") || !strings.Contains(joined, "503") {
+		t.Fatalf("expected failure note, got %v", out.Notes)
+	}
+	prog := progressFromResult("enhance_only", &out)
+	foundFailed := false
+	for _, s := range prog {
+		if s.Step == StepAIEnhance && s.Status == "failed" {
+			foundFailed = true
+			if !strings.Contains(s.Note, "503") {
+				t.Fatalf("failed note=%q", s.Note)
+			}
+		}
+	}
+	if !foundFailed {
+		t.Fatalf("expected ai_enhance failed in progress=%v", prog)
+	}
+}
+
+func TestRunSteps_enhanceMockTimeout(t *testing.T) {
+	t.Parallel()
+	e := &Engine{
+		Completer: stubCompleter{fn: func(_, _ string) (Completion, error) {
+			return Completion{}, context.DeadlineExceeded
+		}},
+	}
+	out, err := e.RunSteps(context.Background(), "co-llm-mock", ProductInput{
+		Mapped:   map[string]any{"name": "Timeout Widget", "description": "slow"},
+		Language: "nl",
+	}, "enhance_only", nil, StepPolicy{AllowAI: true})
+	if err != nil {
+		t.Fatalf("RunSteps should swallow provider timeout: %v", err)
+	}
+	joined := strings.Join(out.Notes, ";")
+	lowerJoined := strings.ToLower(joined)
+	if !strings.Contains(lowerJoined, "timed out") && !strings.Contains(lowerJoined, "deadline") {
+		t.Fatalf("expected timeout note, got %v", out.Notes)
+	}
+	prog := progressFromResult("enhance_only", &out)
+	ok := false
+	for _, s := range prog {
+		if s.Step == StepAIEnhance && s.Status == "failed" {
+			ok = true
+		}
+	}
+	if !ok {
+		t.Fatalf("expected ai_enhance failed, progress=%v", prog)
+	}
+}
+
+func TestOpenAIClient_Complete_httptestHappyPath(t *testing.T) {
+	srv := mockChatCompletionsServer(t, func(w http.ResponseWriter, r *http.Request) {
+		if r.Method != http.MethodPost {
+			t.Fatalf("method=%s", r.Method)
+		}
+		auth := r.Header.Get("Authorization")
+		if !strings.HasPrefix(auth, "Bearer sk-test-") {
+			http.Error(w, "unauthorized", http.StatusUnauthorized)
+			return
+		}
+		_ = json.NewEncoder(w).Encode(map[string]any{
+			"model": "test-small-model",
+			"choices": []map[string]any{
+				{"message": map[string]any{"content": `{"name":"HTTP Shoe","description":"From mock LLM."}`}},
+			},
+			"usage": map[string]any{
+				"prompt_tokens": 3, "completion_tokens": 5, "total_tokens": 8,
+			},
+		})
+	})
+	c := openAIClientForMock(t, srv.URL, 0)
+	comp, err := c.Complete(context.Background(), "sys", "user")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !strings.Contains(comp.Text, "HTTP Shoe") {
+		t.Fatalf("text=%q", comp.Text)
+	}
+	if comp.TotalTokens != 8 {
+		t.Fatalf("tokens=%d", comp.TotalTokens)
+	}
+}
+
+func TestOpenAIClient_Complete_httptestTimeout(t *testing.T) {
+	srv := mockChatCompletionsServer(t, func(w http.ResponseWriter, r *http.Request) {
+		deadline := time.After(300 * time.Millisecond)
+		for {
+			select {
+			case <-r.Context().Done():
+				return
+			case <-deadline:
+				_ = json.NewEncoder(w).Encode(map[string]any{
+					"choices": []map[string]any{
+						{"message": map[string]any{"content": "late"}},
+					},
+				})
+				return
+			case <-time.After(10 * time.Millisecond):
+			}
+		}
+	})
+	c := openAIClientForMock(t, srv.URL, 0)
+	c.HTTPClient.Timeout = 60 * time.Millisecond
+	start := time.Now()
+	_, err := c.Complete(context.Background(), "sys", "user")
+	elapsed := time.Since(start)
+	if err == nil {
+		t.Fatal("expected timeout error")
+	}
+	if elapsed > time.Second {
+		t.Fatalf("timeout too slow: %s err=%v", elapsed, err)
+	}
+}
+
+func TestRunSteps_enhanceViaHTTPLLMMock(t *testing.T) {
+	var calls atomic.Int32
+	srv := mockChatCompletionsServer(t, func(w http.ResponseWriter, r *http.Request) {
+		calls.Add(1)
+		_ = json.NewEncoder(w).Encode(map[string]any{
+			"model": "test-small-model",
+			"choices": []map[string]any{
+				{"message": map[string]any{"content": `{"name":"Pipeline Mock","description":"Happy path via httptest."}`}},
+			},
+			"usage": map[string]any{"total_tokens": 9},
+		})
+	})
+	e := &Engine{
+		Completer: openAIClientForMock(t, srv.URL, 0),
+		Vector:    NoopVectorCategorizer{},
+	}
+	out, err := e.RunSteps(context.Background(), "co-llm-http", ProductInput{
+		Mapped:   map[string]any{"name": "Raw", "description": "desc"},
+		Language: "it",
+	}, "enhance_only", nil, StepPolicy{AllowAI: true})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if calls.Load() < 1 {
+		t.Fatal("expected chat completions call")
+	}
+	if out.ProcessedName != "Pipeline Mock" {
+		t.Fatalf("name=%q", out.ProcessedName)
+	}
+	if out.TotalTokens < 1 {
+		t.Fatalf("tokens=%d", out.TotalTokens)
+	}
+}
+
+func TestRunSteps_enhanceViaHTTPLLMTimeout(t *testing.T) {
+	srv := mockChatCompletionsServer(t, func(w http.ResponseWriter, r *http.Request) {
+		deadline := time.After(300 * time.Millisecond)
+		for {
+			select {
+			case <-r.Context().Done():
+				return
+			case <-deadline:
+				_ = json.NewEncoder(w).Encode(map[string]any{
+					"choices": []map[string]any{
+						{"message": map[string]any{"content": `{"name":"Late","description":"x"}`}},
+					},
+				})
+				return
+			case <-time.After(10 * time.Millisecond):
+			}
+		}
+	})
+	client := openAIClientForMock(t, srv.URL, 0)
+	client.HTTPClient.Timeout = 60 * time.Millisecond
+	e := &Engine{Completer: client}
+	out, err := e.RunSteps(context.Background(), "co-llm-http", ProductInput{
+		Mapped: map[string]any{"name": "Raw", "description": "desc"},
+	}, "enhance_only", nil, StepPolicy{AllowAI: true})
+	if err != nil {
+		t.Fatalf("RunSteps err=%v", err)
+	}
+	joined := strings.Join(out.Notes, ";")
+	if !strings.Contains(joined, "ai_enhance:") {
+		t.Fatalf("expected ai_enhance note, got %v", out.Notes)
+	}
+	prog := progressFromResult("enhance_only", &out)
+	failed := false
+	for _, s := range prog {
+		if s.Step == StepAIEnhance && s.Status == "failed" {
+			failed = true
+		}
+	}
+	if !failed {
+		t.Fatalf("expected failed ai_enhance, progress=%v notes=%v", prog, out.Notes)
+	}
+}
+
+func TestRunSteps_enhanceViaHTTPLLMServerError(t *testing.T) {
+	srv := mockChatCompletionsServer(t, func(w http.ResponseWriter, r *http.Request) {
+		w.WriteHeader(http.StatusBadGateway)
+		_ = json.NewEncoder(w).Encode(map[string]any{
+			"error": map[string]any{"message": "green-chat unavailable"},
+		})
+	})
+	e := &Engine{Completer: openAIClientForMock(t, srv.URL, 0)}
+	out, err := e.RunSteps(context.Background(), "co-llm-http", ProductInput{
+		Mapped: map[string]any{"name": "Raw", "description": "desc"},
+	}, "enhance_only", nil, StepPolicy{AllowAI: true})
+	if err != nil {
+		t.Fatal(err)
+	}
+	joined := strings.Join(out.Notes, ";")
+	if !strings.Contains(joined, "ai_enhance:") {
+		t.Fatalf("notes=%v", out.Notes)
+	}
+	if out.ProcessedName != "Raw" {
+		t.Fatalf("expected original name on failure, got %q", out.ProcessedName)
+	}
+}
+
+func pipelineLLMMockFixtures(t *testing.T, pg *pgxpool.Pool, ctx context.Context, language string) (companyID, userID, rawID uuid.UUID, cleanup func()) {
+	t.Helper()
+	if language == "" {
+		language = "de"
+	}
+	// Prefer demo sandbox users only — never a1-primary / A1 cohort emails.
+	// Exact emails only (no LIKE '%a1%' — false positives and random-user fallback).
+	err := pg.QueryRow(ctx, `
+		SELECT id FROM users
+		WHERE LOWER(COALESCE(email, '')) = 'demo@descrybe.local'
+		LIMIT 1`).Scan(&userID)
+	if err != nil {
+		err = pg.QueryRow(ctx, `
+			SELECT id FROM users
+			WHERE LOWER(COALESCE(email, '')) = 'demo@descrybe.test'
+			LIMIT 1`).Scan(&userID)
+		if err != nil {
+			t.Skip("need demo@descrybe.local or demo@descrybe.test (run seed-demo); refusing random/a1 users")
+		}
+	}
+	companyID = uuid.New()
+	rawID = uuid.New()
+	if _, err := pg.Exec(ctx, `
+		INSERT INTO companies (id, name, language) VALUES ($1, $2, $3)`,
+		companyID, "pipeline-llm-mock-co", language); err != nil {
+		t.Fatal(err)
+	}
+	gtin := fmt.Sprintf("llm-mock-%s", companyID.String()[:8])
+	if _, err := pg.Exec(ctx, `
+		INSERT INTO raw_products (id, company_id, gtin, raw_data, mapped_data, is_processed, processing_status)
+		VALUES ($1, $2, $3, '{}'::jsonb, '{"name":"Raw Widget","description":"original desc"}'::jsonb, false, 'unprocessed')`,
+		rawID, companyID, gtin); err != nil {
+		t.Fatal(err)
+	}
+	cleanup = func() {
+		_, _ = pg.Exec(context.Background(), `DELETE FROM processed_products WHERE company_id = $1`, companyID)
+		_, _ = pg.Exec(context.Background(), `DELETE FROM processing_job_products WHERE job_id IN (SELECT id FROM processing_jobs WHERE company_id = $1)`, companyID)
+		_, _ = pg.Exec(context.Background(), `DELETE FROM processing_jobs WHERE company_id = $1`, companyID)
+		_, _ = pg.Exec(context.Background(), `DELETE FROM raw_products WHERE company_id = $1`, companyID)
+		_, _ = pg.Exec(context.Background(), `DELETE FROM companies WHERE id = $1`, companyID)
+	}
+	return companyID, userID, rawID, cleanup
+}
+
+// TestProcessJob_mockLLMHappyPath runs ProcessJob with a stub Completer (no live LLM, no a1 tenant).
+// Asserts companies.language is loaded and injected into the enhance system prompt.
+func TestProcessJob_mockLLMHappyPath(t *testing.T) {
+	dsn := os.Getenv("DATABASE_URL")
+	if dsn == "" {
+		t.Skip("DATABASE_URL not set")
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), 45*time.Second)
+	defer cancel()
+	pg, err := pgxpool.New(ctx, dsn)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer pg.Close()
+
+	companyID, userID, rawID, cleanup := pipelineLLMMockFixtures(t, pg, ctx, "de")
+	defer cleanup()
+
+	var gotSystem string
+	p := NewPipeline(pg)
+	p.Billing = nil
+	p.Limiter = nil
+	p.AI = nil
+	p.Engine = &Engine{
+		Completer: stubCompleter{fn: func(system, _ string) (Completion, error) {
+			gotSystem = system
+			return Completion{
+				Text:        `{"name":"Happy Mock","description":"Processed by mock LLM."}`,
+				TotalTokens: 6,
+				Model:       "mock",
+			}, nil
+		}},
+		Vector: NoopVectorCategorizer{},
+	}
+
+	jobs, err := p.StartJob(ctx, companyID, userID, []uuid.UUID{rawID}, "enhance_only")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if err := p.ProcessJob(ctx, jobs[0].ID); err != nil {
+		t.Fatal(err)
+	}
+	job, err := p.GetJob(ctx, companyID, jobs[0].ID)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if job.Status != "completed" || job.ProcessedProducts != 1 {
+		t.Fatalf("status=%s processed=%d", job.Status, job.ProcessedProducts)
+	}
+	if !strings.Contains(gotSystem, "German") {
+		t.Fatalf("ProcessJob must inject companies.language into enhance prompt, got %q", gotSystem)
+	}
+	var processedName string
+	if err := pg.QueryRow(ctx, `
+		SELECT processed_name FROM processed_products
+		WHERE company_id = $1 AND raw_product_id = $2`, companyID, rawID).Scan(&processedName); err != nil {
+		t.Fatal(err)
+	}
+	if processedName != "Happy Mock" {
+		t.Fatalf("processed_name=%q", processedName)
+	}
+	foundDone := false
+	for _, s := range job.StepProgress {
+		if s.Step == StepAIEnhance && s.Status == "done" {
+			foundDone = true
+		}
+	}
+	if !foundDone {
+		t.Fatalf("expected ai_enhance done, progress=%v", job.StepProgress)
+	}
+	// Second ProcessJob on a completed job must be a no-op (idempotent / safe with test LLM).
+	if err := p.ProcessJob(ctx, jobs[0].ID); err != nil {
+		t.Fatal(err)
+	}
+	job2, err := p.GetJob(ctx, companyID, jobs[0].ID)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if job2.Status != "completed" || job2.ProcessedProducts != 1 {
+		t.Fatalf("re-run mutated job status=%s processed=%d", job2.Status, job2.ProcessedProducts)
+	}
+}
+
+// TestProcessJob_mockLLMTimeoutSurfacesFailedStep: provider timeout keeps the item
+// deliverable (passthrough title) but marks ai_enhance failed in step_progress.
+// Language is still loaded from the company before the provider error.
+func TestProcessJob_mockLLMTimeoutSurfacesFailedStep(t *testing.T) {
+	dsn := os.Getenv("DATABASE_URL")
+	if dsn == "" {
+		t.Skip("DATABASE_URL not set")
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), 45*time.Second)
+	defer cancel()
+	pg, err := pgxpool.New(ctx, dsn)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer pg.Close()
+
+	companyID, userID, rawID, cleanup := pipelineLLMMockFixtures(t, pg, ctx, "fr")
+	defer cleanup()
+
+	var gotSystem string
+	p := NewPipeline(pg)
+	p.Billing = nil
+	p.Limiter = nil
+	p.AI = nil
+	p.Engine = &Engine{
+		Completer: stubCompleter{fn: func(system, _ string) (Completion, error) {
+			gotSystem = system
+			return Completion{}, context.DeadlineExceeded
+		}},
+		Vector: NoopVectorCategorizer{},
+	}
+
+	jobs, err := p.StartJob(ctx, companyID, userID, []uuid.UUID{rawID}, "enhance_only")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if err := p.ProcessJob(ctx, jobs[0].ID); err != nil {
+		t.Fatal(err)
+	}
+	job, err := p.GetJob(ctx, companyID, jobs[0].ID)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if job.Status != "completed" {
+		t.Fatalf("status=%s (AI timeout is non-fatal for the job item)", job.Status)
+	}
+	if !strings.Contains(gotSystem, "French") {
+		t.Fatalf("timeout path must still inject language before error: %q", gotSystem)
+	}
+	var processedName string
+	if err := pg.QueryRow(ctx, `
+		SELECT processed_name FROM processed_products
+		WHERE company_id = $1 AND raw_product_id = $2`, companyID, rawID).Scan(&processedName); err != nil {
+		t.Fatal(err)
+	}
+	if processedName != "Raw Widget" {
+		t.Fatalf("expected passthrough name, got %q", processedName)
+	}
+	foundFailed := false
+	for _, s := range job.StepProgress {
+		if s.Step == StepAIEnhance && s.Status == "failed" {
+			foundFailed = true
+			note := strings.ToLower(s.Note)
+			if !strings.Contains(note, "timed out") && !strings.Contains(note, "deadline") {
+				t.Fatalf("failed note=%q", s.Note)
+			}
+		}
+	}
+	if !foundFailed {
+		t.Fatalf("expected ai_enhance failed in step_progress=%v", job.StepProgress)
+	}
+}
+
+// TestRunSteps_liveSmallLLMIfConfigured optionally hits OPENAI_BASE_URL (Green Chat / local).
+// Skips when unset or unreachable — CI uses the httptest mocks above.
+func TestRunSteps_liveSmallLLMIfConfigured(t *testing.T) {
+	base := strings.TrimSpace(os.Getenv("OPENAI_BASE_URL"))
+	key := strings.TrimSpace(os.Getenv("OPENAI_API_KEY"))
+	model := strings.TrimSpace(os.Getenv("OPENAI_MODEL"))
+	if base == "" || key == "" {
+		t.Skip("OPENAI_BASE_URL / OPENAI_API_KEY not set")
+	}
+	if model == "" {
+		model = "gpt-4o-mini"
+	}
+	t.Setenv("APP_ENV", "local")
+	c := NewOpenAIClient(key, base, model, 0, 0)
+	if !c.Enabled() {
+		t.Skip("OpenAI client not enabled")
+	}
+
+	probeCtx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
+	defer cancel()
+	req, err := http.NewRequestWithContext(probeCtx, http.MethodGet, strings.TrimRight(base, "/")+"/models", nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	req.Header.Set("Authorization", "Bearer "+key)
+	res, err := c.HTTPClient.Do(req)
+	if err != nil {
+		t.Skipf("LLM unreachable: %v", err)
+	}
+	_ = res.Body.Close()
+	if res.StatusCode >= 500 {
+		t.Skipf("LLM models probe HTTP %d", res.StatusCode)
+	}
+
+	e := &Engine{Completer: c, Vector: NoopVectorCategorizer{}}
+	runCtx, runCancel := context.WithTimeout(context.Background(), 45*time.Second)
+	defer runCancel()
+	out, err := e.RunSteps(runCtx, "co-live-llm", ProductInput{
+		Mapped: map[string]any{
+			"name":        "Live LLM Test Widget",
+			"description": "Short product used only in automated pipeline tests.",
+			"brand":       "DescrybeTest",
+		},
+		Language: "en",
+	}, "enhance_only", nil, StepPolicy{AllowAI: true})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if strings.TrimSpace(out.ProcessedName) == "" {
+		t.Fatalf("empty ProcessedName notes=%v", out.Notes)
+	}
+	joined := strings.Join(out.Notes, ";")
+	if strings.Contains(joined, "ai_enhance: skipped") {
+		t.Fatalf("AI skipped unexpectedly: %v", out.Notes)
+	}
+}