descrybe/apps/api/internal/processing/category_test.go

package processing

import (
	"context"
	"fmt"
	"os"
	"strings"
	"testing"
	"time"

	"github.com/google/uuid"
	"github.com/jackc/pgx/v5/pgxpool"
)

func TestCategoryUniqueIDFromAny(t *testing.T) {
	t.Parallel()
	cases := []struct {
		in   any
		want string
	}{
		{nil, ""},
		{"", ""},
		{"none", ""},
		{"NONE", ""},
		{"50", "50"},
		{50, "50"},
		{float64(50), "50"},
		{map[string]any{"unique_id": "28"}, "28"},
		{map[string]any{"category_unique_id": 48}, "48"},
		{map[string]any{"#text": "120"}, "120"},
		{[]any{map[string]any{"unique_id": "7"}}, "7"},
		// Nested name-only is a candidate token (resolved to unique_id later).
		{map[string]any{"name": "OnlyName"}, "OnlyName"},
	}
	for i, tc := range cases {
		if got := categoryUniqueIDFromAny(tc.in); got != tc.want {
			t.Fatalf("case %d: got %q want %q", i, got, tc.want)
		}
	}
}

func TestRunSteps_full_mappedUniqueIDCategory(t *testing.T) {
	e := &Engine{
		Completer: stubCompleter{fn: func(system, user string) (Completion, error) {
			return Completion{Text: `{"name":"N","description":"D"}`, TotalTokens: 1}, nil
		}},
		Vector: NoopVectorCategorizer{},
	}
	out, err := e.RunSteps(context.Background(), "co", ProductInput{
		GTIN: "8606019604493",
		Mapped: map[string]any{
			"name":        "Cooker",
			"description": "stove",
			"category":    "50",
		},
	}, "full", nil, StepPolicy{AllowAI: true, AllowEPREL: false})
	if err != nil {
		t.Fatal(err)
	}
	if out.Category != "50" {
		t.Fatalf("category=%q want 50", out.Category)
	}
	if src, _ := out.FieldSources["category"].(string); src != "mapped" {
		t.Fatalf("field_sources.category=%v want mapped", out.FieldSources["category"])
	}
}

func TestRunSteps_full_categoryUniqueIDKey(t *testing.T) {
	e := &Engine{Vector: NoopVectorCategorizer{}}
	out, err := e.RunSteps(context.Background(), "co", ProductInput{
		Mapped: map[string]any{
			"name":                "Headphones",
			"category_unique_id":  "48",
		},
	}, "normalize_only", nil, StepPolicy{})
	if err != nil {
		t.Fatal(err)
	}
	if out.Category != "48" {
		t.Fatalf("category=%q want 48", out.Category)
	}
}

func TestRunSteps_full_nestedCategoryUniqueID(t *testing.T) {
	e := &Engine{Vector: NoopVectorCategorizer{}}
	out, err := e.RunSteps(context.Background(), "co", ProductInput{
		Mapped: map[string]any{
			"name": "Dryer",
			"category": map[string]any{
				"unique_id": "52",
				"name":      "Sušilni stroji",
			},
		},
	}, "normalize_only", nil, StepPolicy{})
	if err != nil {
		t.Fatal(err)
	}
	if out.Category != "52" {
		t.Fatalf("category=%q want 52", out.Category)
	}
}

func TestCategoryDisplayLabel_andSync(t *testing.T) {
	t.Parallel()
	out := StepResult{Category: "50"}
	syncCategoryName(&out, map[string]string{"50": "Štedilniki"})
	if out.CategoryName != "Štedilniki" {
		t.Fatalf("CategoryName=%q", out.CategoryName)
	}
	if out.Category != "50" {
		t.Fatalf("Category unique_id mutated: %q", out.Category)
	}
	if got := categoryDisplayLabel(out); got != "Štedilniki" {
		t.Fatalf("display=%q", got)
	}
	synth := synthesizeDescriptionFromTitle("Cooker X", categoryDisplayLabel(out), "sl", nil)
	if strings.Contains(synth, "kategoriji 50") || strings.Contains(synth, " 50") {
		t.Fatalf("synth used unique_id: %q", synth)
	}
	if !strings.Contains(synth, "Štedilniki") {
		t.Fatalf("synth missing display name: %q", synth)
	}
}

func TestFilterCategoryIfInvalid(t *testing.T) {
	t.Parallel()
	out := StepResult{Category: "50", FieldSources: map[string]any{"category": "mapped"}}
	filterCategoryIfInvalid(&out, map[string]struct{}{"50": {}})
	if out.Category != "50" {
		t.Fatalf("valid unique_id cleared: %q", out.Category)
	}
	filterCategoryIfInvalid(&out, map[string]struct{}{"99": {}})
	if out.Category != "" {
		t.Fatalf("invalid unique_id kept: %q", out.Category)
	}
	if src, _ := out.FieldSources["category"].(string); src != "cleared_invalid" {
		t.Fatalf("field_sources.category=%v want cleared_invalid", out.FieldSources["category"])
	}
	out.Category = "50"
	filterCategoryIfInvalid(&out, nil)
	if out.Category != "50" {
		t.Fatalf("empty valid set should skip filter: %q", out.Category)
	}
}

func TestCoerceCategoryToCompanyUniqueID_nameToUID(t *testing.T) {
	t.Parallel()
	names := map[string]string{"50": "Štedilniki", "28": "TV mounts"}
	valid := map[string]struct{}{"50": {}, "28": {}}

	out := StepResult{Category: "Štedilniki"}
	coerceCategoryToCompanyUniqueID(&out, names, valid)
	if out.Category != "50" {
		t.Fatalf("name coerce: Category=%q want 50", out.Category)
	}
	filterCategoryIfInvalid(&out, valid)
	if out.Category != "50" {
		t.Fatalf("after filter: Category=%q want 50", out.Category)
	}

	// Case-insensitive name match.
	out = StepResult{Category: "tv mounts"}
	coerceCategoryToCompanyUniqueID(&out, names, valid)
	if out.Category != "28" {
		t.Fatalf("case-insensitive coerce: Category=%q want 28", out.Category)
	}

	// Already a unique_id — unchanged.
	out = StepResult{Category: "50"}
	coerceCategoryToCompanyUniqueID(&out, names, valid)
	if out.Category != "50" {
		t.Fatalf("uid passthrough: Category=%q", out.Category)
	}

	// Unknown token — left for filter to clear.
	out = StepResult{Category: "not-a-category"}
	coerceCategoryToCompanyUniqueID(&out, names, valid)
	if out.Category != "not-a-category" {
		t.Fatalf("unknown should stay until filter: %q", out.Category)
	}
	filterCategoryIfInvalid(&out, valid)
	if out.Category != "" {
		t.Fatalf("unknown should clear: %q", out.Category)
	}
	if src, _ := out.FieldSources["category"].(string); src != "cleared_invalid" {
		t.Fatalf("field_sources.category=%v want cleared_invalid", out.FieldSources["category"])
	}
}

// Product titles (and nested category.name copies of the title) must never land
// in Category — only taxonomy unique_ids (+ display names from categories).
func TestRunSteps_productTitleNeverBecomesCategory(t *testing.T) {
	t.Parallel()
	const productTitle = "Bosch Serie 6 WAU28PH0BY 9kg White Washing Machine"
	names := map[string]string{"50": "Pralni stroji", "28": "TV mounts"}
	e := &Engine{Vector: NoopVectorCategorizer{}}

	out, err := e.RunSteps(context.Background(), "co", ProductInput{
		GTIN: "4242005191234",
		Name: productTitle,
		Mapped: map[string]any{
			"name": productTitle,
			"category": map[string]any{
				"name": productTitle, // feed pollution: product title as category.name
			},
		},
		CategoryNamesByUID: names,
	}, "normalize_only", nil, StepPolicy{})
	if err != nil {
		t.Fatal(err)
	}
	if out.Category == productTitle || strings.EqualFold(out.Category, productTitle) {
		t.Fatalf("Category must not be product title: %q", out.Category)
	}
	if out.CategoryName == productTitle || strings.EqualFold(out.CategoryName, productTitle) {
		t.Fatalf("CategoryName must not be product title: %q", out.CategoryName)
	}
	if cat := categoryDisplayLabel(out); cat == productTitle || strings.EqualFold(cat, productTitle) {
		t.Fatalf("categoryDisplayLabel leaked product title: %q", cat)
	}
	if out.Category != "" {
		t.Fatalf("Category=%q want empty (unresolved product-title token scrubbed)", out.Category)
	}

	// Plain string category == product title must also be scrubbed.
	out2, err := e.RunSteps(context.Background(), "co", ProductInput{
		GTIN: "4242005191235",
		Name: productTitle,
		Mapped: map[string]any{
			"name":     productTitle,
			"category": productTitle,
		},
		CategoryNamesByUID: names,
	}, "normalize_only", nil, StepPolicy{})
	if err != nil {
		t.Fatal(err)
	}
	if out2.Category != "" {
		t.Fatalf("plain title-as-category kept: %q", out2.Category)
	}

	// Real taxonomy display name still coerces via processOne helpers.
	out3 := StepResult{Category: "Pralni stroji", Name: productTitle, ProcessedName: productTitle}
	valid := map[string]struct{}{"50": {}, "28": {}}
	coerceCategoryToCompanyUniqueID(&out3, names, valid)
	filterCategoryIfInvalid(&out3, valid)
	scrubCategoryPollution(&out3, names, valid)
	syncCategoryName(&out3, names)
	if out3.Category != "50" {
		t.Fatalf("taxonomy name coerce: Category=%q want 50", out3.Category)
	}
	if out3.CategoryName != "Pralni stroji" {
		t.Fatalf("CategoryName=%q want Pralni stroji", out3.CategoryName)
	}
}

func TestResolveCompanyCategoryUniqueID_usedByV1Projection(t *testing.T) {
	t.Parallel()
	names := map[string]string{"50": "Štedilniki"}
	valid := map[string]struct{}{"50": {}}
	// Poll/projection path: mapped name-only token must resolve like processOne.
	if got := resolveCompanyCategoryUniqueID("Štedilniki", names, valid); got != "50" {
		t.Fatalf("got %q want 50", got)
	}
	if got := resolveCompanyCategoryUniqueID("50", names, valid); got != "50" {
		t.Fatalf("uid passthrough got %q", got)
	}
}

func TestCategoryUniqueIDFromAny_nestedNameOnly(t *testing.T) {
	t.Parallel()
	got := categoryUniqueIDFromAny(map[string]any{"name": "Štedilniki"})
	if got != "Štedilniki" {
		t.Fatalf("got %q want Štedilniki", got)
	}
	// unique_id still wins over name.
	got = categoryUniqueIDFromAny(map[string]any{"unique_id": "50", "name": "Štedilniki"})
	if got != "50" {
		t.Fatalf("got %q want 50", got)
	}
}

func TestLoadV1ProcessJobItems_resolvesCategoryName(t *testing.T) {
	dsn := os.Getenv("DATABASE_URL")
	if dsn == "" {
		t.Skip("DATABASE_URL not set")
	}

	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
	defer cancel()

	pg, err := pgxpool.New(ctx, dsn)
	if err != nil {
		t.Fatal(err)
	}
	defer pg.Close()

	companyID := uuid.New()
	userID := uuid.New()
	rawID := uuid.New()
	jobID := uuid.New()
	ppID := uuid.New()
	gtin := "cat-v1-" + companyID.String()[:8]

	if _, err := pg.Exec(ctx, `INSERT INTO companies (id, name) VALUES ($1, $2)`, companyID, "cat-v1-test"); err != nil {
		t.Fatal(err)
	}
	// users.id may be required for processing_jobs — reuse an existing user when possible.
	err = pg.QueryRow(ctx, `SELECT id FROM users ORDER BY created_at DESC LIMIT 1`).Scan(&userID)
	if err != nil {
		t.Skip("no users rows available")
	}
	defer func() {
		_, _ = pg.Exec(context.Background(), `DELETE FROM processing_job_products WHERE job_id = $1`, jobID)
		_, _ = pg.Exec(context.Background(), `DELETE FROM processing_jobs WHERE id = $1`, jobID)
		_, _ = pg.Exec(context.Background(), `DELETE FROM processed_products WHERE company_id = $1`, companyID)
		_, _ = pg.Exec(context.Background(), `DELETE FROM raw_products WHERE company_id = $1`, companyID)
		_, _ = pg.Exec(context.Background(), `DELETE FROM categories WHERE company_id = $1`, companyID)
		_, _ = pg.Exec(context.Background(), `DELETE FROM companies WHERE id = $1`, companyID)
	}()

	if _, err := pg.Exec(ctx, `
		INSERT INTO categories (id, company_id, name, unique_id, is_active)
		VALUES (gen_random_uuid(), $1, 'Štedilniki', '50', true)`, companyID); err != nil {
		t.Fatal(err)
	}
	if _, err := pg.Exec(ctx, `
		INSERT INTO raw_products (id, company_id, gtin, raw_data, mapped_data, is_processed, processing_status)
		VALUES ($1, $2, $3, '{}'::jsonb, '{"category":"50","name":"Cooker"}'::jsonb, true, 'processed')`,
		rawID, companyID, gtin); err != nil {
		t.Fatal(err)
	}
	if _, err := pg.Exec(ctx, `
		INSERT INTO processed_products (
			id, company_id, product_id, name, category, description, processed_name, processed_description,
			raw_product_id, status, attributes, processed_attributes, field_sources
		) VALUES (
			$1, $2, $3, 'Cooker', '50', 'stove', 'Cooker', 'stove',
			$4, 'needs_review', '{}'::jsonb, '{}'::jsonb, '{"category":"mapped"}'::jsonb
		)`, ppID, companyID, gtin, rawID); err != nil {
		t.Fatal(err)
	}
	if _, err := pg.Exec(ctx, `
		INSERT INTO processing_jobs (id, company_id, user_id, status, processing_type, total_products, processed_products)
		VALUES ($1, $2, $3, 'completed', 'full', 1, 1)`, jobID, companyID, userID); err != nil {
		t.Fatal(err)
	}
	if _, err := pg.Exec(ctx, `
		INSERT INTO processing_job_products (id, job_id, raw_product_id, processed_product_id, status)
		VALUES (gen_random_uuid(), $1, $2, $3, 'processed')`, jobID, rawID, ppID); err != nil {
		t.Fatal(err)
	}

	p := NewPipeline(pg)
	items, err := p.LoadV1ProcessJobItems(ctx, companyID, jobID, "full")
	if err != nil {
		t.Fatal(err)
	}
	if len(items) != 1 {
		t.Fatalf("items=%d want 1", len(items))
	}
	if got := fmt.Sprint(items[0]["category"]); got != "Štedilniki" {
		t.Fatalf("category=%v want Štedilniki (display name)", items[0]["category"])
	}
	if got := fmt.Sprint(items[0]["category_id"]); got != "50" {
		t.Fatalf("category_id=%v want 50", items[0]["category_id"])
	}
	if got := fmt.Sprint(items[0]["category_name"]); got != "Štedilniki" {
		t.Fatalf("category_name=%v want Štedilniki", items[0]["category_name"])
	}
}

type stubVectorCategorizer struct {
	enabled bool
	cat     string
	err     error
	calls   int
}

func (s *stubVectorCategorizer) Enabled() bool { return s.enabled }

func (s *stubVectorCategorizer) SuggestCategory(context.Context, string, string, []string) (string, error) {
	s.calls++
	return s.cat, s.err
}

func TestRunSteps_vectorCategoryRequiresAllowAI(t *testing.T) {
	vec := &stubVectorCategorizer{enabled: true, cat: "28"}
	e := &Engine{Vector: vec}
	out, err := e.RunSteps(context.Background(), "co", ProductInput{
		Name: "TV mount",
		Mapped: map[string]any{
			"name": "TV mount",
		},
	}, "normalize_only", nil, StepPolicy{AllowAI: false})
	if err != nil {
		t.Fatal(err)
	}
	if out.Category != "" {
		t.Fatalf("category=%q want empty when AllowAI=false", out.Category)
	}
	if vec.calls != 0 {
		t.Fatalf("vector calls=%d want 0", vec.calls)
	}
	found := false
	for _, n := range out.Notes {
		if strings.Contains(n, "category: unset") && strings.Contains(n, "AI/vector not allowed") {
			found = true
			break
		}
	}
	if !found {
		t.Fatalf("expected unset note, got %v", out.Notes)
	}
}

func TestRunSteps_vectorCategoryWhenAllowAI(t *testing.T) {
	vec := &stubVectorCategorizer{enabled: true, cat: "28"}
	e := &Engine{Vector: vec}
	out, err := e.RunSteps(context.Background(), "co", ProductInput{
		Name: "TV mount",
		Mapped: map[string]any{
			"name": "TV mount",
		},
	}, "normalize_only", nil, StepPolicy{AllowAI: true})
	if err != nil {
		t.Fatal(err)
	}
	if out.Category != "28" {
		t.Fatalf("category=%q want 28", out.Category)
	}
	if src, _ := out.FieldSources["category"].(string); src != "vector" {
		t.Fatalf("field_sources.category=%v want vector", out.FieldSources["category"])
	}
	if vec.calls != 1 {
		t.Fatalf("vector calls=%d want 1", vec.calls)
	}
}

func TestRunSteps_mappedCategorySkipsVector(t *testing.T) {
	vec := &stubVectorCategorizer{enabled: true, cat: "99"}
	e := &Engine{Vector: vec}
	out, err := e.RunSteps(context.Background(), "co", ProductInput{
		Mapped: map[string]any{
			"name":     "Cooker",
			"category": "50",
		},
	}, "normalize_only", nil, StepPolicy{AllowAI: true})
	if err != nil {
		t.Fatal(err)
	}
	if out.Category != "50" {
		t.Fatalf("category=%q want 50", out.Category)
	}
	if vec.calls != 0 {
		t.Fatalf("vector should not run when mapped unique_id present: calls=%d", vec.calls)
	}
}

func TestBackfillProcessedCategoriesFromMapped(t *testing.T) {
	dsn := os.Getenv("DATABASE_URL")
	if dsn == "" {
		t.Skip("DATABASE_URL not set")
	}

	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
	defer cancel()

	pg, err := pgxpool.New(ctx, dsn)
	if err != nil {
		t.Fatal(err)
	}
	defer pg.Close()

	companyID := uuid.New()
	rawID := uuid.New()
	ppID := uuid.New()
	gtin := "cat-bf-" + companyID.String()[:8]

	if _, err := pg.Exec(ctx, `INSERT INTO companies (id, name) VALUES ($1, $2)`, companyID, "cat-backfill-test"); err != nil {
		t.Fatal(err)
	}
	defer func() {
		_, _ = pg.Exec(context.Background(), `DELETE FROM processed_products WHERE company_id = $1`, companyID)
		_, _ = pg.Exec(context.Background(), `DELETE FROM raw_products WHERE company_id = $1`, companyID)
		_, _ = pg.Exec(context.Background(), `DELETE FROM categories WHERE company_id = $1`, companyID)
		_, _ = pg.Exec(context.Background(), `DELETE FROM companies WHERE id = $1`, companyID)
	}()

	if _, err := pg.Exec(ctx, `
		INSERT INTO categories (id, company_id, name, unique_id, is_active)
		VALUES (gen_random_uuid(), $1, 'TV mounts', '28', true)`, companyID); err != nil {
		t.Fatal(err)
	}
	if _, err := pg.Exec(ctx, `
		INSERT INTO raw_products (id, company_id, gtin, raw_data, mapped_data, is_processed, processing_status)
		VALUES ($1, $2, $3, '{}'::jsonb, '{"name":"Mount","category":{"unique_id":"28","name":"TV"}}'::jsonb, true, 'processed')`,
		rawID, companyID, gtin); err != nil {
		t.Fatal(err)
	}
	if _, err := pg.Exec(ctx, `
		INSERT INTO processed_products (
			id, company_id, product_id, name, category, description, processed_name, processed_description,
			raw_product_id, status, attributes, processed_attributes, field_sources
		) VALUES (
			$1, $2, $3, 'Mount', '', 'x', 'Mount', 'x',
			$4, 'needs_review', '{}'::jsonb, '{}'::jsonb, '{}'::jsonb
		)`, ppID, companyID, gtin, rawID); err != nil {
		t.Fatal(err)
	}

	res, err := BackfillProcessedCategoriesFromMapped(ctx, pg, companyID)
	if err != nil {
		t.Fatal(err)
	}
	if res.Updated != 1 {
		t.Fatalf("updated=%d want 1", res.Updated)
	}
	var cat, src string
	if err := pg.QueryRow(ctx, `
		SELECT COALESCE(category, ''), COALESCE(field_sources->>'category', '')
		FROM processed_products WHERE id = $1`, ppID).Scan(&cat, &src); err != nil {
		t.Fatal(err)
	}
	if cat != "28" {
		t.Fatalf("category=%q want 28", cat)
	}
	if src != "mapped_backfill" {
		t.Fatalf("field_sources.category=%q want mapped_backfill", src)
	}

	// Idempotent.
	res2, err := BackfillProcessedCategoriesFromMapped(ctx, pg, companyID)
	if err != nil {
		t.Fatal(err)
	}
	if res2.Updated != 0 {
		t.Fatalf("second backfill updated=%d want 0", res2.Updated)
	}

	withCat, withoutCat, err := CountProcessedCategoryCoverage(ctx, pg, companyID)
	if err != nil {
		t.Fatal(err)
	}
	if withCat != 1 || withoutCat != 0 {
		t.Fatalf("coverage with=%d without=%d", withCat, withoutCat)
	}
}

func TestBackfillMappedCategoriesFromProcessed(t *testing.T) {
	dsn := os.Getenv("DATABASE_URL")
	if dsn == "" {
		t.Skip("DATABASE_URL not set")
	}

	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
	defer cancel()

	pg, err := pgxpool.New(ctx, dsn)
	if err != nil {
		t.Fatal(err)
	}
	defer pg.Close()

	companyID := uuid.New()
	rawID := uuid.New()
	ppID := uuid.New()
	gtin := "map-bf-" + companyID.String()[:8]

	if _, err := pg.Exec(ctx, `INSERT INTO companies (id, name) VALUES ($1, $2)`, companyID, "mapped-backfill-test"); err != nil {
		t.Fatal(err)
	}
	defer func() {
		_, _ = pg.Exec(context.Background(), `DELETE FROM processed_products WHERE company_id = $1`, companyID)
		_, _ = pg.Exec(context.Background(), `DELETE FROM raw_products WHERE company_id = $1`, companyID)
		_, _ = pg.Exec(context.Background(), `DELETE FROM companies WHERE id = $1`, companyID)
	}()

	if _, err := pg.Exec(ctx, `
		INSERT INTO raw_products (id, company_id, gtin, raw_data, mapped_data, is_processed, processing_status)
		VALUES ($1, $2, $3, '{}'::jsonb, '{"name":"Widget"}'::jsonb, true, 'processed')`,
		rawID, companyID, gtin); err != nil {
		t.Fatal(err)
	}
	if _, err := pg.Exec(ctx, `
		INSERT INTO processed_products (
			id, company_id, product_id, name, category, description, processed_name, processed_description,
			raw_product_id, status, attributes, processed_attributes, field_sources
		) VALUES (
			$1, $2, $3, 'Widget', '42', 'x', 'Widget', 'x',
			$4, 'needs_review', '{}'::jsonb, '{}'::jsonb, '{}'::jsonb
		)`, ppID, companyID, gtin, rawID); err != nil {
		t.Fatal(err)
	}

	n, err := BackfillMappedCategoriesFromProcessed(ctx, pg, companyID)
	if err != nil {
		t.Fatal(err)
	}
	if n != 1 {
		t.Fatalf("updated=%d want 1", n)
	}
	var mappedCat string
	if err := pg.QueryRow(ctx, `
		SELECT COALESCE(mapped_data->>'category', '') FROM raw_products WHERE id = $1`, rawID).Scan(&mappedCat); err != nil {
		t.Fatal(err)
	}
	if mappedCat != "42" {
		t.Fatalf("mapped category=%q want 42", mappedCat)
	}

	n2, err := BackfillMappedCategoriesFromProcessed(ctx, pg, companyID)
	if err != nil {
		t.Fatal(err)
	}
	if n2 != 0 {
		t.Fatalf("second backfill updated=%d want 0", n2)
	}

	withCat, withoutCat, taxonomy, err := CountMappedCategoryCoverage(ctx, pg, companyID)
	if err != nil {
		t.Fatal(err)
	}
	if withCat != 1 || withoutCat != 0 || taxonomy != 0 {
		t.Fatalf("mapped coverage with=%d without=%d taxonomy=%d", withCat, withoutCat, taxonomy)
	}
}