package processing import ( "context" "fmt" "os" "strings" "testing" "time" "github.com/google/uuid" "github.com/jackc/pgx/v5/pgxpool" ) func TestCategoryUniqueIDFromAny(t *testing.T) { t.Parallel() cases := []struct { in any want string }{ {nil, ""}, {"", ""}, {"none", ""}, {"NONE", ""}, {"50", "50"}, {50, "50"}, {float64(50), "50"}, {map[string]any{"unique_id": "28"}, "28"}, {map[string]any{"category_unique_id": 48}, "48"}, {map[string]any{"#text": "120"}, "120"}, {[]any{map[string]any{"unique_id": "7"}}, "7"}, // Nested name-only is a candidate token (resolved to unique_id later). {map[string]any{"name": "OnlyName"}, "OnlyName"}, } for i, tc := range cases { if got := categoryUniqueIDFromAny(tc.in); got != tc.want { t.Fatalf("case %d: got %q want %q", i, got, tc.want) } } } func TestRunSteps_full_mappedUniqueIDCategory(t *testing.T) { e := &Engine{ Completer: stubCompleter{fn: func(system, user string) (Completion, error) { return Completion{Text: `{"name":"N","description":"D"}`, TotalTokens: 1}, nil }}, Vector: NoopVectorCategorizer{}, } out, err := e.RunSteps(context.Background(), "co", ProductInput{ GTIN: "8606019604493", Mapped: map[string]any{ "name": "Cooker", "description": "stove", "category": "50", }, }, "full", nil, StepPolicy{AllowAI: true, AllowEPREL: false}) if err != nil { t.Fatal(err) } if out.Category != "50" { t.Fatalf("category=%q want 50", out.Category) } if src, _ := out.FieldSources["category"].(string); src != "mapped" { t.Fatalf("field_sources.category=%v want mapped", out.FieldSources["category"]) } } func TestRunSteps_full_categoryUniqueIDKey(t *testing.T) { e := &Engine{Vector: NoopVectorCategorizer{}} out, err := e.RunSteps(context.Background(), "co", ProductInput{ Mapped: map[string]any{ "name": "Headphones", "category_unique_id": "48", }, }, "normalize_only", nil, StepPolicy{}) if err != nil { t.Fatal(err) } if out.Category != "48" { t.Fatalf("category=%q want 48", out.Category) } } func TestRunSteps_full_nestedCategoryUniqueID(t *testing.T) { e := &Engine{Vector: NoopVectorCategorizer{}} out, err := e.RunSteps(context.Background(), "co", ProductInput{ Mapped: map[string]any{ "name": "Dryer", "category": map[string]any{ "unique_id": "52", "name": "Sušilni stroji", }, }, }, "normalize_only", nil, StepPolicy{}) if err != nil { t.Fatal(err) } if out.Category != "52" { t.Fatalf("category=%q want 52", out.Category) } } func TestCategoryDisplayLabel_andSync(t *testing.T) { t.Parallel() out := StepResult{Category: "50"} syncCategoryName(&out, map[string]string{"50": "Štedilniki"}) if out.CategoryName != "Štedilniki" { t.Fatalf("CategoryName=%q", out.CategoryName) } if out.Category != "50" { t.Fatalf("Category unique_id mutated: %q", out.Category) } if got := categoryDisplayLabel(out); got != "Štedilniki" { t.Fatalf("display=%q", got) } synth := synthesizeDescriptionFromTitle("Cooker X", categoryDisplayLabel(out), "sl", nil) if strings.Contains(synth, "kategoriji 50") || strings.Contains(synth, " 50") { t.Fatalf("synth used unique_id: %q", synth) } if !strings.Contains(synth, "Štedilniki") { t.Fatalf("synth missing display name: %q", synth) } } func TestFilterCategoryIfInvalid(t *testing.T) { t.Parallel() out := StepResult{Category: "50", FieldSources: map[string]any{"category": "mapped"}} filterCategoryIfInvalid(&out, map[string]struct{}{"50": {}}) if out.Category != "50" { t.Fatalf("valid unique_id cleared: %q", out.Category) } filterCategoryIfInvalid(&out, map[string]struct{}{"99": {}}) if out.Category != "" { t.Fatalf("invalid unique_id kept: %q", out.Category) } out.Category = "50" filterCategoryIfInvalid(&out, nil) if out.Category != "50" { t.Fatalf("empty valid set should skip filter: %q", out.Category) } } func TestCoerceCategoryToCompanyUniqueID_nameToUID(t *testing.T) { t.Parallel() names := map[string]string{"50": "Štedilniki", "28": "TV mounts"} valid := map[string]struct{}{"50": {}, "28": {}} out := StepResult{Category: "Štedilniki"} coerceCategoryToCompanyUniqueID(&out, names, valid) if out.Category != "50" { t.Fatalf("name coerce: Category=%q want 50", out.Category) } filterCategoryIfInvalid(&out, valid) if out.Category != "50" { t.Fatalf("after filter: Category=%q want 50", out.Category) } // Case-insensitive name match. out = StepResult{Category: "tv mounts"} coerceCategoryToCompanyUniqueID(&out, names, valid) if out.Category != "28" { t.Fatalf("case-insensitive coerce: Category=%q want 28", out.Category) } // Already a unique_id — unchanged. out = StepResult{Category: "50"} coerceCategoryToCompanyUniqueID(&out, names, valid) if out.Category != "50" { t.Fatalf("uid passthrough: Category=%q", out.Category) } // Unknown token — left for filter to clear. out = StepResult{Category: "not-a-category"} coerceCategoryToCompanyUniqueID(&out, names, valid) if out.Category != "not-a-category" { t.Fatalf("unknown should stay until filter: %q", out.Category) } filterCategoryIfInvalid(&out, valid) if out.Category != "" { t.Fatalf("unknown should clear: %q", out.Category) } } func TestResolveCompanyCategoryUniqueID_usedByV1Projection(t *testing.T) { t.Parallel() names := map[string]string{"50": "Štedilniki"} valid := map[string]struct{}{"50": {}} // Poll/projection path: mapped name-only token must resolve like processOne. if got := resolveCompanyCategoryUniqueID("Štedilniki", names, valid); got != "50" { t.Fatalf("got %q want 50", got) } if got := resolveCompanyCategoryUniqueID("50", names, valid); got != "50" { t.Fatalf("uid passthrough got %q", got) } } func TestCategoryUniqueIDFromAny_nestedNameOnly(t *testing.T) { t.Parallel() got := categoryUniqueIDFromAny(map[string]any{"name": "Štedilniki"}) if got != "Štedilniki" { t.Fatalf("got %q want Štedilniki", got) } // unique_id still wins over name. got = categoryUniqueIDFromAny(map[string]any{"unique_id": "50", "name": "Štedilniki"}) if got != "50" { t.Fatalf("got %q want 50", got) } } func TestLoadV1ProcessJobItems_resolvesCategoryName(t *testing.T) { dsn := os.Getenv("DATABASE_URL") if dsn == "" { t.Skip("DATABASE_URL not set") } ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) defer cancel() pg, err := pgxpool.New(ctx, dsn) if err != nil { t.Fatal(err) } defer pg.Close() companyID := uuid.New() userID := uuid.New() rawID := uuid.New() jobID := uuid.New() ppID := uuid.New() gtin := "cat-v1-" + companyID.String()[:8] if _, err := pg.Exec(ctx, `INSERT INTO companies (id, name) VALUES ($1, $2)`, companyID, "cat-v1-test"); err != nil { t.Fatal(err) } // users.id may be required for processing_jobs — reuse an existing user when possible. err = pg.QueryRow(ctx, `SELECT id FROM users ORDER BY created_at DESC LIMIT 1`).Scan(&userID) if err != nil { t.Skip("no users rows available") } defer func() { _, _ = pg.Exec(context.Background(), `DELETE FROM processing_job_products WHERE job_id = $1`, jobID) _, _ = pg.Exec(context.Background(), `DELETE FROM processing_jobs WHERE id = $1`, jobID) _, _ = pg.Exec(context.Background(), `DELETE FROM processed_products WHERE company_id = $1`, companyID) _, _ = pg.Exec(context.Background(), `DELETE FROM raw_products WHERE company_id = $1`, companyID) _, _ = pg.Exec(context.Background(), `DELETE FROM categories WHERE company_id = $1`, companyID) _, _ = pg.Exec(context.Background(), `DELETE FROM companies WHERE id = $1`, companyID) }() if _, err := pg.Exec(ctx, ` INSERT INTO categories (id, company_id, name, unique_id, is_active) VALUES (gen_random_uuid(), $1, 'Štedilniki', '50', true)`, companyID); err != nil { t.Fatal(err) } if _, err := pg.Exec(ctx, ` INSERT INTO raw_products (id, company_id, gtin, raw_data, mapped_data, is_processed, processing_status) VALUES ($1, $2, $3, '{}'::jsonb, '{"category":"50","name":"Cooker"}'::jsonb, true, 'processed')`, rawID, companyID, gtin); err != nil { t.Fatal(err) } if _, err := pg.Exec(ctx, ` INSERT INTO processed_products ( id, company_id, product_id, name, category, description, processed_name, processed_description, raw_product_id, status, attributes, processed_attributes, field_sources ) VALUES ( $1, $2, $3, 'Cooker', '50', 'stove', 'Cooker', 'stove', $4, 'needs_review', '{}'::jsonb, '{}'::jsonb, '{"category":"mapped"}'::jsonb )`, ppID, companyID, gtin, rawID); err != nil { t.Fatal(err) } if _, err := pg.Exec(ctx, ` INSERT INTO processing_jobs (id, company_id, user_id, status, processing_type, total_products, processed_products) VALUES ($1, $2, $3, 'completed', 'full', 1, 1)`, jobID, companyID, userID); err != nil { t.Fatal(err) } if _, err := pg.Exec(ctx, ` INSERT INTO processing_job_products (id, job_id, raw_product_id, processed_product_id, status) VALUES (gen_random_uuid(), $1, $2, $3, 'processed')`, jobID, rawID, ppID); err != nil { t.Fatal(err) } p := NewPipeline(pg) items, err := p.LoadV1ProcessJobItems(ctx, companyID, jobID, "full") if err != nil { t.Fatal(err) } if len(items) != 1 { t.Fatalf("items=%d want 1", len(items)) } if got := fmt.Sprint(items[0]["category"]); got != "50" { t.Fatalf("category=%v want 50", items[0]["category"]) } if got := fmt.Sprint(items[0]["category_name"]); got != "Štedilniki" { t.Fatalf("category_name=%v want Štedilniki", items[0]["category_name"]) } } type stubVectorCategorizer struct { enabled bool cat string err error calls int } func (s *stubVectorCategorizer) Enabled() bool { return s.enabled } func (s *stubVectorCategorizer) SuggestCategory(context.Context, string, string, []string) (string, error) { s.calls++ return s.cat, s.err } func TestRunSteps_vectorCategoryRequiresAllowAI(t *testing.T) { vec := &stubVectorCategorizer{enabled: true, cat: "28"} e := &Engine{Vector: vec} out, err := e.RunSteps(context.Background(), "co", ProductInput{ Name: "TV mount", Mapped: map[string]any{ "name": "TV mount", }, }, "normalize_only", nil, StepPolicy{AllowAI: false}) if err != nil { t.Fatal(err) } if out.Category != "" { t.Fatalf("category=%q want empty when AllowAI=false", out.Category) } if vec.calls != 0 { t.Fatalf("vector calls=%d want 0", vec.calls) } found := false for _, n := range out.Notes { if strings.Contains(n, "category: unset") && strings.Contains(n, "AI/vector not allowed") { found = true break } } if !found { t.Fatalf("expected unset note, got %v", out.Notes) } } func TestRunSteps_vectorCategoryWhenAllowAI(t *testing.T) { vec := &stubVectorCategorizer{enabled: true, cat: "28"} e := &Engine{Vector: vec} out, err := e.RunSteps(context.Background(), "co", ProductInput{ Name: "TV mount", Mapped: map[string]any{ "name": "TV mount", }, }, "normalize_only", nil, StepPolicy{AllowAI: true}) if err != nil { t.Fatal(err) } if out.Category != "28" { t.Fatalf("category=%q want 28", out.Category) } if src, _ := out.FieldSources["category"].(string); src != "vector" { t.Fatalf("field_sources.category=%v want vector", out.FieldSources["category"]) } if vec.calls != 1 { t.Fatalf("vector calls=%d want 1", vec.calls) } } func TestRunSteps_mappedCategorySkipsVector(t *testing.T) { vec := &stubVectorCategorizer{enabled: true, cat: "99"} e := &Engine{Vector: vec} out, err := e.RunSteps(context.Background(), "co", ProductInput{ Mapped: map[string]any{ "name": "Cooker", "category": "50", }, }, "normalize_only", nil, StepPolicy{AllowAI: true}) if err != nil { t.Fatal(err) } if out.Category != "50" { t.Fatalf("category=%q want 50", out.Category) } if vec.calls != 0 { t.Fatalf("vector should not run when mapped unique_id present: calls=%d", vec.calls) } } func TestBackfillProcessedCategoriesFromMapped(t *testing.T) { dsn := os.Getenv("DATABASE_URL") if dsn == "" { t.Skip("DATABASE_URL not set") } ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) defer cancel() pg, err := pgxpool.New(ctx, dsn) if err != nil { t.Fatal(err) } defer pg.Close() companyID := uuid.New() rawID := uuid.New() ppID := uuid.New() gtin := "cat-bf-" + companyID.String()[:8] if _, err := pg.Exec(ctx, `INSERT INTO companies (id, name) VALUES ($1, $2)`, companyID, "cat-backfill-test"); err != nil { t.Fatal(err) } defer func() { _, _ = pg.Exec(context.Background(), `DELETE FROM processed_products WHERE company_id = $1`, companyID) _, _ = pg.Exec(context.Background(), `DELETE FROM raw_products WHERE company_id = $1`, companyID) _, _ = pg.Exec(context.Background(), `DELETE FROM categories WHERE company_id = $1`, companyID) _, _ = pg.Exec(context.Background(), `DELETE FROM companies WHERE id = $1`, companyID) }() if _, err := pg.Exec(ctx, ` INSERT INTO categories (id, company_id, name, unique_id, is_active) VALUES (gen_random_uuid(), $1, 'TV mounts', '28', true)`, companyID); err != nil { t.Fatal(err) } if _, err := pg.Exec(ctx, ` INSERT INTO raw_products (id, company_id, gtin, raw_data, mapped_data, is_processed, processing_status) VALUES ($1, $2, $3, '{}'::jsonb, '{"name":"Mount","category":{"unique_id":"28","name":"TV"}}'::jsonb, true, 'processed')`, rawID, companyID, gtin); err != nil { t.Fatal(err) } if _, err := pg.Exec(ctx, ` INSERT INTO processed_products ( id, company_id, product_id, name, category, description, processed_name, processed_description, raw_product_id, status, attributes, processed_attributes, field_sources ) VALUES ( $1, $2, $3, 'Mount', '', 'x', 'Mount', 'x', $4, 'needs_review', '{}'::jsonb, '{}'::jsonb, '{}'::jsonb )`, ppID, companyID, gtin, rawID); err != nil { t.Fatal(err) } res, err := BackfillProcessedCategoriesFromMapped(ctx, pg, companyID) if err != nil { t.Fatal(err) } if res.Updated != 1 { t.Fatalf("updated=%d want 1", res.Updated) } var cat, src string if err := pg.QueryRow(ctx, ` SELECT COALESCE(category, ''), COALESCE(field_sources->>'category', '') FROM processed_products WHERE id = $1`, ppID).Scan(&cat, &src); err != nil { t.Fatal(err) } if cat != "28" { t.Fatalf("category=%q want 28", cat) } if src != "mapped_backfill" { t.Fatalf("field_sources.category=%q want mapped_backfill", src) } // Idempotent. res2, err := BackfillProcessedCategoriesFromMapped(ctx, pg, companyID) if err != nil { t.Fatal(err) } if res2.Updated != 0 { t.Fatalf("second backfill updated=%d want 0", res2.Updated) } withCat, withoutCat, err := CountProcessedCategoryCoverage(ctx, pg, companyID) if err != nil { t.Fatal(err) } if withCat != 1 || withoutCat != 0 { t.Fatalf("coverage with=%d without=%d", withCat, withoutCat) } }