diff --git a/apps/api/internal/processing/catalog_fix.go b/apps/api/internal/processing/catalog_fix.go index 0210d53..fbe225b 100644 --- a/apps/api/internal/processing/catalog_fix.go +++ b/apps/api/internal/processing/catalog_fix.go @@ -37,13 +37,14 @@ func ClearWeakEnhanceHashes(ctx context.Context, pool *pgxpool.Pool, companyID u // BackfillCategoriesFromMapped copies mapped_data category unique_ids onto // processed_products.category when the processed category is empty/"none". -// Uses Go-side categoryUniqueIDFromMaps (nested/array shapes) and validates -// against the company taxonomy when categories exist. +// Uses Go-side categoryUniqueIDFromMaps (nested/array shapes), coerces display +// names onto taxonomy unique_ids, and validates against the company taxonomy +// when categories exist. func BackfillCategoriesFromMapped(ctx context.Context, pool *pgxpool.Pool, companyID uuid.UUID) (updated int, err error) { if pool == nil { return 0, fmt.Errorf("nil pool") } - valid, err := loadCompanyCategoryUniqueIDs(ctx, pool, companyID) + valid, namesByUID, err := loadCompanyCategoryTaxonomy(ctx, pool, companyID) if err != nil { return 0, err } @@ -87,10 +88,10 @@ func BackfillCategoriesFromMapped(ctx context.Context, pool *pgxpool.Pool, compa if cat == "" { continue } - if len(valid) > 0 { - if _, ok := valid[cat]; !ok { - continue - } + if resolved := resolveCompanyCategoryUniqueID(cat, namesByUID, valid); resolved != "" { + cat = resolved + } else if len(valid) > 0 { + continue } ct, err := pool.Exec(ctx, ` UPDATE processed_products @@ -121,25 +122,36 @@ func BackfillCategoriesFromMapped(ctx context.Context, pool *pgxpool.Pool, compa } func loadCompanyCategoryUniqueIDs(ctx context.Context, pool *pgxpool.Pool, companyID uuid.UUID) (map[string]struct{}, error) { - out := map[string]struct{}{} + valid, _, err := loadCompanyCategoryTaxonomy(ctx, pool, companyID) + return valid, err +} + +func loadCompanyCategoryTaxonomy(ctx context.Context, pool *pgxpool.Pool, companyID uuid.UUID) (map[string]struct{}, map[string]string, error) { + valid := map[string]struct{}{} + namesByUID := map[string]string{} rows, err := pool.Query(ctx, ` - SELECT unique_id FROM categories + SELECT unique_id, name FROM categories WHERE company_id = $1 AND COALESCE(NULLIF(btrim(unique_id), ''), '') <> ''`, companyID) if err != nil { - return nil, err + return nil, nil, err } defer rows.Close() for rows.Next() { - var uid string - if err := rows.Scan(&uid); err != nil { - return nil, err + var uid, name string + if err := rows.Scan(&uid, &name); err != nil { + return nil, nil, err } uid = strings.TrimSpace(uid) - if uid != "" { - out[uid] = struct{}{} + name = strings.TrimSpace(name) + if uid == "" { + continue + } + valid[uid] = struct{}{} + if name != "" { + namesByUID[uid] = name } } - return out, rows.Err() + return valid, namesByUID, rows.Err() } // FixCatalogHygiene clears poisoned enhance hashes and optionally backfills categories. diff --git a/apps/api/internal/processing/category.go b/apps/api/internal/processing/category.go index a85b99a..9e53522 100644 --- a/apps/api/internal/processing/category.go +++ b/apps/api/internal/processing/category.go @@ -79,6 +79,8 @@ func categoryUniqueIDFromAny(v any) string { for _, k := range []string{ "unique_id", "category_unique_id", "categoryUniqueId", "#text", "text", "id", "value", "code", + // Last: display name only (resolved to unique_id later via taxonomy). + "name", } { if s := categoryUniqueIDFromAny(t[k]); s != "" { return s @@ -163,8 +165,70 @@ func applyCategoryFromMapped(out *StepResult, maps ...map[string]any) { out.FieldSources["category"] = "mapped" } +// resolveCompanyCategoryUniqueID maps a feed/vector/prior category token onto a +// company taxonomy unique_id. Accepts an already-valid unique_id, or a display +// name (case-insensitive) from namesByUID. When valid is non-empty, the result +// must be in that set. Returns "" when unresolvable. +func resolveCompanyCategoryUniqueID(raw string, namesByUID map[string]string, valid map[string]struct{}) string { + raw = strings.TrimSpace(raw) + if raw == "" || strings.EqualFold(raw, "none") { + return "" + } + if len(valid) > 0 { + if _, ok := valid[raw]; ok { + return raw + } + } else if _, ok := namesByUID[raw]; ok { + return raw + } + if len(namesByUID) == 0 { + return "" + } + for uid, name := range namesByUID { + uid = strings.TrimSpace(uid) + if uid == "" { + continue + } + if !strings.EqualFold(strings.TrimSpace(name), raw) { + continue + } + if len(valid) == 0 { + return uid + } + if _, ok := valid[uid]; ok { + return uid + } + } + return "" +} + +// coerceCategoryToCompanyUniqueID rewrites out.Category from a display name (or +// other alias) onto the taxonomy unique_id before filterCategoryIfInvalid. +// Without this, feed/vector category names are wiped even when they match categories.name. +func coerceCategoryToCompanyUniqueID(out *StepResult, namesByUID map[string]string, valid map[string]struct{}) { + if out == nil { + return + } + cat := strings.TrimSpace(out.Category) + if cat == "" { + return + } + resolved := resolveCompanyCategoryUniqueID(cat, namesByUID, valid) + if resolved == "" || resolved == cat { + return + } + out.Category = SanitizeText(resolved) + if out.FieldSources == nil { + out.FieldSources = map[string]any{} + } + if _, ok := out.FieldSources["category"]; !ok { + out.FieldSources["category"] = "name_coerced" + } +} + // filterCategoryIfInvalid clears Category when the company has a known unique_id set // and the value is not in that set. Empty valid set means "skip validation" (tests / no taxonomy). +// Call coerceCategoryToCompanyUniqueID first so display names are not dropped. func filterCategoryIfInvalid(out *StepResult, valid map[string]struct{}) { if out == nil || len(valid) == 0 { return diff --git a/apps/api/internal/processing/category_test.go b/apps/api/internal/processing/category_test.go index 7969b56..cb2f0e5 100644 --- a/apps/api/internal/processing/category_test.go +++ b/apps/api/internal/processing/category_test.go @@ -29,7 +29,8 @@ func TestCategoryUniqueIDFromAny(t *testing.T) { {map[string]any{"category_unique_id": 48}, "48"}, {map[string]any{"#text": "120"}, "120"}, {[]any{map[string]any{"unique_id": "7"}}, "7"}, - {map[string]any{"name": "OnlyName"}, ""}, + // Nested name-only is a candidate token (resolved to unique_id later). + {map[string]any{"name": "OnlyName"}, "OnlyName"}, } for i, tc := range cases { if got := categoryUniqueIDFromAny(tc.in); got != tc.want { @@ -139,6 +140,60 @@ func TestFilterCategoryIfInvalid(t *testing.T) { } } +func TestCoerceCategoryToCompanyUniqueID_nameToUID(t *testing.T) { + t.Parallel() + names := map[string]string{"50": "Štedilniki", "28": "TV mounts"} + valid := map[string]struct{}{"50": {}, "28": {}} + + out := StepResult{Category: "Štedilniki"} + coerceCategoryToCompanyUniqueID(&out, names, valid) + if out.Category != "50" { + t.Fatalf("name coerce: Category=%q want 50", out.Category) + } + filterCategoryIfInvalid(&out, valid) + if out.Category != "50" { + t.Fatalf("after filter: Category=%q want 50", out.Category) + } + + // Case-insensitive name match. + out = StepResult{Category: "tv mounts"} + coerceCategoryToCompanyUniqueID(&out, names, valid) + if out.Category != "28" { + t.Fatalf("case-insensitive coerce: Category=%q want 28", out.Category) + } + + // Already a unique_id — unchanged. + out = StepResult{Category: "50"} + coerceCategoryToCompanyUniqueID(&out, names, valid) + if out.Category != "50" { + t.Fatalf("uid passthrough: Category=%q", out.Category) + } + + // Unknown token — left for filter to clear. + out = StepResult{Category: "not-a-category"} + coerceCategoryToCompanyUniqueID(&out, names, valid) + if out.Category != "not-a-category" { + t.Fatalf("unknown should stay until filter: %q", out.Category) + } + filterCategoryIfInvalid(&out, valid) + if out.Category != "" { + t.Fatalf("unknown should clear: %q", out.Category) + } +} + +func TestCategoryUniqueIDFromAny_nestedNameOnly(t *testing.T) { + t.Parallel() + got := categoryUniqueIDFromAny(map[string]any{"name": "Štedilniki"}) + if got != "Štedilniki" { + t.Fatalf("got %q want Štedilniki", got) + } + // unique_id still wins over name. + got = categoryUniqueIDFromAny(map[string]any{"unique_id": "50", "name": "Štedilniki"}) + if got != "50" { + t.Fatalf("got %q want 50", got) + } +} + func TestLoadV1ProcessJobItems_resolvesCategoryName(t *testing.T) { dsn := os.Getenv("DATABASE_URL") if dsn == "" { diff --git a/apps/api/internal/processing/pipeline.go b/apps/api/internal/processing/pipeline.go index b396a70..b1354bc 100644 --- a/apps/api/internal/processing/pipeline.go +++ b/apps/api/internal/processing/pipeline.go @@ -1462,6 +1462,17 @@ func (p *Pipeline) processOne(ctx context.Context, companyID, jobID uuid.UUID, i if cat := categoryUniqueIDFromMaps(enriched, raw); cat != "" { enriched["category"] = cat } + // Resolve display names → taxonomy unique_id before RunSteps so category + // formulas, enhance overlays, and attribute allowlists key correctly. + if cache != nil { + if resolved := resolveCompanyCategoryUniqueID( + stringFromAny(enriched["category"]), + cache.categoryNamesByUID, + cache.categoryUniqueIDs, + ); resolved != "" { + enriched["category"] = resolved + } + } if gtin == "" { gtin = stringFromMap(enriched, "gtin", "ean", "EAN", "upc") } @@ -1528,8 +1539,10 @@ func (p *Pipeline) processOne(ctx context.Context, companyID, jobID uuid.UUID, i return false, 0, StepResult{}, err } // Persist mapped unique_id only when it exists in the company taxonomy. - // Empty taxonomy set skips filtering (unit tests / companies without categories). + // Coerce display names (feed/vector) → unique_id before filtering so A1-style + // name tokens are not wiped. Empty taxonomy set skips filtering (unit tests). if cache != nil { + coerceCategoryToCompanyUniqueID(&result, cache.categoryNamesByUID, cache.categoryUniqueIDs) filterCategoryIfInvalid(&result, cache.categoryUniqueIDs) syncCategoryName(&result, cache.categoryNamesByUID) } @@ -1577,7 +1590,7 @@ func (p *Pipeline) processOne(ctx context.Context, companyID, jobID uuid.UUID, i cache.noteCreditDebit(p.Billing.EstimateDebit(ctx, "product_processing", result.TotalTokens)) } - processedID, err := p.upsertProcessedProduct(ctx, tx, companyID, it.RawID, gtin, result, attrsJSON, procAttrsJSON, gptJSON, sourcesJSON, providerMode) + processedID, err := p.upsertProcessedProduct(ctx, tx, companyID, it.RawID, gtin, result, attrsJSON, procAttrsJSON, gptJSON, sourcesJSON, providerMode, language) if err != nil { return false, 0, result, err } @@ -1705,13 +1718,18 @@ func (p *Pipeline) upsertProcessedProduct( result StepResult, attrsJSON, procAttrsJSON, gptJSON, sourcesJSON []byte, providerMode string, + primaryLang string, ) (uuid.UUID, error) { localized := result.LocalizedContent if localized == nil { localized = company.LocalizedContent{} } if len(localized) == 0 && (strings.TrimSpace(result.ProcessedName) != "" || strings.TrimSpace(result.ProcessedDescription) != "") { - localized[company.DefaultLanguage] = company.LocalizedFields{ + lang := company.NormalizeLanguage(primaryLang) + if lang == "" || lang == company.LangPromptAny || !company.IsAllowedLanguage(lang) { + lang = company.DefaultLanguage + } + localized[lang] = company.LocalizedFields{ ProcessedName: result.ProcessedName, ProcessedDescription: result.ProcessedDescription, } diff --git a/apps/api/internal/processing/upsert_processed_product_test.go b/apps/api/internal/processing/upsert_processed_product_test.go index edb394d..e9e6fdd 100644 --- a/apps/api/internal/processing/upsert_processed_product_test.go +++ b/apps/api/internal/processing/upsert_processed_product_test.go @@ -116,7 +116,7 @@ func TestUpsertProcessedProduct_concurrentIdempotent(t *testing.T) { for i := 0; i < n; i++ { go func() { defer wg.Done() - id, err := p.upsertProcessedProduct(ctx, nil, companyID, rawID, gtin, result, emptyJSON, emptyJSON, emptyJSON, emptyJSON, AIProviderInternal) + id, err := p.upsertProcessedProduct(ctx, nil, companyID, rawID, gtin, result, emptyJSON, emptyJSON, emptyJSON, emptyJSON, AIProviderInternal, "en") if err != nil { errCh <- err return