foo
\nAttrs missing` + if !CategoryEnhancePromptNeedsRepair(legacy) { + t.Fatal("legacy HTML prompt should need repair") + } +} + +func TestBuiltInProductEnhanceUsesSharedUserTemplate(t *testing.T) { + t.Parallel() + def, ok := DefaultFor(KeyProductEnhance) + if !ok { + t.Fatal("missing product_enhance default") + } + if def.UserTemplate != CategoryEnhanceUserTemplate { + t.Fatalf("UserTemplate must be CategoryEnhanceUserTemplate") + } +} diff --git a/apps/api/internal/aiprompts/service.go b/apps/api/internal/aiprompts/service.go index e03036c..0e395ea 100644 --- a/apps/api/internal/aiprompts/service.go +++ b/apps/api/internal/aiprompts/service.go @@ -87,8 +87,11 @@ func (s *Service) GetBundle(ctx context.Context, companyID uuid.UUID, language s }, nil } -// Resolve returns the effective templates for one key + language -// (custom if enabled for lang, else built-in). No cross-language company fallback. +// Resolve returns the effective templates for one key + language. +// Fallback: requested lang → "*" → company primary language → built-in. +// Language on the result is always the requested (normalized) lang so {{language}} +// still resolves to the content language being generated — templates are shared, +// not duplicated per language. func (s *Service) Resolve(ctx context.Context, companyID uuid.UUID, key, language string) (Resolved, error) { if !ValidPromptKey(key) { return Resolved{}, ErrInvalidKey @@ -101,11 +104,19 @@ func (s *Service) Resolve(ctx context.Context, companyID uuid.UUID, key, languag if err != nil { lang = company.DefaultLanguage } - st, err := s.loadOne(ctx, companyID, key, lang) - if err != nil && !errors.Is(err, pgx.ErrNoRows) { - return Resolved{}, err + primary := company.LoadLanguage(ctx, s.Pool, companyID) + candidates := []string{lang, company.LangPromptAny} + if primary != "" && primary != lang && primary != company.LangPromptAny { + candidates = append(candidates, primary) } - if err == nil && st.isEnabled { + for _, cand := range candidates { + st, err := s.loadOne(ctx, companyID, key, cand) + if err != nil && !errors.Is(err, pgx.ErrNoRows) { + return Resolved{}, err + } + if err != nil || !st.isEnabled { + continue + } sys := strings.TrimSpace(st.systemTemplate) user := strings.TrimSpace(st.userTemplate) if sys == "" { @@ -155,9 +166,15 @@ func (s *Service) Update(ctx context.Context, companyID uuid.UUID, in UpdateInpu if langRaw == "" { langRaw = defaultLang } - lang, err := company.ParseLanguage(langRaw, false) - if err != nil { - return Bundle{}, fmt.Errorf("%w: language %q", ErrInvalidInput, langRaw) + var lang string + if langRaw == company.LangPromptAny { + lang = company.LangPromptAny + } else { + parsed, err := company.ParseLanguage(langRaw, false) + if err != nil { + return Bundle{}, fmt.Errorf("%w: language %q", ErrInvalidInput, langRaw) + } + lang = parsed } lastLang = lang if item.Reset { @@ -196,7 +213,11 @@ func (s *Service) Update(ctx context.Context, companyID uuid.UUID, in UpdateInpu if err := tx.Commit(ctx); err != nil { return Bundle{}, err } - return s.GetBundle(ctx, companyID, lastLang) + bundleLang := lastLang + if bundleLang == company.LangPromptAny { + bundleLang = defaultLang + } + return s.GetBundle(ctx, companyID, bundleLang) } func (s *Service) loadAll(ctx context.Context, companyID uuid.UUID) ([]stored, error) { diff --git a/apps/api/internal/aiprovider/resolve_platform_test.go b/apps/api/internal/aiprovider/resolve_platform_test.go index 65bc1f4..aec687e 100644 --- a/apps/api/internal/aiprovider/resolve_platform_test.go +++ b/apps/api/internal/aiprovider/resolve_platform_test.go @@ -61,3 +61,15 @@ func TestResolvePlatformOpenAI_viaPlatformService(t *testing.T) { t.Fatalf("source=%q", oi.Source) } } + +func TestPlatformModeLabel(t *testing.T) { + if got := platformModeLabel(platformsettings.SourceDB, "https://llm.overloadedbot.org/v1"); got != ModeInternalLabel { + t.Fatalf("db remote: %q", got) + } + if got := platformModeLabel(platformsettings.SourceEnv, "https://api.openai.com/v1"); got != ModeCustomLabel { + t.Fatalf("env remote: %q", got) + } + if got := platformModeLabel(platformsettings.SourceDB, "http://127.0.0.1:18767/v1"); got != ModeCustomLabel { + t.Fatalf("mock loopback: %q", got) + } +} diff --git a/apps/api/internal/aiprovider/roles.go b/apps/api/internal/aiprovider/roles.go index aab090e..fa33445 100644 --- a/apps/api/internal/aiprovider/roles.go +++ b/apps/api/internal/aiprovider/roles.go @@ -4,6 +4,7 @@ import ( "context" "strings" + "github.com/descrybe/descrybe-v2/apps/api/internal/platformsettings" "github.com/descrybe/descrybe-v2/apps/api/internal/processing" "github.com/google/uuid" ) @@ -134,7 +135,7 @@ func (s *Service) resolvePlatformRoleCompleter(ctx context.Context, role string) BaseURL: cfg.BaseURL, Model: model, UsingBYOK: false, - ModeLabel: ModeInternalLabel, + ModeLabel: platformModeLabel(cfg.Source, cfg.BaseURL), }) } } @@ -143,13 +144,14 @@ func (s *Service) resolvePlatformRoleCompleter(ctx context.Context, role string) if role == RoleProcessing { key := strings.TrimSpace(s.Env.OpenAIAPIKey) model := strings.TrimSpace(s.Env.OpenAIModel) + baseURL := strings.TrimSpace(s.Env.OpenAIBaseURL) if key != "" && model != "" { return s.completerFromEndpoint(RoleEndpoint{ APIKey: key, - BaseURL: strings.TrimSpace(s.Env.OpenAIBaseURL), + BaseURL: baseURL, Model: model, UsingBYOK: false, - ModeLabel: ModeInternalLabel, + ModeLabel: platformModeLabel(platformsettings.SourceEnv, baseURL), }) } } diff --git a/apps/api/internal/aiprovider/service.go b/apps/api/internal/aiprovider/service.go index 1582a64..e783194 100644 --- a/apps/api/internal/aiprovider/service.go +++ b/apps/api/internal/aiprovider/service.go @@ -277,6 +277,7 @@ func (s *Service) Resolve(ctx context.Context, companyID uuid.UUID) (Resolved, e if strings.TrimSpace(platform.APIKey) == "" { return Resolved{ModeLabel: ModeInternalLabel, UsingBYOK: false}, nil } + modeLabel := platformModeLabel(platform.Source, platform.BaseURL) client := processing.NewOpenAIClient( platform.APIKey, platform.BaseURL, @@ -284,17 +285,29 @@ func (s *Service) Resolve(ctx context.Context, companyID uuid.UUID) (Resolved, e rpm, retries, ) - client.ModeLabel = ModeInternalLabel + client.ModeLabel = modeLabel if s.HTTPClient != nil { client.HTTPClient = s.HTTPClient } return Resolved{ Completer: client, - ModeLabel: ModeInternalLabel, + ModeLabel: modeLabel, UsingBYOK: false, }, nil } +// platformModeLabel maps platform OpenAI source/base to analytics ModeLabel. +// DB-backed admin settings → internal; env bootstrap or mock/loopback → custom. +func platformModeLabel(source, baseURL string) string { + if processing.IsMockOrLoopbackBaseURL(baseURL) { + return ModeCustomLabel + } + if strings.TrimSpace(source) == platformsettings.SourceEnv { + return ModeCustomLabel + } + return ModeInternalLabel +} + func (s *Service) platformConfigured(ctx context.Context) (bool, error) { oi, err := s.resolvePlatformOpenAI(ctx) if err != nil { diff --git a/apps/api/internal/catalog/clone.go b/apps/api/internal/catalog/clone.go index 7dc180a..57488a4 100644 --- a/apps/api/internal/catalog/clone.go +++ b/apps/api/internal/catalog/clone.go @@ -9,6 +9,10 @@ import ( "github.com/jackc/pgx/v5/pgxpool" ) +// cloneProcessedFieldSourcesSQL copies field_sources but drops enhance_input_hash +// so the destination's first process cannot hash-skip a thin prior description. +const cloneProcessedFieldSourcesSQL = "COALESCE(pp.field_sources, '{}'::jsonb) - 'enhance_input_hash'" + // CloneResult summarizes a company catalog clone (source unchanged). type CloneResult struct { SourceCompanyID uuid.UUID `json:"source_company_id"` @@ -382,7 +386,7 @@ func cloneCompanyCatalog(ctx context.Context, pool *pgxpool.Pool, src, dest uuid pp.meta_description, pp.last_transition_at, pp.structured_description, - pp.field_sources, + ` + cloneProcessedFieldSourcesSQL + `, pp.created_at, now(), COALESCE(pp.ai_provider_mode, 'internal'), @@ -452,6 +456,14 @@ func cloneCompanyCatalog(ctx context.Context, pool *pgxpool.Pool, src, dest uuid return nil, fmt.Errorf("reset raw processing flags: %w", err) } + // Drop skip hashes still present on weak / title-echo localized descriptions + // (field_sources hash is already stripped in the INSERT above). + hashRepair, err := repairWeakEnhanceHashesTx(ctx, tx, dest) + if err != nil { + return nil, fmt.Errorf("repair weak enhance hashes: %w", err) + } + add("weak_enhance_hashes_cleared", hashRepair.Cleared) + if err := tx.Commit(ctx); err != nil { return nil, err } diff --git a/apps/api/internal/catalog/clone_test.go b/apps/api/internal/catalog/clone_test.go index 3195d41..d1f2940 100644 --- a/apps/api/internal/catalog/clone_test.go +++ b/apps/api/internal/catalog/clone_test.go @@ -1,6 +1,7 @@ package catalog import ( + "strings" "testing" "github.com/google/uuid" @@ -24,3 +25,14 @@ func TestValidateCloneCompanies(t *testing.T) { t.Fatalf("unexpected error: %v", err) } } + +func TestCloneFieldSourcesSQLClearsEnhanceHash(t *testing.T) { + t.Parallel() + const want = "enhance_input_hash" + if !strings.Contains(cloneProcessedFieldSourcesSQL, want) { + t.Fatalf("clone field_sources SQL must remove %q; got %q", want, cloneProcessedFieldSourcesSQL) + } + if !strings.Contains(cloneProcessedFieldSourcesSQL, "-") { + t.Fatalf("expected jsonb key removal operator in %q", cloneProcessedFieldSourcesSQL) + } +} diff --git a/apps/api/internal/catalog/fix_catalog.go b/apps/api/internal/catalog/fix_catalog.go new file mode 100644 index 0000000..68e02cb --- /dev/null +++ b/apps/api/internal/catalog/fix_catalog.go @@ -0,0 +1,76 @@ +package catalog + +import ( + "context" + "fmt" + + "github.com/descrybe/descrybe-v2/apps/api/internal/billing" + "github.com/google/uuid" + "github.com/jackc/pgx/v5/pgxpool" +) + +// EnsureCategoryAttributeLinks removes orphan category_attributes rows (missing +// category or attribute) without wiping taxonomy or valid links. Returns orphans +// removed and remaining link count. +func EnsureCategoryAttributeLinks(ctx context.Context, pool *pgxpool.Pool, companyID uuid.UUID) (orphansRemoved int, links int, err error) { + if pool == nil { + return 0, 0, fmt.Errorf("nil pool") + } + ct, err := pool.Exec(ctx, ` + DELETE FROM category_attributes ca + WHERE ca.company_id = $1 + AND ( + NOT EXISTS ( + SELECT 1 FROM categories c + WHERE c.company_id = ca.company_id AND c.unique_id = ca.category_unique_id + ) + OR NOT EXISTS ( + SELECT 1 FROM attributes a + WHERE a.id = ca.attribute_id AND a.company_id = ca.company_id + ) + )`, companyID) + if err != nil { + return 0, 0, fmt.Errorf("purge orphan category_attributes: %w", err) + } + orphansRemoved = int(ct.RowsAffected()) + if err := pool.QueryRow(ctx, ` + SELECT count(*)::int FROM category_attributes WHERE company_id = $1`, companyID).Scan(&links); err != nil { + return orphansRemoved, 0, err + } + return orphansRemoved, links, nil +} + +// RepairCompanyCategoryEnhancePrompts is the company-scoped variant of +// RepairA1DemoCategoryEnhancePrompts: same repairedCategoryEnhancePromptMap +// (sl + "*" with CategoryEnhanceUserTemplate / {{attrs}}), applied to one company. +// Idempotent — already-OK categories count as already_ok, not updated. +func RepairCompanyCategoryEnhancePrompts(ctx context.Context, pool *pgxpool.Pool, companyID uuid.UUID) (updated, alreadyOK, emptySkipped int, err error) { + if pool == nil { + return 0, 0, 0, fmt.Errorf("nil pool") + } + want, err := repairedCategoryEnhancePromptMap() + if err != nil { + return 0, 0, 0, err + } + summary := &RepairCategoryEnhancePromptsResult{ByCompany: map[string]int{}} + if _, err := repairCompanyCategoryEnhancePrompts(ctx, pool, companyID, companyID.String(), want, false, summary); err != nil { + return 0, 0, 0, err + } + return summary.Updated, summary.AlreadyOK, summary.EmptySkipped, nil +} + +// ResolvePlatformDemoCompanyID finds the Platform Demo sandbox (never A1 cohort). +func ResolvePlatformDemoCompanyID(ctx context.Context, pool *pgxpool.Pool) (uuid.UUID, error) { + var id uuid.UUID + err := pool.QueryRow(ctx, ` + SELECT c.id + FROM companies c + WHERE c.name = $1 + AND COALESCE(c.legacy_company_id, '') <> $2 + ORDER BY c.created_at ASC + LIMIT 1`, platformDemoCompanyName, billing.A1LegacyCompanyID).Scan(&id) + if err != nil { + return uuid.Nil, fmt.Errorf("platform demo company not found: %w", err) + } + return id, nil +} diff --git a/apps/api/internal/catalog/fix_catalog_test.go b/apps/api/internal/catalog/fix_catalog_test.go new file mode 100644 index 0000000..5a9c47c --- /dev/null +++ b/apps/api/internal/catalog/fix_catalog_test.go @@ -0,0 +1,45 @@ +package catalog + +import ( + "strings" + "testing" + + "github.com/descrybe/descrybe-v2/apps/api/internal/aiprompts" + "github.com/descrybe/descrybe-v2/apps/api/internal/company" +) + +func TestCategoryEnhanceTemplateHasAttrs(t *testing.T) { + t.Parallel() + tpl := aiprompts.CategoryEnhanceUserTemplate + if !strings.Contains(tpl, "{{attrs}}") { + t.Fatalf("template missing {{attrs}}: %q", tpl) + } + if !aiprompts.CategoryEnhancePromptNeedsRepair("legacy HTML prompt without attrs") { + t.Fatal("expected legacy prompt to need repair") + } + if aiprompts.CategoryEnhancePromptNeedsRepair(tpl) { + t.Fatal("canonical template should not need repair") + } +} + +func TestPlatformDemoNameConstant(t *testing.T) { + t.Parallel() + if platformDemoCompanyName != "Platform Demo" { + t.Fatalf("got %q", platformDemoCompanyName) + } +} + +func TestRepairedCategoryEnhancePromptMapMatchesA1Demo(t *testing.T) { + t.Parallel() + want, err := repairedCategoryEnhancePromptMap() + if err != nil { + t.Fatal(err) + } + tpl := strings.TrimSpace(aiprompts.CategoryEnhanceUserTemplate) + if want["sl"] != tpl || want[company.LangPromptAny] != tpl { + t.Fatalf("want sl+* template, got %#v", want) + } + if !strings.Contains(tpl, "{{attrs}}") { + t.Fatal("template must include {{attrs}}") + } +} diff --git a/apps/api/internal/catalog/product_v1_align.go b/apps/api/internal/catalog/product_v1_align.go new file mode 100644 index 0000000..043dfac --- /dev/null +++ b/apps/api/internal/catalog/product_v1_align.go @@ -0,0 +1,66 @@ +package catalog + +import ( + "strings" + + "github.com/descrybe/descrybe-v2/apps/api/internal/eprel" +) + +// alignProductFieldsWithV1 mirrors V1 process-item field semantics on dashboard +// product payloads for the same EAN: title/name, preferred description, and +// structured eprel. Mutates item in place after SQL scan / feed-spec linking. +func alignProductFieldsWithV1(item map[string]any) { + if item == nil { + return + } + + name := firstNonEmptyString( + item["processed_name"], + item["name"], + item["title"], + ) + if name != "" { + item["name"] = name + item["title"] = name + } + + desc := firstNonEmptyString( + item["processed_description"], + item["description"], + ) + if desc != "" { + item["description"] = desc + } + + eprelVal := eprel.ExtractFromAttrs(asStringAnyMap(item["processed_attributes"])) + if eprelVal == nil { + eprelVal = eprel.ExtractFromAttrs(asStringAnyMap(item["attributes"])) + } + if eprelVal == nil { + if mapped, ok := item["mapped_data"].(map[string]any); ok { + eprelVal = eprel.ExtractFromAttrs(mapped) + } + } + item["eprel"] = eprelVal + if eprelVal != nil { + item["has_eprel"] = true + } +} + +func firstNonEmptyString(vals ...any) string { + for _, v := range vals { + switch t := v.(type) { + case string: + if s := strings.TrimSpace(t); s != "" { + return s + } + case *string: + if t != nil { + if s := strings.TrimSpace(*t); s != "" { + return s + } + } + } + } + return "" +} diff --git a/apps/api/internal/catalog/product_v1_align_test.go b/apps/api/internal/catalog/product_v1_align_test.go new file mode 100644 index 0000000..c738a34 --- /dev/null +++ b/apps/api/internal/catalog/product_v1_align_test.go @@ -0,0 +1,123 @@ +package catalog + +import ( + "testing" + + "github.com/descrybe/descrybe-v2/apps/api/internal/eprel" +) + +func TestAlignProductFieldsWithV1_prefersProcessedAndEprel(t *testing.T) { + item := map[string]any{ + "name": "RAW ALLCAPS TITLE", + "processed_name": "Retail Title", + "description": "raw feed description", + "processed_description": "AI retail description", + "attributes": map[string]any{ + "vzmetenje": "junk", + "brand": "Ostalo", + }, + "processed_attributes": map[string]any{ + "brand": "Ostalo", + "product_model": "W53070", + "eprel_id": "1632113", + "eprel_label": "https://eprel.example/label", + "eprel_energy_class": "A", + }, + "category_name": "Nosilci za TV", + } + alignProductFieldsWithV1(item) + + if item["name"] != "Retail Title" { + t.Fatalf("name=%v want Retail Title", item["name"]) + } + if item["title"] != "Retail Title" { + t.Fatalf("title=%v want Retail Title", item["title"]) + } + if item["description"] != "AI retail description" { + t.Fatalf("description=%v", item["description"]) + } + ep, ok := item["eprel"].(map[string]any) + if !ok { + t.Fatalf("eprel type=%T", item["eprel"]) + } + if ep["id"] != "1632113" || ep["energy_class"] != "A" { + t.Fatalf("eprel=%v", ep) + } + if item["has_eprel"] != true { + t.Fatalf("has_eprel=%v", item["has_eprel"]) + } + // Feed attrs stay available for dual-pane UI; V1 poll uses processed_attributes. + attrs, _ := item["attributes"].(map[string]any) + if attrs["vzmetenje"] != "junk" { + t.Fatalf("feed attributes should remain: %v", attrs) + } + if item["category_name"] != "Nosilci za TV" { + t.Fatalf("category_name mutated") + } +} + +func TestAlignProductFieldsWithV1_matchesProcessItemSemantics(t *testing.T) { + // Same underlying row as a V1 process item would project for one EAN. + processedName := "Pralni stroj Samsung WW90CGC04DAELE, 9kg" + processedDesc := "Tehnologija Ecobubble…" + processedAttrs := map[string]any{ + "brand": "Samsung", + "product_model": "WW90CGC04DAELE", + "eprel_id": "1632113", + "eprel_label": "https://eprel.example/label", + "eprel_pdf": "https://eprel.example/fiche.pdf", + "eprel_energy_class": "A", + "eprel_energy_scale": "A_G", + } + dashboard := map[string]any{ + "name": "RAW FEED TITLE", + "processed_name": processedName, + "description": "raw description", + "processed_description": processedDesc, + "category": "40", + "category_name": "Pralni stroji", + "attributes": map[string]any{"dodatne-informacije": "AI", "brand": "Samsung"}, + "processed_attributes": processedAttrs, + "gtin": "8806095210711", + } + alignProductFieldsWithV1(dashboard) + + // V1 process item projection (subset used by poll clients). + v1Title := processedName + v1Desc := processedDesc + v1Eprel := eprel.ExtractFromAttrs(processedAttrs) + + if dashboard["name"] != v1Title || dashboard["title"] != v1Title { + t.Fatalf("title/name mismatch: dash=%v/%v v1=%v", dashboard["title"], dashboard["name"], v1Title) + } + if dashboard["description"] != v1Desc { + t.Fatalf("description mismatch: dash=%v v1=%v", dashboard["description"], v1Desc) + } + if dashboard["category_name"] != "Pralni stroji" { + t.Fatalf("category_name=%v", dashboard["category_name"]) + } + dashEprel, _ := dashboard["eprel"].(map[string]any) + v1Map, _ := v1Eprel.(map[string]any) + if dashEprel["id"] != v1Map["id"] || dashEprel["energy_class"] != v1Map["energy_class"] { + t.Fatalf("eprel mismatch dash=%v v1=%v", dashEprel, v1Map) + } + // V1 attributes ≈ processed_attributes (allowlist/sanitize aside). + pa, _ := dashboard["processed_attributes"].(map[string]any) + if pa["brand"] != "Samsung" || pa["product_model"] != "WW90CGC04DAELE" { + t.Fatalf("processed_attributes=%v", pa) + } +} + +func TestAlignProductFieldsWithV1_eprelFromMappedFallback(t *testing.T) { + item := map[string]any{ + "processed_name": "Washer", + "mapped_data": map[string]any{ + "eprel_id": "1632113", + }, + } + alignProductFieldsWithV1(item) + ep, ok := item["eprel"].(map[string]any) + if !ok || ep["id"] != "1632113" { + t.Fatalf("eprel=%v", item["eprel"]) + } +} diff --git a/apps/api/internal/catalog/prompt_repair.go b/apps/api/internal/catalog/prompt_repair.go new file mode 100644 index 0000000..844f809 --- /dev/null +++ b/apps/api/internal/catalog/prompt_repair.go @@ -0,0 +1,220 @@ +package catalog + +import ( + "context" + "fmt" + "strings" + + "github.com/descrybe/descrybe-v2/apps/api/internal/aiprompts" + "github.com/descrybe/descrybe-v2/apps/api/internal/billing" + "github.com/descrybe/descrybe-v2/apps/api/internal/company" + "github.com/google/uuid" + "github.com/jackc/pgx/v5/pgxpool" +) + +// Canonical Postgres id for the migrated A1 Slovenija tenant (seed-a1 default). +const a1CompanyID = "604f23a8-b66e-4b21-8b45-0d72b68f4790" + +// platformDemoCompanyName matches migrator / seed-demo standalone demo tenant. +const platformDemoCompanyName = "Platform Demo" + +// RepairCategoryEnhancePromptsResult is the dry-run / apply summary for +// RepairA1DemoCategoryEnhancePrompts. +type RepairCategoryEnhancePromptsResult struct { + CompaniesScanned int `json:"companies_scanned"` + CategoriesSeen int `json:"categories_seen"` + WouldUpdate int `json:"would_update"` + Updated int `json:"updated"` + AlreadyOK int `json:"already_ok"` + EmptySkipped int `json:"empty_skipped"` + ByCompany map[string]int `json:"by_company"` + DryRun bool `json:"dry_run"` +} + +// RepairA1DemoCategoryEnhancePrompts replaces legacy HTML marketing categories.prompt +// values for A1 Slovenija + Platform Demo with aiprompts.CategoryEnhanceUserTemplate. +// +// LOCAL repair only (idempotent): +// - Writes JSON-compatible user overlays under both "sl" (prompt->>'sl') and "*" +// (LangPromptAny) so language stays via {{language}}, not hardcoded-only copy. +// - Touches ONLY categories.prompt — never title_template / description_template +// (unique name/description formulas stay intact; AppendFormulaConstraints encodes +// them as plain-text instructions at enhance render time). +// - dryRun=true: count WouldUpdate only; dryRun=false: apply and set Updated. +// +// Entrypoint: go run ./cmd/repair-category-prompts (-dry-run | -apply). +func RepairA1DemoCategoryEnhancePrompts(ctx context.Context, pool *pgxpool.Pool, dryRun bool) (RepairCategoryEnhancePromptsResult, error) { + out := RepairCategoryEnhancePromptsResult{ + DryRun: dryRun, + ByCompany: map[string]int{}, + } + if pool == nil { + return out, fmt.Errorf("postgres pool is required") + } + + a1ID, err := uuid.Parse(a1CompanyID) + if err != nil { + return out, fmt.Errorf("a1 company id: %w", err) + } + + companyRows, err := pool.Query(ctx, ` + SELECT id, name + FROM companies + WHERE id = $1 + OR name = $2 + OR COALESCE(legacy_company_id, '') = $3 + ORDER BY name`, a1ID, platformDemoCompanyName, billing.A1LegacyCompanyID) + if err != nil { + return out, fmt.Errorf("list target companies: %w", err) + } + defer companyRows.Close() + + type co struct { + id uuid.UUID + name string + } + companies := make([]co, 0, 2) + for companyRows.Next() { + var c co + if err := companyRows.Scan(&c.id, &c.name); err != nil { + return out, err + } + companies = append(companies, c) + } + if err := companyRows.Err(); err != nil { + return out, err + } + out.CompaniesScanned = len(companies) + if len(companies) == 0 { + return out, fmt.Errorf("no A1 / Platform Demo companies found") + } + + want, err := repairedCategoryEnhancePromptMap() + if err != nil { + return out, err + } + + for _, c := range companies { + n, err := repairCompanyCategoryEnhancePrompts(ctx, pool, c.id, c.name, want, dryRun, &out) + if err != nil { + return out, err + } + if n > 0 { + out.ByCompany[c.name] = n + } + } + return out, nil +} + +// repairedCategoryEnhancePromptMap is the canonical stored shape: "sl" (prompt->>'sl' +// for A1/Demo) plus LangPromptAny ("*") so PromptForLanguage resolves for any content +// language. Copy stays language-agnostic via {{language}} — not hardcoded Slovenian. +func repairedCategoryEnhancePromptMap() (company.LangPromptMap, error) { + tpl := strings.TrimSpace(aiprompts.CategoryEnhanceUserTemplate) + if tpl == "" { + return nil, fmt.Errorf("CategoryEnhanceUserTemplate is empty") + } + return company.LangPromptMap{ + "sl": tpl, + company.LangPromptAny: tpl, + }, nil +} + +func categoryEnhancePromptMapOK(m company.LangPromptMap, want company.LangPromptMap) bool { + if !company.HasAnyPrompt(m) || !company.HasAnyPrompt(want) { + return false + } + tpl := strings.TrimSpace(want[company.LangPromptAny]) + if tpl == "" { + tpl = strings.TrimSpace(want["sl"]) + } + if tpl == "" { + return false + } + // Accept already-repaired maps: every non-empty value equals the shared template + // and LangPromptAny (or legacy sl-only) is present. + hasKey := false + for lang, p := range m { + p = strings.TrimSpace(p) + if p == "" { + continue + } + if p != tpl { + return false + } + if lang == company.LangPromptAny || lang == "sl" { + hasKey = true + } + } + return hasKey +} + +func repairCompanyCategoryEnhancePrompts( + ctx context.Context, + pool *pgxpool.Pool, + companyID uuid.UUID, + companyName string, + want company.LangPromptMap, + dryRun bool, + out *RepairCategoryEnhancePromptsResult, +) (int, error) { + rows, err := pool.Query(ctx, ` + SELECT id, COALESCE(prompt, '{}'::jsonb) + FROM categories + WHERE company_id = $1`, companyID) + if err != nil { + return 0, fmt.Errorf("list categories for %s: %w", companyName, err) + } + defer rows.Close() + + updatedHere := 0 + for rows.Next() { + var id uuid.UUID + var raw []byte + if err := rows.Scan(&id, &raw); err != nil { + return updatedHere, err + } + out.CategoriesSeen++ + + m, err := company.DecodeLangPromptMap(raw) + if err != nil { + return updatedHere, fmt.Errorf("decode prompt category=%s company=%s: %w", id, companyName, err) + } + if !company.HasAnyPrompt(m) { + out.EmptySkipped++ + continue + } + if categoryEnhancePromptMapOK(m, want) { + out.AlreadyOK++ + continue + } + + out.WouldUpdate++ + if dryRun { + updatedHere++ + continue + } + + cleaned, err := company.SanitizeLangPromptMap(want, MaxCategoryPromptRunes) + if err != nil { + return updatedHere, fmt.Errorf("sanitize prompt category=%s: %w", id, err) + } + encoded, err := company.EncodeLangPromptMap(cleaned) + if err != nil { + return updatedHere, err + } + ct, err := pool.Exec(ctx, ` + UPDATE categories + SET prompt = $3::jsonb, updated_at = now() + WHERE id = $1 AND company_id = $2`, id, companyID, string(encoded)) + if err != nil { + return updatedHere, fmt.Errorf("update prompt category=%s: %w", id, err) + } + if ct.RowsAffected() == 0 { + return updatedHere, fmt.Errorf("category %s not updated (company mismatch?)", id) + } + out.Updated++ + updatedHere++ + } + return updatedHere, rows.Err() +} diff --git a/apps/api/internal/catalog/prompt_repair_test.go b/apps/api/internal/catalog/prompt_repair_test.go new file mode 100644 index 0000000..4cb8f07 --- /dev/null +++ b/apps/api/internal/catalog/prompt_repair_test.go @@ -0,0 +1,52 @@ +package catalog + +import ( + "strings" + "testing" + + "github.com/descrybe/descrybe-v2/apps/api/internal/aiprompts" + "github.com/descrybe/descrybe-v2/apps/api/internal/company" +) + +func TestRepairedCategoryEnhancePromptMap(t *testing.T) { + t.Parallel() + want, err := repairedCategoryEnhancePromptMap() + if err != nil { + t.Fatal(err) + } + tpl := strings.TrimSpace(aiprompts.CategoryEnhanceUserTemplate) + if want["sl"] != tpl { + t.Fatalf("want prompt->>'sl' = shared template") + } + if want[company.LangPromptAny] != tpl { + t.Fatalf("want * = shared template") + } + if !categoryEnhancePromptMapOK(want, want) { + t.Fatal("canonical map should be OK") + } + legacy := company.LangPromptMap{"sl": "legacy HTML marketing
"} + if categoryEnhancePromptMapOK(legacy, want) { + t.Fatal("legacy HTML must need repair") + } + slOnly := company.LangPromptMap{"sl": tpl} + if !categoryEnhancePromptMapOK(slOnly, want) { + t.Fatal("sl-only repaired map should be OK (idempotent)") + } + starOnly := company.LangPromptMap{company.LangPromptAny: tpl} + if !categoryEnhancePromptMapOK(starOnly, want) { + t.Fatal("*-only repaired map should be OK (idempotent)") + } +} + +func TestRepairTargetsUseSharedTemplate(t *testing.T) { + t.Parallel() + tpl := strings.TrimSpace(aiprompts.CategoryEnhanceUserTemplate) + for _, v := range []string{"name", "description", "attrs", "category", "language"} { + if !strings.Contains(tpl, "{{"+v+"}}") { + t.Fatalf("shared template missing {{%s}}", v) + } + } + if a1CompanyID == "" || platformDemoCompanyName == "" { + t.Fatal("missing company target constants") + } +} diff --git a/apps/api/internal/catalog/raw_v1.go b/apps/api/internal/catalog/raw_v1.go index 4677f1d..0dd330a 100644 --- a/apps/api/internal/catalog/raw_v1.go +++ b/apps/api/internal/catalog/raw_v1.go @@ -161,12 +161,11 @@ func ExtractProductImages(mapped, raw map[string]any) (main string, more []strin break } } - if imgs, ok := merged["images"].([]any); ok && len(imgs) > 0 { - urls := coerceToURLList(imgs) - if main == "" && len(urls) > 0 { + if urls := coerceToURLList(merged["images"]); len(urls) > 0 { + if main == "" { main = urls[0] urls = urls[1:] - } else if len(urls) > 0 && urls[0] == main { + } else if urls[0] == main { urls = urls[1:] } for _, u := range urls { @@ -175,6 +174,11 @@ func ExtractProductImages(mapped, raw map[string]any) (main string, more []strin } } } + // Feeds often leave main_image empty while moreimages/images hold URLs. + if main == "" && len(more) > 0 { + main = more[0] + more = more[1:] + } if main != "" { filtered := more[:0] for _, u := range more { @@ -211,7 +215,7 @@ func coerceToURLString(value any) string { } return "" case map[string]any: - for _, k := range []string{"#text", "__cdata", "@_href", "@_url", "href", "url"} { + for _, k := range []string{"#text", "__cdata", "@_href", "@_url", "href", "url", "src", "link", "image"} { if u := coerceToURLString(v[k]); u != "" { return u } diff --git a/apps/api/internal/catalog/raw_v1_test.go b/apps/api/internal/catalog/raw_v1_test.go index 4be3f2b..4aee82f 100644 --- a/apps/api/internal/catalog/raw_v1_test.go +++ b/apps/api/internal/catalog/raw_v1_test.go @@ -54,3 +54,38 @@ func TestBuildMappedDataFromV1ItemOmitsEmptyNulls(t *testing.T) { t.Fatal("EAN-only must not count as content") } } + +func TestExtractProductImages_promotesMoreimagesWhenMainEmpty(t *testing.T) { + main, more := ExtractProductImages(map[string]any{ + "main_image": "", + "moreimages": "https://cdn.example.com/a.jpg,https://cdn.example.com/b.jpg", + }, nil) + if main != "https://cdn.example.com/a.jpg" { + t.Fatalf("main=%q", main) + } + if len(more) != 1 || more[0] != "https://cdn.example.com/b.jpg" { + t.Fatalf("more=%v", more) + } +} + +func TestExtractProductImages_imagesStringSliceAndSrcObjects(t *testing.T) { + main, more := ExtractProductImages(map[string]any{ + "images": []string{ + "https://cdn.example.com/main.jpg", + "https://cdn.example.com/2.jpg", + }, + }, nil) + if main != "https://cdn.example.com/main.jpg" || len(more) != 1 || more[0] != "https://cdn.example.com/2.jpg" { + t.Fatalf("string slice main=%q more=%v", main, more) + } + + main, more = ExtractProductImages(map[string]any{ + "images": []any{ + map[string]any{"src": "https://cdn.example.com/from-src.jpg"}, + map[string]any{"src": "https://cdn.example.com/from-src-2.jpg"}, + }, + }, nil) + if main != "https://cdn.example.com/from-src.jpg" || len(more) != 1 || more[0] != "https://cdn.example.com/from-src-2.jpg" { + t.Fatalf("src objects main=%q more=%v", main, more) + } +} diff --git a/apps/api/internal/catalog/repair_enhance_hashes.go b/apps/api/internal/catalog/repair_enhance_hashes.go new file mode 100644 index 0000000..2b65423 --- /dev/null +++ b/apps/api/internal/catalog/repair_enhance_hashes.go @@ -0,0 +1,188 @@ +package catalog + +import ( + "context" + "encoding/json" + "fmt" + "strings" + + "github.com/descrybe/descrybe-v2/apps/api/internal/company" + "github.com/google/uuid" + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgxpool" +) + +const enhanceInputHashKey = "enhance_input_hash" + +// RepairWeakEnhanceHashesResult is the detailed report for weak-hash clearing. +type RepairWeakEnhanceHashesResult struct { + Cleared int64 + Scanned int + ClearedRawIDs []uuid.UUID +} + +// RepairWeakEnhanceHashes clears enhance_input_hash from field_sources and +// localized_content for processed products whose descriptions are weak / +// title-echo / filler (so the next enhance cannot hash-skip thin priors). +// Returns the number of products updated. +func RepairWeakEnhanceHashes(ctx context.Context, pool *pgxpool.Pool, companyID uuid.UUID) (int64, error) { + res, err := RepairWeakEnhanceHashesDetailed(ctx, pool, companyID) + return res.Cleared, err +} + +// RepairWeakEnhanceHashesDetailed is RepairWeakEnhanceHashes plus scan/raw-id detail +// for admin Fix A1 hygiene reporting. +func RepairWeakEnhanceHashesDetailed(ctx context.Context, pool *pgxpool.Pool, companyID uuid.UUID) (RepairWeakEnhanceHashesResult, error) { + var out RepairWeakEnhanceHashesResult + if pool == nil { + return out, fmt.Errorf("catalog pool not configured") + } + if companyID == uuid.Nil { + return out, ClientMsg("company_id is required") + } + tx, err := pool.Begin(ctx) + if err != nil { + return out, err + } + defer tx.Rollback(ctx) + + out, err = repairWeakEnhanceHashesTx(ctx, tx, companyID) + if err != nil { + return out, err + } + if err := tx.Commit(ctx); err != nil { + return out, err + } + return out, nil +} + +// RepairWeakEnhanceHashes clears weak enhance skip hashes for the company. +func (s *Service) RepairWeakEnhanceHashes(ctx context.Context, companyID uuid.UUID) (int64, error) { + if s == nil || s.Pool == nil { + return 0, fmt.Errorf("catalog service not configured") + } + return RepairWeakEnhanceHashes(ctx, s.Pool, companyID) +} + +func repairWeakEnhanceHashesTx(ctx context.Context, tx pgx.Tx, companyID uuid.UUID) (RepairWeakEnhanceHashesResult, error) { + var out RepairWeakEnhanceHashesResult + rows, err := tx.Query(ctx, ` + SELECT id, raw_product_id, + COALESCE(name, ''), + COALESCE(description, ''), + COALESCE(processed_name, ''), + COALESCE(processed_description, ''), + COALESCE(field_sources, '{}'::jsonb), + COALESCE(localized_content, '{}'::jsonb) + FROM processed_products + WHERE company_id = $1`, companyID) + if err != nil { + return out, fmt.Errorf("list processed products: %w", err) + } + defer rows.Close() + + type pending struct { + id uuid.UUID + rawID uuid.UUID + fs []byte + loc []byte + } + var updates []pending + + for rows.Next() { + out.Scanned++ + var ( + id, rawID uuid.UUID + name, desc, processedName, processedDesc string + fsRaw, locRaw []byte + ) + if err := rows.Scan(&id, &rawID, &name, &desc, &processedName, &processedDesc, &fsRaw, &locRaw); err != nil { + return out, err + } + fs := map[string]any{} + if len(fsRaw) > 0 { + if err := json.Unmarshal(fsRaw, &fs); err != nil { + return out, err + } + } + if fs == nil { + fs = map[string]any{} + } + loc, err := company.DecodeLocalizedContent(locRaw) + if err != nil { + return out, err + } + + changed := false + primaryDesc := strings.TrimSpace(processedDesc) + if primaryDesc == "" { + primaryDesc = desc + } + primaryName := strings.TrimSpace(processedName) + if primaryName == "" { + primaryName = name + } + + if _, has := fs[enhanceInputHashKey]; has { + if company.IsWeakPriorEnhanceDescription(primaryDesc, primaryName, name) { + delete(fs, enhanceInputHashKey) + changed = true + } + } + + for lang, fields := range loc { + d := strings.TrimSpace(fields.ProcessedDescription) + n := strings.TrimSpace(fields.ProcessedName) + if n == "" { + n = primaryName + } + if strings.TrimSpace(fields.EnhanceInputHash) == "" { + continue + } + if company.IsWeakPriorEnhanceDescription(d, n, primaryName, name) { + fields.EnhanceInputHash = "" + loc[lang] = fields + changed = true + } + } + + if !changed { + continue + } + fsBytes, err := json.Marshal(fs) + if err != nil { + return out, err + } + locBytes, err := company.EncodeLocalizedContent(loc) + if err != nil { + return out, err + } + updates = append(updates, pending{id: id, rawID: rawID, fs: fsBytes, loc: locBytes}) + } + if err := rows.Err(); err != nil { + return out, err + } + rows.Close() + + seenRaw := map[uuid.UUID]struct{}{} + for _, u := range updates { + ct, err := tx.Exec(ctx, ` + UPDATE processed_products + SET field_sources = $2::jsonb, + localized_content = $3::jsonb, + updated_at = now() + WHERE id = $1 AND company_id = $4`, + u.id, u.fs, u.loc, companyID) + if err != nil { + return out, fmt.Errorf("clear enhance hashes product=%s: %w", u.id, err) + } + if ct.RowsAffected() > 0 { + out.Cleared++ + if _, ok := seenRaw[u.rawID]; !ok { + seenRaw[u.rawID] = struct{}{} + out.ClearedRawIDs = append(out.ClearedRawIDs, u.rawID) + } + } + } + return out, nil +} diff --git a/apps/api/internal/catalog/repair_enhance_hashes_test.go b/apps/api/internal/catalog/repair_enhance_hashes_test.go new file mode 100644 index 0000000..68cd7f0 --- /dev/null +++ b/apps/api/internal/catalog/repair_enhance_hashes_test.go @@ -0,0 +1,137 @@ +package catalog + +import ( + "context" + "encoding/json" + "os" + "strings" + "testing" + + "github.com/descrybe/descrybe-v2/apps/api/internal/company" + "github.com/google/uuid" + "github.com/jackc/pgx/v5/pgxpool" +) + +func TestClearWeakEnhanceHashesDecision(t *testing.T) { + t.Parallel() + title := "Acme Widget Pro 24" + weak := title + strong := "Acme Widget Pro 24 is a durable retail widget built for everyday warehouse use with a reinforced frame." + + if !company.IsWeakPriorEnhanceDescription(weak, title) { + t.Fatal("title-echo must be weak") + } + if company.IsWeakPriorEnhanceDescription(strong, title) { + t.Fatal("strong description must not be weak") + } +} + +func TestRepairWeakEnhanceHashes_nilPool(t *testing.T) { + t.Parallel() + _, err := RepairWeakEnhanceHashes(context.Background(), nil, uuid.New()) + if err == nil { + t.Fatal("expected error for nil pool") + } + _, err = RepairWeakEnhanceHashes(context.Background(), &pgxpool.Pool{}, uuid.Nil) + if err == nil { + t.Fatal("expected error for nil company") + } +} + +func TestRepairWeakEnhanceHashes_integration(t *testing.T) { + dsn := strings.TrimSpace(os.Getenv("DATABASE_URL")) + if dsn == "" { + t.Skip("DATABASE_URL not set") + } + ctx := context.Background() + pool, err := pgxpool.New(ctx, dsn) + if err != nil { + t.Fatalf("connect: %v", err) + } + defer pool.Close() + + var companyID uuid.UUID + err = pool.QueryRow(ctx, ` + SELECT id FROM companies + WHERE lower(name) IN ('platform demo', 'demo') + ORDER BY CASE WHEN lower(name) = 'platform demo' THEN 0 ELSE 1 END + LIMIT 1`).Scan(&companyID) + if err != nil { + t.Skipf("Platform Demo company not found: %v", err) + } + + title := "Repair Hash Probe Widget X" + weakDesc := title + hash := "deadbeefcafebabe0123456789abcdef0123456789abcdef0123456789abcdef" + loc := company.LocalizedContent{ + "en": { + ProcessedName: title, + ProcessedDescription: weakDesc, + EnhanceInputHash: hash, + }, + } + locJSON, err := company.EncodeLocalizedContent(loc) + if err != nil { + t.Fatalf("encode loc: %v", err) + } + fs, _ := json.Marshal(map[string]any{enhanceInputHashKey: hash, "name": "ai_enhance"}) + + tx, err := pool.Begin(ctx) + if err != nil { + t.Fatalf("begin: %v", err) + } + defer tx.Rollback(ctx) + + var rawID, ppID uuid.UUID + gtin := "8700999111222" + err = tx.QueryRow(ctx, ` + INSERT INTO raw_products (company_id, gtin, raw_data, mapped_data, is_processed, processing_status) + VALUES ($1, $2, '{}'::jsonb, '{}'::jsonb, true, 'processed') + RETURNING id`, companyID, gtin).Scan(&rawID) + if err != nil { + t.Fatalf("insert raw: %v", err) + } + err = tx.QueryRow(ctx, ` + INSERT INTO processed_products ( + company_id, raw_product_id, product_id, name, processed_name, description, processed_description, + status, attributes, processed_attributes, gpt_response, field_sources, localized_content, ai_provider_mode + ) VALUES ( + $1, $2, $3, $4, $4, $5, $5, 'needs_review', '{}'::jsonb, '{}'::jsonb, '{}'::jsonb, $6::jsonb, $7::jsonb, 'internal' + ) RETURNING id`, companyID, rawID, gtin, title, weakDesc, fs, locJSON).Scan(&ppID) + if err != nil { + t.Fatalf("insert processed: %v", err) + } + + res, err := repairWeakEnhanceHashesTx(ctx, tx, companyID) + if err != nil { + t.Fatalf("repair: %v", err) + } + if res.Cleared < 1 { + t.Fatalf("expected at least 1 cleared, got %d (scanned=%d)", res.Cleared, res.Scanned) + } + + var fsOut, locOut []byte + err = tx.QueryRow(ctx, ` + SELECT COALESCE(field_sources, '{}'::jsonb), COALESCE(localized_content, '{}'::jsonb) + FROM processed_products WHERE id = $1`, ppID).Scan(&fsOut, &locOut) + if err != nil { + t.Fatalf("reload: %v", err) + } + var fsMap map[string]any + _ = json.Unmarshal(fsOut, &fsMap) + if _, ok := fsMap[enhanceInputHashKey]; ok { + t.Fatalf("field_sources hash still present: %v", fsMap[enhanceInputHashKey]) + } + gotLoc, err := company.DecodeLocalizedContent(locOut) + if err != nil { + t.Fatalf("decode loc: %v", err) + } + if h := gotLoc["en"].EnhanceInputHash; h != "" { + t.Fatalf("localized hash still present: %q", h) + } + if gotLoc["en"].ProcessedDescription != weakDesc { + t.Fatalf("description should be preserved, got %q", gotLoc["en"].ProcessedDescription) + } + + // Roll back probe rows — do not leave smoke data on Platform Demo. +} diff --git a/apps/api/internal/catalog/service.go b/apps/api/internal/catalog/service.go index 7b486ce..25b97c6 100644 --- a/apps/api/internal/catalog/service.go +++ b/apps/api/internal/catalog/service.go @@ -260,7 +260,7 @@ func enrichCategoryPrompts(ctx context.Context, pool *pgxpool.Pool, companyID uu } primary := company.LoadLanguage(ctx, pool, companyID) item["prompts"] = prompts - item["prompt"] = company.PromptForLanguage(prompts, primary) + item["prompt"] = company.PromptForLanguage(prompts, primary, primary) item["has_prompt"] = company.HasAnyPrompt(prompts) return item, nil } @@ -668,8 +668,11 @@ func processedProductsCountFromSQL(needsRawJoin bool) string { // Enrichment coverage SQL predicates (alias p = processed_products, r = raw_products). const ( - processedHasNameSQL = `(COALESCE(NULLIF(p.processed_name, ''), NULLIF(p.name, ''), NULLIF(r.mapped_data->>'name', ''), NULLIF(r.mapped_data->>'title', ''), '') <> '')` - processedHasDescriptionSQL = `(COALESCE(NULLIF(p.processed_description, ''), NULLIF(p.description, ''), NULLIF(r.mapped_data->>'description', ''), '') <> '')` + // Prefer processed_* so dashboard product fields match V1 process items for the same EAN. + processedPreferredNameSQL = `COALESCE(NULLIF(p.processed_name, ''), NULLIF(p.name, ''), NULLIF(r.mapped_data->>'name', ''), NULLIF(r.mapped_data->>'title', ''), '')` + processedPreferredDescriptionSQL = `COALESCE(NULLIF(p.processed_description, ''), NULLIF(p.description, ''), NULLIF(r.mapped_data->>'description', ''), '')` + processedHasNameSQL = `(` + processedPreferredNameSQL + ` <> '')` + processedHasDescriptionSQL = `(` + processedPreferredDescriptionSQL + ` <> '')` processedHasCategorySQL = `(COALESCE(NULLIF(p.category, ''), '') <> '' AND lower(p.category) <> 'none')` // Resolve display name / canonical unique_id when products store unique_id, UUID id, or name. processedCategoryResolveJoin = ` @@ -1093,7 +1096,7 @@ func (s *Service) ListProcessedProducts(ctx context.Context, companyID uuid.UUID func(ctx context.Context) ([]map[string]any, error) { rows, err := s.Pool.Query(ctx, fmt.Sprintf(` SELECT p.id, p.product_id, - COALESCE(NULLIF(p.name, ''), NULLIF(r.mapped_data->>'name', ''), NULLIF(r.mapped_data->>'title', ''), '') AS name, + `+processedPreferredNameSQL+` AS name, COALESCE(NULLIF(p.processed_name, ''), '') AS processed_name, p.category, COALESCE(cat.name, NULLIF(BTRIM(p.category), '')) AS category_name, @@ -1102,9 +1105,9 @@ func (s *Service) ListProcessedProducts(ctx context.Context, companyID uuid.UUID f.name AS feed_name, f.last_synced_at AS feed_last_synced_at, r.updated_at AS raw_updated_at, - (COALESCE(NULLIF(p.processed_name, ''), NULLIF(p.name, ''), NULLIF(r.mapped_data->>'name', ''), NULLIF(r.mapped_data->>'title', ''), '') <> '') AS has_name, + (`+processedPreferredNameSQL+` <> '') AS has_name, (COALESCE(NULLIF(p.processed_name, ''), '') <> '') AS has_processed_name, - (COALESCE(NULLIF(p.processed_description, ''), NULLIF(p.description, ''), NULLIF(r.mapped_data->>'description', ''), '') <> '') AS has_description, + (`+processedPreferredDescriptionSQL+` <> '') AS has_description, (COALESCE(NULLIF(p.processed_description, ''), '') <> '') AS has_processed_description, (COALESCE(NULLIF(p.category, ''), '') <> '' AND lower(p.category) <> 'none') AS has_category, `+processedHasAttributesSQL+` AS has_attributes, @@ -1120,7 +1123,7 @@ func (s *Service) ListProcessedProducts(ctx context.Context, companyID uuid.UUID return nil, err } defer rows.Close() - return scanMaps(rows, []string{ + items, err := scanMaps(rows, []string{ "id", "product_id", "name", "processed_name", "category", "category_name", "category_unique_id", "status", "raw_product_id", "feed_id", "gtin", "feed_name", "feed_last_synced_at", "raw_updated_at", @@ -1128,6 +1131,15 @@ func (s *Service) ListProcessedProducts(ctx context.Context, companyID uuid.UUID "has_eprel", "created_at", "updated_at", }) + if err != nil { + return nil, err + } + for _, item := range items { + if name, ok := item["name"].(string); ok && strings.TrimSpace(name) != "" { + item["title"] = name + } + } + return items, nil }, ) } @@ -1172,12 +1184,12 @@ func (s *Service) ListProcessedProductsDetailed(ctx context.Context, companyID u func(ctx context.Context) ([]map[string]any, error) { rows, err := s.Pool.Query(ctx, fmt.Sprintf(` SELECT p.id, p.product_id, - COALESCE(NULLIF(p.name, ''), NULLIF(r.mapped_data->>'name', ''), NULLIF(r.mapped_data->>'title', ''), '') AS name, + `+processedPreferredNameSQL+` AS name, p.category, COALESCE(cat.name, NULLIF(BTRIM(p.category), '')) AS category_name, COALESCE(cat.unique_id, NULLIF(BTRIM(p.category), '')) AS category_unique_id, p.status, p.raw_product_id, p.feed_id, r.gtin, - COALESCE(NULLIF(p.description, ''), NULLIF(r.mapped_data->>'description', ''), '') AS description, + `+processedPreferredDescriptionSQL+` AS description, p.processed_name, p.processed_description, COALESCE(p.meta_title, ''), COALESCE(p.meta_description, ''), p.attributes, p.processed_attributes, r.mapped_data, @@ -1190,7 +1202,7 @@ func (s *Service) ListProcessedProductsDetailed(ctx context.Context, companyID u return nil, err } defer rows.Close() - return scanMaps(rows, []string{ + items, err := scanMaps(rows, []string{ "id", "product_id", "name", "category", "category_name", "category_unique_id", "status", "raw_product_id", "feed_id", "gtin", "description", "processed_name", "processed_description", @@ -1198,6 +1210,13 @@ func (s *Service) ListProcessedProductsDetailed(ctx context.Context, companyID u "attributes", "processed_attributes", "mapped_data", "created_at", "updated_at", }) + if err != nil { + return nil, err + } + for _, item := range items { + alignProductFieldsWithV1(item) + } + return items, nil }, ) } @@ -1205,11 +1224,11 @@ func (s *Service) ListProcessedProductsDetailed(ctx context.Context, companyID u func (s *Service) GetProcessedProduct(ctx context.Context, companyID, id uuid.UUID) (map[string]any, error) { row := s.Pool.QueryRow(ctx, ` SELECT p.id, p.product_id, - COALESCE(NULLIF(p.name, ''), NULLIF(r.mapped_data->>'name', ''), NULLIF(r.mapped_data->>'title', ''), '') AS name, + `+processedPreferredNameSQL+` AS name, p.category, COALESCE(cat.name, NULLIF(BTRIM(p.category), '')) AS category_name, COALESCE(cat.unique_id, NULLIF(BTRIM(p.category), '')) AS category_unique_id, - COALESCE(NULLIF(p.description, ''), NULLIF(r.mapped_data->>'description', ''), '') AS description, + `+processedPreferredDescriptionSQL+` AS description, p.processed_name, p.processed_description, p.status, p.attributes, p.processed_attributes, r.gtin, r.mapped_data, COALESCE(p.feed_id, r.feed_id) AS feed_id, @@ -1241,6 +1260,7 @@ func (s *Service) GetProcessedProduct(ctx context.Context, companyID, id uuid.UU return nil, err } linkFeedSpecificationsIntoProduct(item) + alignProductFieldsWithV1(item) primary := company.LoadLanguage(ctx, s.Pool, companyID) item["content_language"] = primary item["content_languages"] = company.LoadContentLanguages(ctx, s.Pool, companyID) @@ -1303,6 +1323,7 @@ func (s *Service) GetRawProduct(ctx context.Context, companyID, id uuid.UUID) (m return nil, err } linkFeedSpecificationsIntoProduct(item) + alignProductFieldsWithV1(item) primary := company.LoadLanguage(ctx, s.Pool, companyID) item["content_language"] = primary item["content_languages"] = company.LoadContentLanguages(ctx, s.Pool, companyID) diff --git a/apps/api/internal/company/lang_content.go b/apps/api/internal/company/lang_content.go index 12b4361..f9d2cc4 100644 --- a/apps/api/internal/company/lang_content.go +++ b/apps/api/internal/company/lang_content.go @@ -11,7 +11,12 @@ import ( "github.com/jackc/pgx/v5/pgxpool" ) +// LangPromptAny is the wildcard key for a single shared prompt used for every +// content language (avoid duplicating the same text per lang). +const LangPromptAny = "*" + // LangPromptMap is language-code → prompt text for category / template overrides. +// Optional key LangPromptAny ("*") is a shared overlay for all languages. type LangPromptMap map[string]string // LocalizedFields holds AI/output fields for one content language. @@ -27,15 +32,16 @@ type LocalizedFields struct { type LocalizedContent map[string]LocalizedFields // SanitizeLangPromptMap validates language codes, sanitizes prompts, and drops empties. +// Accepts LangPromptAny ("*") as a shared any-language prompt key. func SanitizeLangPromptMap(in map[string]string, maxRunes int) (LangPromptMap, error) { out := make(LangPromptMap) if len(in) == 0 { return out, nil } for lang, prompt := range in { - code, err := ParseLanguage(lang, false) + code, err := parseLangPromptKey(lang) if err != nil { - return nil, fmt.Errorf("unsupported language %q", lang) + return nil, err } p := strings.TrimSpace(security.SanitizePrompt(prompt, maxRunes)) if p == "" { @@ -46,16 +52,46 @@ func SanitizeLangPromptMap(in map[string]string, maxRunes int) (LangPromptMap, e return out, nil } -// PromptForLanguage returns the prompt for lang, or empty if unset. -func PromptForLanguage(m LangPromptMap, lang string) string { +func parseLangPromptKey(raw string) (string, error) { + key := strings.TrimSpace(raw) + if key == LangPromptAny { + return LangPromptAny, nil + } + code, err := ParseLanguage(key, false) + if err != nil { + return "", fmt.Errorf("unsupported language %q", raw) + } + return code, nil +} + +// PromptForLanguage resolves a prompt with fallback: +// requested lang → LangPromptAny ("*") → primary → "" (caller may then use built-in). +// Empty primary skips the primary step. Does not invent cross-lang text beyond this chain. +func PromptForLanguage(m LangPromptMap, lang, primary string) string { if len(m) == 0 { return "" } - code, err := ParseLanguage(lang, true) - if err != nil { - code = DefaultLanguage + try := func(code string) string { + code = strings.TrimSpace(code) + if code == "" { + return "" + } + return strings.TrimSpace(m[code]) } - return strings.TrimSpace(m[code]) + if code, err := ParseLanguage(lang, true); err == nil { + if p := try(code); p != "" { + return p + } + } + if p := try(LangPromptAny); p != "" { + return p + } + if prim, err := ParseLanguage(primary, false); err == nil { + if p := try(prim); p != "" { + return p + } + } + return "" } // HasAnyPrompt reports whether any language has a non-empty prompt. diff --git a/apps/api/internal/company/lang_content_test.go b/apps/api/internal/company/lang_content_test.go index dbee1bd..302db4e 100644 --- a/apps/api/internal/company/lang_content_test.go +++ b/apps/api/internal/company/lang_content_test.go @@ -17,6 +17,7 @@ func TestSanitizeLangPromptMap(t *testing.T) { m, err := SanitizeLangPromptMap(map[string]string{ "SL": " hello {{name}} ", "en": "", + "*": " shared ", }, 100) if err != nil { t.Fatal(err) @@ -27,16 +28,37 @@ func TestSanitizeLangPromptMap(t *testing.T) { if _, ok := m["en"]; ok { t.Fatalf("empty en should be dropped: %#v", m) } + if m[LangPromptAny] != "shared" { + t.Fatalf("wildcard missing: %#v", m) + } } func TestPromptForLanguage(t *testing.T) { t.Parallel() m := LangPromptMap{"sl": "slo", "en": "eng"} - if got := PromptForLanguage(m, "SL"); got != "slo" { + if got := PromptForLanguage(m, "SL", ""); got != "slo" { t.Fatalf("got %q", got) } - if got := PromptForLanguage(m, "de"); got != "" { - t.Fatalf("expected empty, got %q", got) + if got := PromptForLanguage(m, "de", ""); got != "" { + t.Fatalf("expected empty without primary, got %q", got) + } + // sl-only map: exact hit for sl + slOnly := LangPromptMap{"sl": "slo-only"} + if got := PromptForLanguage(slOnly, "sl", "sl"); got != "slo-only" { + t.Fatalf("sl exact: got %q", got) + } + // sl-only map: en request falls back to primary=sl + if got := PromptForLanguage(slOnly, "en", "sl"); got != "slo-only" { + t.Fatalf("en→primary sl: got %q", got) + } + // wildcard before primary + anyMap := LangPromptMap{"*": "shared", "sl": "slo"} + if got := PromptForLanguage(anyMap, "de", "sl"); got != "shared" { + t.Fatalf("wildcard before primary: got %q", got) + } + // explicit lang beats wildcard + if got := PromptForLanguage(anyMap, "sl", "en"); got != "slo" { + t.Fatalf("explicit beats *: got %q", got) } } diff --git a/apps/api/internal/company/weak_desc.go b/apps/api/internal/company/weak_desc.go new file mode 100644 index 0000000..670176d --- /dev/null +++ b/apps/api/internal/company/weak_desc.go @@ -0,0 +1,151 @@ +package company + +import "strings" + +// minUsableProductDescRunes is the floor for a prior description that may hash-skip +// the enhance LLM. Shorter copy is treated as thin and forced through enhance. +const minUsableProductDescRunes = 40 + +// shortBoilerplateDescRunes: short copy without product-fact token overlap is weak +// even when longer than minUsableProductDescRunes (generic one-liners). +const shortBoilerplateDescRunes = 96 + +// weakFillerPhrases are known heuristic / placeholder snippets that must never +// hash-skip enhance (mock-llm / invent fallbacks historically produced these). +var weakFillerPhrases = []string{ + "ready for retail listing", + "quality product ready", + "product description", + "based on available specifications", + "with available specifications", + "available catalog details", + "pripravljeno za prodajo", + "na podlagi razpoložljivih specifikacij", +} + +// IsWeakPriorEnhanceDescription reports empty, too-short, title-echo, known filler, +// or short boilerplate without overlapping tokens from title/fact sources. +func IsWeakPriorEnhanceDescription(priorDesc string, titles ...string) bool { + priorDesc = strings.TrimSpace(priorDesc) + if priorDesc == "" || priorDesc == "World", "Second"}) + if got != "Hello\nWorld\nSecond" { + t.Fatalf("array+html: %q", got) + } + got = PlainDescriptionFromAny(map[string]any{"#text": "Plain & ok"}) + if got != "Plain & ok" { + t.Fatalf("#text: %q", got) + } + if PlainDescriptionFromAny(nil) != "" { + t.Fatal("nil should be empty") + } +} + +func TestNormalizeMapped_descriptionArrayBecomesPlainString(t *testing.T) { + got := NormalizeMapped(map[string]any{ + "description": []any{"Line1
A", "Line2"}, + "name": "Widget", + }, nil) + desc, ok := got["description"].(string) + if !ok { + t.Fatalf("description should be string, got %T %v", got["description"], got["description"]) + } + if desc != "Line1\nA\nLine2" { + t.Fatalf("description=%q", desc) + } +} + +func TestNormalizeMapped_eprelIDAlias(t *testing.T) { + got := NormalizeMapped(map[string]any{"EPRELID": "246834"}, nil) + if got["eprel_id"] != "246834" { + t.Fatalf("eprel_id=%v", got["eprel_id"]) + } +} + +func TestStringFromAny_flattensArray(t *testing.T) { + got := stringFromAny([]any{"a", "b"}) + if got != "a b" { + t.Fatalf("got %q", got) + } +} + +func TestCanonicalizeAttrKey_eprel(t *testing.T) { + if canonicalizeAttrKey("EPRELID") != "eprel_id" { + t.Fatalf("got %q", canonicalizeAttrKey("EPRELID")) + } + if canonicalizeAttrKey("eprel") != "eprel_id" { + t.Fatalf("got %q", canonicalizeAttrKey("eprel")) + } +} + +func TestPlainDescriptionFromStored_jsonArrayString(t *testing.T) { + got := plainDescriptionFromStored(`["Hello
World"]`) + if got != "Hello\nWorld" { + t.Fatalf("got %q", got) + } +} diff --git a/apps/api/internal/processing/live_mock_llm_title_test.go b/apps/api/internal/processing/live_mock_llm_title_test.go new file mode 100644 index 0000000..9e999e4 --- /dev/null +++ b/apps/api/internal/processing/live_mock_llm_title_test.go @@ -0,0 +1,67 @@ +package processing_test + +import ( + "context" + "os" + "strings" + "testing" + "time" + + "github.com/descrybe/descrybe-v2/apps/api/internal/aiprompts" + "github.com/descrybe/descrybe-v2/apps/api/internal/processing" +) + +// Live proof against local mock-llm (OPENAI_BASE_URL). Skipped unless LIVE_MOCK_LLM=1. +func TestLive_MockLLM_rejectsCategoryEnhanceInstructionTitle(t *testing.T) { + if strings.TrimSpace(os.Getenv("LIVE_MOCK_LLM")) == "" { + t.Skip("set LIVE_MOCK_LLM=1 with mock-llm on OPENAI_BASE_URL") + } + base := strings.TrimSpace(os.Getenv("OPENAI_BASE_URL")) + key := strings.TrimSpace(os.Getenv("OPENAI_API_KEY")) + model := strings.TrimSpace(os.Getenv("OPENAI_MODEL")) + if base == "" || key == "" { + t.Fatal("OPENAI_BASE_URL and OPENAI_API_KEY required") + } + if model == "" { + model = "mock-llm" + } + + client := processing.NewOpenAIClient(key, base, model, 0, 2) + eng := &processing.Engine{ + Completer: client, + Vector: processing.NoopVectorCategorizer{}, + } + ctx, cancel := context.WithTimeout(context.Background(), 45*time.Second) + defer cancel() + + out, err := eng.RunSteps(ctx, "live-probe", processing.ProductInput{ + GTIN: "8712285326882", + Name: "Vox SWA-8000W", + Description: "Washer dryer combo", + Mapped: map[string]any{"name": "Vox SWA-8000W", "description": "Washer dryer combo", "category": "demo-electronics", "brand": "Vox"}, + Language: "en", + EnhanceUserTemplate: aiprompts.CategoryEnhanceUserTemplate, + EnhanceSystemTemplate: `Retail product copywriter. Reply with ONLY JSON. Schema: {"name":"string","description":"string"}`, + TitleTemplate: map[string]any{ + "separator": " ", + "elements": []any{ + map[string]any{"type": "variable", "value": "brand"}, + map[string]any{"type": "text", "value": " "}, + map[string]any{"type": "variable", "value": "product_model"}, + }, + }, + }, "enhance_only", nil, processing.StepPolicy{AllowAI: true}) + if err != nil { + t.Fatal(err) + } + t.Logf("ProcessedName=%q AIProviderMode=%q", out.ProcessedName, out.AIProviderMode) + lower := strings.ToLower(out.ProcessedName) + for _, p := range []string{"title formula", "follow any", "short retail title", "use attrs", "schema:", "reply with only json"} { + if strings.Contains(lower, p) { + t.Fatalf("polluted title %q contains %q", out.ProcessedName, p) + } + } + if !strings.Contains(lower, "vox") { + t.Fatalf("ProcessedName=%q want Vox feed title", out.ProcessedName) + } +} diff --git a/apps/api/internal/processing/llm_json.go b/apps/api/internal/processing/llm_json.go index 4243ca3..edc7854 100644 --- a/apps/api/internal/processing/llm_json.go +++ b/apps/api/internal/processing/llm_json.go @@ -11,15 +11,18 @@ import ( // Local / weak-model defaults (8k-class context). See docs/local-llm-tuning.md. const ( DefaultStructuredTemp = 0.2 - MaxTokensEnhance = 350 - MaxTokensSEO = 180 - MaxTokensCampaign = 650 - MaxProductDescRunes = 400 - MaxAttrKeys = 10 - MaxAttrValueRunes = 60 - MaxBrandInjectRunes = 500 - MaxCampaignProducts = 8 - MaxCampaignNameRunes = 80 + // Reasoning-capable OpenAI-compatible models (e.g. code-fast / Qwen3) spend + // hundreds–thousands of tokens in reasoning_content before writing message.content. + // 350 capped mid-thought → empty content → "empty model response". + MaxTokensEnhance = 4096 + MaxTokensSEO = 180 + MaxTokensCampaign = 650 + MaxProductDescRunes = 400 + MaxAttrKeys = 10 + MaxAttrValueRunes = 60 + MaxBrandInjectRunes = 500 + MaxCampaignProducts = 8 + MaxCampaignNameRunes = 80 ) // CompleteOptions tunes a single chat completion for structured tasks. @@ -152,14 +155,14 @@ func CompactAttrs(attrs map[string]any, maxKeys int) map[string]any { keys := make([]string, 0, len(attrs)) for k := range attrs { k = strings.TrimSpace(k) - if k == "" { + if k == "" || isInvalidAttributeKey(k) { continue } keys = append(keys, k) } sort.Strings(keys) // Prefer common retail keys first. - priority := []string{"brand", "Brand", "color", "Color", "material", "Material", "size", "Size", "model", "Model", "gtin", "GTIN", "ean", "EAN"} + priority := []string{"brand", "Brand", "product_model", "color", "Color", "material", "Material", "size", "Size", "model", "Model", "gtin", "GTIN", "ean", "EAN"} ordered := make([]string, 0, len(keys)) seen := map[string]bool{} for _, p := range priority { diff --git a/apps/api/internal/processing/meta.go b/apps/api/internal/processing/meta.go new file mode 100644 index 0000000..4f4c3ac --- /dev/null +++ b/apps/api/internal/processing/meta.go @@ -0,0 +1,150 @@ +package processing + +import ( + "regexp" + "strings" + "unicode" +) + +// Meta title/description caps mirror seo.MetaTitleMaxChars / MetaDescriptionMaxChars. +// Duplicated here so processing can fill meta without importing seo (seo imports processing). +const ( + metaTitleMaxChars = 60 + metaDescriptionMaxChars = 155 +) + +// Bare "|
World
& morebold") if strings.Contains(got, "<") { diff --git a/apps/api/internal/processing/prompt_fallback_test.go b/apps/api/internal/processing/prompt_fallback_test.go index 4d186fa..342d229 100644 --- a/apps/api/internal/processing/prompt_fallback_test.go +++ b/apps/api/internal/processing/prompt_fallback_test.go @@ -1,6 +1,7 @@ package processing import ( + "strings" "testing" "github.com/descrybe/descrybe-v2/apps/api/internal/company" @@ -9,14 +10,18 @@ import ( func TestResolvePromptFallbackChain(t *testing.T) { t.Parallel() // Category override for language wins over company user template. + // Overrides without {{attrs}} get standard context placeholders injected. sys, user := resolveProductPromptTemplates(ProductInput{ EnhanceSystemTemplate: "sys", EnhanceUserTemplate: "company-en", CategoryEnhancePrompt: "cat-sl", Language: "sl", }) - if sys != "sys" || user != "cat-sl" { - t.Fatalf("sys=%q user=%q", sys, user) + if sys != "sys" { + t.Fatalf("sys=%q", sys) + } + if !strings.HasPrefix(user, "cat-sl") || !strings.Contains(user, "{{attrs}}") { + t.Fatalf("sys=%q user=%q want cat-sl + attrs", sys, user) } // Empty category → company template. @@ -28,12 +33,20 @@ func TestResolvePromptFallbackChain(t *testing.T) { t.Fatalf("user=%q", user) } - // categoryEnhancePromptFor: lang-specific only (no cross-lang fallback). + // sl-only category prompt: exact for sl; en falls back to primary=sl. m := map[string]company.LangPromptMap{"audio": {"sl": "slo-prompt"}} - if got := categoryEnhancePromptFor(m, "audio", "sl"); got != "slo-prompt" { - t.Fatalf("got %q", got) + if got := categoryEnhancePromptFor(m, "audio", "sl", "sl"); got != "slo-prompt" { + t.Fatalf("sl exact: got %q", got) } - if got := categoryEnhancePromptFor(m, "audio", "en"); got != "" { - t.Fatalf("cross-lang should be empty, got %q", got) + if got := categoryEnhancePromptFor(m, "audio", "en", "sl"); got != "slo-prompt" { + t.Fatalf("en→primary sl: got %q", got) + } + if got := categoryEnhancePromptFor(m, "audio", "en", "en"); got != "" { + t.Fatalf("en with primary=en and no en/* prompt should be empty, got %q", got) + } + // wildcard shared overlay + m2 := map[string]company.LangPromptMap{"audio": {"*": "shared-overlay"}} + if got := categoryEnhancePromptFor(m2, "audio", "en", "sl"); got != "shared-overlay" { + t.Fatalf("wildcard: got %q", got) } } diff --git a/apps/api/internal/processing/prompt_render.go b/apps/api/internal/processing/prompt_render.go index d49c8a3..c80251e 100644 --- a/apps/api/internal/processing/prompt_render.go +++ b/apps/api/internal/processing/prompt_render.go @@ -12,21 +12,76 @@ func resolveProductPromptTemplates(in ProductInput) (systemTpl, userTpl string) userTpl = strings.TrimSpace(in.EnhanceUserTemplate) // Per-category prompt wins for the user message (company system keeps JSON schema / brand). if cat := strings.TrimSpace(in.CategoryEnhancePrompt); cat != "" { - userTpl = cat + userTpl = ensureCategoryEnhanceUserContext(cat) } def, ok := aiprompts.DefaultFor(aiprompts.KeyProductEnhance) - if !ok { - return systemTpl, userTpl - } - if systemTpl == "" { - systemTpl = def.SystemTemplate - } - if userTpl == "" { - userTpl = def.UserTemplate + if ok { + if systemTpl == "" { + systemTpl = def.SystemTemplate + } + if userTpl == "" { + userTpl = def.UserTemplate + } } + // Category formulas are language-agnostic; inject once into the shared user skeleton. + userTpl = AppendFormulaConstraints(userTpl, in.TitleTemplate, in.DescriptionTemplate) return systemTpl, userTpl } +// thinEnhanceUserInstruction is prepended when Desc is empty or ≈ Name so the model +// builds copy from Attrs+Category instead of echoing the title. +const thinEnhanceUserInstruction = "Build the description from Attrs and Category only; never copy Name." + +// templateHasVar reports whether tpl contains {{name}} (via aiprompts.ExtractVariables). +func templateHasVar(tpl, name string) bool { + for _, v := range aiprompts.ExtractVariables(tpl) { + if v == name { + return true + } + } + return false +} + +// ensureCategoryEnhanceUserContext appends standard product context placeholders when a +// category override omits {{attrs}} (common in DB category prompts). Does not rewrite +// category copy that already includes attrs. +func ensureCategoryEnhanceUserContext(userTpl string) string { + userTpl = strings.TrimSpace(userTpl) + if userTpl == "" || templateHasVar(userTpl, "attrs") { + return userTpl + } + var b strings.Builder + b.WriteString(userTpl) + b.WriteString("\n\n") + first := true + appendLine := func(line string) { + if !first { + b.WriteByte('\n') + } + first = false + b.WriteString(line) + } + if !templateHasVar(userTpl, "category") { + appendLine("Category: {{category}}") + } + if !templateHasVar(userTpl, "name") { + appendLine("Name: {{name}}") + } + if !templateHasVar(userTpl, "description") { + appendLine("Desc: {{description}}") + } + appendLine("Attrs: {{attrs}}") + return b.String() +} + +func isThinEnhanceDescription(desc, name string) bool { + desc = strings.TrimSpace(desc) + if desc == "" || desc == "
/entity markup. func v1PlainDescription(s string) string { + s = strings.TrimSpace(s) + if s == "" { + return "" + } + s = unwrapLegacyDescriptionStored(s) s = strings.TrimSpace(s) if s == "" { return "" } s = html.UnescapeString(s) s = strings.ReplaceAll(s, "\u00a0", " ") + s = strings.ReplaceAll(s, `\u00a0`, " ") s = v1BreakTagRe.ReplaceAllString(s, "\n") s = v1BlockEndRe.ReplaceAllString(s, "\n") s = specsHTMLTagRe.ReplaceAllString(s, " ") @@ -406,6 +520,10 @@ func v1PlainDescription(s string) string { kept := make([]string, 0, len(lines)) for _, line := range lines { line = strings.Join(strings.Fields(line), " ") + line = strings.ReplaceAll(line, " .", ".") + line = strings.ReplaceAll(line, " ,", ",") + line = strings.ReplaceAll(line, " ;", ";") + line = strings.ReplaceAll(line, " :", ":") if line != "" { kept = append(kept, line) } @@ -413,23 +531,67 @@ func v1PlainDescription(s string) string { return strings.TrimSpace(strings.Join(kept, "\n")) } -func extractEPRELFromAttrs(attrs map[string]any) any { - if attrs == nil { - return nil +const maxLegacyDescriptionUnwrapDepth = 4 + +// unwrapLegacyDescriptionStored flattens JSON-encoded legacy description payloads +// (one-element arrays, multi-part arrays, nested arrays, JSON-quoted strings). +// Non-JSON text is returned unchanged. +func unwrapLegacyDescriptionStored(s string) string { + return unwrapLegacyDescriptionStoredN(s, 0) +} + +func unwrapLegacyDescriptionStoredN(s string, depth int) string { + s = strings.TrimSpace(s) + if s == "" || depth > maxLegacyDescriptionUnwrapDepth { + return s } - if e, ok := attrs["eprel"]; ok && e != nil { - return e + if !looksLikeLegacyDescriptionJSON(s) { + return s } - out := map[string]any{} - for _, k := range []string{"label", "pdf", "energy_class", "energy_scale"} { - if v, ok := attrs["eprel_"+k]; ok && v != nil { - out[k] = v + var raw any + if err := json.Unmarshal([]byte(s), &raw); err != nil { + return s + } + flat := flattenLegacyDescriptionAny(raw, depth) + if flat == "" { + if _, ok := raw.([]any); ok { + return "" } + return s } - if len(out) == 0 { - return nil + return flat +} + +func looksLikeLegacyDescriptionJSON(s string) bool { + if len(s) >= 2 && s[0] == '[' && s[len(s)-1] == ']' { + return true } - return out + return len(s) >= 2 && s[0] == '"' && s[len(s)-1] == '"' +} + +func flattenLegacyDescriptionAny(v any, depth int) string { + switch t := v.(type) { + case string: + return unwrapLegacyDescriptionStoredN(t, depth+1) + case []any: + parts := make([]string, 0, len(t)) + for _, el := range t { + if p := flattenLegacyDescriptionAny(el, depth+1); p != "" { + parts = append(parts, p) + } + } + return strings.Join(parts, "\n") + case float64: + return strings.TrimSpace(fmt.Sprint(t)) + case bool: + return fmt.Sprint(t) + default: + return "" + } +} + +func extractEPRELFromAttrs(attrs map[string]any) any { + return eprel.ExtractFromAttrs(attrs) } // ProjectV1ProcessJobItems applies legacy partial-type field projection. @@ -471,6 +633,10 @@ func ProjectV1ProcessJobItems(storedType string, items []V1ProcessJobItem) []V1P projected["category_name"] = item["category_name"] case "title": projected["title"] = item["title"] + projected["name"] = item["name"] + if projected["name"] == nil { + projected["name"] = item["title"] + } projected["meta_title"] = item["meta_title"] case "description": projected["description"] = item["description"] diff --git a/apps/api/internal/processing/v1_legacy_test.go b/apps/api/internal/processing/v1_legacy_test.go index b2b0b4c..5b23570 100644 --- a/apps/api/internal/processing/v1_legacy_test.go +++ b/apps/api/internal/processing/v1_legacy_test.go @@ -76,7 +76,7 @@ func TestProjectV1ProcessJobItemsPartial(t *testing.T) { "processed_product_id": "bbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb", "raw_product_id": "cccccccc-cccc-cccc-cccc-cccccccccccc", "status": "processed", - "title": "T", "meta_title": "MT", + "title": "T", "name": "T", "meta_title": "MT", "description": "D", "attributes": map[string]any{"brand": "X"}, "main_image": "https://example.com/a.jpg", "more_images": []string{"https://example.com/b.jpg"}, "eprel": nil, "category": "cat", "category_name": "Cat", @@ -91,6 +91,9 @@ func TestProjectV1ProcessJobItemsPartial(t *testing.T) { if got["title"] != "T" || got["ean"] != "123" { t.Fatalf("got=%v", got) } + if got["name"] != "T" { + t.Fatalf("name dual-mode alias missing or mismatched: %v", got) + } if _, ok := got["attributes"]; ok { t.Fatalf("attributes should be projected out: %v", got) } @@ -108,6 +111,23 @@ func TestProjectV1ProcessJobItemsPartial(t *testing.T) { } } +func TestProjectV1ProcessJobItemsTitleDerivesName(t *testing.T) { + items := []V1ProcessJobItem{{ + "ean": "123", "id": "bbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb", + "status": "processed", "title": "OnlyTitle", "meta_title": "MT", + }} + out := ProjectV1ProcessJobItems("title", items) + if len(out) != 1 { + t.Fatalf("len=%d", len(out)) + } + if out[0]["title"] != "OnlyTitle" { + t.Fatalf("title=%v", out[0]["title"]) + } + if out[0]["name"] != "OnlyTitle" { + t.Fatalf("name should mirror title when absent: %v", out[0]["name"]) + } +} + func TestV1PlainDescriptionStandalone(t *testing.T) { got := v1PlainDescription("Line1
Line2 & ok") if strings.Contains(got, "<") { @@ -118,6 +138,43 @@ func TestV1PlainDescriptionStandalone(t *testing.T) { } } +func TestV1PlainDescriptionEdgeCases(t *testing.T) { + nbsp := "\u00a0" + cases := []struct { + name string + in string + want string + }{ + {name: "empty", in: "", want: ""}, + {name: "whitespace", in: " \n\t ", want: ""}, + {name: "plain", in: "Hello world", want: "Hello world"}, + {name: "html_br", in: "Hello
World
& morebold", want: "Hello\nWorld\n& more\nbold"}, + {name: "one_element_json_array_html", in: `["Hello
World"]`, want: "Hello\nWorld"}, + {name: "multi_element_array", in: `["Para one.", "Para two."]`, want: "Para one.\nPara two."}, + {name: "nested_array", in: `[["inner
x"]]`, want: "inner\nx"}, + {name: "json_quoted_string", in: `"Plain & quoted"`, want: "Plain & quoted"}, + {name: "empty_array", in: `[]`, want: ""}, + {name: "array_with_nulls", in: `[null, "keep", null, ""]`, want: "keep"}, + {name: "array_with_object_ignored", in: `[{"text":"x"}, "ok"]`, want: "ok"}, + {name: "invalid_bracket_text", in: `[not json`, want: "[not json"}, + {name: "nbsp_and_entities", in: "A" + nbsp + "B C", want: "A B C"}, + {name: "literal_unicode_escape", in: `A\u00a0B`, want: "A B"}, + {name: "double_encoded_array", in: `"[\"Hello
World\"]"`, want: "Hello\nWorld"}, + {name: "whitespace_around_array", in: " [\" spaced \"] ", want: "spaced"}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + got := v1PlainDescription(tc.in) + if got != tc.want { + t.Fatalf("in=%q got=%q want=%q", tc.in, got, tc.want) + } + if strings.Contains(got, "<") { + t.Fatalf("html left: %q", got) + } + }) + } +} + func TestApplyV1ProcessItemIDsDualMode(t *testing.T) { processed := mustParseTestUUID(t, "bbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb") raw := mustParseTestUUID(t, "cccccccc-cccc-cccc-cccc-cccccccccccc") diff --git a/apps/api/internal/processing/v1_process_item.go b/apps/api/internal/processing/v1_process_item.go new file mode 100644 index 0000000..8aba772 --- /dev/null +++ b/apps/api/internal/processing/v1_process_item.go @@ -0,0 +1,456 @@ +package processing + +import ( + "fmt" + "strings" + + "github.com/descrybe/descrybe-v2/apps/api/internal/eprel" +) + +// V1ProcessItemScorecard scores one COMPLETED legacy process item against +// OpenAPI LegacyProcessItem + A1 poll expectations. +type V1ProcessItemScorecard struct { + EAN string `json:"ean"` + Status string `json:"status"` + OK bool `json:"ok"` + Score int `json:"score"` // 0–100 + FailFlags []string `json:"fail_flags,omitempty"` + + HasTitle bool `json:"has_title"` + HasName bool `json:"has_name"` + HasDescription bool `json:"has_description"` + DescriptionIsPlain bool `json:"description_is_plain"` + HasMetaTitle bool `json:"has_meta_title"` + HasMetaDescription bool `json:"has_meta_description"` + HasCategory bool `json:"has_category"` + HasCategoryName bool `json:"has_category_name"` + AttrsClean bool `json:"attrs_clean"` + EPRELOK bool `json:"eprel_ok"` + HasIDs bool `json:"has_ids"` + ImagesOK bool `json:"images_ok"` +} + +// ScoreV1ProcessItemOptions tunes structure checks for a poll item. +type ScoreV1ProcessItemOptions struct { + // MappedCategory, when non-empty, requires item.category to equal it + // (projection must surface mapped unique_id). + MappedCategory string + // Language is used only for documentation of synthesize paths in tests. + Language string +} + +// ScoreV1ProcessCompletedItem scores a successful (or terminal) V1 process item. +// For status != processed it only checks ean/status shape and returns OK when those exist. +func ScoreV1ProcessCompletedItem(item V1ProcessJobItem, opts ScoreV1ProcessItemOptions) V1ProcessItemScorecard { + sc := V1ProcessItemScorecard{ + EAN: strings.TrimSpace(fmt.Sprint(item["ean"])), + Status: strings.TrimSpace(fmt.Sprint(item["status"])), + } + if sc.EAN == "" || sc.EAN == "
{i18n.t("products.edit.images")}
+
+ {#each imageUrls as src, i}
+
+ 
+
+ {/each}
+
+
+ {#if isImage}
+ 
+ {/if}
+
+ {field.value}
+
+
{:else}
{field.value}
{/if}
diff --git a/apps/web/src/lib/components/products/types.ts b/apps/web/src/lib/components/products/types.ts
index 0b9b58d..64204ba 100644
--- a/apps/web/src/lib/components/products/types.ts
+++ b/apps/web/src/lib/components/products/types.ts
@@ -847,6 +847,39 @@ function digAttrBag(
return undefined;
}
+/** Collect http(s) image URLs from mapped_data (main_image + more_images). */
+export function productImageUrls(product: ProductRow | null | undefined): string[] {
+ const mapped = asRecord(product?.mapped_data);
+ if (!mapped) return [];
+ const out: string[] = [];
+ const seen = new Set
-
-
-
+
+
+
+
+