package processing import ( "context" "encoding/json" "fmt" "strings" "github.com/descrybe/descrybe-v2/apps/api/internal/catalog" "github.com/descrybe/descrybe-v2/apps/api/internal/company" "github.com/google/uuid" "github.com/jackc/pgx/v5/pgxpool" ) // CatalogFixResult summarizes in-place catalog hygiene (never clears the catalog). type CatalogFixResult struct { WeakHashesCleared int `json:"weak_hashes_cleared"` CategoriesBackfilled int `json:"categories_backfilled"` DescriptionsNormalized int `json:"descriptions_normalized"` DescriptionsBackfilled int `json:"descriptions_backfilled"` NamesBackfilled int `json:"names_backfilled"` MappedDescriptionsFlat int `json:"mapped_descriptions_flattened"` MetaBackfilled int `json:"meta_backfilled"` ProductsScanned int `json:"products_scanned"` } // ClearWeakEnhanceHashes removes enhance_input_hash from field_sources and // localized_content when the product description is weak (poisoned skip hashes). // Delegates to catalog.RepairWeakEnhanceHashesDetailed (canonical implementation). func ClearWeakEnhanceHashes(ctx context.Context, pool *pgxpool.Pool, companyID uuid.UUID) (cleared int, scanned int, clearedRawIDs []uuid.UUID, err error) { res, err := catalog.RepairWeakEnhanceHashesDetailed(ctx, pool, companyID) if err != nil { return 0, 0, nil, err } return int(res.Cleared), res.Scanned, res.ClearedRawIDs, nil } // BackfillCategoriesFromMapped copies mapped_data category unique_ids onto // processed_products.category when the processed category is empty/"none". // Uses Go-side categoryUniqueIDFromMaps (nested/array shapes), coerces display // names onto taxonomy unique_ids, and validates against the company taxonomy // when categories exist. func BackfillCategoriesFromMapped(ctx context.Context, pool *pgxpool.Pool, companyID uuid.UUID) (updated int, err error) { if pool == nil { return 0, fmt.Errorf("nil pool") } valid, namesByUID, err := loadCompanyCategoryTaxonomy(ctx, pool, companyID) if err != nil { return 0, err } rows, err := pool.Query(ctx, ` SELECT pp.id, COALESCE(pp.category, ''), COALESCE(rp.mapped_data, '{}'::jsonb), COALESCE(rp.raw_data, '{}'::jsonb) FROM processed_products pp JOIN raw_products rp ON rp.id = pp.raw_product_id AND rp.company_id = pp.company_id WHERE pp.company_id = $1 AND ( pp.category IS NULL OR btrim(pp.category) = '' OR lower(btrim(pp.category)) = 'none' )`, companyID) if err != nil { return 0, err } defer rows.Close() for rows.Next() { var ( id uuid.UUID prior string mappedB []byte rawB []byte ) if err := rows.Scan(&id, &prior, &mappedB, &rawB); err != nil { return updated, err } mapped, err := decodeJSONObject(mappedB) if err != nil { return updated, err } raw, err := decodeJSONObject(rawB) if err != nil { return updated, err } cat := categoryUniqueIDFromMaps(mapped, raw) if cat == "" { continue } if resolved := resolveCompanyCategoryUniqueID(cat, namesByUID, valid); resolved != "" { cat = resolved } else if len(valid) > 0 { continue } ct, err := pool.Exec(ctx, ` UPDATE processed_products SET category = $2, field_sources = jsonb_set( COALESCE(field_sources, '{}'::jsonb), '{category}', '"mapped_backfill"'::jsonb, true ), updated_at = now() WHERE id = $1 AND company_id = $3 AND ( category IS NULL OR btrim(category) = '' OR lower(btrim(category)) = 'none' )`, id, cat, companyID) if err != nil { return updated, err } if ct.RowsAffected() > 0 { updated++ } _ = prior } return updated, rows.Err() } func loadCompanyCategoryUniqueIDs(ctx context.Context, pool *pgxpool.Pool, companyID uuid.UUID) (map[string]struct{}, error) { valid, _, err := loadCompanyCategoryTaxonomy(ctx, pool, companyID) return valid, err } func loadCompanyCategoryTaxonomy(ctx context.Context, pool *pgxpool.Pool, companyID uuid.UUID) (map[string]struct{}, map[string]string, error) { valid := map[string]struct{}{} namesByUID := map[string]string{} rows, err := pool.Query(ctx, ` SELECT unique_id, name FROM categories WHERE company_id = $1 AND COALESCE(NULLIF(btrim(unique_id), ''), '') <> ''`, companyID) if err != nil { return nil, nil, err } defer rows.Close() for rows.Next() { var uid, name string if err := rows.Scan(&uid, &name); err != nil { return nil, nil, err } uid = strings.TrimSpace(uid) name = strings.TrimSpace(name) if uid == "" { continue } valid[uid] = struct{}{} if name != "" { namesByUID[uid] = name } } return valid, namesByUID, rows.Err() } // FixCatalogHygiene clears poisoned enhance hashes and optionally backfills categories. func FixCatalogHygiene(ctx context.Context, pool *pgxpool.Pool, companyID uuid.UUID) (CatalogFixResult, error) { out, _, err := FixCatalogHygieneWithIDs(ctx, pool, companyID, true) return out, err } // FixCatalogHygieneWithIDs is FixCatalogHygiene plus raw_product_ids that had weak hashes cleared. func FixCatalogHygieneWithIDs(ctx context.Context, pool *pgxpool.Pool, companyID uuid.UUID, backfillCategories bool) (CatalogFixResult, []uuid.UUID, error) { var out CatalogFixResult cleared, scanned, ids, err := ClearWeakEnhanceHashes(ctx, pool, companyID) if err != nil { return out, nil, err } out.WeakHashesCleared = cleared out.ProductsScanned = scanned flat, err := FlattenMappedDescriptionArrays(ctx, pool, companyID) if err != nil { return out, ids, err } out.MappedDescriptionsFlat = flat if backfillCategories { // Go-side unique_id extraction (nested/array) + taxonomy validation. cats, err := BackfillCategoriesFromMapped(ctx, pool, companyID) if err != nil { return out, ids, err } out.CategoriesBackfilled = cats descN, err := BackfillDescriptionsFromMapped(ctx, pool, companyID) if err != nil { return out, ids, err } out.DescriptionsBackfilled = descN nameN, err := BackfillPollutedNamesFromMapped(ctx, pool, companyID) if err != nil { return out, ids, err } out.NamesBackfilled = nameN metaN, err := BackfillMissingMeta(ctx, pool, companyID) if err != nil { return out, ids, err } out.MetaBackfilled = metaN } descs, err := NormalizeProcessedDescriptions(ctx, pool, companyID) if err != nil { return out, ids, err } out.DescriptionsNormalized = descs return out, ids, nil } // NormalizeProcessedDescriptions rewrites processed_description to the plain-text // form produced by PlainDescriptionFromAny (unwraps JSON arrays, strips HTML). When // processed_description is empty, normalizes from description. Returns rows updated. func NormalizeProcessedDescriptions(ctx context.Context, pool *pgxpool.Pool, companyID uuid.UUID) (int, error) { if pool == nil { return 0, fmt.Errorf("normalize descriptions: nil pool") } rows, err := pool.Query(ctx, ` SELECT id, COALESCE(description, ''), COALESCE(processed_description, '') FROM processed_products WHERE company_id = $1`, companyID) if err != nil { return 0, fmt.Errorf("normalize descriptions query: %w", err) } defer rows.Close() updated := 0 for rows.Next() { var ( id uuid.UUID desc, processedDesc string ) if err := rows.Scan(&id, &desc, &processedDesc); err != nil { return updated, fmt.Errorf("normalize descriptions scan: %w", err) } raw := strings.TrimSpace(processedDesc) if raw == "" { raw = strings.TrimSpace(desc) } if raw == "" { continue } plain := plainDescriptionFromStored(raw) if plain == "" || plain == strings.TrimSpace(processedDesc) { continue } ct, err := pool.Exec(ctx, ` UPDATE processed_products SET processed_description = $2, updated_at = now() WHERE id = $1 AND company_id = $3`, id, plain, companyID) if err != nil { return updated, fmt.Errorf("normalize descriptions update %s: %w", id, err) } updated += int(ct.RowsAffected()) } if err := rows.Err(); err != nil { return updated, fmt.Errorf("normalize descriptions rows: %w", err) } return updated, nil } // plainDescriptionFromStored handles DB text that may itself be a JSON array/string. func plainDescriptionFromStored(raw string) string { raw = strings.TrimSpace(raw) if raw == "" { return "" } if strings.HasPrefix(raw, "[") || strings.HasPrefix(raw, "{") { var decoded any if err := json.Unmarshal([]byte(raw), &decoded); err == nil { if s := PlainDescriptionFromAny(decoded); s != "" { return s } } } return PlainDescriptionFromAny(raw) } // RecommendReprocessRawProductIDs returns how many products should be reprocessed // after Fix A1 (weak desc and/or missing enhance hash and/or empty category), plus // an optional sample of raw_product_ids (limit 0 = no sample). func RecommendReprocessRawProductIDs(ctx context.Context, pool *pgxpool.Pool, companyID uuid.UUID, prefer []uuid.UUID, sampleLimit int) (needed int, sample []uuid.UUID, err error) { if pool == nil { return 0, nil, fmt.Errorf("nil pool") } seen := map[uuid.UUID]struct{}{} add := func(id uuid.UUID) { if id == uuid.Nil { return } if _, ok := seen[id]; ok { return } seen[id] = struct{}{} needed++ if sampleLimit > 0 && len(sample) < sampleLimit { sample = append(sample, id) } } for _, id := range prefer { add(id) } rows, err := pool.Query(ctx, ` SELECT raw_product_id, COALESCE(name, ''), COALESCE(description, ''), COALESCE(processed_name, ''), COALESCE(processed_description, ''), COALESCE(category, ''), COALESCE(field_sources, '{}'::jsonb), COALESCE(localized_content, '{}'::jsonb) FROM processed_products WHERE company_id = $1`, companyID) if err != nil { return 0, nil, err } defer rows.Close() for rows.Next() { var ( rawID uuid.UUID name, desc, processedName, processedDesc, category string fsRaw, locRaw []byte ) if err := rows.Scan(&rawID, &name, &desc, &processedName, &processedDesc, &category, &fsRaw, &locRaw); err != nil { return needed, sample, err } primaryDesc := strings.TrimSpace(processedDesc) if primaryDesc == "" { primaryDesc = desc } primaryName := strings.TrimSpace(processedName) if primaryName == "" { primaryName = name } fs, err := decodeJSONObject(fsRaw) if err != nil { return needed, sample, err } loc, err := company.DecodeLocalizedContent(locRaw) if err != nil { return needed, sample, err } needs := false if strings.TrimSpace(category) == "" { needs = true } if isPromptLabelTitle(primaryName) || isPromptLabelTitle(processedName) || isPromptLabelTitle(name) { needs = true } if company.ShouldRefuseEnhanceHashSkip(primaryDesc, primaryName, name) { needs = true } if _, has := fs[FieldEnhanceInputHash]; !has { // Missing hash after weak clear / never enhanced — recommend when desc weak or empty category. if company.ShouldRefuseEnhanceHashSkip(primaryDesc, primaryName, name) || strings.TrimSpace(category) == "" { needs = true } } for _, fields := range loc { d := strings.TrimSpace(fields.ProcessedDescription) n := strings.TrimSpace(fields.ProcessedName) if n == "" { n = primaryName } if fields.EnhanceInputHash == "" && company.ShouldRefuseEnhanceHashSkip(d, n, primaryName, name) { needs = true break } } if needs { add(rawID) } } return needed, sample, rows.Err() } func decodeJSONObject(raw []byte) (map[string]any, error) { out := map[string]any{} if len(raw) == 0 { return out, nil } if err := json.Unmarshal(raw, &out); err != nil { return nil, err } if out == nil { out = map[string]any{} } return out, nil }