package processing import ( "context" "encoding/json" "fmt" "strings" "github.com/google/uuid" "github.com/jackc/pgx/v5/pgxpool" ) // BackfillDescriptionsFromMapped fills empty processed_products.description from // linked raw mapped_data (mirrors migrator backfillProcessedDescriptionsFromMapped) // and coerces array/HTML shapes via PlainDescriptionFromAny. func BackfillDescriptionsFromMapped(ctx context.Context, pool *pgxpool.Pool, companyID uuid.UUID) (updated int, err error) { if pool == nil { return 0, fmt.Errorf("description backfill: nil pool") } if companyID == uuid.Nil { return 0, fmt.Errorf("description backfill: empty company id") } rows, err := pool.Query(ctx, ` SELECT p.id, COALESCE(p.description, ''), COALESCE(r.mapped_data, '{}'::jsonb), COALESCE(r.raw_data, '{}'::jsonb) FROM processed_products p INNER JOIN raw_products r ON r.id = p.raw_product_id AND r.company_id = p.company_id WHERE p.company_id = $1 AND COALESCE(NULLIF(BTRIM(p.description), ''), '') = ''`, companyID) if err != nil { return 0, fmt.Errorf("description backfill query: %w", err) } defer rows.Close() for rows.Next() { var ( id uuid.UUID prior string mappedB []byte rawB []byte ) if err := rows.Scan(&id, &prior, &mappedB, &rawB); err != nil { return updated, fmt.Errorf("description backfill scan: %w", err) } mapped, err := decodeJSONObject(mappedB) if err != nil { return updated, err } raw, err := decodeJSONObject(rawB) if err != nil { return updated, err } desc := PlainDescriptionFromAny(mapped["description"]) if desc == "" { desc = PlainDescriptionFromAny(mapped["desc"]) } if desc == "" { desc = PlainDescriptionFromAny(mapped["body"]) } if desc == "" { desc = PlainDescriptionFromAny(raw["description"]) } if desc == "" { continue } ct, err := pool.Exec(ctx, ` UPDATE processed_products SET description = $2, updated_at = now() WHERE id = $1 AND company_id = $3 AND COALESCE(NULLIF(BTRIM(description), ''), '') = ''`, id, desc, companyID) if err != nil { return updated, fmt.Errorf("description backfill update %s: %w", id, err) } if ct.RowsAffected() > 0 { updated++ } _ = prior } return updated, rows.Err() } // mappedFeedTitle picks a clean name/title from mapped or raw feed maps. func mappedFeedTitle(mapped, raw map[string]any) string { for _, src := range []map[string]any{mapped, raw} { if src == nil { continue } for _, key := range []string{"name", "title", "product_name", "product_title"} { v := strings.TrimSpace(stringFromAny(src[key])) if v == "" || v == "" || isPromptLabelTitle(v) { continue } return SanitizeOutput(v) } } return "" } // BackfillPollutedNamesFromMapped clears processed_name / name when they match // prompt-leakage patterns (Title formula / short retail title / …) and replaces // them with mapped_data name/title when available. Next Load/FixCatalog or // reprocess then shows the feed title instead of instruction text. func BackfillPollutedNamesFromMapped(ctx context.Context, pool *pgxpool.Pool, companyID uuid.UUID) (updated int, err error) { if pool == nil { return 0, fmt.Errorf("name pollution backfill: nil pool") } if companyID == uuid.Nil { return 0, fmt.Errorf("name pollution backfill: empty company id") } rows, err := pool.Query(ctx, ` SELECT p.id, COALESCE(p.name, ''), COALESCE(p.processed_name, ''), COALESCE(r.mapped_data, '{}'::jsonb), COALESCE(r.raw_data, '{}'::jsonb) FROM processed_products p INNER JOIN raw_products r ON r.id = p.raw_product_id AND r.company_id = p.company_id WHERE p.company_id = $1`, companyID) if err != nil { return 0, fmt.Errorf("name pollution backfill query: %w", err) } defer rows.Close() for rows.Next() { var ( id uuid.UUID name, processedName string mappedB, rawB []byte ) if err := rows.Scan(&id, &name, &processedName, &mappedB, &rawB); err != nil { return updated, fmt.Errorf("name pollution backfill scan: %w", err) } namePolluted := isPromptLabelTitle(name) procPolluted := isPromptLabelTitle(processedName) if !namePolluted && !procPolluted { continue } mapped, err := decodeJSONObject(mappedB) if err != nil { return updated, err } raw, err := decodeJSONObject(rawB) if err != nil { return updated, err } feed := mappedFeedTitle(mapped, raw) if feed == "" { continue } newName := name newProcessed := processedName if namePolluted { newName = feed } if procPolluted { newProcessed = feed } if newName == name && newProcessed == processedName { continue } ct, err := pool.Exec(ctx, ` UPDATE processed_products SET name = $2, processed_name = $3, field_sources = jsonb_set( COALESCE(field_sources, '{}'::jsonb), '{name}', '"mapped_pollution_backfill"'::jsonb, true ), updated_at = now() WHERE id = $1 AND company_id = $4`, id, newName, newProcessed, companyID) if err != nil { return updated, fmt.Errorf("name pollution backfill update %s: %w", id, err) } if ct.RowsAffected() > 0 { updated++ } } return updated, rows.Err() } // BackfillMissingMeta fills empty meta_title / meta_description using the same // free template fillMetaFromResult uses at process time (no AI / no credits). func BackfillMissingMeta(ctx context.Context, pool *pgxpool.Pool, companyID uuid.UUID) (updated int, err error) { if pool == nil { return 0, fmt.Errorf("meta backfill: nil pool") } if companyID == uuid.Nil { return 0, fmt.Errorf("meta backfill: empty company id") } rows, err := pool.Query(ctx, ` SELECT id, COALESCE(name, ''), COALESCE(processed_name, ''), COALESCE(category, ''), COALESCE(description, ''), COALESCE(processed_description, ''), COALESCE(meta_title, ''), COALESCE(meta_description, ''), COALESCE(attributes, '{}'::jsonb), COALESCE(processed_attributes, '{}'::jsonb) FROM processed_products WHERE company_id = $1 AND ( COALESCE(NULLIF(BTRIM(meta_title), ''), '') = '' OR COALESCE(NULLIF(BTRIM(meta_description), ''), '') = '' OR BTRIM(meta_title) ~ '\|\s+[0-9]+$' OR LOWER(meta_title) LIKE '%short retail title%' OR LOWER(meta_title) LIKE '%title formula%' OR LOWER(meta_title) LIKE '%follow any%' OR LOWER(meta_title) LIKE '%constraints that follow%' OR LOWER(meta_title) LIKE '%prefer 1-3%' OR LOWER(meta_title) LIKE '%reply with only json%' OR LOWER(meta_title) LIKE '%your reply is parsed as json%' OR LOWER(meta_description) LIKE '%prefer 1-3%' OR LOWER(meta_description) LIKE '%short retail title%' OR LOWER(meta_description) LIKE '%title formula%' OR LOWER(meta_description) LIKE '%do not emit%' )`, companyID) if err != nil { return 0, fmt.Errorf("meta backfill query: %w", err) } defer rows.Close() for rows.Next() { var ( id uuid.UUID name, processedName, category string desc, processedDesc, metaTitle, metaDesc string attrsJSON, procAttrsJSON []byte ) if err := rows.Scan(&id, &name, &processedName, &category, &desc, &processedDesc, &metaTitle, &metaDesc, &attrsJSON, &procAttrsJSON); err != nil { return updated, fmt.Errorf("meta backfill scan: %w", err) } attrs := decodeAttrMap(attrsJSON) procAttrs := decodeAttrMap(procAttrsJSON) synthTitle, synthDesc := fillMetaFromResult(StepResult{ Name: name, ProcessedName: processedName, Category: category, Description: PlainDescriptionFromAny(desc), ProcessedDescription: PlainDescriptionFromAny(processedDesc), Attributes: attrs, ProcessedAttributes: procAttrs, }) newTitle := strings.TrimSpace(metaTitle) newDesc := strings.TrimSpace(metaDesc) if newTitle == "" || isPoisonedMetaTitle(newTitle) { newTitle = synthTitle } if newDesc == "" || isPromptLeakageTitle(newDesc) || (isPoisonedMetaTitle(strings.TrimSpace(metaTitle)) && (isWeakPriorEnhanceDescription(newDesc, name, processedName) || isPromptLeakageTitle(newDesc))) { newDesc = synthDesc } if newTitle == strings.TrimSpace(metaTitle) && newDesc == strings.TrimSpace(metaDesc) { continue } if newTitle == "" && newDesc == "" { continue } ct, err := pool.Exec(ctx, ` UPDATE processed_products SET meta_title = CASE WHEN COALESCE(NULLIF(BTRIM(meta_title), ''), '') = '' OR BTRIM(meta_title) ~ '\|\s+[0-9]+$' OR LOWER(meta_title) LIKE '%short retail title%' OR LOWER(meta_title) LIKE '%title formula%' OR LOWER(meta_title) LIKE '%follow any%' OR LOWER(meta_title) LIKE '%constraints that follow%' OR LOWER(meta_title) LIKE '%prefer 1-3%' OR LOWER(meta_title) LIKE '%reply with only json%' OR LOWER(meta_title) LIKE '%your reply is parsed as json%' THEN NULLIF(BTRIM($2), '') ELSE meta_title END, meta_description = CASE WHEN COALESCE(NULLIF(BTRIM(meta_description), ''), '') = '' THEN NULLIF(BTRIM($3), '') WHEN LOWER(meta_description) LIKE '%prefer 1-3%' OR LOWER(meta_description) LIKE '%short retail title%' OR LOWER(meta_description) LIKE '%title formula%' OR LOWER(meta_description) LIKE '%do not emit%' THEN NULLIF(BTRIM($3), '') WHEN ( BTRIM(meta_title) ~ '\|\s+[0-9]+$' OR LOWER(meta_title) LIKE '%short retail title%' OR LOWER(meta_title) LIKE '%title formula%' OR LOWER(meta_title) LIKE '%follow any%' OR LOWER(meta_title) LIKE '%constraints that follow%' OR LOWER(meta_title) LIKE '%prefer 1-3%' OR LOWER(meta_title) LIKE '%reply with only json%' OR LOWER(meta_title) LIKE '%your reply is parsed as json%' ) AND ( char_length(BTRIM(meta_description)) < 40 OR LOWER(meta_description) LIKE '%ready for retail listing%' OR LOWER(meta_description) LIKE '%quality product ready%' OR LOWER(meta_description) LIKE '%product description%' OR LOWER(meta_description) LIKE '%based on available specifications%' OR LOWER(meta_description) LIKE '%with available specifications%' OR LOWER(meta_description) LIKE '%available catalog details%' OR LOWER(meta_description) LIKE '%pripravljeno za prodajo%' OR LOWER(meta_description) LIKE '%na podlagi razpoložljivih specifikacij%' OR LOWER(meta_description) LIKE '%prefer 1-3%' ) THEN NULLIF(BTRIM($3), '') ELSE meta_description END, updated_at = now() WHERE id = $1 AND company_id = $4`, id, newTitle, newDesc, companyID) if err != nil { return updated, fmt.Errorf("meta backfill update %s: %w", id, err) } if ct.RowsAffected() > 0 { updated++ } } return updated, rows.Err() } // FlattenMappedDescriptionArrays rewrites mapped_data.description when it is a // JSON array / HTML blob into a plain string (local admin Fix A1 hygiene). func FlattenMappedDescriptionArrays(ctx context.Context, pool *pgxpool.Pool, companyID uuid.UUID) (updated int, err error) { if pool == nil { return 0, fmt.Errorf("flatten mapped descriptions: nil pool") } rows, err := pool.Query(ctx, ` SELECT id, COALESCE(mapped_data, '{}'::jsonb) FROM raw_products WHERE company_id = $1 AND mapped_data ? 'description' AND jsonb_typeof(mapped_data->'description') <> 'string'`, companyID) if err != nil { return 0, fmt.Errorf("flatten mapped descriptions query: %w", err) } defer rows.Close() for rows.Next() { var id uuid.UUID var mappedB []byte if err := rows.Scan(&id, &mappedB); err != nil { return updated, err } mapped, err := decodeJSONObject(mappedB) if err != nil { return updated, err } rawDesc := mapped["description"] plain := PlainDescriptionFromAny(rawDesc) if plain == "" { continue } // Skip when already a plain string equal to conversion (shouldn't hit query). if s, ok := rawDesc.(string); ok && v1PlainDescription(s) == plain { continue } mapped["description"] = plain encoded, err := json.Marshal(mapped) if err != nil { return updated, err } ct, err := pool.Exec(ctx, ` UPDATE raw_products SET mapped_data = $2::jsonb, updated_at = now() WHERE id = $1 AND company_id = $3`, id, string(encoded), companyID) if err != nil { return updated, fmt.Errorf("flatten mapped description %s: %w", id, err) } if ct.RowsAffected() > 0 { updated++ } } return updated, rows.Err() }