383 lines
12 KiB
Go
383 lines
12 KiB
Go
package processing
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"fmt"
|
|
"strings"
|
|
|
|
"github.com/google/uuid"
|
|
"github.com/jackc/pgx/v5/pgxpool"
|
|
)
|
|
|
|
// BackfillDescriptionsFromMapped fills empty processed_products.description from
|
|
// linked raw mapped_data (mirrors migrator backfillProcessedDescriptionsFromMapped)
|
|
// and coerces array/HTML shapes via PlainDescriptionFromAny.
|
|
func BackfillDescriptionsFromMapped(ctx context.Context, pool *pgxpool.Pool, companyID uuid.UUID) (updated int, err error) {
|
|
if pool == nil {
|
|
return 0, fmt.Errorf("description backfill: nil pool")
|
|
}
|
|
if companyID == uuid.Nil {
|
|
return 0, fmt.Errorf("description backfill: empty company id")
|
|
}
|
|
rows, err := pool.Query(ctx, `
|
|
SELECT p.id,
|
|
COALESCE(p.description, ''),
|
|
COALESCE(r.mapped_data, '{}'::jsonb),
|
|
COALESCE(r.raw_data, '{}'::jsonb)
|
|
FROM processed_products p
|
|
INNER JOIN raw_products r ON r.id = p.raw_product_id AND r.company_id = p.company_id
|
|
WHERE p.company_id = $1
|
|
AND COALESCE(NULLIF(BTRIM(p.description), ''), '') = ''`, companyID)
|
|
if err != nil {
|
|
return 0, fmt.Errorf("description backfill query: %w", err)
|
|
}
|
|
defer rows.Close()
|
|
|
|
for rows.Next() {
|
|
var (
|
|
id uuid.UUID
|
|
prior string
|
|
mappedB []byte
|
|
rawB []byte
|
|
)
|
|
if err := rows.Scan(&id, &prior, &mappedB, &rawB); err != nil {
|
|
return updated, fmt.Errorf("description backfill scan: %w", err)
|
|
}
|
|
mapped, err := decodeJSONObject(mappedB)
|
|
if err != nil {
|
|
return updated, err
|
|
}
|
|
raw, err := decodeJSONObject(rawB)
|
|
if err != nil {
|
|
return updated, err
|
|
}
|
|
desc := PlainDescriptionFromAny(mapped["description"])
|
|
if desc == "" {
|
|
desc = PlainDescriptionFromAny(mapped["desc"])
|
|
}
|
|
if desc == "" {
|
|
desc = PlainDescriptionFromAny(mapped["body"])
|
|
}
|
|
if desc == "" {
|
|
desc = PlainDescriptionFromAny(raw["description"])
|
|
}
|
|
if desc == "" {
|
|
continue
|
|
}
|
|
ct, err := pool.Exec(ctx, `
|
|
UPDATE processed_products
|
|
SET description = $2,
|
|
updated_at = now()
|
|
WHERE id = $1 AND company_id = $3
|
|
AND COALESCE(NULLIF(BTRIM(description), ''), '') = ''`,
|
|
id, desc, companyID)
|
|
if err != nil {
|
|
return updated, fmt.Errorf("description backfill update %s: %w", id, err)
|
|
}
|
|
if ct.RowsAffected() > 0 {
|
|
updated++
|
|
}
|
|
_ = prior
|
|
}
|
|
return updated, rows.Err()
|
|
}
|
|
|
|
// mappedFeedTitle picks a clean name/title from mapped or raw feed maps.
|
|
func mappedFeedTitle(mapped, raw map[string]any) string {
|
|
for _, src := range []map[string]any{mapped, raw} {
|
|
if src == nil {
|
|
continue
|
|
}
|
|
for _, key := range []string{"name", "title", "product_name", "product_title"} {
|
|
v := strings.TrimSpace(stringFromAny(src[key]))
|
|
if v == "" || v == "<nil>" || isPromptLabelTitle(v) {
|
|
continue
|
|
}
|
|
return SanitizeOutput(v)
|
|
}
|
|
}
|
|
return ""
|
|
}
|
|
|
|
// BackfillPollutedNamesFromMapped clears processed_name / name when they match
|
|
// prompt-leakage patterns (Title formula / short retail title / …) and replaces
|
|
// them with mapped_data name/title when available. Next Load/FixCatalog or
|
|
// reprocess then shows the feed title instead of instruction text.
|
|
func BackfillPollutedNamesFromMapped(ctx context.Context, pool *pgxpool.Pool, companyID uuid.UUID) (updated int, err error) {
|
|
if pool == nil {
|
|
return 0, fmt.Errorf("name pollution backfill: nil pool")
|
|
}
|
|
if companyID == uuid.Nil {
|
|
return 0, fmt.Errorf("name pollution backfill: empty company id")
|
|
}
|
|
rows, err := pool.Query(ctx, `
|
|
SELECT p.id,
|
|
COALESCE(p.name, ''),
|
|
COALESCE(p.processed_name, ''),
|
|
COALESCE(r.mapped_data, '{}'::jsonb),
|
|
COALESCE(r.raw_data, '{}'::jsonb)
|
|
FROM processed_products p
|
|
INNER JOIN raw_products r ON r.id = p.raw_product_id AND r.company_id = p.company_id
|
|
WHERE p.company_id = $1`, companyID)
|
|
if err != nil {
|
|
return 0, fmt.Errorf("name pollution backfill query: %w", err)
|
|
}
|
|
defer rows.Close()
|
|
|
|
for rows.Next() {
|
|
var (
|
|
id uuid.UUID
|
|
name, processedName string
|
|
mappedB, rawB []byte
|
|
)
|
|
if err := rows.Scan(&id, &name, &processedName, &mappedB, &rawB); err != nil {
|
|
return updated, fmt.Errorf("name pollution backfill scan: %w", err)
|
|
}
|
|
namePolluted := isPromptLabelTitle(name)
|
|
procPolluted := isPromptLabelTitle(processedName)
|
|
if !namePolluted && !procPolluted {
|
|
continue
|
|
}
|
|
mapped, err := decodeJSONObject(mappedB)
|
|
if err != nil {
|
|
return updated, err
|
|
}
|
|
raw, err := decodeJSONObject(rawB)
|
|
if err != nil {
|
|
return updated, err
|
|
}
|
|
feed := mappedFeedTitle(mapped, raw)
|
|
if feed == "" {
|
|
continue
|
|
}
|
|
newName := name
|
|
newProcessed := processedName
|
|
if namePolluted {
|
|
newName = feed
|
|
}
|
|
if procPolluted {
|
|
newProcessed = feed
|
|
}
|
|
if newName == name && newProcessed == processedName {
|
|
continue
|
|
}
|
|
ct, err := pool.Exec(ctx, `
|
|
UPDATE processed_products
|
|
SET name = $2,
|
|
processed_name = $3,
|
|
field_sources = jsonb_set(
|
|
COALESCE(field_sources, '{}'::jsonb),
|
|
'{name}',
|
|
'"mapped_pollution_backfill"'::jsonb,
|
|
true
|
|
),
|
|
updated_at = now()
|
|
WHERE id = $1 AND company_id = $4`,
|
|
id, newName, newProcessed, companyID)
|
|
if err != nil {
|
|
return updated, fmt.Errorf("name pollution backfill update %s: %w", id, err)
|
|
}
|
|
if ct.RowsAffected() > 0 {
|
|
updated++
|
|
}
|
|
}
|
|
return updated, rows.Err()
|
|
}
|
|
|
|
// BackfillMissingMeta fills empty meta_title / meta_description using the same
|
|
// free template fillMetaFromResult uses at process time (no AI / no credits).
|
|
func BackfillMissingMeta(ctx context.Context, pool *pgxpool.Pool, companyID uuid.UUID) (updated int, err error) {
|
|
if pool == nil {
|
|
return 0, fmt.Errorf("meta backfill: nil pool")
|
|
}
|
|
if companyID == uuid.Nil {
|
|
return 0, fmt.Errorf("meta backfill: empty company id")
|
|
}
|
|
rows, err := pool.Query(ctx, `
|
|
SELECT id,
|
|
COALESCE(name, ''),
|
|
COALESCE(processed_name, ''),
|
|
COALESCE(category, ''),
|
|
COALESCE(description, ''),
|
|
COALESCE(processed_description, ''),
|
|
COALESCE(meta_title, ''),
|
|
COALESCE(meta_description, ''),
|
|
COALESCE(attributes, '{}'::jsonb),
|
|
COALESCE(processed_attributes, '{}'::jsonb)
|
|
FROM processed_products
|
|
WHERE company_id = $1
|
|
AND (
|
|
COALESCE(NULLIF(BTRIM(meta_title), ''), '') = ''
|
|
OR COALESCE(NULLIF(BTRIM(meta_description), ''), '') = ''
|
|
OR BTRIM(meta_title) ~ '\|\s+[0-9]+$'
|
|
OR LOWER(meta_title) LIKE '%short retail title%'
|
|
OR LOWER(meta_title) LIKE '%title formula%'
|
|
OR LOWER(meta_title) LIKE '%follow any%'
|
|
OR LOWER(meta_title) LIKE '%constraints that follow%'
|
|
OR LOWER(meta_title) LIKE '%prefer 1-3%'
|
|
OR LOWER(meta_title) LIKE '%reply with only json%'
|
|
OR LOWER(meta_title) LIKE '%your reply is parsed as json%'
|
|
OR LOWER(meta_description) LIKE '%prefer 1-3%'
|
|
OR LOWER(meta_description) LIKE '%short retail title%'
|
|
OR LOWER(meta_description) LIKE '%title formula%'
|
|
OR LOWER(meta_description) LIKE '%do not emit%'
|
|
)`, companyID)
|
|
if err != nil {
|
|
return 0, fmt.Errorf("meta backfill query: %w", err)
|
|
}
|
|
defer rows.Close()
|
|
|
|
for rows.Next() {
|
|
var (
|
|
id uuid.UUID
|
|
name, processedName, category string
|
|
desc, processedDesc, metaTitle, metaDesc string
|
|
attrsJSON, procAttrsJSON []byte
|
|
)
|
|
if err := rows.Scan(&id, &name, &processedName, &category, &desc, &processedDesc,
|
|
&metaTitle, &metaDesc, &attrsJSON, &procAttrsJSON); err != nil {
|
|
return updated, fmt.Errorf("meta backfill scan: %w", err)
|
|
}
|
|
attrs := decodeAttrMap(attrsJSON)
|
|
procAttrs := decodeAttrMap(procAttrsJSON)
|
|
synthTitle, synthDesc := fillMetaFromResult(StepResult{
|
|
Name: name,
|
|
ProcessedName: processedName,
|
|
Category: category,
|
|
Description: PlainDescriptionFromAny(desc),
|
|
ProcessedDescription: PlainDescriptionFromAny(processedDesc),
|
|
Attributes: attrs,
|
|
ProcessedAttributes: procAttrs,
|
|
})
|
|
newTitle := strings.TrimSpace(metaTitle)
|
|
newDesc := strings.TrimSpace(metaDesc)
|
|
if newTitle == "" || isPoisonedMetaTitle(newTitle) {
|
|
newTitle = synthTitle
|
|
}
|
|
if newDesc == "" || isPromptLeakageTitle(newDesc) ||
|
|
(isPoisonedMetaTitle(strings.TrimSpace(metaTitle)) &&
|
|
(isWeakPriorEnhanceDescription(newDesc, name, processedName) || isPromptLeakageTitle(newDesc))) {
|
|
newDesc = synthDesc
|
|
}
|
|
if newTitle == strings.TrimSpace(metaTitle) && newDesc == strings.TrimSpace(metaDesc) {
|
|
continue
|
|
}
|
|
if newTitle == "" && newDesc == "" {
|
|
continue
|
|
}
|
|
ct, err := pool.Exec(ctx, `
|
|
UPDATE processed_products
|
|
SET meta_title = CASE
|
|
WHEN COALESCE(NULLIF(BTRIM(meta_title), ''), '') = ''
|
|
OR BTRIM(meta_title) ~ '\|\s+[0-9]+$'
|
|
OR LOWER(meta_title) LIKE '%short retail title%'
|
|
OR LOWER(meta_title) LIKE '%title formula%'
|
|
OR LOWER(meta_title) LIKE '%follow any%'
|
|
OR LOWER(meta_title) LIKE '%constraints that follow%'
|
|
OR LOWER(meta_title) LIKE '%prefer 1-3%'
|
|
OR LOWER(meta_title) LIKE '%reply with only json%'
|
|
OR LOWER(meta_title) LIKE '%your reply is parsed as json%'
|
|
THEN NULLIF(BTRIM($2), '')
|
|
ELSE meta_title
|
|
END,
|
|
meta_description = CASE
|
|
WHEN COALESCE(NULLIF(BTRIM(meta_description), ''), '') = '' THEN NULLIF(BTRIM($3), '')
|
|
WHEN LOWER(meta_description) LIKE '%prefer 1-3%'
|
|
OR LOWER(meta_description) LIKE '%short retail title%'
|
|
OR LOWER(meta_description) LIKE '%title formula%'
|
|
OR LOWER(meta_description) LIKE '%do not emit%'
|
|
THEN NULLIF(BTRIM($3), '')
|
|
WHEN (
|
|
BTRIM(meta_title) ~ '\|\s+[0-9]+$'
|
|
OR LOWER(meta_title) LIKE '%short retail title%'
|
|
OR LOWER(meta_title) LIKE '%title formula%'
|
|
OR LOWER(meta_title) LIKE '%follow any%'
|
|
OR LOWER(meta_title) LIKE '%constraints that follow%'
|
|
OR LOWER(meta_title) LIKE '%prefer 1-3%'
|
|
OR LOWER(meta_title) LIKE '%reply with only json%'
|
|
OR LOWER(meta_title) LIKE '%your reply is parsed as json%'
|
|
)
|
|
AND (
|
|
char_length(BTRIM(meta_description)) < 40
|
|
OR LOWER(meta_description) LIKE '%ready for retail listing%'
|
|
OR LOWER(meta_description) LIKE '%quality product ready%'
|
|
OR LOWER(meta_description) LIKE '%product description%'
|
|
OR LOWER(meta_description) LIKE '%based on available specifications%'
|
|
OR LOWER(meta_description) LIKE '%with available specifications%'
|
|
OR LOWER(meta_description) LIKE '%available catalog details%'
|
|
OR LOWER(meta_description) LIKE '%pripravljeno za prodajo%'
|
|
OR LOWER(meta_description) LIKE '%na podlagi razpoložljivih specifikacij%'
|
|
OR LOWER(meta_description) LIKE '%prefer 1-3%'
|
|
)
|
|
THEN NULLIF(BTRIM($3), '')
|
|
ELSE meta_description
|
|
END,
|
|
updated_at = now()
|
|
WHERE id = $1 AND company_id = $4`,
|
|
id, newTitle, newDesc, companyID)
|
|
if err != nil {
|
|
return updated, fmt.Errorf("meta backfill update %s: %w", id, err)
|
|
}
|
|
if ct.RowsAffected() > 0 {
|
|
updated++
|
|
}
|
|
}
|
|
return updated, rows.Err()
|
|
}
|
|
|
|
// FlattenMappedDescriptionArrays rewrites mapped_data.description when it is a
|
|
// JSON array / HTML blob into a plain string (local admin Fix A1 hygiene).
|
|
func FlattenMappedDescriptionArrays(ctx context.Context, pool *pgxpool.Pool, companyID uuid.UUID) (updated int, err error) {
|
|
if pool == nil {
|
|
return 0, fmt.Errorf("flatten mapped descriptions: nil pool")
|
|
}
|
|
rows, err := pool.Query(ctx, `
|
|
SELECT id, COALESCE(mapped_data, '{}'::jsonb)
|
|
FROM raw_products
|
|
WHERE company_id = $1
|
|
AND mapped_data ? 'description'
|
|
AND jsonb_typeof(mapped_data->'description') <> 'string'`, companyID)
|
|
if err != nil {
|
|
return 0, fmt.Errorf("flatten mapped descriptions query: %w", err)
|
|
}
|
|
defer rows.Close()
|
|
|
|
for rows.Next() {
|
|
var id uuid.UUID
|
|
var mappedB []byte
|
|
if err := rows.Scan(&id, &mappedB); err != nil {
|
|
return updated, err
|
|
}
|
|
mapped, err := decodeJSONObject(mappedB)
|
|
if err != nil {
|
|
return updated, err
|
|
}
|
|
rawDesc := mapped["description"]
|
|
plain := PlainDescriptionFromAny(rawDesc)
|
|
if plain == "" {
|
|
continue
|
|
}
|
|
// Skip when already a plain string equal to conversion (shouldn't hit query).
|
|
if s, ok := rawDesc.(string); ok && v1PlainDescription(s) == plain {
|
|
continue
|
|
}
|
|
mapped["description"] = plain
|
|
encoded, err := json.Marshal(mapped)
|
|
if err != nil {
|
|
return updated, err
|
|
}
|
|
ct, err := pool.Exec(ctx, `
|
|
UPDATE raw_products
|
|
SET mapped_data = $2::jsonb, updated_at = now()
|
|
WHERE id = $1 AND company_id = $3`, id, string(encoded), companyID)
|
|
if err != nil {
|
|
return updated, fmt.Errorf("flatten mapped description %s: %w", id, err)
|
|
}
|
|
if ct.RowsAffected() > 0 {
|
|
updated++
|
|
}
|
|
}
|
|
return updated, rows.Err()
|
|
}
|