fixes
This commit is contained in:
@@ -0,0 +1,382 @@
|
||||
package processing
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"github.com/google/uuid"
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
)
|
||||
|
||||
// BackfillDescriptionsFromMapped fills empty processed_products.description from
|
||||
// linked raw mapped_data (mirrors migrator backfillProcessedDescriptionsFromMapped)
|
||||
// and coerces array/HTML shapes via PlainDescriptionFromAny.
|
||||
func BackfillDescriptionsFromMapped(ctx context.Context, pool *pgxpool.Pool, companyID uuid.UUID) (updated int, err error) {
|
||||
if pool == nil {
|
||||
return 0, fmt.Errorf("description backfill: nil pool")
|
||||
}
|
||||
if companyID == uuid.Nil {
|
||||
return 0, fmt.Errorf("description backfill: empty company id")
|
||||
}
|
||||
rows, err := pool.Query(ctx, `
|
||||
SELECT p.id,
|
||||
COALESCE(p.description, ''),
|
||||
COALESCE(r.mapped_data, '{}'::jsonb),
|
||||
COALESCE(r.raw_data, '{}'::jsonb)
|
||||
FROM processed_products p
|
||||
INNER JOIN raw_products r ON r.id = p.raw_product_id AND r.company_id = p.company_id
|
||||
WHERE p.company_id = $1
|
||||
AND COALESCE(NULLIF(BTRIM(p.description), ''), '') = ''`, companyID)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("description backfill query: %w", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
for rows.Next() {
|
||||
var (
|
||||
id uuid.UUID
|
||||
prior string
|
||||
mappedB []byte
|
||||
rawB []byte
|
||||
)
|
||||
if err := rows.Scan(&id, &prior, &mappedB, &rawB); err != nil {
|
||||
return updated, fmt.Errorf("description backfill scan: %w", err)
|
||||
}
|
||||
mapped, err := decodeJSONObject(mappedB)
|
||||
if err != nil {
|
||||
return updated, err
|
||||
}
|
||||
raw, err := decodeJSONObject(rawB)
|
||||
if err != nil {
|
||||
return updated, err
|
||||
}
|
||||
desc := PlainDescriptionFromAny(mapped["description"])
|
||||
if desc == "" {
|
||||
desc = PlainDescriptionFromAny(mapped["desc"])
|
||||
}
|
||||
if desc == "" {
|
||||
desc = PlainDescriptionFromAny(mapped["body"])
|
||||
}
|
||||
if desc == "" {
|
||||
desc = PlainDescriptionFromAny(raw["description"])
|
||||
}
|
||||
if desc == "" {
|
||||
continue
|
||||
}
|
||||
ct, err := pool.Exec(ctx, `
|
||||
UPDATE processed_products
|
||||
SET description = $2,
|
||||
updated_at = now()
|
||||
WHERE id = $1 AND company_id = $3
|
||||
AND COALESCE(NULLIF(BTRIM(description), ''), '') = ''`,
|
||||
id, desc, companyID)
|
||||
if err != nil {
|
||||
return updated, fmt.Errorf("description backfill update %s: %w", id, err)
|
||||
}
|
||||
if ct.RowsAffected() > 0 {
|
||||
updated++
|
||||
}
|
||||
_ = prior
|
||||
}
|
||||
return updated, rows.Err()
|
||||
}
|
||||
|
||||
// mappedFeedTitle picks a clean name/title from mapped or raw feed maps.
|
||||
func mappedFeedTitle(mapped, raw map[string]any) string {
|
||||
for _, src := range []map[string]any{mapped, raw} {
|
||||
if src == nil {
|
||||
continue
|
||||
}
|
||||
for _, key := range []string{"name", "title", "product_name", "product_title"} {
|
||||
v := strings.TrimSpace(stringFromAny(src[key]))
|
||||
if v == "" || v == "<nil>" || isPromptLabelTitle(v) {
|
||||
continue
|
||||
}
|
||||
return SanitizeOutput(v)
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// BackfillPollutedNamesFromMapped clears processed_name / name when they match
|
||||
// prompt-leakage patterns (Title formula / short retail title / …) and replaces
|
||||
// them with mapped_data name/title when available. Next Load/FixCatalog or
|
||||
// reprocess then shows the feed title instead of instruction text.
|
||||
func BackfillPollutedNamesFromMapped(ctx context.Context, pool *pgxpool.Pool, companyID uuid.UUID) (updated int, err error) {
|
||||
if pool == nil {
|
||||
return 0, fmt.Errorf("name pollution backfill: nil pool")
|
||||
}
|
||||
if companyID == uuid.Nil {
|
||||
return 0, fmt.Errorf("name pollution backfill: empty company id")
|
||||
}
|
||||
rows, err := pool.Query(ctx, `
|
||||
SELECT p.id,
|
||||
COALESCE(p.name, ''),
|
||||
COALESCE(p.processed_name, ''),
|
||||
COALESCE(r.mapped_data, '{}'::jsonb),
|
||||
COALESCE(r.raw_data, '{}'::jsonb)
|
||||
FROM processed_products p
|
||||
INNER JOIN raw_products r ON r.id = p.raw_product_id AND r.company_id = p.company_id
|
||||
WHERE p.company_id = $1`, companyID)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("name pollution backfill query: %w", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
for rows.Next() {
|
||||
var (
|
||||
id uuid.UUID
|
||||
name, processedName string
|
||||
mappedB, rawB []byte
|
||||
)
|
||||
if err := rows.Scan(&id, &name, &processedName, &mappedB, &rawB); err != nil {
|
||||
return updated, fmt.Errorf("name pollution backfill scan: %w", err)
|
||||
}
|
||||
namePolluted := isPromptLabelTitle(name)
|
||||
procPolluted := isPromptLabelTitle(processedName)
|
||||
if !namePolluted && !procPolluted {
|
||||
continue
|
||||
}
|
||||
mapped, err := decodeJSONObject(mappedB)
|
||||
if err != nil {
|
||||
return updated, err
|
||||
}
|
||||
raw, err := decodeJSONObject(rawB)
|
||||
if err != nil {
|
||||
return updated, err
|
||||
}
|
||||
feed := mappedFeedTitle(mapped, raw)
|
||||
if feed == "" {
|
||||
continue
|
||||
}
|
||||
newName := name
|
||||
newProcessed := processedName
|
||||
if namePolluted {
|
||||
newName = feed
|
||||
}
|
||||
if procPolluted {
|
||||
newProcessed = feed
|
||||
}
|
||||
if newName == name && newProcessed == processedName {
|
||||
continue
|
||||
}
|
||||
ct, err := pool.Exec(ctx, `
|
||||
UPDATE processed_products
|
||||
SET name = $2,
|
||||
processed_name = $3,
|
||||
field_sources = jsonb_set(
|
||||
COALESCE(field_sources, '{}'::jsonb),
|
||||
'{name}',
|
||||
'"mapped_pollution_backfill"'::jsonb,
|
||||
true
|
||||
),
|
||||
updated_at = now()
|
||||
WHERE id = $1 AND company_id = $4`,
|
||||
id, newName, newProcessed, companyID)
|
||||
if err != nil {
|
||||
return updated, fmt.Errorf("name pollution backfill update %s: %w", id, err)
|
||||
}
|
||||
if ct.RowsAffected() > 0 {
|
||||
updated++
|
||||
}
|
||||
}
|
||||
return updated, rows.Err()
|
||||
}
|
||||
|
||||
// BackfillMissingMeta fills empty meta_title / meta_description using the same
|
||||
// free template fillMetaFromResult uses at process time (no AI / no credits).
|
||||
func BackfillMissingMeta(ctx context.Context, pool *pgxpool.Pool, companyID uuid.UUID) (updated int, err error) {
|
||||
if pool == nil {
|
||||
return 0, fmt.Errorf("meta backfill: nil pool")
|
||||
}
|
||||
if companyID == uuid.Nil {
|
||||
return 0, fmt.Errorf("meta backfill: empty company id")
|
||||
}
|
||||
rows, err := pool.Query(ctx, `
|
||||
SELECT id,
|
||||
COALESCE(name, ''),
|
||||
COALESCE(processed_name, ''),
|
||||
COALESCE(category, ''),
|
||||
COALESCE(description, ''),
|
||||
COALESCE(processed_description, ''),
|
||||
COALESCE(meta_title, ''),
|
||||
COALESCE(meta_description, ''),
|
||||
COALESCE(attributes, '{}'::jsonb),
|
||||
COALESCE(processed_attributes, '{}'::jsonb)
|
||||
FROM processed_products
|
||||
WHERE company_id = $1
|
||||
AND (
|
||||
COALESCE(NULLIF(BTRIM(meta_title), ''), '') = ''
|
||||
OR COALESCE(NULLIF(BTRIM(meta_description), ''), '') = ''
|
||||
OR BTRIM(meta_title) ~ '\|\s+[0-9]+$'
|
||||
OR LOWER(meta_title) LIKE '%short retail title%'
|
||||
OR LOWER(meta_title) LIKE '%title formula%'
|
||||
OR LOWER(meta_title) LIKE '%follow any%'
|
||||
OR LOWER(meta_title) LIKE '%constraints that follow%'
|
||||
OR LOWER(meta_title) LIKE '%prefer 1-3%'
|
||||
OR LOWER(meta_title) LIKE '%reply with only json%'
|
||||
OR LOWER(meta_title) LIKE '%your reply is parsed as json%'
|
||||
OR LOWER(meta_description) LIKE '%prefer 1-3%'
|
||||
OR LOWER(meta_description) LIKE '%short retail title%'
|
||||
OR LOWER(meta_description) LIKE '%title formula%'
|
||||
OR LOWER(meta_description) LIKE '%do not emit%'
|
||||
)`, companyID)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("meta backfill query: %w", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
for rows.Next() {
|
||||
var (
|
||||
id uuid.UUID
|
||||
name, processedName, category string
|
||||
desc, processedDesc, metaTitle, metaDesc string
|
||||
attrsJSON, procAttrsJSON []byte
|
||||
)
|
||||
if err := rows.Scan(&id, &name, &processedName, &category, &desc, &processedDesc,
|
||||
&metaTitle, &metaDesc, &attrsJSON, &procAttrsJSON); err != nil {
|
||||
return updated, fmt.Errorf("meta backfill scan: %w", err)
|
||||
}
|
||||
attrs := decodeAttrMap(attrsJSON)
|
||||
procAttrs := decodeAttrMap(procAttrsJSON)
|
||||
synthTitle, synthDesc := fillMetaFromResult(StepResult{
|
||||
Name: name,
|
||||
ProcessedName: processedName,
|
||||
Category: category,
|
||||
Description: PlainDescriptionFromAny(desc),
|
||||
ProcessedDescription: PlainDescriptionFromAny(processedDesc),
|
||||
Attributes: attrs,
|
||||
ProcessedAttributes: procAttrs,
|
||||
})
|
||||
newTitle := strings.TrimSpace(metaTitle)
|
||||
newDesc := strings.TrimSpace(metaDesc)
|
||||
if newTitle == "" || isPoisonedMetaTitle(newTitle) {
|
||||
newTitle = synthTitle
|
||||
}
|
||||
if newDesc == "" || isPromptLeakageTitle(newDesc) ||
|
||||
(isPoisonedMetaTitle(strings.TrimSpace(metaTitle)) &&
|
||||
(isWeakPriorEnhanceDescription(newDesc, name, processedName) || isPromptLeakageTitle(newDesc))) {
|
||||
newDesc = synthDesc
|
||||
}
|
||||
if newTitle == strings.TrimSpace(metaTitle) && newDesc == strings.TrimSpace(metaDesc) {
|
||||
continue
|
||||
}
|
||||
if newTitle == "" && newDesc == "" {
|
||||
continue
|
||||
}
|
||||
ct, err := pool.Exec(ctx, `
|
||||
UPDATE processed_products
|
||||
SET meta_title = CASE
|
||||
WHEN COALESCE(NULLIF(BTRIM(meta_title), ''), '') = ''
|
||||
OR BTRIM(meta_title) ~ '\|\s+[0-9]+$'
|
||||
OR LOWER(meta_title) LIKE '%short retail title%'
|
||||
OR LOWER(meta_title) LIKE '%title formula%'
|
||||
OR LOWER(meta_title) LIKE '%follow any%'
|
||||
OR LOWER(meta_title) LIKE '%constraints that follow%'
|
||||
OR LOWER(meta_title) LIKE '%prefer 1-3%'
|
||||
OR LOWER(meta_title) LIKE '%reply with only json%'
|
||||
OR LOWER(meta_title) LIKE '%your reply is parsed as json%'
|
||||
THEN NULLIF(BTRIM($2), '')
|
||||
ELSE meta_title
|
||||
END,
|
||||
meta_description = CASE
|
||||
WHEN COALESCE(NULLIF(BTRIM(meta_description), ''), '') = '' THEN NULLIF(BTRIM($3), '')
|
||||
WHEN LOWER(meta_description) LIKE '%prefer 1-3%'
|
||||
OR LOWER(meta_description) LIKE '%short retail title%'
|
||||
OR LOWER(meta_description) LIKE '%title formula%'
|
||||
OR LOWER(meta_description) LIKE '%do not emit%'
|
||||
THEN NULLIF(BTRIM($3), '')
|
||||
WHEN (
|
||||
BTRIM(meta_title) ~ '\|\s+[0-9]+$'
|
||||
OR LOWER(meta_title) LIKE '%short retail title%'
|
||||
OR LOWER(meta_title) LIKE '%title formula%'
|
||||
OR LOWER(meta_title) LIKE '%follow any%'
|
||||
OR LOWER(meta_title) LIKE '%constraints that follow%'
|
||||
OR LOWER(meta_title) LIKE '%prefer 1-3%'
|
||||
OR LOWER(meta_title) LIKE '%reply with only json%'
|
||||
OR LOWER(meta_title) LIKE '%your reply is parsed as json%'
|
||||
)
|
||||
AND (
|
||||
char_length(BTRIM(meta_description)) < 40
|
||||
OR LOWER(meta_description) LIKE '%ready for retail listing%'
|
||||
OR LOWER(meta_description) LIKE '%quality product ready%'
|
||||
OR LOWER(meta_description) LIKE '%product description%'
|
||||
OR LOWER(meta_description) LIKE '%based on available specifications%'
|
||||
OR LOWER(meta_description) LIKE '%with available specifications%'
|
||||
OR LOWER(meta_description) LIKE '%available catalog details%'
|
||||
OR LOWER(meta_description) LIKE '%pripravljeno za prodajo%'
|
||||
OR LOWER(meta_description) LIKE '%na podlagi razpoložljivih specifikacij%'
|
||||
OR LOWER(meta_description) LIKE '%prefer 1-3%'
|
||||
)
|
||||
THEN NULLIF(BTRIM($3), '')
|
||||
ELSE meta_description
|
||||
END,
|
||||
updated_at = now()
|
||||
WHERE id = $1 AND company_id = $4`,
|
||||
id, newTitle, newDesc, companyID)
|
||||
if err != nil {
|
||||
return updated, fmt.Errorf("meta backfill update %s: %w", id, err)
|
||||
}
|
||||
if ct.RowsAffected() > 0 {
|
||||
updated++
|
||||
}
|
||||
}
|
||||
return updated, rows.Err()
|
||||
}
|
||||
|
||||
// FlattenMappedDescriptionArrays rewrites mapped_data.description when it is a
|
||||
// JSON array / HTML blob into a plain string (local admin Fix A1 hygiene).
|
||||
func FlattenMappedDescriptionArrays(ctx context.Context, pool *pgxpool.Pool, companyID uuid.UUID) (updated int, err error) {
|
||||
if pool == nil {
|
||||
return 0, fmt.Errorf("flatten mapped descriptions: nil pool")
|
||||
}
|
||||
rows, err := pool.Query(ctx, `
|
||||
SELECT id, COALESCE(mapped_data, '{}'::jsonb)
|
||||
FROM raw_products
|
||||
WHERE company_id = $1
|
||||
AND mapped_data ? 'description'
|
||||
AND jsonb_typeof(mapped_data->'description') <> 'string'`, companyID)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("flatten mapped descriptions query: %w", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
for rows.Next() {
|
||||
var id uuid.UUID
|
||||
var mappedB []byte
|
||||
if err := rows.Scan(&id, &mappedB); err != nil {
|
||||
return updated, err
|
||||
}
|
||||
mapped, err := decodeJSONObject(mappedB)
|
||||
if err != nil {
|
||||
return updated, err
|
||||
}
|
||||
rawDesc := mapped["description"]
|
||||
plain := PlainDescriptionFromAny(rawDesc)
|
||||
if plain == "" {
|
||||
continue
|
||||
}
|
||||
// Skip when already a plain string equal to conversion (shouldn't hit query).
|
||||
if s, ok := rawDesc.(string); ok && v1PlainDescription(s) == plain {
|
||||
continue
|
||||
}
|
||||
mapped["description"] = plain
|
||||
encoded, err := json.Marshal(mapped)
|
||||
if err != nil {
|
||||
return updated, err
|
||||
}
|
||||
ct, err := pool.Exec(ctx, `
|
||||
UPDATE raw_products
|
||||
SET mapped_data = $2::jsonb, updated_at = now()
|
||||
WHERE id = $1 AND company_id = $3`, id, string(encoded), companyID)
|
||||
if err != nil {
|
||||
return updated, fmt.Errorf("flatten mapped description %s: %w", id, err)
|
||||
}
|
||||
if ct.RowsAffected() > 0 {
|
||||
updated++
|
||||
}
|
||||
}
|
||||
return updated, rows.Err()
|
||||
}
|
||||
Reference in New Issue
Block a user