fixes

2026-08-16 16:57:36 +02:00
parent 96f8c1115c
commit 532d439c41
106 changed files with 10147 additions and 494 deletions
@@ -0,0 +1,386 @@
+package processing
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"strings"
+
+	"github.com/descrybe/descrybe-v2/apps/api/internal/catalog"
+	"github.com/descrybe/descrybe-v2/apps/api/internal/company"
+	"github.com/google/uuid"
+	"github.com/jackc/pgx/v5/pgxpool"
+)
+
+// CatalogFixResult summarizes in-place catalog hygiene (never clears the catalog).
+type CatalogFixResult struct {
+	WeakHashesCleared         int `json:"weak_hashes_cleared"`
+	CategoriesBackfilled      int `json:"categories_backfilled"`
+	DescriptionsNormalized    int `json:"descriptions_normalized"`
+	DescriptionsBackfilled    int `json:"descriptions_backfilled"`
+	NamesBackfilled           int `json:"names_backfilled"`
+	MappedDescriptionsFlat    int `json:"mapped_descriptions_flattened"`
+	MetaBackfilled            int `json:"meta_backfilled"`
+	ProductsScanned           int `json:"products_scanned"`
+}
+
+// ClearWeakEnhanceHashes removes enhance_input_hash from field_sources and
+// localized_content when the product description is weak (poisoned skip hashes).
+// Delegates to catalog.RepairWeakEnhanceHashesDetailed (canonical implementation).
+func ClearWeakEnhanceHashes(ctx context.Context, pool *pgxpool.Pool, companyID uuid.UUID) (cleared int, scanned int, clearedRawIDs []uuid.UUID, err error) {
+	res, err := catalog.RepairWeakEnhanceHashesDetailed(ctx, pool, companyID)
+	if err != nil {
+		return 0, 0, nil, err
+	}
+	return int(res.Cleared), res.Scanned, res.ClearedRawIDs, nil
+}
+
+// BackfillCategoriesFromMapped copies mapped_data category unique_ids onto
+// processed_products.category when the processed category is empty/"none".
+// Uses Go-side categoryUniqueIDFromMaps (nested/array shapes) and validates
+// against the company taxonomy when categories exist.
+func BackfillCategoriesFromMapped(ctx context.Context, pool *pgxpool.Pool, companyID uuid.UUID) (updated int, err error) {
+	if pool == nil {
+		return 0, fmt.Errorf("nil pool")
+	}
+	valid, err := loadCompanyCategoryUniqueIDs(ctx, pool, companyID)
+	if err != nil {
+		return 0, err
+	}
+	rows, err := pool.Query(ctx, `
+		SELECT pp.id,
+			COALESCE(pp.category, ''),
+			COALESCE(rp.mapped_data, '{}'::jsonb),
+			COALESCE(rp.raw_data, '{}'::jsonb)
+		FROM processed_products pp
+		JOIN raw_products rp ON rp.id = pp.raw_product_id AND rp.company_id = pp.company_id
+		WHERE pp.company_id = $1
+		  AND (
+			pp.category IS NULL
+			OR btrim(pp.category) = ''
+			OR lower(btrim(pp.category)) = 'none'
+		  )`, companyID)
+	if err != nil {
+		return 0, err
+	}
+	defer rows.Close()
+
+	for rows.Next() {
+		var (
+			id      uuid.UUID
+			prior   string
+			mappedB []byte
+			rawB    []byte
+		)
+		if err := rows.Scan(&id, &prior, &mappedB, &rawB); err != nil {
+			return updated, err
+		}
+		mapped, err := decodeJSONObject(mappedB)
+		if err != nil {
+			return updated, err
+		}
+		raw, err := decodeJSONObject(rawB)
+		if err != nil {
+			return updated, err
+		}
+		cat := categoryUniqueIDFromMaps(mapped, raw)
+		if cat == "" {
+			continue
+		}
+		if len(valid) > 0 {
+			if _, ok := valid[cat]; !ok {
+				continue
+			}
+		}
+		ct, err := pool.Exec(ctx, `
+			UPDATE processed_products
+			SET category = $2,
+			    field_sources = jsonb_set(
+					COALESCE(field_sources, '{}'::jsonb),
+					'{category}',
+					'"mapped_backfill"'::jsonb,
+					true
+				),
+			    updated_at = now()
+			WHERE id = $1 AND company_id = $3
+			  AND (
+				category IS NULL
+				OR btrim(category) = ''
+				OR lower(btrim(category)) = 'none'
+			  )`,
+			id, cat, companyID)
+		if err != nil {
+			return updated, err
+		}
+		if ct.RowsAffected() > 0 {
+			updated++
+		}
+		_ = prior
+	}
+	return updated, rows.Err()
+}
+
+func loadCompanyCategoryUniqueIDs(ctx context.Context, pool *pgxpool.Pool, companyID uuid.UUID) (map[string]struct{}, error) {
+	out := map[string]struct{}{}
+	rows, err := pool.Query(ctx, `
+		SELECT unique_id FROM categories
+		WHERE company_id = $1 AND COALESCE(NULLIF(btrim(unique_id), ''), '') <> ''`, companyID)
+	if err != nil {
+		return nil, err
+	}
+	defer rows.Close()
+	for rows.Next() {
+		var uid string
+		if err := rows.Scan(&uid); err != nil {
+			return nil, err
+		}
+		uid = strings.TrimSpace(uid)
+		if uid != "" {
+			out[uid] = struct{}{}
+		}
+	}
+	return out, rows.Err()
+}
+
+// FixCatalogHygiene clears poisoned enhance hashes and optionally backfills categories.
+func FixCatalogHygiene(ctx context.Context, pool *pgxpool.Pool, companyID uuid.UUID) (CatalogFixResult, error) {
+	out, _, err := FixCatalogHygieneWithIDs(ctx, pool, companyID, true)
+	return out, err
+}
+
+// FixCatalogHygieneWithIDs is FixCatalogHygiene plus raw_product_ids that had weak hashes cleared.
+func FixCatalogHygieneWithIDs(ctx context.Context, pool *pgxpool.Pool, companyID uuid.UUID, backfillCategories bool) (CatalogFixResult, []uuid.UUID, error) {
+	var out CatalogFixResult
+	cleared, scanned, ids, err := ClearWeakEnhanceHashes(ctx, pool, companyID)
+	if err != nil {
+		return out, nil, err
+	}
+	out.WeakHashesCleared = cleared
+	out.ProductsScanned = scanned
+
+	flat, err := FlattenMappedDescriptionArrays(ctx, pool, companyID)
+	if err != nil {
+		return out, ids, err
+	}
+	out.MappedDescriptionsFlat = flat
+
+	if backfillCategories {
+		// Go-side unique_id extraction (nested/array) + taxonomy validation.
+		cats, err := BackfillCategoriesFromMapped(ctx, pool, companyID)
+		if err != nil {
+			return out, ids, err
+		}
+		out.CategoriesBackfilled = cats
+
+		descN, err := BackfillDescriptionsFromMapped(ctx, pool, companyID)
+		if err != nil {
+			return out, ids, err
+		}
+		out.DescriptionsBackfilled = descN
+
+		nameN, err := BackfillPollutedNamesFromMapped(ctx, pool, companyID)
+		if err != nil {
+			return out, ids, err
+		}
+		out.NamesBackfilled = nameN
+
+		metaN, err := BackfillMissingMeta(ctx, pool, companyID)
+		if err != nil {
+			return out, ids, err
+		}
+		out.MetaBackfilled = metaN
+	}
+
+	descs, err := NormalizeProcessedDescriptions(ctx, pool, companyID)
+	if err != nil {
+		return out, ids, err
+	}
+	out.DescriptionsNormalized = descs
+	return out, ids, nil
+}
+
+// NormalizeProcessedDescriptions rewrites processed_description to the plain-text
+// form produced by PlainDescriptionFromAny (unwraps JSON arrays, strips HTML). When
+// processed_description is empty, normalizes from description. Returns rows updated.
+func NormalizeProcessedDescriptions(ctx context.Context, pool *pgxpool.Pool, companyID uuid.UUID) (int, error) {
+	if pool == nil {
+		return 0, fmt.Errorf("normalize descriptions: nil pool")
+	}
+	rows, err := pool.Query(ctx, `
+		SELECT id,
+			COALESCE(description, ''),
+			COALESCE(processed_description, '')
+		FROM processed_products
+		WHERE company_id = $1`, companyID)
+	if err != nil {
+		return 0, fmt.Errorf("normalize descriptions query: %w", err)
+	}
+	defer rows.Close()
+
+	updated := 0
+	for rows.Next() {
+		var (
+			id                  uuid.UUID
+			desc, processedDesc string
+		)
+		if err := rows.Scan(&id, &desc, &processedDesc); err != nil {
+			return updated, fmt.Errorf("normalize descriptions scan: %w", err)
+		}
+		raw := strings.TrimSpace(processedDesc)
+		if raw == "" {
+			raw = strings.TrimSpace(desc)
+		}
+		if raw == "" {
+			continue
+		}
+		plain := plainDescriptionFromStored(raw)
+		if plain == "" || plain == strings.TrimSpace(processedDesc) {
+			continue
+		}
+		ct, err := pool.Exec(ctx, `
+			UPDATE processed_products
+			SET processed_description = $2,
+			    updated_at = now()
+			WHERE id = $1 AND company_id = $3`, id, plain, companyID)
+		if err != nil {
+			return updated, fmt.Errorf("normalize descriptions update %s: %w", id, err)
+		}
+		updated += int(ct.RowsAffected())
+	}
+	if err := rows.Err(); err != nil {
+		return updated, fmt.Errorf("normalize descriptions rows: %w", err)
+	}
+	return updated, nil
+}
+
+// plainDescriptionFromStored handles DB text that may itself be a JSON array/string.
+func plainDescriptionFromStored(raw string) string {
+	raw = strings.TrimSpace(raw)
+	if raw == "" {
+		return ""
+	}
+	if strings.HasPrefix(raw, "[") || strings.HasPrefix(raw, "{") {
+		var decoded any
+		if err := json.Unmarshal([]byte(raw), &decoded); err == nil {
+			if s := PlainDescriptionFromAny(decoded); s != "" {
+				return s
+			}
+		}
+	}
+	return PlainDescriptionFromAny(raw)
+}
+
+// RecommendReprocessRawProductIDs returns how many products should be reprocessed
+// after Fix A1 (weak desc and/or missing enhance hash and/or empty category), plus
+// an optional sample of raw_product_ids (limit 0 = no sample).
+func RecommendReprocessRawProductIDs(ctx context.Context, pool *pgxpool.Pool, companyID uuid.UUID, prefer []uuid.UUID, sampleLimit int) (needed int, sample []uuid.UUID, err error) {
+	if pool == nil {
+		return 0, nil, fmt.Errorf("nil pool")
+	}
+	seen := map[uuid.UUID]struct{}{}
+	add := func(id uuid.UUID) {
+		if id == uuid.Nil {
+			return
+		}
+		if _, ok := seen[id]; ok {
+			return
+		}
+		seen[id] = struct{}{}
+		needed++
+		if sampleLimit > 0 && len(sample) < sampleLimit {
+			sample = append(sample, id)
+		}
+	}
+	for _, id := range prefer {
+		add(id)
+	}
+
+	rows, err := pool.Query(ctx, `
+		SELECT raw_product_id,
+			COALESCE(name, ''),
+			COALESCE(description, ''),
+			COALESCE(processed_name, ''),
+			COALESCE(processed_description, ''),
+			COALESCE(category, ''),
+			COALESCE(field_sources, '{}'::jsonb),
+			COALESCE(localized_content, '{}'::jsonb)
+		FROM processed_products
+		WHERE company_id = $1`, companyID)
+	if err != nil {
+		return 0, nil, err
+	}
+	defer rows.Close()
+
+	for rows.Next() {
+		var (
+			rawID                                  uuid.UUID
+			name, desc, processedName, processedDesc, category string
+			fsRaw, locRaw                          []byte
+		)
+		if err := rows.Scan(&rawID, &name, &desc, &processedName, &processedDesc, &category, &fsRaw, &locRaw); err != nil {
+			return needed, sample, err
+		}
+		primaryDesc := strings.TrimSpace(processedDesc)
+		if primaryDesc == "" {
+			primaryDesc = desc
+		}
+		primaryName := strings.TrimSpace(processedName)
+		if primaryName == "" {
+			primaryName = name
+		}
+		fs, err := decodeJSONObject(fsRaw)
+		if err != nil {
+			return needed, sample, err
+		}
+		loc, err := company.DecodeLocalizedContent(locRaw)
+		if err != nil {
+			return needed, sample, err
+		}
+
+		needs := false
+		if strings.TrimSpace(category) == "" {
+			needs = true
+		}
+		if isPromptLabelTitle(primaryName) || isPromptLabelTitle(processedName) || isPromptLabelTitle(name) {
+			needs = true
+		}
+		if isWeakPriorEnhanceDescription(primaryDesc, primaryName, name) {
+			needs = true
+		}
+		if _, has := fs[FieldEnhanceInputHash]; !has {
+			// Missing hash after weak clear / never enhanced — recommend when desc weak or empty category.
+			if isWeakPriorEnhanceDescription(primaryDesc, primaryName, name) || strings.TrimSpace(category) == "" {
+				needs = true
+			}
+		}
+		for _, fields := range loc {
+			d := strings.TrimSpace(fields.ProcessedDescription)
+			n := strings.TrimSpace(fields.ProcessedName)
+			if n == "" {
+				n = primaryName
+			}
+			if fields.EnhanceInputHash == "" && isWeakPriorEnhanceDescription(d, n, primaryName, name) {
+				needs = true
+				break
+			}
+		}
+		if needs {
+			add(rawID)
+		}
+	}
+	return needed, sample, rows.Err()
+}
+
+func decodeJSONObject(raw []byte) (map[string]any, error) {
+	out := map[string]any{}
+	if len(raw) == 0 {
+		return out, nil
+	}
+	if err := json.Unmarshal(raw, &out); err != nil {
+		return nil, err
+	}
+	if out == nil {
+		out = map[string]any{}
+	}
+	return out, nil
+}