descrybe/apps/api/internal/processing/catalog_fix.go

package processing

import (
	"context"
	"encoding/json"
	"fmt"
	"strings"

	"github.com/descrybe/descrybe-v2/apps/api/internal/catalog"
	"github.com/descrybe/descrybe-v2/apps/api/internal/company"
	"github.com/google/uuid"
	"github.com/jackc/pgx/v5/pgxpool"
)

// CatalogFixResult summarizes in-place catalog hygiene (never clears the catalog).
type CatalogFixResult struct {
	WeakHashesCleared         int `json:"weak_hashes_cleared"`
	CategoriesBackfilled      int `json:"categories_backfilled"`
	DescriptionsNormalized    int `json:"descriptions_normalized"`
	DescriptionsBackfilled    int `json:"descriptions_backfilled"`
	NamesBackfilled           int `json:"names_backfilled"`
	MappedDescriptionsFlat    int `json:"mapped_descriptions_flattened"`
	MetaBackfilled            int `json:"meta_backfilled"`
	ProductsScanned           int `json:"products_scanned"`
}

// ClearWeakEnhanceHashes removes enhance_input_hash from field_sources and
// localized_content when the product description is weak (poisoned skip hashes).
// Delegates to catalog.RepairWeakEnhanceHashesDetailed (canonical implementation).
func ClearWeakEnhanceHashes(ctx context.Context, pool *pgxpool.Pool, companyID uuid.UUID) (cleared int, scanned int, clearedRawIDs []uuid.UUID, err error) {
	res, err := catalog.RepairWeakEnhanceHashesDetailed(ctx, pool, companyID)
	if err != nil {
		return 0, 0, nil, err
	}
	return int(res.Cleared), res.Scanned, res.ClearedRawIDs, nil
}

// BackfillCategoriesFromMapped copies mapped_data category unique_ids onto
// processed_products.category when the processed category is empty/"none".
// Uses Go-side categoryUniqueIDFromMaps (nested/array shapes), coerces display
// names onto taxonomy unique_ids, and validates against the company taxonomy
// when categories exist.
func BackfillCategoriesFromMapped(ctx context.Context, pool *pgxpool.Pool, companyID uuid.UUID) (updated int, err error) {
	if pool == nil {
		return 0, fmt.Errorf("nil pool")
	}
	valid, namesByUID, err := loadCompanyCategoryTaxonomy(ctx, pool, companyID)
	if err != nil {
		return 0, err
	}
	rows, err := pool.Query(ctx, `
		SELECT pp.id,
			COALESCE(pp.category, ''),
			COALESCE(rp.mapped_data, '{}'::jsonb),
			COALESCE(rp.raw_data, '{}'::jsonb)
		FROM processed_products pp
		JOIN raw_products rp ON rp.id = pp.raw_product_id AND rp.company_id = pp.company_id
		WHERE pp.company_id = $1
		  AND (
			pp.category IS NULL
			OR btrim(pp.category) = ''
			OR lower(btrim(pp.category)) = 'none'
		  )`, companyID)
	if err != nil {
		return 0, err
	}
	defer rows.Close()

	for rows.Next() {
		var (
			id      uuid.UUID
			prior   string
			mappedB []byte
			rawB    []byte
		)
		if err := rows.Scan(&id, &prior, &mappedB, &rawB); err != nil {
			return updated, err
		}
		mapped, err := decodeJSONObject(mappedB)
		if err != nil {
			return updated, err
		}
		raw, err := decodeJSONObject(rawB)
		if err != nil {
			return updated, err
		}
		cat := categoryUniqueIDFromMaps(mapped, raw)
		if cat == "" {
			continue
		}
		if resolved := resolveCompanyCategoryUniqueID(cat, namesByUID, valid); resolved != "" {
			cat = resolved
		} else if len(valid) > 0 {
			continue
		}
		ct, err := pool.Exec(ctx, `
			UPDATE processed_products
			SET category = $2,
			    field_sources = jsonb_set(
					COALESCE(field_sources, '{}'::jsonb),
					'{category}',
					'"mapped_backfill"'::jsonb,
					true
				),
			    updated_at = now()
			WHERE id = $1 AND company_id = $3
			  AND (
				category IS NULL
				OR btrim(category) = ''
				OR lower(btrim(category)) = 'none'
			  )`,
			id, cat, companyID)
		if err != nil {
			return updated, err
		}
		if ct.RowsAffected() > 0 {
			updated++
		}
		_ = prior
	}
	return updated, rows.Err()
}

func loadCompanyCategoryUniqueIDs(ctx context.Context, pool *pgxpool.Pool, companyID uuid.UUID) (map[string]struct{}, error) {
	valid, _, err := loadCompanyCategoryTaxonomy(ctx, pool, companyID)
	return valid, err
}

func loadCompanyCategoryTaxonomy(ctx context.Context, pool *pgxpool.Pool, companyID uuid.UUID) (map[string]struct{}, map[string]string, error) {
	valid := map[string]struct{}{}
	namesByUID := map[string]string{}
	rows, err := pool.Query(ctx, `
		SELECT unique_id, name FROM categories
		WHERE company_id = $1 AND COALESCE(NULLIF(btrim(unique_id), ''), '') <> ''`, companyID)
	if err != nil {
		return nil, nil, err
	}
	defer rows.Close()
	for rows.Next() {
		var uid, name string
		if err := rows.Scan(&uid, &name); err != nil {
			return nil, nil, err
		}
		uid = strings.TrimSpace(uid)
		name = strings.TrimSpace(name)
		if uid == "" {
			continue
		}
		valid[uid] = struct{}{}
		if name != "" {
			namesByUID[uid] = name
		}
	}
	return valid, namesByUID, rows.Err()
}

// FixCatalogHygiene clears poisoned enhance hashes and optionally backfills categories.
func FixCatalogHygiene(ctx context.Context, pool *pgxpool.Pool, companyID uuid.UUID) (CatalogFixResult, error) {
	out, _, err := FixCatalogHygieneWithIDs(ctx, pool, companyID, true)
	return out, err
}

// FixCatalogHygieneWithIDs is FixCatalogHygiene plus raw_product_ids that had weak hashes cleared.
func FixCatalogHygieneWithIDs(ctx context.Context, pool *pgxpool.Pool, companyID uuid.UUID, backfillCategories bool) (CatalogFixResult, []uuid.UUID, error) {
	var out CatalogFixResult
	cleared, scanned, ids, err := ClearWeakEnhanceHashes(ctx, pool, companyID)
	if err != nil {
		return out, nil, err
	}
	out.WeakHashesCleared = cleared
	out.ProductsScanned = scanned

	flat, err := FlattenMappedDescriptionArrays(ctx, pool, companyID)
	if err != nil {
		return out, ids, err
	}
	out.MappedDescriptionsFlat = flat

	if backfillCategories {
		// Go-side unique_id extraction (nested/array) + taxonomy validation.
		cats, err := BackfillCategoriesFromMapped(ctx, pool, companyID)
		if err != nil {
			return out, ids, err
		}
		out.CategoriesBackfilled = cats

		descN, err := BackfillDescriptionsFromMapped(ctx, pool, companyID)
		if err != nil {
			return out, ids, err
		}
		out.DescriptionsBackfilled = descN

		nameN, err := BackfillPollutedNamesFromMapped(ctx, pool, companyID)
		if err != nil {
			return out, ids, err
		}
		out.NamesBackfilled = nameN

		metaN, err := BackfillMissingMeta(ctx, pool, companyID)
		if err != nil {
			return out, ids, err
		}
		out.MetaBackfilled = metaN
	}

	descs, err := NormalizeProcessedDescriptions(ctx, pool, companyID)
	if err != nil {
		return out, ids, err
	}
	out.DescriptionsNormalized = descs
	return out, ids, nil
}

// NormalizeProcessedDescriptions rewrites processed_description to the plain-text
// form produced by PlainDescriptionFromAny (unwraps JSON arrays, strips HTML). When
// processed_description is empty, normalizes from description. Returns rows updated.
func NormalizeProcessedDescriptions(ctx context.Context, pool *pgxpool.Pool, companyID uuid.UUID) (int, error) {
	if pool == nil {
		return 0, fmt.Errorf("normalize descriptions: nil pool")
	}
	rows, err := pool.Query(ctx, `
		SELECT id,
			COALESCE(description, ''),
			COALESCE(processed_description, '')
		FROM processed_products
		WHERE company_id = $1`, companyID)
	if err != nil {
		return 0, fmt.Errorf("normalize descriptions query: %w", err)
	}
	defer rows.Close()

	updated := 0
	for rows.Next() {
		var (
			id                  uuid.UUID
			desc, processedDesc string
		)
		if err := rows.Scan(&id, &desc, &processedDesc); err != nil {
			return updated, fmt.Errorf("normalize descriptions scan: %w", err)
		}
		raw := strings.TrimSpace(processedDesc)
		if raw == "" {
			raw = strings.TrimSpace(desc)
		}
		if raw == "" {
			continue
		}
		plain := plainDescriptionFromStored(raw)
		if plain == "" || plain == strings.TrimSpace(processedDesc) {
			continue
		}
		ct, err := pool.Exec(ctx, `
			UPDATE processed_products
			SET processed_description = $2,
			    updated_at = now()
			WHERE id = $1 AND company_id = $3`, id, plain, companyID)
		if err != nil {
			return updated, fmt.Errorf("normalize descriptions update %s: %w", id, err)
		}
		updated += int(ct.RowsAffected())
	}
	if err := rows.Err(); err != nil {
		return updated, fmt.Errorf("normalize descriptions rows: %w", err)
	}
	return updated, nil
}

// plainDescriptionFromStored handles DB text that may itself be a JSON array/string.
func plainDescriptionFromStored(raw string) string {
	raw = strings.TrimSpace(raw)
	if raw == "" {
		return ""
	}
	if strings.HasPrefix(raw, "[") || strings.HasPrefix(raw, "{") {
		var decoded any
		if err := json.Unmarshal([]byte(raw), &decoded); err == nil {
			if s := PlainDescriptionFromAny(decoded); s != "" {
				return s
			}
		}
	}
	return PlainDescriptionFromAny(raw)
}

// RecommendReprocessRawProductIDs returns how many products should be reprocessed
// after Fix A1 (weak desc and/or missing enhance hash and/or empty category), plus
// an optional sample of raw_product_ids (limit 0 = no sample).
func RecommendReprocessRawProductIDs(ctx context.Context, pool *pgxpool.Pool, companyID uuid.UUID, prefer []uuid.UUID, sampleLimit int) (needed int, sample []uuid.UUID, err error) {
	if pool == nil {
		return 0, nil, fmt.Errorf("nil pool")
	}
	seen := map[uuid.UUID]struct{}{}
	add := func(id uuid.UUID) {
		if id == uuid.Nil {
			return
		}
		if _, ok := seen[id]; ok {
			return
		}
		seen[id] = struct{}{}
		needed++
		if sampleLimit > 0 && len(sample) < sampleLimit {
			sample = append(sample, id)
		}
	}
	for _, id := range prefer {
		add(id)
	}

	rows, err := pool.Query(ctx, `
		SELECT raw_product_id,
			COALESCE(name, ''),
			COALESCE(description, ''),
			COALESCE(processed_name, ''),
			COALESCE(processed_description, ''),
			COALESCE(category, ''),
			COALESCE(field_sources, '{}'::jsonb),
			COALESCE(localized_content, '{}'::jsonb)
		FROM processed_products
		WHERE company_id = $1`, companyID)
	if err != nil {
		return 0, nil, err
	}
	defer rows.Close()

	for rows.Next() {
		var (
			rawID                                  uuid.UUID
			name, desc, processedName, processedDesc, category string
			fsRaw, locRaw                          []byte
		)
		if err := rows.Scan(&rawID, &name, &desc, &processedName, &processedDesc, &category, &fsRaw, &locRaw); err != nil {
			return needed, sample, err
		}
		primaryDesc := strings.TrimSpace(processedDesc)
		if primaryDesc == "" {
			primaryDesc = desc
		}
		primaryName := strings.TrimSpace(processedName)
		if primaryName == "" {
			primaryName = name
		}
		fs, err := decodeJSONObject(fsRaw)
		if err != nil {
			return needed, sample, err
		}
		loc, err := company.DecodeLocalizedContent(locRaw)
		if err != nil {
			return needed, sample, err
		}

		needs := false
		if strings.TrimSpace(category) == "" {
			needs = true
		}
		if isPromptLabelTitle(primaryName) || isPromptLabelTitle(processedName) || isPromptLabelTitle(name) {
			needs = true
		}
		if isWeakPriorEnhanceDescription(primaryDesc, primaryName, name) {
			needs = true
		}
		if _, has := fs[FieldEnhanceInputHash]; !has {
			// Missing hash after weak clear / never enhanced — recommend when desc weak or empty category.
			if isWeakPriorEnhanceDescription(primaryDesc, primaryName, name) || strings.TrimSpace(category) == "" {
				needs = true
			}
		}
		for _, fields := range loc {
			d := strings.TrimSpace(fields.ProcessedDescription)
			n := strings.TrimSpace(fields.ProcessedName)
			if n == "" {
				n = primaryName
			}
			if fields.EnhanceInputHash == "" && isWeakPriorEnhanceDescription(d, n, primaryName, name) {
				needs = true
				break
			}
		}
		if needs {
			add(rawID)
		}
	}
	return needed, sample, rows.Err()
}

func decodeJSONObject(raw []byte) (map[string]any, error) {
	out := map[string]any{}
	if len(raw) == 0 {
		return out, nil
	}
	if err := json.Unmarshal(raw, &out); err != nil {
		return nil, err
	}
	if out == nil {
		out = map[string]any{}
	}
	return out, nil
}