descrybe/apps/api/internal/processing/v1_legacy.go

package processing

import (
	"context"
	"encoding/json"
	"fmt"
	"html"
	"regexp"
	"strings"

	"github.com/descrybe/descrybe-v2/apps/api/internal/catalog"
	"github.com/descrybe/descrybe-v2/apps/api/internal/eprel"
	"github.com/google/uuid"
)

var v1PartialSteps = []string{"category", "title", "description", "attributes"}

const v1MetaDescriptionMaxChars = 155

var (
	v1BreakTagRe = regexp.MustCompile(`(?i)<br\s*/?>`)
	v1BlockEndRe = regexp.MustCompile(`(?i)</(p|div|li|h[1-6]|tr)>`)
)

// ParseV1ProcessingType mirrors legacy parseV1ProcessingTypeFromBody.
// Accepts string ("full" / step), JSON array of steps, or nil (defaults to full).
func ParseV1ProcessingType(raw any) (storageValue string, responseValue any, err error) {
	if raw == nil {
		return "full", "full", nil
	}
	switch v := raw.(type) {
	case string:
		trimmed := strings.TrimSpace(v)
		if trimmed == "" {
			return "full", "full", nil
		}
		if strings.Contains(trimmed, ",") {
			return "", nil, fmt.Errorf("Invalid processing_type. Use \"full\", a single step (%s), or an array of steps, e.g. [\"title\",\"attributes\"].", strings.Join(v1PartialSteps, ", "))
		}
		normalized := strings.ToLower(trimmed)
		if normalized == "full" {
			return "full", "full", nil
		}
		if normalized == "both" || normalized == "search" {
			return "", nil, fmt.Errorf("Invalid processing_type. Use \"full\", a single step (%s), or an array of steps, e.g. [\"title\",\"attributes\"].", strings.Join(v1PartialSteps, ", "))
		}
		step := normalizeV1Step(normalized)
		if step != "" {
			return step, step, nil
		}
		// Dual-mode: accept v2 dashboard types (normalize_only, enhance_only, …).
		if isV2ProcessingType(normalized) {
			return normalized, normalized, nil
		}
		return "", nil, fmt.Errorf("Invalid processing_type. Use \"full\", a single step (%s), or an array of steps, e.g. [\"title\",\"attributes\"].", strings.Join(v1PartialSteps, ", "))
	case []any:
		if len(v) == 0 {
			return "", nil, fmt.Errorf("Invalid processing_type. Use \"full\", a single step (%s), or an array of steps, e.g. [\"title\",\"attributes\"].", strings.Join(v1PartialSteps, ", "))
		}
		steps := make([]string, 0, len(v))
		seen := map[string]struct{}{}
		for _, entry := range v {
			step := normalizeV1Step(fmt.Sprint(entry))
			if step == "" {
				return "", nil, fmt.Errorf("Invalid processing_type. Use \"full\", a single step (%s), or an array of steps, e.g. [\"title\",\"attributes\"].", strings.Join(v1PartialSteps, ", "))
			}
			if _, ok := seen[step]; ok {
				continue
			}
			seen[step] = struct{}{}
			steps = append(steps, step)
		}
		if len(steps) == 1 {
			return steps[0], steps[0], nil
		}
		b, _ := json.Marshal(steps)
		return string(b), steps, nil
	default:
		return "", nil, fmt.Errorf("Invalid processing_type. Use \"full\", a single step (%s), or an array of steps, e.g. [\"title\",\"attributes\"].", strings.Join(v1PartialSteps, ", "))
	}
}

func isV2ProcessingType(normalized string) bool {
	switch normalized {
	case "normalize_only", "enhance", "enhance_only", "enhance-only",
		"attributes_only", "specs", "specifications",
		"eprel", "eprel_only", "categorize", "categorize_only", "categorize_enhance":
		return true
	default:
		return false
	}
}

func normalizeV1Step(raw string) string {
	token := strings.ToLower(strings.TrimSpace(raw))
	if token == "name" {
		token = "title"
	}
	for _, s := range v1PartialSteps {
		if s == token {
			return s
		}
	}
	return ""
}

// ProcessingTypeForAPIResponse echoes the stored job type as string or []string.
func ProcessingTypeForAPIResponse(stored string) any {
	normalized := strings.ToLower(strings.TrimSpace(stored))
	if normalized == "" || normalized == "full" {
		return "full"
	}
	if normalized == "both" {
		return "both"
	}
	if strings.HasPrefix(normalized, "[") {
		var parsed []any
		if err := json.Unmarshal([]byte(stored), &parsed); err == nil {
			out := make([]string, 0, len(parsed))
			for _, e := range parsed {
				if step := normalizeV1Step(fmt.Sprint(e)); step != "" {
					out = append(out, step)
				} else {
					out = append(out, strings.ToLower(strings.TrimSpace(fmt.Sprint(e))))
				}
			}
			return out
		}
	}
	if step := normalizeV1Step(normalized); step != "" {
		return step
	}
	return normalized
}

// MapJobStatusForV1 uppercases pipeline statuses toward the legacy public API.
func MapJobStatusForV1(status string) string {
	switch strings.ToLower(strings.TrimSpace(status)) {
	case "pending":
		return "PENDING"
	case "running", "processing", "queued":
		return "PROCESSING"
	case "completed", "success", "done", "finished", "processed":
		return "COMPLETED"
	case "failed", "error":
		return "FAILED"
	case "cancelled", "canceled":
		return "CANCELLED"
	default:
		return strings.ToUpper(strings.TrimSpace(status))
	}
}

// JobStatusIncludesProducts reports whether a finished job should expose processed product items.
func JobStatusIncludesProducts(status string) bool {
	return MapJobStatusForV1(status) == "COMPLETED"
}

// FormatJobStatusResponse returns job JSON, optionally enriched with processed product items.
// Additive only: existing Job fields are preserved; items/total_items appear when includeItems.
func FormatJobStatusResponse(job Job, items []V1ProcessJobItem, includeItems bool) any {
	if !includeItems {
		return job
	}
	raw, err := json.Marshal(job)
	if err != nil {
		return job
	}
	out := map[string]any{}
	if err := json.Unmarshal(raw, &out); err != nil {
		return job
	}
	if items == nil {
		items = []V1ProcessJobItem{}
	}
	out["items"] = items
	out["total_items"] = len(items)
	return out
}

// MapV1JobItemStatus normalizes processing_job_products.status for legacy poll items.
func MapV1JobItemStatus(raw string, hasProcessed bool) string {
	switch strings.ToLower(strings.TrimSpace(raw)) {
	case "processed", "completed", "success", "done":
		return "processed"
	case "failed", "error":
		return "failed"
	case "cancelled", "canceled", "skipped":
		return "cancelled"
	case "processing", "running":
		return "processing"
	case "pending", "queued":
		return "pending"
	default:
		if hasProcessed {
			return "processed"
		}
		return "not_found"
	}
}

// V1ProcessJobItem is one projected product in a legacy GET /products/process/{id} response.
type V1ProcessJobItem map[string]any

// LoadV1ProcessJobItems loads and projects job products for the legacy GET status response.
func (p *Pipeline) LoadV1ProcessJobItems(ctx context.Context, companyID, jobID uuid.UUID, processingType string) ([]V1ProcessJobItem, error) {
	if p == nil || p.Pool == nil {
		return nil, fmt.Errorf("pipeline not configured")
	}
	allowedAttrs := loadCompanyAttributeKeySet(ctx, p, companyID)
	categoryNames := loadCompanyCategoryNameMap(ctx, p, companyID)
	rows, err := p.Pool.Query(ctx, `
		SELECT
			COALESCE(r.gtin, p.product_id, '') AS ean,
			p.category,
			c.name AS category_name,
			p.processed_name,
			p.meta_title,
			p.meta_description,
			COALESCE(p.processed_description, p.description) AS description,
			p.processed_attributes,
			r.mapped_data,
			r.raw_data,
			pjp.status AS item_status,
			pjp.error AS item_error,
			p.id AS processed_id,
			pjp.raw_product_id AS raw_product_id
		FROM processing_job_products pjp
		LEFT JOIN raw_products r ON r.id = pjp.raw_product_id
		LEFT JOIN processed_products p
			ON p.raw_product_id = pjp.raw_product_id AND p.company_id = $2
		LEFT JOIN categories c
			ON c.unique_id = p.category AND c.company_id = $2
		WHERE pjp.job_id = $1
		ORDER BY pjp.created_at ASC, pjp.id ASC`, jobID, companyID)
	if err != nil {
		return nil, err
	}
	defer rows.Close()

	full := make([]V1ProcessJobItem, 0)
	for rows.Next() {
		var (
			ean, itemStatus                                             string
			category, categoryName, title, metaTitle, metaDesc, descTxt *string
			attrsJSON, mappedJSON, rawJSON                              []byte
			itemError                                                   *string
			processedID                                                 *uuid.UUID
			rawProductID                                                *uuid.UUID
		)
		if err := rows.Scan(
			&ean, &category, &categoryName, &title, &metaTitle, &metaDesc, &descTxt,
			&attrsJSON, &mappedJSON, &rawJSON, &itemStatus, &itemError, &processedID, &rawProductID,
		); err != nil {
			return nil, err
		}
		if processedID == nil {
			st := MapV1JobItemStatus(itemStatus, false)
			if st == "processed" || st == "processing" || st == "pending" {
				st = "not_found"
			}
			item := V1ProcessJobItem{
				"ean":    ean,
				"status": st,
				"error":  "Product data not available",
			}
			if itemError != nil && *itemError != "" {
				item["error"] = *itemError
			}
			applyV1ProcessItemIDs(item, nil, rawProductID)
			full = append(full, item)
			continue
		}

		attrs := map[string]any{}
		if len(attrsJSON) > 0 {
			var raw any
			if err := json.Unmarshal(attrsJSON, &raw); err == nil {
				switch t := raw.(type) {
				case map[string]any:
					attrs = t
				case []any:
					for _, entry := range t {
						if m, ok := entry.(map[string]any); ok {
							if k, ok := m["key"].(string); ok && k != "" {
								attrs[k] = m
							}
						}
					}
				}
			}
		}

		var mapped, rawData map[string]any
		_ = json.Unmarshal(mappedJSON, &mapped)
		_ = json.Unmarshal(rawJSON, &rawData)
		main, more := catalog.ExtractProductImages(mapped, rawData)

		plainDesc := ""
		if descTxt != nil {
			plainDesc = v1PlainDescription(*descTxt)
		}

		eprelVal := extractEPRELFromAttrs(attrs)
		attrs = SanitizeV1ProcessAttributesAllowed(attrs, allowedAttrs)

		titleStr := derefStringPtr(title)
		catStr := derefStringPtr(category)
		catNameStr := derefStringPtr(categoryName)
		// Projection gap: when processed.category is empty but mapped/raw carries a
		// unique_id, surface it (and resolve category_name from company taxonomy).
		if catStr == "" {
			catStr = categoryUniqueIDFromMaps(mapped, rawData)
		}
		// Coerce display names → taxonomy unique_id (same as processOne / d161b28).
		if catStr != "" && len(categoryNames) > 0 {
			valid := make(map[string]struct{}, len(categoryNames))
			for uid := range categoryNames {
				valid[uid] = struct{}{}
			}
			if resolved := resolveCompanyCategoryUniqueID(catStr, categoryNames, valid); resolved != "" {
				catStr = resolved
			}
		}
		if catNameStr == "" && catStr != "" {
			if name, ok := categoryNames[catStr]; ok && name != "" {
				catNameStr = name
			} else if nested, ok := mapped["category"].(map[string]any); ok {
				// Only accept nested display names that resolve in company taxonomy —
				// never product title / title-formula leftovers from feed objects.
				for _, k := range []string{"name", "category_name"} {
					s := strings.TrimSpace(stringFromAny(nested[k]))
					if s == "" || isUnusableCategoryValue(s, titleStr) {
						continue
					}
					uid := resolveCompanyCategoryUniqueID(s, categoryNames, nil)
					if uid == "" {
						continue
					}
					if resolvedName := strings.TrimSpace(categoryNames[uid]); resolvedName != "" {
						catNameStr = resolvedName
					} else {
						catNameStr = s
					}
					break
				}
			}
		}

		metaTitleOut := nullIfEmptyPtr(metaTitle)
		metaDescOut := nullIfEmptyPtr(metaDesc)
		if s, ok := metaTitleOut.(string); ok && isPoisonedMetaTitle(s) {
			metaTitleOut = nil
			// Poisoned title refresh: also drop empty/weak/leakage stub meta_description.
			if ds, ok := metaDescOut.(string); ok && (ds == "" || isWeakPriorEnhanceDescription(ds, titleStr) || isPromptLeakageTitle(ds)) {
				metaDescOut = nil
			}
		}
		if ds, ok := metaDescOut.(string); ok && isPromptLeakageTitle(ds) {
			metaDescOut = nil
		}
		catLabel := catNameStr
		if catLabel == "" {
			catLabel = catStr
		}
		if (metaTitleOut == nil || metaDescOut == nil) && (titleStr != "" || plainDesc != "" || catLabel != "") {
			synthTitle, synthDesc := fillMetaFromResult(StepResult{
				Name:                 titleStr,
				ProcessedName:        titleStr,
				Category:             catStr,
				CategoryName:         catNameStr,
				Description:          plainDesc,
				ProcessedDescription: plainDesc,
				Attributes:           attrs,
				ProcessedAttributes:  attrs,
			})
			if metaTitleOut == nil {
				if synthTitle != "" {
					metaTitleOut = synthTitle
				} else {
					metaTitleOut = nullIfEmptyPtr(title)
				}
			}
			if metaDescOut == nil {
				if synthDesc != "" {
					metaDescOut = synthDesc
				} else if plainDesc != "" {
					metaDescOut = truncateMetaDescription(plainDesc, v1MetaDescriptionMaxChars)
				}
			}
		} else if metaTitleOut == nil {
			metaTitleOut = nullIfEmptyPtr(title)
		}

		var description any
		if plainDesc != "" {
			description = plainDesc
		} else {
			description = nil
		}
		var catOut, catNameOut any
		if catStr != "" {
			catOut = catStr
		}
		if catNameStr != "" {
			catNameOut = catNameStr
		}

		titleOut := nullIfEmptyPtr(title)
		item := V1ProcessJobItem{
			"ean":              ean,
			"status":           MapV1JobItemStatus(itemStatus, true),
			"category":         catOut,
			"category_name":    catNameOut,
			"title":            titleOut,
			"name":             titleOut,
			"meta_title":       metaTitleOut,
			"meta_description": metaDescOut,
			"description":      description,
			"attributes":       nil,
			"main_image":       nil,
			"more_images":      nil,
			"eprel":            eprelVal,
		}
		applyV1ProcessItemIDs(item, processedID, rawProductID)
		if itemError != nil && *itemError != "" {
			item["error"] = *itemError
		}
		if len(attrs) > 0 {
			item["attributes"] = attrs
		}
		if main != "" {
			item["main_image"] = main
		}
		if len(more) > 0 {
			item["more_images"] = more
		}
		item = EnforceV1ProcessCompletedItem(item, "", allowedAttrs)
		full = append(full, item)
	}
	if err := rows.Err(); err != nil {
		return nil, err
	}
	return ProjectV1ProcessJobItems(processingType, full), nil
}

func nullIfEmptyPtr(s *string) any {
	if s == nil || strings.TrimSpace(*s) == "" {
		return nil
	}
	return *s
}

func derefStringPtr(s *string) string {
	if s == nil {
		return ""
	}
	return strings.TrimSpace(*s)
}

// loadCompanyAttributeKeySet returns canonicalized attribute_key values for the company.
// On query failure returns an empty (non-nil) set so FilterAttributesByAllowed still
// restricts to coreCharacteristicAttrKeys only.
func loadCompanyAttributeKeySet(ctx context.Context, p *Pipeline, companyID uuid.UUID) map[string]struct{} {
	out := map[string]struct{}{}
	if p == nil || p.Pool == nil {
		return out
	}
	rows, err := p.Pool.Query(ctx, `
		SELECT attribute_key
		FROM attributes
		WHERE company_id = $1 AND COALESCE(attribute_key, '') <> ''`, companyID)
	if err != nil {
		return out
	}
	defer rows.Close()
	for rows.Next() {
		var key string
		if err := rows.Scan(&key); err != nil {
			continue
		}
		canon := canonicalizeAttrKey(key)
		if canon == "" {
			continue
		}
		out[canon] = struct{}{}
		out[strings.ToLower(strings.TrimSpace(key))] = struct{}{}
	}
	return out
}

// loadCompanyCategoryNameMap returns unique_id → display name for the company.
func loadCompanyCategoryNameMap(ctx context.Context, p *Pipeline, companyID uuid.UUID) map[string]string {
	out := map[string]string{}
	if p == nil || p.Pool == nil {
		return out
	}
	rows, err := p.Pool.Query(ctx, `
		SELECT unique_id, name
		FROM categories
		WHERE company_id = $1 AND COALESCE(unique_id, '') <> ''`, companyID)
	if err != nil {
		return out
	}
	defer rows.Close()
	for rows.Next() {
		var uid, name string
		if err := rows.Scan(&uid, &name); err != nil {
			continue
		}
		uid = strings.TrimSpace(uid)
		name = strings.TrimSpace(name)
		if uid == "" || name == "" {
			continue
		}
		out[uid] = name
	}
	return out
}

// v1PlainDescription normalizes legacy stored descriptions into a single plain-text
// string for the process poll (description is a string, not a one-element array).
// Handles JSON array/string encodings (e.g. mapped_data->>'description' when the
// feed value was an array) and HTML/<br>/entity markup.
func v1PlainDescription(s string) string {
	s = strings.TrimSpace(s)
	if s == "" {
		return ""
	}
	s = unwrapLegacyDescriptionStored(s)
	s = strings.TrimSpace(s)
	if s == "" {
		return ""
	}
	s = html.UnescapeString(s)
	s = strings.ReplaceAll(s, "\u00a0", " ")
	s = strings.ReplaceAll(s, `\u00a0`, " ")
	s = v1BreakTagRe.ReplaceAllString(s, "\n")
	s = v1BlockEndRe.ReplaceAllString(s, "\n")
	s = specsHTMLTagRe.ReplaceAllString(s, " ")
	s = html.UnescapeString(s)
	lines := strings.Split(s, "\n")
	kept := make([]string, 0, len(lines))
	for _, line := range lines {
		line = strings.Join(strings.Fields(line), " ")
		line = strings.ReplaceAll(line, " .", ".")
		line = strings.ReplaceAll(line, " ,", ",")
		line = strings.ReplaceAll(line, " ;", ";")
		line = strings.ReplaceAll(line, " :", ":")
		if line != "" {
			kept = append(kept, line)
		}
	}
	return strings.TrimSpace(strings.Join(kept, "\n"))
}

const maxLegacyDescriptionUnwrapDepth = 4

// unwrapLegacyDescriptionStored flattens JSON-encoded legacy description payloads
// (one-element arrays, multi-part arrays, nested arrays, JSON-quoted strings).
// Non-JSON text is returned unchanged.
func unwrapLegacyDescriptionStored(s string) string {
	return unwrapLegacyDescriptionStoredN(s, 0)
}

func unwrapLegacyDescriptionStoredN(s string, depth int) string {
	s = strings.TrimSpace(s)
	if s == "" || depth > maxLegacyDescriptionUnwrapDepth {
		return s
	}
	if !looksLikeLegacyDescriptionJSON(s) {
		return s
	}
	var raw any
	if err := json.Unmarshal([]byte(s), &raw); err != nil {
		return s
	}
	flat := flattenLegacyDescriptionAny(raw, depth)
	if flat == "" {
		if _, ok := raw.([]any); ok {
			return ""
		}
		return s
	}
	return flat
}

func looksLikeLegacyDescriptionJSON(s string) bool {
	if len(s) >= 2 && s[0] == '[' && s[len(s)-1] == ']' {
		return true
	}
	return len(s) >= 2 && s[0] == '"' && s[len(s)-1] == '"'
}

func flattenLegacyDescriptionAny(v any, depth int) string {
	switch t := v.(type) {
	case string:
		return unwrapLegacyDescriptionStoredN(t, depth+1)
	case []any:
		parts := make([]string, 0, len(t))
		for _, el := range t {
			if p := flattenLegacyDescriptionAny(el, depth+1); p != "" {
				parts = append(parts, p)
			}
		}
		return strings.Join(parts, "\n")
	case float64:
		return strings.TrimSpace(fmt.Sprint(t))
	case bool:
		return fmt.Sprint(t)
	default:
		return ""
	}
}

func extractEPRELFromAttrs(attrs map[string]any) any {
	return eprel.ExtractFromAttrs(attrs)
}

// ProjectV1ProcessJobItems applies legacy partial-type field projection.
func ProjectV1ProcessJobItems(storedType string, items []V1ProcessJobItem) []V1ProcessJobItem {
	resolved := resolveV1Steps(storedType)
	if resolved.isFull {
		return items
	}
	out := make([]V1ProcessJobItem, 0, len(items))
	for _, item := range items {
		if _, hasID := item["id"]; !hasID {
			if status, _ := item["status"].(string); status == "not_found" || status == "failed" || status == "cancelled" {
				out = append(out, item)
				continue
			}
		}
		main, _ := item["main_image"].(string)
		var more []string
		if m, ok := item["more_images"].([]string); ok {
			more = m
		} else if arr, ok := item["more_images"].([]any); ok {
			for _, e := range arr {
				if s, ok := e.(string); ok {
					more = append(more, s)
				}
			}
		}
		eprel := item["eprel"]
		ean, _ := item["ean"].(string)
		projected := withItemMeta(V1ProcessJobItem{"ean": ean}, item)
		if len(resolved.steps) == 0 {
			out = append(out, withAlwaysIncluded(projected, main, more, eprel))
			continue
		}
		for step := range resolved.steps {
			switch step {
			case "category":
				projected["category"] = item["category"]
				projected["category_name"] = item["category_name"]
			case "title":
				projected["title"] = item["title"]
				projected["name"] = item["name"]
				if projected["name"] == nil {
					projected["name"] = item["title"]
				}
				projected["meta_title"] = item["meta_title"]
			case "description":
				projected["description"] = item["description"]
				projected["meta_description"] = item["meta_description"]
			case "attributes":
				projected["attributes"] = item["attributes"]
			}
		}
		out = append(out, withAlwaysIncluded(projected, main, more, eprel))
	}
	return out
}

func withItemMeta(dst, src V1ProcessJobItem) V1ProcessJobItem {
	for _, k := range []string{"status", "error", "id", "processed_product_id", "raw_product_id"} {
		if v, ok := src[k]; ok {
			dst[k] = v
		}
	}
	return dst
}

// applyV1ProcessItemIDs sets legacy id (= processed UUID) plus additive dual-mode aliases.
// id is preserved for existing integrators; processed_product_id mirrors it; raw_product_id is raw_products.id.
func applyV1ProcessItemIDs(item V1ProcessJobItem, processedID, rawProductID *uuid.UUID) {
	if processedID != nil {
		s := processedID.String()
		item["id"] = s
		item["processed_product_id"] = s
	}
	if rawProductID != nil {
		item["raw_product_id"] = rawProductID.String()
	}
}

func withAlwaysIncluded(item V1ProcessJobItem, main string, more []string, eprel any) V1ProcessJobItem {
	if main != "" {
		item["main_image"] = main
	} else {
		item["main_image"] = nil
	}
	if len(more) > 0 {
		item["more_images"] = more
	} else {
		item["more_images"] = nil
	}
	if eprel == nil {
		item["eprel"] = nil
	} else {
		item["eprel"] = eprel
	}
	return item
}

type v1ResolvedSteps struct {
	isFull bool
	steps  map[string]struct{}
}

func resolveV1Steps(stored string) v1ResolvedSteps {
	normalized := strings.ToLower(strings.TrimSpace(stored))
	if normalized == "" || normalized == "full" {
		return v1ResolvedSteps{isFull: true, steps: map[string]struct{}{}}
	}
	if normalized == "both" {
		return v1ResolvedSteps{steps: map[string]struct{}{"title": {}, "description": {}}}
	}
	if strings.HasPrefix(normalized, "[") {
		var parsed []any
		if err := json.Unmarshal([]byte(stored), &parsed); err == nil {
			steps := map[string]struct{}{}
			for _, e := range parsed {
				if step := normalizeV1Step(fmt.Sprint(e)); step != "" {
					steps[step] = struct{}{}
				}
			}
			return v1ResolvedSteps{steps: steps}
		}
	}
	if step := normalizeV1Step(normalized); step != "" {
		return v1ResolvedSteps{steps: map[string]struct{}{step: {}}}
	}
	// Unknown stored types (normalize_only, enhance_only, …) → treat as full projection.
	return v1ResolvedSteps{isFull: true, steps: map[string]struct{}{}}
}