descrybe/apps/api/internal/processing/normalize.go

package processing

import (
	"fmt"
	"strings"
)

// knownKeyAliases maps vendor / feed keys onto canonical standard-field keys.
var knownKeyAliases = map[string]string{
	"ean":              "gtin",
	"ean13":            "gtin",
	"barcode":          "gtin",
	"sku":              "sku",
	"product_name":     "name",
	"title":            "name",
	"producttitle":     "name",
	"desc":             "description",
	"body":             "description",
	"shortdescription": "description",
	"product_type":     "category",
	"producttype":      "category",
	"brand_name":       "brand",
	"manufacturer":     "brand",
	"mainimage":        "image",
	"main_image":       "image",
	"image_url":        "image",
	"imageurl":         "image",
	"purchaseprice":    "price",
	"purchase_price":   "price",
	"sellingprice":     "price",
	"netwidth":         "width",
	"netheight":        "height",
	"netdepth":         "depth",
	"netmass":          "weight",
	"weight_kg":        "weight",
	"eprelid":          "eprel_id",
	"stockstatus":      "stock_status",
	"stock":            "stock",
	"spec":             "specifications",
	"specs":            "specifications",
	"specification":    "specifications",
}

// NormalizeMapped flattens aliases, trims strings, drops empty/#text wrappers,
// and coerces obvious zero-dimension placeholders to empty (not fake "0").
func NormalizeMapped(mapped, raw map[string]any) map[string]any {
	out := make(map[string]any)
	mergeNormalized(out, raw)
	mergeNormalized(out, mapped) // mapped wins
	return out
}

func mergeNormalized(dst, src map[string]any) {
	if src == nil {
		return
	}
	for k, v := range src {
		canon := canonicalizeKey(k)
		nv := normalizeValue(canon, v)
		if nv == nil {
			continue
		}
		if _, exists := dst[canon]; exists && isEmptyValue(nv) {
			continue
		}
		dst[canon] = nv
	}
}

func canonicalizeKey(k string) string {
	compact := strings.ToLower(strings.TrimSpace(k))
	compact = strings.ReplaceAll(compact, "-", "_")
	compact = strings.ReplaceAll(compact, " ", "_")
	noUnderscore := strings.ReplaceAll(compact, "_", "")
	if alias, ok := knownKeyAliases[compact]; ok {
		return alias
	}
	if alias, ok := knownKeyAliases[noUnderscore]; ok {
		return alias
	}
	return compact
}

func normalizeValue(key string, v any) any {
	if v == nil {
		return nil
	}
	// Descriptions must be plain scalars (never leftover ["…"] / HTML arrays).
	if isDescriptionFieldKey(key) {
		if s := PlainDescriptionFromAny(v); s != "" {
			return SanitizeText(s)
		}
		return nil
	}
	switch t := v.(type) {
	case string:
		s := strings.TrimSpace(t)
		if s == "" {
			return nil
		}
		if isDimensionKey(key) && isZeroishString(s) {
			return nil
		}
		return SanitizeText(s)
	case float64:
		if isDimensionKey(key) && t == 0 {
			return nil
		}
		return t
	case float32:
		if isDimensionKey(key) && t == 0 {
			return nil
		}
		return float64(t)
	case int:
		if isDimensionKey(key) && t == 0 {
			return nil
		}
		return t
	case int64:
		if isDimensionKey(key) && t == 0 {
			return nil
		}
		return t
	case bool:
		return t
	case map[string]any:
		if text, ok := t["#text"]; ok {
			return normalizeValue(key, text)
		}
		if text, ok := t["text"]; ok {
			return normalizeValue(key, text)
		}
		// Feed/XML category objects often carry unique_id — flatten to a scalar
		// so StepResult.Category / processed_products.category get the code.
		if key == "category" {
			if id := categoryUniqueIDFromAny(t); id != "" {
				return id
			}
		}
		nested := make(map[string]any, len(t))
		for nk, nv := range t {
			if nn := normalizeValue(canonicalizeKey(nk), nv); nn != nil {
				nested[canonicalizeKey(nk)] = nn
			}
		}
		if len(nested) == 0 {
			return nil
		}
		return nested
	case []any:
		if len(t) == 0 {
			return nil
		}
		// Scalar-ish fields (name, brand, eprel_id, …): collapse one-element /
		// joinable arrays so process steps never see []any where a string is expected.
		if flat := coerceScalarText(t); flat != "" && looksLikeScalarFieldArray(t) {
			if isDimensionKey(key) && isZeroishString(flat) {
				return nil
			}
			return SanitizeText(flat)
		}
		out := make([]any, 0, len(t))
		for _, item := range t {
			if nn := normalizeValue(key, item); nn != nil {
				out = append(out, nn)
			}
		}
		if len(out) == 0 {
			return nil
		}
		return out
	default:
		s := strings.TrimSpace(fmt.Sprint(t))
		if s == "" || s == "<nil>" {
			return nil
		}
		if isDimensionKey(key) && isZeroishString(s) {
			return nil
		}
		return SanitizeText(s)
	}
}

// looksLikeScalarFieldArray is true when every element coerces to a short scalar
// (string/number/#text) — not nested attribute/spec objects.
func looksLikeScalarFieldArray(items []any) bool {
	if len(items) == 0 {
		return false
	}
	for _, item := range items {
		switch t := item.(type) {
		case nil:
			continue
		case string, float64, float32, int, int64, int32, bool:
			continue
		case map[string]any:
			if _, ok := t["#text"]; ok {
				continue
			}
			if _, ok := t["text"]; ok {
				continue
			}
			if _, ok := t["value"]; ok {
				continue
			}
			return false
		case []any, []string:
			return false
		default:
			if _, ok := item.(jsonNumberStringer); ok {
				continue
			}
			return false
		}
	}
	return true
}

func isDimensionKey(key string) bool {
	switch key {
	case "width", "height", "depth", "weight", "length", "net_width", "net_height", "net_depth", "net_mass":
		return true
	default:
		return false
	}
}

func isZeroishString(s string) bool {
	s = strings.TrimSpace(strings.ToLower(s))
	if s == "" {
		return false
	}
	if s == "0" || s == "0.0" || s == "0,0" || s == "0.00" {
		return true
	}
	// Strip common unit suffixes (m, cm, mm, kg, g) then re-check numeric zero.
	trimmed := s
	for _, u := range []string{"kg", "cm", "mm", "g", "m"} {
		if strings.HasSuffix(trimmed, u) {
			trimmed = strings.TrimSpace(strings.TrimSuffix(trimmed, u))
			break
		}
	}
	trimmed = strings.ReplaceAll(trimmed, " ", "")
	if trimmed == "" {
		return false
	}
	// 0 / 0.0 / 0,00 / 0.0000 / 0,0000
	onlyZero := true
	sawDigit := false
	for _, r := range trimmed {
		switch r {
		case '0':
			sawDigit = true
		case '.', ',':
			continue
		default:
			onlyZero = false
		}
	}
	return onlyZero && sawDigit
}

func isEmptyValue(v any) bool {
	if v == nil {
		return true
	}
	switch t := v.(type) {
	case string:
		return strings.TrimSpace(t) == ""
	case map[string]any:
		return len(t) == 0
	case []any:
		return len(t) == 0
	default:
		return false
	}
}