descrybe/apps/api/internal/processing/specs.go

package processing

import (
	"encoding/xml"
	"fmt"
	"html"
	"regexp"
	"strings"
)

var (
	htmlLiRe       = regexp.MustCompile(`(?is)<li[^>]*>(.*?)(?:</li\s*>|</\s*>)`)
	specsHTMLTagRe = regexp.MustCompile(`(?is)<[^>]+>`)
	csvLikeRe      = regexp.MustCompile(`(?m)^\s*([^:=\n\r]{1,120})\s*[:=]\s*(.+?)\s*$`)
	bulletLineRe   = regexp.MustCompile(`(?m)^\s*[-•*]\s*([^:=\n\r]{1,120})\s*[:=]\s*(.+?)\s*$`)
)

// Caps for locale/spec parsing — unbounded FindAll on huge CDATA can OOM the worker.
const (
	maxSpecInputBytes = 200_000
	maxSpecPairs      = 500
)

func capSpecInput(s string) string {
	if len(s) <= maxSpecInputBytes {
		return s
	}
	return s[:maxSpecInputBytes]
}

// ParseSpecifications extracts attribute key/values from tree XML, CDATA HTML, CSV-like, or nested maps.
func ParseSpecifications(v any) map[string]any {
	attrs := map[string]any{}
	parseSpecsInto(attrs, v)
	return attrs
}

func parseSpecsInto(dst map[string]any, v any) {
	if v == nil {
		return
	}
	switch t := v.(type) {
	case map[string]any:
		// Already structured attributes / grouped specs
		if looksLikeAttrMap(t) {
			for k, val := range t {
				lk := strings.ToLower(strings.TrimSpace(k))
				if isReservedProductKey(lk) || isInvalidAttributeKey(k) {
					continue
				}
				if s := stringifySpecValue(val); s != "" {
					key := SanitizeOutput(k)
					if key == "" || isInvalidAttributeKey(key) {
						continue
					}
					dst[key] = s
				} else if nested, ok := val.(map[string]any); ok {
					parseSpecsInto(dst, nested)
				}
			}
			return
		}
		for k, val := range t {
			lk := strings.ToLower(k)
			if strings.Contains(lk, "spec") {
				parseSpecsInto(dst, val)
				continue
			}
			if s := stringifySpecValue(val); s != "" && !isReservedProductKey(lk) && !isInvalidAttributeKey(k) {
				dst[SanitizeOutput(k)] = s
			}
		}
	case []any:
		for _, item := range t {
			parseSpecsInto(dst, item)
		}
	case string:
		s := strings.TrimSpace(t)
		if s == "" {
			return
		}
		if strings.Contains(s, "<") {
			parseHTMLSpecs(dst, s)
			if len(dst) > 0 {
				return
			}
			parseXMLTreeSpecs(dst, s)
			if len(dst) > 0 {
				return
			}
		}
		parseCSVLikeSpecs(dst, s)
	default:
		s := strings.TrimSpace(fmt.Sprint(t))
		if s != "" && s != "<nil>" {
			parseSpecsInto(dst, s)
		}
	}
}

func looksLikeAttrMap(m map[string]any) bool {
	if len(m) == 0 {
		return false
	}
	scalar := 0
	for _, v := range m {
		switch v.(type) {
		case string, float64, float32, int, int64, bool:
			scalar++
		}
	}
	return scalar >= len(m)/2
}

func isReservedProductKey(k string) bool {
	switch strings.ToLower(strings.TrimSpace(k)) {
	case "name", "title", "description", "gtin", "ean", "category", "category_unique_id",
		"price", "purchaseprice", "purchase_price", "sellingprice", "selling_price",
		"image", "main_image", "mainimage", "moreimages", "more_images", "images",
		"image_url", "imageurl", "image_link", "imagelink", "additional_image_urls",
		"additional_image_link", "videourl", "video_url",
		"stock", "stockstatus", "stock_status", "availability",
		"id", "sku", "officiallink", "official_link", "service",
		"specifications", "specs", "specification",
		"raw", "mapped", "search":
		return true
	default:
		return false
	}
}

// isInvalidAttributeKey rejects garbage keys from bad feed specs (e.g. ":").
func isInvalidAttributeKey(k string) bool {
	k = strings.TrimSpace(k)
	if k == "" || k == ":" || k == "=" || k == "-" || k == "_" {
		return true
	}
	// Must contain at least one letter after sanitize.
	hasLetter := false
	for _, r := range strings.ToLower(k) {
		if r >= 'a' && r <= 'z' {
			hasLetter = true
			break
		}
	}
	return !hasLetter
}

// coreCharacteristicAttrKeys are always allowed on V1 process items even when
// the company has no matching attributes row (common dims/brand/model).
var coreCharacteristicAttrKeys = map[string]struct{}{
	"brand": {}, "product_model": {}, "warranty": {},
	"width": {}, "height": {}, "depth": {}, "weight": {},
	"energy_class": {}, "color": {}, "material": {},
}

// SanitizeProductAttributes keeps characteristic attrs for API/storage and drops
// core product fields that belong on the V1 item root (title, description, images, …).
func SanitizeProductAttributes(attrs map[string]any) map[string]any {
	return sanitizeProductAttributes(attrs, false)
}

// SanitizeV1ProcessAttributes is the stricter poll projection: also drops eprel_*
// keys (those are exposed on item.eprel). Without an allowlist, feed junk specs
// (zavora, vzmetenje, …) can remain — prefer SanitizeV1ProcessAttributesAllowed.
func SanitizeV1ProcessAttributes(attrs map[string]any) map[string]any {
	return sanitizeProductAttributes(attrs, true)
}

// SanitizeV1ProcessAttributesAllowed sanitizes then keeps only core characteristic
// keys plus company attribute_key values (canonicalized).
func SanitizeV1ProcessAttributesAllowed(attrs map[string]any, allowed map[string]struct{}) map[string]any {
	return FilterAttributesByAllowed(sanitizeProductAttributes(attrs, true), allowed)
}

// FilterAttributesByAllowed keeps coreCharacteristicAttrKeys plus keys present in
// allowed (after canonicalizeAttrKey). When allowed is nil, returns attrs unchanged.
func FilterAttributesByAllowed(attrs map[string]any, allowed map[string]struct{}) map[string]any {
	if len(attrs) == 0 {
		return map[string]any{}
	}
	if allowed == nil {
		return attrs
	}
	out := make(map[string]any, len(attrs))
	for k, v := range attrs {
		canon := canonicalizeAttrKey(k)
		if canon == "" {
			continue
		}
		if _, ok := coreCharacteristicAttrKeys[canon]; ok {
			out[canon] = v
			continue
		}
		if _, ok := allowed[canon]; ok {
			out[canon] = v
			continue
		}
		if _, ok := allowed[strings.ToLower(strings.TrimSpace(k))]; ok {
			out[canon] = v
		}
	}
	return out
}

func sanitizeProductAttributes(attrs map[string]any, dropEPREL bool) map[string]any {
	if len(attrs) == 0 {
		return map[string]any{}
	}
	out := make(map[string]any, len(attrs))
	for k, v := range attrs {
		key := strings.TrimSpace(SanitizeOutput(k))
		if key == "" || isReservedProductKey(key) || isInvalidAttributeKey(key) {
			continue
		}
		if dropEPREL && strings.HasPrefix(strings.ToLower(key), "eprel") {
			continue
		}
		s := stringifySpecValue(v)
		if s == "" || s == "<nil>" {
			continue
		}
		// Prefer canonical dimension/model keys when aliases collide.
		canon := canonicalizeAttrKey(key)
		if canon == "" || isReservedProductKey(canon) || isInvalidAttributeKey(canon) {
			continue
		}
		if dropEPREL && strings.HasPrefix(strings.ToLower(canon), "eprel") {
			continue
		}
		if isDimensionKey(canon) && isZeroishString(s) {
			continue
		}
		if _, exists := out[canon]; exists && (canon != key) {
			continue
		}
		out[canon] = SanitizeOutput(s)
	}
	return out
}

func canonicalizeAttrKey(k string) string {
	compact := strings.ToLower(strings.TrimSpace(k))
	compact = strings.ReplaceAll(compact, "-", "_")
	compact = strings.ReplaceAll(compact, " ", "_")
	noUnderscore := strings.ReplaceAll(compact, "_", "")
	switch noUnderscore {
	case "netwidth", "width", "sirina":
		return "width"
	case "netheight", "height", "visina":
		return "height"
	case "netdepth", "depth", "globina":
		return "depth"
	case "netmass", "weight", "mass", "teza":
		return "weight"
	case "productmodel", "model":
		return "product_model"
	case "energijskirazred", "energyclass":
		return "energy_class"
	default:
		return compact
	}
}

func parseHTMLSpecs(dst map[string]any, s string) {
	s = capSpecInput(s)
	matches := htmlLiRe.FindAllStringSubmatch(s, maxSpecPairs)
	for _, m := range matches {
		if len(m) < 2 {
			continue
		}
		text := strings.TrimSpace(html.UnescapeString(specsHTMLTagRe.ReplaceAllString(m[1], "")))
		if text == "" {
			continue
		}
		key, val := splitLabelValue(text)
		attrKey := attributeKeyFromLabel(key)
		if attrKey != "" && val != "" && !isReservedProductKey(attrKey) && !isInvalidAttributeKey(attrKey) {
			dst[attrKey] = SanitizeOutput(val)
		}
	}
	if len(matches) == 0 {
		// Fallback: strip tags and parse as CSV-like / bullets
		plain := strings.TrimSpace(html.UnescapeString(specsHTMLTagRe.ReplaceAllString(s, "\n")))
		parseCSVLikeSpecs(dst, plain)
	}
}

func parseXMLTreeSpecs(dst map[string]any, s string) {
	type node struct {
		XMLName xml.Name
		Attrs   []xml.Attr `xml:",any,attr"`
		Content string     `xml:",chardata"`
		Nodes   []node     `xml:",any"`
	}
	// Wrap fragment so arbitrary roots parse.
	wrapped := "<specs>" + s + "</specs>"
	var root node
	if err := xml.Unmarshal([]byte(wrapped), &root); err != nil {
		return
	}
	var walk func(n node, path string)
	walk = func(n node, path string) {
		name := n.XMLName.Local
		if name == "" {
			name = path
		}
		text := strings.TrimSpace(n.Content)
		if len(n.Nodes) == 0 && text != "" && name != "" && name != "specs" {
			dst[SanitizeOutput(name)] = SanitizeOutput(text)
			return
		}
		// Common pattern: <spec name="Color">Red</spec> or <item><name/><value/>
		attrName := ""
		for _, a := range n.Attrs {
			an := strings.ToLower(a.Name.Local)
			if an == "name" || an == "key" || an == "label" {
				attrName = a.Value
			}
		}
		if attrName != "" && text != "" {
			dst[SanitizeOutput(attrName)] = SanitizeOutput(text)
		}
		childName, childVal := "", ""
		for _, c := range n.Nodes {
			ln := strings.ToLower(c.XMLName.Local)
			ct := strings.TrimSpace(c.Content)
			if ln == "name" || ln == "key" || ln == "label" {
				childName = ct
			}
			if ln == "value" || ln == "val" {
				childVal = ct
			}
			walk(c, c.XMLName.Local)
		}
		if childName != "" && childVal != "" {
			dst[SanitizeOutput(childName)] = SanitizeOutput(childVal)
		}
	}
	walk(root, "")
}

func parseCSVLikeSpecs(dst map[string]any, s string) {
	s = capSpecInput(s)
	for _, re := range []*regexp.Regexp{bulletLineRe, csvLikeRe} {
		for _, m := range re.FindAllStringSubmatch(s, maxSpecPairs) {
			if len(m) < 3 {
				continue
			}
			key := attributeKeyFromLabel(strings.TrimSpace(m[1]))
			val := strings.TrimSpace(m[2])
			if key != "" && val != "" && !isReservedProductKey(key) && !isInvalidAttributeKey(key) {
				dst[key] = SanitizeOutput(val)
			}
		}
	}
}

func splitLabelValue(text string) (string, string) {
	for _, sep := range []string{":", " - ", " – ", "="} {
		if i := strings.Index(text, sep); i > 0 {
			return strings.TrimSpace(text[:i]), strings.TrimSpace(text[i+len(sep):])
		}
	}
	return "", ""
}

func stringifySpecValue(v any) string {
	if v == nil {
		return ""
	}
	switch t := v.(type) {
	case string:
		return strings.TrimSpace(t)
	case float64, float32, int, int64, bool:
		return strings.TrimSpace(fmt.Sprint(t))
	default:
		return ""
	}
}

// attributeKeyFromLabel turns "Energijski razred" into "energijski-razred",
// and maps known locale/shipping aliases onto standard snake_case keys.
func attributeKeyFromLabel(label string) string {
	label = strings.TrimSpace(label)
	if label == "" {
		return ""
	}
	var b strings.Builder
	b.Grow(len(label))
	prevHyphen := false
	for _, r := range strings.ToLower(label) {
		switch r {
		case 'š', 'ś', 'ş':
			r = 's'
		case 'č', 'ć', 'ç':
			r = 'c'
		case 'ž', 'ź', 'ż':
			r = 'z'
		case 'đ':
			r = 'd'
		case 'ä', 'á', 'à', 'â', 'ã', 'å':
			r = 'a'
		case 'ë', 'é', 'è', 'ê':
			r = 'e'
		case 'ï', 'í', 'ì', 'î':
			r = 'i'
		case 'ö', 'ó', 'ò', 'ô', 'õ':
			r = 'o'
		case 'ü', 'ú', 'ù', 'û':
			r = 'u'
		case 'ý', 'ÿ':
			r = 'y'
		}
		switch {
		case r >= 'a' && r <= 'z', r >= '0' && r <= '9':
			b.WriteRune(r)
			prevHyphen = false
		case r == ' ' || r == '_' || r == '/' || r == '\\' || r == '.' || r == ':' || r == '-':
			if !prevHyphen && b.Len() > 0 {
				b.WriteByte('-')
				prevHyphen = true
			}
		}
	}
	slug := strings.Trim(b.String(), "-")
	compact := strings.ReplaceAll(slug, "-", "")
	switch compact {
	case "visina", "height", "netheight":
		return "net_height"
	case "sirina", "width", "netwidth":
		return "net_width"
	case "globina", "depth", "netdepth":
		return "net_depth"
	case "netmass", "mass", "weight", "teza":
		return "net_mass"
	case "productmodel", "model":
		return "product_model"
	case "eprelid", "eprel":
		return "eprel_id"
	case "energyclass", "energijskirazred":
		return "energy_class"
	}
	if slug == "" || len(compact) < 2 {
		return ""
	}
	hasLetter := false
	for _, r := range compact {
		if r >= 'a' && r <= 'z' {
			hasLetter = true
			break
		}
	}
	if !hasLetter {
		return ""
	}
	switch compact {
	case "true", "false", "yes", "no", "null", "undefined", "none", "n", "y":
		return ""
	}
	return slug
}