descrybe/apps/api/internal/feeds/specs.go

package feeds

import (
	"html"
	"regexp"
	"strings"
	"unicode"
)

const (
	maxSpecPairs       = 200
	maxSpecRawBytes    = 64 << 10 // 64 KiB per specifications blob
	maxSpecLabelRunes  = 120
	maxSpecValueRunes  = 2000
)

var (
	// Accept </li> and broken </> closers seen in A1 feed CDATA.
	reHTMLLi     = regexp.MustCompile(`(?is)<li\b[^>]*>(.*?)(?:</li\s*>|</\s*>)`)
	reHTMLTag    = regexp.MustCompile(`(?is)<[^>]+>`)
	reMultiSpace = regexp.MustCompile(`\s+`)
)

// SpecPair is one label/value extracted from a specifications blob.
type SpecPair struct {
	Label string
	Value string
}

// ParseSpecifications accepts nested-expanded text, CDATA HTML lists, or flat
// CSV-like strings. Empty / missing / blank HTML returns nil (not an error).
func ParseSpecifications(raw string) []SpecPair {
	raw = strings.TrimSpace(raw)
	if raw == "" || len(raw) > maxSpecRawBytes {
		return nil
	}
	if isEmptySpecBlob(raw) {
		return nil
	}

	var pairs []SpecPair
	switch {
	case looksLikeHTMLList(raw):
		pairs = parseHTMLSpecList(raw)
	case looksLikeFlatSpecs(raw):
		pairs = parseFlatSpecs(raw)
	default:
		// Single "Label: value" line still counts as flat.
		if p, ok := splitLabelValue(raw); ok {
			pairs = []SpecPair{p}
		}
	}
	return clampSpecPairs(pairs)
}

// expandSpecificationFields mutates row: for specification-like keys whose
// value is HTML/flat text, add nested paths key/Label → value. Nested XML
// children are already present as key/child from the XML walker.
func expandSpecificationFields(row feedRow) {
	if len(row) == 0 {
		return
	}
	keys := make([]string, 0, 8)
	for k, v := range row {
		if !isSpecFieldKey(k) {
			continue
		}
		if strings.TrimSpace(v) == "" {
			continue
		}
		// Already has nested children — leave tree as-is; still parse text if useful.
		if hasPrefixedChildren(row, k) && !looksLikeHTMLList(v) && !looksLikeFlatSpecs(v) {
			continue
		}
		keys = append(keys, k)
	}
	for _, k := range keys {
		pairs := ParseSpecifications(row[k])
		for _, p := range pairs {
			seg := sanitizePathSegment(p.Label)
			if seg == "" {
				continue
			}
			path := k + "/" + seg
			if prev, ok := row[path]; ok && strings.TrimSpace(prev) != "" {
				continue
			}
			row[path] = p.Value
		}
	}
}

func isSpecFieldKey(key string) bool {
	leaf := strings.ToLower(leafName(key))
	leaf = strings.TrimPrefix(leaf, "@")
	switch leaf {
	case "specifications", "specification", "specs", "spec", "features", "feature", "attributes_raw":
		return true
	}
	return strings.Contains(leaf, "specification")
}

func hasPrefixedChildren(row feedRow, prefix string) bool {
	prefix = strings.TrimSuffix(prefix, "/") + "/"
	for k := range row {
		if strings.HasPrefix(k, prefix) {
			return true
		}
	}
	return false
}

func collectPrefixed(row feedRow, prefix string) map[string]string {
	prefix = strings.TrimSuffix(strings.TrimSpace(prefix), "/")
	if prefix == "" {
		return nil
	}
	p := prefix + "/"
	out := map[string]string{}
	for k, v := range row {
		if !strings.HasPrefix(k, p) {
			continue
		}
		rest := k[len(p):]
		if rest == "" || strings.Contains(rest, "/") {
			continue
		}
		v = strings.TrimSpace(v)
		if v == "" {
			continue
		}
		out[rest] = v
	}
	if len(out) == 0 {
		return nil
	}
	return out
}

func isEmptySpecBlob(raw string) bool {
	stripped := strings.TrimSpace(reHTMLTag.ReplaceAllString(raw, " "))
	stripped = html.UnescapeString(stripped)
	stripped = strings.TrimSpace(reMultiSpace.ReplaceAllString(stripped, " "))
	return stripped == ""
}

func looksLikeHTMLList(raw string) bool {
	lower := strings.ToLower(raw)
	return strings.Contains(lower, "<li") || (strings.Contains(lower, "<ul") && strings.Contains(lower, "</ul>"))
}

func looksLikeFlatSpecs(raw string) bool {
	if looksLikeHTMLList(raw) {
		return false
	}
	// Multiple label:value pairs separated by ; | newline or comma between pairs.
	if strings.Count(raw, ":") >= 2 {
		return true
	}
	if strings.Count(raw, "=") >= 2 && (strings.Contains(raw, ";") || strings.Contains(raw, "|") || strings.Contains(raw, "\n")) {
		return true
	}
	if strings.Contains(raw, ";") && strings.Contains(raw, ":") {
		return true
	}
	if strings.Contains(raw, "|") && strings.Contains(raw, ":") {
		return true
	}
	if strings.Contains(raw, "\n") && strings.Contains(raw, ":") {
		return true
	}
	// Quoted CSV-ish pairs: "Brand","Acme"; "Model","X1"
	if strings.Count(raw, `"`) >= 4 && (strings.Contains(raw, ";") || strings.Contains(raw, ",")) {
		return true
	}
	return false
}

func parseHTMLSpecList(raw string) []SpecPair {
	matches := reHTMLLi.FindAllStringSubmatch(raw, maxSpecPairs+1)
	if len(matches) == 0 {
		// Fallback: strip tags and try flat parse.
		plain := strings.TrimSpace(reHTMLTag.ReplaceAllString(raw, "\n"))
		plain = html.UnescapeString(plain)
		return parseFlatSpecs(plain)
	}
	out := make([]SpecPair, 0, len(matches))
	for _, m := range matches {
		inner := strings.TrimSpace(reHTMLTag.ReplaceAllString(m[1], " "))
		inner = html.UnescapeString(inner)
		inner = strings.TrimSpace(reMultiSpace.ReplaceAllString(inner, " "))
		if inner == "" {
			continue
		}
		if p, ok := splitLabelValue(inner); ok {
			out = append(out, p)
		}
		// Bare list items without "Label: value" are skipped — suppliers should
		// send explicit pairs; inventing key→"true" produces junk attributes.
		if len(out) >= maxSpecPairs {
			break
		}
	}
	return out
}

func parseFlatSpecs(raw string) []SpecPair {
	raw = strings.ReplaceAll(raw, "\r\n", "\n")
	raw = strings.ReplaceAll(raw, "\r", "\n")

	chunks := splitSpecChunks(raw)
	out := make([]SpecPair, 0, len(chunks))
	for _, chunk := range chunks {
		chunk = strings.TrimSpace(chunk)
		if chunk == "" {
			continue
		}
		// CSV-ish "Label","value" or Label,value
		if strings.Contains(chunk, ",") {
			if p, ok := parseCSVSpecChunk(chunk); ok {
				out = append(out, p)
				if len(out) >= maxSpecPairs {
					break
				}
				continue
			}
		}
		if p, ok := splitLabelValue(chunk); ok {
			out = append(out, p)
		}
		if len(out) >= maxSpecPairs {
			break
		}
	}
	return out
}

func splitSpecChunks(raw string) []string {
	// Prefer strong separators first.
	for _, sep := range []string{"\n", ";", "|"} {
		if strings.Contains(raw, sep) {
			return strings.Split(raw, sep)
		}
	}
	// Comma only when it looks like paired entries (has colon/equals).
	if strings.Contains(raw, ",") && (strings.Contains(raw, ":") || strings.Contains(raw, "=")) {
		return strings.Split(raw, ",")
	}
	return []string{raw}
}

func parseCSVSpecChunk(chunk string) (SpecPair, bool) {
	parts := strings.SplitN(chunk, ",", 2)
	if len(parts) != 2 {
		return SpecPair{}, false
	}
	label := strings.Trim(strings.TrimSpace(parts[0]), `"'`)
	value := strings.Trim(strings.TrimSpace(parts[1]), `"'`)
	if label == "" || value == "" {
		return SpecPair{}, false
	}
	return SpecPair{
		Label: truncateRunes(label, maxSpecLabelRunes),
		Value: truncateRunes(value, maxSpecValueRunes),
	}, true
}

func splitLabelValue(s string) (SpecPair, bool) {
	s = strings.TrimSpace(s)
	if s == "" {
		return SpecPair{}, false
	}
	// Prefer "Label: value" / "Label：value" / "Label - value" / "Label = value"
	for _, sep := range []string{":", "：", "=", "–", "—"} {
		if i := strings.Index(s, sep); i > 0 {
			label := strings.TrimSpace(s[:i])
			value := strings.TrimSpace(s[i+len(sep):])
			if label != "" && value != "" && !looksLikeURLScheme(label) {
				return SpecPair{
					Label: truncateRunes(label, maxSpecLabelRunes),
					Value: truncateRunes(value, maxSpecValueRunes),
				}, true
			}
		}
	}
	// "Label - value" with spaces (avoid splitting hyphenated words alone)
	if i := strings.Index(s, " - "); i > 0 {
		label := strings.TrimSpace(s[:i])
		value := strings.TrimSpace(s[i+3:])
		if label != "" && value != "" {
			return SpecPair{
				Label: truncateRunes(label, maxSpecLabelRunes),
				Value: truncateRunes(value, maxSpecValueRunes),
			}, true
		}
	}
	return SpecPair{}, false
}

func looksLikeURLScheme(label string) bool {
	lower := strings.ToLower(strings.TrimSpace(label))
	return lower == "http" || lower == "https" || lower == "ftp"
}

func sanitizePathSegment(label string) string {
	label = strings.TrimSpace(label)
	if label == "" {
		return ""
	}
	var b strings.Builder
	b.Grow(len(label))
	prevUS := false
	for _, r := range label {
		switch {
		case unicode.IsLetter(r) || unicode.IsDigit(r):
			b.WriteRune(r)
			prevUS = false
		case r == '_' || r == '-' || r == '.':
			b.WriteRune(r)
			prevUS = false
		case unicode.IsSpace(r) || r == '/' || r == '\\':
			if !prevUS && b.Len() > 0 {
				b.WriteByte('_')
				prevUS = true
			}
		default:
			// drop punctuation
		}
	}
	out := strings.Trim(b.String(), "._-")
	return truncateRunes(out, maxSpecLabelRunes)
}

func clampSpecPairs(pairs []SpecPair) []SpecPair {
	if len(pairs) == 0 {
		return nil
	}
	if len(pairs) > maxSpecPairs {
		pairs = pairs[:maxSpecPairs]
	}
	seen := make(map[string]struct{}, len(pairs))
	out := make([]SpecPair, 0, len(pairs))
	for _, p := range pairs {
		p.Label = strings.TrimSpace(p.Label)
		p.Value = strings.TrimSpace(p.Value)
		if p.Label == "" || p.Value == "" {
			continue
		}
		key := strings.ToLower(p.Label)
		if _, ok := seen[key]; ok {
			continue
		}
		seen[key] = struct{}{}
		out = append(out, p)
	}
	if len(out) == 0 {
		return nil
	}
	return out
}

func truncateRunes(s string, max int) string {
	if max <= 0 || s == "" {
		return s
	}
	n := 0
	for i := range s {
		if n == max {
			return s[:i]
		}
		n++
	}
	return s
}

func specsToMap(pairs []SpecPair) map[string]string {
	if len(pairs) == 0 {
		return nil
	}
	out := make(map[string]string, len(pairs))
	for _, p := range pairs {
		key := CanonicalAttributeKey(p.Label)
		if key == "" || strings.TrimSpace(p.Value) == "" {
			continue
		}
		out[key] = p.Value
	}
	if len(out) == 0 {
		return nil
	}
	return out
}

// compactAttributeKey strips separators for alias lookup (net_height / net-height / netheight → netheight).
func compactAttributeKey(key string) string {
	var b strings.Builder
	b.Grow(len(key))
	for _, r := range strings.ToLower(key) {
		r = foldLatinRune(r)
		if (r >= 'a' && r <= 'z') || (r >= '0' && r <= '9') {
			b.WriteRune(r)
		}
	}
	return b.String()
}

// Known supplier / locale labels → Descrybe standard field keys (snake_case).
// Freeform specs keep kebab-case from AttributeKeyFromLabel.
var attributeKeyAliases = map[string]string{
	"visina":       "net_height",
	"height":       "net_height",
	"netheight":    "net_height",
	"sirina":       "net_width",
	"width":        "net_width",
	"netwidth":     "net_width",
	"globina":      "net_depth",
	"depth":        "net_depth",
	"netdepth":     "net_depth",
	"netmass":      "net_mass",
	"mass":         "net_mass",
	"weight":       "net_mass",
	"teza":         "net_mass",
	"productmodel": "product_model",
	"model":        "product_model",
	"eprelid":      "eprel_id",
	"eprel":        "eprel_id",
	"energyclass":  "energy_class",
	"energijskirazred": "energy_class",
}

// IsValidAttributeKey rejects empty / punctuation-only / boolean junk keys.
func IsValidAttributeKey(key string) bool {
	key = strings.TrimSpace(key)
	if key == "" || len(key) < 2 {
		return false
	}
	compact := compactAttributeKey(key)
	if len(compact) < 2 {
		return false
	}
	hasLetter := false
	for _, r := range compact {
		if r >= 'a' && r <= 'z' {
			hasLetter = true
			break
		}
	}
	if !hasLetter {
		return false
	}
	switch compact {
	case "true", "false", "yes", "no", "null", "undefined", "none", "n", "y":
		return false
	}
	return true
}

// CanonicalAttributeKey normalizes a human or feed label to a stable attribute key.
// Known dimension/identity aliases map to STANDARD_FIELDS snake_case; other labels
// become kebab-case. Invalid / junk labels return "".
func CanonicalAttributeKey(label string) string {
	slug := AttributeKeyFromLabel(label)
	compact := compactAttributeKey(slug)
	if compact == "" {
		compact = compactAttributeKey(label)
	}
	if alias, ok := attributeKeyAliases[compact]; ok {
		return alias
	}
	if slug == "" || !IsValidAttributeKey(slug) {
		return ""
	}
	return slug
}

// AttributeKeyFromLabel turns a human spec label into a kebab-case attribute_key
// (e.g. "Energijski razred" → "energijski-razred") matching company attributes.
func AttributeKeyFromLabel(label string) string {
	label = strings.TrimSpace(label)
	if label == "" {
		return ""
	}
	var b strings.Builder
	b.Grow(len(label))
	prevHyphen := false
	for _, r := range strings.ToLower(label) {
		r = foldLatinRune(r)
		switch {
		case r >= 'a' && r <= 'z', r >= '0' && r <= '9':
			b.WriteRune(r)
			prevHyphen = false
		case unicode.IsSpace(r) || r == '_' || r == '/' || r == '\\' || r == '.' || r == ':' || r == '-':
			if !prevHyphen && b.Len() > 0 {
				b.WriteByte('-')
				prevHyphen = true
			}
		default:
			// drop other punctuation
		}
	}
	return strings.Trim(b.String(), "-")
}

func foldLatinRune(r rune) rune {
	switch r {
	case 'š', 'ś', 'ş':
		return 's'
	case 'č', 'ć', 'ç':
		return 'c'
	case 'ž', 'ź', 'ż':
		return 'z'
	case 'đ':
		return 'd'
	case 'ň', 'ń':
		return 'n'
	case 'ř':
		return 'r'
	case 'ť':
		return 't'
	case 'ď':
		return 'd'
	case 'ľ', 'ĺ':
		return 'l'
	case 'ä', 'á', 'à', 'â', 'ã', 'å':
		return 'a'
	case 'ë', 'é', 'è', 'ê':
		return 'e'
	case 'ï', 'í', 'ì', 'î':
		return 'i'
	case 'ö', 'ó', 'ò', 'ô', 'õ':
		return 'o'
	case 'ü', 'ú', 'ù', 'û':
		return 'u'
	case 'ý', 'ÿ':
		return 'y'
	default:
		return r
	}
}