package processing
import (
"encoding/xml"
"fmt"
"html"
"regexp"
"strings"
)
var (
htmlLiRe = regexp.MustCompile(`(?is)
]*>(.*?)(?:|\s*>)`)
specsHTMLTagRe = regexp.MustCompile(`(?is)<[^>]+>`)
csvLikeRe = regexp.MustCompile(`(?m)^\s*([^:=\n\r]{1,120})\s*[:=]\s*(.+?)\s*$`)
bulletLineRe = regexp.MustCompile(`(?m)^\s*[-•*]\s*([^:=\n\r]{1,120})\s*[:=]\s*(.+?)\s*$`)
// GTIN/EAN/UPC masquerading as product_model (feed mapping mistakes).
barcodeLikeModelRe = regexp.MustCompile(`^\d{8,14}$`)
)
// Caps for locale/spec parsing — unbounded FindAll on huge CDATA can OOM the worker.
const (
maxSpecInputBytes = 200_000
maxSpecPairs = 500
)
func capSpecInput(s string) string {
if len(s) <= maxSpecInputBytes {
return s
}
return s[:maxSpecInputBytes]
}
// ParseSpecifications extracts attribute key/values from tree XML, CDATA HTML, CSV-like, or nested maps.
func ParseSpecifications(v any) map[string]any {
attrs := map[string]any{}
parseSpecsInto(attrs, v)
return attrs
}
func parseSpecsInto(dst map[string]any, v any) {
if v == nil {
return
}
switch t := v.(type) {
case map[string]any:
// Already structured attributes / grouped specs
if looksLikeAttrMap(t) {
for k, val := range t {
lk := strings.ToLower(strings.TrimSpace(k))
if isReservedProductKey(lk) || isInvalidAttributeKey(k) {
continue
}
if s := stringifySpecValue(val); s != "" {
key := SanitizeOutput(k)
if key == "" || isInvalidAttributeKey(key) {
continue
}
dst[key] = s
} else if nested, ok := val.(map[string]any); ok {
parseSpecsInto(dst, nested)
}
}
return
}
for k, val := range t {
lk := strings.ToLower(k)
if strings.Contains(lk, "spec") {
parseSpecsInto(dst, val)
continue
}
if s := stringifySpecValue(val); s != "" && !isReservedProductKey(lk) && !isInvalidAttributeKey(k) {
dst[SanitizeOutput(k)] = s
}
}
case []any:
for _, item := range t {
parseSpecsInto(dst, item)
}
case string:
s := strings.TrimSpace(t)
if s == "" {
return
}
if strings.Contains(s, "<") {
parseHTMLSpecs(dst, s)
if len(dst) > 0 {
return
}
parseXMLTreeSpecs(dst, s)
if len(dst) > 0 {
return
}
}
parseCSVLikeSpecs(dst, s)
default:
s := strings.TrimSpace(fmt.Sprint(t))
if s != "" && s != "" {
parseSpecsInto(dst, s)
}
}
}
func looksLikeAttrMap(m map[string]any) bool {
if len(m) == 0 {
return false
}
scalar := 0
for _, v := range m {
switch v.(type) {
case string, float64, float32, int, int64, bool:
scalar++
}
}
return scalar >= len(m)/2
}
func isReservedProductKey(k string) bool {
switch strings.ToLower(strings.TrimSpace(k)) {
case "name", "title", "description", "gtin", "ean", "category", "category_unique_id",
"price", "purchaseprice", "purchase_price", "sellingprice", "selling_price",
"image", "main_image", "mainimage", "moreimages", "more_images", "images",
"image_url", "imageurl", "image_link", "imagelink", "additional_image_urls",
"additional_image_link", "videourl", "video_url",
"stock", "stockstatus", "stock_status", "availability",
"id", "sku", "officiallink", "official_link", "service",
"specifications", "specs", "specification",
"raw", "mapped", "search":
return true
default:
return false
}
}
// isInvalidAttributeKey rejects garbage keys from bad feed specs (e.g. ":").
func isInvalidAttributeKey(k string) bool {
k = strings.TrimSpace(k)
if k == "" || k == ":" || k == "=" || k == "-" || k == "_" {
return true
}
// Must contain at least one letter after sanitize.
hasLetter := false
for _, r := range strings.ToLower(k) {
if r >= 'a' && r <= 'z' {
hasLetter = true
break
}
}
return !hasLetter
}
// coreCharacteristicAttrKeys are always allowed on V1 process items even when
// the company has no matching attributes row (common dims/brand/model).
var coreCharacteristicAttrKeys = map[string]struct{}{
"brand": {}, "product_model": {}, "warranty": {},
"width": {}, "height": {}, "depth": {}, "weight": {},
"energy_class": {}, "color": {}, "material": {},
}
// SanitizeProductAttributes keeps characteristic attrs for API/storage and drops
// core product fields that belong on the V1 item root (title, description, images, …).
func SanitizeProductAttributes(attrs map[string]any) map[string]any {
return sanitizeProductAttributes(attrs, false)
}
// SanitizeV1ProcessAttributes is the stricter poll projection: also drops eprel_*
// keys (those are exposed on item.eprel). Without an allowlist, feed junk specs
// (zavora, vzmetenje, …) can remain — prefer SanitizeV1ProcessAttributesAllowed.
func SanitizeV1ProcessAttributes(attrs map[string]any) map[string]any {
return sanitizeProductAttributes(attrs, true)
}
// SanitizeV1ProcessAttributesAllowed sanitizes then keeps only core characteristic
// keys plus company attribute_key values (canonicalized).
func SanitizeV1ProcessAttributesAllowed(attrs map[string]any, allowed map[string]struct{}) map[string]any {
return FilterAttributesByAllowed(sanitizeProductAttributes(attrs, true), allowed)
}
// AttrsForEnhance prepares attributes for AI enhance prompts / hashes.
// Always drops reserved and invalid keys. When allowed is non-nil (including empty),
// keeps only coreCharacteristicAttrKeys plus the allowlist (category_attributes).
// When allowed is nil, returns sanitized attrs unchanged (unit-test fallback).
func AttrsForEnhance(attrs map[string]any, allowed map[string]struct{}) map[string]any {
cleaned := sanitizeProductAttributes(attrs, true)
if allowed == nil {
return cleaned
}
cleaned = MapAttrsOntoAllowedKeys(cleaned, allowed)
return FilterAttributesByAllowed(cleaned, allowed)
}
// ensureEnergyClassFromEPREL copies eprel_energy_class → energy_class when the
// core energy_class key is empty so AttrsForEnhance (which dropEPREL strips
// eprel_*) still surfaces the label class to the model.
func ensureEnergyClassFromEPREL(attrs map[string]any) {
if attrs == nil {
return
}
if v, ok := attrs["energy_class"]; ok {
if s := strings.TrimSpace(fmt.Sprint(v)); s != "" && s != "" {
return
}
}
for _, k := range []string{"eprel_energy_class", "eprelEnergyClass"} {
if v, ok := attrs[k]; ok {
s := strings.TrimSpace(fmt.Sprint(v))
if s != "" && s != "" {
attrs["energy_class"] = s
return
}
}
}
}
// AttrsForPersist prepares attributes for DB storage (attributes / processed_attributes).
// Same category allowlist semantics as AttrsForEnhance, but uses SanitizeProductAttributes
// (keeps eprel_* for poll extractEPRELFromAttrs) then FilterAttributesByAllowed.
// Feed/spec keys are remapped onto category_attributes (formula) keys when possible so
// values under near-miss labels (velikost-zaslona → diagonala_zaslona) are kept.
// When allowed is nil, returns sanitized attrs unchanged (unit-test fallback).
func AttrsForPersist(attrs map[string]any, allowed map[string]struct{}) map[string]any {
cleaned := SanitizeProductAttributes(attrs)
if allowed == nil {
return cleaned
}
cleaned = MapAttrsOntoAllowedKeys(cleaned, allowed)
filtered := FilterAttributesByAllowed(cleaned, allowed)
// Preserve eprel_* that SanitizeProductAttributes kept (not on category allowlists).
for k, v := range cleaned {
if strings.HasPrefix(strings.ToLower(strings.TrimSpace(k)), "eprel") {
filtered[k] = v
}
}
return filtered
}
// enhanceAllowedAttrKeys prefers category_attributes for the resolved category;
// falls back to AllowedAttrKeys (nil = sanitize-only for unit tests).
func enhanceAllowedAttrKeys(in ProductInput, categoryUID string) map[string]struct{} {
if in.CategoryAttrKeys != nil {
return allowedAttrKeysFromSets(in.CategoryAttrKeys, categoryUID)
}
return in.AllowedAttrKeys
}
// FilterAttributesByAllowed keeps coreCharacteristicAttrKeys plus keys present in
// allowed (after canonicalizeAttrKey). When allowed is nil, returns attrs unchanged.
func FilterAttributesByAllowed(attrs map[string]any, allowed map[string]struct{}) map[string]any {
if len(attrs) == 0 {
return map[string]any{}
}
if allowed == nil {
return attrs
}
out := make(map[string]any, len(attrs))
for k, v := range attrs {
canon := canonicalizeAttrKey(k)
if canon == "" {
continue
}
if _, ok := coreCharacteristicAttrKeys[canon]; ok {
out[canon] = v
continue
}
if _, ok := allowed[canon]; ok {
out[canon] = v
continue
}
if _, ok := allowed[strings.ToLower(strings.TrimSpace(k))]; ok {
out[canon] = v
}
}
return out
}
func sanitizeProductAttributes(attrs map[string]any, dropEPREL bool) map[string]any {
if len(attrs) == 0 {
return map[string]any{}
}
out := make(map[string]any, len(attrs))
for k, v := range attrs {
key := strings.TrimSpace(SanitizeOutput(k))
if key == "" || isReservedProductKey(key) || isInvalidAttributeKey(key) {
continue
}
if dropEPREL && strings.HasPrefix(strings.ToLower(key), "eprel") {
continue
}
// Keep nested eprel object for storage / extractEPRELFromAttrs (stringify would drop maps).
if !dropEPREL && strings.EqualFold(key, "eprel") {
if m, ok := v.(map[string]any); ok && len(m) > 0 {
nested := make(map[string]any, len(m))
for nk, nv := range m {
nk = strings.TrimSpace(nk)
if nk == "" || nv == nil {
continue
}
if s, ok := nv.(string); ok {
s = strings.TrimSpace(SanitizeOutput(s))
if s == "" {
continue
}
nested[nk] = s
continue
}
nested[nk] = nv
}
if len(nested) > 0 {
out["eprel"] = nested
}
}
continue
}
s := stringifySpecValue(v)
if s == "" || s == "" {
continue
}
// Prefer canonical dimension/model keys when aliases collide.
canon := canonicalizeAttrKey(key)
if canon == "" || isReservedProductKey(canon) || isInvalidAttributeKey(canon) {
continue
}
if dropEPREL && strings.HasPrefix(strings.ToLower(canon), "eprel") {
continue
}
if isDimensionKey(canon) && isZeroishString(s) {
continue
}
// Drop barcode-as-model (e.g. mapped product_model = GTIN).
if canon == "product_model" && barcodeLikeModelRe.MatchString(strings.TrimSpace(s)) {
continue
}
if _, exists := out[canon]; exists && (canon != key) {
continue
}
out[canon] = SanitizeOutput(s)
}
return out
}
func canonicalizeAttrKey(k string) string {
compact := strings.ToLower(strings.TrimSpace(k))
compact = strings.ReplaceAll(compact, "-", "_")
compact = strings.ReplaceAll(compact, " ", "_")
noUnderscore := strings.ReplaceAll(compact, "_", "")
switch noUnderscore {
case "netwidth", "width", "sirina":
return "width"
case "netheight", "height", "visina":
return "height"
case "netdepth", "depth", "globina":
return "depth"
case "netmass", "weight", "mass", "teza":
return "weight"
case "productmodel", "model":
return "product_model"
case "energijskirazred", "energyclass":
return "energy_class"
case "eprelid", "eprel":
return "eprel_id"
default:
return compact
}
}
func parseHTMLSpecs(dst map[string]any, s string) {
s = capSpecInput(s)
matches := htmlLiRe.FindAllStringSubmatch(s, maxSpecPairs)
for _, m := range matches {
if len(m) < 2 {
continue
}
text := strings.TrimSpace(html.UnescapeString(specsHTMLTagRe.ReplaceAllString(m[1], "")))
if text == "" {
continue
}
key, val := splitLabelValue(text)
attrKey := attributeKeyFromLabel(key)
if attrKey != "" && val != "" && !isReservedProductKey(attrKey) && !isInvalidAttributeKey(attrKey) {
dst[attrKey] = SanitizeOutput(val)
}
}
if len(matches) == 0 {
// Fallback: strip tags and parse as CSV-like / bullets
plain := strings.TrimSpace(html.UnescapeString(specsHTMLTagRe.ReplaceAllString(s, "\n")))
parseCSVLikeSpecs(dst, plain)
}
}
func parseXMLTreeSpecs(dst map[string]any, s string) {
type node struct {
XMLName xml.Name
Attrs []xml.Attr `xml:",any,attr"`
Content string `xml:",chardata"`
Nodes []node `xml:",any"`
}
// Wrap fragment so arbitrary roots parse.
wrapped := "" + s + ""
var root node
if err := xml.Unmarshal([]byte(wrapped), &root); err != nil {
return
}
var walk func(n node, path string)
walk = func(n node, path string) {
name := n.XMLName.Local
if name == "" {
name = path
}
text := strings.TrimSpace(n.Content)
if len(n.Nodes) == 0 && text != "" && name != "" && name != "specs" {
dst[SanitizeOutput(name)] = SanitizeOutput(text)
return
}
// Common pattern: Red or -
attrName := ""
for _, a := range n.Attrs {
an := strings.ToLower(a.Name.Local)
if an == "name" || an == "key" || an == "label" {
attrName = a.Value
}
}
if attrName != "" && text != "" {
dst[SanitizeOutput(attrName)] = SanitizeOutput(text)
}
childName, childVal := "", ""
for _, c := range n.Nodes {
ln := strings.ToLower(c.XMLName.Local)
ct := strings.TrimSpace(c.Content)
if ln == "name" || ln == "key" || ln == "label" {
childName = ct
}
if ln == "value" || ln == "val" {
childVal = ct
}
walk(c, c.XMLName.Local)
}
if childName != "" && childVal != "" {
dst[SanitizeOutput(childName)] = SanitizeOutput(childVal)
}
}
walk(root, "")
}
func parseCSVLikeSpecs(dst map[string]any, s string) {
s = capSpecInput(s)
for _, re := range []*regexp.Regexp{bulletLineRe, csvLikeRe} {
for _, m := range re.FindAllStringSubmatch(s, maxSpecPairs) {
if len(m) < 3 {
continue
}
key := attributeKeyFromLabel(strings.TrimSpace(m[1]))
val := strings.TrimSpace(m[2])
if key != "" && val != "" && !isReservedProductKey(key) && !isInvalidAttributeKey(key) {
dst[key] = SanitizeOutput(val)
}
}
}
}
func splitLabelValue(text string) (string, string) {
for _, sep := range []string{":", " - ", " – ", "="} {
if i := strings.Index(text, sep); i > 0 {
return strings.TrimSpace(text[:i]), strings.TrimSpace(text[i+len(sep):])
}
}
return "", ""
}
func stringifySpecValue(v any) string {
if v == nil {
return ""
}
switch t := v.(type) {
case string:
return strings.TrimSpace(t)
case float64, float32, int, int64, bool:
return strings.TrimSpace(fmt.Sprint(t))
default:
return ""
}
}
// attributeKeyFromLabel turns "Energijski razred" into "energijski-razred",
// and maps known locale/shipping aliases onto standard snake_case keys.
func attributeKeyFromLabel(label string) string {
label = strings.TrimSpace(label)
if label == "" {
return ""
}
var b strings.Builder
b.Grow(len(label))
prevHyphen := false
for _, r := range strings.ToLower(label) {
switch r {
case 'š', 'ś', 'ş':
r = 's'
case 'č', 'ć', 'ç':
r = 'c'
case 'ž', 'ź', 'ż':
r = 'z'
case 'đ':
r = 'd'
case 'ä', 'á', 'à', 'â', 'ã', 'å':
r = 'a'
case 'ë', 'é', 'è', 'ê':
r = 'e'
case 'ï', 'í', 'ì', 'î':
r = 'i'
case 'ö', 'ó', 'ò', 'ô', 'õ':
r = 'o'
case 'ü', 'ú', 'ù', 'û':
r = 'u'
case 'ý', 'ÿ':
r = 'y'
}
switch {
case r >= 'a' && r <= 'z', r >= '0' && r <= '9':
b.WriteRune(r)
prevHyphen = false
case r == ' ' || r == '_' || r == '/' || r == '\\' || r == '.' || r == ':' || r == '-':
if !prevHyphen && b.Len() > 0 {
b.WriteByte('-')
prevHyphen = true
}
}
}
slug := strings.Trim(b.String(), "-")
compact := strings.ReplaceAll(slug, "-", "")
switch compact {
case "visina", "height", "netheight":
return "net_height"
case "sirina", "width", "netwidth":
return "net_width"
case "globina", "depth", "netdepth":
return "net_depth"
case "netmass", "mass", "weight", "teza":
return "net_mass"
case "productmodel", "model":
return "product_model"
case "eprelid", "eprel":
return "eprel_id"
case "energyclass", "energijskirazred":
return "energy_class"
}
if slug == "" || len(compact) < 2 {
return ""
}
hasLetter := false
for _, r := range compact {
if r >= 'a' && r <= 'z' {
hasLetter = true
break
}
}
if !hasLetter {
return ""
}
switch compact {
case "true", "false", "yes", "no", "null", "undefined", "none", "n", "y":
return ""
}
return slug
}