package processing import ( "encoding/xml" "fmt" "html" "regexp" "strings" ) var ( htmlLiRe = regexp.MustCompile(`(?is)]*>(.*?)(?:|)`) specsHTMLTagRe = regexp.MustCompile(`(?is)<[^>]+>`) csvLikeRe = regexp.MustCompile(`(?m)^\s*([^:=\n\r]{1,120})\s*[:=]\s*(.+?)\s*$`) bulletLineRe = regexp.MustCompile(`(?m)^\s*[-•*]\s*([^:=\n\r]{1,120})\s*[:=]\s*(.+?)\s*$`) ) // Caps for locale/spec parsing — unbounded FindAll on huge CDATA can OOM the worker. const ( maxSpecInputBytes = 200_000 maxSpecPairs = 500 ) func capSpecInput(s string) string { if len(s) <= maxSpecInputBytes { return s } return s[:maxSpecInputBytes] } // ParseSpecifications extracts attribute key/values from tree XML, CDATA HTML, CSV-like, or nested maps. func ParseSpecifications(v any) map[string]any { attrs := map[string]any{} parseSpecsInto(attrs, v) return attrs } func parseSpecsInto(dst map[string]any, v any) { if v == nil { return } switch t := v.(type) { case map[string]any: // Already structured attributes / grouped specs if looksLikeAttrMap(t) { for k, val := range t { if s := stringifySpecValue(val); s != "" { dst[SanitizeOutput(k)] = s } else if nested, ok := val.(map[string]any); ok { parseSpecsInto(dst, nested) } } return } for k, val := range t { lk := strings.ToLower(k) if strings.Contains(lk, "spec") { parseSpecsInto(dst, val) continue } if s := stringifySpecValue(val); s != "" && !isReservedProductKey(lk) { dst[SanitizeOutput(k)] = s } } case []any: for _, item := range t { parseSpecsInto(dst, item) } case string: s := strings.TrimSpace(t) if s == "" { return } if strings.Contains(s, "<") { parseHTMLSpecs(dst, s) if len(dst) > 0 { return } parseXMLTreeSpecs(dst, s) if len(dst) > 0 { return } } parseCSVLikeSpecs(dst, s) default: s := strings.TrimSpace(fmt.Sprint(t)) if s != "" && s != "" { parseSpecsInto(dst, s) } } } func looksLikeAttrMap(m map[string]any) bool { if len(m) == 0 { return false } scalar := 0 for _, v := range m { switch v.(type) { case string, float64, float32, int, int64, bool: scalar++ } } return scalar >= len(m)/2 } func isReservedProductKey(k string) bool { switch k { case "name", "title", "description", "gtin", "ean", "brand", "category", "price", "image", "stock", "eprel_id", "specifications", "raw", "mapped": return true default: return false } } func parseHTMLSpecs(dst map[string]any, s string) { s = capSpecInput(s) matches := htmlLiRe.FindAllStringSubmatch(s, maxSpecPairs) for _, m := range matches { if len(m) < 2 { continue } text := strings.TrimSpace(html.UnescapeString(specsHTMLTagRe.ReplaceAllString(m[1], ""))) if text == "" { continue } key, val := splitLabelValue(text) if key != "" && val != "" { dst[attributeKeyFromLabel(key)] = SanitizeOutput(val) } } if len(matches) == 0 { // Fallback: strip tags and parse as CSV-like / bullets plain := strings.TrimSpace(html.UnescapeString(specsHTMLTagRe.ReplaceAllString(s, "\n"))) parseCSVLikeSpecs(dst, plain) } } func parseXMLTreeSpecs(dst map[string]any, s string) { type node struct { XMLName xml.Name Attrs []xml.Attr `xml:",any,attr"` Content string `xml:",chardata"` Nodes []node `xml:",any"` } // Wrap fragment so arbitrary roots parse. wrapped := "" + s + "" var root node if err := xml.Unmarshal([]byte(wrapped), &root); err != nil { return } var walk func(n node, path string) walk = func(n node, path string) { name := n.XMLName.Local if name == "" { name = path } text := strings.TrimSpace(n.Content) if len(n.Nodes) == 0 && text != "" && name != "" && name != "specs" { dst[SanitizeOutput(name)] = SanitizeOutput(text) return } // Common pattern: Red or attrName := "" for _, a := range n.Attrs { an := strings.ToLower(a.Name.Local) if an == "name" || an == "key" || an == "label" { attrName = a.Value } } if attrName != "" && text != "" { dst[SanitizeOutput(attrName)] = SanitizeOutput(text) } childName, childVal := "", "" for _, c := range n.Nodes { ln := strings.ToLower(c.XMLName.Local) ct := strings.TrimSpace(c.Content) if ln == "name" || ln == "key" || ln == "label" { childName = ct } if ln == "value" || ln == "val" { childVal = ct } walk(c, c.XMLName.Local) } if childName != "" && childVal != "" { dst[SanitizeOutput(childName)] = SanitizeOutput(childVal) } } walk(root, "") } func parseCSVLikeSpecs(dst map[string]any, s string) { s = capSpecInput(s) for _, re := range []*regexp.Regexp{bulletLineRe, csvLikeRe} { for _, m := range re.FindAllStringSubmatch(s, maxSpecPairs) { if len(m) < 3 { continue } key := strings.TrimSpace(m[1]) val := strings.TrimSpace(m[2]) if key != "" && val != "" { dst[attributeKeyFromLabel(key)] = SanitizeOutput(val) } } } } func splitLabelValue(text string) (string, string) { for _, sep := range []string{":", " - ", " – ", "="} { if i := strings.Index(text, sep); i > 0 { return strings.TrimSpace(text[:i]), strings.TrimSpace(text[i+len(sep):]) } } return "", "" } func stringifySpecValue(v any) string { if v == nil { return "" } switch t := v.(type) { case string: return strings.TrimSpace(t) case float64, float32, int, int64, bool: return strings.TrimSpace(fmt.Sprint(t)) default: return "" } } // attributeKeyFromLabel turns "Energijski razred" into "energijski-razred", // and maps known locale/shipping aliases onto standard snake_case keys. func attributeKeyFromLabel(label string) string { label = strings.TrimSpace(label) if label == "" { return "" } var b strings.Builder b.Grow(len(label)) prevHyphen := false for _, r := range strings.ToLower(label) { switch r { case 'š', 'ś', 'ş': r = 's' case 'č', 'ć', 'ç': r = 'c' case 'ž', 'ź', 'ż': r = 'z' case 'đ': r = 'd' case 'ä', 'á', 'à', 'â', 'ã', 'å': r = 'a' case 'ë', 'é', 'è', 'ê': r = 'e' case 'ï', 'í', 'ì', 'î': r = 'i' case 'ö', 'ó', 'ò', 'ô', 'õ': r = 'o' case 'ü', 'ú', 'ù', 'û': r = 'u' case 'ý', 'ÿ': r = 'y' } switch { case r >= 'a' && r <= 'z', r >= '0' && r <= '9': b.WriteRune(r) prevHyphen = false case r == ' ' || r == '_' || r == '/' || r == '\\' || r == '.' || r == ':' || r == '-': if !prevHyphen && b.Len() > 0 { b.WriteByte('-') prevHyphen = true } } } slug := strings.Trim(b.String(), "-") compact := strings.ReplaceAll(slug, "-", "") switch compact { case "visina", "height", "netheight": return "net_height" case "sirina", "width", "netwidth": return "net_width" case "globina", "depth", "netdepth": return "net_depth" case "netmass", "mass", "weight", "teza": return "net_mass" case "productmodel", "model": return "product_model" case "eprelid", "eprel": return "eprel_id" case "energyclass", "energijskirazred": return "energy_class" } if slug == "" || len(compact) < 2 { return "" } hasLetter := false for _, r := range compact { if r >= 'a' && r <= 'z' { hasLetter = true break } } if !hasLetter { return "" } switch compact { case "true", "false", "yes", "no", "null", "undefined", "none", "n", "y": return "" } return slug }