462 lines
12 KiB
Go
462 lines
12 KiB
Go
package processing
|
||
|
||
import (
|
||
"encoding/xml"
|
||
"fmt"
|
||
"html"
|
||
"regexp"
|
||
"strings"
|
||
)
|
||
|
||
var (
|
||
htmlLiRe = regexp.MustCompile(`(?is)<li[^>]*>(.*?)(?:</li\s*>|</\s*>)`)
|
||
specsHTMLTagRe = regexp.MustCompile(`(?is)<[^>]+>`)
|
||
csvLikeRe = regexp.MustCompile(`(?m)^\s*([^:=\n\r]{1,120})\s*[:=]\s*(.+?)\s*$`)
|
||
bulletLineRe = regexp.MustCompile(`(?m)^\s*[-•*]\s*([^:=\n\r]{1,120})\s*[:=]\s*(.+?)\s*$`)
|
||
)
|
||
|
||
// Caps for locale/spec parsing — unbounded FindAll on huge CDATA can OOM the worker.
|
||
const (
|
||
maxSpecInputBytes = 200_000
|
||
maxSpecPairs = 500
|
||
)
|
||
|
||
func capSpecInput(s string) string {
|
||
if len(s) <= maxSpecInputBytes {
|
||
return s
|
||
}
|
||
return s[:maxSpecInputBytes]
|
||
}
|
||
|
||
// ParseSpecifications extracts attribute key/values from tree XML, CDATA HTML, CSV-like, or nested maps.
|
||
func ParseSpecifications(v any) map[string]any {
|
||
attrs := map[string]any{}
|
||
parseSpecsInto(attrs, v)
|
||
return attrs
|
||
}
|
||
|
||
func parseSpecsInto(dst map[string]any, v any) {
|
||
if v == nil {
|
||
return
|
||
}
|
||
switch t := v.(type) {
|
||
case map[string]any:
|
||
// Already structured attributes / grouped specs
|
||
if looksLikeAttrMap(t) {
|
||
for k, val := range t {
|
||
lk := strings.ToLower(strings.TrimSpace(k))
|
||
if isReservedProductKey(lk) || isInvalidAttributeKey(k) {
|
||
continue
|
||
}
|
||
if s := stringifySpecValue(val); s != "" {
|
||
key := SanitizeOutput(k)
|
||
if key == "" || isInvalidAttributeKey(key) {
|
||
continue
|
||
}
|
||
dst[key] = s
|
||
} else if nested, ok := val.(map[string]any); ok {
|
||
parseSpecsInto(dst, nested)
|
||
}
|
||
}
|
||
return
|
||
}
|
||
for k, val := range t {
|
||
lk := strings.ToLower(k)
|
||
if strings.Contains(lk, "spec") {
|
||
parseSpecsInto(dst, val)
|
||
continue
|
||
}
|
||
if s := stringifySpecValue(val); s != "" && !isReservedProductKey(lk) && !isInvalidAttributeKey(k) {
|
||
dst[SanitizeOutput(k)] = s
|
||
}
|
||
}
|
||
case []any:
|
||
for _, item := range t {
|
||
parseSpecsInto(dst, item)
|
||
}
|
||
case string:
|
||
s := strings.TrimSpace(t)
|
||
if s == "" {
|
||
return
|
||
}
|
||
if strings.Contains(s, "<") {
|
||
parseHTMLSpecs(dst, s)
|
||
if len(dst) > 0 {
|
||
return
|
||
}
|
||
parseXMLTreeSpecs(dst, s)
|
||
if len(dst) > 0 {
|
||
return
|
||
}
|
||
}
|
||
parseCSVLikeSpecs(dst, s)
|
||
default:
|
||
s := strings.TrimSpace(fmt.Sprint(t))
|
||
if s != "" && s != "<nil>" {
|
||
parseSpecsInto(dst, s)
|
||
}
|
||
}
|
||
}
|
||
|
||
func looksLikeAttrMap(m map[string]any) bool {
|
||
if len(m) == 0 {
|
||
return false
|
||
}
|
||
scalar := 0
|
||
for _, v := range m {
|
||
switch v.(type) {
|
||
case string, float64, float32, int, int64, bool:
|
||
scalar++
|
||
}
|
||
}
|
||
return scalar >= len(m)/2
|
||
}
|
||
|
||
func isReservedProductKey(k string) bool {
|
||
switch strings.ToLower(strings.TrimSpace(k)) {
|
||
case "name", "title", "description", "gtin", "ean", "category", "category_unique_id",
|
||
"price", "purchaseprice", "purchase_price", "sellingprice", "selling_price",
|
||
"image", "main_image", "mainimage", "moreimages", "more_images", "images",
|
||
"image_url", "imageurl", "image_link", "imagelink", "additional_image_urls",
|
||
"additional_image_link", "videourl", "video_url",
|
||
"stock", "stockstatus", "stock_status", "availability",
|
||
"id", "sku", "officiallink", "official_link", "service",
|
||
"specifications", "specs", "specification",
|
||
"raw", "mapped", "search":
|
||
return true
|
||
default:
|
||
return false
|
||
}
|
||
}
|
||
|
||
// isInvalidAttributeKey rejects garbage keys from bad feed specs (e.g. ":").
|
||
func isInvalidAttributeKey(k string) bool {
|
||
k = strings.TrimSpace(k)
|
||
if k == "" || k == ":" || k == "=" || k == "-" || k == "_" {
|
||
return true
|
||
}
|
||
// Must contain at least one letter after sanitize.
|
||
hasLetter := false
|
||
for _, r := range strings.ToLower(k) {
|
||
if r >= 'a' && r <= 'z' {
|
||
hasLetter = true
|
||
break
|
||
}
|
||
}
|
||
return !hasLetter
|
||
}
|
||
|
||
// coreCharacteristicAttrKeys are always allowed on V1 process items even when
|
||
// the company has no matching attributes row (common dims/brand/model).
|
||
var coreCharacteristicAttrKeys = map[string]struct{}{
|
||
"brand": {}, "product_model": {}, "warranty": {},
|
||
"width": {}, "height": {}, "depth": {}, "weight": {},
|
||
"energy_class": {}, "color": {}, "material": {},
|
||
}
|
||
|
||
// SanitizeProductAttributes keeps characteristic attrs for API/storage and drops
|
||
// core product fields that belong on the V1 item root (title, description, images, …).
|
||
func SanitizeProductAttributes(attrs map[string]any) map[string]any {
|
||
return sanitizeProductAttributes(attrs, false)
|
||
}
|
||
|
||
// SanitizeV1ProcessAttributes is the stricter poll projection: also drops eprel_*
|
||
// keys (those are exposed on item.eprel). Without an allowlist, feed junk specs
|
||
// (zavora, vzmetenje, …) can remain — prefer SanitizeV1ProcessAttributesAllowed.
|
||
func SanitizeV1ProcessAttributes(attrs map[string]any) map[string]any {
|
||
return sanitizeProductAttributes(attrs, true)
|
||
}
|
||
|
||
// SanitizeV1ProcessAttributesAllowed sanitizes then keeps only core characteristic
|
||
// keys plus company attribute_key values (canonicalized).
|
||
func SanitizeV1ProcessAttributesAllowed(attrs map[string]any, allowed map[string]struct{}) map[string]any {
|
||
return FilterAttributesByAllowed(sanitizeProductAttributes(attrs, true), allowed)
|
||
}
|
||
|
||
// FilterAttributesByAllowed keeps coreCharacteristicAttrKeys plus keys present in
|
||
// allowed (after canonicalizeAttrKey). When allowed is nil, returns attrs unchanged.
|
||
func FilterAttributesByAllowed(attrs map[string]any, allowed map[string]struct{}) map[string]any {
|
||
if len(attrs) == 0 {
|
||
return map[string]any{}
|
||
}
|
||
if allowed == nil {
|
||
return attrs
|
||
}
|
||
out := make(map[string]any, len(attrs))
|
||
for k, v := range attrs {
|
||
canon := canonicalizeAttrKey(k)
|
||
if canon == "" {
|
||
continue
|
||
}
|
||
if _, ok := coreCharacteristicAttrKeys[canon]; ok {
|
||
out[canon] = v
|
||
continue
|
||
}
|
||
if _, ok := allowed[canon]; ok {
|
||
out[canon] = v
|
||
continue
|
||
}
|
||
if _, ok := allowed[strings.ToLower(strings.TrimSpace(k))]; ok {
|
||
out[canon] = v
|
||
}
|
||
}
|
||
return out
|
||
}
|
||
|
||
func sanitizeProductAttributes(attrs map[string]any, dropEPREL bool) map[string]any {
|
||
if len(attrs) == 0 {
|
||
return map[string]any{}
|
||
}
|
||
out := make(map[string]any, len(attrs))
|
||
for k, v := range attrs {
|
||
key := strings.TrimSpace(SanitizeOutput(k))
|
||
if key == "" || isReservedProductKey(key) || isInvalidAttributeKey(key) {
|
||
continue
|
||
}
|
||
if dropEPREL && strings.HasPrefix(strings.ToLower(key), "eprel") {
|
||
continue
|
||
}
|
||
s := stringifySpecValue(v)
|
||
if s == "" || s == "<nil>" {
|
||
continue
|
||
}
|
||
// Prefer canonical dimension/model keys when aliases collide.
|
||
canon := canonicalizeAttrKey(key)
|
||
if canon == "" || isReservedProductKey(canon) || isInvalidAttributeKey(canon) {
|
||
continue
|
||
}
|
||
if dropEPREL && strings.HasPrefix(strings.ToLower(canon), "eprel") {
|
||
continue
|
||
}
|
||
if isDimensionKey(canon) && isZeroishString(s) {
|
||
continue
|
||
}
|
||
if _, exists := out[canon]; exists && (canon != key) {
|
||
continue
|
||
}
|
||
out[canon] = SanitizeOutput(s)
|
||
}
|
||
return out
|
||
}
|
||
|
||
func canonicalizeAttrKey(k string) string {
|
||
compact := strings.ToLower(strings.TrimSpace(k))
|
||
compact = strings.ReplaceAll(compact, "-", "_")
|
||
compact = strings.ReplaceAll(compact, " ", "_")
|
||
noUnderscore := strings.ReplaceAll(compact, "_", "")
|
||
switch noUnderscore {
|
||
case "netwidth", "width", "sirina":
|
||
return "width"
|
||
case "netheight", "height", "visina":
|
||
return "height"
|
||
case "netdepth", "depth", "globina":
|
||
return "depth"
|
||
case "netmass", "weight", "mass", "teza":
|
||
return "weight"
|
||
case "productmodel", "model":
|
||
return "product_model"
|
||
case "energijskirazred", "energyclass":
|
||
return "energy_class"
|
||
default:
|
||
return compact
|
||
}
|
||
}
|
||
|
||
func parseHTMLSpecs(dst map[string]any, s string) {
|
||
s = capSpecInput(s)
|
||
matches := htmlLiRe.FindAllStringSubmatch(s, maxSpecPairs)
|
||
for _, m := range matches {
|
||
if len(m) < 2 {
|
||
continue
|
||
}
|
||
text := strings.TrimSpace(html.UnescapeString(specsHTMLTagRe.ReplaceAllString(m[1], "")))
|
||
if text == "" {
|
||
continue
|
||
}
|
||
key, val := splitLabelValue(text)
|
||
attrKey := attributeKeyFromLabel(key)
|
||
if attrKey != "" && val != "" && !isReservedProductKey(attrKey) && !isInvalidAttributeKey(attrKey) {
|
||
dst[attrKey] = SanitizeOutput(val)
|
||
}
|
||
}
|
||
if len(matches) == 0 {
|
||
// Fallback: strip tags and parse as CSV-like / bullets
|
||
plain := strings.TrimSpace(html.UnescapeString(specsHTMLTagRe.ReplaceAllString(s, "\n")))
|
||
parseCSVLikeSpecs(dst, plain)
|
||
}
|
||
}
|
||
|
||
func parseXMLTreeSpecs(dst map[string]any, s string) {
|
||
type node struct {
|
||
XMLName xml.Name
|
||
Attrs []xml.Attr `xml:",any,attr"`
|
||
Content string `xml:",chardata"`
|
||
Nodes []node `xml:",any"`
|
||
}
|
||
// Wrap fragment so arbitrary roots parse.
|
||
wrapped := "<specs>" + s + "</specs>"
|
||
var root node
|
||
if err := xml.Unmarshal([]byte(wrapped), &root); err != nil {
|
||
return
|
||
}
|
||
var walk func(n node, path string)
|
||
walk = func(n node, path string) {
|
||
name := n.XMLName.Local
|
||
if name == "" {
|
||
name = path
|
||
}
|
||
text := strings.TrimSpace(n.Content)
|
||
if len(n.Nodes) == 0 && text != "" && name != "" && name != "specs" {
|
||
dst[SanitizeOutput(name)] = SanitizeOutput(text)
|
||
return
|
||
}
|
||
// Common pattern: <spec name="Color">Red</spec> or <item><name/><value/>
|
||
attrName := ""
|
||
for _, a := range n.Attrs {
|
||
an := strings.ToLower(a.Name.Local)
|
||
if an == "name" || an == "key" || an == "label" {
|
||
attrName = a.Value
|
||
}
|
||
}
|
||
if attrName != "" && text != "" {
|
||
dst[SanitizeOutput(attrName)] = SanitizeOutput(text)
|
||
}
|
||
childName, childVal := "", ""
|
||
for _, c := range n.Nodes {
|
||
ln := strings.ToLower(c.XMLName.Local)
|
||
ct := strings.TrimSpace(c.Content)
|
||
if ln == "name" || ln == "key" || ln == "label" {
|
||
childName = ct
|
||
}
|
||
if ln == "value" || ln == "val" {
|
||
childVal = ct
|
||
}
|
||
walk(c, c.XMLName.Local)
|
||
}
|
||
if childName != "" && childVal != "" {
|
||
dst[SanitizeOutput(childName)] = SanitizeOutput(childVal)
|
||
}
|
||
}
|
||
walk(root, "")
|
||
}
|
||
|
||
func parseCSVLikeSpecs(dst map[string]any, s string) {
|
||
s = capSpecInput(s)
|
||
for _, re := range []*regexp.Regexp{bulletLineRe, csvLikeRe} {
|
||
for _, m := range re.FindAllStringSubmatch(s, maxSpecPairs) {
|
||
if len(m) < 3 {
|
||
continue
|
||
}
|
||
key := attributeKeyFromLabel(strings.TrimSpace(m[1]))
|
||
val := strings.TrimSpace(m[2])
|
||
if key != "" && val != "" && !isReservedProductKey(key) && !isInvalidAttributeKey(key) {
|
||
dst[key] = SanitizeOutput(val)
|
||
}
|
||
}
|
||
}
|
||
}
|
||
|
||
func splitLabelValue(text string) (string, string) {
|
||
for _, sep := range []string{":", " - ", " – ", "="} {
|
||
if i := strings.Index(text, sep); i > 0 {
|
||
return strings.TrimSpace(text[:i]), strings.TrimSpace(text[i+len(sep):])
|
||
}
|
||
}
|
||
return "", ""
|
||
}
|
||
|
||
func stringifySpecValue(v any) string {
|
||
if v == nil {
|
||
return ""
|
||
}
|
||
switch t := v.(type) {
|
||
case string:
|
||
return strings.TrimSpace(t)
|
||
case float64, float32, int, int64, bool:
|
||
return strings.TrimSpace(fmt.Sprint(t))
|
||
default:
|
||
return ""
|
||
}
|
||
}
|
||
|
||
// attributeKeyFromLabel turns "Energijski razred" into "energijski-razred",
|
||
// and maps known locale/shipping aliases onto standard snake_case keys.
|
||
func attributeKeyFromLabel(label string) string {
|
||
label = strings.TrimSpace(label)
|
||
if label == "" {
|
||
return ""
|
||
}
|
||
var b strings.Builder
|
||
b.Grow(len(label))
|
||
prevHyphen := false
|
||
for _, r := range strings.ToLower(label) {
|
||
switch r {
|
||
case 'š', 'ś', 'ş':
|
||
r = 's'
|
||
case 'č', 'ć', 'ç':
|
||
r = 'c'
|
||
case 'ž', 'ź', 'ż':
|
||
r = 'z'
|
||
case 'đ':
|
||
r = 'd'
|
||
case 'ä', 'á', 'à', 'â', 'ã', 'å':
|
||
r = 'a'
|
||
case 'ë', 'é', 'è', 'ê':
|
||
r = 'e'
|
||
case 'ï', 'í', 'ì', 'î':
|
||
r = 'i'
|
||
case 'ö', 'ó', 'ò', 'ô', 'õ':
|
||
r = 'o'
|
||
case 'ü', 'ú', 'ù', 'û':
|
||
r = 'u'
|
||
case 'ý', 'ÿ':
|
||
r = 'y'
|
||
}
|
||
switch {
|
||
case r >= 'a' && r <= 'z', r >= '0' && r <= '9':
|
||
b.WriteRune(r)
|
||
prevHyphen = false
|
||
case r == ' ' || r == '_' || r == '/' || r == '\\' || r == '.' || r == ':' || r == '-':
|
||
if !prevHyphen && b.Len() > 0 {
|
||
b.WriteByte('-')
|
||
prevHyphen = true
|
||
}
|
||
}
|
||
}
|
||
slug := strings.Trim(b.String(), "-")
|
||
compact := strings.ReplaceAll(slug, "-", "")
|
||
switch compact {
|
||
case "visina", "height", "netheight":
|
||
return "net_height"
|
||
case "sirina", "width", "netwidth":
|
||
return "net_width"
|
||
case "globina", "depth", "netdepth":
|
||
return "net_depth"
|
||
case "netmass", "mass", "weight", "teza":
|
||
return "net_mass"
|
||
case "productmodel", "model":
|
||
return "product_model"
|
||
case "eprelid", "eprel":
|
||
return "eprel_id"
|
||
case "energyclass", "energijskirazred":
|
||
return "energy_class"
|
||
}
|
||
if slug == "" || len(compact) < 2 {
|
||
return ""
|
||
}
|
||
hasLetter := false
|
||
for _, r := range compact {
|
||
if r >= 'a' && r <= 'z' {
|
||
hasLetter = true
|
||
break
|
||
}
|
||
}
|
||
if !hasLetter {
|
||
return ""
|
||
}
|
||
switch compact {
|
||
case "true", "false", "yes", "no", "null", "undefined", "none", "n", "y":
|
||
return ""
|
||
}
|
||
return slug
|
||
} |