Initial commit of Descrybe v2 without local scratch artifacts.
Drop one-shot tmp/axe scripts and agent i18n scratch so the Gitea tree is deployable.
This commit is contained in:
@@ -0,0 +1,313 @@
|
||||
package processing
|
||||
|
||||
import (
|
||||
"encoding/xml"
|
||||
"fmt"
|
||||
"html"
|
||||
"regexp"
|
||||
"strings"
|
||||
)
|
||||
|
||||
var (
|
||||
htmlLiRe = regexp.MustCompile(`(?is)<li[^>]*>(.*?)(?:</li\s*>|</\s*>)`)
|
||||
specsHTMLTagRe = regexp.MustCompile(`(?is)<[^>]+>`)
|
||||
csvLikeRe = regexp.MustCompile(`(?m)^\s*([^:=\n\r]{1,120})\s*[:=]\s*(.+?)\s*$`)
|
||||
bulletLineRe = regexp.MustCompile(`(?m)^\s*[-•*]\s*([^:=\n\r]{1,120})\s*[:=]\s*(.+?)\s*$`)
|
||||
)
|
||||
|
||||
// Caps for locale/spec parsing — unbounded FindAll on huge CDATA can OOM the worker.
|
||||
const (
|
||||
maxSpecInputBytes = 200_000
|
||||
maxSpecPairs = 500
|
||||
)
|
||||
|
||||
func capSpecInput(s string) string {
|
||||
if len(s) <= maxSpecInputBytes {
|
||||
return s
|
||||
}
|
||||
return s[:maxSpecInputBytes]
|
||||
}
|
||||
|
||||
// ParseSpecifications extracts attribute key/values from tree XML, CDATA HTML, CSV-like, or nested maps.
|
||||
func ParseSpecifications(v any) map[string]any {
|
||||
attrs := map[string]any{}
|
||||
parseSpecsInto(attrs, v)
|
||||
return attrs
|
||||
}
|
||||
|
||||
func parseSpecsInto(dst map[string]any, v any) {
|
||||
if v == nil {
|
||||
return
|
||||
}
|
||||
switch t := v.(type) {
|
||||
case map[string]any:
|
||||
// Already structured attributes / grouped specs
|
||||
if looksLikeAttrMap(t) {
|
||||
for k, val := range t {
|
||||
if s := stringifySpecValue(val); s != "" {
|
||||
dst[SanitizeOutput(k)] = s
|
||||
} else if nested, ok := val.(map[string]any); ok {
|
||||
parseSpecsInto(dst, nested)
|
||||
}
|
||||
}
|
||||
return
|
||||
}
|
||||
for k, val := range t {
|
||||
lk := strings.ToLower(k)
|
||||
if strings.Contains(lk, "spec") {
|
||||
parseSpecsInto(dst, val)
|
||||
continue
|
||||
}
|
||||
if s := stringifySpecValue(val); s != "" && !isReservedProductKey(lk) {
|
||||
dst[SanitizeOutput(k)] = s
|
||||
}
|
||||
}
|
||||
case []any:
|
||||
for _, item := range t {
|
||||
parseSpecsInto(dst, item)
|
||||
}
|
||||
case string:
|
||||
s := strings.TrimSpace(t)
|
||||
if s == "" {
|
||||
return
|
||||
}
|
||||
if strings.Contains(s, "<") {
|
||||
parseHTMLSpecs(dst, s)
|
||||
if len(dst) > 0 {
|
||||
return
|
||||
}
|
||||
parseXMLTreeSpecs(dst, s)
|
||||
if len(dst) > 0 {
|
||||
return
|
||||
}
|
||||
}
|
||||
parseCSVLikeSpecs(dst, s)
|
||||
default:
|
||||
s := strings.TrimSpace(fmt.Sprint(t))
|
||||
if s != "" && s != "<nil>" {
|
||||
parseSpecsInto(dst, s)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func looksLikeAttrMap(m map[string]any) bool {
|
||||
if len(m) == 0 {
|
||||
return false
|
||||
}
|
||||
scalar := 0
|
||||
for _, v := range m {
|
||||
switch v.(type) {
|
||||
case string, float64, float32, int, int64, bool:
|
||||
scalar++
|
||||
}
|
||||
}
|
||||
return scalar >= len(m)/2
|
||||
}
|
||||
|
||||
func isReservedProductKey(k string) bool {
|
||||
switch k {
|
||||
case "name", "title", "description", "gtin", "ean", "brand", "category",
|
||||
"price", "image", "stock", "eprel_id", "specifications", "raw", "mapped":
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
func parseHTMLSpecs(dst map[string]any, s string) {
|
||||
s = capSpecInput(s)
|
||||
matches := htmlLiRe.FindAllStringSubmatch(s, maxSpecPairs)
|
||||
for _, m := range matches {
|
||||
if len(m) < 2 {
|
||||
continue
|
||||
}
|
||||
text := strings.TrimSpace(html.UnescapeString(specsHTMLTagRe.ReplaceAllString(m[1], "")))
|
||||
if text == "" {
|
||||
continue
|
||||
}
|
||||
key, val := splitLabelValue(text)
|
||||
if key != "" && val != "" {
|
||||
dst[attributeKeyFromLabel(key)] = SanitizeOutput(val)
|
||||
}
|
||||
}
|
||||
if len(matches) == 0 {
|
||||
// Fallback: strip tags and parse as CSV-like / bullets
|
||||
plain := strings.TrimSpace(html.UnescapeString(specsHTMLTagRe.ReplaceAllString(s, "\n")))
|
||||
parseCSVLikeSpecs(dst, plain)
|
||||
}
|
||||
}
|
||||
|
||||
func parseXMLTreeSpecs(dst map[string]any, s string) {
|
||||
type node struct {
|
||||
XMLName xml.Name
|
||||
Attrs []xml.Attr `xml:",any,attr"`
|
||||
Content string `xml:",chardata"`
|
||||
Nodes []node `xml:",any"`
|
||||
}
|
||||
// Wrap fragment so arbitrary roots parse.
|
||||
wrapped := "<specs>" + s + "</specs>"
|
||||
var root node
|
||||
if err := xml.Unmarshal([]byte(wrapped), &root); err != nil {
|
||||
return
|
||||
}
|
||||
var walk func(n node, path string)
|
||||
walk = func(n node, path string) {
|
||||
name := n.XMLName.Local
|
||||
if name == "" {
|
||||
name = path
|
||||
}
|
||||
text := strings.TrimSpace(n.Content)
|
||||
if len(n.Nodes) == 0 && text != "" && name != "" && name != "specs" {
|
||||
dst[SanitizeOutput(name)] = SanitizeOutput(text)
|
||||
return
|
||||
}
|
||||
// Common pattern: <spec name="Color">Red</spec> or <item><name/><value/>
|
||||
attrName := ""
|
||||
for _, a := range n.Attrs {
|
||||
an := strings.ToLower(a.Name.Local)
|
||||
if an == "name" || an == "key" || an == "label" {
|
||||
attrName = a.Value
|
||||
}
|
||||
}
|
||||
if attrName != "" && text != "" {
|
||||
dst[SanitizeOutput(attrName)] = SanitizeOutput(text)
|
||||
}
|
||||
childName, childVal := "", ""
|
||||
for _, c := range n.Nodes {
|
||||
ln := strings.ToLower(c.XMLName.Local)
|
||||
ct := strings.TrimSpace(c.Content)
|
||||
if ln == "name" || ln == "key" || ln == "label" {
|
||||
childName = ct
|
||||
}
|
||||
if ln == "value" || ln == "val" {
|
||||
childVal = ct
|
||||
}
|
||||
walk(c, c.XMLName.Local)
|
||||
}
|
||||
if childName != "" && childVal != "" {
|
||||
dst[SanitizeOutput(childName)] = SanitizeOutput(childVal)
|
||||
}
|
||||
}
|
||||
walk(root, "")
|
||||
}
|
||||
|
||||
func parseCSVLikeSpecs(dst map[string]any, s string) {
|
||||
s = capSpecInput(s)
|
||||
for _, re := range []*regexp.Regexp{bulletLineRe, csvLikeRe} {
|
||||
for _, m := range re.FindAllStringSubmatch(s, maxSpecPairs) {
|
||||
if len(m) < 3 {
|
||||
continue
|
||||
}
|
||||
key := strings.TrimSpace(m[1])
|
||||
val := strings.TrimSpace(m[2])
|
||||
if key != "" && val != "" {
|
||||
dst[attributeKeyFromLabel(key)] = SanitizeOutput(val)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func splitLabelValue(text string) (string, string) {
|
||||
for _, sep := range []string{":", " - ", " – ", "="} {
|
||||
if i := strings.Index(text, sep); i > 0 {
|
||||
return strings.TrimSpace(text[:i]), strings.TrimSpace(text[i+len(sep):])
|
||||
}
|
||||
}
|
||||
return "", ""
|
||||
}
|
||||
|
||||
func stringifySpecValue(v any) string {
|
||||
if v == nil {
|
||||
return ""
|
||||
}
|
||||
switch t := v.(type) {
|
||||
case string:
|
||||
return strings.TrimSpace(t)
|
||||
case float64, float32, int, int64, bool:
|
||||
return strings.TrimSpace(fmt.Sprint(t))
|
||||
default:
|
||||
return ""
|
||||
}
|
||||
}
|
||||
|
||||
// attributeKeyFromLabel turns "Energijski razred" into "energijski-razred",
|
||||
// and maps known locale/shipping aliases onto standard snake_case keys.
|
||||
func attributeKeyFromLabel(label string) string {
|
||||
label = strings.TrimSpace(label)
|
||||
if label == "" {
|
||||
return ""
|
||||
}
|
||||
var b strings.Builder
|
||||
b.Grow(len(label))
|
||||
prevHyphen := false
|
||||
for _, r := range strings.ToLower(label) {
|
||||
switch r {
|
||||
case 'š', 'ś', 'ş':
|
||||
r = 's'
|
||||
case 'č', 'ć', 'ç':
|
||||
r = 'c'
|
||||
case 'ž', 'ź', 'ż':
|
||||
r = 'z'
|
||||
case 'đ':
|
||||
r = 'd'
|
||||
case 'ä', 'á', 'à', 'â', 'ã', 'å':
|
||||
r = 'a'
|
||||
case 'ë', 'é', 'è', 'ê':
|
||||
r = 'e'
|
||||
case 'ï', 'í', 'ì', 'î':
|
||||
r = 'i'
|
||||
case 'ö', 'ó', 'ò', 'ô', 'õ':
|
||||
r = 'o'
|
||||
case 'ü', 'ú', 'ù', 'û':
|
||||
r = 'u'
|
||||
case 'ý', 'ÿ':
|
||||
r = 'y'
|
||||
}
|
||||
switch {
|
||||
case r >= 'a' && r <= 'z', r >= '0' && r <= '9':
|
||||
b.WriteRune(r)
|
||||
prevHyphen = false
|
||||
case r == ' ' || r == '_' || r == '/' || r == '\\' || r == '.' || r == ':' || r == '-':
|
||||
if !prevHyphen && b.Len() > 0 {
|
||||
b.WriteByte('-')
|
||||
prevHyphen = true
|
||||
}
|
||||
}
|
||||
}
|
||||
slug := strings.Trim(b.String(), "-")
|
||||
compact := strings.ReplaceAll(slug, "-", "")
|
||||
switch compact {
|
||||
case "visina", "height", "netheight":
|
||||
return "net_height"
|
||||
case "sirina", "width", "netwidth":
|
||||
return "net_width"
|
||||
case "globina", "depth", "netdepth":
|
||||
return "net_depth"
|
||||
case "netmass", "mass", "weight", "teza":
|
||||
return "net_mass"
|
||||
case "productmodel", "model":
|
||||
return "product_model"
|
||||
case "eprelid", "eprel":
|
||||
return "eprel_id"
|
||||
case "energyclass", "energijskirazred":
|
||||
return "energy_class"
|
||||
}
|
||||
if slug == "" || len(compact) < 2 {
|
||||
return ""
|
||||
}
|
||||
hasLetter := false
|
||||
for _, r := range compact {
|
||||
if r >= 'a' && r <= 'z' {
|
||||
hasLetter = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if !hasLetter {
|
||||
return ""
|
||||
}
|
||||
switch compact {
|
||||
case "true", "false", "yes", "no", "null", "undefined", "none", "n", "y":
|
||||
return ""
|
||||
}
|
||||
return slug
|
||||
}
|
||||
Reference in New Issue
Block a user