Initial commit of Descrybe v2 without local scratch artifacts.
Drop one-shot tmp/axe scripts and agent i18n scratch so the Gitea tree is deployable.
This commit is contained in:
@@ -0,0 +1,539 @@
|
||||
package feeds
|
||||
|
||||
import (
|
||||
"html"
|
||||
"regexp"
|
||||
"strings"
|
||||
"unicode"
|
||||
)
|
||||
|
||||
const (
|
||||
maxSpecPairs = 200
|
||||
maxSpecRawBytes = 64 << 10 // 64 KiB per specifications blob
|
||||
maxSpecLabelRunes = 120
|
||||
maxSpecValueRunes = 2000
|
||||
)
|
||||
|
||||
var (
|
||||
// Accept </li> and broken </> closers seen in A1 feed CDATA.
|
||||
reHTMLLi = regexp.MustCompile(`(?is)<li\b[^>]*>(.*?)(?:</li\s*>|</\s*>)`)
|
||||
reHTMLTag = regexp.MustCompile(`(?is)<[^>]+>`)
|
||||
reMultiSpace = regexp.MustCompile(`\s+`)
|
||||
)
|
||||
|
||||
// SpecPair is one label/value extracted from a specifications blob.
|
||||
type SpecPair struct {
|
||||
Label string
|
||||
Value string
|
||||
}
|
||||
|
||||
// ParseSpecifications accepts nested-expanded text, CDATA HTML lists, or flat
|
||||
// CSV-like strings. Empty / missing / blank HTML returns nil (not an error).
|
||||
func ParseSpecifications(raw string) []SpecPair {
|
||||
raw = strings.TrimSpace(raw)
|
||||
if raw == "" || len(raw) > maxSpecRawBytes {
|
||||
return nil
|
||||
}
|
||||
if isEmptySpecBlob(raw) {
|
||||
return nil
|
||||
}
|
||||
|
||||
var pairs []SpecPair
|
||||
switch {
|
||||
case looksLikeHTMLList(raw):
|
||||
pairs = parseHTMLSpecList(raw)
|
||||
case looksLikeFlatSpecs(raw):
|
||||
pairs = parseFlatSpecs(raw)
|
||||
default:
|
||||
// Single "Label: value" line still counts as flat.
|
||||
if p, ok := splitLabelValue(raw); ok {
|
||||
pairs = []SpecPair{p}
|
||||
}
|
||||
}
|
||||
return clampSpecPairs(pairs)
|
||||
}
|
||||
|
||||
// expandSpecificationFields mutates row: for specification-like keys whose
|
||||
// value is HTML/flat text, add nested paths key/Label → value. Nested XML
|
||||
// children are already present as key/child from the XML walker.
|
||||
func expandSpecificationFields(row feedRow) {
|
||||
if len(row) == 0 {
|
||||
return
|
||||
}
|
||||
keys := make([]string, 0, 8)
|
||||
for k, v := range row {
|
||||
if !isSpecFieldKey(k) {
|
||||
continue
|
||||
}
|
||||
if strings.TrimSpace(v) == "" {
|
||||
continue
|
||||
}
|
||||
// Already has nested children — leave tree as-is; still parse text if useful.
|
||||
if hasPrefixedChildren(row, k) && !looksLikeHTMLList(v) && !looksLikeFlatSpecs(v) {
|
||||
continue
|
||||
}
|
||||
keys = append(keys, k)
|
||||
}
|
||||
for _, k := range keys {
|
||||
pairs := ParseSpecifications(row[k])
|
||||
for _, p := range pairs {
|
||||
seg := sanitizePathSegment(p.Label)
|
||||
if seg == "" {
|
||||
continue
|
||||
}
|
||||
path := k + "/" + seg
|
||||
if prev, ok := row[path]; ok && strings.TrimSpace(prev) != "" {
|
||||
continue
|
||||
}
|
||||
row[path] = p.Value
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func isSpecFieldKey(key string) bool {
|
||||
leaf := strings.ToLower(leafName(key))
|
||||
leaf = strings.TrimPrefix(leaf, "@")
|
||||
switch leaf {
|
||||
case "specifications", "specification", "specs", "spec", "features", "feature", "attributes_raw":
|
||||
return true
|
||||
}
|
||||
return strings.Contains(leaf, "specification")
|
||||
}
|
||||
|
||||
func hasPrefixedChildren(row feedRow, prefix string) bool {
|
||||
prefix = strings.TrimSuffix(prefix, "/") + "/"
|
||||
for k := range row {
|
||||
if strings.HasPrefix(k, prefix) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func collectPrefixed(row feedRow, prefix string) map[string]string {
|
||||
prefix = strings.TrimSuffix(strings.TrimSpace(prefix), "/")
|
||||
if prefix == "" {
|
||||
return nil
|
||||
}
|
||||
p := prefix + "/"
|
||||
out := map[string]string{}
|
||||
for k, v := range row {
|
||||
if !strings.HasPrefix(k, p) {
|
||||
continue
|
||||
}
|
||||
rest := k[len(p):]
|
||||
if rest == "" || strings.Contains(rest, "/") {
|
||||
continue
|
||||
}
|
||||
v = strings.TrimSpace(v)
|
||||
if v == "" {
|
||||
continue
|
||||
}
|
||||
out[rest] = v
|
||||
}
|
||||
if len(out) == 0 {
|
||||
return nil
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func isEmptySpecBlob(raw string) bool {
|
||||
stripped := strings.TrimSpace(reHTMLTag.ReplaceAllString(raw, " "))
|
||||
stripped = html.UnescapeString(stripped)
|
||||
stripped = strings.TrimSpace(reMultiSpace.ReplaceAllString(stripped, " "))
|
||||
return stripped == ""
|
||||
}
|
||||
|
||||
func looksLikeHTMLList(raw string) bool {
|
||||
lower := strings.ToLower(raw)
|
||||
return strings.Contains(lower, "<li") || (strings.Contains(lower, "<ul") && strings.Contains(lower, "</ul>"))
|
||||
}
|
||||
|
||||
func looksLikeFlatSpecs(raw string) bool {
|
||||
if looksLikeHTMLList(raw) {
|
||||
return false
|
||||
}
|
||||
// Multiple label:value pairs separated by ; | newline or comma between pairs.
|
||||
if strings.Count(raw, ":") >= 2 {
|
||||
return true
|
||||
}
|
||||
if strings.Count(raw, "=") >= 2 && (strings.Contains(raw, ";") || strings.Contains(raw, "|") || strings.Contains(raw, "\n")) {
|
||||
return true
|
||||
}
|
||||
if strings.Contains(raw, ";") && strings.Contains(raw, ":") {
|
||||
return true
|
||||
}
|
||||
if strings.Contains(raw, "|") && strings.Contains(raw, ":") {
|
||||
return true
|
||||
}
|
||||
if strings.Contains(raw, "\n") && strings.Contains(raw, ":") {
|
||||
return true
|
||||
}
|
||||
// Quoted CSV-ish pairs: "Brand","Acme"; "Model","X1"
|
||||
if strings.Count(raw, `"`) >= 4 && (strings.Contains(raw, ";") || strings.Contains(raw, ",")) {
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func parseHTMLSpecList(raw string) []SpecPair {
|
||||
matches := reHTMLLi.FindAllStringSubmatch(raw, maxSpecPairs+1)
|
||||
if len(matches) == 0 {
|
||||
// Fallback: strip tags and try flat parse.
|
||||
plain := strings.TrimSpace(reHTMLTag.ReplaceAllString(raw, "\n"))
|
||||
plain = html.UnescapeString(plain)
|
||||
return parseFlatSpecs(plain)
|
||||
}
|
||||
out := make([]SpecPair, 0, len(matches))
|
||||
for _, m := range matches {
|
||||
inner := strings.TrimSpace(reHTMLTag.ReplaceAllString(m[1], " "))
|
||||
inner = html.UnescapeString(inner)
|
||||
inner = strings.TrimSpace(reMultiSpace.ReplaceAllString(inner, " "))
|
||||
if inner == "" {
|
||||
continue
|
||||
}
|
||||
if p, ok := splitLabelValue(inner); ok {
|
||||
out = append(out, p)
|
||||
}
|
||||
// Bare list items without "Label: value" are skipped — suppliers should
|
||||
// send explicit pairs; inventing key→"true" produces junk attributes.
|
||||
if len(out) >= maxSpecPairs {
|
||||
break
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func parseFlatSpecs(raw string) []SpecPair {
|
||||
raw = strings.ReplaceAll(raw, "\r\n", "\n")
|
||||
raw = strings.ReplaceAll(raw, "\r", "\n")
|
||||
|
||||
chunks := splitSpecChunks(raw)
|
||||
out := make([]SpecPair, 0, len(chunks))
|
||||
for _, chunk := range chunks {
|
||||
chunk = strings.TrimSpace(chunk)
|
||||
if chunk == "" {
|
||||
continue
|
||||
}
|
||||
// CSV-ish "Label","value" or Label,value
|
||||
if strings.Contains(chunk, ",") {
|
||||
if p, ok := parseCSVSpecChunk(chunk); ok {
|
||||
out = append(out, p)
|
||||
if len(out) >= maxSpecPairs {
|
||||
break
|
||||
}
|
||||
continue
|
||||
}
|
||||
}
|
||||
if p, ok := splitLabelValue(chunk); ok {
|
||||
out = append(out, p)
|
||||
}
|
||||
if len(out) >= maxSpecPairs {
|
||||
break
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func splitSpecChunks(raw string) []string {
|
||||
// Prefer strong separators first.
|
||||
for _, sep := range []string{"\n", ";", "|"} {
|
||||
if strings.Contains(raw, sep) {
|
||||
return strings.Split(raw, sep)
|
||||
}
|
||||
}
|
||||
// Comma only when it looks like paired entries (has colon/equals).
|
||||
if strings.Contains(raw, ",") && (strings.Contains(raw, ":") || strings.Contains(raw, "=")) {
|
||||
return strings.Split(raw, ",")
|
||||
}
|
||||
return []string{raw}
|
||||
}
|
||||
|
||||
func parseCSVSpecChunk(chunk string) (SpecPair, bool) {
|
||||
parts := strings.SplitN(chunk, ",", 2)
|
||||
if len(parts) != 2 {
|
||||
return SpecPair{}, false
|
||||
}
|
||||
label := strings.Trim(strings.TrimSpace(parts[0]), `"'`)
|
||||
value := strings.Trim(strings.TrimSpace(parts[1]), `"'`)
|
||||
if label == "" || value == "" {
|
||||
return SpecPair{}, false
|
||||
}
|
||||
return SpecPair{
|
||||
Label: truncateRunes(label, maxSpecLabelRunes),
|
||||
Value: truncateRunes(value, maxSpecValueRunes),
|
||||
}, true
|
||||
}
|
||||
|
||||
func splitLabelValue(s string) (SpecPair, bool) {
|
||||
s = strings.TrimSpace(s)
|
||||
if s == "" {
|
||||
return SpecPair{}, false
|
||||
}
|
||||
// Prefer "Label: value" / "Label：value" / "Label - value" / "Label = value"
|
||||
for _, sep := range []string{":", "：", "=", "–", "—"} {
|
||||
if i := strings.Index(s, sep); i > 0 {
|
||||
label := strings.TrimSpace(s[:i])
|
||||
value := strings.TrimSpace(s[i+len(sep):])
|
||||
if label != "" && value != "" && !looksLikeURLScheme(label) {
|
||||
return SpecPair{
|
||||
Label: truncateRunes(label, maxSpecLabelRunes),
|
||||
Value: truncateRunes(value, maxSpecValueRunes),
|
||||
}, true
|
||||
}
|
||||
}
|
||||
}
|
||||
// "Label - value" with spaces (avoid splitting hyphenated words alone)
|
||||
if i := strings.Index(s, " - "); i > 0 {
|
||||
label := strings.TrimSpace(s[:i])
|
||||
value := strings.TrimSpace(s[i+3:])
|
||||
if label != "" && value != "" {
|
||||
return SpecPair{
|
||||
Label: truncateRunes(label, maxSpecLabelRunes),
|
||||
Value: truncateRunes(value, maxSpecValueRunes),
|
||||
}, true
|
||||
}
|
||||
}
|
||||
return SpecPair{}, false
|
||||
}
|
||||
|
||||
func looksLikeURLScheme(label string) bool {
|
||||
lower := strings.ToLower(strings.TrimSpace(label))
|
||||
return lower == "http" || lower == "https" || lower == "ftp"
|
||||
}
|
||||
|
||||
func sanitizePathSegment(label string) string {
|
||||
label = strings.TrimSpace(label)
|
||||
if label == "" {
|
||||
return ""
|
||||
}
|
||||
var b strings.Builder
|
||||
b.Grow(len(label))
|
||||
prevUS := false
|
||||
for _, r := range label {
|
||||
switch {
|
||||
case unicode.IsLetter(r) || unicode.IsDigit(r):
|
||||
b.WriteRune(r)
|
||||
prevUS = false
|
||||
case r == '_' || r == '-' || r == '.':
|
||||
b.WriteRune(r)
|
||||
prevUS = false
|
||||
case unicode.IsSpace(r) || r == '/' || r == '\\':
|
||||
if !prevUS && b.Len() > 0 {
|
||||
b.WriteByte('_')
|
||||
prevUS = true
|
||||
}
|
||||
default:
|
||||
// drop punctuation
|
||||
}
|
||||
}
|
||||
out := strings.Trim(b.String(), "._-")
|
||||
return truncateRunes(out, maxSpecLabelRunes)
|
||||
}
|
||||
|
||||
func clampSpecPairs(pairs []SpecPair) []SpecPair {
|
||||
if len(pairs) == 0 {
|
||||
return nil
|
||||
}
|
||||
if len(pairs) > maxSpecPairs {
|
||||
pairs = pairs[:maxSpecPairs]
|
||||
}
|
||||
seen := make(map[string]struct{}, len(pairs))
|
||||
out := make([]SpecPair, 0, len(pairs))
|
||||
for _, p := range pairs {
|
||||
p.Label = strings.TrimSpace(p.Label)
|
||||
p.Value = strings.TrimSpace(p.Value)
|
||||
if p.Label == "" || p.Value == "" {
|
||||
continue
|
||||
}
|
||||
key := strings.ToLower(p.Label)
|
||||
if _, ok := seen[key]; ok {
|
||||
continue
|
||||
}
|
||||
seen[key] = struct{}{}
|
||||
out = append(out, p)
|
||||
}
|
||||
if len(out) == 0 {
|
||||
return nil
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func truncateRunes(s string, max int) string {
|
||||
if max <= 0 || s == "" {
|
||||
return s
|
||||
}
|
||||
n := 0
|
||||
for i := range s {
|
||||
if n == max {
|
||||
return s[:i]
|
||||
}
|
||||
n++
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
func specsToMap(pairs []SpecPair) map[string]string {
|
||||
if len(pairs) == 0 {
|
||||
return nil
|
||||
}
|
||||
out := make(map[string]string, len(pairs))
|
||||
for _, p := range pairs {
|
||||
key := CanonicalAttributeKey(p.Label)
|
||||
if key == "" || strings.TrimSpace(p.Value) == "" {
|
||||
continue
|
||||
}
|
||||
out[key] = p.Value
|
||||
}
|
||||
if len(out) == 0 {
|
||||
return nil
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// compactAttributeKey strips separators for alias lookup (net_height / net-height / netheight → netheight).
|
||||
func compactAttributeKey(key string) string {
|
||||
var b strings.Builder
|
||||
b.Grow(len(key))
|
||||
for _, r := range strings.ToLower(key) {
|
||||
r = foldLatinRune(r)
|
||||
if (r >= 'a' && r <= 'z') || (r >= '0' && r <= '9') {
|
||||
b.WriteRune(r)
|
||||
}
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
|
||||
// Known supplier / locale labels → Descrybe standard field keys (snake_case).
|
||||
// Freeform specs keep kebab-case from AttributeKeyFromLabel.
|
||||
var attributeKeyAliases = map[string]string{
|
||||
"visina": "net_height",
|
||||
"height": "net_height",
|
||||
"netheight": "net_height",
|
||||
"sirina": "net_width",
|
||||
"width": "net_width",
|
||||
"netwidth": "net_width",
|
||||
"globina": "net_depth",
|
||||
"depth": "net_depth",
|
||||
"netdepth": "net_depth",
|
||||
"netmass": "net_mass",
|
||||
"mass": "net_mass",
|
||||
"weight": "net_mass",
|
||||
"teza": "net_mass",
|
||||
"productmodel": "product_model",
|
||||
"model": "product_model",
|
||||
"eprelid": "eprel_id",
|
||||
"eprel": "eprel_id",
|
||||
"energyclass": "energy_class",
|
||||
"energijskirazred": "energy_class",
|
||||
}
|
||||
|
||||
// IsValidAttributeKey rejects empty / punctuation-only / boolean junk keys.
|
||||
func IsValidAttributeKey(key string) bool {
|
||||
key = strings.TrimSpace(key)
|
||||
if key == "" || len(key) < 2 {
|
||||
return false
|
||||
}
|
||||
compact := compactAttributeKey(key)
|
||||
if len(compact) < 2 {
|
||||
return false
|
||||
}
|
||||
hasLetter := false
|
||||
for _, r := range compact {
|
||||
if r >= 'a' && r <= 'z' {
|
||||
hasLetter = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if !hasLetter {
|
||||
return false
|
||||
}
|
||||
switch compact {
|
||||
case "true", "false", "yes", "no", "null", "undefined", "none", "n", "y":
|
||||
return false
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// CanonicalAttributeKey normalizes a human or feed label to a stable attribute key.
|
||||
// Known dimension/identity aliases map to STANDARD_FIELDS snake_case; other labels
|
||||
// become kebab-case. Invalid / junk labels return "".
|
||||
func CanonicalAttributeKey(label string) string {
|
||||
slug := AttributeKeyFromLabel(label)
|
||||
compact := compactAttributeKey(slug)
|
||||
if compact == "" {
|
||||
compact = compactAttributeKey(label)
|
||||
}
|
||||
if alias, ok := attributeKeyAliases[compact]; ok {
|
||||
return alias
|
||||
}
|
||||
if slug == "" || !IsValidAttributeKey(slug) {
|
||||
return ""
|
||||
}
|
||||
return slug
|
||||
}
|
||||
|
||||
// AttributeKeyFromLabel turns a human spec label into a kebab-case attribute_key
|
||||
// (e.g. "Energijski razred" → "energijski-razred") matching company attributes.
|
||||
func AttributeKeyFromLabel(label string) string {
|
||||
label = strings.TrimSpace(label)
|
||||
if label == "" {
|
||||
return ""
|
||||
}
|
||||
var b strings.Builder
|
||||
b.Grow(len(label))
|
||||
prevHyphen := false
|
||||
for _, r := range strings.ToLower(label) {
|
||||
r = foldLatinRune(r)
|
||||
switch {
|
||||
case r >= 'a' && r <= 'z', r >= '0' && r <= '9':
|
||||
b.WriteRune(r)
|
||||
prevHyphen = false
|
||||
case unicode.IsSpace(r) || r == '_' || r == '/' || r == '\\' || r == '.' || r == ':' || r == '-':
|
||||
if !prevHyphen && b.Len() > 0 {
|
||||
b.WriteByte('-')
|
||||
prevHyphen = true
|
||||
}
|
||||
default:
|
||||
// drop other punctuation
|
||||
}
|
||||
}
|
||||
return strings.Trim(b.String(), "-")
|
||||
}
|
||||
|
||||
func foldLatinRune(r rune) rune {
|
||||
switch r {
|
||||
case 'š', 'ś', 'ş':
|
||||
return 's'
|
||||
case 'č', 'ć', 'ç':
|
||||
return 'c'
|
||||
case 'ž', 'ź', 'ż':
|
||||
return 'z'
|
||||
case 'đ':
|
||||
return 'd'
|
||||
case 'ň', 'ń':
|
||||
return 'n'
|
||||
case 'ř':
|
||||
return 'r'
|
||||
case 'ť':
|
||||
return 't'
|
||||
case 'ď':
|
||||
return 'd'
|
||||
case 'ľ', 'ĺ':
|
||||
return 'l'
|
||||
case 'ä', 'á', 'à', 'â', 'ã', 'å':
|
||||
return 'a'
|
||||
case 'ë', 'é', 'è', 'ê':
|
||||
return 'e'
|
||||
case 'ï', 'í', 'ì', 'î':
|
||||
return 'i'
|
||||
case 'ö', 'ó', 'ò', 'ô', 'õ':
|
||||
return 'o'
|
||||
case 'ü', 'ú', 'ù', 'û':
|
||||
return 'u'
|
||||
case 'ý', 'ÿ':
|
||||
return 'y'
|
||||
default:
|
||||
return r
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user