greeneclipse
8580c996c3
Drop one-shot tmp/axe scripts and agent i18n scratch so the Gitea tree is deployable.
540 lines
13 KiB
Go
540 lines
13 KiB
Go
package feeds
|
||
|
||
import (
|
||
"html"
|
||
"regexp"
|
||
"strings"
|
||
"unicode"
|
||
)
|
||
|
||
const (
|
||
maxSpecPairs = 200
|
||
maxSpecRawBytes = 64 << 10 // 64 KiB per specifications blob
|
||
maxSpecLabelRunes = 120
|
||
maxSpecValueRunes = 2000
|
||
)
|
||
|
||
var (
|
||
// Accept </li> and broken </> closers seen in A1 feed CDATA.
|
||
reHTMLLi = regexp.MustCompile(`(?is)<li\b[^>]*>(.*?)(?:</li\s*>|</\s*>)`)
|
||
reHTMLTag = regexp.MustCompile(`(?is)<[^>]+>`)
|
||
reMultiSpace = regexp.MustCompile(`\s+`)
|
||
)
|
||
|
||
// SpecPair is one label/value extracted from a specifications blob.
|
||
type SpecPair struct {
|
||
Label string
|
||
Value string
|
||
}
|
||
|
||
// ParseSpecifications accepts nested-expanded text, CDATA HTML lists, or flat
|
||
// CSV-like strings. Empty / missing / blank HTML returns nil (not an error).
|
||
func ParseSpecifications(raw string) []SpecPair {
|
||
raw = strings.TrimSpace(raw)
|
||
if raw == "" || len(raw) > maxSpecRawBytes {
|
||
return nil
|
||
}
|
||
if isEmptySpecBlob(raw) {
|
||
return nil
|
||
}
|
||
|
||
var pairs []SpecPair
|
||
switch {
|
||
case looksLikeHTMLList(raw):
|
||
pairs = parseHTMLSpecList(raw)
|
||
case looksLikeFlatSpecs(raw):
|
||
pairs = parseFlatSpecs(raw)
|
||
default:
|
||
// Single "Label: value" line still counts as flat.
|
||
if p, ok := splitLabelValue(raw); ok {
|
||
pairs = []SpecPair{p}
|
||
}
|
||
}
|
||
return clampSpecPairs(pairs)
|
||
}
|
||
|
||
// expandSpecificationFields mutates row: for specification-like keys whose
|
||
// value is HTML/flat text, add nested paths key/Label → value. Nested XML
|
||
// children are already present as key/child from the XML walker.
|
||
func expandSpecificationFields(row feedRow) {
|
||
if len(row) == 0 {
|
||
return
|
||
}
|
||
keys := make([]string, 0, 8)
|
||
for k, v := range row {
|
||
if !isSpecFieldKey(k) {
|
||
continue
|
||
}
|
||
if strings.TrimSpace(v) == "" {
|
||
continue
|
||
}
|
||
// Already has nested children — leave tree as-is; still parse text if useful.
|
||
if hasPrefixedChildren(row, k) && !looksLikeHTMLList(v) && !looksLikeFlatSpecs(v) {
|
||
continue
|
||
}
|
||
keys = append(keys, k)
|
||
}
|
||
for _, k := range keys {
|
||
pairs := ParseSpecifications(row[k])
|
||
for _, p := range pairs {
|
||
seg := sanitizePathSegment(p.Label)
|
||
if seg == "" {
|
||
continue
|
||
}
|
||
path := k + "/" + seg
|
||
if prev, ok := row[path]; ok && strings.TrimSpace(prev) != "" {
|
||
continue
|
||
}
|
||
row[path] = p.Value
|
||
}
|
||
}
|
||
}
|
||
|
||
func isSpecFieldKey(key string) bool {
|
||
leaf := strings.ToLower(leafName(key))
|
||
leaf = strings.TrimPrefix(leaf, "@")
|
||
switch leaf {
|
||
case "specifications", "specification", "specs", "spec", "features", "feature", "attributes_raw":
|
||
return true
|
||
}
|
||
return strings.Contains(leaf, "specification")
|
||
}
|
||
|
||
func hasPrefixedChildren(row feedRow, prefix string) bool {
|
||
prefix = strings.TrimSuffix(prefix, "/") + "/"
|
||
for k := range row {
|
||
if strings.HasPrefix(k, prefix) {
|
||
return true
|
||
}
|
||
}
|
||
return false
|
||
}
|
||
|
||
func collectPrefixed(row feedRow, prefix string) map[string]string {
|
||
prefix = strings.TrimSuffix(strings.TrimSpace(prefix), "/")
|
||
if prefix == "" {
|
||
return nil
|
||
}
|
||
p := prefix + "/"
|
||
out := map[string]string{}
|
||
for k, v := range row {
|
||
if !strings.HasPrefix(k, p) {
|
||
continue
|
||
}
|
||
rest := k[len(p):]
|
||
if rest == "" || strings.Contains(rest, "/") {
|
||
continue
|
||
}
|
||
v = strings.TrimSpace(v)
|
||
if v == "" {
|
||
continue
|
||
}
|
||
out[rest] = v
|
||
}
|
||
if len(out) == 0 {
|
||
return nil
|
||
}
|
||
return out
|
||
}
|
||
|
||
func isEmptySpecBlob(raw string) bool {
|
||
stripped := strings.TrimSpace(reHTMLTag.ReplaceAllString(raw, " "))
|
||
stripped = html.UnescapeString(stripped)
|
||
stripped = strings.TrimSpace(reMultiSpace.ReplaceAllString(stripped, " "))
|
||
return stripped == ""
|
||
}
|
||
|
||
func looksLikeHTMLList(raw string) bool {
|
||
lower := strings.ToLower(raw)
|
||
return strings.Contains(lower, "<li") || (strings.Contains(lower, "<ul") && strings.Contains(lower, "</ul>"))
|
||
}
|
||
|
||
func looksLikeFlatSpecs(raw string) bool {
|
||
if looksLikeHTMLList(raw) {
|
||
return false
|
||
}
|
||
// Multiple label:value pairs separated by ; | newline or comma between pairs.
|
||
if strings.Count(raw, ":") >= 2 {
|
||
return true
|
||
}
|
||
if strings.Count(raw, "=") >= 2 && (strings.Contains(raw, ";") || strings.Contains(raw, "|") || strings.Contains(raw, "\n")) {
|
||
return true
|
||
}
|
||
if strings.Contains(raw, ";") && strings.Contains(raw, ":") {
|
||
return true
|
||
}
|
||
if strings.Contains(raw, "|") && strings.Contains(raw, ":") {
|
||
return true
|
||
}
|
||
if strings.Contains(raw, "\n") && strings.Contains(raw, ":") {
|
||
return true
|
||
}
|
||
// Quoted CSV-ish pairs: "Brand","Acme"; "Model","X1"
|
||
if strings.Count(raw, `"`) >= 4 && (strings.Contains(raw, ";") || strings.Contains(raw, ",")) {
|
||
return true
|
||
}
|
||
return false
|
||
}
|
||
|
||
func parseHTMLSpecList(raw string) []SpecPair {
|
||
matches := reHTMLLi.FindAllStringSubmatch(raw, maxSpecPairs+1)
|
||
if len(matches) == 0 {
|
||
// Fallback: strip tags and try flat parse.
|
||
plain := strings.TrimSpace(reHTMLTag.ReplaceAllString(raw, "\n"))
|
||
plain = html.UnescapeString(plain)
|
||
return parseFlatSpecs(plain)
|
||
}
|
||
out := make([]SpecPair, 0, len(matches))
|
||
for _, m := range matches {
|
||
inner := strings.TrimSpace(reHTMLTag.ReplaceAllString(m[1], " "))
|
||
inner = html.UnescapeString(inner)
|
||
inner = strings.TrimSpace(reMultiSpace.ReplaceAllString(inner, " "))
|
||
if inner == "" {
|
||
continue
|
||
}
|
||
if p, ok := splitLabelValue(inner); ok {
|
||
out = append(out, p)
|
||
}
|
||
// Bare list items without "Label: value" are skipped — suppliers should
|
||
// send explicit pairs; inventing key→"true" produces junk attributes.
|
||
if len(out) >= maxSpecPairs {
|
||
break
|
||
}
|
||
}
|
||
return out
|
||
}
|
||
|
||
func parseFlatSpecs(raw string) []SpecPair {
|
||
raw = strings.ReplaceAll(raw, "\r\n", "\n")
|
||
raw = strings.ReplaceAll(raw, "\r", "\n")
|
||
|
||
chunks := splitSpecChunks(raw)
|
||
out := make([]SpecPair, 0, len(chunks))
|
||
for _, chunk := range chunks {
|
||
chunk = strings.TrimSpace(chunk)
|
||
if chunk == "" {
|
||
continue
|
||
}
|
||
// CSV-ish "Label","value" or Label,value
|
||
if strings.Contains(chunk, ",") {
|
||
if p, ok := parseCSVSpecChunk(chunk); ok {
|
||
out = append(out, p)
|
||
if len(out) >= maxSpecPairs {
|
||
break
|
||
}
|
||
continue
|
||
}
|
||
}
|
||
if p, ok := splitLabelValue(chunk); ok {
|
||
out = append(out, p)
|
||
}
|
||
if len(out) >= maxSpecPairs {
|
||
break
|
||
}
|
||
}
|
||
return out
|
||
}
|
||
|
||
func splitSpecChunks(raw string) []string {
|
||
// Prefer strong separators first.
|
||
for _, sep := range []string{"\n", ";", "|"} {
|
||
if strings.Contains(raw, sep) {
|
||
return strings.Split(raw, sep)
|
||
}
|
||
}
|
||
// Comma only when it looks like paired entries (has colon/equals).
|
||
if strings.Contains(raw, ",") && (strings.Contains(raw, ":") || strings.Contains(raw, "=")) {
|
||
return strings.Split(raw, ",")
|
||
}
|
||
return []string{raw}
|
||
}
|
||
|
||
func parseCSVSpecChunk(chunk string) (SpecPair, bool) {
|
||
parts := strings.SplitN(chunk, ",", 2)
|
||
if len(parts) != 2 {
|
||
return SpecPair{}, false
|
||
}
|
||
label := strings.Trim(strings.TrimSpace(parts[0]), `"'`)
|
||
value := strings.Trim(strings.TrimSpace(parts[1]), `"'`)
|
||
if label == "" || value == "" {
|
||
return SpecPair{}, false
|
||
}
|
||
return SpecPair{
|
||
Label: truncateRunes(label, maxSpecLabelRunes),
|
||
Value: truncateRunes(value, maxSpecValueRunes),
|
||
}, true
|
||
}
|
||
|
||
func splitLabelValue(s string) (SpecPair, bool) {
|
||
s = strings.TrimSpace(s)
|
||
if s == "" {
|
||
return SpecPair{}, false
|
||
}
|
||
// Prefer "Label: value" / "Label：value" / "Label - value" / "Label = value"
|
||
for _, sep := range []string{":", "：", "=", "–", "—"} {
|
||
if i := strings.Index(s, sep); i > 0 {
|
||
label := strings.TrimSpace(s[:i])
|
||
value := strings.TrimSpace(s[i+len(sep):])
|
||
if label != "" && value != "" && !looksLikeURLScheme(label) {
|
||
return SpecPair{
|
||
Label: truncateRunes(label, maxSpecLabelRunes),
|
||
Value: truncateRunes(value, maxSpecValueRunes),
|
||
}, true
|
||
}
|
||
}
|
||
}
|
||
// "Label - value" with spaces (avoid splitting hyphenated words alone)
|
||
if i := strings.Index(s, " - "); i > 0 {
|
||
label := strings.TrimSpace(s[:i])
|
||
value := strings.TrimSpace(s[i+3:])
|
||
if label != "" && value != "" {
|
||
return SpecPair{
|
||
Label: truncateRunes(label, maxSpecLabelRunes),
|
||
Value: truncateRunes(value, maxSpecValueRunes),
|
||
}, true
|
||
}
|
||
}
|
||
return SpecPair{}, false
|
||
}
|
||
|
||
func looksLikeURLScheme(label string) bool {
|
||
lower := strings.ToLower(strings.TrimSpace(label))
|
||
return lower == "http" || lower == "https" || lower == "ftp"
|
||
}
|
||
|
||
func sanitizePathSegment(label string) string {
|
||
label = strings.TrimSpace(label)
|
||
if label == "" {
|
||
return ""
|
||
}
|
||
var b strings.Builder
|
||
b.Grow(len(label))
|
||
prevUS := false
|
||
for _, r := range label {
|
||
switch {
|
||
case unicode.IsLetter(r) || unicode.IsDigit(r):
|
||
b.WriteRune(r)
|
||
prevUS = false
|
||
case r == '_' || r == '-' || r == '.':
|
||
b.WriteRune(r)
|
||
prevUS = false
|
||
case unicode.IsSpace(r) || r == '/' || r == '\\':
|
||
if !prevUS && b.Len() > 0 {
|
||
b.WriteByte('_')
|
||
prevUS = true
|
||
}
|
||
default:
|
||
// drop punctuation
|
||
}
|
||
}
|
||
out := strings.Trim(b.String(), "._-")
|
||
return truncateRunes(out, maxSpecLabelRunes)
|
||
}
|
||
|
||
func clampSpecPairs(pairs []SpecPair) []SpecPair {
|
||
if len(pairs) == 0 {
|
||
return nil
|
||
}
|
||
if len(pairs) > maxSpecPairs {
|
||
pairs = pairs[:maxSpecPairs]
|
||
}
|
||
seen := make(map[string]struct{}, len(pairs))
|
||
out := make([]SpecPair, 0, len(pairs))
|
||
for _, p := range pairs {
|
||
p.Label = strings.TrimSpace(p.Label)
|
||
p.Value = strings.TrimSpace(p.Value)
|
||
if p.Label == "" || p.Value == "" {
|
||
continue
|
||
}
|
||
key := strings.ToLower(p.Label)
|
||
if _, ok := seen[key]; ok {
|
||
continue
|
||
}
|
||
seen[key] = struct{}{}
|
||
out = append(out, p)
|
||
}
|
||
if len(out) == 0 {
|
||
return nil
|
||
}
|
||
return out
|
||
}
|
||
|
||
func truncateRunes(s string, max int) string {
|
||
if max <= 0 || s == "" {
|
||
return s
|
||
}
|
||
n := 0
|
||
for i := range s {
|
||
if n == max {
|
||
return s[:i]
|
||
}
|
||
n++
|
||
}
|
||
return s
|
||
}
|
||
|
||
func specsToMap(pairs []SpecPair) map[string]string {
|
||
if len(pairs) == 0 {
|
||
return nil
|
||
}
|
||
out := make(map[string]string, len(pairs))
|
||
for _, p := range pairs {
|
||
key := CanonicalAttributeKey(p.Label)
|
||
if key == "" || strings.TrimSpace(p.Value) == "" {
|
||
continue
|
||
}
|
||
out[key] = p.Value
|
||
}
|
||
if len(out) == 0 {
|
||
return nil
|
||
}
|
||
return out
|
||
}
|
||
|
||
// compactAttributeKey strips separators for alias lookup (net_height / net-height / netheight → netheight).
|
||
func compactAttributeKey(key string) string {
|
||
var b strings.Builder
|
||
b.Grow(len(key))
|
||
for _, r := range strings.ToLower(key) {
|
||
r = foldLatinRune(r)
|
||
if (r >= 'a' && r <= 'z') || (r >= '0' && r <= '9') {
|
||
b.WriteRune(r)
|
||
}
|
||
}
|
||
return b.String()
|
||
}
|
||
|
||
// Known supplier / locale labels → Descrybe standard field keys (snake_case).
|
||
// Freeform specs keep kebab-case from AttributeKeyFromLabel.
|
||
var attributeKeyAliases = map[string]string{
|
||
"visina": "net_height",
|
||
"height": "net_height",
|
||
"netheight": "net_height",
|
||
"sirina": "net_width",
|
||
"width": "net_width",
|
||
"netwidth": "net_width",
|
||
"globina": "net_depth",
|
||
"depth": "net_depth",
|
||
"netdepth": "net_depth",
|
||
"netmass": "net_mass",
|
||
"mass": "net_mass",
|
||
"weight": "net_mass",
|
||
"teza": "net_mass",
|
||
"productmodel": "product_model",
|
||
"model": "product_model",
|
||
"eprelid": "eprel_id",
|
||
"eprel": "eprel_id",
|
||
"energyclass": "energy_class",
|
||
"energijskirazred": "energy_class",
|
||
}
|
||
|
||
// IsValidAttributeKey rejects empty / punctuation-only / boolean junk keys.
|
||
func IsValidAttributeKey(key string) bool {
|
||
key = strings.TrimSpace(key)
|
||
if key == "" || len(key) < 2 {
|
||
return false
|
||
}
|
||
compact := compactAttributeKey(key)
|
||
if len(compact) < 2 {
|
||
return false
|
||
}
|
||
hasLetter := false
|
||
for _, r := range compact {
|
||
if r >= 'a' && r <= 'z' {
|
||
hasLetter = true
|
||
break
|
||
}
|
||
}
|
||
if !hasLetter {
|
||
return false
|
||
}
|
||
switch compact {
|
||
case "true", "false", "yes", "no", "null", "undefined", "none", "n", "y":
|
||
return false
|
||
}
|
||
return true
|
||
}
|
||
|
||
// CanonicalAttributeKey normalizes a human or feed label to a stable attribute key.
|
||
// Known dimension/identity aliases map to STANDARD_FIELDS snake_case; other labels
|
||
// become kebab-case. Invalid / junk labels return "".
|
||
func CanonicalAttributeKey(label string) string {
|
||
slug := AttributeKeyFromLabel(label)
|
||
compact := compactAttributeKey(slug)
|
||
if compact == "" {
|
||
compact = compactAttributeKey(label)
|
||
}
|
||
if alias, ok := attributeKeyAliases[compact]; ok {
|
||
return alias
|
||
}
|
||
if slug == "" || !IsValidAttributeKey(slug) {
|
||
return ""
|
||
}
|
||
return slug
|
||
}
|
||
|
||
// AttributeKeyFromLabel turns a human spec label into a kebab-case attribute_key
|
||
// (e.g. "Energijski razred" → "energijski-razred") matching company attributes.
|
||
func AttributeKeyFromLabel(label string) string {
|
||
label = strings.TrimSpace(label)
|
||
if label == "" {
|
||
return ""
|
||
}
|
||
var b strings.Builder
|
||
b.Grow(len(label))
|
||
prevHyphen := false
|
||
for _, r := range strings.ToLower(label) {
|
||
r = foldLatinRune(r)
|
||
switch {
|
||
case r >= 'a' && r <= 'z', r >= '0' && r <= '9':
|
||
b.WriteRune(r)
|
||
prevHyphen = false
|
||
case unicode.IsSpace(r) || r == '_' || r == '/' || r == '\\' || r == '.' || r == ':' || r == '-':
|
||
if !prevHyphen && b.Len() > 0 {
|
||
b.WriteByte('-')
|
||
prevHyphen = true
|
||
}
|
||
default:
|
||
// drop other punctuation
|
||
}
|
||
}
|
||
return strings.Trim(b.String(), "-")
|
||
}
|
||
|
||
func foldLatinRune(r rune) rune {
|
||
switch r {
|
||
case 'š', 'ś', 'ş':
|
||
return 's'
|
||
case 'č', 'ć', 'ç':
|
||
return 'c'
|
||
case 'ž', 'ź', 'ż':
|
||
return 'z'
|
||
case 'đ':
|
||
return 'd'
|
||
case 'ň', 'ń':
|
||
return 'n'
|
||
case 'ř':
|
||
return 'r'
|
||
case 'ť':
|
||
return 't'
|
||
case 'ď':
|
||
return 'd'
|
||
case 'ľ', 'ĺ':
|
||
return 'l'
|
||
case 'ä', 'á', 'à', 'â', 'ã', 'å':
|
||
return 'a'
|
||
case 'ë', 'é', 'è', 'ê':
|
||
return 'e'
|
||
case 'ï', 'í', 'ì', 'î':
|
||
return 'i'
|
||
case 'ö', 'ó', 'ò', 'ô', 'õ':
|
||
return 'o'
|
||
case 'ü', 'ú', 'ù', 'û':
|
||
return 'u'
|
||
case 'ý', 'ÿ':
|
||
return 'y'
|
||
default:
|
||
return r
|
||
}
|
||
}
|