319 lines
7.6 KiB
Go
319 lines
7.6 KiB
Go
|
package processing
|
||
|
|
|
||
|
|
import (
|
||
|
|
"regexp"
|
||
|
|
"strconv"
|
||
|
|
"strings"
|
||
|
|
)
|
||
|
|
|
||
|
|
var (
|
||
|
|
enrichCDATAWrapRe = regexp.MustCompile(`(?is)^\s*<!\[CDATA\[(.*)\]\]>\s*$`)
|
||
|
|
enrichMassValueRe = regexp.MustCompile(`(?i)^\s*([0-9]+(?:[.,][0-9]+)?)\s*([a-zµμ]+)?\s*$`)
|
||
|
|
enrichNbspRe = regexp.MustCompile(`(?i) | `)
|
||
|
|
enrichDimKeyRe = regexp.MustCompile(`(?i)^(net)?(width|height|depth|length|dimension)s?$`)
|
||
|
|
enrichZeroNumRe = regexp.MustCompile(`^\s*0+(?:[.,]0+)?\s*$`)
|
||
|
|
enrichHTMLTagRe = regexp.MustCompile(`(?is)<[^>]*>`)
|
||
|
|
)
|
||
|
|
|
||
|
|
// Availability values normalized from vendor stockStatus text.
|
||
|
|
const (
|
||
|
|
AvailabilityInStock = "in_stock"
|
||
|
|
AvailabilityOutOfStock = "out_of_stock"
|
||
|
|
AvailabilityPreorder = "preorder"
|
||
|
|
AvailabilityBackorder = "backorder"
|
||
|
|
AvailabilityLimited = "limited"
|
||
|
|
)
|
||
|
|
|
||
|
|
// EnrichMapped returns a processing-time copy of mapped feed fields with
|
||
|
|
// derived fills and cleanup. Sync/map must keep raw values unchanged.
|
||
|
|
//
|
||
|
|
// Complements NormalizeMapped (alias flatten / zero dims) with Janus-style rules:
|
||
|
|
// gtin←EAN, title←name, strip empty CDATA/HTML, parse netMass+unit,
|
||
|
|
// stockStatus→availability enum. Does not invent empty EPRELID/mainImage.
|
||
|
|
func EnrichMapped(mapped map[string]any) map[string]any {
|
||
|
|
if mapped == nil {
|
||
|
|
return map[string]any{}
|
||
|
|
}
|
||
|
|
out := make(map[string]any, len(mapped)+4)
|
||
|
|
for k, v := range mapped {
|
||
|
|
out[k] = v
|
||
|
|
}
|
||
|
|
|
||
|
|
stripEmptyMarkup(out)
|
||
|
|
dropZeroDimensions(out)
|
||
|
|
deriveGTIN(out)
|
||
|
|
deriveTitle(out)
|
||
|
|
parseNetMass(out)
|
||
|
|
applyAvailabilityFromStock(out)
|
||
|
|
return out
|
||
|
|
}
|
||
|
|
|
||
|
|
func stripEmptyMarkup(m map[string]any) {
|
||
|
|
for k, v := range m {
|
||
|
|
s, ok := enrichAsString(v)
|
||
|
|
if !ok {
|
||
|
|
continue
|
||
|
|
}
|
||
|
|
cleaned := cleanMarkupValue(s)
|
||
|
|
if cleaned == "" {
|
||
|
|
delete(m, k)
|
||
|
|
continue
|
||
|
|
}
|
||
|
|
if cleaned != s {
|
||
|
|
m[k] = cleaned
|
||
|
|
}
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
func cleanMarkupValue(s string) string {
|
||
|
|
s = strings.TrimSpace(s)
|
||
|
|
if s == "" {
|
||
|
|
return ""
|
||
|
|
}
|
||
|
|
if sub := enrichCDATAWrapRe.FindStringSubmatch(s); len(sub) == 2 {
|
||
|
|
s = strings.TrimSpace(sub[1])
|
||
|
|
}
|
||
|
|
if strings.EqualFold(s, "<![CDATA[]]>") || strings.EqualFold(s, "<![CDATA[ ]]>") {
|
||
|
|
return ""
|
||
|
|
}
|
||
|
|
plain := enrichNbspRe.ReplaceAllString(s, " ")
|
||
|
|
// Reuse specs package htmlTagRe via stripping through a local call pattern:
|
||
|
|
// htmlTagRe lives in specs.go — strip tags with a dedicated helper.
|
||
|
|
plain = stripHTMLTags(plain)
|
||
|
|
plain = strings.Join(strings.Fields(plain), " ")
|
||
|
|
if plain == "" {
|
||
|
|
return ""
|
||
|
|
}
|
||
|
|
return strings.TrimSpace(s)
|
||
|
|
}
|
||
|
|
|
||
|
|
func stripHTMLTags(s string) string {
|
||
|
|
return enrichHTMLTagRe.ReplaceAllString(s, " ")
|
||
|
|
}
|
||
|
|
|
||
|
|
func dropZeroDimensions(m map[string]any) {
|
||
|
|
for k, v := range m {
|
||
|
|
leaf := enrichLeafKey(k)
|
||
|
|
if !enrichDimKeyRe.MatchString(leaf) {
|
||
|
|
continue
|
||
|
|
}
|
||
|
|
if enrichIsZeroValue(v) {
|
||
|
|
delete(m, k)
|
||
|
|
}
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
func deriveGTIN(m map[string]any) {
|
||
|
|
if hasNonEmptyEnrich(m, "gtin", "GTIN") {
|
||
|
|
return
|
||
|
|
}
|
||
|
|
for _, key := range []string{"ean", "EAN", "upc", "UPC", "barcode", "Barcode"} {
|
||
|
|
if s, ok := enrichAsString(m[key]); ok && s != "" {
|
||
|
|
m["gtin"] = s
|
||
|
|
return
|
||
|
|
}
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
func deriveTitle(m map[string]any) {
|
||
|
|
if hasNonEmptyEnrich(m, "title", "Title") {
|
||
|
|
return
|
||
|
|
}
|
||
|
|
for _, key := range []string{"name", "Name", "product_name", "productName", "ProductName"} {
|
||
|
|
if s, ok := enrichAsString(m[key]); ok && s != "" {
|
||
|
|
m["title"] = s
|
||
|
|
return
|
||
|
|
}
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
func parseNetMass(m map[string]any) {
|
||
|
|
var raw any
|
||
|
|
var srcKey string
|
||
|
|
for _, key := range []string{"netMass", "net_mass", "NetMass", "weight", "Weight", "mass"} {
|
||
|
|
if v, ok := m[key]; ok {
|
||
|
|
raw = v
|
||
|
|
srcKey = key
|
||
|
|
break
|
||
|
|
}
|
||
|
|
}
|
||
|
|
if raw == nil {
|
||
|
|
return
|
||
|
|
}
|
||
|
|
s, ok := enrichAsString(raw)
|
||
|
|
if !ok || s == "" {
|
||
|
|
return
|
||
|
|
}
|
||
|
|
sub := enrichMassValueRe.FindStringSubmatch(s)
|
||
|
|
if len(sub) < 2 {
|
||
|
|
return
|
||
|
|
}
|
||
|
|
num := strings.ReplaceAll(sub[1], ",", ".")
|
||
|
|
f, err := strconv.ParseFloat(num, 64)
|
||
|
|
if err != nil {
|
||
|
|
return
|
||
|
|
}
|
||
|
|
if f == 0 {
|
||
|
|
delete(m, srcKey)
|
||
|
|
return
|
||
|
|
}
|
||
|
|
unit := ""
|
||
|
|
if len(sub) >= 3 {
|
||
|
|
unit = normalizeMassUnit(sub[2])
|
||
|
|
}
|
||
|
|
m["net_mass"] = f
|
||
|
|
if unit != "" {
|
||
|
|
m["net_mass_unit"] = unit
|
||
|
|
}
|
||
|
|
if unit != "" {
|
||
|
|
m["weight"] = strings.TrimSpace(num + " " + unit)
|
||
|
|
} else {
|
||
|
|
m["weight"] = num
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
func normalizeMassUnit(u string) string {
|
||
|
|
u = strings.ToLower(strings.TrimSpace(u))
|
||
|
|
u = strings.ReplaceAll(u, "μ", "u")
|
||
|
|
u = strings.ReplaceAll(u, "µ", "u")
|
||
|
|
switch u {
|
||
|
|
case "kg", "kilogram", "kilograms":
|
||
|
|
return "kg"
|
||
|
|
case "g", "gram", "grams":
|
||
|
|
return "g"
|
||
|
|
case "mg", "milligram", "milligrams":
|
||
|
|
return "mg"
|
||
|
|
case "lb", "lbs", "pound", "pounds":
|
||
|
|
return "lb"
|
||
|
|
case "oz", "ounce", "ounces":
|
||
|
|
return "oz"
|
||
|
|
case "t", "ton", "tonne", "tonnes":
|
||
|
|
return "t"
|
||
|
|
case "ug", "mcg":
|
||
|
|
return "ug"
|
||
|
|
default:
|
||
|
|
return u
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
func applyAvailabilityFromStock(m map[string]any) {
|
||
|
|
var raw any
|
||
|
|
for _, key := range []string{"stockStatus", "stock_status", "StockStatus", "availability", "Availability"} {
|
||
|
|
if v, ok := m[key]; ok {
|
||
|
|
raw = v
|
||
|
|
break
|
||
|
|
}
|
||
|
|
}
|
||
|
|
if raw == nil {
|
||
|
|
return
|
||
|
|
}
|
||
|
|
s, ok := enrichAsString(raw)
|
||
|
|
if !ok || s == "" {
|
||
|
|
return
|
||
|
|
}
|
||
|
|
if avail := MapStockStatus(s); avail != "" {
|
||
|
|
m["availability"] = avail
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
// MapStockStatus maps vendor stock text onto a stable availability enum.
|
||
|
|
func MapStockStatus(s string) string {
|
||
|
|
n := normalizeStockToken(s)
|
||
|
|
if n == "" {
|
||
|
|
return ""
|
||
|
|
}
|
||
|
|
switch {
|
||
|
|
case n == "instock" || n == "in_stock" || n == "available" || n == "nazalogi" ||
|
||
|
|
n == "naskladiscu" || n == "auflager" || n == "yes" || n == "1" || n == "true":
|
||
|
|
return AvailabilityInStock
|
||
|
|
case n == "outofstock" || n == "out_of_stock" || n == "unavailable" || n == "ninazalogi" ||
|
||
|
|
n == "soldout" || n == "no" || n == "0" || n == "false":
|
||
|
|
return AvailabilityOutOfStock
|
||
|
|
case strings.Contains(n, "preorder") || strings.Contains(n, "pre_order"):
|
||
|
|
return AvailabilityPreorder
|
||
|
|
case strings.Contains(n, "backorder") || strings.Contains(n, "back_order"):
|
||
|
|
return AvailabilityBackorder
|
||
|
|
case strings.Contains(n, "limited") || n == "lowstock" || n == "low_stock":
|
||
|
|
return AvailabilityLimited
|
||
|
|
case strings.Contains(n, "instock") || strings.Contains(n, "in_stock") || strings.Contains(n, "available"):
|
||
|
|
return AvailabilityInStock
|
||
|
|
case strings.Contains(n, "outofstock") || strings.Contains(n, "out_of_stock"):
|
||
|
|
return AvailabilityOutOfStock
|
||
|
|
default:
|
||
|
|
return ""
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
func normalizeStockToken(s string) string {
|
||
|
|
s = strings.ToLower(strings.TrimSpace(s))
|
||
|
|
s = strings.ReplaceAll(s, "-", "")
|
||
|
|
s = strings.ReplaceAll(s, " ", "")
|
||
|
|
s = strings.ReplaceAll(s, "_", "")
|
||
|
|
replacer := strings.NewReplacer(
|
||
|
|
"č", "c", "ć", "c", "š", "s", "ž", "z", "đ", "d",
|
||
|
|
"ä", "a", "ö", "o", "ü", "u", "ß", "ss",
|
||
|
|
)
|
||
|
|
return replacer.Replace(s)
|
||
|
|
}
|
||
|
|
|
||
|
|
func hasNonEmptyEnrich(m map[string]any, keys ...string) bool {
|
||
|
|
for _, k := range keys {
|
||
|
|
if s, ok := enrichAsString(m[k]); ok && s != "" {
|
||
|
|
return true
|
||
|
|
}
|
||
|
|
}
|
||
|
|
return false
|
||
|
|
}
|
||
|
|
|
||
|
|
func enrichAsString(v any) (string, bool) {
|
||
|
|
switch t := v.(type) {
|
||
|
|
case string:
|
||
|
|
return strings.TrimSpace(t), true
|
||
|
|
case float64:
|
||
|
|
if t == float64(int64(t)) {
|
||
|
|
return strconv.FormatInt(int64(t), 10), true
|
||
|
|
}
|
||
|
|
return strconv.FormatFloat(t, 'f', -1, 64), true
|
||
|
|
case float32:
|
||
|
|
return strconv.FormatFloat(float64(t), 'f', -1, 64), true
|
||
|
|
case int:
|
||
|
|
return strconv.Itoa(t), true
|
||
|
|
case int64:
|
||
|
|
return strconv.FormatInt(t, 10), true
|
||
|
|
default:
|
||
|
|
return "", false
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
func enrichIsZeroValue(v any) bool {
|
||
|
|
switch t := v.(type) {
|
||
|
|
case nil:
|
||
|
|
return true
|
||
|
|
case string:
|
||
|
|
return enrichZeroNumRe.MatchString(t) || strings.TrimSpace(t) == "" || isZeroishString(t)
|
||
|
|
case float64:
|
||
|
|
return t == 0
|
||
|
|
case float32:
|
||
|
|
return t == 0
|
||
|
|
case int:
|
||
|
|
return t == 0
|
||
|
|
case int64:
|
||
|
|
return t == 0
|
||
|
|
case int32:
|
||
|
|
return t == 0
|
||
|
|
default:
|
||
|
|
if s, ok := enrichAsString(v); ok {
|
||
|
|
return enrichZeroNumRe.MatchString(s) || isZeroishString(s)
|
||
|
|
}
|
||
|
|
return false
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
func enrichLeafKey(k string) string {
|
||
|
|
k = strings.TrimSpace(k)
|
||
|
|
if i := strings.LastIndex(k, "/"); i >= 0 {
|
||
|
|
k = k[i+1:]
|
||
|
|
}
|
||
|
|
return k
|
||
|
|
}
|