Initial commit of Descrybe v2 without local scratch artifacts.
Drop one-shot tmp/axe scripts and agent i18n scratch so the Gitea tree is deployable.
This commit is contained in:
@@ -0,0 +1,318 @@
|
||||
package processing
|
||||
|
||||
import (
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
)
|
||||
|
||||
var (
|
||||
enrichCDATAWrapRe = regexp.MustCompile(`(?is)^\s*<!\[CDATA\[(.*)\]\]>\s*$`)
|
||||
enrichMassValueRe = regexp.MustCompile(`(?i)^\s*([0-9]+(?:[.,][0-9]+)?)\s*([a-zµμ]+)?\s*$`)
|
||||
enrichNbspRe = regexp.MustCompile(`(?i) | `)
|
||||
enrichDimKeyRe = regexp.MustCompile(`(?i)^(net)?(width|height|depth|length|dimension)s?$`)
|
||||
enrichZeroNumRe = regexp.MustCompile(`^\s*0+(?:[.,]0+)?\s*$`)
|
||||
enrichHTMLTagRe = regexp.MustCompile(`(?is)<[^>]*>`)
|
||||
)
|
||||
|
||||
// Availability values normalized from vendor stockStatus text.
|
||||
const (
|
||||
AvailabilityInStock = "in_stock"
|
||||
AvailabilityOutOfStock = "out_of_stock"
|
||||
AvailabilityPreorder = "preorder"
|
||||
AvailabilityBackorder = "backorder"
|
||||
AvailabilityLimited = "limited"
|
||||
)
|
||||
|
||||
// EnrichMapped returns a processing-time copy of mapped feed fields with
|
||||
// derived fills and cleanup. Sync/map must keep raw values unchanged.
|
||||
//
|
||||
// Complements NormalizeMapped (alias flatten / zero dims) with Janus-style rules:
|
||||
// gtin←EAN, title←name, strip empty CDATA/HTML, parse netMass+unit,
|
||||
// stockStatus→availability enum. Does not invent empty EPRELID/mainImage.
|
||||
func EnrichMapped(mapped map[string]any) map[string]any {
|
||||
if mapped == nil {
|
||||
return map[string]any{}
|
||||
}
|
||||
out := make(map[string]any, len(mapped)+4)
|
||||
for k, v := range mapped {
|
||||
out[k] = v
|
||||
}
|
||||
|
||||
stripEmptyMarkup(out)
|
||||
dropZeroDimensions(out)
|
||||
deriveGTIN(out)
|
||||
deriveTitle(out)
|
||||
parseNetMass(out)
|
||||
applyAvailabilityFromStock(out)
|
||||
return out
|
||||
}
|
||||
|
||||
func stripEmptyMarkup(m map[string]any) {
|
||||
for k, v := range m {
|
||||
s, ok := enrichAsString(v)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
cleaned := cleanMarkupValue(s)
|
||||
if cleaned == "" {
|
||||
delete(m, k)
|
||||
continue
|
||||
}
|
||||
if cleaned != s {
|
||||
m[k] = cleaned
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func cleanMarkupValue(s string) string {
|
||||
s = strings.TrimSpace(s)
|
||||
if s == "" {
|
||||
return ""
|
||||
}
|
||||
if sub := enrichCDATAWrapRe.FindStringSubmatch(s); len(sub) == 2 {
|
||||
s = strings.TrimSpace(sub[1])
|
||||
}
|
||||
if strings.EqualFold(s, "<![CDATA[]]>") || strings.EqualFold(s, "<![CDATA[ ]]>") {
|
||||
return ""
|
||||
}
|
||||
plain := enrichNbspRe.ReplaceAllString(s, " ")
|
||||
// Reuse specs package htmlTagRe via stripping through a local call pattern:
|
||||
// htmlTagRe lives in specs.go — strip tags with a dedicated helper.
|
||||
plain = stripHTMLTags(plain)
|
||||
plain = strings.Join(strings.Fields(plain), " ")
|
||||
if plain == "" {
|
||||
return ""
|
||||
}
|
||||
return strings.TrimSpace(s)
|
||||
}
|
||||
|
||||
func stripHTMLTags(s string) string {
|
||||
return enrichHTMLTagRe.ReplaceAllString(s, " ")
|
||||
}
|
||||
|
||||
func dropZeroDimensions(m map[string]any) {
|
||||
for k, v := range m {
|
||||
leaf := enrichLeafKey(k)
|
||||
if !enrichDimKeyRe.MatchString(leaf) {
|
||||
continue
|
||||
}
|
||||
if enrichIsZeroValue(v) {
|
||||
delete(m, k)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func deriveGTIN(m map[string]any) {
|
||||
if hasNonEmptyEnrich(m, "gtin", "GTIN") {
|
||||
return
|
||||
}
|
||||
for _, key := range []string{"ean", "EAN", "upc", "UPC", "barcode", "Barcode"} {
|
||||
if s, ok := enrichAsString(m[key]); ok && s != "" {
|
||||
m["gtin"] = s
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func deriveTitle(m map[string]any) {
|
||||
if hasNonEmptyEnrich(m, "title", "Title") {
|
||||
return
|
||||
}
|
||||
for _, key := range []string{"name", "Name", "product_name", "productName", "ProductName"} {
|
||||
if s, ok := enrichAsString(m[key]); ok && s != "" {
|
||||
m["title"] = s
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func parseNetMass(m map[string]any) {
|
||||
var raw any
|
||||
var srcKey string
|
||||
for _, key := range []string{"netMass", "net_mass", "NetMass", "weight", "Weight", "mass"} {
|
||||
if v, ok := m[key]; ok {
|
||||
raw = v
|
||||
srcKey = key
|
||||
break
|
||||
}
|
||||
}
|
||||
if raw == nil {
|
||||
return
|
||||
}
|
||||
s, ok := enrichAsString(raw)
|
||||
if !ok || s == "" {
|
||||
return
|
||||
}
|
||||
sub := enrichMassValueRe.FindStringSubmatch(s)
|
||||
if len(sub) < 2 {
|
||||
return
|
||||
}
|
||||
num := strings.ReplaceAll(sub[1], ",", ".")
|
||||
f, err := strconv.ParseFloat(num, 64)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
if f == 0 {
|
||||
delete(m, srcKey)
|
||||
return
|
||||
}
|
||||
unit := ""
|
||||
if len(sub) >= 3 {
|
||||
unit = normalizeMassUnit(sub[2])
|
||||
}
|
||||
m["net_mass"] = f
|
||||
if unit != "" {
|
||||
m["net_mass_unit"] = unit
|
||||
}
|
||||
if unit != "" {
|
||||
m["weight"] = strings.TrimSpace(num + " " + unit)
|
||||
} else {
|
||||
m["weight"] = num
|
||||
}
|
||||
}
|
||||
|
||||
func normalizeMassUnit(u string) string {
|
||||
u = strings.ToLower(strings.TrimSpace(u))
|
||||
u = strings.ReplaceAll(u, "μ", "u")
|
||||
u = strings.ReplaceAll(u, "µ", "u")
|
||||
switch u {
|
||||
case "kg", "kilogram", "kilograms":
|
||||
return "kg"
|
||||
case "g", "gram", "grams":
|
||||
return "g"
|
||||
case "mg", "milligram", "milligrams":
|
||||
return "mg"
|
||||
case "lb", "lbs", "pound", "pounds":
|
||||
return "lb"
|
||||
case "oz", "ounce", "ounces":
|
||||
return "oz"
|
||||
case "t", "ton", "tonne", "tonnes":
|
||||
return "t"
|
||||
case "ug", "mcg":
|
||||
return "ug"
|
||||
default:
|
||||
return u
|
||||
}
|
||||
}
|
||||
|
||||
func applyAvailabilityFromStock(m map[string]any) {
|
||||
var raw any
|
||||
for _, key := range []string{"stockStatus", "stock_status", "StockStatus", "availability", "Availability"} {
|
||||
if v, ok := m[key]; ok {
|
||||
raw = v
|
||||
break
|
||||
}
|
||||
}
|
||||
if raw == nil {
|
||||
return
|
||||
}
|
||||
s, ok := enrichAsString(raw)
|
||||
if !ok || s == "" {
|
||||
return
|
||||
}
|
||||
if avail := MapStockStatus(s); avail != "" {
|
||||
m["availability"] = avail
|
||||
}
|
||||
}
|
||||
|
||||
// MapStockStatus maps vendor stock text onto a stable availability enum.
|
||||
func MapStockStatus(s string) string {
|
||||
n := normalizeStockToken(s)
|
||||
if n == "" {
|
||||
return ""
|
||||
}
|
||||
switch {
|
||||
case n == "instock" || n == "in_stock" || n == "available" || n == "nazalogi" ||
|
||||
n == "naskladiscu" || n == "auflager" || n == "yes" || n == "1" || n == "true":
|
||||
return AvailabilityInStock
|
||||
case n == "outofstock" || n == "out_of_stock" || n == "unavailable" || n == "ninazalogi" ||
|
||||
n == "soldout" || n == "no" || n == "0" || n == "false":
|
||||
return AvailabilityOutOfStock
|
||||
case strings.Contains(n, "preorder") || strings.Contains(n, "pre_order"):
|
||||
return AvailabilityPreorder
|
||||
case strings.Contains(n, "backorder") || strings.Contains(n, "back_order"):
|
||||
return AvailabilityBackorder
|
||||
case strings.Contains(n, "limited") || n == "lowstock" || n == "low_stock":
|
||||
return AvailabilityLimited
|
||||
case strings.Contains(n, "instock") || strings.Contains(n, "in_stock") || strings.Contains(n, "available"):
|
||||
return AvailabilityInStock
|
||||
case strings.Contains(n, "outofstock") || strings.Contains(n, "out_of_stock"):
|
||||
return AvailabilityOutOfStock
|
||||
default:
|
||||
return ""
|
||||
}
|
||||
}
|
||||
|
||||
func normalizeStockToken(s string) string {
|
||||
s = strings.ToLower(strings.TrimSpace(s))
|
||||
s = strings.ReplaceAll(s, "-", "")
|
||||
s = strings.ReplaceAll(s, " ", "")
|
||||
s = strings.ReplaceAll(s, "_", "")
|
||||
replacer := strings.NewReplacer(
|
||||
"č", "c", "ć", "c", "š", "s", "ž", "z", "đ", "d",
|
||||
"ä", "a", "ö", "o", "ü", "u", "ß", "ss",
|
||||
)
|
||||
return replacer.Replace(s)
|
||||
}
|
||||
|
||||
func hasNonEmptyEnrich(m map[string]any, keys ...string) bool {
|
||||
for _, k := range keys {
|
||||
if s, ok := enrichAsString(m[k]); ok && s != "" {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func enrichAsString(v any) (string, bool) {
|
||||
switch t := v.(type) {
|
||||
case string:
|
||||
return strings.TrimSpace(t), true
|
||||
case float64:
|
||||
if t == float64(int64(t)) {
|
||||
return strconv.FormatInt(int64(t), 10), true
|
||||
}
|
||||
return strconv.FormatFloat(t, 'f', -1, 64), true
|
||||
case float32:
|
||||
return strconv.FormatFloat(float64(t), 'f', -1, 64), true
|
||||
case int:
|
||||
return strconv.Itoa(t), true
|
||||
case int64:
|
||||
return strconv.FormatInt(t, 10), true
|
||||
default:
|
||||
return "", false
|
||||
}
|
||||
}
|
||||
|
||||
func enrichIsZeroValue(v any) bool {
|
||||
switch t := v.(type) {
|
||||
case nil:
|
||||
return true
|
||||
case string:
|
||||
return enrichZeroNumRe.MatchString(t) || strings.TrimSpace(t) == "" || isZeroishString(t)
|
||||
case float64:
|
||||
return t == 0
|
||||
case float32:
|
||||
return t == 0
|
||||
case int:
|
||||
return t == 0
|
||||
case int64:
|
||||
return t == 0
|
||||
case int32:
|
||||
return t == 0
|
||||
default:
|
||||
if s, ok := enrichAsString(v); ok {
|
||||
return enrichZeroNumRe.MatchString(s) || isZeroishString(s)
|
||||
}
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
func enrichLeafKey(k string) string {
|
||||
k = strings.TrimSpace(k)
|
||||
if i := strings.LastIndex(k, "/"); i >= 0 {
|
||||
k = k[i+1:]
|
||||
}
|
||||
return k
|
||||
}
|
||||
Reference in New Issue
Block a user