diff --git a/apps/api/internal/processing/pipeline_steps_test.go b/apps/api/internal/processing/pipeline_steps_test.go
index a754f88..26bad9d 100644
--- a/apps/api/internal/processing/pipeline_steps_test.go
+++ b/apps/api/internal/processing/pipeline_steps_test.go
@@ -30,6 +30,66 @@ func TestParseSpecifications_htmlAndCSV(t *testing.T) {
}
}
+func TestParseSpecifications_skipsGarbageAndReservedKeys(t *testing.T) {
+ attrs := ParseSpecifications(map[string]any{
+ ":": "true",
+ "zavora": "disc",
+ "name": "Should not appear",
+ "description": "Should not appear",
+ "brand": "Vox",
+ })
+ if _, ok := attrs[":"]; ok {
+ t.Fatalf("colon key must be dropped: %v", attrs)
+ }
+ if _, ok := attrs["name"]; ok {
+ t.Fatalf("name must be reserved: %v", attrs)
+ }
+ if attrs["zavora"] != "disc" {
+ t.Fatalf("zavora=%v", attrs["zavora"])
+ }
+ if attrs["brand"] != "Vox" {
+ t.Fatalf("brand=%v", attrs["brand"])
+ }
+}
+
+func TestSanitizeProductAttributes(t *testing.T) {
+ got := SanitizeProductAttributes(map[string]any{
+ "name": "TV Mount",
+ "description": "
html",
+ "gtin": "123",
+ "id": "102544",
+ "purchaseprice": "10",
+ "main_image": "https://x",
+ ":": "true",
+ "brand": "Ostalo",
+ "productmodel": "W53070",
+ "netwidth": "0.6 m",
+ "width": "0.6 m",
+ "warranty": "60 mesecev",
+ })
+ if _, ok := got["name"]; ok {
+ t.Fatalf("name must be stripped: %v", got)
+ }
+ if _, ok := got["description"]; ok {
+ t.Fatalf("description must be stripped: %v", got)
+ }
+ if _, ok := got[":"]; ok {
+ t.Fatalf("invalid key must be stripped: %v", got)
+ }
+ if got["brand"] != "Ostalo" {
+ t.Fatalf("brand=%v", got["brand"])
+ }
+ if got["product_model"] != "W53070" {
+ t.Fatalf("product_model=%v", got["product_model"])
+ }
+ if got["width"] != "0.6 m" {
+ t.Fatalf("width=%v", got["width"])
+ }
+ if _, ok := got["netwidth"]; ok {
+ t.Fatalf("alias netwidth should collapse to width: %v", got)
+ }
+}
+
func TestFillMissingFields_brandAndDims(t *testing.T) {
m := FillMissingFields(map[string]any{
"name": "Nike Air 30x20x10 cm",
diff --git a/apps/api/internal/processing/specs.go b/apps/api/internal/processing/specs.go
index 07665d0..ba1adbe 100644
--- a/apps/api/internal/processing/specs.go
+++ b/apps/api/internal/processing/specs.go
@@ -44,8 +44,16 @@ func parseSpecsInto(dst map[string]any, v any) {
// Already structured attributes / grouped specs
if looksLikeAttrMap(t) {
for k, val := range t {
+ lk := strings.ToLower(strings.TrimSpace(k))
+ if isReservedProductKey(lk) || isInvalidAttributeKey(k) {
+ continue
+ }
if s := stringifySpecValue(val); s != "" {
- dst[SanitizeOutput(k)] = s
+ key := SanitizeOutput(k)
+ if key == "" || isInvalidAttributeKey(key) {
+ continue
+ }
+ dst[key] = s
} else if nested, ok := val.(map[string]any); ok {
parseSpecsInto(dst, nested)
}
@@ -58,7 +66,7 @@ func parseSpecsInto(dst map[string]any, v any) {
parseSpecsInto(dst, val)
continue
}
- if s := stringifySpecValue(val); s != "" && !isReservedProductKey(lk) {
+ if s := stringifySpecValue(val); s != "" && !isReservedProductKey(lk) && !isInvalidAttributeKey(k) {
dst[SanitizeOutput(k)] = s
}
}
@@ -105,15 +113,110 @@ func looksLikeAttrMap(m map[string]any) bool {
}
func isReservedProductKey(k string) bool {
- switch k {
- case "name", "title", "description", "gtin", "ean", "brand", "category",
- "price", "image", "stock", "eprel_id", "specifications", "raw", "mapped":
+ switch strings.ToLower(strings.TrimSpace(k)) {
+ case "name", "title", "description", "gtin", "ean", "category", "category_unique_id",
+ "price", "purchaseprice", "purchase_price", "sellingprice", "selling_price",
+ "image", "main_image", "mainimage", "moreimages", "more_images", "images",
+ "image_url", "imageurl", "image_link", "imagelink", "additional_image_urls",
+ "additional_image_link", "videourl", "video_url",
+ "stock", "stockstatus", "stock_status", "availability",
+ "id", "sku", "officiallink", "official_link", "service",
+ "specifications", "specs", "specification",
+ "raw", "mapped", "search":
return true
default:
return false
}
}
+// isInvalidAttributeKey rejects garbage keys from bad feed specs (e.g. ":").
+func isInvalidAttributeKey(k string) bool {
+ k = strings.TrimSpace(k)
+ if k == "" || k == ":" || k == "=" || k == "-" || k == "_" {
+ return true
+ }
+ // Must contain at least one letter after sanitize.
+ hasLetter := false
+ for _, r := range strings.ToLower(k) {
+ if r >= 'a' && r <= 'z' {
+ hasLetter = true
+ break
+ }
+ }
+ return !hasLetter
+}
+
+// SanitizeProductAttributes keeps characteristic attrs for API/storage and drops
+// core product fields that belong on the V1 item root (title, description, images, …).
+func SanitizeProductAttributes(attrs map[string]any) map[string]any {
+ return sanitizeProductAttributes(attrs, false)
+}
+
+// SanitizeV1ProcessAttributes is the stricter poll projection: also drops eprel_*
+// keys (those are exposed on item.eprel).
+func SanitizeV1ProcessAttributes(attrs map[string]any) map[string]any {
+ return sanitizeProductAttributes(attrs, true)
+}
+
+func sanitizeProductAttributes(attrs map[string]any, dropEPREL bool) map[string]any {
+ if len(attrs) == 0 {
+ return map[string]any{}
+ }
+ out := make(map[string]any, len(attrs))
+ for k, v := range attrs {
+ key := strings.TrimSpace(SanitizeOutput(k))
+ if key == "" || isReservedProductKey(key) || isInvalidAttributeKey(key) {
+ continue
+ }
+ if dropEPREL && strings.HasPrefix(strings.ToLower(key), "eprel") {
+ continue
+ }
+ s := stringifySpecValue(v)
+ if s == "" || s == "" {
+ continue
+ }
+ // Prefer canonical dimension/model keys when aliases collide.
+ canon := canonicalizeAttrKey(key)
+ if canon == "" || isReservedProductKey(canon) || isInvalidAttributeKey(canon) {
+ continue
+ }
+ if dropEPREL && strings.HasPrefix(strings.ToLower(canon), "eprel") {
+ continue
+ }
+ if isDimensionKey(canon) && isZeroishString(s) {
+ continue
+ }
+ if _, exists := out[canon]; exists && (canon != key) {
+ continue
+ }
+ out[canon] = SanitizeOutput(s)
+ }
+ return out
+}
+
+func canonicalizeAttrKey(k string) string {
+ compact := strings.ToLower(strings.TrimSpace(k))
+ compact = strings.ReplaceAll(compact, "-", "_")
+ compact = strings.ReplaceAll(compact, " ", "_")
+ noUnderscore := strings.ReplaceAll(compact, "_", "")
+ switch noUnderscore {
+ case "netwidth", "width", "sirina":
+ return "width"
+ case "netheight", "height", "visina":
+ return "height"
+ case "netdepth", "depth", "globina":
+ return "depth"
+ case "netmass", "weight", "mass", "teza":
+ return "weight"
+ case "productmodel", "model":
+ return "product_model"
+ case "energijskirazred", "energyclass":
+ return "energy_class"
+ default:
+ return compact
+ }
+}
+
func parseHTMLSpecs(dst map[string]any, s string) {
s = capSpecInput(s)
matches := htmlLiRe.FindAllStringSubmatch(s, maxSpecPairs)
@@ -126,8 +229,9 @@ func parseHTMLSpecs(dst map[string]any, s string) {
continue
}
key, val := splitLabelValue(text)
- if key != "" && val != "" {
- dst[attributeKeyFromLabel(key)] = SanitizeOutput(val)
+ attrKey := attributeKeyFromLabel(key)
+ if attrKey != "" && val != "" && !isReservedProductKey(attrKey) && !isInvalidAttributeKey(attrKey) {
+ dst[attrKey] = SanitizeOutput(val)
}
}
if len(matches) == 0 {
@@ -198,10 +302,10 @@ func parseCSVLikeSpecs(dst map[string]any, s string) {
if len(m) < 3 {
continue
}
- key := strings.TrimSpace(m[1])
+ key := attributeKeyFromLabel(strings.TrimSpace(m[1]))
val := strings.TrimSpace(m[2])
- if key != "" && val != "" {
- dst[attributeKeyFromLabel(key)] = SanitizeOutput(val)
+ if key != "" && val != "" && !isReservedProductKey(key) && !isInvalidAttributeKey(key) {
+ dst[key] = SanitizeOutput(val)
}
}
}
diff --git a/apps/api/internal/processing/steps.go b/apps/api/internal/processing/steps.go
index 60eed57..40a5248 100644
--- a/apps/api/internal/processing/steps.go
+++ b/apps/api/internal/processing/steps.go
@@ -75,8 +75,9 @@ func (e *Engine) RunSteps(ctx context.Context, companyID string, in ProductInput
}
}
attrs = parsed
- out.Attributes = attrs
- out.ProcessedAttributes = attrs
+ out.Attributes = SanitizeProductAttributes(attrs)
+ out.ProcessedAttributes = out.Attributes
+ attrs = out.Attributes
out.FieldSources["attributes"] = "specifications"
appendStepLog(out.GPTResponse, StepParseSpecs, map[string]any{
"count": len(attrs),
@@ -100,12 +101,15 @@ func (e *Engine) RunSteps(ctx context.Context, companyID string, in ProductInput
in.Description,
)
out.Category = stringFromAny(normalized["category"])
- // Promote filled scalar fields into attributes when useful
- promote := []string{"brand", "width", "height", "depth", "weight", "gtin", "stock_status"}
+ // Promote characteristic fields only — never core product identity/content
+ // (those belong on the V1 item root: title, description, ean, images, …).
+ promote := []string{"brand", "width", "height", "depth", "weight", "product_model", "warranty"}
for _, f := range in.StandardFields {
- if f.Key != "" {
- promote = append(promote, f.Key)
+ k := strings.TrimSpace(f.Key)
+ if k == "" || isReservedProductKey(k) || isInvalidAttributeKey(k) {
+ continue
}
+ promote = append(promote, k)
}
seen := map[string]bool{}
for _, k := range promote {
@@ -120,6 +124,7 @@ func (e *Engine) RunSteps(ctx context.Context, companyID string, in ProductInput
out.FieldSources[k] = "fill_fields"
}
}
+ attrs = SanitizeProductAttributes(attrs)
out.Attributes = attrs
out.ProcessedAttributes = attrs
appendStepLog(out.GPTResponse, StepFillFields, map[string]any{
@@ -411,7 +416,9 @@ func (e *Engine) RunSteps(ctx context.Context, companyID string, in ProductInput
if out.Attributes == nil {
out.Attributes = map[string]any{}
}
- if out.ProcessedAttributes == nil {
+ out.Attributes = SanitizeProductAttributes(out.Attributes)
+ out.ProcessedAttributes = SanitizeProductAttributes(out.ProcessedAttributes)
+ if len(out.ProcessedAttributes) == 0 {
out.ProcessedAttributes = out.Attributes
}
if out.AIProviderMode == "" {
diff --git a/apps/api/internal/processing/v1_legacy.go b/apps/api/internal/processing/v1_legacy.go
index af3b09b..8c40e5d 100644
--- a/apps/api/internal/processing/v1_legacy.go
+++ b/apps/api/internal/processing/v1_legacy.go
@@ -291,6 +291,9 @@ func (p *Pipeline) LoadV1ProcessJobItems(ctx context.Context, companyID, jobID u
description = nil
}
+ eprelVal := extractEPRELFromAttrs(attrs)
+ attrs = SanitizeV1ProcessAttributes(attrs)
+
item := V1ProcessJobItem{
"ean": ean,
"status": MapV1JobItemStatus(itemStatus, true),
@@ -303,7 +306,7 @@ func (p *Pipeline) LoadV1ProcessJobItems(ctx context.Context, companyID, jobID u
"attributes": nil,
"main_image": nil,
"more_images": nil,
- "eprel": extractEPRELFromAttrs(attrs),
+ "eprel": eprelVal,
}
applyV1ProcessItemIDs(item, processedID, rawProductID)
if itemError != nil && *itemError != "" {