From 1d3a9d5abd6afd60c4a1e9c43f573014aa953dfd Mon Sep 17 00:00:00 2001 From: GreenEclipse Date: Sun, 16 Aug 2026 12:23:14 +0200 Subject: [PATCH] fix --- .../processing/pipeline_steps_test.go | 60 +++++++++ apps/api/internal/processing/specs.go | 124 ++++++++++++++++-- apps/api/internal/processing/steps.go | 21 ++- apps/api/internal/processing/v1_legacy.go | 5 +- 4 files changed, 192 insertions(+), 18 deletions(-) diff --git a/apps/api/internal/processing/pipeline_steps_test.go b/apps/api/internal/processing/pipeline_steps_test.go index a754f88..26bad9d 100644 --- a/apps/api/internal/processing/pipeline_steps_test.go +++ b/apps/api/internal/processing/pipeline_steps_test.go @@ -30,6 +30,66 @@ func TestParseSpecifications_htmlAndCSV(t *testing.T) { } } +func TestParseSpecifications_skipsGarbageAndReservedKeys(t *testing.T) { + attrs := ParseSpecifications(map[string]any{ + ":": "true", + "zavora": "disc", + "name": "Should not appear", + "description": "Should not appear", + "brand": "Vox", + }) + if _, ok := attrs[":"]; ok { + t.Fatalf("colon key must be dropped: %v", attrs) + } + if _, ok := attrs["name"]; ok { + t.Fatalf("name must be reserved: %v", attrs) + } + if attrs["zavora"] != "disc" { + t.Fatalf("zavora=%v", attrs["zavora"]) + } + if attrs["brand"] != "Vox" { + t.Fatalf("brand=%v", attrs["brand"]) + } +} + +func TestSanitizeProductAttributes(t *testing.T) { + got := SanitizeProductAttributes(map[string]any{ + "name": "TV Mount", + "description": "
html", + "gtin": "123", + "id": "102544", + "purchaseprice": "10", + "main_image": "https://x", + ":": "true", + "brand": "Ostalo", + "productmodel": "W53070", + "netwidth": "0.6 m", + "width": "0.6 m", + "warranty": "60 mesecev", + }) + if _, ok := got["name"]; ok { + t.Fatalf("name must be stripped: %v", got) + } + if _, ok := got["description"]; ok { + t.Fatalf("description must be stripped: %v", got) + } + if _, ok := got[":"]; ok { + t.Fatalf("invalid key must be stripped: %v", got) + } + if got["brand"] != "Ostalo" { + t.Fatalf("brand=%v", got["brand"]) + } + if got["product_model"] != "W53070" { + t.Fatalf("product_model=%v", got["product_model"]) + } + if got["width"] != "0.6 m" { + t.Fatalf("width=%v", got["width"]) + } + if _, ok := got["netwidth"]; ok { + t.Fatalf("alias netwidth should collapse to width: %v", got) + } +} + func TestFillMissingFields_brandAndDims(t *testing.T) { m := FillMissingFields(map[string]any{ "name": "Nike Air 30x20x10 cm", diff --git a/apps/api/internal/processing/specs.go b/apps/api/internal/processing/specs.go index 07665d0..ba1adbe 100644 --- a/apps/api/internal/processing/specs.go +++ b/apps/api/internal/processing/specs.go @@ -44,8 +44,16 @@ func parseSpecsInto(dst map[string]any, v any) { // Already structured attributes / grouped specs if looksLikeAttrMap(t) { for k, val := range t { + lk := strings.ToLower(strings.TrimSpace(k)) + if isReservedProductKey(lk) || isInvalidAttributeKey(k) { + continue + } if s := stringifySpecValue(val); s != "" { - dst[SanitizeOutput(k)] = s + key := SanitizeOutput(k) + if key == "" || isInvalidAttributeKey(key) { + continue + } + dst[key] = s } else if nested, ok := val.(map[string]any); ok { parseSpecsInto(dst, nested) } @@ -58,7 +66,7 @@ func parseSpecsInto(dst map[string]any, v any) { parseSpecsInto(dst, val) continue } - if s := stringifySpecValue(val); s != "" && !isReservedProductKey(lk) { + if s := stringifySpecValue(val); s != "" && !isReservedProductKey(lk) && !isInvalidAttributeKey(k) { dst[SanitizeOutput(k)] = s } } @@ -105,15 +113,110 @@ func looksLikeAttrMap(m map[string]any) bool { } func isReservedProductKey(k string) bool { - switch k { - case "name", "title", "description", "gtin", "ean", "brand", "category", - "price", "image", "stock", "eprel_id", "specifications", "raw", "mapped": + switch strings.ToLower(strings.TrimSpace(k)) { + case "name", "title", "description", "gtin", "ean", "category", "category_unique_id", + "price", "purchaseprice", "purchase_price", "sellingprice", "selling_price", + "image", "main_image", "mainimage", "moreimages", "more_images", "images", + "image_url", "imageurl", "image_link", "imagelink", "additional_image_urls", + "additional_image_link", "videourl", "video_url", + "stock", "stockstatus", "stock_status", "availability", + "id", "sku", "officiallink", "official_link", "service", + "specifications", "specs", "specification", + "raw", "mapped", "search": return true default: return false } } +// isInvalidAttributeKey rejects garbage keys from bad feed specs (e.g. ":"). +func isInvalidAttributeKey(k string) bool { + k = strings.TrimSpace(k) + if k == "" || k == ":" || k == "=" || k == "-" || k == "_" { + return true + } + // Must contain at least one letter after sanitize. + hasLetter := false + for _, r := range strings.ToLower(k) { + if r >= 'a' && r <= 'z' { + hasLetter = true + break + } + } + return !hasLetter +} + +// SanitizeProductAttributes keeps characteristic attrs for API/storage and drops +// core product fields that belong on the V1 item root (title, description, images, …). +func SanitizeProductAttributes(attrs map[string]any) map[string]any { + return sanitizeProductAttributes(attrs, false) +} + +// SanitizeV1ProcessAttributes is the stricter poll projection: also drops eprel_* +// keys (those are exposed on item.eprel). +func SanitizeV1ProcessAttributes(attrs map[string]any) map[string]any { + return sanitizeProductAttributes(attrs, true) +} + +func sanitizeProductAttributes(attrs map[string]any, dropEPREL bool) map[string]any { + if len(attrs) == 0 { + return map[string]any{} + } + out := make(map[string]any, len(attrs)) + for k, v := range attrs { + key := strings.TrimSpace(SanitizeOutput(k)) + if key == "" || isReservedProductKey(key) || isInvalidAttributeKey(key) { + continue + } + if dropEPREL && strings.HasPrefix(strings.ToLower(key), "eprel") { + continue + } + s := stringifySpecValue(v) + if s == "" || s == "" { + continue + } + // Prefer canonical dimension/model keys when aliases collide. + canon := canonicalizeAttrKey(key) + if canon == "" || isReservedProductKey(canon) || isInvalidAttributeKey(canon) { + continue + } + if dropEPREL && strings.HasPrefix(strings.ToLower(canon), "eprel") { + continue + } + if isDimensionKey(canon) && isZeroishString(s) { + continue + } + if _, exists := out[canon]; exists && (canon != key) { + continue + } + out[canon] = SanitizeOutput(s) + } + return out +} + +func canonicalizeAttrKey(k string) string { + compact := strings.ToLower(strings.TrimSpace(k)) + compact = strings.ReplaceAll(compact, "-", "_") + compact = strings.ReplaceAll(compact, " ", "_") + noUnderscore := strings.ReplaceAll(compact, "_", "") + switch noUnderscore { + case "netwidth", "width", "sirina": + return "width" + case "netheight", "height", "visina": + return "height" + case "netdepth", "depth", "globina": + return "depth" + case "netmass", "weight", "mass", "teza": + return "weight" + case "productmodel", "model": + return "product_model" + case "energijskirazred", "energyclass": + return "energy_class" + default: + return compact + } +} + func parseHTMLSpecs(dst map[string]any, s string) { s = capSpecInput(s) matches := htmlLiRe.FindAllStringSubmatch(s, maxSpecPairs) @@ -126,8 +229,9 @@ func parseHTMLSpecs(dst map[string]any, s string) { continue } key, val := splitLabelValue(text) - if key != "" && val != "" { - dst[attributeKeyFromLabel(key)] = SanitizeOutput(val) + attrKey := attributeKeyFromLabel(key) + if attrKey != "" && val != "" && !isReservedProductKey(attrKey) && !isInvalidAttributeKey(attrKey) { + dst[attrKey] = SanitizeOutput(val) } } if len(matches) == 0 { @@ -198,10 +302,10 @@ func parseCSVLikeSpecs(dst map[string]any, s string) { if len(m) < 3 { continue } - key := strings.TrimSpace(m[1]) + key := attributeKeyFromLabel(strings.TrimSpace(m[1])) val := strings.TrimSpace(m[2]) - if key != "" && val != "" { - dst[attributeKeyFromLabel(key)] = SanitizeOutput(val) + if key != "" && val != "" && !isReservedProductKey(key) && !isInvalidAttributeKey(key) { + dst[key] = SanitizeOutput(val) } } } diff --git a/apps/api/internal/processing/steps.go b/apps/api/internal/processing/steps.go index 60eed57..40a5248 100644 --- a/apps/api/internal/processing/steps.go +++ b/apps/api/internal/processing/steps.go @@ -75,8 +75,9 @@ func (e *Engine) RunSteps(ctx context.Context, companyID string, in ProductInput } } attrs = parsed - out.Attributes = attrs - out.ProcessedAttributes = attrs + out.Attributes = SanitizeProductAttributes(attrs) + out.ProcessedAttributes = out.Attributes + attrs = out.Attributes out.FieldSources["attributes"] = "specifications" appendStepLog(out.GPTResponse, StepParseSpecs, map[string]any{ "count": len(attrs), @@ -100,12 +101,15 @@ func (e *Engine) RunSteps(ctx context.Context, companyID string, in ProductInput in.Description, ) out.Category = stringFromAny(normalized["category"]) - // Promote filled scalar fields into attributes when useful - promote := []string{"brand", "width", "height", "depth", "weight", "gtin", "stock_status"} + // Promote characteristic fields only — never core product identity/content + // (those belong on the V1 item root: title, description, ean, images, …). + promote := []string{"brand", "width", "height", "depth", "weight", "product_model", "warranty"} for _, f := range in.StandardFields { - if f.Key != "" { - promote = append(promote, f.Key) + k := strings.TrimSpace(f.Key) + if k == "" || isReservedProductKey(k) || isInvalidAttributeKey(k) { + continue } + promote = append(promote, k) } seen := map[string]bool{} for _, k := range promote { @@ -120,6 +124,7 @@ func (e *Engine) RunSteps(ctx context.Context, companyID string, in ProductInput out.FieldSources[k] = "fill_fields" } } + attrs = SanitizeProductAttributes(attrs) out.Attributes = attrs out.ProcessedAttributes = attrs appendStepLog(out.GPTResponse, StepFillFields, map[string]any{ @@ -411,7 +416,9 @@ func (e *Engine) RunSteps(ctx context.Context, companyID string, in ProductInput if out.Attributes == nil { out.Attributes = map[string]any{} } - if out.ProcessedAttributes == nil { + out.Attributes = SanitizeProductAttributes(out.Attributes) + out.ProcessedAttributes = SanitizeProductAttributes(out.ProcessedAttributes) + if len(out.ProcessedAttributes) == 0 { out.ProcessedAttributes = out.Attributes } if out.AIProviderMode == "" { diff --git a/apps/api/internal/processing/v1_legacy.go b/apps/api/internal/processing/v1_legacy.go index af3b09b..8c40e5d 100644 --- a/apps/api/internal/processing/v1_legacy.go +++ b/apps/api/internal/processing/v1_legacy.go @@ -291,6 +291,9 @@ func (p *Pipeline) LoadV1ProcessJobItems(ctx context.Context, companyID, jobID u description = nil } + eprelVal := extractEPRELFromAttrs(attrs) + attrs = SanitizeV1ProcessAttributes(attrs) + item := V1ProcessJobItem{ "ean": ean, "status": MapV1JobItemStatus(itemStatus, true), @@ -303,7 +306,7 @@ func (p *Pipeline) LoadV1ProcessJobItems(ctx context.Context, companyID, jobID u "attributes": nil, "main_image": nil, "more_images": nil, - "eprel": extractEPRELFromAttrs(attrs), + "eprel": eprelVal, } applyV1ProcessItemIDs(item, processedID, rawProductID) if itemError != nil && *itemError != "" {