greeneclipse/descrybe
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fix

Browse Source
This commit is contained in:
greeneclipse
2026-08-16 12:23:14 +02:00
parent 52fb129957
commit 1d3a9d5abd
4 changed files with 192 additions and 18 deletions
Download Patch File Download Diff File Expand all files Collapse all files
apps/api/internal/processing/pipeline_steps_test.go
+60
View File
@@ -30,6 +30,66 @@ func TestParseSpecifications_htmlAndCSV(t *testing.T) {
}
}
func TestParseSpecifications_skipsGarbageAndReservedKeys(t *testing.T) {
attrs := ParseSpecifications(map[string]any{
":": "true",
"zavora": "disc",
"name": "Should not appear",
"description": "Should not appear",
"brand": "Vox",
})
if _, ok := attrs[":"]; ok {
t.Fatalf("colon key must be dropped: %v", attrs)
}
if _, ok := attrs["name"]; ok {
t.Fatalf("name must be reserved: %v", attrs)
}
if attrs["zavora"] != "disc" {
t.Fatalf("zavora=%v", attrs["zavora"])
}
if attrs["brand"] != "Vox" {
t.Fatalf("brand=%v", attrs["brand"])
}
}
func TestSanitizeProductAttributes(t *testing.T) {
got := SanitizeProductAttributes(map[string]any{
"name": "TV Mount",
"description": "<br>html",
"gtin": "123",
"id": "102544",
"purchaseprice": "10",
"main_image": "https://x",
":": "true",
"brand": "Ostalo",
"productmodel": "W53070",
"netwidth": "0.6 m",
"width": "0.6 m",
"warranty": "60 mesecev",
})
if _, ok := got["name"]; ok {
t.Fatalf("name must be stripped: %v", got)
}
if _, ok := got["description"]; ok {
t.Fatalf("description must be stripped: %v", got)
}
if _, ok := got[":"]; ok {
t.Fatalf("invalid key must be stripped: %v", got)
}
if got["brand"] != "Ostalo" {
t.Fatalf("brand=%v", got["brand"])
}
if got["product_model"] != "W53070" {
t.Fatalf("product_model=%v", got["product_model"])
}
if got["width"] != "0.6 m" {
t.Fatalf("width=%v", got["width"])
}
if _, ok := got["netwidth"]; ok {
t.Fatalf("alias netwidth should collapse to width: %v", got)
}
}
func TestFillMissingFields_brandAndDims(t *testing.T) {
m := FillMissingFields(map[string]any{
"name": "Nike Air 30x20x10 cm",
apps/api/internal/processing/specs.go
+114 -10
View File
@@ -44,8 +44,16 @@ func parseSpecsInto(dst map[string]any, v any) {
// Already structured attributes / grouped specs
if looksLikeAttrMap(t) {
for k, val := range t {
lk := strings.ToLower(strings.TrimSpace(k))
if isReservedProductKey(lk) || isInvalidAttributeKey(k) {
continue
}
if s := stringifySpecValue(val); s != "" {
dst[SanitizeOutput(k)] = s
key := SanitizeOutput(k)
if key == "" || isInvalidAttributeKey(key) {
continue
}
dst[key] = s
} else if nested, ok := val.(map[string]any); ok {
parseSpecsInto(dst, nested)
}
@@ -58,7 +66,7 @@ func parseSpecsInto(dst map[string]any, v any) {
parseSpecsInto(dst, val)
continue
}
if s := stringifySpecValue(val); s != "" && !isReservedProductKey(lk) {
if s := stringifySpecValue(val); s != "" && !isReservedProductKey(lk) && !isInvalidAttributeKey(k) {
dst[SanitizeOutput(k)] = s
}
}
@@ -105,15 +113,110 @@ func looksLikeAttrMap(m map[string]any) bool {
}
func isReservedProductKey(k string) bool {
switch k {
case "name", "title", "description", "gtin", "ean", "brand", "category",
"price", "image", "stock", "eprel_id", "specifications", "raw", "mapped":
switch strings.ToLower(strings.TrimSpace(k)) {
case "name", "title", "description", "gtin", "ean", "category", "category_unique_id",
"price", "purchaseprice", "purchase_price", "sellingprice", "selling_price",
"image", "main_image", "mainimage", "moreimages", "more_images", "images",
"image_url", "imageurl", "image_link", "imagelink", "additional_image_urls",
"additional_image_link", "videourl", "video_url",
"stock", "stockstatus", "stock_status", "availability",
"id", "sku", "officiallink", "official_link", "service",
"specifications", "specs", "specification",
"raw", "mapped", "search":
return true
default:
return false
}
}
// isInvalidAttributeKey rejects garbage keys from bad feed specs (e.g. ":").
func isInvalidAttributeKey(k string) bool {
k = strings.TrimSpace(k)
if k == "" || k == ":" || k == "=" || k == "-" || k == "_" {
return true
}
// Must contain at least one letter after sanitize.
hasLetter := false
for _, r := range strings.ToLower(k) {
if r >= 'a' && r <= 'z' {
hasLetter = true
break
}
}
return !hasLetter
}
// SanitizeProductAttributes keeps characteristic attrs for API/storage and drops
// core product fields that belong on the V1 item root (title, description, images, …).
func SanitizeProductAttributes(attrs map[string]any) map[string]any {
return sanitizeProductAttributes(attrs, false)
}
// SanitizeV1ProcessAttributes is the stricter poll projection: also drops eprel_*
// keys (those are exposed on item.eprel).
func SanitizeV1ProcessAttributes(attrs map[string]any) map[string]any {
return sanitizeProductAttributes(attrs, true)
}
func sanitizeProductAttributes(attrs map[string]any, dropEPREL bool) map[string]any {
if len(attrs) == 0 {
return map[string]any{}
}
out := make(map[string]any, len(attrs))
for k, v := range attrs {
key := strings.TrimSpace(SanitizeOutput(k))
if key == "" || isReservedProductKey(key) || isInvalidAttributeKey(key) {
continue
}
if dropEPREL && strings.HasPrefix(strings.ToLower(key), "eprel") {
continue
}
s := stringifySpecValue(v)
if s == "" || s == "<nil>" {
continue
}
// Prefer canonical dimension/model keys when aliases collide.
canon := canonicalizeAttrKey(key)
if canon == "" || isReservedProductKey(canon) || isInvalidAttributeKey(canon) {
continue
}
if dropEPREL && strings.HasPrefix(strings.ToLower(canon), "eprel") {
continue
}
if isDimensionKey(canon) && isZeroishString(s) {
continue
}
if _, exists := out[canon]; exists && (canon != key) {
continue
}
out[canon] = SanitizeOutput(s)
}
return out
}
func canonicalizeAttrKey(k string) string {
compact := strings.ToLower(strings.TrimSpace(k))
compact = strings.ReplaceAll(compact, "-", "_")
compact = strings.ReplaceAll(compact, " ", "_")
noUnderscore := strings.ReplaceAll(compact, "_", "")
switch noUnderscore {
case "netwidth", "width", "sirina":
return "width"
case "netheight", "height", "visina":
return "height"
case "netdepth", "depth", "globina":
return "depth"
case "netmass", "weight", "mass", "teza":
return "weight"
case "productmodel", "model":
return "product_model"
case "energijskirazred", "energyclass":
return "energy_class"
default:
return compact
}
}
func parseHTMLSpecs(dst map[string]any, s string) {
s = capSpecInput(s)
matches := htmlLiRe.FindAllStringSubmatch(s, maxSpecPairs)
@@ -126,8 +229,9 @@ func parseHTMLSpecs(dst map[string]any, s string) {
continue
}
key, val := splitLabelValue(text)
if key != "" && val != "" {
dst[attributeKeyFromLabel(key)] = SanitizeOutput(val)
attrKey := attributeKeyFromLabel(key)
if attrKey != "" && val != "" && !isReservedProductKey(attrKey) && !isInvalidAttributeKey(attrKey) {
dst[attrKey] = SanitizeOutput(val)
}
}
if len(matches) == 0 {
@@ -198,10 +302,10 @@ func parseCSVLikeSpecs(dst map[string]any, s string) {
if len(m) < 3 {
continue
}
key := strings.TrimSpace(m[1])
key := attributeKeyFromLabel(strings.TrimSpace(m[1]))
val := strings.TrimSpace(m[2])
if key != "" && val != "" {
dst[attributeKeyFromLabel(key)] = SanitizeOutput(val)
if key != "" && val != "" && !isReservedProductKey(key) && !isInvalidAttributeKey(key) {
dst[key] = SanitizeOutput(val)
}
}
}
apps/api/internal/processing/steps.go
+14 -7
View File
@@ -75,8 +75,9 @@ func (e *Engine) RunSteps(ctx context.Context, companyID string, in ProductInput
}
}
attrs = parsed
out.Attributes = attrs
out.ProcessedAttributes = attrs
out.Attributes = SanitizeProductAttributes(attrs)
out.ProcessedAttributes = out.Attributes
attrs = out.Attributes
out.FieldSources["attributes"] = "specifications"
appendStepLog(out.GPTResponse, StepParseSpecs, map[string]any{
"count": len(attrs),
@@ -100,12 +101,15 @@ func (e *Engine) RunSteps(ctx context.Context, companyID string, in ProductInput
in.Description,
)
out.Category = stringFromAny(normalized["category"])
// Promote filled scalar fields into attributes when useful
promote := []string{"brand", "width", "height", "depth", "weight", "gtin", "stock_status"}
// Promote characteristic fields only — never core product identity/content
// (those belong on the V1 item root: title, description, ean, images, …).
promote := []string{"brand", "width", "height", "depth", "weight", "product_model", "warranty"}
for _, f := range in.StandardFields {
if f.Key != "" {
promote = append(promote, f.Key)
k := strings.TrimSpace(f.Key)
if k == "" || isReservedProductKey(k) || isInvalidAttributeKey(k) {
continue
}
promote = append(promote, k)
}
seen := map[string]bool{}
for _, k := range promote {
@@ -120,6 +124,7 @@ func (e *Engine) RunSteps(ctx context.Context, companyID string, in ProductInput
out.FieldSources[k] = "fill_fields"
}
}
attrs = SanitizeProductAttributes(attrs)
out.Attributes = attrs
out.ProcessedAttributes = attrs
appendStepLog(out.GPTResponse, StepFillFields, map[string]any{
@@ -411,7 +416,9 @@ func (e *Engine) RunSteps(ctx context.Context, companyID string, in ProductInput
if out.Attributes == nil {
out.Attributes = map[string]any{}
}
if out.ProcessedAttributes == nil {
out.Attributes = SanitizeProductAttributes(out.Attributes)
out.ProcessedAttributes = SanitizeProductAttributes(out.ProcessedAttributes)
if len(out.ProcessedAttributes) == 0 {
out.ProcessedAttributes = out.Attributes
}
if out.AIProviderMode == "" {
apps/api/internal/processing/v1_legacy.go
+4 -1
View File
@@ -291,6 +291,9 @@ func (p *Pipeline) LoadV1ProcessJobItems(ctx context.Context, companyID, jobID u
description = nil
}
eprelVal := extractEPRELFromAttrs(attrs)
attrs = SanitizeV1ProcessAttributes(attrs)
item := V1ProcessJobItem{
"ean": ean,
"status": MapV1JobItemStatus(itemStatus, true),
@@ -303,7 +306,7 @@ func (p *Pipeline) LoadV1ProcessJobItems(ctx context.Context, companyID, jobID u
"attributes": nil,
"main_image": nil,
"more_images": nil,
"eprel": extractEPRELFromAttrs(attrs),
"eprel": eprelVal,
}
applyV1ProcessItemIDs(item, processedID, rawProductID)
if itemError != nil && *itemError != "" {