From 96f8c1115c7ab5d18c757725380b0d2633f8f8a7 Mon Sep 17 00:00:00 2001 From: GreenEclipse Date: Sun, 16 Aug 2026 12:47:06 +0200 Subject: [PATCH] update --- apps/api/internal/httpapi/v1_openapi.go | 11 +-- .../processing/pipeline_steps_test.go | 44 ++++++++++ apps/api/internal/processing/specs.go | 47 +++++++++- apps/api/internal/processing/v1_legacy.go | 87 +++++++++++++++++-- .../api/internal/processing/v1_legacy_test.go | 13 ++- 5 files changed, 187 insertions(+), 15 deletions(-) diff --git a/apps/api/internal/httpapi/v1_openapi.go b/apps/api/internal/httpapi/v1_openapi.go index ffc5536..3ac2a15 100644 --- a/apps/api/internal/httpapi/v1_openapi.go +++ b/apps/api/internal/httpapi/v1_openapi.go @@ -6296,13 +6296,9 @@ components: type: string nullable: true description: + type: string nullable: true - oneOf: - - type: string - - type: array - items: - type: string - attributes: + description: Plain-text product description (HTML stripped). attributes: type: object additionalProperties: true nullable: true @@ -6414,8 +6410,7 @@ components: title: Sony WH-1000XM5 Wireless Noise Cancelling Headphones Black meta_title: Sony WH-1000XM5 | Noise Cancelling Headphones meta_description: Industry-leading noise cancellation with up to 30 hours battery life. - description: - - Industry-leading noise cancellation with up to 30 hours battery life. + description: Industry-leading noise cancellation with up to 30 hours battery life. attributes: color: Black brand: Sony diff --git a/apps/api/internal/processing/pipeline_steps_test.go b/apps/api/internal/processing/pipeline_steps_test.go index 26bad9d..8bd1a3d 100644 --- a/apps/api/internal/processing/pipeline_steps_test.go +++ b/apps/api/internal/processing/pipeline_steps_test.go @@ -90,6 +90,50 @@ func TestSanitizeProductAttributes(t *testing.T) { } } +func TestFilterAttributesByAllowed(t *testing.T) { + in := map[string]any{ + "brand": "Ostalo", + "zavora": "Mehanska", + "vzmetenje": "Spredaj", + "nosilnost": "40 kg", + "width": "0.6 m", + } + allowed := map[string]struct{}{ + "nosilnost": {}, + "barva": {}, + } + got := FilterAttributesByAllowed(in, allowed) + if got["brand"] != "Ostalo" || got["width"] != "0.6 m" { + t.Fatalf("core keys missing: %v", got) + } + if got["nosilnost"] != "40 kg" { + t.Fatalf("allowed key missing: %v", got) + } + if _, ok := got["zavora"]; ok { + t.Fatalf("junk spec should be dropped: %v", got) + } + if _, ok := got["vzmetenje"]; ok { + t.Fatalf("junk spec should be dropped: %v", got) + } + unchanged := FilterAttributesByAllowed(in, nil) + if unchanged["zavora"] != "Mehanska" { + t.Fatalf("nil allowlist should keep all: %v", unchanged) + } +} + +func TestV1PlainDescription(t *testing.T) { + got := v1PlainDescription("Hello
World
& more

bold") + if strings.Contains(got, "<") { + t.Fatalf("html should be stripped: %q", got) + } + if !strings.Contains(got, "Hello") || !strings.Contains(got, "World") || !strings.Contains(got, "bold") { + t.Fatalf("got=%q", got) + } + if !strings.Contains(got, "& more") { + t.Fatalf("expected unescaped amp: %q", got) + } +} + func TestFillMissingFields_brandAndDims(t *testing.T) { m := FillMissingFields(map[string]any{ "name": "Nike Air 30x20x10 cm", diff --git a/apps/api/internal/processing/specs.go b/apps/api/internal/processing/specs.go index ba1adbe..6e71759 100644 --- a/apps/api/internal/processing/specs.go +++ b/apps/api/internal/processing/specs.go @@ -146,6 +146,14 @@ func isInvalidAttributeKey(k string) bool { return !hasLetter } +// coreCharacteristicAttrKeys are always allowed on V1 process items even when +// the company has no matching attributes row (common dims/brand/model). +var coreCharacteristicAttrKeys = map[string]struct{}{ + "brand": {}, "product_model": {}, "warranty": {}, + "width": {}, "height": {}, "depth": {}, "weight": {}, + "energy_class": {}, "color": {}, "material": {}, +} + // SanitizeProductAttributes keeps characteristic attrs for API/storage and drops // core product fields that belong on the V1 item root (title, description, images, …). func SanitizeProductAttributes(attrs map[string]any) map[string]any { @@ -153,11 +161,48 @@ func SanitizeProductAttributes(attrs map[string]any) map[string]any { } // SanitizeV1ProcessAttributes is the stricter poll projection: also drops eprel_* -// keys (those are exposed on item.eprel). +// keys (those are exposed on item.eprel). Without an allowlist, feed junk specs +// (zavora, vzmetenje, …) can remain — prefer SanitizeV1ProcessAttributesAllowed. func SanitizeV1ProcessAttributes(attrs map[string]any) map[string]any { return sanitizeProductAttributes(attrs, true) } +// SanitizeV1ProcessAttributesAllowed sanitizes then keeps only core characteristic +// keys plus company attribute_key values (canonicalized). +func SanitizeV1ProcessAttributesAllowed(attrs map[string]any, allowed map[string]struct{}) map[string]any { + return FilterAttributesByAllowed(sanitizeProductAttributes(attrs, true), allowed) +} + +// FilterAttributesByAllowed keeps coreCharacteristicAttrKeys plus keys present in +// allowed (after canonicalizeAttrKey). When allowed is nil, returns attrs unchanged. +func FilterAttributesByAllowed(attrs map[string]any, allowed map[string]struct{}) map[string]any { + if len(attrs) == 0 { + return map[string]any{} + } + if allowed == nil { + return attrs + } + out := make(map[string]any, len(attrs)) + for k, v := range attrs { + canon := canonicalizeAttrKey(k) + if canon == "" { + continue + } + if _, ok := coreCharacteristicAttrKeys[canon]; ok { + out[canon] = v + continue + } + if _, ok := allowed[canon]; ok { + out[canon] = v + continue + } + if _, ok := allowed[strings.ToLower(strings.TrimSpace(k))]; ok { + out[canon] = v + } + } + return out +} + func sanitizeProductAttributes(attrs map[string]any, dropEPREL bool) map[string]any { if len(attrs) == 0 { return map[string]any{} diff --git a/apps/api/internal/processing/v1_legacy.go b/apps/api/internal/processing/v1_legacy.go index 8c40e5d..0f62643 100644 --- a/apps/api/internal/processing/v1_legacy.go +++ b/apps/api/internal/processing/v1_legacy.go @@ -4,6 +4,8 @@ import ( "context" "encoding/json" "fmt" + "html" + "regexp" "strings" "github.com/descrybe/descrybe-v2/apps/api/internal/catalog" @@ -12,6 +14,12 @@ import ( var v1PartialSteps = []string{"category", "title", "description", "attributes"} +const v1MetaDescriptionMaxChars = 155 + +var ( + v1BreakTagRe = regexp.MustCompile(`(?i)`) + v1BlockEndRe = regexp.MustCompile(`(?i)`) +) // ParseV1ProcessingType mirrors legacy parseV1ProcessingTypeFromBody. // Accepts string ("full" / step), JSON array of steps, or nil (defaults to full). func ParseV1ProcessingType(raw any) (storageValue string, responseValue any, err error) { @@ -197,6 +205,7 @@ func (p *Pipeline) LoadV1ProcessJobItems(ctx context.Context, companyID, jobID u if p == nil || p.Pool == nil { return nil, fmt.Errorf("pipeline not configured") } + allowedAttrs := loadCompanyAttributeKeySet(ctx, p, companyID) rows, err := p.Pool.Query(ctx, ` SELECT COALESCE(r.gtin, p.product_id, '') AS ean, @@ -284,15 +293,28 @@ func (p *Pipeline) LoadV1ProcessJobItems(ctx context.Context, companyID, jobID u _ = json.Unmarshal(rawJSON, &rawData) main, more := catalog.ExtractProductImages(mapped, rawData) + plainDesc := "" + if descTxt != nil { + plainDesc = v1PlainDescription(*descTxt) + } var description any - if descTxt != nil && strings.TrimSpace(*descTxt) != "" { - description = []string{*descTxt} + if plainDesc != "" { + description = plainDesc } else { description = nil } eprelVal := extractEPRELFromAttrs(attrs) - attrs = SanitizeV1ProcessAttributes(attrs) + attrs = SanitizeV1ProcessAttributesAllowed(attrs, allowedAttrs) + + metaTitleOut := nullIfEmptyPtr(metaTitle) + if metaTitleOut == nil { + metaTitleOut = nullIfEmptyPtr(title) + } + metaDescOut := nullIfEmptyPtr(metaDesc) + if metaDescOut == nil && plainDesc != "" { + metaDescOut = truncateRunes(plainDesc, v1MetaDescriptionMaxChars) + } item := V1ProcessJobItem{ "ean": ean, @@ -300,8 +322,8 @@ func (p *Pipeline) LoadV1ProcessJobItems(ctx context.Context, companyID, jobID u "category": nullIfEmptyPtr(category), "category_name": nullIfEmptyPtr(categoryName), "title": nullIfEmptyPtr(title), - "meta_title": nullIfEmptyPtr(metaTitle), - "meta_description": nullIfEmptyPtr(metaDesc), + "meta_title": metaTitleOut, + "meta_description": metaDescOut, "description": description, "attributes": nil, "main_image": nil, @@ -336,6 +358,61 @@ func nullIfEmptyPtr(s *string) any { return *s } +// loadCompanyAttributeKeySet returns canonicalized attribute_key values for the company. +// On query failure returns an empty (non-nil) set so FilterAttributesByAllowed still +// restricts to coreCharacteristicAttrKeys only. +func loadCompanyAttributeKeySet(ctx context.Context, p *Pipeline, companyID uuid.UUID) map[string]struct{} { + out := map[string]struct{}{} + if p == nil || p.Pool == nil { + return out + } + rows, err := p.Pool.Query(ctx, ` + SELECT attribute_key + FROM attributes + WHERE company_id = $1 AND COALESCE(attribute_key, '') <> ''`, companyID) + if err != nil { + return out + } + defer rows.Close() + for rows.Next() { + var key string + if err := rows.Scan(&key); err != nil { + continue + } + canon := canonicalizeAttrKey(key) + if canon == "" { + continue + } + out[canon] = struct{}{} + out[strings.ToLower(strings.TrimSpace(key))] = struct{}{} + } + return out +} + +// v1PlainDescription normalizes feed HTML into a single plain-text string for the +// legacy process poll (description is a string, not a one-element array). +func v1PlainDescription(s string) string { + s = strings.TrimSpace(s) + if s == "" { + return "" + } + s = html.UnescapeString(s) + s = strings.ReplaceAll(s, "\u00a0", " ") + s = v1BreakTagRe.ReplaceAllString(s, "\n") + s = v1BlockEndRe.ReplaceAllString(s, "\n") + s = specsHTMLTagRe.ReplaceAllString(s, " ") + s = html.UnescapeString(s) + lines := strings.Split(s, "\n") + kept := make([]string, 0, len(lines)) + for _, line := range lines { + line = strings.Join(strings.Fields(line), " ") + if line != "" { + kept = append(kept, line) + } + } + return strings.TrimSpace(strings.Join(kept, "\n")) +} + func extractEPRELFromAttrs(attrs map[string]any) any { if attrs == nil { return nil diff --git a/apps/api/internal/processing/v1_legacy_test.go b/apps/api/internal/processing/v1_legacy_test.go index 096f7f4..b2b0b4c 100644 --- a/apps/api/internal/processing/v1_legacy_test.go +++ b/apps/api/internal/processing/v1_legacy_test.go @@ -2,6 +2,7 @@ package processing import ( "encoding/json" + "strings" "testing" "github.com/google/uuid" @@ -76,7 +77,7 @@ func TestProjectV1ProcessJobItemsPartial(t *testing.T) { "raw_product_id": "cccccccc-cccc-cccc-cccc-cccccccccccc", "status": "processed", "title": "T", "meta_title": "MT", - "description": []string{"D"}, "attributes": map[string]any{"brand": "X"}, + "description": "D", "attributes": map[string]any{"brand": "X"}, "main_image": "https://example.com/a.jpg", "more_images": []string{"https://example.com/b.jpg"}, "eprel": nil, "category": "cat", "category_name": "Cat", }} @@ -107,6 +108,16 @@ func TestProjectV1ProcessJobItemsPartial(t *testing.T) { } } +func TestV1PlainDescriptionStandalone(t *testing.T) { + got := v1PlainDescription("Line1
Line2 & ok") + if strings.Contains(got, "<") { + t.Fatalf("html left: %q", got) + } + if !strings.Contains(got, "Line1") || !strings.Contains(got, "Line2") || !strings.Contains(got, "& ok") { + t.Fatalf("got=%q", got) + } +} + func TestApplyV1ProcessItemIDsDualMode(t *testing.T) { processed := mustParseTestUUID(t, "bbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb") raw := mustParseTestUUID(t, "cccccccc-cccc-cccc-cccc-cccccccccccc")