diff --git a/apps/api/internal/httpapi/v1_openapi.go b/apps/api/internal/httpapi/v1_openapi.go
index ffc5536..3ac2a15 100644
--- a/apps/api/internal/httpapi/v1_openapi.go
+++ b/apps/api/internal/httpapi/v1_openapi.go
@@ -6296,13 +6296,9 @@ components:
type: string
nullable: true
description:
+ type: string
nullable: true
- oneOf:
- - type: string
- - type: array
- items:
- type: string
- attributes:
+ description: Plain-text product description (HTML stripped). attributes:
type: object
additionalProperties: true
nullable: true
@@ -6414,8 +6410,7 @@ components:
title: Sony WH-1000XM5 Wireless Noise Cancelling Headphones Black
meta_title: Sony WH-1000XM5 | Noise Cancelling Headphones
meta_description: Industry-leading noise cancellation with up to 30 hours battery life.
- description:
- - Industry-leading noise cancellation with up to 30 hours battery life.
+ description: Industry-leading noise cancellation with up to 30 hours battery life.
attributes:
color: Black
brand: Sony
diff --git a/apps/api/internal/processing/pipeline_steps_test.go b/apps/api/internal/processing/pipeline_steps_test.go
index 26bad9d..8bd1a3d 100644
--- a/apps/api/internal/processing/pipeline_steps_test.go
+++ b/apps/api/internal/processing/pipeline_steps_test.go
@@ -90,6 +90,50 @@ func TestSanitizeProductAttributes(t *testing.T) {
}
}
+func TestFilterAttributesByAllowed(t *testing.T) {
+ in := map[string]any{
+ "brand": "Ostalo",
+ "zavora": "Mehanska",
+ "vzmetenje": "Spredaj",
+ "nosilnost": "40 kg",
+ "width": "0.6 m",
+ }
+ allowed := map[string]struct{}{
+ "nosilnost": {},
+ "barva": {},
+ }
+ got := FilterAttributesByAllowed(in, allowed)
+ if got["brand"] != "Ostalo" || got["width"] != "0.6 m" {
+ t.Fatalf("core keys missing: %v", got)
+ }
+ if got["nosilnost"] != "40 kg" {
+ t.Fatalf("allowed key missing: %v", got)
+ }
+ if _, ok := got["zavora"]; ok {
+ t.Fatalf("junk spec should be dropped: %v", got)
+ }
+ if _, ok := got["vzmetenje"]; ok {
+ t.Fatalf("junk spec should be dropped: %v", got)
+ }
+ unchanged := FilterAttributesByAllowed(in, nil)
+ if unchanged["zavora"] != "Mehanska" {
+ t.Fatalf("nil allowlist should keep all: %v", unchanged)
+ }
+}
+
+func TestV1PlainDescription(t *testing.T) {
+ got := v1PlainDescription("Hello
World
& more
bold")
+ if strings.Contains(got, "<") {
+ t.Fatalf("html should be stripped: %q", got)
+ }
+ if !strings.Contains(got, "Hello") || !strings.Contains(got, "World") || !strings.Contains(got, "bold") {
+ t.Fatalf("got=%q", got)
+ }
+ if !strings.Contains(got, "& more") {
+ t.Fatalf("expected unescaped amp: %q", got)
+ }
+}
+
func TestFillMissingFields_brandAndDims(t *testing.T) {
m := FillMissingFields(map[string]any{
"name": "Nike Air 30x20x10 cm",
diff --git a/apps/api/internal/processing/specs.go b/apps/api/internal/processing/specs.go
index ba1adbe..6e71759 100644
--- a/apps/api/internal/processing/specs.go
+++ b/apps/api/internal/processing/specs.go
@@ -146,6 +146,14 @@ func isInvalidAttributeKey(k string) bool {
return !hasLetter
}
+// coreCharacteristicAttrKeys are always allowed on V1 process items even when
+// the company has no matching attributes row (common dims/brand/model).
+var coreCharacteristicAttrKeys = map[string]struct{}{
+ "brand": {}, "product_model": {}, "warranty": {},
+ "width": {}, "height": {}, "depth": {}, "weight": {},
+ "energy_class": {}, "color": {}, "material": {},
+}
+
// SanitizeProductAttributes keeps characteristic attrs for API/storage and drops
// core product fields that belong on the V1 item root (title, description, images, …).
func SanitizeProductAttributes(attrs map[string]any) map[string]any {
@@ -153,11 +161,48 @@ func SanitizeProductAttributes(attrs map[string]any) map[string]any {
}
// SanitizeV1ProcessAttributes is the stricter poll projection: also drops eprel_*
-// keys (those are exposed on item.eprel).
+// keys (those are exposed on item.eprel). Without an allowlist, feed junk specs
+// (zavora, vzmetenje, …) can remain — prefer SanitizeV1ProcessAttributesAllowed.
func SanitizeV1ProcessAttributes(attrs map[string]any) map[string]any {
return sanitizeProductAttributes(attrs, true)
}
+// SanitizeV1ProcessAttributesAllowed sanitizes then keeps only core characteristic
+// keys plus company attribute_key values (canonicalized).
+func SanitizeV1ProcessAttributesAllowed(attrs map[string]any, allowed map[string]struct{}) map[string]any {
+ return FilterAttributesByAllowed(sanitizeProductAttributes(attrs, true), allowed)
+}
+
+// FilterAttributesByAllowed keeps coreCharacteristicAttrKeys plus keys present in
+// allowed (after canonicalizeAttrKey). When allowed is nil, returns attrs unchanged.
+func FilterAttributesByAllowed(attrs map[string]any, allowed map[string]struct{}) map[string]any {
+ if len(attrs) == 0 {
+ return map[string]any{}
+ }
+ if allowed == nil {
+ return attrs
+ }
+ out := make(map[string]any, len(attrs))
+ for k, v := range attrs {
+ canon := canonicalizeAttrKey(k)
+ if canon == "" {
+ continue
+ }
+ if _, ok := coreCharacteristicAttrKeys[canon]; ok {
+ out[canon] = v
+ continue
+ }
+ if _, ok := allowed[canon]; ok {
+ out[canon] = v
+ continue
+ }
+ if _, ok := allowed[strings.ToLower(strings.TrimSpace(k))]; ok {
+ out[canon] = v
+ }
+ }
+ return out
+}
+
func sanitizeProductAttributes(attrs map[string]any, dropEPREL bool) map[string]any {
if len(attrs) == 0 {
return map[string]any{}
diff --git a/apps/api/internal/processing/v1_legacy.go b/apps/api/internal/processing/v1_legacy.go
index 8c40e5d..0f62643 100644
--- a/apps/api/internal/processing/v1_legacy.go
+++ b/apps/api/internal/processing/v1_legacy.go
@@ -4,6 +4,8 @@ import (
"context"
"encoding/json"
"fmt"
+ "html"
+ "regexp"
"strings"
"github.com/descrybe/descrybe-v2/apps/api/internal/catalog"
@@ -12,6 +14,12 @@ import (
var v1PartialSteps = []string{"category", "title", "description", "attributes"}
+const v1MetaDescriptionMaxChars = 155
+
+var (
+ v1BreakTagRe = regexp.MustCompile(`(?i)
`)
+ v1BlockEndRe = regexp.MustCompile(`(?i)(p|div|li|h[1-6]|tr)>`)
+)
// ParseV1ProcessingType mirrors legacy parseV1ProcessingTypeFromBody.
// Accepts string ("full" / step), JSON array of steps, or nil (defaults to full).
func ParseV1ProcessingType(raw any) (storageValue string, responseValue any, err error) {
@@ -197,6 +205,7 @@ func (p *Pipeline) LoadV1ProcessJobItems(ctx context.Context, companyID, jobID u
if p == nil || p.Pool == nil {
return nil, fmt.Errorf("pipeline not configured")
}
+ allowedAttrs := loadCompanyAttributeKeySet(ctx, p, companyID)
rows, err := p.Pool.Query(ctx, `
SELECT
COALESCE(r.gtin, p.product_id, '') AS ean,
@@ -284,15 +293,28 @@ func (p *Pipeline) LoadV1ProcessJobItems(ctx context.Context, companyID, jobID u
_ = json.Unmarshal(rawJSON, &rawData)
main, more := catalog.ExtractProductImages(mapped, rawData)
+ plainDesc := ""
+ if descTxt != nil {
+ plainDesc = v1PlainDescription(*descTxt)
+ }
var description any
- if descTxt != nil && strings.TrimSpace(*descTxt) != "" {
- description = []string{*descTxt}
+ if plainDesc != "" {
+ description = plainDesc
} else {
description = nil
}
eprelVal := extractEPRELFromAttrs(attrs)
- attrs = SanitizeV1ProcessAttributes(attrs)
+ attrs = SanitizeV1ProcessAttributesAllowed(attrs, allowedAttrs)
+
+ metaTitleOut := nullIfEmptyPtr(metaTitle)
+ if metaTitleOut == nil {
+ metaTitleOut = nullIfEmptyPtr(title)
+ }
+ metaDescOut := nullIfEmptyPtr(metaDesc)
+ if metaDescOut == nil && plainDesc != "" {
+ metaDescOut = truncateRunes(plainDesc, v1MetaDescriptionMaxChars)
+ }
item := V1ProcessJobItem{
"ean": ean,
@@ -300,8 +322,8 @@ func (p *Pipeline) LoadV1ProcessJobItems(ctx context.Context, companyID, jobID u
"category": nullIfEmptyPtr(category),
"category_name": nullIfEmptyPtr(categoryName),
"title": nullIfEmptyPtr(title),
- "meta_title": nullIfEmptyPtr(metaTitle),
- "meta_description": nullIfEmptyPtr(metaDesc),
+ "meta_title": metaTitleOut,
+ "meta_description": metaDescOut,
"description": description,
"attributes": nil,
"main_image": nil,
@@ -336,6 +358,61 @@ func nullIfEmptyPtr(s *string) any {
return *s
}
+// loadCompanyAttributeKeySet returns canonicalized attribute_key values for the company.
+// On query failure returns an empty (non-nil) set so FilterAttributesByAllowed still
+// restricts to coreCharacteristicAttrKeys only.
+func loadCompanyAttributeKeySet(ctx context.Context, p *Pipeline, companyID uuid.UUID) map[string]struct{} {
+ out := map[string]struct{}{}
+ if p == nil || p.Pool == nil {
+ return out
+ }
+ rows, err := p.Pool.Query(ctx, `
+ SELECT attribute_key
+ FROM attributes
+ WHERE company_id = $1 AND COALESCE(attribute_key, '') <> ''`, companyID)
+ if err != nil {
+ return out
+ }
+ defer rows.Close()
+ for rows.Next() {
+ var key string
+ if err := rows.Scan(&key); err != nil {
+ continue
+ }
+ canon := canonicalizeAttrKey(key)
+ if canon == "" {
+ continue
+ }
+ out[canon] = struct{}{}
+ out[strings.ToLower(strings.TrimSpace(key))] = struct{}{}
+ }
+ return out
+}
+
+// v1PlainDescription normalizes feed HTML into a single plain-text string for the
+// legacy process poll (description is a string, not a one-element array).
+func v1PlainDescription(s string) string {
+ s = strings.TrimSpace(s)
+ if s == "" {
+ return ""
+ }
+ s = html.UnescapeString(s)
+ s = strings.ReplaceAll(s, "\u00a0", " ")
+ s = v1BreakTagRe.ReplaceAllString(s, "\n")
+ s = v1BlockEndRe.ReplaceAllString(s, "\n")
+ s = specsHTMLTagRe.ReplaceAllString(s, " ")
+ s = html.UnescapeString(s)
+ lines := strings.Split(s, "\n")
+ kept := make([]string, 0, len(lines))
+ for _, line := range lines {
+ line = strings.Join(strings.Fields(line), " ")
+ if line != "" {
+ kept = append(kept, line)
+ }
+ }
+ return strings.TrimSpace(strings.Join(kept, "\n"))
+}
+
func extractEPRELFromAttrs(attrs map[string]any) any {
if attrs == nil {
return nil
diff --git a/apps/api/internal/processing/v1_legacy_test.go b/apps/api/internal/processing/v1_legacy_test.go
index 096f7f4..b2b0b4c 100644
--- a/apps/api/internal/processing/v1_legacy_test.go
+++ b/apps/api/internal/processing/v1_legacy_test.go
@@ -2,6 +2,7 @@ package processing
import (
"encoding/json"
+ "strings"
"testing"
"github.com/google/uuid"
@@ -76,7 +77,7 @@ func TestProjectV1ProcessJobItemsPartial(t *testing.T) {
"raw_product_id": "cccccccc-cccc-cccc-cccc-cccccccccccc",
"status": "processed",
"title": "T", "meta_title": "MT",
- "description": []string{"D"}, "attributes": map[string]any{"brand": "X"},
+ "description": "D", "attributes": map[string]any{"brand": "X"},
"main_image": "https://example.com/a.jpg", "more_images": []string{"https://example.com/b.jpg"},
"eprel": nil, "category": "cat", "category_name": "Cat",
}}
@@ -107,6 +108,16 @@ func TestProjectV1ProcessJobItemsPartial(t *testing.T) {
}
}
+func TestV1PlainDescriptionStandalone(t *testing.T) {
+ got := v1PlainDescription("Line1
Line2 & ok")
+ if strings.Contains(got, "<") {
+ t.Fatalf("html left: %q", got)
+ }
+ if !strings.Contains(got, "Line1") || !strings.Contains(got, "Line2") || !strings.Contains(got, "& ok") {
+ t.Fatalf("got=%q", got)
+ }
+}
+
func TestApplyV1ProcessItemIDsDualMode(t *testing.T) {
processed := mustParseTestUUID(t, "bbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb")
raw := mustParseTestUUID(t, "cccccccc-cccc-cccc-cccc-cccccccccccc")