update

apps/api/internal/processing/v1_legacy.go

@@ -4,6 +4,8 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
+	"html"
+	"regexp"
 	"strings"
 
 	"github.com/descrybe/descrybe-v2/apps/api/internal/catalog"
@@ -12,6 +14,12 @@ import (
 
 var v1PartialSteps = []string{"category", "title", "description", "attributes"}
 
+const v1MetaDescriptionMaxChars = 155
+
+var (
+	v1BreakTagRe = regexp.MustCompile(`(?i)<br\s*/?>`)
+	v1BlockEndRe = regexp.MustCompile(`(?i)</(p|div|li|h[1-6]|tr)>`)
+)
 // ParseV1ProcessingType mirrors legacy parseV1ProcessingTypeFromBody.
 // Accepts string ("full" / step), JSON array of steps, or nil (defaults to full).
 func ParseV1ProcessingType(raw any) (storageValue string, responseValue any, err error) {
@@ -197,6 +205,7 @@ func (p *Pipeline) LoadV1ProcessJobItems(ctx context.Context, companyID, jobID u
 	if p == nil || p.Pool == nil {
 		return nil, fmt.Errorf("pipeline not configured")
 	}
+	allowedAttrs := loadCompanyAttributeKeySet(ctx, p, companyID)
 	rows, err := p.Pool.Query(ctx, `
 		SELECT
 			COALESCE(r.gtin, p.product_id, '') AS ean,
@@ -284,15 +293,28 @@ func (p *Pipeline) LoadV1ProcessJobItems(ctx context.Context, companyID, jobID u
 		_ = json.Unmarshal(rawJSON, &rawData)
 		main, more := catalog.ExtractProductImages(mapped, rawData)
 
+		plainDesc := ""
+		if descTxt != nil {
+			plainDesc = v1PlainDescription(*descTxt)
+		}
 		var description any
-		if descTxt != nil && strings.TrimSpace(*descTxt) != "" {
-			description = []string{*descTxt}
+		if plainDesc != "" {
+			description = plainDesc
 		} else {
 			description = nil
 		}
 
 		eprelVal := extractEPRELFromAttrs(attrs)
-		attrs = SanitizeV1ProcessAttributes(attrs)
+		attrs = SanitizeV1ProcessAttributesAllowed(attrs, allowedAttrs)
+
+		metaTitleOut := nullIfEmptyPtr(metaTitle)
+		if metaTitleOut == nil {
+			metaTitleOut = nullIfEmptyPtr(title)
+		}
+		metaDescOut := nullIfEmptyPtr(metaDesc)
+		if metaDescOut == nil && plainDesc != "" {
+			metaDescOut = truncateRunes(plainDesc, v1MetaDescriptionMaxChars)
+		}
 
 		item := V1ProcessJobItem{
 			"ean":              ean,
@@ -300,8 +322,8 @@ func (p *Pipeline) LoadV1ProcessJobItems(ctx context.Context, companyID, jobID u
 			"category":         nullIfEmptyPtr(category),
 			"category_name":    nullIfEmptyPtr(categoryName),
 			"title":            nullIfEmptyPtr(title),
-			"meta_title":       nullIfEmptyPtr(metaTitle),
-			"meta_description": nullIfEmptyPtr(metaDesc),
+			"meta_title":       metaTitleOut,
+			"meta_description": metaDescOut,
 			"description":      description,
 			"attributes":       nil,
 			"main_image":       nil,
@@ -336,6 +358,61 @@ func nullIfEmptyPtr(s *string) any {
 	return *s
 }
 
+// loadCompanyAttributeKeySet returns canonicalized attribute_key values for the company.
+// On query failure returns an empty (non-nil) set so FilterAttributesByAllowed still
+// restricts to coreCharacteristicAttrKeys only.
+func loadCompanyAttributeKeySet(ctx context.Context, p *Pipeline, companyID uuid.UUID) map[string]struct{} {
+	out := map[string]struct{}{}
+	if p == nil || p.Pool == nil {
+		return out
+	}
+	rows, err := p.Pool.Query(ctx, `
+		SELECT attribute_key
+		FROM attributes
+		WHERE company_id = $1 AND COALESCE(attribute_key, '') <> ''`, companyID)
+	if err != nil {
+		return out
+	}
+	defer rows.Close()
+	for rows.Next() {
+		var key string
+		if err := rows.Scan(&key); err != nil {
+			continue
+		}
+		canon := canonicalizeAttrKey(key)
+		if canon == "" {
+			continue
+		}
+		out[canon] = struct{}{}
+		out[strings.ToLower(strings.TrimSpace(key))] = struct{}{}
+	}
+	return out
+}
+
+// v1PlainDescription normalizes feed HTML into a single plain-text string for the
+// legacy process poll (description is a string, not a one-element array).
+func v1PlainDescription(s string) string {
+	s = strings.TrimSpace(s)
+	if s == "" {
+		return ""
+	}
+	s = html.UnescapeString(s)
+	s = strings.ReplaceAll(s, "\u00a0", " ")
+	s = v1BreakTagRe.ReplaceAllString(s, "\n")
+	s = v1BlockEndRe.ReplaceAllString(s, "\n")
+	s = specsHTMLTagRe.ReplaceAllString(s, " ")
+	s = html.UnescapeString(s)
+	lines := strings.Split(s, "\n")
+	kept := make([]string, 0, len(lines))
+	for _, line := range lines {
+		line = strings.Join(strings.Fields(line), " ")
+		if line != "" {
+			kept = append(kept, line)
+		}
+	}
+	return strings.TrimSpace(strings.Join(kept, "\n"))
+}
+
 func extractEPRELFromAttrs(attrs map[string]any) any {
 	if attrs == nil {
 		return nil