greeneclipse/descrybe
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

update

Browse Source
This commit is contained in:
greeneclipse
2026-08-16 12:47:06 +02:00
parent 1d3a9d5abd
commit 96f8c1115c
5 changed files with 187 additions and 15 deletions
Download Patch File Download Diff File Expand all files Collapse all files
apps/api/internal/httpapi/v1_openapi.go
+3 -8
View File
@@ -6296,13 +6296,9 @@ components:
type: string
nullable: true
description:
nullable: true
oneOf:
- type: string
- type: array
items:
type: string
attributes:
nullable: true
description: Plain-text product description (HTML stripped). attributes:
type: object
additionalProperties: true
nullable: true
@@ -6414,8 +6410,7 @@ components:
title: Sony WH-1000XM5 Wireless Noise Cancelling Headphones Black
meta_title: Sony WH-1000XM5 | Noise Cancelling Headphones
meta_description: Industry-leading noise cancellation with up to 30 hours battery life.
description:
- Industry-leading noise cancellation with up to 30 hours battery life.
description: Industry-leading noise cancellation with up to 30 hours battery life.
attributes:
color: Black
brand: Sony
apps/api/internal/processing/pipeline_steps_test.go
+44
View File
@@ -90,6 +90,50 @@ func TestSanitizeProductAttributes(t *testing.T) {
}
}
func TestFilterAttributesByAllowed(t *testing.T) {
in := map[string]any{
"brand": "Ostalo",
"zavora": "Mehanska",
"vzmetenje": "Spredaj",
"nosilnost": "40 kg",
"width": "0.6 m",
}
allowed := map[string]struct{}{
"nosilnost": {},
"barva": {},
}
got := FilterAttributesByAllowed(in, allowed)
if got["brand"] != "Ostalo" || got["width"] != "0.6 m" {
t.Fatalf("core keys missing: %v", got)
}
if got["nosilnost"] != "40 kg" {
t.Fatalf("allowed key missing: %v", got)
}
if _, ok := got["zavora"]; ok {
t.Fatalf("junk spec should be dropped: %v", got)
}
if _, ok := got["vzmetenje"]; ok {
t.Fatalf("junk spec should be dropped: %v", got)
}
unchanged := FilterAttributesByAllowed(in, nil)
if unchanged["zavora"] != "Mehanska" {
t.Fatalf("nil allowlist should keep all: %v", unchanged)
}
}
func TestV1PlainDescription(t *testing.T) {
got := v1PlainDescription("Hello<br>World<br/>&amp; more</p><b>bold</b>")
if strings.Contains(got, "<") {
t.Fatalf("html should be stripped: %q", got)
}
if !strings.Contains(got, "Hello") || !strings.Contains(got, "World") || !strings.Contains(got, "bold") {
t.Fatalf("got=%q", got)
}
if !strings.Contains(got, "& more") {
t.Fatalf("expected unescaped amp: %q", got)
}
}
func TestFillMissingFields_brandAndDims(t *testing.T) {
m := FillMissingFields(map[string]any{
"name": "Nike Air 30x20x10 cm",
apps/api/internal/processing/specs.go
+46 -1
View File
@@ -146,6 +146,14 @@ func isInvalidAttributeKey(k string) bool {
return !hasLetter
}
// coreCharacteristicAttrKeys are always allowed on V1 process items even when
// the company has no matching attributes row (common dims/brand/model).
var coreCharacteristicAttrKeys = map[string]struct{}{
"brand": {}, "product_model": {}, "warranty": {},
"width": {}, "height": {}, "depth": {}, "weight": {},
"energy_class": {}, "color": {}, "material": {},
}
// SanitizeProductAttributes keeps characteristic attrs for API/storage and drops
// core product fields that belong on the V1 item root (title, description, images, …).
func SanitizeProductAttributes(attrs map[string]any) map[string]any {
@@ -153,11 +161,48 @@ func SanitizeProductAttributes(attrs map[string]any) map[string]any {
}
// SanitizeV1ProcessAttributes is the stricter poll projection: also drops eprel_*
// keys (those are exposed on item.eprel).
// keys (those are exposed on item.eprel). Without an allowlist, feed junk specs
// (zavora, vzmetenje, …) can remain — prefer SanitizeV1ProcessAttributesAllowed.
func SanitizeV1ProcessAttributes(attrs map[string]any) map[string]any {
return sanitizeProductAttributes(attrs, true)
}
// SanitizeV1ProcessAttributesAllowed sanitizes then keeps only core characteristic
// keys plus company attribute_key values (canonicalized).
func SanitizeV1ProcessAttributesAllowed(attrs map[string]any, allowed map[string]struct{}) map[string]any {
return FilterAttributesByAllowed(sanitizeProductAttributes(attrs, true), allowed)
}
// FilterAttributesByAllowed keeps coreCharacteristicAttrKeys plus keys present in
// allowed (after canonicalizeAttrKey). When allowed is nil, returns attrs unchanged.
func FilterAttributesByAllowed(attrs map[string]any, allowed map[string]struct{}) map[string]any {
if len(attrs) == 0 {
return map[string]any{}
}
if allowed == nil {
return attrs
}
out := make(map[string]any, len(attrs))
for k, v := range attrs {
canon := canonicalizeAttrKey(k)
if canon == "" {
continue
}
if _, ok := coreCharacteristicAttrKeys[canon]; ok {
out[canon] = v
continue
}
if _, ok := allowed[canon]; ok {
out[canon] = v
continue
}
if _, ok := allowed[strings.ToLower(strings.TrimSpace(k))]; ok {
out[canon] = v
}
}
return out
}
func sanitizeProductAttributes(attrs map[string]any, dropEPREL bool) map[string]any {
if len(attrs) == 0 {
return map[string]any{}
apps/api/internal/processing/v1_legacy.go
+82 -5
View File
@@ -4,6 +4,8 @@ import (
"context"
"encoding/json"
"fmt"
"html"
"regexp"
"strings"
"github.com/descrybe/descrybe-v2/apps/api/internal/catalog"
@@ -12,6 +14,12 @@ import (
var v1PartialSteps = []string{"category", "title", "description", "attributes"}
const v1MetaDescriptionMaxChars = 155
var (
v1BreakTagRe = regexp.MustCompile(`(?i)<br\s*/?>`)
v1BlockEndRe = regexp.MustCompile(`(?i)</(p|div|li|h[1-6]|tr)>`)
)
// ParseV1ProcessingType mirrors legacy parseV1ProcessingTypeFromBody.
// Accepts string ("full" / step), JSON array of steps, or nil (defaults to full).
func ParseV1ProcessingType(raw any) (storageValue string, responseValue any, err error) {
@@ -197,6 +205,7 @@ func (p *Pipeline) LoadV1ProcessJobItems(ctx context.Context, companyID, jobID u
if p == nil || p.Pool == nil {
return nil, fmt.Errorf("pipeline not configured")
}
allowedAttrs := loadCompanyAttributeKeySet(ctx, p, companyID)
rows, err := p.Pool.Query(ctx, `
SELECT
COALESCE(r.gtin, p.product_id, '') AS ean,
@@ -284,15 +293,28 @@ func (p *Pipeline) LoadV1ProcessJobItems(ctx context.Context, companyID, jobID u
_ = json.Unmarshal(rawJSON, &rawData)
main, more := catalog.ExtractProductImages(mapped, rawData)
plainDesc := ""
if descTxt != nil {
plainDesc = v1PlainDescription(*descTxt)
}
var description any
if descTxt != nil && strings.TrimSpace(*descTxt) != "" {
description = []string{*descTxt}
if plainDesc != "" {
description = plainDesc
} else {
description = nil
}
eprelVal := extractEPRELFromAttrs(attrs)
attrs = SanitizeV1ProcessAttributes(attrs)
attrs = SanitizeV1ProcessAttributesAllowed(attrs, allowedAttrs)
metaTitleOut := nullIfEmptyPtr(metaTitle)
if metaTitleOut == nil {
metaTitleOut = nullIfEmptyPtr(title)
}
metaDescOut := nullIfEmptyPtr(metaDesc)
if metaDescOut == nil && plainDesc != "" {
metaDescOut = truncateRunes(plainDesc, v1MetaDescriptionMaxChars)
}
item := V1ProcessJobItem{
"ean": ean,
@@ -300,8 +322,8 @@ func (p *Pipeline) LoadV1ProcessJobItems(ctx context.Context, companyID, jobID u
"category": nullIfEmptyPtr(category),
"category_name": nullIfEmptyPtr(categoryName),
"title": nullIfEmptyPtr(title),
"meta_title": nullIfEmptyPtr(metaTitle),
"meta_description": nullIfEmptyPtr(metaDesc),
"meta_title": metaTitleOut,
"meta_description": metaDescOut,
"description": description,
"attributes": nil,
"main_image": nil,
@@ -336,6 +358,61 @@ func nullIfEmptyPtr(s *string) any {
return *s
}
// loadCompanyAttributeKeySet returns canonicalized attribute_key values for the company.
// On query failure returns an empty (non-nil) set so FilterAttributesByAllowed still
// restricts to coreCharacteristicAttrKeys only.
func loadCompanyAttributeKeySet(ctx context.Context, p *Pipeline, companyID uuid.UUID) map[string]struct{} {
out := map[string]struct{}{}
if p == nil || p.Pool == nil {
return out
}
rows, err := p.Pool.Query(ctx, `
SELECT attribute_key
FROM attributes
WHERE company_id = $1 AND COALESCE(attribute_key, '') <> ''`, companyID)
if err != nil {
return out
}
defer rows.Close()
for rows.Next() {
var key string
if err := rows.Scan(&key); err != nil {
continue
}
canon := canonicalizeAttrKey(key)
if canon == "" {
continue
}
out[canon] = struct{}{}
out[strings.ToLower(strings.TrimSpace(key))] = struct{}{}
}
return out
}
// v1PlainDescription normalizes feed HTML into a single plain-text string for the
// legacy process poll (description is a string, not a one-element array).
func v1PlainDescription(s string) string {
s = strings.TrimSpace(s)
if s == "" {
return ""
}
s = html.UnescapeString(s)
s = strings.ReplaceAll(s, "\u00a0", " ")
s = v1BreakTagRe.ReplaceAllString(s, "\n")
s = v1BlockEndRe.ReplaceAllString(s, "\n")
s = specsHTMLTagRe.ReplaceAllString(s, " ")
s = html.UnescapeString(s)
lines := strings.Split(s, "\n")
kept := make([]string, 0, len(lines))
for _, line := range lines {
line = strings.Join(strings.Fields(line), " ")
if line != "" {
kept = append(kept, line)
}
}
return strings.TrimSpace(strings.Join(kept, "\n"))
}
func extractEPRELFromAttrs(attrs map[string]any) any {
if attrs == nil {
return nil
apps/api/internal/processing/v1_legacy_test.go
+12 -1
View File
@@ -2,6 +2,7 @@ package processing
import (
"encoding/json"
"strings"
"testing"
"github.com/google/uuid"
@@ -76,7 +77,7 @@ func TestProjectV1ProcessJobItemsPartial(t *testing.T) {
"raw_product_id": "cccccccc-cccc-cccc-cccc-cccccccccccc",
"status": "processed",
"title": "T", "meta_title": "MT",
"description": []string{"D"}, "attributes": map[string]any{"brand": "X"},
"description": "D", "attributes": map[string]any{"brand": "X"},
"main_image": "https://example.com/a.jpg", "more_images": []string{"https://example.com/b.jpg"},
"eprel": nil, "category": "cat", "category_name": "Cat",
}}
@@ -107,6 +108,16 @@ func TestProjectV1ProcessJobItemsPartial(t *testing.T) {
}
}
func TestV1PlainDescriptionStandalone(t *testing.T) {
got := v1PlainDescription("Line1<br>Line2 &amp; ok")
if strings.Contains(got, "<") {
t.Fatalf("html left: %q", got)
}
if !strings.Contains(got, "Line1") || !strings.Contains(got, "Line2") || !strings.Contains(got, "& ok") {
t.Fatalf("got=%q", got)
}
}
func TestApplyV1ProcessItemIDsDualMode(t *testing.T) {
processed := mustParseTestUUID(t, "bbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb")
raw := mustParseTestUUID(t, "cccccccc-cccc-cccc-cccc-cccccccccccc")