greeneclipse/descrybe
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fixes

Browse Source
This commit is contained in:
greeneclipse
2026-08-16 16:57:36 +02:00
parent 96f8c1115c
commit 532d439c41
106 changed files with 10147 additions and 494 deletions
Download Patch File Download Diff File Expand all files Collapse all files
apps/api/internal/processing/v1_legacy.go
+193 -27
View File
@@ -9,6 +9,7 @@ import (
"strings"
"github.com/descrybe/descrybe-v2/apps/api/internal/catalog"
"github.com/descrybe/descrybe-v2/apps/api/internal/eprel"
"github.com/google/uuid"
)
@@ -206,6 +207,7 @@ func (p *Pipeline) LoadV1ProcessJobItems(ctx context.Context, companyID, jobID u
return nil, fmt.Errorf("pipeline not configured")
}
allowedAttrs := loadCompanyAttributeKeySet(ctx, p, companyID)
categoryNames := loadCompanyCategoryNameMap(ctx, p, companyID)
rows, err := p.Pool.Query(ctx, `
SELECT
COALESCE(r.gtin, p.product_id, '') AS ean,
@@ -297,31 +299,98 @@ func (p *Pipeline) LoadV1ProcessJobItems(ctx context.Context, companyID, jobID u
if descTxt != nil {
plainDesc = v1PlainDescription(*descTxt)
}
eprelVal := extractEPRELFromAttrs(attrs)
attrs = SanitizeV1ProcessAttributesAllowed(attrs, allowedAttrs)
titleStr := derefStringPtr(title)
catStr := derefStringPtr(category)
catNameStr := derefStringPtr(categoryName)
// Projection gap: when processed.category is empty but mapped/raw carries a
// unique_id, surface it (and resolve category_name from company taxonomy).
if catStr == "" {
catStr = categoryUniqueIDFromMaps(mapped, rawData)
}
if catNameStr == "" && catStr != "" {
if name, ok := categoryNames[catStr]; ok && name != "" {
catNameStr = name
} else if nested, ok := mapped["category"].(map[string]any); ok {
for _, k := range []string{"name", "category_name", "title"} {
if s := strings.TrimSpace(stringFromAny(nested[k])); s != "" {
catNameStr = s
break
}
}
}
}
metaTitleOut := nullIfEmptyPtr(metaTitle)
metaDescOut := nullIfEmptyPtr(metaDesc)
if s, ok := metaTitleOut.(string); ok && isPoisonedMetaTitle(s) {
metaTitleOut = nil
// Poisoned title refresh: also drop empty/weak/leakage stub meta_description.
if ds, ok := metaDescOut.(string); ok && (ds == "" || isWeakPriorEnhanceDescription(ds, titleStr) || isPromptLeakageTitle(ds)) {
metaDescOut = nil
}
}
if ds, ok := metaDescOut.(string); ok && isPromptLeakageTitle(ds) {
metaDescOut = nil
}
catLabel := catNameStr
if catLabel == "" {
catLabel = catStr
}
if (metaTitleOut == nil || metaDescOut == nil) && (titleStr != "" || plainDesc != "" || catLabel != "") {
synthTitle, synthDesc := fillMetaFromResult(StepResult{
Name: titleStr,
ProcessedName: titleStr,
Category: catStr,
CategoryName: catNameStr,
Description: plainDesc,
ProcessedDescription: plainDesc,
Attributes: attrs,
ProcessedAttributes: attrs,
})
if metaTitleOut == nil {
if synthTitle != "" {
metaTitleOut = synthTitle
} else {
metaTitleOut = nullIfEmptyPtr(title)
}
}
if metaDescOut == nil {
if synthDesc != "" {
metaDescOut = synthDesc
} else if plainDesc != "" {
metaDescOut = truncateMetaDescription(plainDesc, v1MetaDescriptionMaxChars)
}
}
} else if metaTitleOut == nil {
metaTitleOut = nullIfEmptyPtr(title)
}
var description any
if plainDesc != "" {
description = plainDesc
} else {
description = nil
}
eprelVal := extractEPRELFromAttrs(attrs)
attrs = SanitizeV1ProcessAttributesAllowed(attrs, allowedAttrs)
metaTitleOut := nullIfEmptyPtr(metaTitle)
if metaTitleOut == nil {
metaTitleOut = nullIfEmptyPtr(title)
var catOut, catNameOut any
if catStr != "" {
catOut = catStr
}
metaDescOut := nullIfEmptyPtr(metaDesc)
if metaDescOut == nil && plainDesc != "" {
metaDescOut = truncateRunes(plainDesc, v1MetaDescriptionMaxChars)
if catNameStr != "" {
catNameOut = catNameStr
}
titleOut := nullIfEmptyPtr(title)
item := V1ProcessJobItem{
"ean": ean,
"status": MapV1JobItemStatus(itemStatus, true),
"category": nullIfEmptyPtr(category),
"category_name": nullIfEmptyPtr(categoryName),
"title": nullIfEmptyPtr(title),
"category": catOut,
"category_name": catNameOut,
"title": titleOut,
"name": titleOut,
"meta_title": metaTitleOut,
"meta_description": metaDescOut,
"description": description,
@@ -343,6 +412,7 @@ func (p *Pipeline) LoadV1ProcessJobItems(ctx context.Context, companyID, jobID u
if len(more) > 0 {
item["more_images"] = more
}
item = EnforceV1ProcessCompletedItem(item, "", allowedAttrs)
full = append(full, item)
}
if err := rows.Err(); err != nil {
@@ -358,6 +428,13 @@ func nullIfEmptyPtr(s *string) any {
return *s
}
func derefStringPtr(s *string) string {
if s == nil {
return ""
}
return strings.TrimSpace(*s)
}
// loadCompanyAttributeKeySet returns canonicalized attribute_key values for the company.
// On query failure returns an empty (non-nil) set so FilterAttributesByAllowed still
// restricts to coreCharacteristicAttrKeys only.
@@ -389,15 +466,52 @@ func loadCompanyAttributeKeySet(ctx context.Context, p *Pipeline, companyID uuid
return out
}
// v1PlainDescription normalizes feed HTML into a single plain-text string for the
// legacy process poll (description is a string, not a one-element array).
// loadCompanyCategoryNameMap returns unique_id → display name for the company.
func loadCompanyCategoryNameMap(ctx context.Context, p *Pipeline, companyID uuid.UUID) map[string]string {
out := map[string]string{}
if p == nil || p.Pool == nil {
return out
}
rows, err := p.Pool.Query(ctx, `
SELECT unique_id, name
FROM categories
WHERE company_id = $1 AND COALESCE(unique_id, '') <> ''`, companyID)
if err != nil {
return out
}
defer rows.Close()
for rows.Next() {
var uid, name string
if err := rows.Scan(&uid, &name); err != nil {
continue
}
uid = strings.TrimSpace(uid)
name = strings.TrimSpace(name)
if uid == "" || name == "" {
continue
}
out[uid] = name
}
return out
}
// v1PlainDescription normalizes legacy stored descriptions into a single plain-text
// string for the process poll (description is a string, not a one-element array).
// Handles JSON array/string encodings (e.g. mapped_data->>'description' when the
// feed value was an array) and HTML/<br>/entity markup.
func v1PlainDescription(s string) string {
s = strings.TrimSpace(s)
if s == "" {
return ""
}
s = unwrapLegacyDescriptionStored(s)
s = strings.TrimSpace(s)
if s == "" {
return ""
}
s = html.UnescapeString(s)
s = strings.ReplaceAll(s, "\u00a0", " ")
s = strings.ReplaceAll(s, `\u00a0`, " ")
s = v1BreakTagRe.ReplaceAllString(s, "\n")
s = v1BlockEndRe.ReplaceAllString(s, "\n")
s = specsHTMLTagRe.ReplaceAllString(s, " ")
@@ -406,6 +520,10 @@ func v1PlainDescription(s string) string {
kept := make([]string, 0, len(lines))
for _, line := range lines {
line = strings.Join(strings.Fields(line), " ")
line = strings.ReplaceAll(line, " .", ".")
line = strings.ReplaceAll(line, " ,", ",")
line = strings.ReplaceAll(line, " ;", ";")
line = strings.ReplaceAll(line, " :", ":")
if line != "" {
kept = append(kept, line)
}
@@ -413,23 +531,67 @@ func v1PlainDescription(s string) string {
return strings.TrimSpace(strings.Join(kept, "\n"))
}
func extractEPRELFromAttrs(attrs map[string]any) any {
if attrs == nil {
return nil
const maxLegacyDescriptionUnwrapDepth = 4
// unwrapLegacyDescriptionStored flattens JSON-encoded legacy description payloads
// (one-element arrays, multi-part arrays, nested arrays, JSON-quoted strings).
// Non-JSON text is returned unchanged.
func unwrapLegacyDescriptionStored(s string) string {
return unwrapLegacyDescriptionStoredN(s, 0)
}
func unwrapLegacyDescriptionStoredN(s string, depth int) string {
s = strings.TrimSpace(s)
if s == "" || depth > maxLegacyDescriptionUnwrapDepth {
return s
}
if e, ok := attrs["eprel"]; ok && e != nil {
return e
if !looksLikeLegacyDescriptionJSON(s) {
return s
}
out := map[string]any{}
for _, k := range []string{"label", "pdf", "energy_class", "energy_scale"} {
if v, ok := attrs["eprel_"+k]; ok && v != nil {
out[k] = v
var raw any
if err := json.Unmarshal([]byte(s), &raw); err != nil {
return s
}
flat := flattenLegacyDescriptionAny(raw, depth)
if flat == "" {
if _, ok := raw.([]any); ok {
return ""
}
return s
}
if len(out) == 0 {
return nil
return flat
}
func looksLikeLegacyDescriptionJSON(s string) bool {
if len(s) >= 2 && s[0] == '[' && s[len(s)-1] == ']' {
return true
}
return out
return len(s) >= 2 && s[0] == '"' && s[len(s)-1] == '"'
}
func flattenLegacyDescriptionAny(v any, depth int) string {
switch t := v.(type) {
case string:
return unwrapLegacyDescriptionStoredN(t, depth+1)
case []any:
parts := make([]string, 0, len(t))
for _, el := range t {
if p := flattenLegacyDescriptionAny(el, depth+1); p != "" {
parts = append(parts, p)
}
}
return strings.Join(parts, "\n")
case float64:
return strings.TrimSpace(fmt.Sprint(t))
case bool:
return fmt.Sprint(t)
default:
return ""
}
}
func extractEPRELFromAttrs(attrs map[string]any) any {
return eprel.ExtractFromAttrs(attrs)
}
// ProjectV1ProcessJobItems applies legacy partial-type field projection.
@@ -471,6 +633,10 @@ func ProjectV1ProcessJobItems(storedType string, items []V1ProcessJobItem) []V1P
projected["category_name"] = item["category_name"]
case "title":
projected["title"] = item["title"]
projected["name"] = item["name"]
if projected["name"] == nil {
projected["name"] = item["title"]
}
projected["meta_title"] = item["meta_title"]
case "description":
projected["description"] = item["description"]