fix
This commit is contained in:
@@ -0,0 +1,236 @@
|
||||
// Temporary local probe for A1/Demo V1 projection cleanup. Do not commit.
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"time"
|
||||
|
||||
"github.com/descrybe/descrybe-v2/apps/api/internal/billing"
|
||||
"github.com/descrybe/descrybe-v2/apps/api/internal/processing"
|
||||
"github.com/google/uuid"
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
)
|
||||
|
||||
func main() {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute)
|
||||
defer cancel()
|
||||
dsn := os.Getenv("DATABASE_URL")
|
||||
if dsn == "" {
|
||||
dsn = "postgres://descrybe:descrybe@localhost:5433/descrybe?sslmode=disable"
|
||||
}
|
||||
pool, err := pgxpool.New(ctx, dsn)
|
||||
if err != nil {
|
||||
fatal(err)
|
||||
}
|
||||
defer pool.Close()
|
||||
|
||||
outDir := filepath.Join("..", "..", ".codehelper", "_a1_v1_response_20260817")
|
||||
if abs, err := filepath.Abs(outDir); err == nil {
|
||||
outDir = abs
|
||||
}
|
||||
_ = os.MkdirAll(outDir, 0o755)
|
||||
|
||||
type companyRow struct {
|
||||
ID uuid.UUID
|
||||
Name string
|
||||
Legacy string
|
||||
}
|
||||
companies := []companyRow{}
|
||||
rows, err := pool.Query(ctx, `
|
||||
SELECT id, name, COALESCE(legacy_company_id, '')
|
||||
FROM companies
|
||||
WHERE legacy_company_id = $1
|
||||
OR lower(name) LIKE '%demo%'
|
||||
ORDER BY CASE WHEN legacy_company_id = $1 THEN 0 ELSE 1 END, name
|
||||
LIMIT 6`, billing.A1LegacyCompanyID)
|
||||
if err != nil {
|
||||
fatal(err)
|
||||
}
|
||||
for rows.Next() {
|
||||
var c companyRow
|
||||
if err := rows.Scan(&c.ID, &c.Name, &c.Legacy); err != nil {
|
||||
fatal(err)
|
||||
}
|
||||
companies = append(companies, c)
|
||||
}
|
||||
rows.Close()
|
||||
if len(companies) == 0 {
|
||||
fatal(fmt.Errorf("no A1/Demo companies found"))
|
||||
}
|
||||
|
||||
p := processing.NewPipeline(pool)
|
||||
eans := []string{"8806091734365", "8422160053580"}
|
||||
summary := map[string]any{"generated_at": time.Now().UTC().Format(time.RFC3339), "companies": []any{}}
|
||||
|
||||
for _, co := range companies {
|
||||
omitMeta := billing.IsA1CohortCompany(co.Legacy, co.Name)
|
||||
coSummary := map[string]any{
|
||||
"company_id": co.ID.String(),
|
||||
"name": co.Name,
|
||||
"legacy_id": co.Legacy,
|
||||
"omit_seo_meta": omitMeta,
|
||||
"items": []any{},
|
||||
}
|
||||
|
||||
// Prefer existing processed rows for the sample EANs; fall back to any 2 recent.
|
||||
type prod struct {
|
||||
EAN, Title, Desc, Cat, CatName string
|
||||
Attrs []byte
|
||||
}
|
||||
prods := []prod{}
|
||||
q := `
|
||||
SELECT COALESCE(p.product_id,''), COALESCE(p.processed_name,p.name,''),
|
||||
COALESCE(p.processed_description, p.description, ''),
|
||||
COALESCE(p.category,''), COALESCE(c.name,''),
|
||||
COALESCE(p.processed_attributes, p.attributes, '{}'::jsonb)
|
||||
FROM processed_products p
|
||||
LEFT JOIN categories c ON c.unique_id = p.category AND c.company_id = p.company_id
|
||||
WHERE p.company_id = $1 AND p.product_id = ANY($2)
|
||||
ORDER BY p.updated_at DESC NULLS LAST
|
||||
LIMIT 4`
|
||||
pr, err := pool.Query(ctx, q, co.ID, eans)
|
||||
if err != nil {
|
||||
fatal(err)
|
||||
}
|
||||
for pr.Next() {
|
||||
var x prod
|
||||
if err := pr.Scan(&x.EAN, &x.Title, &x.Desc, &x.Cat, &x.CatName, &x.Attrs); err != nil {
|
||||
fatal(err)
|
||||
}
|
||||
prods = append(prods, x)
|
||||
}
|
||||
pr.Close()
|
||||
if len(prods) < 2 {
|
||||
pr2, err := pool.Query(ctx, `
|
||||
SELECT COALESCE(p.product_id,''), COALESCE(p.processed_name,p.name,''),
|
||||
COALESCE(p.processed_description, p.description, ''),
|
||||
COALESCE(p.category,''), COALESCE(c.name,''),
|
||||
COALESCE(p.processed_attributes, p.attributes, '{}'::jsonb)
|
||||
FROM processed_products p
|
||||
LEFT JOIN categories c ON c.unique_id = p.category AND c.company_id = p.company_id
|
||||
WHERE p.company_id = $1 AND COALESCE(p.processed_name,p.name,'') <> ''
|
||||
ORDER BY p.updated_at DESC NULLS LAST
|
||||
LIMIT 2`, co.ID)
|
||||
if err != nil {
|
||||
fatal(err)
|
||||
}
|
||||
for pr2.Next() {
|
||||
var x prod
|
||||
if err := pr2.Scan(&x.EAN, &x.Title, &x.Desc, &x.Cat, &x.CatName, &x.Attrs); err != nil {
|
||||
fatal(err)
|
||||
}
|
||||
prods = append(prods, x)
|
||||
}
|
||||
pr2.Close()
|
||||
}
|
||||
if len(prods) == 0 {
|
||||
coSummary["note"] = "no processed products"
|
||||
summary["companies"] = append(summary["companies"].([]any), coSummary)
|
||||
continue
|
||||
}
|
||||
|
||||
items := []processing.V1ProcessJobItem{}
|
||||
for _, x := range prods {
|
||||
attrs := map[string]any{}
|
||||
_ = json.Unmarshal(x.Attrs, &attrs)
|
||||
item := processing.V1ProcessJobItem{
|
||||
"ean": x.EAN,
|
||||
"status": "processed",
|
||||
"title": x.Title,
|
||||
"name": x.Title,
|
||||
"description": x.Desc,
|
||||
"category": x.Cat,
|
||||
"category_name": x.CatName,
|
||||
"attributes": attrs,
|
||||
"eprel": nil,
|
||||
"id": uuid.New().String(),
|
||||
}
|
||||
item["processed_product_id"] = item["id"]
|
||||
item["raw_product_id"] = uuid.New().String()
|
||||
if !omitMeta {
|
||||
item["meta_title"] = nil
|
||||
item["meta_description"] = nil
|
||||
}
|
||||
item = processing.EnforceV1ProcessCompletedItemOpts(item, processing.EnforceV1Opts{
|
||||
Language: "sl",
|
||||
Allowed: map[string]struct{}{},
|
||||
OmitSEOMeta: omitMeta,
|
||||
})
|
||||
items = append(items, item)
|
||||
}
|
||||
|
||||
payload := map[string]any{
|
||||
"data": map[string]any{
|
||||
"status": "COMPLETED",
|
||||
"processing_type": "full",
|
||||
"total_items": len(items),
|
||||
"items": items,
|
||||
"company": co.Name,
|
||||
"omit_seo_meta": omitMeta,
|
||||
},
|
||||
}
|
||||
b, err := json.MarshalIndent(payload, "", " ")
|
||||
if err != nil {
|
||||
fatal(err)
|
||||
}
|
||||
label := "demo"
|
||||
if omitMeta {
|
||||
label = "a1"
|
||||
}
|
||||
fname := fmt.Sprintf("%s_%s.json", label, co.ID.String()[:8])
|
||||
path := filepath.Join(outDir, fname)
|
||||
if err := os.WriteFile(path, b, 0o644); err != nil {
|
||||
fatal(err)
|
||||
}
|
||||
coSummary["sample_path"] = path
|
||||
coSummary["item_count"] = len(items)
|
||||
coSummary["items"] = items
|
||||
summary["companies"] = append(summary["companies"].([]any), coSummary)
|
||||
fmt.Println("wrote", path, "items=", len(items), "omit_meta=", omitMeta)
|
||||
|
||||
// Also exercise LoadV1 against the latest completed job for this company when present.
|
||||
var jobID uuid.UUID
|
||||
err = pool.QueryRow(ctx, `
|
||||
SELECT id FROM processing_jobs
|
||||
WHERE company_id = $1 AND status IN ('completed','COMPLETED')
|
||||
ORDER BY updated_at DESC NULLS LAST
|
||||
LIMIT 1`, co.ID).Scan(&jobID)
|
||||
if err == nil {
|
||||
loaded, loadErr := p.LoadV1ProcessJobItems(ctx, co.ID, jobID, "full")
|
||||
if loadErr == nil && len(loaded) > 0 {
|
||||
if len(loaded) > 2 {
|
||||
loaded = loaded[:2]
|
||||
}
|
||||
live := map[string]any{
|
||||
"data": map[string]any{
|
||||
"process_id": jobID.String(),
|
||||
"status": "COMPLETED",
|
||||
"processing_type": "full",
|
||||
"total_items": len(loaded),
|
||||
"items": loaded,
|
||||
"company": co.Name,
|
||||
"omit_seo_meta": omitMeta,
|
||||
"source": "LoadV1ProcessJobItems",
|
||||
},
|
||||
}
|
||||
lb, _ := json.MarshalIndent(live, "", " ")
|
||||
lpath := filepath.Join(outDir, fmt.Sprintf("%s_live_job_%s.json", label, jobID.String()[:8]))
|
||||
_ = os.WriteFile(lpath, lb, 0o644)
|
||||
fmt.Println("wrote", lpath, "live_items=", len(loaded))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
sb, _ := json.MarshalIndent(summary, "", " ")
|
||||
_ = os.WriteFile(filepath.Join(outDir, "summary.json"), sb, 0o644)
|
||||
fmt.Println("done", outDir)
|
||||
}
|
||||
|
||||
func fatal(err error) {
|
||||
fmt.Fprintln(os.Stderr, err)
|
||||
os.Exit(1)
|
||||
}
|
||||
@@ -0,0 +1,84 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
|
||||
"github.com/descrybe/descrybe-v2/apps/api/internal/processing"
|
||||
)
|
||||
|
||||
func main() {
|
||||
badItems := []processing.V1ProcessJobItem{
|
||||
{
|
||||
"ean": "8806091734365",
|
||||
"status": "processed",
|
||||
"title": "LG Ameriški hladilnikGSXV80PZLE",
|
||||
"name": "LG Ameriški hladilnikGSXV80PZLE",
|
||||
"category": "11",
|
||||
"category_name": "Hladilniki",
|
||||
"description": "<h1>LG Ameriški hladilnikGSXV80PZLE</h1><p>LG Ameriški hladilnikGSXV80PZLE je izdelek v kategoriji Hladilniki znamke LG. Ključne specifikacije: width 179, height 91.3, depth 73.5.</p><h2>Ključne lastnosti</h2><ul><li>width 179</li><li>height 91.3</li><li>depth 73.5</li></ul><p>LG Ameriški hladilnikGSXV80PZLE je izdelek v kategoriji Hladilniki znamke LG. Ključne specifikacije: width 179, height 91.3, depth 73.5.</p><p>LG Ameriški hladilnikGSXV80PZLE je izdelek v kategoriji Hladilniki znamke LG. Ključne specifikacije: width 179, height 91.3, depth 73.5.</p><p>LG Ameriški hladilnikGSXV80PZLE je izdelek v kategoriji Hladilniki znamke LG. Ključne specifikacije: width 179, height 91.3, depth 73.5.</p>",
|
||||
"attributes": map[string]any{"brand": "LG", "depth": "73.5", "energy_class": "E", "height": "91.3", "warranty": "24 mesecev", "width": "179"},
|
||||
"meta_title": "LG Ameriški hladilnikGSXV80PZLE | Hladilniki",
|
||||
"meta_description": "x",
|
||||
"eprel": map[string]any{"id": "1158009", "energy_class": "E"},
|
||||
"id": "1bc20e89-3779-4c66-8617-8e72555cd363",
|
||||
"processed_product_id": "1bc20e89-3779-4c66-8617-8e72555cd363",
|
||||
"raw_product_id": "fff13ee3-e78c-4185-b524-b64128bbc38b",
|
||||
"main_image": "https://example/lg.jpg",
|
||||
"more_images": nil,
|
||||
},
|
||||
{
|
||||
"ean": "8422160053580",
|
||||
"status": "processed",
|
||||
"title": "Ufesa digitalni zračni cvrtnik 23L Magnum",
|
||||
"name": "Ufesa digitalni zračni cvrtnik 23L Magnum",
|
||||
"category": "112",
|
||||
"category_name": "Ostali aparati",
|
||||
"description": "<h1>Ufesa digitalni zračni cvrtnik 23L Magnum</h1><p>Ufesa digitalni zračni cvrtnik 23L Magnum je izdelek v kategoriji Ostali aparati znamke UFESA. Ključne specifikacije: width 0,00m, height 0,00m, depth 0,00m.</p><h2>Ključne lastnosti</h2><ul><li>width 0,00m</li><li>height 0,00m</li><li>depth 0,00m</li><li>weight 11,0000kg</li></ul><p>Ufesa digitalni zračni cvrtnik 23L Magnum je izdelek v kategoriji Ostali aparati znamke UFESA. Ključne specifikacije: width 0,00m, height 0,00m, depth 0,00m.</p>",
|
||||
"attributes": map[string]any{"brand": "UFESA", "depth": "0,00m", "height": "0,00m", "warranty": "24 mesecev", "weight": "11,0000kg", "width": "0,00m"},
|
||||
"meta_title": "x",
|
||||
"meta_description": "y",
|
||||
"eprel": nil,
|
||||
"id": "c708b505-dbf0-40c5-ba09-022d15daff97",
|
||||
"processed_product_id": "c708b505-dbf0-40c5-ba09-022d15daff97",
|
||||
"raw_product_id": "fff6c685-931b-4fa5-846b-89b2503a5a77",
|
||||
"main_image": "https://example/ufesa.jpg",
|
||||
"more_images": []string{"https://example/2.jpg"},
|
||||
},
|
||||
}
|
||||
cleaned := make([]processing.V1ProcessJobItem, 0, len(badItems))
|
||||
for _, it := range badItems {
|
||||
cleaned = append(cleaned, processing.EnforceV1ProcessCompletedItemOpts(it, processing.EnforceV1Opts{
|
||||
Language: "sl",
|
||||
Allowed: map[string]struct{}{},
|
||||
OmitSEOMeta: true,
|
||||
}))
|
||||
}
|
||||
out := map[string]any{
|
||||
"data": map[string]any{
|
||||
"process_id": "13b97eab-2329-4735-82fa-3afe263849fa",
|
||||
"status": "COMPLETED",
|
||||
"processing_type": "full",
|
||||
"total_items": len(cleaned),
|
||||
"items": cleaned,
|
||||
"note": "projection cleanup of production sample (A1 OmitSEOMeta)",
|
||||
},
|
||||
}
|
||||
b, err := json.MarshalIndent(out, "", " ")
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
path := filepath.Join("..", "..", ".codehelper", "_a1_v1_response_20260817", "production_sample_cleaned.json")
|
||||
if err := os.WriteFile(path, b, 0o644); err != nil {
|
||||
panic(err)
|
||||
}
|
||||
fmt.Println("wrote", path)
|
||||
for _, it := range cleaned {
|
||||
fmt.Printf("ean=%v\ntitle=%v\ncategory=%v category_id=%v\nmeta_present=%v/%v\nattrs=%v\ndesc=%v\n\n",
|
||||
it["ean"], it["title"], it["category"], it["category_id"],
|
||||
it["meta_title"] != nil, it["meta_description"] != nil,
|
||||
it["attributes"], it["description"])
|
||||
}
|
||||
}
|
||||
@@ -929,7 +929,8 @@ paths:
|
||||
processed_product_id: bbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb
|
||||
raw_product_id: cccccccc-cccc-cccc-cccc-cccccccccccc
|
||||
status: processed
|
||||
category: '50'
|
||||
category: Cookers
|
||||
category_id: '50'
|
||||
category_name: Cookers
|
||||
title: VOX electric cooker EHT 6020 WG
|
||||
name: VOX electric cooker EHT 6020 WG
|
||||
@@ -6278,12 +6279,15 @@ components:
|
||||
type: object
|
||||
description: |
|
||||
One COMPLETED legacy process line (A1 / public contract). Successful items
|
||||
expose category as categories.unique_id, a description string that may
|
||||
include category formula HTML (h1/h2/h3/h4, p, ul — never a JSON array),
|
||||
SEO meta_title / meta_description (plain text), optional eprel object or
|
||||
expose category as the human-readable display name (category_id holds
|
||||
categories.unique_id), a description string that may include category formula
|
||||
HTML (h1/h2/h3/h4, p, ul — never a JSON array), optional SEO meta_title /
|
||||
meta_description (plain text; omitted for A1 cohort), optional eprel object or
|
||||
null, clean attributes, images, and dual-mode ids.
|
||||
Product display name is title; additive name mirrors the same processed title
|
||||
(dual-mode for scorecards / legacy clients that read name).
|
||||
Property order prefers human-readable fields first (ean, title, category,
|
||||
description, attributes, images, eprel) with internal ids last.
|
||||
required:
|
||||
- ean
|
||||
properties:
|
||||
@@ -6308,6 +6312,14 @@ components:
|
||||
raw_products.id for this job line. Use with POST /process raw_product_ids
|
||||
or dashboard catalog APIs. Present whenever the job product row exists.
|
||||
category:
|
||||
type: string
|
||||
nullable: true
|
||||
description: |
|
||||
Human-readable category display name (same value as category_name when
|
||||
resolved). Prefer this for UI. Machine unique_id is category_id.
|
||||
ASSUMPTION: historically this field held categories.unique_id; clients
|
||||
that need the opaque id must read category_id (additive).
|
||||
category_id:
|
||||
type: string
|
||||
nullable: true
|
||||
description: |
|
||||
@@ -6316,7 +6328,8 @@ components:
|
||||
category_name:
|
||||
type: string
|
||||
nullable: true
|
||||
description: Human-readable category display name (not the unique_id).
|
||||
description: |
|
||||
Human-readable category display name (mirrors category when both are set).
|
||||
title:
|
||||
type: string
|
||||
nullable: true
|
||||
@@ -6335,12 +6348,14 @@ components:
|
||||
description: |
|
||||
SEO title. Filled from processing meta or synthesized from title / category
|
||||
when empty so successful items are not left with null meta.
|
||||
Omitted for A1 cohort (SEO meta is not used).
|
||||
meta_description:
|
||||
type: string
|
||||
nullable: true
|
||||
description: |
|
||||
SEO description (word-safe truncate). Distinct from body description when
|
||||
possible; synthesized from plain description when DB meta is empty.
|
||||
Omitted for A1 cohort (SEO meta is not used).
|
||||
description:
|
||||
type: string
|
||||
nullable: true
|
||||
@@ -6466,7 +6481,8 @@ components:
|
||||
processed_product_id: bbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb
|
||||
raw_product_id: cccccccc-cccc-cccc-cccc-cccccccccccc
|
||||
status: processed
|
||||
category: '50'
|
||||
category: Cookers
|
||||
category_id: '50'
|
||||
category_name: Cookers
|
||||
title: VOX electric cooker EHT 6020 WG
|
||||
name: VOX electric cooker EHT 6020 WG
|
||||
|
||||
@@ -126,6 +126,9 @@ type ProductInput struct {
|
||||
// CategoryUniqueID is the taxonomy unique_id for overlay/formula keys when
|
||||
// the enhance display category argument is a localized name.
|
||||
CategoryUniqueID string
|
||||
// OmitSEOMeta skips meta_title / meta_description enhance + free-template fill
|
||||
// (A1 cohort does not use SEO meta fields).
|
||||
OmitSEOMeta bool
|
||||
}
|
||||
|
||||
// CategoryFormulas holds optional title/description templates for one category key.
|
||||
|
||||
@@ -359,8 +359,11 @@ func TestLoadV1ProcessJobItems_resolvesCategoryName(t *testing.T) {
|
||||
if len(items) != 1 {
|
||||
t.Fatalf("items=%d want 1", len(items))
|
||||
}
|
||||
if got := fmt.Sprint(items[0]["category"]); got != "50" {
|
||||
t.Fatalf("category=%v want 50", items[0]["category"])
|
||||
if got := fmt.Sprint(items[0]["category"]); got != "Štedilniki" {
|
||||
t.Fatalf("category=%v want Štedilniki (display name)", items[0]["category"])
|
||||
}
|
||||
if got := fmt.Sprint(items[0]["category_id"]); got != "50" {
|
||||
t.Fatalf("category_id=%v want 50", items[0]["category_id"])
|
||||
}
|
||||
if got := fmt.Sprint(items[0]["category_name"]); got != "Štedilniki" {
|
||||
t.Fatalf("category_name=%v want Štedilniki", items[0]["category_name"])
|
||||
|
||||
@@ -375,9 +375,9 @@ func TestRunSteps_synthPriorHashDoesNotSkipReprocess(t *testing.T) {
|
||||
}
|
||||
normName := "Vogel's WALL 3245 TV Wall Mount"
|
||||
normDesc := "A mount with enough mapped detail for hashing inputs."
|
||||
synthPrior := synthesizeDescriptionFromTitle(normName, "TV Mounts", "en", map[string]any{
|
||||
"brand": "Vogel's", "width": "45 cm", "max_load": "40 kg",
|
||||
})
|
||||
// Use legacy invent phrasing as the stored prior — new factual fallback must not
|
||||
// look like invent, but old catalog rows still do and must force re-enhance.
|
||||
synthPrior := normName + " is a TV Mounts product from Vogel's. Key specs: width 45 cm, max_load 40 kg."
|
||||
if !company.LooksLikeHeuristicSynthesize(synthPrior) {
|
||||
t.Fatalf("expected invent synth prior, got %q", synthPrior)
|
||||
}
|
||||
|
||||
@@ -17,12 +17,15 @@ import (
|
||||
// Meta instructions come from description_template.metaTitle / metaDescription
|
||||
// (legacy A1 / cats.json) and are distinct from HTML description sections.
|
||||
// Attribute allowlist / formula-key guidance is AppendAttributeConstraints.
|
||||
func AppendFormulaConstraints(userTpl string, titleTemplate, descriptionTemplate any) string {
|
||||
// When omitSEOMeta is true (A1 cohort), meta formula blocks are skipped.
|
||||
func AppendFormulaConstraints(userTpl string, titleTemplate, descriptionTemplate any, omitSEOMeta bool) string {
|
||||
userTpl = strings.TrimSpace(userTpl)
|
||||
blocks := []string{
|
||||
FormatTitleFormulaConstraint(titleTemplate),
|
||||
FormatDescriptionFormulaConstraint(descriptionTemplate),
|
||||
FormatMetaFormulaConstraint(descriptionTemplate),
|
||||
}
|
||||
if !omitSEOMeta {
|
||||
blocks = append(blocks, FormatMetaFormulaConstraint(descriptionTemplate))
|
||||
}
|
||||
var joined strings.Builder
|
||||
for _, block := range blocks {
|
||||
|
||||
@@ -228,7 +228,38 @@ func isDimensionKey(key string) bool {
|
||||
|
||||
func isZeroishString(s string) bool {
|
||||
s = strings.TrimSpace(strings.ToLower(s))
|
||||
return s == "0" || s == "0.0" || s == "0,0" || s == "0.00"
|
||||
if s == "" {
|
||||
return false
|
||||
}
|
||||
if s == "0" || s == "0.0" || s == "0,0" || s == "0.00" {
|
||||
return true
|
||||
}
|
||||
// Strip common unit suffixes (m, cm, mm, kg, g) then re-check numeric zero.
|
||||
trimmed := s
|
||||
for _, u := range []string{"kg", "cm", "mm", "g", "m"} {
|
||||
if strings.HasSuffix(trimmed, u) {
|
||||
trimmed = strings.TrimSpace(strings.TrimSuffix(trimmed, u))
|
||||
break
|
||||
}
|
||||
}
|
||||
trimmed = strings.ReplaceAll(trimmed, " ", "")
|
||||
if trimmed == "" {
|
||||
return false
|
||||
}
|
||||
// 0 / 0.0 / 0,00 / 0.0000 / 0,0000
|
||||
onlyZero := true
|
||||
sawDigit := false
|
||||
for _, r := range trimmed {
|
||||
switch r {
|
||||
case '0':
|
||||
sawDigit = true
|
||||
case '.', ',':
|
||||
continue
|
||||
default:
|
||||
onlyZero = false
|
||||
}
|
||||
}
|
||||
return onlyZero && sawDigit
|
||||
}
|
||||
|
||||
func isEmptyValue(v any) bool {
|
||||
|
||||
@@ -879,6 +879,8 @@ type jobScopedCache struct {
|
||||
canUseAI bool
|
||||
allowEPREL bool
|
||||
remainingCredits int
|
||||
// omitSEOMeta skips meta_title / meta_description enhance + free-template fill (A1).
|
||||
omitSEOMeta bool
|
||||
}
|
||||
|
||||
func (c *jobScopedCache) stepPolicy() StepPolicy {
|
||||
@@ -927,6 +929,7 @@ func (p *Pipeline) loadJobScopedCache(ctx context.Context, companyID, jobID uuid
|
||||
}
|
||||
}
|
||||
}
|
||||
cache.omitSEOMeta = companyOmitsSEOMeta(ctx, p, companyID)
|
||||
if p.Prompts != nil {
|
||||
cache.enhanceByLang = make(map[string]PromptTemplates, len(cache.contentLanguages)+1)
|
||||
langs := cache.contentLanguages
|
||||
@@ -1526,6 +1529,9 @@ func (p *Pipeline) processOne(ctx context.Context, companyID, jobID uuid.UUID, i
|
||||
CompanyID: companyID.String(),
|
||||
RawProductID: it.RawID.String(),
|
||||
}
|
||||
if cache != nil {
|
||||
in.OmitSEOMeta = cache.omitSEOMeta
|
||||
}
|
||||
if it.hydrated {
|
||||
if it.hasPrior {
|
||||
in.PriorProcessedName = it.priorName
|
||||
@@ -1592,7 +1598,18 @@ func (p *Pipeline) processOne(ctx context.Context, companyID, jobID uuid.UUID, i
|
||||
result.ProcessedAttributes = result.Attributes
|
||||
}
|
||||
// Free template SEO meta when enhance did not emit meta_* (no FillMetaAI / no extra credits).
|
||||
if strings.TrimSpace(result.MetaTitle) == "" || strings.TrimSpace(result.MetaDescription) == "" {
|
||||
// A1 cohort omits SEO meta entirely (not needed for their storefront).
|
||||
if cache != nil && cache.omitSEOMeta {
|
||||
result.MetaTitle = ""
|
||||
result.MetaDescription = ""
|
||||
if result.LocalizedContent != nil {
|
||||
for lang, lf := range result.LocalizedContent {
|
||||
lf.MetaTitle = ""
|
||||
lf.MetaDescription = ""
|
||||
result.LocalizedContent[lang] = lf
|
||||
}
|
||||
}
|
||||
} else if strings.TrimSpace(result.MetaTitle) == "" || strings.TrimSpace(result.MetaDescription) == "" {
|
||||
mt, md := fillMetaFromResult(result)
|
||||
if strings.TrimSpace(result.MetaTitle) == "" {
|
||||
result.MetaTitle = mt
|
||||
|
||||
@@ -25,7 +25,7 @@ func resolveProductPromptTemplates(in ProductInput) (systemTpl, userTpl string)
|
||||
}
|
||||
}
|
||||
// Category formulas are language-agnostic; inject once into the shared user skeleton.
|
||||
userTpl = AppendFormulaConstraints(userTpl, in.TitleTemplate, in.DescriptionTemplate)
|
||||
userTpl = AppendFormulaConstraints(userTpl, in.TitleTemplate, in.DescriptionTemplate, in.OmitSEOMeta)
|
||||
// Category attribute allowlist + title-formula keys guide JSON "attrs" extraction.
|
||||
allowed := enhanceAllowedAttrKeys(in, in.CategoryUniqueID)
|
||||
userTpl = AppendAttributeConstraints(userTpl, allowed, in.TitleTemplate)
|
||||
@@ -34,7 +34,9 @@ func resolveProductPromptTemplates(in ProductInput) (systemTpl, userTpl string)
|
||||
systemTpl = AppendDescriptionFormulaSystemOverride(systemTpl, in.DescriptionTemplate)
|
||||
// Same for title_template vs "short retail title" — formulas win for name structure.
|
||||
systemTpl = AppendTitleFormulaSystemOverride(systemTpl, in.TitleTemplate)
|
||||
systemTpl = AppendMetaFormulaSystemOverride(systemTpl, in.DescriptionTemplate)
|
||||
if !in.OmitSEOMeta {
|
||||
systemTpl = AppendMetaFormulaSystemOverride(systemTpl, in.DescriptionTemplate)
|
||||
}
|
||||
// categories.prompt applies to name, description, and attrs (not description-only).
|
||||
systemTpl = AppendCategoryEnhanceSystemOverlay(systemTpl, catPrompt)
|
||||
return systemTpl, userTpl
|
||||
|
||||
@@ -358,6 +358,9 @@ func (e *Engine) RunSteps(ctx context.Context, companyID string, in ProductInput
|
||||
}
|
||||
weakDesc := descriptionNeedsEnhanceRepair(desc, descTpl, name)
|
||||
enhanceMetaTitle, enhanceMetaDesc := enhanceMetaFromRaw(raw)
|
||||
if in.OmitSEOMeta {
|
||||
enhanceMetaTitle, enhanceMetaDesc = "", ""
|
||||
}
|
||||
// Only persist enhance_input_hash for quality ok / hash-skip unchanged.
|
||||
// Never copy input_hash from error/passthrough/thin/synthesized meta.
|
||||
persistHash := ""
|
||||
@@ -1391,7 +1394,7 @@ func preferredProductTitle(gtin string, candidates ...string) string {
|
||||
if c == "" || c == "<nil>" || isPromptLabelTitle(c) {
|
||||
continue
|
||||
}
|
||||
usable = append(usable, c)
|
||||
usable = append(usable, ensureReadableTitleSpacing(c))
|
||||
}
|
||||
for _, c := range usable {
|
||||
if isBrandOnlyTitleAmong(c, usable) {
|
||||
@@ -1479,6 +1482,9 @@ func preferredProductDescription(title string, candidates ...string) string {
|
||||
if containsWeakFillerPhrase(c) {
|
||||
continue
|
||||
}
|
||||
if company.LooksLikeHeuristicSynthesize(c) {
|
||||
continue
|
||||
}
|
||||
return SanitizeOutput(c)
|
||||
}
|
||||
return ""
|
||||
@@ -1508,15 +1514,20 @@ func attrLookupCI(attrs map[string]any, keys ...string) string {
|
||||
}
|
||||
|
||||
func formatAttrDimParts(attrs map[string]any, maxParts int) []string {
|
||||
return formatAttrDimPartsLang(attrs, maxParts, "")
|
||||
}
|
||||
|
||||
func formatAttrDimPartsLang(attrs map[string]any, maxParts int, language string) []string {
|
||||
if attrs == nil || maxParts <= 0 {
|
||||
return nil
|
||||
}
|
||||
prefer := []string{
|
||||
"width", "height", "depth", "weight", "max_load", "max load", "load_capacity",
|
||||
"vesa", "screen_size", "diagonal", "color", "material", "size",
|
||||
"vesa", "screen_size", "diagonal", "color", "material", "size", "energy_class", "warranty",
|
||||
}
|
||||
parts := make([]string, 0, maxParts)
|
||||
seen := map[string]struct{}{}
|
||||
sl := isSlovenianContentLanguage(language)
|
||||
add := func(k, v string) {
|
||||
k = strings.TrimSpace(k)
|
||||
v = strings.TrimSpace(v)
|
||||
@@ -1527,8 +1538,11 @@ func formatAttrDimParts(attrs map[string]any, maxParts int) []string {
|
||||
if _, ok := seen[lk]; ok {
|
||||
return
|
||||
}
|
||||
if isDimensionKey(canonicalizeAttrKey(k)) && isZeroishString(v) {
|
||||
return
|
||||
}
|
||||
seen[lk] = struct{}{}
|
||||
parts = append(parts, fmt.Sprintf("%s %s", k, v))
|
||||
parts = append(parts, fmt.Sprintf("%s: %s", attrDimLabel(k, sl), v))
|
||||
}
|
||||
for _, k := range prefer {
|
||||
if len(parts) >= maxParts {
|
||||
@@ -1541,9 +1555,61 @@ func formatAttrDimParts(attrs map[string]any, maxParts int) []string {
|
||||
return parts
|
||||
}
|
||||
|
||||
func attrDimLabel(key string, slovenian bool) string {
|
||||
canon := canonicalizeAttrKey(key)
|
||||
if slovenian {
|
||||
switch canon {
|
||||
case "width":
|
||||
return "Širina"
|
||||
case "height":
|
||||
return "Višina"
|
||||
case "depth":
|
||||
return "Globina"
|
||||
case "weight":
|
||||
return "Teža"
|
||||
case "energy_class":
|
||||
return "Energijski razred"
|
||||
case "warranty":
|
||||
return "Garancija"
|
||||
case "color":
|
||||
return "Barva"
|
||||
case "material":
|
||||
return "Material"
|
||||
case "size", "screen_size", "diagonal":
|
||||
return "Velikost"
|
||||
}
|
||||
}
|
||||
switch canon {
|
||||
case "width":
|
||||
return "Width"
|
||||
case "height":
|
||||
return "Height"
|
||||
case "depth":
|
||||
return "Depth"
|
||||
case "weight":
|
||||
return "Weight"
|
||||
case "energy_class":
|
||||
return "Energy class"
|
||||
case "warranty":
|
||||
return "Warranty"
|
||||
case "max_load", "load_capacity":
|
||||
return "Max load"
|
||||
case "screen_size", "diagonal":
|
||||
return "Screen size"
|
||||
case "product_model":
|
||||
return "Model"
|
||||
default:
|
||||
if canon == "" {
|
||||
return key
|
||||
}
|
||||
return strings.ReplaceAll(canon, "_", " ")
|
||||
}
|
||||
}
|
||||
|
||||
// synthesizeProductDescription prefers a category description_template skeleton
|
||||
// (HTML section types) when present; otherwise falls back to plain title synthesize.
|
||||
func synthesizeProductDescription(title, category, language string, attrs map[string]any, descriptionTemplate any) string {
|
||||
title = ensureReadableTitleSpacing(title)
|
||||
if sections, ok := parseDescriptionFormulaSections(descriptionTemplate); ok && len(sections) > 0 {
|
||||
if out := synthesizeDescriptionFromFormula(title, category, language, attrs, sections); out != "" {
|
||||
return out
|
||||
@@ -1555,23 +1621,30 @@ func synthesizeProductDescription(title, category, language string, attrs map[st
|
||||
// synthesizeDescriptionFromFormula builds a minimal HTML description matching
|
||||
// category description_template section types so timeout/fallback still respects
|
||||
// A1 category structure (unlike plain title synthesize).
|
||||
// Duplicate paragraph/list section types are emitted once to avoid invent-boilerplate
|
||||
// repetition (legacy bug: every <p> repeated the same synthesize sentence).
|
||||
func synthesizeDescriptionFromFormula(title, category, language string, attrs map[string]any, sections []descriptionFormulaSection) string {
|
||||
title = strings.TrimSpace(title)
|
||||
title = ensureReadableTitleSpacing(strings.TrimSpace(title))
|
||||
if title == "" || title == "<nil>" || isPromptLabelTitle(title) {
|
||||
return ""
|
||||
}
|
||||
base := synthesizeDescriptionFromTitle(title, category, language, attrs)
|
||||
if base == "" {
|
||||
dims := formatAttrDimPartsLang(attrs, 6, language)
|
||||
intro := factualDescriptionIntro(title, category, language, attrs)
|
||||
if intro == "" {
|
||||
return ""
|
||||
}
|
||||
dims := formatAttrDimParts(attrs, 6)
|
||||
var b strings.Builder
|
||||
paraUsed := false
|
||||
listUsed := false
|
||||
headingLevels := map[string]bool{}
|
||||
for _, s := range sections {
|
||||
typ := strings.ToLower(strings.TrimSpace(s.Type))
|
||||
switch typ {
|
||||
case "h1", "h2", "h3", "h4":
|
||||
if headingLevels[typ] {
|
||||
continue
|
||||
}
|
||||
headingLevels[typ] = true
|
||||
heading := title
|
||||
if typ != "h1" {
|
||||
if isSlovenianContentLanguage(language) {
|
||||
@@ -1582,37 +1655,42 @@ func synthesizeDescriptionFromFormula(title, category, language string, attrs ma
|
||||
}
|
||||
fmt.Fprintf(&b, "<%s>%s</%s>", typ, SanitizeOutput(heading), typ)
|
||||
case "ul":
|
||||
b.WriteString("<ul>")
|
||||
if listUsed {
|
||||
continue
|
||||
}
|
||||
items := dims
|
||||
if len(items) == 0 {
|
||||
items = []string{base}
|
||||
if secondary := factualSecondaryFacts(language, attrs); len(secondary) > 0 {
|
||||
items = secondary
|
||||
} else {
|
||||
items = []string{intro}
|
||||
}
|
||||
}
|
||||
b.WriteString("<ul>")
|
||||
for _, it := range items {
|
||||
fmt.Fprintf(&b, "<li>%s</li>", SanitizeOutput(it))
|
||||
}
|
||||
b.WriteString("</ul>")
|
||||
listUsed = true
|
||||
default: // p and unknown → paragraph
|
||||
body := base
|
||||
if paraUsed && len(dims) > 0 && !listUsed {
|
||||
if isSlovenianContentLanguage(language) {
|
||||
body = "Ključne specifikacije: " + strings.Join(dims, ", ") + "."
|
||||
} else {
|
||||
body = "Key specs: " + strings.Join(dims, ", ") + "."
|
||||
}
|
||||
default: // p and unknown → paragraph (once)
|
||||
if paraUsed {
|
||||
continue
|
||||
}
|
||||
fmt.Fprintf(&b, "<p>%s</p>", SanitizeOutput(body))
|
||||
fmt.Fprintf(&b, "<p>%s</p>", SanitizeOutput(intro))
|
||||
paraUsed = true
|
||||
}
|
||||
}
|
||||
return SanitizeOutput(b.String())
|
||||
out := strings.TrimSpace(b.String())
|
||||
if out == "" {
|
||||
return ""
|
||||
}
|
||||
return SanitizeOutput(out)
|
||||
}
|
||||
|
||||
// synthesizeDescriptionFromTitle builds a short factual fallback when enhance
|
||||
// returns empty/title-echo/filler copy. Uses title, category, brand, model, and
|
||||
// key dims. language is a content-language code (en/sl/…) or English label.
|
||||
func synthesizeDescriptionFromTitle(title, category, language string, attrs map[string]any) string {
|
||||
title = strings.TrimSpace(title)
|
||||
// factualDescriptionIntro builds a short non-invent paragraph for formula fallback.
|
||||
// Intentionally avoids heuristicSynthesizePhrases ("je izdelek v kategoriji", …).
|
||||
func factualDescriptionIntro(title, category, language string, attrs map[string]any) string {
|
||||
title = ensureReadableTitleSpacing(strings.TrimSpace(title))
|
||||
if title == "" || title == "<nil>" || isPromptLabelTitle(title) {
|
||||
return ""
|
||||
}
|
||||
@@ -1622,59 +1700,73 @@ func synthesizeDescriptionFromTitle(title, category, language string, attrs map[
|
||||
}
|
||||
brand := attrLookupCI(attrs, "brand")
|
||||
model := attrLookupCI(attrs, "product_model", "model", "sku")
|
||||
dims := formatAttrDimParts(attrs, 3)
|
||||
dims := formatAttrDimPartsLang(attrs, 3, language)
|
||||
sl := isSlovenianContentLanguage(language)
|
||||
|
||||
var b strings.Builder
|
||||
b.WriteString(title)
|
||||
if sl {
|
||||
b.WriteString(title)
|
||||
switch {
|
||||
case cat != "" && brand != "":
|
||||
fmt.Fprintf(&b, " je izdelek v kategoriji %s znamke %s", cat, brand)
|
||||
case cat != "":
|
||||
fmt.Fprintf(&b, " je izdelek v kategoriji %s", cat)
|
||||
case brand != "" && cat != "":
|
||||
fmt.Fprintf(&b, " — %s, znamka %s", cat, brand)
|
||||
case brand != "":
|
||||
fmt.Fprintf(&b, " je izdelek znamke %s", brand)
|
||||
fmt.Fprintf(&b, " — znamka %s", brand)
|
||||
case cat != "":
|
||||
fmt.Fprintf(&b, " — %s", cat)
|
||||
default:
|
||||
b.WriteString(" je katalogski izdelek z znanimi atributi")
|
||||
b.WriteString(" — katalogski izdelek")
|
||||
}
|
||||
if model != "" && !strings.Contains(strings.ToLower(title), strings.ToLower(model)) {
|
||||
fmt.Fprintf(&b, " (model %s)", model)
|
||||
}
|
||||
if len(dims) > 0 {
|
||||
fmt.Fprintf(&b, ". Ključne specifikacije: %s", strings.Join(dims, ", "))
|
||||
fmt.Fprintf(&b, ". %s", strings.Join(dims, ", "))
|
||||
}
|
||||
b.WriteByte('.')
|
||||
} else {
|
||||
b.WriteString(title)
|
||||
switch {
|
||||
case cat != "" && brand != "":
|
||||
fmt.Fprintf(&b, " is a %s product from %s", cat, brand)
|
||||
case cat != "":
|
||||
fmt.Fprintf(&b, " is listed in the %s category", cat)
|
||||
case brand != "" && cat != "":
|
||||
fmt.Fprintf(&b, " — %s from %s", cat, brand)
|
||||
case brand != "":
|
||||
fmt.Fprintf(&b, " is a product from %s", brand)
|
||||
fmt.Fprintf(&b, " — from %s", brand)
|
||||
case cat != "":
|
||||
fmt.Fprintf(&b, " — %s", cat)
|
||||
default:
|
||||
b.WriteString(" is a catalog product with the known attributes")
|
||||
b.WriteString(" — catalog product")
|
||||
}
|
||||
if model != "" && !strings.Contains(strings.ToLower(title), strings.ToLower(model)) {
|
||||
fmt.Fprintf(&b, " (model %s)", model)
|
||||
}
|
||||
if len(dims) > 0 {
|
||||
fmt.Fprintf(&b, ". Key specs: %s", strings.Join(dims, ", "))
|
||||
fmt.Fprintf(&b, ". %s", strings.Join(dims, ", "))
|
||||
}
|
||||
b.WriteByte('.')
|
||||
}
|
||||
out := SanitizeOutput(b.String())
|
||||
// Never emit sole retail-filler when title/attrs exist — strip legacy phrase if any helper reintroduces it.
|
||||
if containsWeakFillerPhrase(out) {
|
||||
out = strings.TrimSpace(strings.ReplaceAll(out, "Ready for retail listing.", ""))
|
||||
out = strings.TrimSpace(strings.ReplaceAll(out, "ready for retail listing.", ""))
|
||||
out = strings.TrimSpace(strings.Trim(out, ".")) + "."
|
||||
return SanitizeOutput(b.String())
|
||||
}
|
||||
|
||||
func factualSecondaryFacts(language string, attrs map[string]any) []string {
|
||||
prefer := []string{"energy_class", "warranty", "color", "material"}
|
||||
sl := isSlovenianContentLanguage(language)
|
||||
var out []string
|
||||
for _, k := range prefer {
|
||||
v := attrLookupCI(attrs, k)
|
||||
if v == "" || isZeroishString(v) {
|
||||
continue
|
||||
}
|
||||
out = append(out, fmt.Sprintf("%s: %s", attrDimLabel(k, sl), v))
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// synthesizeDescriptionFromTitle builds a short factual fallback when enhance
|
||||
// returns empty/title-echo/filler copy. Uses title, category, brand, model, and
|
||||
// key dims. language is a content-language code (en/sl/…) or English label.
|
||||
// Prefer synthesizeProductDescription when a description_template is available.
|
||||
func synthesizeDescriptionFromTitle(title, category, language string, attrs map[string]any) string {
|
||||
return factualDescriptionIntro(title, category, language, attrs)
|
||||
}
|
||||
|
||||
func isSlovenianContentLanguage(raw string) bool {
|
||||
raw = strings.TrimSpace(raw)
|
||||
if raw == "" {
|
||||
|
||||
@@ -0,0 +1,22 @@
|
||||
package processing
|
||||
|
||||
import (
|
||||
"regexp"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// Glue between a lowercase letter and an UPPERCASE model token
|
||||
// (e.g. "hladilnikGSXV80PZLE" → "hladilnik GSXV80PZLE").
|
||||
// Digits are excluded from the left side so "GSXV80PZLE" is not split at "0P".
|
||||
var reGlueUpperModel = regexp.MustCompile(`(\p{Ll})([\p{Lu}][\p{Lu}\p{Nd}]{2,})`)
|
||||
|
||||
// ensureReadableTitleSpacing inserts missing spaces before glued model codes and
|
||||
// collapses whitespace. Safe for already-spaced retail titles.
|
||||
func ensureReadableTitleSpacing(title string) string {
|
||||
title = strings.TrimSpace(title)
|
||||
if title == "" || title == "<nil>" {
|
||||
return title
|
||||
}
|
||||
out := reGlueUpperModel.ReplaceAllString(title, "$1 $2")
|
||||
return strings.Join(strings.Fields(out), " ")
|
||||
}
|
||||
@@ -0,0 +1,66 @@
|
||||
package processing
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestEnsureReadableTitleSpacing(t *testing.T) {
|
||||
t.Parallel()
|
||||
cases := []struct {
|
||||
in, want string
|
||||
}{
|
||||
{"LG Ameriški hladilnikGSXV80PZLE", "LG Ameriški hladilnik GSXV80PZLE"},
|
||||
{"LG Ameriški hladilnik GSXV80PZLE", "LG Ameriški hladilnik GSXV80PZLE"},
|
||||
{"Ufesa Magnum", "Ufesa Magnum"},
|
||||
{"", ""},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
got := ensureReadableTitleSpacing(tc.in)
|
||||
if got != tc.want {
|
||||
t.Fatalf("in=%q got=%q want=%q", tc.in, got, tc.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestIsZeroishString_units(t *testing.T) {
|
||||
t.Parallel()
|
||||
for _, s := range []string{"0", "0.0", "0,00", "0,00m", "0.00cm", "0,0000kg"} {
|
||||
if !isZeroishString(s) {
|
||||
t.Fatalf("expected zeroish %q", s)
|
||||
}
|
||||
}
|
||||
for _, s := range []string{"11,0000kg", "73.5", "179", "0.5"} {
|
||||
if isZeroishString(s) {
|
||||
t.Fatalf("expected non-zero %q", s)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestV1ProcessJobItemMarshalJSON_order(t *testing.T) {
|
||||
t.Parallel()
|
||||
item := V1ProcessJobItem{
|
||||
"attributes": map[string]any{"brand": "LG"},
|
||||
"ean": "8806091734365",
|
||||
"title": "LG Fridge",
|
||||
"category": "Hladilniki",
|
||||
"category_id": "11",
|
||||
"processed_product_id": "bbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb",
|
||||
"status": "processed",
|
||||
}
|
||||
b, err := item.MarshalJSON()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
s := string(b)
|
||||
eanAt := strings.Index(s, `"ean"`)
|
||||
titleAt := strings.Index(s, `"title"`)
|
||||
attrsAt := strings.Index(s, `"attributes"`)
|
||||
ppAt := strings.Index(s, `"processed_product_id"`)
|
||||
if eanAt < 0 || titleAt < 0 || attrsAt < 0 || ppAt < 0 {
|
||||
t.Fatalf("missing keys in %s", s)
|
||||
}
|
||||
if !(eanAt < titleAt && titleAt < attrsAt && attrsAt < ppAt) {
|
||||
t.Fatalf("bad key order in %s", s)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,97 @@
|
||||
package processing
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
)
|
||||
|
||||
// v1ProcessItemKeyOrder is the human-readable LegacyProcessItem property order.
|
||||
// Important catalog fields first; internal ids / SEO last.
|
||||
var v1ProcessItemKeyOrder = []string{
|
||||
"ean",
|
||||
"title",
|
||||
"name",
|
||||
"category",
|
||||
"category_id",
|
||||
"category_name",
|
||||
"description",
|
||||
"attributes",
|
||||
"main_image",
|
||||
"more_images",
|
||||
"eprel",
|
||||
"status",
|
||||
"error",
|
||||
"meta_title",
|
||||
"meta_description",
|
||||
"id",
|
||||
"processed_product_id",
|
||||
"raw_product_id",
|
||||
}
|
||||
|
||||
// MarshalJSON emits LegacyProcessItem keys in a stable human-readable order.
|
||||
// ASSUMPTION: JSON object key order is part of the V1 readability contract for A1.
|
||||
func (item V1ProcessJobItem) MarshalJSON() ([]byte, error) {
|
||||
if item == nil {
|
||||
return []byte("null"), nil
|
||||
}
|
||||
var buf bytes.Buffer
|
||||
buf.WriteByte('{')
|
||||
first := true
|
||||
writePair := func(k string, v any) error {
|
||||
if !first {
|
||||
buf.WriteByte(',')
|
||||
}
|
||||
first = false
|
||||
kb, err := json.Marshal(k)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
vb, err := json.Marshal(v)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
buf.Write(kb)
|
||||
buf.WriteByte(':')
|
||||
buf.Write(vb)
|
||||
return nil
|
||||
}
|
||||
seen := map[string]struct{}{}
|
||||
for _, k := range v1ProcessItemKeyOrder {
|
||||
v, ok := item[k]
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
seen[k] = struct{}{}
|
||||
if err := writePair(k, v); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
// Preserve any unexpected keys deterministically (sorted via encoding/json map).
|
||||
extras := map[string]any{}
|
||||
for k, v := range item {
|
||||
if _, ok := seen[k]; ok {
|
||||
continue
|
||||
}
|
||||
extras[k] = v
|
||||
}
|
||||
if len(extras) > 0 {
|
||||
eb, err := json.Marshal(extras)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// eb is `{...}`; splice inner pairs.
|
||||
inner := bytes.TrimSpace(eb)
|
||||
if len(inner) >= 2 && inner[0] == '{' && inner[len(inner)-1] == '}' {
|
||||
inner = inner[1 : len(inner)-1]
|
||||
if len(bytes.TrimSpace(inner)) > 0 {
|
||||
if !first {
|
||||
buf.WriteByte(',')
|
||||
}
|
||||
first = false
|
||||
buf.Write(inner)
|
||||
}
|
||||
}
|
||||
}
|
||||
buf.WriteByte('}')
|
||||
return buf.Bytes(), nil
|
||||
}
|
||||
@@ -8,7 +8,9 @@ import (
|
||||
"regexp"
|
||||
"strings"
|
||||
|
||||
"github.com/descrybe/descrybe-v2/apps/api/internal/billing"
|
||||
"github.com/descrybe/descrybe-v2/apps/api/internal/catalog"
|
||||
"github.com/descrybe/descrybe-v2/apps/api/internal/company"
|
||||
"github.com/descrybe/descrybe-v2/apps/api/internal/eprel"
|
||||
"github.com/google/uuid"
|
||||
)
|
||||
@@ -209,6 +211,11 @@ func (p *Pipeline) LoadV1ProcessJobItems(ctx context.Context, companyID, jobID u
|
||||
}
|
||||
allowedAttrs := loadCompanyAttributeKeySet(ctx, p, companyID)
|
||||
categoryNames := loadCompanyCategoryNameMap(ctx, p, companyID)
|
||||
omitSEOMeta := companyOmitsSEOMeta(ctx, p, companyID)
|
||||
language := ""
|
||||
if p != nil {
|
||||
language = company.LoadLanguage(ctx, p.Pool, companyID)
|
||||
}
|
||||
rows, err := p.Pool.Query(ctx, `
|
||||
SELECT
|
||||
COALESCE(r.gtin, p.product_id, '') AS ean,
|
||||
@@ -356,49 +363,53 @@ func (p *Pipeline) LoadV1ProcessJobItems(ctx context.Context, companyID, jobID u
|
||||
}
|
||||
}
|
||||
|
||||
metaTitleOut := nullIfEmptyPtr(metaTitle)
|
||||
metaDescOut := nullIfEmptyPtr(metaDesc)
|
||||
if s, ok := metaTitleOut.(string); ok && isPoisonedMetaTitle(s) {
|
||||
metaTitleOut = nil
|
||||
// Poisoned title refresh: also drop empty/weak/leakage stub meta_description.
|
||||
if ds, ok := metaDescOut.(string); ok && (ds == "" || isWeakPriorEnhanceDescription(ds, titleStr) || isPromptLeakageTitle(ds)) {
|
||||
titleStr = ensureReadableTitleSpacing(titleStr)
|
||||
|
||||
var metaTitleOut, metaDescOut any
|
||||
if !omitSEOMeta {
|
||||
metaTitleOut = nullIfEmptyPtr(metaTitle)
|
||||
metaDescOut = nullIfEmptyPtr(metaDesc)
|
||||
if s, ok := metaTitleOut.(string); ok && isPoisonedMetaTitle(s) {
|
||||
metaTitleOut = nil
|
||||
if ds, ok := metaDescOut.(string); ok && (ds == "" || isWeakPriorEnhanceDescription(ds, titleStr) || isPromptLeakageTitle(ds)) {
|
||||
metaDescOut = nil
|
||||
}
|
||||
}
|
||||
if ds, ok := metaDescOut.(string); ok && isPromptLeakageTitle(ds) {
|
||||
metaDescOut = nil
|
||||
}
|
||||
}
|
||||
if ds, ok := metaDescOut.(string); ok && isPromptLeakageTitle(ds) {
|
||||
metaDescOut = nil
|
||||
}
|
||||
catLabel := catNameStr
|
||||
if catLabel == "" {
|
||||
catLabel = catStr
|
||||
}
|
||||
if (metaTitleOut == nil || metaDescOut == nil) && (titleStr != "" || descOut != "" || catLabel != "") {
|
||||
synthTitle, synthDesc := fillMetaFromResult(StepResult{
|
||||
Name: titleStr,
|
||||
ProcessedName: titleStr,
|
||||
Category: catStr,
|
||||
CategoryName: catNameStr,
|
||||
Description: descOut,
|
||||
ProcessedDescription: descOut,
|
||||
Attributes: attrs,
|
||||
ProcessedAttributes: attrs,
|
||||
})
|
||||
if metaTitleOut == nil {
|
||||
if synthTitle != "" {
|
||||
metaTitleOut = synthTitle
|
||||
} else {
|
||||
metaTitleOut = nullIfEmptyPtr(title)
|
||||
}
|
||||
catLabel := catNameStr
|
||||
if catLabel == "" {
|
||||
catLabel = catStr
|
||||
}
|
||||
if metaDescOut == nil {
|
||||
if synthDesc != "" {
|
||||
metaDescOut = synthDesc
|
||||
} else if descOut != "" {
|
||||
metaDescOut = truncateMetaDescription(v1PlainDescription(descOut), v1MetaDescriptionMaxChars)
|
||||
if (metaTitleOut == nil || metaDescOut == nil) && (titleStr != "" || descOut != "" || catLabel != "") {
|
||||
synthTitle, synthDesc := fillMetaFromResult(StepResult{
|
||||
Name: titleStr,
|
||||
ProcessedName: titleStr,
|
||||
Category: catStr,
|
||||
CategoryName: catNameStr,
|
||||
Description: descOut,
|
||||
ProcessedDescription: descOut,
|
||||
Attributes: attrs,
|
||||
ProcessedAttributes: attrs,
|
||||
})
|
||||
if metaTitleOut == nil {
|
||||
if synthTitle != "" {
|
||||
metaTitleOut = synthTitle
|
||||
} else {
|
||||
metaTitleOut = nullIfEmptyPtr(title)
|
||||
}
|
||||
}
|
||||
if metaDescOut == nil {
|
||||
if synthDesc != "" {
|
||||
metaDescOut = synthDesc
|
||||
} else if descOut != "" {
|
||||
metaDescOut = truncateMetaDescription(v1PlainDescription(descOut), v1MetaDescriptionMaxChars)
|
||||
}
|
||||
}
|
||||
} else if metaTitleOut == nil {
|
||||
metaTitleOut = nullIfEmptyPtr(title)
|
||||
}
|
||||
} else if metaTitleOut == nil {
|
||||
metaTitleOut = nullIfEmptyPtr(title)
|
||||
}
|
||||
|
||||
var description any
|
||||
@@ -407,32 +418,37 @@ func (p *Pipeline) LoadV1ProcessJobItems(ctx context.Context, companyID, jobID u
|
||||
} else {
|
||||
description = nil
|
||||
}
|
||||
var catOut, catNameOut any
|
||||
if catStr != "" {
|
||||
catOut = catStr
|
||||
}
|
||||
// ASSUMPTION: V1 `category` is the display name; `category_id` is unique_id
|
||||
// (additive). `category_name` mirrors the display name for dual-mode clients.
|
||||
var catNameOut, catIDOut any
|
||||
if catNameStr != "" {
|
||||
catNameOut = catNameStr
|
||||
}
|
||||
if catStr != "" {
|
||||
catIDOut = catStr
|
||||
}
|
||||
|
||||
var titleOut any
|
||||
if titleStr != "" {
|
||||
titleOut = titleStr
|
||||
}
|
||||
item := V1ProcessJobItem{
|
||||
"ean": ean,
|
||||
"status": MapV1JobItemStatus(itemStatus, true),
|
||||
"category": catOut,
|
||||
"category_name": catNameOut,
|
||||
"title": titleOut,
|
||||
"name": titleOut,
|
||||
"meta_title": metaTitleOut,
|
||||
"meta_description": metaDescOut,
|
||||
"description": description,
|
||||
"attributes": nil,
|
||||
"main_image": nil,
|
||||
"more_images": nil,
|
||||
"eprel": eprelVal,
|
||||
"ean": ean,
|
||||
"status": MapV1JobItemStatus(itemStatus, true),
|
||||
"category": catNameOut,
|
||||
"category_id": catIDOut,
|
||||
"category_name": catNameOut,
|
||||
"title": titleOut,
|
||||
"name": titleOut,
|
||||
"description": description,
|
||||
"attributes": nil,
|
||||
"main_image": nil,
|
||||
"more_images": nil,
|
||||
"eprel": eprelVal,
|
||||
}
|
||||
if !omitSEOMeta {
|
||||
item["meta_title"] = metaTitleOut
|
||||
item["meta_description"] = metaDescOut
|
||||
}
|
||||
applyV1ProcessItemIDs(item, processedID, rawProductID)
|
||||
if itemError != nil && *itemError != "" {
|
||||
@@ -447,7 +463,11 @@ func (p *Pipeline) LoadV1ProcessJobItems(ctx context.Context, companyID, jobID u
|
||||
if len(more) > 0 {
|
||||
item["more_images"] = more
|
||||
}
|
||||
item = EnforceV1ProcessCompletedItem(item, "", allowedAttrs)
|
||||
item = EnforceV1ProcessCompletedItemOpts(item, EnforceV1Opts{
|
||||
Language: language,
|
||||
Allowed: allowedAttrs,
|
||||
OmitSEOMeta: omitSEOMeta,
|
||||
})
|
||||
full = append(full, item)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
@@ -463,6 +483,22 @@ func nullIfEmptyPtr(s *string) any {
|
||||
return *s
|
||||
}
|
||||
|
||||
// companyOmitsSEOMeta is true for the A1 cohort (no meta_title / meta_description).
|
||||
func companyOmitsSEOMeta(ctx context.Context, p *Pipeline, companyID uuid.UUID) bool {
|
||||
if p == nil || p.Pool == nil {
|
||||
return false
|
||||
}
|
||||
var legacy string
|
||||
err := p.Pool.QueryRow(ctx, `
|
||||
SELECT COALESCE(legacy_company_id, '')
|
||||
FROM companies
|
||||
WHERE id = $1`, companyID).Scan(&legacy)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
return billing.IsA1CohortCompany(legacy, "")
|
||||
}
|
||||
|
||||
func derefStringPtr(s *string) string {
|
||||
if s == nil {
|
||||
return ""
|
||||
@@ -685,6 +721,7 @@ func ProjectV1ProcessJobItems(storedType string, items []V1ProcessJobItem) []V1P
|
||||
switch step {
|
||||
case "category":
|
||||
projected["category"] = item["category"]
|
||||
projected["category_id"] = item["category_id"]
|
||||
projected["category_name"] = item["category_name"]
|
||||
case "title":
|
||||
projected["title"] = item["title"]
|
||||
@@ -692,10 +729,14 @@ func ProjectV1ProcessJobItems(storedType string, items []V1ProcessJobItem) []V1P
|
||||
if projected["name"] == nil {
|
||||
projected["name"] = item["title"]
|
||||
}
|
||||
projected["meta_title"] = item["meta_title"]
|
||||
if _, ok := item["meta_title"]; ok {
|
||||
projected["meta_title"] = item["meta_title"]
|
||||
}
|
||||
case "description":
|
||||
projected["description"] = item["description"]
|
||||
projected["meta_description"] = item["meta_description"]
|
||||
if _, ok := item["meta_description"]; ok {
|
||||
projected["meta_description"] = item["meta_description"]
|
||||
}
|
||||
case "attributes":
|
||||
projected["attributes"] = item["attributes"]
|
||||
}
|
||||
|
||||
@@ -4,6 +4,7 @@ import (
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"github.com/descrybe/descrybe-v2/apps/api/internal/company"
|
||||
"github.com/descrybe/descrybe-v2/apps/api/internal/eprel"
|
||||
)
|
||||
|
||||
@@ -32,11 +33,13 @@ type V1ProcessItemScorecard struct {
|
||||
|
||||
// ScoreV1ProcessItemOptions tunes structure checks for a poll item.
|
||||
type ScoreV1ProcessItemOptions struct {
|
||||
// MappedCategory, when non-empty, requires item.category to equal it
|
||||
// (projection must surface mapped unique_id).
|
||||
// MappedCategory, when non-empty, requires item.category_id (or legacy
|
||||
// item.category unique_id) to equal it.
|
||||
MappedCategory string
|
||||
// Language is used only for documentation of synthesize paths in tests.
|
||||
Language string
|
||||
// OmitSEOMeta skips meta_title / meta_description presence checks (A1 cohort).
|
||||
OmitSEOMeta bool
|
||||
}
|
||||
|
||||
// ScoreV1ProcessCompletedItem scores a successful (or terminal) V1 process item.
|
||||
@@ -89,28 +92,35 @@ func ScoreV1ProcessCompletedItem(item V1ProcessJobItem, opts ScoreV1ProcessItemO
|
||||
|
||||
sc.HasMetaTitle = stringFromItem(item, "meta_title") != ""
|
||||
sc.HasMetaDescription = stringFromItem(item, "meta_description") != ""
|
||||
if sc.HasTitle && !sc.HasMetaTitle {
|
||||
sc.FailFlags = append(sc.FailFlags, "meta_title_missing")
|
||||
}
|
||||
if sc.HasTitle && !sc.HasMetaDescription {
|
||||
sc.FailFlags = append(sc.FailFlags, "meta_description_missing")
|
||||
}
|
||||
if md := stringFromItem(item, "meta_description"); md != "" && containsWeakFillerPhrase(md) {
|
||||
sc.FailFlags = append(sc.FailFlags, "meta_description_weak_filler")
|
||||
if !opts.OmitSEOMeta {
|
||||
if sc.HasTitle && !sc.HasMetaTitle {
|
||||
sc.FailFlags = append(sc.FailFlags, "meta_title_missing")
|
||||
}
|
||||
if sc.HasTitle && !sc.HasMetaDescription {
|
||||
sc.FailFlags = append(sc.FailFlags, "meta_description_missing")
|
||||
}
|
||||
if md := stringFromItem(item, "meta_description"); md != "" && containsWeakFillerPhrase(md) {
|
||||
sc.FailFlags = append(sc.FailFlags, "meta_description_weak_filler")
|
||||
}
|
||||
}
|
||||
|
||||
cat := stringFromItem(item, "category")
|
||||
catID := stringFromItem(item, "category_id")
|
||||
catName := stringFromItem(item, "category_name")
|
||||
sc.HasCategory = cat != ""
|
||||
sc.HasCategoryName = catName != ""
|
||||
sc.HasCategory = cat != "" || catID != ""
|
||||
sc.HasCategoryName = catName != "" || (cat != "" && catID != "" && cat != catID)
|
||||
mappedCat := strings.TrimSpace(opts.MappedCategory)
|
||||
if mappedCat != "" {
|
||||
if cat == "" {
|
||||
gotUID := catID
|
||||
if gotUID == "" {
|
||||
gotUID = cat // legacy: category held unique_id
|
||||
}
|
||||
if gotUID == "" {
|
||||
sc.FailFlags = append(sc.FailFlags, "category_missing_though_mapped")
|
||||
} else if cat != mappedCat {
|
||||
} else if gotUID != mappedCat && cat != mappedCat {
|
||||
sc.FailFlags = append(sc.FailFlags, "category_mismatch_mapped")
|
||||
}
|
||||
if cat != "" && catName == "" {
|
||||
if catName == "" && (cat == "" || cat == gotUID) {
|
||||
sc.FailFlags = append(sc.FailFlags, "category_name_missing")
|
||||
}
|
||||
}
|
||||
@@ -182,14 +192,28 @@ func ScoreV1ProcessCompletedItem(item V1ProcessJobItem, opts ScoreV1ProcessItemO
|
||||
return sc
|
||||
}
|
||||
|
||||
// EnforceV1ProcessCompletedItem fills LegacyProcessItem projection gaps for a
|
||||
// successful item: nonempty description when title exists (formula HTML
|
||||
// preserved), meta_*, clean attributes, eprel object|null, and image key shapes.
|
||||
// Category must already be set by the caller when mapped provides a unique_id.
|
||||
//
|
||||
// EnforceV1ProcessCompletedItem fills LegacyProcessItem projection gaps for a successful item:
|
||||
// nonempty description when title exists (formula HTML preserved), readable title spacing,
|
||||
// category display name as primary, optional SEO meta (unless omitSEOMeta), clean attrs.
|
||||
// allowed is the company attribute_key set (canonicalized). When nil, only
|
||||
// coreCharacteristicAttrKeys are kept (never leak feed junk like zavora).
|
||||
func EnforceV1ProcessCompletedItem(item V1ProcessJobItem, language string, allowed map[string]struct{}) V1ProcessJobItem {
|
||||
return EnforceV1ProcessCompletedItemOpts(item, EnforceV1Opts{
|
||||
Language: language,
|
||||
Allowed: allowed,
|
||||
})
|
||||
}
|
||||
|
||||
// EnforceV1Opts configures V1 completed-item projection.
|
||||
type EnforceV1Opts struct {
|
||||
Language string
|
||||
Allowed map[string]struct{}
|
||||
OmitSEOMeta bool
|
||||
DescriptionTemplate any
|
||||
}
|
||||
|
||||
// EnforceV1ProcessCompletedItemOpts is the options-aware EnforceV1 entrypoint.
|
||||
func EnforceV1ProcessCompletedItemOpts(item V1ProcessJobItem, opts EnforceV1Opts) V1ProcessJobItem {
|
||||
if item == nil {
|
||||
return item
|
||||
}
|
||||
@@ -198,10 +222,11 @@ func EnforceV1ProcessCompletedItem(item V1ProcessJobItem, language string, allow
|
||||
return item
|
||||
}
|
||||
|
||||
attrs, _ := attrsMapFromItem(item)
|
||||
allowed := opts.Allowed
|
||||
if allowed == nil {
|
||||
allowed = map[string]struct{}{}
|
||||
}
|
||||
attrs, _ := attrsMapFromItem(item)
|
||||
attrs = SanitizeV1ProcessAttributesAllowed(attrs, allowed)
|
||||
if len(attrs) > 0 {
|
||||
item["attributes"] = attrs
|
||||
@@ -209,7 +234,7 @@ func EnforceV1ProcessCompletedItem(item V1ProcessJobItem, language string, allow
|
||||
item["attributes"] = nil
|
||||
}
|
||||
|
||||
title := stringFromItem(item, "title")
|
||||
title := ensureReadableTitleSpacing(stringFromItem(item, "title"))
|
||||
if title != "" {
|
||||
item["title"] = title
|
||||
item["name"] = title
|
||||
@@ -217,18 +242,26 @@ func EnforceV1ProcessCompletedItem(item V1ProcessJobItem, language string, allow
|
||||
item["title"] = nil
|
||||
item["name"] = nil
|
||||
}
|
||||
cat := stringFromItem(item, "category")
|
||||
catName := stringFromItem(item, "category_name")
|
||||
|
||||
catID, catName := projectV1CategoryFields(item)
|
||||
catLabel := catName
|
||||
if catLabel == "" {
|
||||
catLabel = cat
|
||||
catLabel = catID
|
||||
}
|
||||
|
||||
desc, _ := descriptionFromItem(item)
|
||||
// Empty, weak, or title-echo copy must be replaced — never leave description==title.
|
||||
// Formula HTML that satisfies multi-section templates is kept as-is.
|
||||
if title != "" && (desc == "" || isWeakPriorEnhanceDescription(desc, title) || descriptionEchoesTitle(desc, title)) {
|
||||
if synth := synthesizeDescriptionFromTitle(title, catLabel, language, attrs); synth != "" {
|
||||
needsDesc := title != "" && (desc == "" ||
|
||||
isWeakPriorEnhanceDescription(desc, title) ||
|
||||
descriptionEchoesTitle(desc, title) ||
|
||||
company.LooksLikeHeuristicSynthesize(desc))
|
||||
if needsDesc {
|
||||
tpl := opts.DescriptionTemplate
|
||||
if tpl == nil {
|
||||
tpl = inferDescriptionTemplateFromHTML(desc)
|
||||
}
|
||||
if synth := synthesizeProductDescription(title, catLabel, opts.Language, attrs, tpl); synth != "" {
|
||||
desc = synth
|
||||
} else if synth := synthesizeDescriptionFromTitle(title, catLabel, opts.Language, attrs); synth != "" {
|
||||
desc = synth
|
||||
}
|
||||
}
|
||||
@@ -238,66 +271,58 @@ func EnforceV1ProcessCompletedItem(item V1ProcessJobItem, language string, allow
|
||||
item["description"] = nil
|
||||
}
|
||||
|
||||
metaTitle := stringFromItem(item, "meta_title")
|
||||
metaDesc := stringFromItem(item, "meta_description")
|
||||
needMetaTitle := metaTitle == "" || isPoisonedMetaTitle(metaTitle)
|
||||
// Empty, weak stub (Ready for retail…), title-echo, or prompt-leakage — refresh.
|
||||
// When meta_title is poisoned, also force refresh of weak/leakage meta_description.
|
||||
needMetaDesc := metaDesc == "" ||
|
||||
isWeakPriorEnhanceDescription(metaDesc, title) ||
|
||||
containsWeakFillerPhrase(metaDesc) ||
|
||||
isPromptLeakageTitle(metaDesc) ||
|
||||
(title != "" && descriptionEchoesTitle(metaDesc, title))
|
||||
if needMetaTitle && isPoisonedMetaTitle(metaTitle) &&
|
||||
(metaDesc == "" || isWeakPriorEnhanceDescription(metaDesc, title) || isPromptLeakageTitle(metaDesc)) {
|
||||
needMetaDesc = true
|
||||
}
|
||||
if title != "" || desc != "" || cat != "" || catName != "" {
|
||||
synthTitle, synthDesc := fillMetaFromResult(StepResult{
|
||||
Name: title,
|
||||
ProcessedName: title,
|
||||
Category: cat,
|
||||
CategoryName: catName,
|
||||
Description: desc,
|
||||
ProcessedDescription: desc,
|
||||
Attributes: attrs,
|
||||
ProcessedAttributes: attrs,
|
||||
})
|
||||
if needMetaTitle {
|
||||
if synthTitle != "" {
|
||||
metaTitle = synthTitle
|
||||
} else {
|
||||
metaTitle = title
|
||||
if opts.OmitSEOMeta {
|
||||
delete(item, "meta_title")
|
||||
delete(item, "meta_description")
|
||||
} else {
|
||||
metaTitle := stringFromItem(item, "meta_title")
|
||||
metaDesc := stringFromItem(item, "meta_description")
|
||||
needMetaTitle := metaTitle == "" || isPoisonedMetaTitle(metaTitle)
|
||||
needMetaDesc := metaDesc == "" ||
|
||||
isWeakPriorEnhanceDescription(metaDesc, title) ||
|
||||
containsWeakFillerPhrase(metaDesc) ||
|
||||
isPromptLeakageTitle(metaDesc) ||
|
||||
(title != "" && descriptionEchoesTitle(metaDesc, title))
|
||||
if needMetaTitle && isPoisonedMetaTitle(metaTitle) &&
|
||||
(metaDesc == "" || isWeakPriorEnhanceDescription(metaDesc, title) || isPromptLeakageTitle(metaDesc)) {
|
||||
needMetaDesc = true
|
||||
}
|
||||
if title != "" || desc != "" || catID != "" || catName != "" {
|
||||
synthTitle, synthDesc := fillMetaFromResult(StepResult{
|
||||
Name: title,
|
||||
ProcessedName: title,
|
||||
Category: catID,
|
||||
CategoryName: catName,
|
||||
Description: desc,
|
||||
ProcessedDescription: desc,
|
||||
Attributes: attrs,
|
||||
ProcessedAttributes: attrs,
|
||||
})
|
||||
if needMetaTitle {
|
||||
if synthTitle != "" {
|
||||
metaTitle = synthTitle
|
||||
} else {
|
||||
metaTitle = title
|
||||
}
|
||||
}
|
||||
if needMetaDesc {
|
||||
if synthDesc != "" {
|
||||
metaDesc = synthDesc
|
||||
} else if desc != "" {
|
||||
metaDesc = truncateMetaDescription(desc, v1MetaDescriptionMaxChars)
|
||||
}
|
||||
}
|
||||
}
|
||||
if needMetaDesc {
|
||||
if synthDesc != "" {
|
||||
metaDesc = synthDesc
|
||||
} else if desc != "" {
|
||||
metaDesc = truncateMetaDescription(desc, v1MetaDescriptionMaxChars)
|
||||
}
|
||||
if metaTitle != "" {
|
||||
item["meta_title"] = metaTitle
|
||||
} else {
|
||||
item["meta_title"] = nil
|
||||
}
|
||||
if metaDesc != "" {
|
||||
item["meta_description"] = metaDesc
|
||||
} else {
|
||||
item["meta_description"] = nil
|
||||
}
|
||||
}
|
||||
if metaTitle != "" {
|
||||
item["meta_title"] = metaTitle
|
||||
} else {
|
||||
item["meta_title"] = nil
|
||||
}
|
||||
if metaDesc != "" {
|
||||
item["meta_description"] = metaDesc
|
||||
} else {
|
||||
item["meta_description"] = nil
|
||||
}
|
||||
|
||||
if cat != "" {
|
||||
item["category"] = cat
|
||||
} else {
|
||||
item["category"] = nil
|
||||
}
|
||||
if catName != "" {
|
||||
item["category_name"] = catName
|
||||
} else {
|
||||
item["category_name"] = nil
|
||||
}
|
||||
|
||||
item["eprel"] = normalizeEPRELValue(item["eprel"])
|
||||
@@ -318,6 +343,61 @@ func EnforceV1ProcessCompletedItem(item V1ProcessJobItem, language string, allow
|
||||
return item
|
||||
}
|
||||
|
||||
// projectV1CategoryFields sets category=display name, category_id=unique_id,
|
||||
// category_name=display name. Returns (unique_id, display_name).
|
||||
func projectV1CategoryFields(item V1ProcessJobItem) (catID, catName string) {
|
||||
catID = stringFromItem(item, "category_id")
|
||||
catName = stringFromItem(item, "category_name")
|
||||
cat := stringFromItem(item, "category")
|
||||
if catID == "" {
|
||||
// Legacy: category held unique_id when category_name differed.
|
||||
if catName != "" && cat != "" && cat != catName {
|
||||
catID = cat
|
||||
} else if catName == "" && cat != "" {
|
||||
catID = cat
|
||||
}
|
||||
}
|
||||
if catName == "" && cat != "" && cat != catID {
|
||||
catName = cat
|
||||
}
|
||||
if catName != "" {
|
||||
item["category"] = catName
|
||||
item["category_name"] = catName
|
||||
} else {
|
||||
item["category"] = nil
|
||||
item["category_name"] = nil
|
||||
}
|
||||
if catID != "" {
|
||||
item["category_id"] = catID
|
||||
} else {
|
||||
delete(item, "category_id")
|
||||
}
|
||||
return catID, catName
|
||||
}
|
||||
|
||||
// inferDescriptionTemplateFromHTML rebuilds a minimal description_template from
|
||||
// tags already present so invent-garbage HTML can be re-synthesized without a DB lookup.
|
||||
func inferDescriptionTemplateFromHTML(html string) any {
|
||||
lower := strings.ToLower(html)
|
||||
var sections []map[string]any
|
||||
for _, typ := range []string{"h1", "h2", "h3", "h4", "p", "ul"} {
|
||||
if strings.Contains(lower, "<"+typ) {
|
||||
sections = append(sections, map[string]any{"type": typ})
|
||||
}
|
||||
}
|
||||
if len(sections) == 0 {
|
||||
return map[string]any{
|
||||
"sections": []map[string]any{
|
||||
{"type": "h1"},
|
||||
{"type": "p"},
|
||||
{"type": "h2"},
|
||||
{"type": "ul"},
|
||||
},
|
||||
}
|
||||
}
|
||||
return map[string]any{"sections": sections}
|
||||
}
|
||||
|
||||
func stringFromItem(item V1ProcessJobItem, key string) string {
|
||||
if item == nil {
|
||||
return ""
|
||||
|
||||
@@ -165,8 +165,11 @@ func TestEnforceV1ProcessCompletedItem_categoryFromCallerPreserved(t *testing.T)
|
||||
"eprel": nil,
|
||||
}
|
||||
out := EnforceV1ProcessCompletedItem(item, "", nil)
|
||||
if out["category"] != "50" {
|
||||
t.Fatalf("category=%v", out["category"])
|
||||
if out["category"] != "Štedilniki" {
|
||||
t.Fatalf("category=%v want display name", out["category"])
|
||||
}
|
||||
if out["category_id"] != "50" {
|
||||
t.Fatalf("category_id=%v want 50", out["category_id"])
|
||||
}
|
||||
if out["category_name"] != "Štedilniki" {
|
||||
t.Fatalf("category_name=%v", out["category_name"])
|
||||
|
||||
@@ -80,29 +80,35 @@ func TestSynthesizeDescriptionFromTitle_brandModelCategory(t *testing.T) {
|
||||
}
|
||||
en := synthesizeDescriptionFromTitle(title, "TV Mounts", "en", attrs)
|
||||
if en == "" {
|
||||
t.Fatal("expected nonempty English invent")
|
||||
t.Fatal("expected nonempty English factual fallback")
|
||||
}
|
||||
if strings.Contains(strings.ToLower(en), "ready for retail listing") {
|
||||
t.Fatalf("must not emit retail filler: %q", en)
|
||||
}
|
||||
if strings.Contains(strings.ToLower(en), "is a ") && strings.Contains(strings.ToLower(en), " product from ") {
|
||||
t.Fatalf("must not emit invent boilerplate: %q", en)
|
||||
}
|
||||
for _, need := range []string{"Vogel", "TV Mounts", "45 cm", "40 kg"} {
|
||||
if !strings.Contains(en, need) {
|
||||
t.Fatalf("English invent missing %q: %q", need, en)
|
||||
t.Fatalf("English fallback missing %q: %q", need, en)
|
||||
}
|
||||
}
|
||||
if isWeakPriorEnhanceDescription(en, title) {
|
||||
t.Fatalf("factual invent must not be weak: %q", en)
|
||||
t.Fatalf("factual fallback must not be weak: %q", en)
|
||||
}
|
||||
|
||||
sl := synthesizeDescriptionFromTitle(title, "TV Mounts", "sl", attrs)
|
||||
if sl == "" || !strings.Contains(sl, "kategoriji") {
|
||||
t.Fatalf("expected Slovenian invent, got %q", sl)
|
||||
if sl == "" || !strings.Contains(sl, "znamka") {
|
||||
t.Fatalf("expected Slovenian factual fallback, got %q", sl)
|
||||
}
|
||||
if strings.Contains(strings.ToLower(sl), "je izdelek v kategoriji") {
|
||||
t.Fatalf("must not emit invent boilerplate: %q", sl)
|
||||
}
|
||||
if strings.Contains(strings.ToLower(sl), "ready for retail listing") {
|
||||
t.Fatalf("SL invent must not emit EN filler: %q", sl)
|
||||
t.Fatalf("SL fallback must not emit EN filler: %q", sl)
|
||||
}
|
||||
if isWeakPriorEnhanceDescription(sl, title) {
|
||||
t.Fatalf("SL factual invent must not be weak: %q", sl)
|
||||
t.Fatalf("SL factual fallback must not be weak: %q", sl)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -134,7 +140,7 @@ func TestInventHeuristicDescription_fromBrandModelCategory(t *testing.T) {
|
||||
|
||||
slSystem := "Write name and description in Slovenian. Return JSON with \"name\"."
|
||||
sl := inventHeuristicDescription(slSystem, user, title)
|
||||
if !strings.Contains(sl, "kategoriji") {
|
||||
t.Fatalf("expected SL invent from system language, got %q", sl)
|
||||
if !strings.Contains(sl, "znamka") {
|
||||
t.Fatalf("expected SL factual invent from system language, got %q", sl)
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user