This commit is contained in:
2026-08-17 01:30:28 +02:00
parent c2429873b3
commit 0dceb3a404
18 changed files with 1021 additions and 219 deletions
+3
View File
@@ -126,6 +126,9 @@ type ProductInput struct {
// CategoryUniqueID is the taxonomy unique_id for overlay/formula keys when
// the enhance display category argument is a localized name.
CategoryUniqueID string
// OmitSEOMeta skips meta_title / meta_description enhance + free-template fill
// (A1 cohort does not use SEO meta fields).
OmitSEOMeta bool
}
// CategoryFormulas holds optional title/description templates for one category key.
@@ -359,8 +359,11 @@ func TestLoadV1ProcessJobItems_resolvesCategoryName(t *testing.T) {
if len(items) != 1 {
t.Fatalf("items=%d want 1", len(items))
}
if got := fmt.Sprint(items[0]["category"]); got != "50" {
t.Fatalf("category=%v want 50", items[0]["category"])
if got := fmt.Sprint(items[0]["category"]); got != "Štedilniki" {
t.Fatalf("category=%v want Štedilniki (display name)", items[0]["category"])
}
if got := fmt.Sprint(items[0]["category_id"]); got != "50" {
t.Fatalf("category_id=%v want 50", items[0]["category_id"])
}
if got := fmt.Sprint(items[0]["category_name"]); got != "Štedilniki" {
t.Fatalf("category_name=%v want Štedilniki", items[0]["category_name"])
@@ -375,9 +375,9 @@ func TestRunSteps_synthPriorHashDoesNotSkipReprocess(t *testing.T) {
}
normName := "Vogel's WALL 3245 TV Wall Mount"
normDesc := "A mount with enough mapped detail for hashing inputs."
synthPrior := synthesizeDescriptionFromTitle(normName, "TV Mounts", "en", map[string]any{
"brand": "Vogel's", "width": "45 cm", "max_load": "40 kg",
})
// Use legacy invent phrasing as the stored prior — new factual fallback must not
// look like invent, but old catalog rows still do and must force re-enhance.
synthPrior := normName + " is a TV Mounts product from Vogel's. Key specs: width 45 cm, max_load 40 kg."
if !company.LooksLikeHeuristicSynthesize(synthPrior) {
t.Fatalf("expected invent synth prior, got %q", synthPrior)
}
@@ -17,12 +17,15 @@ import (
// Meta instructions come from description_template.metaTitle / metaDescription
// (legacy A1 / cats.json) and are distinct from HTML description sections.
// Attribute allowlist / formula-key guidance is AppendAttributeConstraints.
func AppendFormulaConstraints(userTpl string, titleTemplate, descriptionTemplate any) string {
// When omitSEOMeta is true (A1 cohort), meta formula blocks are skipped.
func AppendFormulaConstraints(userTpl string, titleTemplate, descriptionTemplate any, omitSEOMeta bool) string {
userTpl = strings.TrimSpace(userTpl)
blocks := []string{
FormatTitleFormulaConstraint(titleTemplate),
FormatDescriptionFormulaConstraint(descriptionTemplate),
FormatMetaFormulaConstraint(descriptionTemplate),
}
if !omitSEOMeta {
blocks = append(blocks, FormatMetaFormulaConstraint(descriptionTemplate))
}
var joined strings.Builder
for _, block := range blocks {
+32 -1
View File
@@ -228,7 +228,38 @@ func isDimensionKey(key string) bool {
func isZeroishString(s string) bool {
s = strings.TrimSpace(strings.ToLower(s))
return s == "0" || s == "0.0" || s == "0,0" || s == "0.00"
if s == "" {
return false
}
if s == "0" || s == "0.0" || s == "0,0" || s == "0.00" {
return true
}
// Strip common unit suffixes (m, cm, mm, kg, g) then re-check numeric zero.
trimmed := s
for _, u := range []string{"kg", "cm", "mm", "g", "m"} {
if strings.HasSuffix(trimmed, u) {
trimmed = strings.TrimSpace(strings.TrimSuffix(trimmed, u))
break
}
}
trimmed = strings.ReplaceAll(trimmed, " ", "")
if trimmed == "" {
return false
}
// 0 / 0.0 / 0,00 / 0.0000 / 0,0000
onlyZero := true
sawDigit := false
for _, r := range trimmed {
switch r {
case '0':
sawDigit = true
case '.', ',':
continue
default:
onlyZero = false
}
}
return onlyZero && sawDigit
}
func isEmptyValue(v any) bool {
+18 -1
View File
@@ -879,6 +879,8 @@ type jobScopedCache struct {
canUseAI bool
allowEPREL bool
remainingCredits int
// omitSEOMeta skips meta_title / meta_description enhance + free-template fill (A1).
omitSEOMeta bool
}
func (c *jobScopedCache) stepPolicy() StepPolicy {
@@ -927,6 +929,7 @@ func (p *Pipeline) loadJobScopedCache(ctx context.Context, companyID, jobID uuid
}
}
}
cache.omitSEOMeta = companyOmitsSEOMeta(ctx, p, companyID)
if p.Prompts != nil {
cache.enhanceByLang = make(map[string]PromptTemplates, len(cache.contentLanguages)+1)
langs := cache.contentLanguages
@@ -1526,6 +1529,9 @@ func (p *Pipeline) processOne(ctx context.Context, companyID, jobID uuid.UUID, i
CompanyID: companyID.String(),
RawProductID: it.RawID.String(),
}
if cache != nil {
in.OmitSEOMeta = cache.omitSEOMeta
}
if it.hydrated {
if it.hasPrior {
in.PriorProcessedName = it.priorName
@@ -1592,7 +1598,18 @@ func (p *Pipeline) processOne(ctx context.Context, companyID, jobID uuid.UUID, i
result.ProcessedAttributes = result.Attributes
}
// Free template SEO meta when enhance did not emit meta_* (no FillMetaAI / no extra credits).
if strings.TrimSpace(result.MetaTitle) == "" || strings.TrimSpace(result.MetaDescription) == "" {
// A1 cohort omits SEO meta entirely (not needed for their storefront).
if cache != nil && cache.omitSEOMeta {
result.MetaTitle = ""
result.MetaDescription = ""
if result.LocalizedContent != nil {
for lang, lf := range result.LocalizedContent {
lf.MetaTitle = ""
lf.MetaDescription = ""
result.LocalizedContent[lang] = lf
}
}
} else if strings.TrimSpace(result.MetaTitle) == "" || strings.TrimSpace(result.MetaDescription) == "" {
mt, md := fillMetaFromResult(result)
if strings.TrimSpace(result.MetaTitle) == "" {
result.MetaTitle = mt
@@ -25,7 +25,7 @@ func resolveProductPromptTemplates(in ProductInput) (systemTpl, userTpl string)
}
}
// Category formulas are language-agnostic; inject once into the shared user skeleton.
userTpl = AppendFormulaConstraints(userTpl, in.TitleTemplate, in.DescriptionTemplate)
userTpl = AppendFormulaConstraints(userTpl, in.TitleTemplate, in.DescriptionTemplate, in.OmitSEOMeta)
// Category attribute allowlist + title-formula keys guide JSON "attrs" extraction.
allowed := enhanceAllowedAttrKeys(in, in.CategoryUniqueID)
userTpl = AppendAttributeConstraints(userTpl, allowed, in.TitleTemplate)
@@ -34,7 +34,9 @@ func resolveProductPromptTemplates(in ProductInput) (systemTpl, userTpl string)
systemTpl = AppendDescriptionFormulaSystemOverride(systemTpl, in.DescriptionTemplate)
// Same for title_template vs "short retail title" — formulas win for name structure.
systemTpl = AppendTitleFormulaSystemOverride(systemTpl, in.TitleTemplate)
systemTpl = AppendMetaFormulaSystemOverride(systemTpl, in.DescriptionTemplate)
if !in.OmitSEOMeta {
systemTpl = AppendMetaFormulaSystemOverride(systemTpl, in.DescriptionTemplate)
}
// categories.prompt applies to name, description, and attrs (not description-only).
systemTpl = AppendCategoryEnhanceSystemOverlay(systemTpl, catPrompt)
return systemTpl, userTpl
+139 -47
View File
@@ -358,6 +358,9 @@ func (e *Engine) RunSteps(ctx context.Context, companyID string, in ProductInput
}
weakDesc := descriptionNeedsEnhanceRepair(desc, descTpl, name)
enhanceMetaTitle, enhanceMetaDesc := enhanceMetaFromRaw(raw)
if in.OmitSEOMeta {
enhanceMetaTitle, enhanceMetaDesc = "", ""
}
// Only persist enhance_input_hash for quality ok / hash-skip unchanged.
// Never copy input_hash from error/passthrough/thin/synthesized meta.
persistHash := ""
@@ -1391,7 +1394,7 @@ func preferredProductTitle(gtin string, candidates ...string) string {
if c == "" || c == "<nil>" || isPromptLabelTitle(c) {
continue
}
usable = append(usable, c)
usable = append(usable, ensureReadableTitleSpacing(c))
}
for _, c := range usable {
if isBrandOnlyTitleAmong(c, usable) {
@@ -1479,6 +1482,9 @@ func preferredProductDescription(title string, candidates ...string) string {
if containsWeakFillerPhrase(c) {
continue
}
if company.LooksLikeHeuristicSynthesize(c) {
continue
}
return SanitizeOutput(c)
}
return ""
@@ -1508,15 +1514,20 @@ func attrLookupCI(attrs map[string]any, keys ...string) string {
}
func formatAttrDimParts(attrs map[string]any, maxParts int) []string {
return formatAttrDimPartsLang(attrs, maxParts, "")
}
func formatAttrDimPartsLang(attrs map[string]any, maxParts int, language string) []string {
if attrs == nil || maxParts <= 0 {
return nil
}
prefer := []string{
"width", "height", "depth", "weight", "max_load", "max load", "load_capacity",
"vesa", "screen_size", "diagonal", "color", "material", "size",
"vesa", "screen_size", "diagonal", "color", "material", "size", "energy_class", "warranty",
}
parts := make([]string, 0, maxParts)
seen := map[string]struct{}{}
sl := isSlovenianContentLanguage(language)
add := func(k, v string) {
k = strings.TrimSpace(k)
v = strings.TrimSpace(v)
@@ -1527,8 +1538,11 @@ func formatAttrDimParts(attrs map[string]any, maxParts int) []string {
if _, ok := seen[lk]; ok {
return
}
if isDimensionKey(canonicalizeAttrKey(k)) && isZeroishString(v) {
return
}
seen[lk] = struct{}{}
parts = append(parts, fmt.Sprintf("%s %s", k, v))
parts = append(parts, fmt.Sprintf("%s: %s", attrDimLabel(k, sl), v))
}
for _, k := range prefer {
if len(parts) >= maxParts {
@@ -1541,9 +1555,61 @@ func formatAttrDimParts(attrs map[string]any, maxParts int) []string {
return parts
}
func attrDimLabel(key string, slovenian bool) string {
canon := canonicalizeAttrKey(key)
if slovenian {
switch canon {
case "width":
return "Širina"
case "height":
return "Višina"
case "depth":
return "Globina"
case "weight":
return "Teža"
case "energy_class":
return "Energijski razred"
case "warranty":
return "Garancija"
case "color":
return "Barva"
case "material":
return "Material"
case "size", "screen_size", "diagonal":
return "Velikost"
}
}
switch canon {
case "width":
return "Width"
case "height":
return "Height"
case "depth":
return "Depth"
case "weight":
return "Weight"
case "energy_class":
return "Energy class"
case "warranty":
return "Warranty"
case "max_load", "load_capacity":
return "Max load"
case "screen_size", "diagonal":
return "Screen size"
case "product_model":
return "Model"
default:
if canon == "" {
return key
}
return strings.ReplaceAll(canon, "_", " ")
}
}
// synthesizeProductDescription prefers a category description_template skeleton
// (HTML section types) when present; otherwise falls back to plain title synthesize.
func synthesizeProductDescription(title, category, language string, attrs map[string]any, descriptionTemplate any) string {
title = ensureReadableTitleSpacing(title)
if sections, ok := parseDescriptionFormulaSections(descriptionTemplate); ok && len(sections) > 0 {
if out := synthesizeDescriptionFromFormula(title, category, language, attrs, sections); out != "" {
return out
@@ -1555,23 +1621,30 @@ func synthesizeProductDescription(title, category, language string, attrs map[st
// synthesizeDescriptionFromFormula builds a minimal HTML description matching
// category description_template section types so timeout/fallback still respects
// A1 category structure (unlike plain title synthesize).
// Duplicate paragraph/list section types are emitted once to avoid invent-boilerplate
// repetition (legacy bug: every <p> repeated the same synthesize sentence).
func synthesizeDescriptionFromFormula(title, category, language string, attrs map[string]any, sections []descriptionFormulaSection) string {
title = strings.TrimSpace(title)
title = ensureReadableTitleSpacing(strings.TrimSpace(title))
if title == "" || title == "<nil>" || isPromptLabelTitle(title) {
return ""
}
base := synthesizeDescriptionFromTitle(title, category, language, attrs)
if base == "" {
dims := formatAttrDimPartsLang(attrs, 6, language)
intro := factualDescriptionIntro(title, category, language, attrs)
if intro == "" {
return ""
}
dims := formatAttrDimParts(attrs, 6)
var b strings.Builder
paraUsed := false
listUsed := false
headingLevels := map[string]bool{}
for _, s := range sections {
typ := strings.ToLower(strings.TrimSpace(s.Type))
switch typ {
case "h1", "h2", "h3", "h4":
if headingLevels[typ] {
continue
}
headingLevels[typ] = true
heading := title
if typ != "h1" {
if isSlovenianContentLanguage(language) {
@@ -1582,37 +1655,42 @@ func synthesizeDescriptionFromFormula(title, category, language string, attrs ma
}
fmt.Fprintf(&b, "<%s>%s</%s>", typ, SanitizeOutput(heading), typ)
case "ul":
b.WriteString("<ul>")
if listUsed {
continue
}
items := dims
if len(items) == 0 {
items = []string{base}
if secondary := factualSecondaryFacts(language, attrs); len(secondary) > 0 {
items = secondary
} else {
items = []string{intro}
}
}
b.WriteString("<ul>")
for _, it := range items {
fmt.Fprintf(&b, "<li>%s</li>", SanitizeOutput(it))
}
b.WriteString("</ul>")
listUsed = true
default: // p and unknown → paragraph
body := base
if paraUsed && len(dims) > 0 && !listUsed {
if isSlovenianContentLanguage(language) {
body = "Ključne specifikacije: " + strings.Join(dims, ", ") + "."
} else {
body = "Key specs: " + strings.Join(dims, ", ") + "."
}
default: // p and unknown → paragraph (once)
if paraUsed {
continue
}
fmt.Fprintf(&b, "<p>%s</p>", SanitizeOutput(body))
fmt.Fprintf(&b, "<p>%s</p>", SanitizeOutput(intro))
paraUsed = true
}
}
return SanitizeOutput(b.String())
out := strings.TrimSpace(b.String())
if out == "" {
return ""
}
return SanitizeOutput(out)
}
// synthesizeDescriptionFromTitle builds a short factual fallback when enhance
// returns empty/title-echo/filler copy. Uses title, category, brand, model, and
// key dims. language is a content-language code (en/sl/…) or English label.
func synthesizeDescriptionFromTitle(title, category, language string, attrs map[string]any) string {
title = strings.TrimSpace(title)
// factualDescriptionIntro builds a short non-invent paragraph for formula fallback.
// Intentionally avoids heuristicSynthesizePhrases ("je izdelek v kategoriji", …).
func factualDescriptionIntro(title, category, language string, attrs map[string]any) string {
title = ensureReadableTitleSpacing(strings.TrimSpace(title))
if title == "" || title == "<nil>" || isPromptLabelTitle(title) {
return ""
}
@@ -1622,59 +1700,73 @@ func synthesizeDescriptionFromTitle(title, category, language string, attrs map[
}
brand := attrLookupCI(attrs, "brand")
model := attrLookupCI(attrs, "product_model", "model", "sku")
dims := formatAttrDimParts(attrs, 3)
dims := formatAttrDimPartsLang(attrs, 3, language)
sl := isSlovenianContentLanguage(language)
var b strings.Builder
b.WriteString(title)
if sl {
b.WriteString(title)
switch {
case cat != "" && brand != "":
fmt.Fprintf(&b, " je izdelek v kategoriji %s znamke %s", cat, brand)
case cat != "":
fmt.Fprintf(&b, " je izdelek v kategoriji %s", cat)
case brand != "" && cat != "":
fmt.Fprintf(&b, " %s, znamka %s", cat, brand)
case brand != "":
fmt.Fprintf(&b, " je izdelek znamke %s", brand)
fmt.Fprintf(&b, " znamka %s", brand)
case cat != "":
fmt.Fprintf(&b, " — %s", cat)
default:
b.WriteString(" je katalogski izdelek z znanimi atributi")
b.WriteString(" katalogski izdelek")
}
if model != "" && !strings.Contains(strings.ToLower(title), strings.ToLower(model)) {
fmt.Fprintf(&b, " (model %s)", model)
}
if len(dims) > 0 {
fmt.Fprintf(&b, ". Ključne specifikacije: %s", strings.Join(dims, ", "))
fmt.Fprintf(&b, ". %s", strings.Join(dims, ", "))
}
b.WriteByte('.')
} else {
b.WriteString(title)
switch {
case cat != "" && brand != "":
fmt.Fprintf(&b, " is a %s product from %s", cat, brand)
case cat != "":
fmt.Fprintf(&b, " is listed in the %s category", cat)
case brand != "" && cat != "":
fmt.Fprintf(&b, " — %s from %s", cat, brand)
case brand != "":
fmt.Fprintf(&b, " is a product from %s", brand)
fmt.Fprintf(&b, " from %s", brand)
case cat != "":
fmt.Fprintf(&b, " — %s", cat)
default:
b.WriteString(" is a catalog product with the known attributes")
b.WriteString(" catalog product")
}
if model != "" && !strings.Contains(strings.ToLower(title), strings.ToLower(model)) {
fmt.Fprintf(&b, " (model %s)", model)
}
if len(dims) > 0 {
fmt.Fprintf(&b, ". Key specs: %s", strings.Join(dims, ", "))
fmt.Fprintf(&b, ". %s", strings.Join(dims, ", "))
}
b.WriteByte('.')
}
out := SanitizeOutput(b.String())
// Never emit sole retail-filler when title/attrs exist — strip legacy phrase if any helper reintroduces it.
if containsWeakFillerPhrase(out) {
out = strings.TrimSpace(strings.ReplaceAll(out, "Ready for retail listing.", ""))
out = strings.TrimSpace(strings.ReplaceAll(out, "ready for retail listing.", ""))
out = strings.TrimSpace(strings.Trim(out, ".")) + "."
return SanitizeOutput(b.String())
}
func factualSecondaryFacts(language string, attrs map[string]any) []string {
prefer := []string{"energy_class", "warranty", "color", "material"}
sl := isSlovenianContentLanguage(language)
var out []string
for _, k := range prefer {
v := attrLookupCI(attrs, k)
if v == "" || isZeroishString(v) {
continue
}
out = append(out, fmt.Sprintf("%s: %s", attrDimLabel(k, sl), v))
}
return out
}
// synthesizeDescriptionFromTitle builds a short factual fallback when enhance
// returns empty/title-echo/filler copy. Uses title, category, brand, model, and
// key dims. language is a content-language code (en/sl/…) or English label.
// Prefer synthesizeProductDescription when a description_template is available.
func synthesizeDescriptionFromTitle(title, category, language string, attrs map[string]any) string {
return factualDescriptionIntro(title, category, language, attrs)
}
func isSlovenianContentLanguage(raw string) bool {
raw = strings.TrimSpace(raw)
if raw == "" {
@@ -0,0 +1,22 @@
package processing
import (
"regexp"
"strings"
)
// Glue between a lowercase letter and an UPPERCASE model token
// (e.g. "hladilnikGSXV80PZLE" → "hladilnik GSXV80PZLE").
// Digits are excluded from the left side so "GSXV80PZLE" is not split at "0P".
var reGlueUpperModel = regexp.MustCompile(`(\p{Ll})([\p{Lu}][\p{Lu}\p{Nd}]{2,})`)
// ensureReadableTitleSpacing inserts missing spaces before glued model codes and
// collapses whitespace. Safe for already-spaced retail titles.
func ensureReadableTitleSpacing(title string) string {
title = strings.TrimSpace(title)
if title == "" || title == "<nil>" {
return title
}
out := reGlueUpperModel.ReplaceAllString(title, "$1 $2")
return strings.Join(strings.Fields(out), " ")
}
@@ -0,0 +1,66 @@
package processing
import (
"strings"
"testing"
)
func TestEnsureReadableTitleSpacing(t *testing.T) {
t.Parallel()
cases := []struct {
in, want string
}{
{"LG Ameriški hladilnikGSXV80PZLE", "LG Ameriški hladilnik GSXV80PZLE"},
{"LG Ameriški hladilnik GSXV80PZLE", "LG Ameriški hladilnik GSXV80PZLE"},
{"Ufesa Magnum", "Ufesa Magnum"},
{"", ""},
}
for _, tc := range cases {
got := ensureReadableTitleSpacing(tc.in)
if got != tc.want {
t.Fatalf("in=%q got=%q want=%q", tc.in, got, tc.want)
}
}
}
func TestIsZeroishString_units(t *testing.T) {
t.Parallel()
for _, s := range []string{"0", "0.0", "0,00", "0,00m", "0.00cm", "0,0000kg"} {
if !isZeroishString(s) {
t.Fatalf("expected zeroish %q", s)
}
}
for _, s := range []string{"11,0000kg", "73.5", "179", "0.5"} {
if isZeroishString(s) {
t.Fatalf("expected non-zero %q", s)
}
}
}
func TestV1ProcessJobItemMarshalJSON_order(t *testing.T) {
t.Parallel()
item := V1ProcessJobItem{
"attributes": map[string]any{"brand": "LG"},
"ean": "8806091734365",
"title": "LG Fridge",
"category": "Hladilniki",
"category_id": "11",
"processed_product_id": "bbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb",
"status": "processed",
}
b, err := item.MarshalJSON()
if err != nil {
t.Fatal(err)
}
s := string(b)
eanAt := strings.Index(s, `"ean"`)
titleAt := strings.Index(s, `"title"`)
attrsAt := strings.Index(s, `"attributes"`)
ppAt := strings.Index(s, `"processed_product_id"`)
if eanAt < 0 || titleAt < 0 || attrsAt < 0 || ppAt < 0 {
t.Fatalf("missing keys in %s", s)
}
if !(eanAt < titleAt && titleAt < attrsAt && attrsAt < ppAt) {
t.Fatalf("bad key order in %s", s)
}
}
@@ -0,0 +1,97 @@
package processing
import (
"bytes"
"encoding/json"
)
// v1ProcessItemKeyOrder is the human-readable LegacyProcessItem property order.
// Important catalog fields first; internal ids / SEO last.
var v1ProcessItemKeyOrder = []string{
"ean",
"title",
"name",
"category",
"category_id",
"category_name",
"description",
"attributes",
"main_image",
"more_images",
"eprel",
"status",
"error",
"meta_title",
"meta_description",
"id",
"processed_product_id",
"raw_product_id",
}
// MarshalJSON emits LegacyProcessItem keys in a stable human-readable order.
// ASSUMPTION: JSON object key order is part of the V1 readability contract for A1.
func (item V1ProcessJobItem) MarshalJSON() ([]byte, error) {
if item == nil {
return []byte("null"), nil
}
var buf bytes.Buffer
buf.WriteByte('{')
first := true
writePair := func(k string, v any) error {
if !first {
buf.WriteByte(',')
}
first = false
kb, err := json.Marshal(k)
if err != nil {
return err
}
vb, err := json.Marshal(v)
if err != nil {
return err
}
buf.Write(kb)
buf.WriteByte(':')
buf.Write(vb)
return nil
}
seen := map[string]struct{}{}
for _, k := range v1ProcessItemKeyOrder {
v, ok := item[k]
if !ok {
continue
}
seen[k] = struct{}{}
if err := writePair(k, v); err != nil {
return nil, err
}
}
// Preserve any unexpected keys deterministically (sorted via encoding/json map).
extras := map[string]any{}
for k, v := range item {
if _, ok := seen[k]; ok {
continue
}
extras[k] = v
}
if len(extras) > 0 {
eb, err := json.Marshal(extras)
if err != nil {
return nil, err
}
// eb is `{...}`; splice inner pairs.
inner := bytes.TrimSpace(eb)
if len(inner) >= 2 && inner[0] == '{' && inner[len(inner)-1] == '}' {
inner = inner[1 : len(inner)-1]
if len(bytes.TrimSpace(inner)) > 0 {
if !first {
buf.WriteByte(',')
}
first = false
buf.Write(inner)
}
}
}
buf.WriteByte('}')
return buf.Bytes(), nil
}
+99 -58
View File
@@ -8,7 +8,9 @@ import (
"regexp"
"strings"
"github.com/descrybe/descrybe-v2/apps/api/internal/billing"
"github.com/descrybe/descrybe-v2/apps/api/internal/catalog"
"github.com/descrybe/descrybe-v2/apps/api/internal/company"
"github.com/descrybe/descrybe-v2/apps/api/internal/eprel"
"github.com/google/uuid"
)
@@ -209,6 +211,11 @@ func (p *Pipeline) LoadV1ProcessJobItems(ctx context.Context, companyID, jobID u
}
allowedAttrs := loadCompanyAttributeKeySet(ctx, p, companyID)
categoryNames := loadCompanyCategoryNameMap(ctx, p, companyID)
omitSEOMeta := companyOmitsSEOMeta(ctx, p, companyID)
language := ""
if p != nil {
language = company.LoadLanguage(ctx, p.Pool, companyID)
}
rows, err := p.Pool.Query(ctx, `
SELECT
COALESCE(r.gtin, p.product_id, '') AS ean,
@@ -356,49 +363,53 @@ func (p *Pipeline) LoadV1ProcessJobItems(ctx context.Context, companyID, jobID u
}
}
metaTitleOut := nullIfEmptyPtr(metaTitle)
metaDescOut := nullIfEmptyPtr(metaDesc)
if s, ok := metaTitleOut.(string); ok && isPoisonedMetaTitle(s) {
metaTitleOut = nil
// Poisoned title refresh: also drop empty/weak/leakage stub meta_description.
if ds, ok := metaDescOut.(string); ok && (ds == "" || isWeakPriorEnhanceDescription(ds, titleStr) || isPromptLeakageTitle(ds)) {
titleStr = ensureReadableTitleSpacing(titleStr)
var metaTitleOut, metaDescOut any
if !omitSEOMeta {
metaTitleOut = nullIfEmptyPtr(metaTitle)
metaDescOut = nullIfEmptyPtr(metaDesc)
if s, ok := metaTitleOut.(string); ok && isPoisonedMetaTitle(s) {
metaTitleOut = nil
if ds, ok := metaDescOut.(string); ok && (ds == "" || isWeakPriorEnhanceDescription(ds, titleStr) || isPromptLeakageTitle(ds)) {
metaDescOut = nil
}
}
if ds, ok := metaDescOut.(string); ok && isPromptLeakageTitle(ds) {
metaDescOut = nil
}
}
if ds, ok := metaDescOut.(string); ok && isPromptLeakageTitle(ds) {
metaDescOut = nil
}
catLabel := catNameStr
if catLabel == "" {
catLabel = catStr
}
if (metaTitleOut == nil || metaDescOut == nil) && (titleStr != "" || descOut != "" || catLabel != "") {
synthTitle, synthDesc := fillMetaFromResult(StepResult{
Name: titleStr,
ProcessedName: titleStr,
Category: catStr,
CategoryName: catNameStr,
Description: descOut,
ProcessedDescription: descOut,
Attributes: attrs,
ProcessedAttributes: attrs,
})
if metaTitleOut == nil {
if synthTitle != "" {
metaTitleOut = synthTitle
} else {
metaTitleOut = nullIfEmptyPtr(title)
}
catLabel := catNameStr
if catLabel == "" {
catLabel = catStr
}
if metaDescOut == nil {
if synthDesc != "" {
metaDescOut = synthDesc
} else if descOut != "" {
metaDescOut = truncateMetaDescription(v1PlainDescription(descOut), v1MetaDescriptionMaxChars)
if (metaTitleOut == nil || metaDescOut == nil) && (titleStr != "" || descOut != "" || catLabel != "") {
synthTitle, synthDesc := fillMetaFromResult(StepResult{
Name: titleStr,
ProcessedName: titleStr,
Category: catStr,
CategoryName: catNameStr,
Description: descOut,
ProcessedDescription: descOut,
Attributes: attrs,
ProcessedAttributes: attrs,
})
if metaTitleOut == nil {
if synthTitle != "" {
metaTitleOut = synthTitle
} else {
metaTitleOut = nullIfEmptyPtr(title)
}
}
if metaDescOut == nil {
if synthDesc != "" {
metaDescOut = synthDesc
} else if descOut != "" {
metaDescOut = truncateMetaDescription(v1PlainDescription(descOut), v1MetaDescriptionMaxChars)
}
}
} else if metaTitleOut == nil {
metaTitleOut = nullIfEmptyPtr(title)
}
} else if metaTitleOut == nil {
metaTitleOut = nullIfEmptyPtr(title)
}
var description any
@@ -407,32 +418,37 @@ func (p *Pipeline) LoadV1ProcessJobItems(ctx context.Context, companyID, jobID u
} else {
description = nil
}
var catOut, catNameOut any
if catStr != "" {
catOut = catStr
}
// ASSUMPTION: V1 `category` is the display name; `category_id` is unique_id
// (additive). `category_name` mirrors the display name for dual-mode clients.
var catNameOut, catIDOut any
if catNameStr != "" {
catNameOut = catNameStr
}
if catStr != "" {
catIDOut = catStr
}
var titleOut any
if titleStr != "" {
titleOut = titleStr
}
item := V1ProcessJobItem{
"ean": ean,
"status": MapV1JobItemStatus(itemStatus, true),
"category": catOut,
"category_name": catNameOut,
"title": titleOut,
"name": titleOut,
"meta_title": metaTitleOut,
"meta_description": metaDescOut,
"description": description,
"attributes": nil,
"main_image": nil,
"more_images": nil,
"eprel": eprelVal,
"ean": ean,
"status": MapV1JobItemStatus(itemStatus, true),
"category": catNameOut,
"category_id": catIDOut,
"category_name": catNameOut,
"title": titleOut,
"name": titleOut,
"description": description,
"attributes": nil,
"main_image": nil,
"more_images": nil,
"eprel": eprelVal,
}
if !omitSEOMeta {
item["meta_title"] = metaTitleOut
item["meta_description"] = metaDescOut
}
applyV1ProcessItemIDs(item, processedID, rawProductID)
if itemError != nil && *itemError != "" {
@@ -447,7 +463,11 @@ func (p *Pipeline) LoadV1ProcessJobItems(ctx context.Context, companyID, jobID u
if len(more) > 0 {
item["more_images"] = more
}
item = EnforceV1ProcessCompletedItem(item, "", allowedAttrs)
item = EnforceV1ProcessCompletedItemOpts(item, EnforceV1Opts{
Language: language,
Allowed: allowedAttrs,
OmitSEOMeta: omitSEOMeta,
})
full = append(full, item)
}
if err := rows.Err(); err != nil {
@@ -463,6 +483,22 @@ func nullIfEmptyPtr(s *string) any {
return *s
}
// companyOmitsSEOMeta is true for the A1 cohort (no meta_title / meta_description).
func companyOmitsSEOMeta(ctx context.Context, p *Pipeline, companyID uuid.UUID) bool {
if p == nil || p.Pool == nil {
return false
}
var legacy string
err := p.Pool.QueryRow(ctx, `
SELECT COALESCE(legacy_company_id, '')
FROM companies
WHERE id = $1`, companyID).Scan(&legacy)
if err != nil {
return false
}
return billing.IsA1CohortCompany(legacy, "")
}
func derefStringPtr(s *string) string {
if s == nil {
return ""
@@ -685,6 +721,7 @@ func ProjectV1ProcessJobItems(storedType string, items []V1ProcessJobItem) []V1P
switch step {
case "category":
projected["category"] = item["category"]
projected["category_id"] = item["category_id"]
projected["category_name"] = item["category_name"]
case "title":
projected["title"] = item["title"]
@@ -692,10 +729,14 @@ func ProjectV1ProcessJobItems(storedType string, items []V1ProcessJobItem) []V1P
if projected["name"] == nil {
projected["name"] = item["title"]
}
projected["meta_title"] = item["meta_title"]
if _, ok := item["meta_title"]; ok {
projected["meta_title"] = item["meta_title"]
}
case "description":
projected["description"] = item["description"]
projected["meta_description"] = item["meta_description"]
if _, ok := item["meta_description"]; ok {
projected["meta_description"] = item["meta_description"]
}
case "attributes":
projected["attributes"] = item["attributes"]
}
+166 -86
View File
@@ -4,6 +4,7 @@ import (
"fmt"
"strings"
"github.com/descrybe/descrybe-v2/apps/api/internal/company"
"github.com/descrybe/descrybe-v2/apps/api/internal/eprel"
)
@@ -32,11 +33,13 @@ type V1ProcessItemScorecard struct {
// ScoreV1ProcessItemOptions tunes structure checks for a poll item.
type ScoreV1ProcessItemOptions struct {
// MappedCategory, when non-empty, requires item.category to equal it
// (projection must surface mapped unique_id).
// MappedCategory, when non-empty, requires item.category_id (or legacy
// item.category unique_id) to equal it.
MappedCategory string
// Language is used only for documentation of synthesize paths in tests.
Language string
// OmitSEOMeta skips meta_title / meta_description presence checks (A1 cohort).
OmitSEOMeta bool
}
// ScoreV1ProcessCompletedItem scores a successful (or terminal) V1 process item.
@@ -89,28 +92,35 @@ func ScoreV1ProcessCompletedItem(item V1ProcessJobItem, opts ScoreV1ProcessItemO
sc.HasMetaTitle = stringFromItem(item, "meta_title") != ""
sc.HasMetaDescription = stringFromItem(item, "meta_description") != ""
if sc.HasTitle && !sc.HasMetaTitle {
sc.FailFlags = append(sc.FailFlags, "meta_title_missing")
}
if sc.HasTitle && !sc.HasMetaDescription {
sc.FailFlags = append(sc.FailFlags, "meta_description_missing")
}
if md := stringFromItem(item, "meta_description"); md != "" && containsWeakFillerPhrase(md) {
sc.FailFlags = append(sc.FailFlags, "meta_description_weak_filler")
if !opts.OmitSEOMeta {
if sc.HasTitle && !sc.HasMetaTitle {
sc.FailFlags = append(sc.FailFlags, "meta_title_missing")
}
if sc.HasTitle && !sc.HasMetaDescription {
sc.FailFlags = append(sc.FailFlags, "meta_description_missing")
}
if md := stringFromItem(item, "meta_description"); md != "" && containsWeakFillerPhrase(md) {
sc.FailFlags = append(sc.FailFlags, "meta_description_weak_filler")
}
}
cat := stringFromItem(item, "category")
catID := stringFromItem(item, "category_id")
catName := stringFromItem(item, "category_name")
sc.HasCategory = cat != ""
sc.HasCategoryName = catName != ""
sc.HasCategory = cat != "" || catID != ""
sc.HasCategoryName = catName != "" || (cat != "" && catID != "" && cat != catID)
mappedCat := strings.TrimSpace(opts.MappedCategory)
if mappedCat != "" {
if cat == "" {
gotUID := catID
if gotUID == "" {
gotUID = cat // legacy: category held unique_id
}
if gotUID == "" {
sc.FailFlags = append(sc.FailFlags, "category_missing_though_mapped")
} else if cat != mappedCat {
} else if gotUID != mappedCat && cat != mappedCat {
sc.FailFlags = append(sc.FailFlags, "category_mismatch_mapped")
}
if cat != "" && catName == "" {
if catName == "" && (cat == "" || cat == gotUID) {
sc.FailFlags = append(sc.FailFlags, "category_name_missing")
}
}
@@ -182,14 +192,28 @@ func ScoreV1ProcessCompletedItem(item V1ProcessJobItem, opts ScoreV1ProcessItemO
return sc
}
// EnforceV1ProcessCompletedItem fills LegacyProcessItem projection gaps for a
// successful item: nonempty description when title exists (formula HTML
// preserved), meta_*, clean attributes, eprel object|null, and image key shapes.
// Category must already be set by the caller when mapped provides a unique_id.
//
// EnforceV1ProcessCompletedItem fills LegacyProcessItem projection gaps for a successful item:
// nonempty description when title exists (formula HTML preserved), readable title spacing,
// category display name as primary, optional SEO meta (unless omitSEOMeta), clean attrs.
// allowed is the company attribute_key set (canonicalized). When nil, only
// coreCharacteristicAttrKeys are kept (never leak feed junk like zavora).
func EnforceV1ProcessCompletedItem(item V1ProcessJobItem, language string, allowed map[string]struct{}) V1ProcessJobItem {
return EnforceV1ProcessCompletedItemOpts(item, EnforceV1Opts{
Language: language,
Allowed: allowed,
})
}
// EnforceV1Opts configures V1 completed-item projection.
type EnforceV1Opts struct {
Language string
Allowed map[string]struct{}
OmitSEOMeta bool
DescriptionTemplate any
}
// EnforceV1ProcessCompletedItemOpts is the options-aware EnforceV1 entrypoint.
func EnforceV1ProcessCompletedItemOpts(item V1ProcessJobItem, opts EnforceV1Opts) V1ProcessJobItem {
if item == nil {
return item
}
@@ -198,10 +222,11 @@ func EnforceV1ProcessCompletedItem(item V1ProcessJobItem, language string, allow
return item
}
attrs, _ := attrsMapFromItem(item)
allowed := opts.Allowed
if allowed == nil {
allowed = map[string]struct{}{}
}
attrs, _ := attrsMapFromItem(item)
attrs = SanitizeV1ProcessAttributesAllowed(attrs, allowed)
if len(attrs) > 0 {
item["attributes"] = attrs
@@ -209,7 +234,7 @@ func EnforceV1ProcessCompletedItem(item V1ProcessJobItem, language string, allow
item["attributes"] = nil
}
title := stringFromItem(item, "title")
title := ensureReadableTitleSpacing(stringFromItem(item, "title"))
if title != "" {
item["title"] = title
item["name"] = title
@@ -217,18 +242,26 @@ func EnforceV1ProcessCompletedItem(item V1ProcessJobItem, language string, allow
item["title"] = nil
item["name"] = nil
}
cat := stringFromItem(item, "category")
catName := stringFromItem(item, "category_name")
catID, catName := projectV1CategoryFields(item)
catLabel := catName
if catLabel == "" {
catLabel = cat
catLabel = catID
}
desc, _ := descriptionFromItem(item)
// Empty, weak, or title-echo copy must be replaced — never leave description==title.
// Formula HTML that satisfies multi-section templates is kept as-is.
if title != "" && (desc == "" || isWeakPriorEnhanceDescription(desc, title) || descriptionEchoesTitle(desc, title)) {
if synth := synthesizeDescriptionFromTitle(title, catLabel, language, attrs); synth != "" {
needsDesc := title != "" && (desc == "" ||
isWeakPriorEnhanceDescription(desc, title) ||
descriptionEchoesTitle(desc, title) ||
company.LooksLikeHeuristicSynthesize(desc))
if needsDesc {
tpl := opts.DescriptionTemplate
if tpl == nil {
tpl = inferDescriptionTemplateFromHTML(desc)
}
if synth := synthesizeProductDescription(title, catLabel, opts.Language, attrs, tpl); synth != "" {
desc = synth
} else if synth := synthesizeDescriptionFromTitle(title, catLabel, opts.Language, attrs); synth != "" {
desc = synth
}
}
@@ -238,66 +271,58 @@ func EnforceV1ProcessCompletedItem(item V1ProcessJobItem, language string, allow
item["description"] = nil
}
metaTitle := stringFromItem(item, "meta_title")
metaDesc := stringFromItem(item, "meta_description")
needMetaTitle := metaTitle == "" || isPoisonedMetaTitle(metaTitle)
// Empty, weak stub (Ready for retail…), title-echo, or prompt-leakage — refresh.
// When meta_title is poisoned, also force refresh of weak/leakage meta_description.
needMetaDesc := metaDesc == "" ||
isWeakPriorEnhanceDescription(metaDesc, title) ||
containsWeakFillerPhrase(metaDesc) ||
isPromptLeakageTitle(metaDesc) ||
(title != "" && descriptionEchoesTitle(metaDesc, title))
if needMetaTitle && isPoisonedMetaTitle(metaTitle) &&
(metaDesc == "" || isWeakPriorEnhanceDescription(metaDesc, title) || isPromptLeakageTitle(metaDesc)) {
needMetaDesc = true
}
if title != "" || desc != "" || cat != "" || catName != "" {
synthTitle, synthDesc := fillMetaFromResult(StepResult{
Name: title,
ProcessedName: title,
Category: cat,
CategoryName: catName,
Description: desc,
ProcessedDescription: desc,
Attributes: attrs,
ProcessedAttributes: attrs,
})
if needMetaTitle {
if synthTitle != "" {
metaTitle = synthTitle
} else {
metaTitle = title
if opts.OmitSEOMeta {
delete(item, "meta_title")
delete(item, "meta_description")
} else {
metaTitle := stringFromItem(item, "meta_title")
metaDesc := stringFromItem(item, "meta_description")
needMetaTitle := metaTitle == "" || isPoisonedMetaTitle(metaTitle)
needMetaDesc := metaDesc == "" ||
isWeakPriorEnhanceDescription(metaDesc, title) ||
containsWeakFillerPhrase(metaDesc) ||
isPromptLeakageTitle(metaDesc) ||
(title != "" && descriptionEchoesTitle(metaDesc, title))
if needMetaTitle && isPoisonedMetaTitle(metaTitle) &&
(metaDesc == "" || isWeakPriorEnhanceDescription(metaDesc, title) || isPromptLeakageTitle(metaDesc)) {
needMetaDesc = true
}
if title != "" || desc != "" || catID != "" || catName != "" {
synthTitle, synthDesc := fillMetaFromResult(StepResult{
Name: title,
ProcessedName: title,
Category: catID,
CategoryName: catName,
Description: desc,
ProcessedDescription: desc,
Attributes: attrs,
ProcessedAttributes: attrs,
})
if needMetaTitle {
if synthTitle != "" {
metaTitle = synthTitle
} else {
metaTitle = title
}
}
if needMetaDesc {
if synthDesc != "" {
metaDesc = synthDesc
} else if desc != "" {
metaDesc = truncateMetaDescription(desc, v1MetaDescriptionMaxChars)
}
}
}
if needMetaDesc {
if synthDesc != "" {
metaDesc = synthDesc
} else if desc != "" {
metaDesc = truncateMetaDescription(desc, v1MetaDescriptionMaxChars)
}
if metaTitle != "" {
item["meta_title"] = metaTitle
} else {
item["meta_title"] = nil
}
if metaDesc != "" {
item["meta_description"] = metaDesc
} else {
item["meta_description"] = nil
}
}
if metaTitle != "" {
item["meta_title"] = metaTitle
} else {
item["meta_title"] = nil
}
if metaDesc != "" {
item["meta_description"] = metaDesc
} else {
item["meta_description"] = nil
}
if cat != "" {
item["category"] = cat
} else {
item["category"] = nil
}
if catName != "" {
item["category_name"] = catName
} else {
item["category_name"] = nil
}
item["eprel"] = normalizeEPRELValue(item["eprel"])
@@ -318,6 +343,61 @@ func EnforceV1ProcessCompletedItem(item V1ProcessJobItem, language string, allow
return item
}
// projectV1CategoryFields sets category=display name, category_id=unique_id,
// category_name=display name. Returns (unique_id, display_name).
func projectV1CategoryFields(item V1ProcessJobItem) (catID, catName string) {
catID = stringFromItem(item, "category_id")
catName = stringFromItem(item, "category_name")
cat := stringFromItem(item, "category")
if catID == "" {
// Legacy: category held unique_id when category_name differed.
if catName != "" && cat != "" && cat != catName {
catID = cat
} else if catName == "" && cat != "" {
catID = cat
}
}
if catName == "" && cat != "" && cat != catID {
catName = cat
}
if catName != "" {
item["category"] = catName
item["category_name"] = catName
} else {
item["category"] = nil
item["category_name"] = nil
}
if catID != "" {
item["category_id"] = catID
} else {
delete(item, "category_id")
}
return catID, catName
}
// inferDescriptionTemplateFromHTML rebuilds a minimal description_template from
// tags already present so invent-garbage HTML can be re-synthesized without a DB lookup.
func inferDescriptionTemplateFromHTML(html string) any {
lower := strings.ToLower(html)
var sections []map[string]any
for _, typ := range []string{"h1", "h2", "h3", "h4", "p", "ul"} {
if strings.Contains(lower, "<"+typ) {
sections = append(sections, map[string]any{"type": typ})
}
}
if len(sections) == 0 {
return map[string]any{
"sections": []map[string]any{
{"type": "h1"},
{"type": "p"},
{"type": "h2"},
{"type": "ul"},
},
}
}
return map[string]any{"sections": sections}
}
func stringFromItem(item V1ProcessJobItem, key string) string {
if item == nil {
return ""
@@ -165,8 +165,11 @@ func TestEnforceV1ProcessCompletedItem_categoryFromCallerPreserved(t *testing.T)
"eprel": nil,
}
out := EnforceV1ProcessCompletedItem(item, "", nil)
if out["category"] != "50" {
t.Fatalf("category=%v", out["category"])
if out["category"] != "Štedilniki" {
t.Fatalf("category=%v want display name", out["category"])
}
if out["category_id"] != "50" {
t.Fatalf("category_id=%v want 50", out["category_id"])
}
if out["category_name"] != "Štedilniki" {
t.Fatalf("category_name=%v", out["category_name"])
+15 -9
View File
@@ -80,29 +80,35 @@ func TestSynthesizeDescriptionFromTitle_brandModelCategory(t *testing.T) {
}
en := synthesizeDescriptionFromTitle(title, "TV Mounts", "en", attrs)
if en == "" {
t.Fatal("expected nonempty English invent")
t.Fatal("expected nonempty English factual fallback")
}
if strings.Contains(strings.ToLower(en), "ready for retail listing") {
t.Fatalf("must not emit retail filler: %q", en)
}
if strings.Contains(strings.ToLower(en), "is a ") && strings.Contains(strings.ToLower(en), " product from ") {
t.Fatalf("must not emit invent boilerplate: %q", en)
}
for _, need := range []string{"Vogel", "TV Mounts", "45 cm", "40 kg"} {
if !strings.Contains(en, need) {
t.Fatalf("English invent missing %q: %q", need, en)
t.Fatalf("English fallback missing %q: %q", need, en)
}
}
if isWeakPriorEnhanceDescription(en, title) {
t.Fatalf("factual invent must not be weak: %q", en)
t.Fatalf("factual fallback must not be weak: %q", en)
}
sl := synthesizeDescriptionFromTitle(title, "TV Mounts", "sl", attrs)
if sl == "" || !strings.Contains(sl, "kategoriji") {
t.Fatalf("expected Slovenian invent, got %q", sl)
if sl == "" || !strings.Contains(sl, "znamka") {
t.Fatalf("expected Slovenian factual fallback, got %q", sl)
}
if strings.Contains(strings.ToLower(sl), "je izdelek v kategoriji") {
t.Fatalf("must not emit invent boilerplate: %q", sl)
}
if strings.Contains(strings.ToLower(sl), "ready for retail listing") {
t.Fatalf("SL invent must not emit EN filler: %q", sl)
t.Fatalf("SL fallback must not emit EN filler: %q", sl)
}
if isWeakPriorEnhanceDescription(sl, title) {
t.Fatalf("SL factual invent must not be weak: %q", sl)
t.Fatalf("SL factual fallback must not be weak: %q", sl)
}
}
@@ -134,7 +140,7 @@ func TestInventHeuristicDescription_fromBrandModelCategory(t *testing.T) {
slSystem := "Write name and description in Slovenian. Return JSON with \"name\"."
sl := inventHeuristicDescription(slSystem, user, title)
if !strings.Contains(sl, "kategoriji") {
t.Fatalf("expected SL invent from system language, got %q", sl)
if !strings.Contains(sl, "znamka") {
t.Fatalf("expected SL factual invent from system language, got %q", sl)
}
}