This commit is contained in:
2026-08-16 23:42:00 +02:00
parent 6e77154228
commit 92f046b542
32 changed files with 1443 additions and 105 deletions
+3 -1
View File
@@ -89,7 +89,9 @@ type ProductInput struct {
EnhanceSystemTemplate string
EnhanceUserTemplate string
// CategoryEnhancePrompt overrides EnhanceUserTemplate when non-empty
// (resolved for the active language before enhance).
// (resolved for the active language before enhance). Applied to BOTH name
// and description via user framing + system overlay; formulas still win
// for structure.
CategoryEnhancePrompt string
// CategoryPromptsByLang maps lower(name|unique_id) → lang → category override prompt
// (JSON/overlay text with {{attrs}}/{{category}}; works with repaired A1 overlays).
@@ -0,0 +1,120 @@
package processing
import (
"context"
"encoding/json"
"strings"
"testing"
)
func TestAttrsFromEnhanceObj_andMerge(t *testing.T) {
t.Parallel()
obj := map[string]any{
"name": "Monitor",
"attrs": map[string]any{
"diagonala_zaslona": "27\"",
"zavora": "junk",
"brand": "Acme",
},
}
got := attrsFromEnhanceObj(obj)
if got["brand"] != "Acme" || got["diagonala_zaslona"] != "27\"" {
t.Fatalf("attrsFromEnhanceObj=%v", got)
}
allowed := map[string]struct{}{
"diagonala_zaslona": {},
"vrsta_panela": {},
}
merged := mergeEnhanceAttrsInto(map[string]any{
"brand": "Acme",
"product_model": "X1",
}, got, allowed)
if merged["diagonala_zaslona"] != "27\"" {
t.Fatalf("expected remapped/validated diagonala, got %v", merged)
}
if _, ok := merged["zavora"]; ok {
t.Fatalf("zavora must be dropped by allowlist: %v", merged)
}
if merged["brand"] != "Acme" || merged["product_model"] != "X1" {
t.Fatalf("core/base attrs must remain: %v", merged)
}
if mergeEnhanceAttrsInto(map[string]any{"a": 1}, nil, allowed) != nil {
t.Fatal("empty llm attrs must yield nil (no change)")
}
}
func TestEnhanceAttrsFromRaw_roundTrip(t *testing.T) {
t.Parallel()
meta := map[string]any{"status": "ok"}
attachEnhanceAttrs(meta, map[string]any{"brand": "Bosch", "nosilnost": "40 kg"})
got := enhanceAttrsFromRaw(meta)
if got["brand"] != "Bosch" || got["nosilnost"] != "40 kg" {
t.Fatalf("round-trip attrs=%v", got)
}
}
func TestRunSteps_enhanceMergesValidatedAttrs(t *testing.T) {
t.Parallel()
payload := mustJSON(map[string]any{
"name": "Acme UltraView 27",
"description": "<h2>Acme UltraView 27</h2><p>27 inch IPS monitor for desk work with clear specs.</p><ul><li>27 inch</li><li>IPS panel</li></ul>",
"meta_title": "Acme UltraView 27 | IPS",
"meta_description": "Shop Acme UltraView 27 IPS monitor.",
"attrs": map[string]any{
"diagonala_zaslona": "27\"",
"vrsta_panela": "IPS",
"zavora": "must-drop",
},
})
e := &Engine{
Completer: stubCompleter{fn: func(system, user string) (Completion, error) {
return Completion{Text: payload, TotalTokens: 12, Raw: map[string]any{"ok": true}}, nil
}},
Vector: NoopVectorCategorizer{},
}
out, err := e.RunSteps(context.Background(), "co", ProductInput{
Name: "UltraView 27 Gaming Monitor",
Description: "A solid IPS desk monitor with factory specs.",
Mapped: map[string]any{
"category_unique_id": "28",
"name": "UltraView 27 Gaming Monitor",
"attributes": map[string]any{
"brand": "Acme",
},
},
CategoryNamesByUID: map[string]string{"28": "Monitorji"},
CategoryUniqueID: "28",
CategoryAttrKeys: map[string]map[string]struct{}{
"28": {"diagonala_zaslona": {}, "vrsta_panela": {}},
},
CategoryEnhancePrompt: "Focus on panel technology for monitors.",
Language: "en",
}, "full", nil, StepPolicy{AllowAI: true})
if err != nil {
t.Fatal(err)
}
pa := out.ProcessedAttributes
if pa["diagonala_zaslona"] != "27\"" || pa["vrsta_panela"] != "IPS" {
t.Fatalf("expected LLM attrs merged: %v notes=%v gpt=%v", pa, out.Notes, out.GPTResponse)
}
if _, ok := pa["zavora"]; ok {
t.Fatalf("zavora must not survive allowlist: %v", pa)
}
if pa["brand"] != "Acme" {
t.Fatalf("feed brand must remain: %v", pa)
}
if out.FieldSources["attributes"] != "ai_enhance" {
t.Fatalf("field source=%v", out.FieldSources["attributes"])
}
if !strings.Contains(out.ProcessedName, "UltraView") {
t.Fatalf("name=%q", out.ProcessedName)
}
}
func mustJSON(v any) string {
b, err := json.Marshal(v)
if err != nil {
panic(err)
}
return string(b)
}
+24 -4
View File
@@ -211,9 +211,10 @@ func FixCatalogHygieneWithIDs(ctx context.Context, pool *pgxpool.Pool, companyID
return out, ids, nil
}
// NormalizeProcessedDescriptions rewrites processed_description to the plain-text
// form produced by PlainDescriptionFromAny (unwraps JSON arrays, strips HTML). When
// processed_description is empty, normalizes from description. Returns rows updated.
// NormalizeProcessedDescriptions rewrites processed_description to a single
// string form via DescriptionFromAny (unwraps JSON arrays, preserves formula
// HTML). When processed_description is empty, normalizes from description.
// Returns rows updated.
func NormalizeProcessedDescriptions(ctx context.Context, pool *pgxpool.Pool, companyID uuid.UUID) (int, error) {
if pool == nil {
return 0, fmt.Errorf("normalize descriptions: nil pool")
@@ -245,7 +246,7 @@ func NormalizeProcessedDescriptions(ctx context.Context, pool *pgxpool.Pool, com
if raw == "" {
continue
}
plain := plainDescriptionFromStored(raw)
plain := descriptionFromStored(raw)
if plain == "" || plain == strings.TrimSpace(processedDesc) {
continue
}
@@ -265,7 +266,26 @@ func NormalizeProcessedDescriptions(ctx context.Context, pool *pgxpool.Pool, com
return updated, nil
}
// descriptionFromStored handles DB text that may itself be a JSON array/string,
// preserving formula HTML tags for process/API consumers.
func descriptionFromStored(raw string) string {
raw = strings.TrimSpace(raw)
if raw == "" {
return ""
}
if strings.HasPrefix(raw, "[") || strings.HasPrefix(raw, "{") {
var decoded any
if err := json.Unmarshal([]byte(raw), &decoded); err == nil {
if s := DescriptionFromAny(decoded); s != "" {
return s
}
}
}
return DescriptionFromAny(raw)
}
// plainDescriptionFromStored handles DB text that may itself be a JSON array/string.
// Strips HTML — prefer descriptionFromStored when formula markup must be kept.
func plainDescriptionFromStored(raw string) string {
raw = strings.TrimSpace(raw)
if raw == "" {
@@ -82,7 +82,8 @@ func formatAvailableCategoriesList(opts []categoryOption) string {
return strings.TrimSpace(b.String())
}
// ProductCategorizeSystem is the built-in system prompt for taxonomy selection.
// ProductCategorizeSystem is the built-in system prompt for taxonomy selection
// (aiprompts.RoleCategorize — pick company unique_id; separate from enhance overlay).
const ProductCategorizeSystem = `Product categorization expert.
Rules:
- Reply with ONLY a single JSON object (no markdown, no prose, no reasoning)
@@ -80,6 +80,9 @@ func TestA1StyleDescriptionFormulaInResolvedPrompts(t *testing.T) {
if !strings.Contains(sysTpl, "Description formula") {
t.Fatalf("system missing formula override: %s", sysTpl)
}
if !strings.Contains(sysTpl, `When the user message includes a "Title formula"`) {
t.Fatalf("system missing title formula override: %s", sysTpl)
}
if !strings.Contains(userTpl, "Description formula") || !strings.Contains(userTpl, "- h1: Naziv izdelka") || !strings.Contains(userTpl, "- h2: Podnaslov prednosti") {
t.Fatalf("user missing description formula: %s", userTpl)
}
+235 -14
View File
@@ -3,38 +3,146 @@ package processing
import (
"encoding/json"
"fmt"
"sort"
"strconv"
"strings"
"github.com/descrybe/descrybe-v2/apps/api/internal/company"
)
// AppendFormulaConstraints appends language-agnostic title/description formula
// AppendFormulaConstraints appends language-agnostic title/description/meta formula
// guidance to the enhance user template (before {{var}} render). Empty templates
// are no-ops. Shared skeleton stays in company/built-in prompts; formulas only
// constrain structure for the active language via {{language}} elsewhere.
// Meta instructions come from description_template.metaTitle / metaDescription
// (legacy A1 / cats.json) and are distinct from HTML description sections.
// Attribute allowlist / formula-key guidance is AppendAttributeConstraints.
func AppendFormulaConstraints(userTpl string, titleTemplate, descriptionTemplate any) string {
userTpl = strings.TrimSpace(userTpl)
titleBlock := FormatTitleFormulaConstraint(titleTemplate)
descBlock := FormatDescriptionFormulaConstraint(descriptionTemplate)
if titleBlock == "" && descBlock == "" {
blocks := []string{
FormatTitleFormulaConstraint(titleTemplate),
FormatDescriptionFormulaConstraint(descriptionTemplate),
FormatMetaFormulaConstraint(descriptionTemplate),
}
var joined strings.Builder
for _, block := range blocks {
if block == "" {
continue
}
if joined.Len() > 0 {
joined.WriteString("\n\n")
}
joined.WriteString(block)
}
if joined.Len() == 0 {
return userTpl
}
if userTpl == "" {
return joined.String()
}
return userTpl + "\n\n" + joined.String()
}
// MaxAttrAllowlistPromptKeys caps Allowed attribute keys listed in enhance prompts.
const MaxAttrAllowlistPromptKeys = 40
// AppendAttributeConstraints appends category_attributes allowlist + title-formula
// variable keys so the model can emit JSON "attrs" validated via AttrsForEnhance.
// No-op when both allowlist and formula keys are empty.
func AppendAttributeConstraints(userTpl string, allowed map[string]struct{}, titleTemplate any) string {
userTpl = strings.TrimSpace(userTpl)
block := FormatAttributeAllowlistConstraint(allowed, titleTemplate)
if block == "" {
return userTpl
}
if userTpl == "" {
return block
}
if strings.Contains(userTpl, `Allowed attribute keys (JSON "attrs" object`) {
return userTpl
}
return userTpl + "\n\n" + block
}
// FormatAttributeAllowlistConstraint lists category allowlist keys and title-formula
// attr slots for the Attributes enhance role. When allowed is nil, only formula
// keys are listed (unit-test / sanitize-only paths).
func FormatAttributeAllowlistConstraint(allowed map[string]struct{}, titleTemplate any) string {
keys := preferredAttrKeyList(allowed, MaxAttrAllowlistPromptKeys)
formulaKeys := titleFormulaAttrKeys(titleTemplate)
if len(keys) == 0 && len(formulaKeys) == 0 {
return ""
}
var b strings.Builder
if userTpl != "" {
b.WriteString(userTpl)
b.WriteString("\n\n")
b.WriteString("Allowed attribute keys (JSON \"attrs\" object — remap feed labels onto these; omit unknowns):\n")
if len(keys) > 0 {
b.WriteString("- category: ")
b.WriteString(strings.Join(keys, ", "))
b.WriteByte('\n')
} else {
b.WriteString("- category: (core characteristics only — brand, product_model, dims, …)\n")
}
if titleBlock != "" {
b.WriteString(titleBlock)
if descBlock != "" {
b.WriteString("\n\n")
if len(formulaKeys) > 0 {
b.WriteString("- title formula slots: ")
b.WriteString(strings.Join(formulaKeys, ", "))
b.WriteByte('\n')
}
b.WriteString("Fill missing keys only from Name/Description/Category/Attrs evidence; never invent specs or zero dimensions.")
return strings.TrimSpace(b.String())
}
func preferredAttrKeyList(allowed map[string]struct{}, maxKeys int) []string {
if len(allowed) == 0 || maxKeys <= 0 {
return nil
}
prefer := preferredAllowedKeys(allowed)
seen := map[string]struct{}{}
out := make([]string, 0, len(prefer))
for _, k := range prefer {
k = strings.TrimSpace(k)
if k == "" {
continue
}
if _, ok := seen[k]; ok {
continue
}
seen[k] = struct{}{}
out = append(out, k)
}
if descBlock != "" {
b.WriteString(descBlock)
sort.Strings(out)
if len(out) > maxKeys {
out = out[:maxKeys]
}
return b.String()
return out
}
func titleFormulaAttrKeys(template any) []string {
elements, _, ok := parseTitleFormula(template)
if !ok || len(elements) == 0 {
return nil
}
seen := map[string]struct{}{}
var out []string
for _, el := range elements {
if el.Type != "variable" {
continue
}
k := strings.TrimSpace(el.Value)
if k == "" {
continue
}
canon := canonicalizeAttrKey(k)
if canon == "" {
canon = strings.ToLower(k)
}
if _, ok := seen[canon]; ok {
continue
}
seen[canon] = struct{}{}
out = append(out, canon)
}
sort.Strings(out)
return out
}
// FormatTitleFormulaConstraint turns categories.title_template into enhance-prompt
@@ -70,6 +178,30 @@ func FormatTitleFormulaConstraint(template any) string {
return strings.TrimSpace(b.String())
}
// FormatMetaFormulaConstraint turns description_template.metaTitle / metaDescription
// (cats.json / A1 SEO instructions) into enhance-prompt constraints distinct from
// HTML description sections.
func FormatMetaFormulaConstraint(template any) string {
metaTitle, metaDesc := parseMetaFormulaInstructions(template)
if metaTitle == "" && metaDesc == "" {
return ""
}
var b strings.Builder
b.WriteString("SEO meta formula (REQUIRED — distinct from description HTML; plain text only):\n")
b.WriteString("Emit meta_title and meta_description as separate JSON fields in {{language}}.\n")
if metaTitle != "" {
fmt.Fprintf(&b, "- meta_title (50-60 chars): %s\n", metaTitle)
} else {
b.WriteString("- meta_title: 50-60 chars, product name + main benefit or use case\n")
}
if metaDesc != "" {
fmt.Fprintf(&b, "- meta_description (120-155 chars): %s\n", metaDesc)
} else {
b.WriteString("- meta_description: 120-155 chars, factual SEO snippet; never HTML\n")
}
return strings.TrimSpace(b.String())
}
// FormatDescriptionFormulaConstraint turns categories.description_template sections
// into bullet instructions for the enhance user prompt.
func FormatDescriptionFormulaConstraint(template any) string {
@@ -120,6 +252,74 @@ func AppendDescriptionFormulaSystemOverride(systemTpl string, descriptionTemplat
return systemTpl + "\n" + descriptionFormulaSystemOverride
}
// titleFormulaSystemOverride is appended to the enhance system template when a
// category title_template is present so company/built-in "short retail title"
// rules cannot ignore the Title formula (mirrors description formula override).
const titleFormulaSystemOverride = `When the user message includes a "Title formula", obey that structure for "name" over any shorter "short retail title" guidance: build name from Attrs in the given element order, keep literal text as written, write name in {{language}}. Never ignore the Title formula.`
// AppendTitleFormulaSystemOverride strengthens the system prompt when a title
// formula is active. No-op when template is empty/unparseable.
func AppendTitleFormulaSystemOverride(systemTpl string, titleTemplate any) string {
if FormatTitleFormulaConstraint(titleTemplate) == "" {
return strings.TrimSpace(systemTpl)
}
systemTpl = strings.TrimSpace(systemTpl)
if systemTpl == "" {
return titleFormulaSystemOverride
}
if strings.Contains(systemTpl, `When the user message includes a "Title formula"`) {
return systemTpl
}
return systemTpl + "\n" + titleFormulaSystemOverride
}
// categoryEnhanceSystemOverlay is appended when categories.prompt (enhance overlay)
// is active so free-form category copy drives title, description, AND attributes —
// not description alone — while Title/Description formulas keep structural precedence.
const categoryEnhanceSystemOverlay = `When the user message includes category guidance, apply it to "name", "description", and "attrs" (tone/focus for the title and body, plus attribute extraction onto Allowed attribute keys). Structural Title/Description formula blocks and Allowed attribute keys in the user message still take precedence when present.`
// AppendCategoryEnhanceSystemOverlay strengthens the system prompt when a
// per-category enhance overlay (categories.prompt) is set. No-op when empty.
func AppendCategoryEnhanceSystemOverlay(systemTpl, categoryPrompt string) string {
if strings.TrimSpace(categoryPrompt) == "" {
return strings.TrimSpace(systemTpl)
}
systemTpl = strings.TrimSpace(systemTpl)
if systemTpl == "" {
return categoryEnhanceSystemOverlay
}
marker := `apply it to "name", "description", and "attrs"`
if strings.Contains(systemTpl, marker) {
return systemTpl
}
// Upgrade older name+description-only overlay without stacking duplicates.
legacy := `apply it to BOTH "name" and "description"`
if strings.Contains(systemTpl, legacy) {
return strings.Replace(systemTpl, legacy, marker, 1)
}
return systemTpl + "\n" + categoryEnhanceSystemOverlay
}
// metaFormulaSystemOverride is appended when description_template carries metaTitle
// / metaDescription instructions so enhance emits SEO fields separately from HTML.
const metaFormulaSystemOverride = `When the user message includes an "SEO meta formula", emit meta_title and meta_description as plain SEO text (not HTML) distinct from description. Obey the character guidance in the formula.`
// AppendMetaFormulaSystemOverride strengthens the system prompt when meta formula
// instructions are present on description_template. No-op when absent.
func AppendMetaFormulaSystemOverride(systemTpl string, descriptionTemplate any) string {
if FormatMetaFormulaConstraint(descriptionTemplate) == "" {
return strings.TrimSpace(systemTpl)
}
systemTpl = strings.TrimSpace(systemTpl)
if systemTpl == "" {
return metaFormulaSystemOverride
}
if strings.Contains(systemTpl, "SEO meta formula") {
return systemTpl
}
return systemTpl + "\n" + metaFormulaSystemOverride
}
// descriptionSatisfiesFormula reports whether desc includes the HTML tags required
// by categories.description_template sections. No formula → always true.
func descriptionSatisfiesFormula(desc string, template any) bool {
@@ -219,6 +419,27 @@ func parseDescriptionFormulaSections(template any) ([]descriptionFormulaSection,
return out, len(out) > 0
}
// parseMetaFormulaInstructions reads metaTitle / metaDescription instruction
// strings from description_template (camelCase as stored by the category UI).
func parseMetaFormulaInstructions(template any) (metaTitle, metaDescription string) {
if template == nil {
return "", ""
}
obj, err := asObjectMap(template)
if err != nil || obj == nil {
return "", ""
}
metaTitle = strings.TrimSpace(stringFromAny(obj["metaTitle"]))
if metaTitle == "" {
metaTitle = strings.TrimSpace(stringFromAny(obj["meta_title"]))
}
metaDescription = strings.TrimSpace(stringFromAny(obj["metaDescription"]))
if metaDescription == "" {
metaDescription = strings.TrimSpace(stringFromAny(obj["meta_description"]))
}
return metaTitle, metaDescription
}
// categoryFormulasFor resolves title/description templates for a category key
// (unique_id or name). Explicit ProductInput fields win over the job cache map.
func categoryFormulasFor(in ProductInput, category string) (title, description any) {
@@ -76,6 +76,118 @@ func TestAppendDescriptionFormulaSystemOverride(t *testing.T) {
}
}
func TestAppendTitleFormulaSystemOverride(t *testing.T) {
t.Parallel()
title := map[string]any{
"separator": " ",
"elements": []any{
map[string]any{"type": "variable", "value": "brand"},
map[string]any{"type": "text", "value": "Pro"},
},
}
sys, user := resolveProductPromptTemplates(ProductInput{
EnhanceSystemTemplate: "Retail product copywriter.\n- name: short retail title",
EnhanceUserTemplate: "Name: {{name}}",
TitleTemplate: title,
})
if !strings.Contains(user, "Title formula") || !strings.Contains(user, "attr [brand]") {
t.Fatalf("missing user title formula: %s", user)
}
if !strings.Contains(sys, `When the user message includes a "Title formula"`) || !strings.Contains(sys, "short retail title") {
t.Fatalf("system should keep company text and append title override: %s", sys)
}
if AppendTitleFormulaSystemOverride("plain", nil) != "plain" {
t.Fatal("empty title formula must be no-op")
}
dup := AppendTitleFormulaSystemOverride(sys, title)
if strings.Count(dup, `When the user message includes a "Title formula"`) != 1 {
t.Fatalf("title override must be idempotent: %s", dup)
}
}
func TestAppendCategoryEnhanceSystemOverlay(t *testing.T) {
t.Parallel()
if AppendCategoryEnhanceSystemOverlay("plain", "") != "plain" {
t.Fatal("empty category prompt must be no-op")
}
got := AppendCategoryEnhanceSystemOverlay("Retail system.", "Focus on panel tech.")
if !strings.Contains(got, "Retail system.") || !strings.Contains(got, `apply it to "name", "description", and "attrs"`) {
t.Fatalf("expected category system overlay: %s", got)
}
if AppendCategoryEnhanceSystemOverlay(got, "again") != got {
t.Fatal("category overlay must be idempotent")
}
legacy := AppendCategoryEnhanceSystemOverlay("Retail.", "x")
// Force-inject legacy marker then upgrade.
legacy = "Retail.\nWhen the user message includes category guidance, apply it to BOTH \"name\" and \"description\" (tone)."
upgraded := AppendCategoryEnhanceSystemOverlay(legacy, "Focus")
if !strings.Contains(upgraded, `apply it to "name", "description", and "attrs"`) {
t.Fatalf("legacy overlay should upgrade to attrs: %s", upgraded)
}
}
func TestFormatAttributeAllowlistConstraint(t *testing.T) {
t.Parallel()
allowed := map[string]struct{}{
"diagonala_zaslona": {},
"vrsta_panela": {},
}
title := map[string]any{
"separator": " ",
"elements": []any{
map[string]any{"type": "variable", "value": "brand"},
map[string]any{"type": "variable", "value": "product_model"},
},
}
got := FormatAttributeAllowlistConstraint(allowed, title)
if !strings.Contains(got, "Allowed attribute keys") {
t.Fatalf("missing header: %s", got)
}
if !strings.Contains(got, "diagonala_zaslona") || !strings.Contains(got, "vrsta_panela") {
t.Fatalf("missing category keys: %s", got)
}
if !strings.Contains(got, "brand") || !strings.Contains(got, "product_model") {
t.Fatalf("missing formula keys: %s", got)
}
if FormatAttributeAllowlistConstraint(nil, nil) != "" {
t.Fatal("empty allowlist+formula must be no-op")
}
appended := AppendAttributeConstraints("Attrs: {{attrs}}", allowed, title)
if !strings.Contains(appended, "Allowed attribute keys") || !strings.HasPrefix(appended, "Attrs:") {
t.Fatalf("append failed: %s", appended)
}
if AppendAttributeConstraints(appended, allowed, title) != appended {
t.Fatal("attribute constraints must be idempotent")
}
}
func TestResolveProductPromptTemplates_includesAttributeAllowlist(t *testing.T) {
t.Parallel()
_, user := resolveProductPromptTemplates(ProductInput{
EnhanceUserTemplate: "Name: {{name}}",
CategoryUniqueID: "50",
CategoryAttrKeys: map[string]map[string]struct{}{
"50": {"diagonala_zaslona": {}, "vrsta_panela": {}},
},
TitleTemplate: map[string]any{
"separator": " ",
"elements": []any{
map[string]any{"type": "variable", "value": "brand"},
},
},
CategoryEnhancePrompt: "Focus on panel tech for monitors.",
})
if !strings.Contains(user, "Allowed attribute keys") {
t.Fatalf("missing attribute allowlist in user tpl: %s", user)
}
if !strings.Contains(user, "diagonala_zaslona") {
t.Fatalf("missing category attr key: %s", user)
}
if !strings.Contains(user, "Focus on panel tech") {
t.Fatalf("missing category overlay: %s", user)
}
}
func TestDescriptionSatisfiesFormula(t *testing.T) {
t.Parallel()
tpl := map[string]any{
@@ -119,8 +231,10 @@ func TestAppendFormulaConstraints_injectedIntoResolve(t *testing.T) {
"sections": []any{
map[string]any{"type": "p", "instructions": "Factual summary"},
},
"metaTitle": "Include product name and main benefit (50-60 chars)",
"metaDescription": "Write a short Meta Description spotlighting key specs (120-155 chars)",
}
_, user := resolveProductPromptTemplates(ProductInput{
sys, user := resolveProductPromptTemplates(ProductInput{
EnhanceUserTemplate: "Name: {{name}}\nAttrs: {{attrs}}",
TitleTemplate: title,
DescriptionTemplate: desc,
@@ -134,6 +248,12 @@ func TestAppendFormulaConstraints_injectedIntoResolve(t *testing.T) {
if !strings.Contains(user, "Description formula") || !strings.Contains(user, "- p: Factual summary") {
t.Fatalf("missing description formula: %s", user)
}
if !strings.Contains(user, "SEO meta formula") || !strings.Contains(user, "meta_title (50-60 chars)") {
t.Fatalf("missing SEO meta formula: %s", user)
}
if !strings.Contains(sys, "SEO meta formula") {
t.Fatalf("system missing meta override: %s", sys)
}
// Render substitutes {{language}} in formula blocks.
_, rendered := RenderProductEnhancePrompts(
"Write in {{language}}.", user,
@@ -148,6 +268,29 @@ func TestAppendFormulaConstraints_injectedIntoResolve(t *testing.T) {
}
}
func TestFormatMetaFormulaConstraint(t *testing.T) {
t.Parallel()
got := FormatMetaFormulaConstraint(map[string]any{
"metaTitle": "Create a concise Meta Title with product name",
"metaDescription": "Write a short Meta Description for search results",
"sections": []any{
map[string]any{"type": "p", "instructions": "Body"},
},
})
if !strings.Contains(got, "SEO meta formula") {
t.Fatalf("missing header: %s", got)
}
if !strings.Contains(got, "meta_title (50-60 chars): Create a concise Meta Title") {
t.Fatalf("missing meta_title: %s", got)
}
if !strings.Contains(got, "meta_description (120-155 chars): Write a short Meta Description") {
t.Fatalf("missing meta_description: %s", got)
}
if FormatMetaFormulaConstraint(map[string]any{"sections": []any{}}) != "" {
t.Fatal("sections-only template must not invent SEO meta formula")
}
}
func TestCategoryFormulasFor_explicitWins(t *testing.T) {
t.Parallel()
in := ProductInput{
+45 -1
View File
@@ -19,7 +19,8 @@ func isDescriptionFieldKey(key string) bool {
// PlainDescriptionFromAny coerces legacy description shapes to a single plain
// string: string, {#text}, or arrays of those (joined with newlines). HTML is
// stripped via v1PlainDescription so poll/store never keep ["…"] or markup blobs.
// stripped via v1PlainDescription — use for meta/SEO and feed normalize only.
// V1 process poll / formula bodies use DescriptionFromAny (keeps markup).
func PlainDescriptionFromAny(v any) string {
if v == nil {
return ""
@@ -59,6 +60,49 @@ func PlainDescriptionFromAny(v any) string {
}
}
// DescriptionFromAny coerces legacy description shapes to a single string while
// preserving formula HTML (h1/h2/p/ul/…). Unwraps JSON arrays / #text wrappers
// but does not strip tags — used by V1 process item projection and catalog
// normalize when enhance/formula HTML must reach clients and stay in DB.
func DescriptionFromAny(v any) string {
if v == nil {
return ""
}
switch t := v.(type) {
case string:
return v1PreserveDescription(t)
case []string:
parts := make([]string, 0, len(t))
for _, s := range t {
if p := v1PreserveDescription(s); p != "" {
parts = append(parts, p)
}
}
return strings.Join(parts, "\n")
case []any:
parts := make([]string, 0, len(t))
for _, item := range t {
if p := DescriptionFromAny(item); p != "" {
parts = append(parts, p)
}
}
return strings.Join(parts, "\n")
case map[string]any:
for _, k := range []string{"#text", "text", "value", "description", "body"} {
if p := DescriptionFromAny(t[k]); p != "" {
return p
}
}
return ""
default:
s := strings.TrimSpace(fmt.Sprint(t))
if s == "" || s == "<nil>" {
return ""
}
return v1PreserveDescription(s)
}
}
// coerceScalarText flattens arrays / #text wrappers into a trimmed string without
// HTML stripping (names, brands, stock labels, …).
func coerceScalarText(v any) string {
@@ -1,6 +1,9 @@
package processing
import "testing"
import (
"strings"
"testing"
)
func TestPlainDescriptionFromAny_arrayAndHTML(t *testing.T) {
got := PlainDescriptionFromAny([]any{"Hello<br>World", "Second"})
@@ -16,6 +19,24 @@ func TestPlainDescriptionFromAny_arrayAndHTML(t *testing.T) {
}
}
func TestDescriptionFromAny_preservesFormulaHTML(t *testing.T) {
htmlDesc := `<h1>Anker Soundcore Space One Pro</h1><p>Zložljive ANC slušalke.</p><ul><li>Bluetooth</li></ul>`
got := DescriptionFromAny(htmlDesc)
if !strings.Contains(got, "<h1>") || !strings.Contains(got, "<p>") || !strings.Contains(got, "<ul>") {
t.Fatalf("expected formula HTML preserved, got %q", got)
}
// Array unwrap should still keep tags.
got = DescriptionFromAny([]any{htmlDesc})
if !strings.Contains(got, "<h1>") || !strings.Contains(got, "</p>") {
t.Fatalf("array unwrap should keep HTML, got %q", got)
}
// Plain path still strips (meta/SEO / feed normalize).
plain := PlainDescriptionFromAny(htmlDesc)
if strings.Contains(plain, "<h1>") || strings.Contains(plain, "<p>") {
t.Fatalf("PlainDescriptionFromAny should strip tags, got %q", plain)
}
}
func TestNormalizeMapped_descriptionArrayBecomesPlainString(t *testing.T) {
got := NormalizeMapped(map[string]any{
"description": []any{"Line1<br/>A", "Line2"},
+1 -1
View File
@@ -212,7 +212,7 @@ func ProductEnhanceUser(category, name, description string, attrs map[string]any
b.WriteString(SanitizeText(category))
b.WriteString("\nName: ")
b.WriteString(SanitizeText(truncateRunes(name, 200)))
b.WriteString("\nDesc: ")
b.WriteString("\nDescription: ")
b.WriteString(SanitizeText(truncateRunes(description, MaxProductDescRunes)))
compact := CompactAttrs(attrs, MaxAttrKeys)
if len(compact) > 0 {
+56
View File
@@ -1,6 +1,7 @@
package processing
import (
"fmt"
"regexp"
"strings"
"unicode"
@@ -148,3 +149,58 @@ func metaBrandFromAttrs(bags ...map[string]any) string {
}
return ""
}
// metaFieldsFromEnhanceObj extracts plain SEO meta from an enhance JSON object.
// Accepts snake_case and camelCase keys (legacy A1 <metaDescription> / cats.json).
// Strips HTML and rejects prompt-leakage titles; empty when missing.
func metaFieldsFromEnhanceObj(obj map[string]any) (title, description string) {
if obj == nil {
return "", ""
}
title = strings.TrimSpace(SanitizeOutput(fmt.Sprint(obj["meta_title"])))
if title == "" || title == "<nil>" {
title = strings.TrimSpace(SanitizeOutput(fmt.Sprint(obj["metaTitle"])))
}
description = strings.TrimSpace(SanitizeOutput(fmt.Sprint(obj["meta_description"])))
if description == "" || description == "<nil>" {
description = strings.TrimSpace(SanitizeOutput(fmt.Sprint(obj["metaDescription"])))
}
if title == "<nil>" {
title = ""
}
if description == "<nil>" {
description = ""
}
title = stripMetaTags(title)
description = stripMetaTags(description)
if isPromptLabelTitle(title) || isPromptLeakageTitle(title) {
title = ""
}
if isPromptLabelTitle(description) || isPromptLeakageTitle(description) {
description = ""
}
if title != "" {
title = truncateMetaRunes(title, metaTitleMaxChars)
}
if description != "" {
description = truncateMetaDescription(description, metaDescriptionMaxChars)
}
return title, description
}
// enhanceMetaFromRaw reads meta_title / meta_description stashed on enhance raw meta.
func enhanceMetaFromRaw(raw any) (title, description string) {
m, ok := raw.(map[string]any)
if !ok || m == nil {
return "", ""
}
title = strings.TrimSpace(fmt.Sprint(m["meta_title"]))
description = strings.TrimSpace(fmt.Sprint(m["meta_description"]))
if title == "<nil>" {
title = ""
}
if description == "<nil>" {
description = ""
}
return title, description
}
+61
View File
@@ -1,6 +1,7 @@
package processing
import (
"context"
"strings"
"testing"
"unicode/utf8"
@@ -82,6 +83,66 @@ func TestTruncateMetaDescription_collapsesSpace(t *testing.T) {
}
}
func TestMetaFieldsFromEnhanceObj_snakeAndCamel(t *testing.T) {
t.Parallel()
mt, md := metaFieldsFromEnhanceObj(map[string]any{
"meta_title": "Acme Widget Pro | Daily Use",
"meta_description": "Shop Acme Widget Pro for reliable everyday performance.",
})
if mt != "Acme Widget Pro | Daily Use" {
t.Fatalf("meta_title=%q", mt)
}
if !strings.Contains(md, "Acme Widget Pro") {
t.Fatalf("meta_description=%q", md)
}
mt, md = metaFieldsFromEnhanceObj(map[string]any{
"metaTitle": "<b>Camel Title</b>",
"metaDescription": "<p>Camel desc with enough characters for a real SEO snippet about the product.</p>",
})
if strings.Contains(mt, "<") || mt == "" {
t.Fatalf("camel meta_title should be plain: %q", mt)
}
if strings.Contains(md, "<") || md == "" {
t.Fatalf("camel meta_description should be plain: %q", md)
}
mt, md = metaFieldsFromEnhanceObj(map[string]any{
"meta_title": "short retail title; follow any Title formula",
})
if mt != "" || md != "" {
t.Fatalf("leakage meta must be dropped: %q %q", mt, md)
}
}
func TestRunSteps_enhancePersistsMetaFields(t *testing.T) {
e := &Engine{Completer: stubCompleter{fn: func(system, user string) (Completion, error) {
return Completion{Text: `{
"name":"Acme Widget Pro",
"description":"<h2>Acme Widget Pro</h2><p>Durable widget for everyday use.</p>",
"meta_title":"Acme Widget Pro | Durable Daily Use",
"meta_description":"Shop Acme Widget Pro for reliable everyday performance. Clear specs and fast delivery."
}`, TotalTokens: 12}, nil
}}}
out, err := e.RunSteps(context.Background(), "c1", ProductInput{
Mapped: map[string]any{"name": "Raw", "description": "old", "category": "tools"},
Language: "en",
}, "full", nil, StepPolicy{AllowAI: true})
if err != nil {
t.Fatal(err)
}
if out.MetaTitle != "Acme Widget Pro | Durable Daily Use" {
t.Fatalf("MetaTitle=%q", out.MetaTitle)
}
if !strings.Contains(out.MetaDescription, "Shop Acme Widget Pro") {
t.Fatalf("MetaDescription=%q", out.MetaDescription)
}
if !strings.Contains(out.ProcessedDescription, "<h2>") {
t.Fatalf("description should stay HTML: %q", out.ProcessedDescription)
}
if strings.Contains(out.MetaDescription, "<") {
t.Fatalf("meta_description must be plain: %q", out.MetaDescription)
}
}
func TestV1PollMetaFallback_usesTemplateWhenEmpty(t *testing.T) {
t.Parallel()
title := "Cordless Drill"
+13 -1
View File
@@ -663,7 +663,19 @@ func (h HeuristicCompleter) Complete(_ context.Context, system, user string) (Co
if desc == "" || descriptionEchoesTitle(desc, name) || isWeakPriorEnhanceDescription(desc, name) {
desc = inventHeuristicDescription(system, user, name)
}
b, _ := json.Marshal(map[string]string{"name": name, "description": desc})
payload := map[string]any{"name": name, "description": desc}
if strings.Contains(systemL, "meta_title") {
payload["meta_title"] = truncateRunes(name, 60)
payload["meta_description"] = truncateRunes(stripMetaTags(desc), 155)
}
if strings.Contains(systemL, `"attrs"`) {
if attrs := CompactAttrs(parseAttrsFromPrompt(user), MaxAttrKeys); len(attrs) > 0 {
payload["attrs"] = attrs
} else {
payload["attrs"] = map[string]any{}
}
}
b, _ := json.Marshal(payload)
text = string(b)
case strings.Contains(systemL, "attributes") && strings.Contains(systemL, "json"):
text = `{"material":"unknown","brand":"unknown"}`
+26 -2
View File
@@ -1591,8 +1591,32 @@ func (p *Pipeline) processOne(ctx context.Context, companyID, jobID uuid.UUID, i
if len(result.ProcessedAttributes) == 0 {
result.ProcessedAttributes = result.Attributes
}
// Free template SEO meta (no FillMetaAI / no extra credits).
result.MetaTitle, result.MetaDescription = fillMetaFromResult(result)
// Free template SEO meta when enhance did not emit meta_* (no FillMetaAI / no extra credits).
if strings.TrimSpace(result.MetaTitle) == "" || strings.TrimSpace(result.MetaDescription) == "" {
mt, md := fillMetaFromResult(result)
if strings.TrimSpace(result.MetaTitle) == "" {
result.MetaTitle = mt
}
if strings.TrimSpace(result.MetaDescription) == "" {
result.MetaDescription = md
}
}
// Keep primary localized meta in sync with row-level fields used by upsert.
if primary := company.NormalizeLanguage(language); primary != "" && result.LocalizedContent != nil {
lf := company.FieldsForLanguage(result.LocalizedContent, primary)
changed := false
if lf.MetaTitle == "" && result.MetaTitle != "" {
lf.MetaTitle = result.MetaTitle
changed = true
}
if lf.MetaDescription == "" && result.MetaDescription != "" {
lf.MetaDescription = result.MetaDescription
changed = true
}
if changed {
result.LocalizedContent[primary] = lf
}
}
attrsJSON, procAttrsJSON, gptJSON, sourcesJSON, err := marshalProcessOnePayload(result)
if err != nil {
@@ -17,12 +17,18 @@ func TestResolvePromptFallbackChain(t *testing.T) {
CategoryEnhancePrompt: "cat-sl",
Language: "sl",
})
if sys != "sys" {
t.Fatalf("sys=%q", sys)
if !strings.Contains(sys, "sys") {
t.Fatalf("sys=%q want company system kept", sys)
}
if !strings.HasPrefix(user, "cat-sl") || !strings.Contains(user, "{{attrs}}") {
if !strings.Contains(sys, `apply it to "name", "description", and "attrs"`) {
t.Fatalf("sys=%q want category overlay for title+description+attrs", sys)
}
if !strings.Contains(user, "cat-sl") || !strings.Contains(user, "{{attrs}}") {
t.Fatalf("sys=%q user=%q want cat-sl + attrs", sys, user)
}
if !strings.Contains(user, "applies to name, description, and attrs") {
t.Fatalf("user=%q want title framing", user)
}
// Empty category → company template.
_, user = resolveProductPromptTemplates(ProductInput{
+40 -7
View File
@@ -10,9 +10,10 @@ import (
func resolveProductPromptTemplates(in ProductInput) (systemTpl, userTpl string) {
systemTpl = strings.TrimSpace(in.EnhanceSystemTemplate)
userTpl = strings.TrimSpace(in.EnhanceUserTemplate)
catPrompt := strings.TrimSpace(in.CategoryEnhancePrompt)
// Per-category prompt wins for the user message (company system keeps JSON schema / brand).
if cat := strings.TrimSpace(in.CategoryEnhancePrompt); cat != "" {
userTpl = ensureCategoryEnhanceUserContext(cat)
if catPrompt != "" {
userTpl = ensureCategoryEnhanceUserContext(catPrompt)
}
def, ok := aiprompts.DefaultFor(aiprompts.KeyProductEnhance)
if ok {
@@ -25,9 +26,17 @@ func resolveProductPromptTemplates(in ProductInput) (systemTpl, userTpl string)
}
// Category formulas are language-agnostic; inject once into the shared user skeleton.
userTpl = AppendFormulaConstraints(userTpl, in.TitleTemplate, in.DescriptionTemplate)
// Category attribute allowlist + title-formula keys guide JSON "attrs" extraction.
allowed := enhanceAllowedAttrKeys(in, in.CategoryUniqueID)
userTpl = AppendAttributeConstraints(userTpl, allowed, in.TitleTemplate)
// Company/built-in system prompts often say "1-2 sentences"; when a category
// description formula exists, override that so process matches A1 category defs.
systemTpl = AppendDescriptionFormulaSystemOverride(systemTpl, in.DescriptionTemplate)
// Same for title_template vs "short retail title" — formulas win for name structure.
systemTpl = AppendTitleFormulaSystemOverride(systemTpl, in.TitleTemplate)
systemTpl = AppendMetaFormulaSystemOverride(systemTpl, in.DescriptionTemplate)
// categories.prompt applies to name, description, and attrs (not description-only).
systemTpl = AppendCategoryEnhanceSystemOverlay(systemTpl, catPrompt)
return systemTpl, userTpl
}
@@ -45,12 +54,36 @@ func templateHasVar(tpl, name string) bool {
return false
}
// ensureCategoryEnhanceUserContext appends standard product context placeholders when a
// category override omits {{attrs}} (common in DB category prompts). Does not rewrite
// category copy that already includes attrs.
// categoryEnhanceUserOverlayPrefix frames free-form categories.prompt text so the
// model applies it to name, description, and attrs (not description alone). Canonical
// CategoryEnhanceUserTemplate already covers all roles — skip double-framing there.
const categoryEnhanceUserOverlayPrefix = "Category guidance (applies to name, description, and attrs; obey Title/Description formulas and Allowed attribute keys when present):\n"
func categoryEnhancePromptAlreadyCoversTitle(userTpl string) bool {
lower := strings.ToLower(userTpl)
if strings.Contains(lower, "applies to name, description, and attrs") {
return true
}
if strings.Contains(lower, "applies to name and description") {
return true
}
// Repaired / built-in overlay: explicit name + Title formula bullets / role sections.
return strings.Contains(lower, `"name"`) && (strings.Contains(lower, "title formula") || strings.Contains(lower, "--- title ---"))
}
// ensureCategoryEnhanceUserContext frames category overlay for title+description,
// then appends standard product context placeholders when a category override omits
// {{attrs}} (common in DB category prompts). Does not rewrite category copy that
// already includes attrs (aside from optional title framing).
func ensureCategoryEnhanceUserContext(userTpl string) string {
userTpl = strings.TrimSpace(userTpl)
if userTpl == "" || templateHasVar(userTpl, "attrs") {
if userTpl == "" {
return userTpl
}
if !categoryEnhancePromptAlreadyCoversTitle(userTpl) {
userTpl = categoryEnhanceUserOverlayPrefix + userTpl
}
if templateHasVar(userTpl, "attrs") {
return userTpl
}
var b strings.Builder
@@ -71,7 +104,7 @@ func ensureCategoryEnhanceUserContext(userTpl string) string {
appendLine("Name: {{name}}")
}
if !templateHasVar(userTpl, "description") {
appendLine("Desc: {{description}}")
appendLine("Description: {{description}}")
}
appendLine("Attrs: {{attrs}}")
return b.String()
@@ -15,11 +15,17 @@ func TestResolveProductPromptTemplates_categoryOverridesUser(t *testing.T) {
EnhanceUserTemplate: "company user",
CategoryEnhancePrompt: "category user {{description}}",
})
if sys != "sys {{brand_voice}}" {
t.Fatalf("system=%q", sys)
if !strings.Contains(sys, "sys {{brand_voice}}") {
t.Fatalf("system=%q want company system kept", sys)
}
if !strings.HasPrefix(user, "category user {{description}}") {
t.Fatalf("user=%q want category override prefix", user)
if !strings.Contains(sys, `apply it to "name", "description", and "attrs"`) {
t.Fatalf("system=%q want category title+description+attrs overlay", sys)
}
if !strings.Contains(user, "category user {{description}}") {
t.Fatalf("user=%q want category override text", user)
}
if !strings.Contains(user, "applies to name, description, and attrs") {
t.Fatalf("user=%q want title+description+attrs framing", user)
}
if !strings.Contains(user, "{{attrs}}") {
t.Fatalf("user=%q want injected {{attrs}}", user)
@@ -34,8 +40,68 @@ func TestResolveProductPromptTemplates_categoryWithAttrsUnchanged(t *testing.T)
_, user := resolveProductPromptTemplates(ProductInput{
CategoryEnhancePrompt: "Write copy.\nAttrs: {{attrs}}\nName: {{name}}",
})
if user != "Write copy.\nAttrs: {{attrs}}\nName: {{name}}" {
t.Fatalf("user=%q want unchanged when attrs present", user)
if !strings.Contains(user, "Write copy.\nAttrs: {{attrs}}\nName: {{name}}") {
t.Fatalf("user=%q want category text preserved when attrs present", user)
}
if !strings.Contains(user, "applies to name, description, and attrs") {
t.Fatalf("user=%q want title framing on free-form overlay", user)
}
}
func TestResolveProductPromptTemplates_categoryCanonicalSkipsFrame(t *testing.T) {
t.Parallel()
_, user := resolveProductPromptTemplates(ProductInput{
CategoryEnhancePrompt: aiprompts.CategoryEnhanceUserTemplate,
})
if strings.Count(user, "applies to name, description, and attrs") != 0 ||
strings.Count(user, "applies to name and description") != 0 {
t.Fatalf("canonical template must not get double framing: %q", user)
}
if !strings.Contains(user, "Title formula") || !strings.Contains(user, `"name"`) {
t.Fatalf("canonical template should retain name guidance: %q", user)
}
}
func TestResolveProductPromptTemplates_categoryWithTitleFormula(t *testing.T) {
t.Parallel()
title := map[string]any{
"separator": " ",
"elements": []any{
map[string]any{"type": "variable", "value": "brand"},
map[string]any{"type": "variable", "value": "product_model"},
},
}
desc := map[string]any{
"sections": []any{
map[string]any{"type": "p", "instructions": "Factual summary"},
},
}
sys, user := resolveProductPromptTemplates(ProductInput{
EnhanceSystemTemplate: "Retail copywriter.\n- name: short retail title\n- description: 1-2 sentences",
CategoryEnhancePrompt: "Emphasize energy class and Slovenian retail tone.",
TitleTemplate: title,
DescriptionTemplate: desc,
})
if !strings.Contains(user, "Emphasize energy class") {
t.Fatalf("missing category prompt: %s", user)
}
if !strings.Contains(user, "applies to name, description, and attrs") {
t.Fatalf("missing category title framing: %s", user)
}
if !strings.Contains(user, "Title formula") || !strings.Contains(user, "attr [brand]") {
t.Fatalf("missing title formula: %s", user)
}
if !strings.Contains(user, "Description formula") || !strings.Contains(user, "- p: Factual summary") {
t.Fatalf("missing description formula: %s", user)
}
if !strings.Contains(sys, `When the user message includes a "Title formula"`) {
t.Fatalf("missing title system override: %s", sys)
}
if !strings.Contains(sys, "Description formula") {
t.Fatalf("missing description system override: %s", sys)
}
if !strings.Contains(sys, `apply it to "name", "description", and "attrs"`) {
t.Fatalf("missing category system overlay: %s", sys)
}
}
+8 -1
View File
@@ -10,6 +10,11 @@ import (
const maxPromptFieldRunes = 4000
// maxOutputFieldRunes bounds stored / polled model text (titles + multi-section
// formula HTML descriptions). Must stay well above MaxTokensEnhance (~16k tokens)
// so SanitizeOutput does not chop JSON completion bodies or A1 HTML mid-string.
const maxOutputFieldRunes = 120000
var controlOrInject = regexp.MustCompile(`(?i)(ignore\s+(all\s+)?(previous|prior|above)|disregard\s+(all\s+)?(previous|prior)|forget\s+(all\s+)?(previous|prior)|system\s*:|assistant\s*:|<\s*/?\s*script)`)
// SanitizeText strips control chars, truncates, and soft-neutralizes prompt-injection phrases.
@@ -31,6 +36,8 @@ func SanitizeText(s string) string {
}
// SanitizeOutput keeps model text printable and bounded for storage/UI.
// Uses a higher rune cap than SanitizeText so formula HTML and full enhance
// JSON completions are not truncated at 4k.
func SanitizeOutput(s string) string {
s = strings.TrimSpace(s)
if s == "" {
@@ -43,7 +50,7 @@ func SanitizeOutput(s string) string {
b.WriteRune(r)
}
}
return truncateRunes(b.String(), maxPromptFieldRunes)
return truncateRunes(b.String(), maxOutputFieldRunes)
}
func truncateRunes(s string, max int) string {
+222 -4
View File
@@ -303,6 +303,8 @@ func (e *Engine) RunSteps(ctx context.Context, companyID string, in ProductInput
CategoryEnhancePrompt: catPrompt,
TitleTemplate: titleTpl,
DescriptionTemplate: descTpl,
AllowedAttrKeys: in.AllowedAttrKeys,
CategoryAttrKeys: in.CategoryAttrKeys,
PriorEnhanceHash: priorHash,
PriorProcessedName: priorName,
PriorProcessedDescription: priorDesc,
@@ -355,6 +357,7 @@ func (e *Engine) RunSteps(ctx context.Context, companyID string, in ProductInput
}
}
weakDesc := descriptionNeedsEnhanceRepair(desc, descTpl, name)
enhanceMetaTitle, enhanceMetaDesc := enhanceMetaFromRaw(raw)
// Only persist enhance_input_hash for quality ok / hash-skip unchanged.
// Never copy input_hash from error/passthrough/thin/synthesized meta.
persistHash := ""
@@ -380,8 +383,8 @@ func (e *Engine) RunSteps(ctx context.Context, companyID string, in ProductInput
ProcessedName: name,
ProcessedDescription: desc,
EnhanceInputHash: persistHash,
MetaTitle: company.FieldsForLanguage(localized, lang).MetaTitle,
MetaDescription: company.FieldsForLanguage(localized, lang).MetaDescription,
MetaTitle: enhanceMetaTitle,
MetaDescription: enhanceMetaDesc,
}
// Preserve existing meta when re-enhancing titles only.
// Never keep a bare "| <unique_id>" poisoned meta_title.
@@ -410,6 +413,23 @@ func (e *Engine) RunSteps(ctx context.Context, companyID string, in ProductInput
if desc != "" {
out.Description = desc
}
if lf := localized[lang]; lf.MetaTitle != "" {
out.MetaTitle = lf.MetaTitle
}
if lf := localized[lang]; lf.MetaDescription != "" {
out.MetaDescription = lf.MetaDescription
}
// Merge LLM attrs onto pipeline attrs (validated against category keys).
// Attrs are independent of title/description soft-fail (synthesized/refused).
if err == nil && status != "failed" && status != "parse_failed" {
if merged := mergeEnhanceAttrsInto(attrs, enhanceAttrsFromRaw(raw), enhanceAllowedAttrKeys(in, out.Category)); len(merged) > 0 {
attrs = merged
enhanceAttrs = AttrsForEnhance(attrs, enhanceAllowedAttrKeys(in, out.Category))
out.Attributes = attrs
out.ProcessedAttributes = attrs
out.FieldSources["attributes"] = "ai_enhance"
}
}
if persistHash != "" {
out.FieldSources[FieldEnhanceInputHash] = persistHash
} else {
@@ -818,6 +838,8 @@ func (e *Engine) enhance(ctx context.Context, in ProductInput, category string,
}
llmName := SanitizeOutput(fmt.Sprint(obj["name"]))
llmDesc := SanitizeOutput(fmt.Sprint(obj["description"]))
llmMetaTitle, llmMetaDesc := metaFieldsFromEnhanceObj(obj)
llmAttrs := attrsFromEnhanceObj(obj)
// Never prefer-fallback to originals for empty/leakage LLM fields — that stamped
// status=ok + input_hash on garbage enhance output (prod: 30272ms "ok", wrong copy).
if reason := llmEnhanceHardRefuseReason(llmName, llmDesc); reason != "" {
@@ -846,6 +868,8 @@ func (e *Engine) enhance(ctx context.Context, in ProductInput, category string,
"reason": reason,
"raw": comp.Raw,
}
attachEnhanceMeta(meta, llmMetaTitle, llmMetaDesc)
attachEnhanceAttrs(meta, llmAttrs)
if forceReason != "" {
meta["forced_reenhance"] = true
meta["force_reason"] = forceReason
@@ -878,6 +902,16 @@ func (e *Engine) enhance(ctx context.Context, in ProductInput, category string,
if llmEnhanceHardRefuseReason(n2, d2) == "" {
name = preferredProductTitle(in.GTIN, n2, name, in.Name, in.PriorProcessedName)
desc = preferredProductDescription(name, d2)
mt2, md2 := metaFieldsFromEnhanceObj(obj2)
if mt2 != "" {
llmMetaTitle = mt2
}
if md2 != "" {
llmMetaDesc = md2
}
if a2 := attrsFromEnhanceObj(obj2); len(a2) > 0 {
llmAttrs = a2
}
if comp2.Raw != nil {
comp.Raw = comp2.Raw
}
@@ -903,6 +937,8 @@ func (e *Engine) enhance(ctx context.Context, in ProductInput, category string,
"status": "ok",
"raw": comp.Raw,
}
attachEnhanceMeta(meta, llmMetaTitle, llmMetaDesc)
attachEnhanceAttrs(meta, llmAttrs)
if forceReason != "" {
meta["reason"] = forceReason
meta["forced_reenhance"] = true
@@ -942,6 +978,115 @@ func (e *Engine) enhance(ctx context.Context, in ProductInput, category string,
return name, desc, comp.TotalTokens, meta, nil
}
// attachEnhanceMeta stores parsed SEO fields on enhance raw meta for RunSteps.
func attachEnhanceMeta(meta map[string]any, title, description string) {
if meta == nil {
return
}
if title != "" {
meta["meta_title"] = title
}
if description != "" {
meta["meta_description"] = description
}
}
// attachEnhanceAttrs stores parsed attrs on enhance raw meta for RunSteps merge.
func attachEnhanceAttrs(meta map[string]any, attrs map[string]any) {
if meta == nil || len(attrs) == 0 {
return
}
meta["attrs"] = attrs
}
// attrsFromEnhanceObj extracts an attrs/attributes object from enhance JSON.
func attrsFromEnhanceObj(obj map[string]any) map[string]any {
if obj == nil {
return nil
}
raw := obj["attrs"]
if raw == nil {
raw = obj["attributes"]
}
return coerceAttrMap(raw)
}
// enhanceAttrsFromRaw reads attrs stashed on enhance raw meta by attachEnhanceAttrs.
func enhanceAttrsFromRaw(raw any) map[string]any {
m, ok := raw.(map[string]any)
if !ok || m == nil {
return nil
}
if v, ok := m["attrs"]; ok {
return coerceAttrMap(v)
}
return nil
}
// coerceAttrMap normalizes JSON object / map[string]string into map[string]any.
func coerceAttrMap(raw any) map[string]any {
switch v := raw.(type) {
case map[string]any:
if len(v) == 0 {
return nil
}
out := make(map[string]any, len(v))
for k, val := range v {
k = strings.TrimSpace(k)
if k == "" || val == nil {
continue
}
out[k] = val
}
if len(out) == 0 {
return nil
}
return out
case map[string]string:
if len(v) == 0 {
return nil
}
out := make(map[string]any, len(v))
for k, val := range v {
k = strings.TrimSpace(k)
val = strings.TrimSpace(val)
if k == "" || val == "" {
continue
}
out[k] = val
}
if len(out) == 0 {
return nil
}
return out
default:
return nil
}
}
// mergeEnhanceAttrsInto merges LLM attrs onto base after AttrsForEnhance validation
// (MapAttrsOntoAllowedKeys + category allowlist). Empty LLM attrs → nil (no change).
func mergeEnhanceAttrsInto(base, llmAttrs map[string]any, allowed map[string]struct{}) map[string]any {
if len(llmAttrs) == 0 {
return nil
}
validated := AttrsForEnhance(llmAttrs, allowed)
if len(validated) == 0 {
return nil
}
out := make(map[string]any, len(base)+len(validated))
for k, v := range base {
out[k] = v
}
for k, v := range validated {
if !attrValuePresent(v) {
continue
}
out[k] = v
}
return out
}
// llmEnhanceHardRefuseReason reports empty/leakage LLM fields that must never be
// accepted as quality enhance (even when originals could fill the gap).
func llmEnhanceHardRefuseReason(name, desc string) string {
@@ -1042,7 +1187,8 @@ func isPromptLabelTitle(s string) bool {
lower := strings.ToLower(s)
for _, label := range []string{
"category", "name", "desc", "description",
"attrs", "attributes", "current name", "current description",
"meta", "meta_title", "meta_description", "metadescription",
"attrs", "attributes", "current name", "current description", "current meta",
} {
if lower == label || lower == label+":" {
return true
@@ -1092,6 +1238,7 @@ var promptLeakagePhrases = []string{
"reply with only json",
"your reply is parsed as json",
"description formula",
"seo meta formula",
"build name from attrs",
"prefer attrs values",
"order matters; join with",
@@ -1114,6 +1261,23 @@ var promptLeakagePhrases = []string{
"1-2 sentences",
"keep literal text as written",
"build name from attrs using this structure",
"category guidance",
"applies to name and description",
"apply it to both",
"never ignore the title formula",
// Role-sectioned CategoryEnhanceUserTemplate / KeySEOMeta markers.
"--- title ---",
"--- end title ---",
"--- description ---",
"--- end description ---",
"--- meta ---",
"--- end meta ---",
"--- attributes ---",
"--- end attributes ---",
"role: title",
"role: description",
"role: meta",
"role: attributes",
}
var promptLeakageLongKeywords = []string{
@@ -1217,21 +1381,75 @@ func scrubCategoryPollution(out *StepResult, namesByUID map[string]string, valid
}
// preferredProductTitle picks the first usable title, skipping empty values,
// prompt-label echoes like "Category:", and instruction-text leakage.
// prompt-label echoes like "Category:", instruction-text leakage, and brand-only
// stubs when a longer product name exists among the candidates (e.g. "ANKER"
// vs "Anker Soundcore Space One Pro").
func preferredProductTitle(gtin string, candidates ...string) string {
usable := make([]string, 0, len(candidates))
for _, c := range candidates {
c = strings.TrimSpace(c)
if c == "" || c == "<nil>" || isPromptLabelTitle(c) {
continue
}
usable = append(usable, c)
}
for _, c := range usable {
if isBrandOnlyTitleAmong(c, usable) {
continue
}
return SanitizeOutput(c)
}
// All remaining candidates are brand-only relative to each other — pick longest.
best := ""
for _, c := range usable {
if len([]rune(c)) > len([]rune(best)) {
best = c
}
}
if best != "" {
return SanitizeOutput(best)
}
if strings.TrimSpace(gtin) != "" {
return SanitizeText("Product " + strings.TrimSpace(gtin))
}
return "Product"
}
// isBrandOnlyTitleAmong is true when candidate looks like a brand stub that a
// longer candidate expands (prefix / first-token match).
func isBrandOnlyTitleAmong(candidate string, all []string) bool {
c := strings.TrimSpace(candidate)
if c == "" {
return false
}
words := strings.Fields(c)
if len(words) > 2 {
return false
}
if len(words) == 1 && len([]rune(c)) > 40 {
return false
}
clower := strings.ToLower(c)
for _, other := range all {
o := strings.TrimSpace(other)
if o == "" || strings.EqualFold(o, c) {
continue
}
if len([]rune(o)) <= len([]rune(c)) {
continue
}
olower := strings.ToLower(o)
if strings.HasPrefix(olower, clower+" ") || strings.HasPrefix(olower, clower+"-") {
return true
}
oWords := strings.Fields(o)
if len(words) == 1 && len(oWords) >= 2 && strings.EqualFold(oWords[0], words[0]) {
return true
}
}
return false
}
// Thin wrappers keep processing call sites stable; logic lives in company so
// catalog.RepairWeakEnhanceHashes can reuse it without an import cycle.
func isWeakPriorEnhanceDescription(priorDesc string, titles ...string) bool {
+18 -2
View File
@@ -202,6 +202,22 @@ func TestPreferredProductTitle_skipsFormulaLeakage(t *testing.T) {
}
}
func TestPreferredProductTitle_skipsBrandOnlyWhenFullNameExists(t *testing.T) {
got := preferredProductTitle("1", "ANKER", "Anker Soundcore Space One Pro")
if got != "Anker Soundcore Space One Pro" {
t.Fatalf("got %q want full product name", got)
}
got = preferredProductTitle("1", "Anker", "Anker Soundcore Space One Pro Wireless")
if got != "Anker Soundcore Space One Pro Wireless" {
t.Fatalf("got %q", got)
}
// Brand alone when no longer candidate.
got = preferredProductTitle("1", "ANKER")
if got != "ANKER" {
t.Fatalf("brand-only alone got %q", got)
}
}
func TestIsPromptLabelTitle(t *testing.T) {
cases := map[string]bool{
"Category:": true,
@@ -390,13 +406,13 @@ func TestHeuristicCompleter_keepsNameWhenFormulaConstraintsPresent(t *testing.T)
h := HeuristicCompleter{}
// Shape matches aiprompts.CategoryEnhanceUserTemplate: instruction "- name:"
// appears BEFORE the real "Name:" field — labeledPromptValue must skip leakage.
user := `Your reply is parsed as JSON {"name":"string","description":"string"} only (system schema). Write name and description in English (do not hardcode a language).
user := `Your reply is parsed as JSON {"name":"string","description":"string","meta_title":"string","meta_description":"string"} only (system schema). Write all fields in English (do not hardcode a language).
- name: short retail title; follow any Title formula constraints that follow; use Attrs
- description: prefer 1-3 factual paragraphs as ONE string
Category: demo-electronics
Name: Vox SWA-8000W
Desc: Washer
Description: Washer
Attrs: {"brand":"Vox"}
Title formula (order matters; join with " "). Build name from Attrs using this structure:
+45 -13
View File
@@ -296,15 +296,24 @@ func (p *Pipeline) LoadV1ProcessJobItems(ctx context.Context, companyID, jobID u
_ = json.Unmarshal(rawJSON, &rawData)
main, more := catalog.ExtractProductImages(mapped, rawData)
plainDesc := ""
// Preserve formula HTML from processed_description; meta synthesis strips tags.
descOut := ""
if descTxt != nil {
plainDesc = v1PlainDescription(*descTxt)
descOut = v1PreserveDescription(*descTxt)
}
eprelVal := extractEPRELFromAttrs(attrs)
attrs = SanitizeV1ProcessAttributesAllowed(attrs, allowedAttrs)
titleStr := derefStringPtr(title)
// Prefer full product name from mapped/raw when processed title is brand-only
// (e.g. LLM returned "ANKER" while feed has "Anker Soundcore Space One Pro").
titleStr = preferredProductTitle(ean, titleStr,
stringFromAny(mapped["name"]),
stringFromAny(mapped["title"]),
stringFromAny(rawData["name"]),
stringFromAny(rawData["title"]),
)
catStr := derefStringPtr(category)
catNameStr := derefStringPtr(categoryName)
// Projection gap: when processed.category is empty but mapped/raw carries a
@@ -363,14 +372,14 @@ func (p *Pipeline) LoadV1ProcessJobItems(ctx context.Context, companyID, jobID u
if catLabel == "" {
catLabel = catStr
}
if (metaTitleOut == nil || metaDescOut == nil) && (titleStr != "" || plainDesc != "" || catLabel != "") {
if (metaTitleOut == nil || metaDescOut == nil) && (titleStr != "" || descOut != "" || catLabel != "") {
synthTitle, synthDesc := fillMetaFromResult(StepResult{
Name: titleStr,
ProcessedName: titleStr,
Category: catStr,
CategoryName: catNameStr,
Description: plainDesc,
ProcessedDescription: plainDesc,
Description: descOut,
ProcessedDescription: descOut,
Attributes: attrs,
ProcessedAttributes: attrs,
})
@@ -384,8 +393,8 @@ func (p *Pipeline) LoadV1ProcessJobItems(ctx context.Context, companyID, jobID u
if metaDescOut == nil {
if synthDesc != "" {
metaDescOut = synthDesc
} else if plainDesc != "" {
metaDescOut = truncateMetaDescription(plainDesc, v1MetaDescriptionMaxChars)
} else if descOut != "" {
metaDescOut = truncateMetaDescription(v1PlainDescription(descOut), v1MetaDescriptionMaxChars)
}
}
} else if metaTitleOut == nil {
@@ -393,8 +402,8 @@ func (p *Pipeline) LoadV1ProcessJobItems(ctx context.Context, companyID, jobID u
}
var description any
if plainDesc != "" {
description = plainDesc
if descOut != "" {
description = descOut
} else {
description = nil
}
@@ -406,7 +415,10 @@ func (p *Pipeline) LoadV1ProcessJobItems(ctx context.Context, companyID, jobID u
catNameOut = catNameStr
}
titleOut := nullIfEmptyPtr(title)
var titleOut any
if titleStr != "" {
titleOut = titleStr
}
item := V1ProcessJobItem{
"ean": ean,
"status": MapV1JobItemStatus(itemStatus, true),
@@ -518,10 +530,30 @@ func loadCompanyCategoryNameMap(ctx context.Context, p *Pipeline, companyID uuid
return out
}
// v1PreserveDescription unwraps legacy JSON-encoded description payloads and
// normalizes entities, but keeps formula HTML tags (h1/h2/p/ul/…) for the V1
// process poll and DB-facing normalize. Prefer this over v1PlainDescription when
// enhance / category description_template markup must reach API clients.
func v1PreserveDescription(s string) string {
s = strings.TrimSpace(s)
if s == "" {
return ""
}
s = unwrapLegacyDescriptionStored(s)
s = strings.TrimSpace(s)
if s == "" {
return ""
}
s = html.UnescapeString(s)
s = strings.ReplaceAll(s, "\u00a0", " ")
s = strings.ReplaceAll(s, `\u00a0`, " ")
return strings.TrimSpace(s)
}
// v1PlainDescription normalizes legacy stored descriptions into a single plain-text
// string for the process poll (description is a string, not a one-element array).
// Handles JSON array/string encodings (e.g. mapped_data->>'description' when the
// feed value was an array) and HTML/<br>/entity markup.
// string (meta/SEO, feed normalize). Handles JSON array/string encodings and
// strips HTML/<br>/entity markup. Do not use for V1 item.description when formula
// HTML must be preserved — use v1PreserveDescription / DescriptionFromAny.
func v1PlainDescription(s string) string {
s = strings.TrimSpace(s)
if s == "" {
@@ -183,9 +183,9 @@ func ScoreV1ProcessCompletedItem(item V1ProcessJobItem, opts ScoreV1ProcessItemO
}
// EnforceV1ProcessCompletedItem fills LegacyProcessItem projection gaps for a
// successful item: plain nonempty description when title exists, meta_*, clean
// attributes, eprel object|null, and image key shapes. Category must already be
// set by the caller when mapped provides a unique_id.
// successful item: nonempty description when title exists (formula HTML
// preserved), meta_*, clean attributes, eprel object|null, and image key shapes.
// Category must already be set by the caller when mapped provides a unique_id.
//
// allowed is the company attribute_key set (canonicalized). When nil, only
// coreCharacteristicAttrKeys are kept (never leak feed junk like zavora).
@@ -224,8 +224,9 @@ func EnforceV1ProcessCompletedItem(item V1ProcessJobItem, language string, allow
catLabel = cat
}
desc, _ := plainDescriptionFromItem(item)
desc, _ := descriptionFromItem(item)
// Empty, weak, or title-echo copy must be replaced — never leave description==title.
// Formula HTML that satisfies multi-section templates is kept as-is.
if title != "" && (desc == "" || isWeakPriorEnhanceDescription(desc, title) || descriptionEchoesTitle(desc, title)) {
if synth := synthesizeDescriptionFromTitle(title, catLabel, language, attrs); synth != "" {
desc = synth
@@ -324,6 +325,38 @@ func stringFromItem(item V1ProcessJobItem, key string) string {
return strings.TrimSpace(stringFromAny(item[key]))
}
func descriptionFromItem(item V1ProcessJobItem) (string, bool) {
if item == nil {
return "", true
}
raw := item["description"]
if raw == nil {
return "", true
}
switch raw.(type) {
case string:
return DescriptionFromAny(raw), true
case []any, []string:
// Legacy mistake: description as array — coerce to string, keep HTML.
if s := DescriptionFromAny(raw); s != "" {
return s, true
}
return "", false
case map[string]any:
if s := DescriptionFromAny(raw); s != "" {
return s, true
}
return "", false
default:
s := strings.TrimSpace(fmt.Sprint(raw))
if s == "" || s == "<nil>" {
return "", true
}
return "", false
}
}
// plainDescriptionFromItem is kept for callers that need SEO/plain text (meta).
func plainDescriptionFromItem(item V1ProcessJobItem) (string, bool) {
if item == nil {
return "", true
@@ -336,7 +369,6 @@ func plainDescriptionFromItem(item V1ProcessJobItem) (string, bool) {
case string:
return PlainDescriptionFromAny(raw), true
case []any, []string:
// Legacy mistake: description as array — convert to plain string.
if s := PlainDescriptionFromAny(raw); s != "" {
return s, true
}
@@ -317,3 +317,33 @@ func TestEnforceV1ProcessCompletedItem_refreshesPromptLeakageMeta(t *testing.T)
t.Fatalf("prompt-leakage meta_description preserved: %q", md)
}
}
func TestEnforceV1ProcessCompletedItem_preservesFormulaHTML(t *testing.T) {
t.Parallel()
htmlDesc := `<h1>Anker Soundcore Space One Pro</h1><p>Zložljive ANC slušalke z bogatim zvokom.</p><ul><li>Bluetooth 5.3</li></ul>`
item := V1ProcessJobItem{
"ean": "1",
"status": "processed",
"id": "bbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb",
"processed_product_id": "bbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb",
"raw_product_id": "cccccccc-cccc-cccc-cccc-cccccccccccc",
"title": "Anker Soundcore Space One Pro",
"description": htmlDesc,
"category": "48",
"category_name": "Slušalke",
"attributes": map[string]any{"brand": "Anker"},
"eprel": nil,
}
out := EnforceV1ProcessCompletedItem(item, "sl", nil)
desc, _ := out["description"].(string)
for _, tag := range []string{"<h1>", "<p>", "<ul>"} {
if !strings.Contains(desc, tag) {
t.Fatalf("EnforceV1 must keep formula HTML %q, got %q", tag, desc)
}
}
// meta_description must stay plain (no tags).
md := fmt.Sprint(out["meta_description"])
if strings.Contains(md, "<h1>") || strings.Contains(md, "<p>") {
t.Fatalf("meta_description should be plain, got %q", md)
}
}