This commit is contained in:
2026-08-16 16:57:36 +02:00
parent 96f8c1115c
commit 532d439c41
106 changed files with 10147 additions and 494 deletions
+44 -8
View File
@@ -11,7 +11,12 @@ import (
"github.com/jackc/pgx/v5/pgxpool"
)
// LangPromptAny is the wildcard key for a single shared prompt used for every
// content language (avoid duplicating the same text per lang).
const LangPromptAny = "*"
// LangPromptMap is language-code → prompt text for category / template overrides.
// Optional key LangPromptAny ("*") is a shared overlay for all languages.
type LangPromptMap map[string]string
// LocalizedFields holds AI/output fields for one content language.
@@ -27,15 +32,16 @@ type LocalizedFields struct {
type LocalizedContent map[string]LocalizedFields
// SanitizeLangPromptMap validates language codes, sanitizes prompts, and drops empties.
// Accepts LangPromptAny ("*") as a shared any-language prompt key.
func SanitizeLangPromptMap(in map[string]string, maxRunes int) (LangPromptMap, error) {
out := make(LangPromptMap)
if len(in) == 0 {
return out, nil
}
for lang, prompt := range in {
code, err := ParseLanguage(lang, false)
code, err := parseLangPromptKey(lang)
if err != nil {
return nil, fmt.Errorf("unsupported language %q", lang)
return nil, err
}
p := strings.TrimSpace(security.SanitizePrompt(prompt, maxRunes))
if p == "" {
@@ -46,16 +52,46 @@ func SanitizeLangPromptMap(in map[string]string, maxRunes int) (LangPromptMap, e
return out, nil
}
// PromptForLanguage returns the prompt for lang, or empty if unset.
func PromptForLanguage(m LangPromptMap, lang string) string {
func parseLangPromptKey(raw string) (string, error) {
key := strings.TrimSpace(raw)
if key == LangPromptAny {
return LangPromptAny, nil
}
code, err := ParseLanguage(key, false)
if err != nil {
return "", fmt.Errorf("unsupported language %q", raw)
}
return code, nil
}
// PromptForLanguage resolves a prompt with fallback:
// requested lang → LangPromptAny ("*") → primary → "" (caller may then use built-in).
// Empty primary skips the primary step. Does not invent cross-lang text beyond this chain.
func PromptForLanguage(m LangPromptMap, lang, primary string) string {
if len(m) == 0 {
return ""
}
code, err := ParseLanguage(lang, true)
if err != nil {
code = DefaultLanguage
try := func(code string) string {
code = strings.TrimSpace(code)
if code == "" {
return ""
}
return strings.TrimSpace(m[code])
}
return strings.TrimSpace(m[code])
if code, err := ParseLanguage(lang, true); err == nil {
if p := try(code); p != "" {
return p
}
}
if p := try(LangPromptAny); p != "" {
return p
}
if prim, err := ParseLanguage(primary, false); err == nil {
if p := try(prim); p != "" {
return p
}
}
return ""
}
// HasAnyPrompt reports whether any language has a non-empty prompt.
+25 -3
View File
@@ -17,6 +17,7 @@ func TestSanitizeLangPromptMap(t *testing.T) {
m, err := SanitizeLangPromptMap(map[string]string{
"SL": " hello {{name}} ",
"en": "",
"*": " shared ",
}, 100)
if err != nil {
t.Fatal(err)
@@ -27,16 +28,37 @@ func TestSanitizeLangPromptMap(t *testing.T) {
if _, ok := m["en"]; ok {
t.Fatalf("empty en should be dropped: %#v", m)
}
if m[LangPromptAny] != "shared" {
t.Fatalf("wildcard missing: %#v", m)
}
}
func TestPromptForLanguage(t *testing.T) {
t.Parallel()
m := LangPromptMap{"sl": "slo", "en": "eng"}
if got := PromptForLanguage(m, "SL"); got != "slo" {
if got := PromptForLanguage(m, "SL", ""); got != "slo" {
t.Fatalf("got %q", got)
}
if got := PromptForLanguage(m, "de"); got != "" {
t.Fatalf("expected empty, got %q", got)
if got := PromptForLanguage(m, "de", ""); got != "" {
t.Fatalf("expected empty without primary, got %q", got)
}
// sl-only map: exact hit for sl
slOnly := LangPromptMap{"sl": "slo-only"}
if got := PromptForLanguage(slOnly, "sl", "sl"); got != "slo-only" {
t.Fatalf("sl exact: got %q", got)
}
// sl-only map: en request falls back to primary=sl
if got := PromptForLanguage(slOnly, "en", "sl"); got != "slo-only" {
t.Fatalf("en→primary sl: got %q", got)
}
// wildcard before primary
anyMap := LangPromptMap{"*": "shared", "sl": "slo"}
if got := PromptForLanguage(anyMap, "de", "sl"); got != "shared" {
t.Fatalf("wildcard before primary: got %q", got)
}
// explicit lang beats wildcard
if got := PromptForLanguage(anyMap, "sl", "en"); got != "slo" {
t.Fatalf("explicit beats *: got %q", got)
}
}
+151
View File
@@ -0,0 +1,151 @@
package company
import "strings"
// minUsableProductDescRunes is the floor for a prior description that may hash-skip
// the enhance LLM. Shorter copy is treated as thin and forced through enhance.
const minUsableProductDescRunes = 40
// shortBoilerplateDescRunes: short copy without product-fact token overlap is weak
// even when longer than minUsableProductDescRunes (generic one-liners).
const shortBoilerplateDescRunes = 96
// weakFillerPhrases are known heuristic / placeholder snippets that must never
// hash-skip enhance (mock-llm / invent fallbacks historically produced these).
var weakFillerPhrases = []string{
"ready for retail listing",
"quality product ready",
"product description",
"based on available specifications",
"with available specifications",
"available catalog details",
"pripravljeno za prodajo",
"na podlagi razpoložljivih specifikacij",
}
// IsWeakPriorEnhanceDescription reports empty, too-short, title-echo, known filler,
// or short boilerplate without overlapping tokens from title/fact sources.
func IsWeakPriorEnhanceDescription(priorDesc string, titles ...string) bool {
priorDesc = strings.TrimSpace(priorDesc)
if priorDesc == "" || priorDesc == "<nil>" {
return true
}
if len([]rune(priorDesc)) < minUsableProductDescRunes {
return true
}
for _, title := range titles {
if DescriptionEchoesTitle(priorDesc, title) {
return true
}
}
if ContainsWeakFillerPhrase(priorDesc) {
return true
}
if len([]rune(priorDesc)) <= shortBoilerplateDescRunes &&
!descriptionOverlapsProductFacts(priorDesc, titles...) {
return true
}
return false
}
// ContainsWeakFillerPhrase reports known placeholder / invent-fallback snippets.
func ContainsWeakFillerPhrase(desc string) bool {
lower := strings.ToLower(desc)
for _, p := range weakFillerPhrases {
if p != "" && strings.Contains(lower, p) {
return true
}
}
return false
}
var weakDescStopTokens = map[string]struct{}{
"a": {}, "an": {}, "the": {}, "and": {}, "or": {}, "of": {}, "in": {}, "on": {}, "for": {},
"to": {}, "with": {}, "from": {}, "by": {}, "is": {}, "are": {}, "this": {}, "that": {},
"product": {}, "products": {}, "category": {}, "description": {}, "specs": {}, "spec": {},
"key": {}, "je": {}, "v": {}, "za": {}, "z": {}, "iz": {}, "ter": {}, "ali": {},
}
func significantDescTokens(s string) map[string]struct{} {
s = strings.ToLower(s)
out := map[string]struct{}{}
var b strings.Builder
flush := func() {
tok := b.String()
b.Reset()
if len(tok) < 3 {
return
}
if _, stop := weakDescStopTokens[tok]; stop {
return
}
out[tok] = struct{}{}
}
for _, r := range s {
if (r >= 'a' && r <= 'z') || (r >= '0' && r <= '9') || r == '_' {
b.WriteRune(r)
continue
}
// Keep Latin-extended letters (Slovenian čšž etc.) via unicode letter-ish: non-space punctuation splits.
if r > 127 && ((r >= 'à' && r <= 'ÿ') || r == 'č' || r == 'š' || r == 'ž' || r == 'ć' || r == 'đ') {
b.WriteRune(r)
continue
}
flush()
}
flush()
return out
}
// descriptionOverlapsProductFacts is true when desc shares significant tokens with
// any fact source (title, brand, model, …).
func descriptionOverlapsProductFacts(desc string, factSources ...string) bool {
descToks := significantDescTokens(desc)
if len(descToks) == 0 {
return false
}
overlap := 0
seen := map[string]struct{}{}
for _, src := range factSources {
for tok := range significantDescTokens(src) {
if _, ok := descToks[tok]; !ok {
continue
}
if _, dup := seen[tok]; dup {
continue
}
seen[tok] = struct{}{}
overlap++
if overlap >= 2 || len(tok) >= 5 {
return true
}
}
}
return overlap >= 1 && len(seen) >= 1 && len([]rune(desc)) >= minUsableProductDescRunes+20
}
func normalizeForEchoCompare(s string) string {
return strings.Join(strings.Fields(strings.ToLower(strings.TrimSpace(s))), " ")
}
// DescriptionEchoesTitle is true when desc is EqualFold or near-equal to title
// (whitespace-normalized, or one contains the other with only a tiny length delta).
func DescriptionEchoesTitle(desc, title string) bool {
d := normalizeForEchoCompare(desc)
t := normalizeForEchoCompare(title)
if d == "" || t == "" {
return false
}
if d == t {
return true
}
shorter, longer := d, t
if len(d) > len(t) {
shorter, longer = t, d
}
if !strings.Contains(longer, shorter) {
return false
}
delta := len(longer) - len(shorter)
return delta <= 8 && float64(len(shorter))/float64(len(longer)) >= 0.85
}