fixes
This commit is contained in:
@@ -11,7 +11,12 @@ import (
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
)
|
||||
|
||||
// LangPromptAny is the wildcard key for a single shared prompt used for every
|
||||
// content language (avoid duplicating the same text per lang).
|
||||
const LangPromptAny = "*"
|
||||
|
||||
// LangPromptMap is language-code → prompt text for category / template overrides.
|
||||
// Optional key LangPromptAny ("*") is a shared overlay for all languages.
|
||||
type LangPromptMap map[string]string
|
||||
|
||||
// LocalizedFields holds AI/output fields for one content language.
|
||||
@@ -27,15 +32,16 @@ type LocalizedFields struct {
|
||||
type LocalizedContent map[string]LocalizedFields
|
||||
|
||||
// SanitizeLangPromptMap validates language codes, sanitizes prompts, and drops empties.
|
||||
// Accepts LangPromptAny ("*") as a shared any-language prompt key.
|
||||
func SanitizeLangPromptMap(in map[string]string, maxRunes int) (LangPromptMap, error) {
|
||||
out := make(LangPromptMap)
|
||||
if len(in) == 0 {
|
||||
return out, nil
|
||||
}
|
||||
for lang, prompt := range in {
|
||||
code, err := ParseLanguage(lang, false)
|
||||
code, err := parseLangPromptKey(lang)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("unsupported language %q", lang)
|
||||
return nil, err
|
||||
}
|
||||
p := strings.TrimSpace(security.SanitizePrompt(prompt, maxRunes))
|
||||
if p == "" {
|
||||
@@ -46,16 +52,46 @@ func SanitizeLangPromptMap(in map[string]string, maxRunes int) (LangPromptMap, e
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// PromptForLanguage returns the prompt for lang, or empty if unset.
|
||||
func PromptForLanguage(m LangPromptMap, lang string) string {
|
||||
func parseLangPromptKey(raw string) (string, error) {
|
||||
key := strings.TrimSpace(raw)
|
||||
if key == LangPromptAny {
|
||||
return LangPromptAny, nil
|
||||
}
|
||||
code, err := ParseLanguage(key, false)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("unsupported language %q", raw)
|
||||
}
|
||||
return code, nil
|
||||
}
|
||||
|
||||
// PromptForLanguage resolves a prompt with fallback:
|
||||
// requested lang → LangPromptAny ("*") → primary → "" (caller may then use built-in).
|
||||
// Empty primary skips the primary step. Does not invent cross-lang text beyond this chain.
|
||||
func PromptForLanguage(m LangPromptMap, lang, primary string) string {
|
||||
if len(m) == 0 {
|
||||
return ""
|
||||
}
|
||||
code, err := ParseLanguage(lang, true)
|
||||
if err != nil {
|
||||
code = DefaultLanguage
|
||||
try := func(code string) string {
|
||||
code = strings.TrimSpace(code)
|
||||
if code == "" {
|
||||
return ""
|
||||
}
|
||||
return strings.TrimSpace(m[code])
|
||||
}
|
||||
return strings.TrimSpace(m[code])
|
||||
if code, err := ParseLanguage(lang, true); err == nil {
|
||||
if p := try(code); p != "" {
|
||||
return p
|
||||
}
|
||||
}
|
||||
if p := try(LangPromptAny); p != "" {
|
||||
return p
|
||||
}
|
||||
if prim, err := ParseLanguage(primary, false); err == nil {
|
||||
if p := try(prim); p != "" {
|
||||
return p
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// HasAnyPrompt reports whether any language has a non-empty prompt.
|
||||
|
||||
@@ -17,6 +17,7 @@ func TestSanitizeLangPromptMap(t *testing.T) {
|
||||
m, err := SanitizeLangPromptMap(map[string]string{
|
||||
"SL": " hello {{name}} ",
|
||||
"en": "",
|
||||
"*": " shared ",
|
||||
}, 100)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
@@ -27,16 +28,37 @@ func TestSanitizeLangPromptMap(t *testing.T) {
|
||||
if _, ok := m["en"]; ok {
|
||||
t.Fatalf("empty en should be dropped: %#v", m)
|
||||
}
|
||||
if m[LangPromptAny] != "shared" {
|
||||
t.Fatalf("wildcard missing: %#v", m)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPromptForLanguage(t *testing.T) {
|
||||
t.Parallel()
|
||||
m := LangPromptMap{"sl": "slo", "en": "eng"}
|
||||
if got := PromptForLanguage(m, "SL"); got != "slo" {
|
||||
if got := PromptForLanguage(m, "SL", ""); got != "slo" {
|
||||
t.Fatalf("got %q", got)
|
||||
}
|
||||
if got := PromptForLanguage(m, "de"); got != "" {
|
||||
t.Fatalf("expected empty, got %q", got)
|
||||
if got := PromptForLanguage(m, "de", ""); got != "" {
|
||||
t.Fatalf("expected empty without primary, got %q", got)
|
||||
}
|
||||
// sl-only map: exact hit for sl
|
||||
slOnly := LangPromptMap{"sl": "slo-only"}
|
||||
if got := PromptForLanguage(slOnly, "sl", "sl"); got != "slo-only" {
|
||||
t.Fatalf("sl exact: got %q", got)
|
||||
}
|
||||
// sl-only map: en request falls back to primary=sl
|
||||
if got := PromptForLanguage(slOnly, "en", "sl"); got != "slo-only" {
|
||||
t.Fatalf("en→primary sl: got %q", got)
|
||||
}
|
||||
// wildcard before primary
|
||||
anyMap := LangPromptMap{"*": "shared", "sl": "slo"}
|
||||
if got := PromptForLanguage(anyMap, "de", "sl"); got != "shared" {
|
||||
t.Fatalf("wildcard before primary: got %q", got)
|
||||
}
|
||||
// explicit lang beats wildcard
|
||||
if got := PromptForLanguage(anyMap, "sl", "en"); got != "slo" {
|
||||
t.Fatalf("explicit beats *: got %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,151 @@
|
||||
package company
|
||||
|
||||
import "strings"
|
||||
|
||||
// minUsableProductDescRunes is the floor for a prior description that may hash-skip
|
||||
// the enhance LLM. Shorter copy is treated as thin and forced through enhance.
|
||||
const minUsableProductDescRunes = 40
|
||||
|
||||
// shortBoilerplateDescRunes: short copy without product-fact token overlap is weak
|
||||
// even when longer than minUsableProductDescRunes (generic one-liners).
|
||||
const shortBoilerplateDescRunes = 96
|
||||
|
||||
// weakFillerPhrases are known heuristic / placeholder snippets that must never
|
||||
// hash-skip enhance (mock-llm / invent fallbacks historically produced these).
|
||||
var weakFillerPhrases = []string{
|
||||
"ready for retail listing",
|
||||
"quality product ready",
|
||||
"product description",
|
||||
"based on available specifications",
|
||||
"with available specifications",
|
||||
"available catalog details",
|
||||
"pripravljeno za prodajo",
|
||||
"na podlagi razpoložljivih specifikacij",
|
||||
}
|
||||
|
||||
// IsWeakPriorEnhanceDescription reports empty, too-short, title-echo, known filler,
|
||||
// or short boilerplate without overlapping tokens from title/fact sources.
|
||||
func IsWeakPriorEnhanceDescription(priorDesc string, titles ...string) bool {
|
||||
priorDesc = strings.TrimSpace(priorDesc)
|
||||
if priorDesc == "" || priorDesc == "<nil>" {
|
||||
return true
|
||||
}
|
||||
if len([]rune(priorDesc)) < minUsableProductDescRunes {
|
||||
return true
|
||||
}
|
||||
for _, title := range titles {
|
||||
if DescriptionEchoesTitle(priorDesc, title) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
if ContainsWeakFillerPhrase(priorDesc) {
|
||||
return true
|
||||
}
|
||||
if len([]rune(priorDesc)) <= shortBoilerplateDescRunes &&
|
||||
!descriptionOverlapsProductFacts(priorDesc, titles...) {
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// ContainsWeakFillerPhrase reports known placeholder / invent-fallback snippets.
|
||||
func ContainsWeakFillerPhrase(desc string) bool {
|
||||
lower := strings.ToLower(desc)
|
||||
for _, p := range weakFillerPhrases {
|
||||
if p != "" && strings.Contains(lower, p) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
var weakDescStopTokens = map[string]struct{}{
|
||||
"a": {}, "an": {}, "the": {}, "and": {}, "or": {}, "of": {}, "in": {}, "on": {}, "for": {},
|
||||
"to": {}, "with": {}, "from": {}, "by": {}, "is": {}, "are": {}, "this": {}, "that": {},
|
||||
"product": {}, "products": {}, "category": {}, "description": {}, "specs": {}, "spec": {},
|
||||
"key": {}, "je": {}, "v": {}, "za": {}, "z": {}, "iz": {}, "ter": {}, "ali": {},
|
||||
}
|
||||
|
||||
func significantDescTokens(s string) map[string]struct{} {
|
||||
s = strings.ToLower(s)
|
||||
out := map[string]struct{}{}
|
||||
var b strings.Builder
|
||||
flush := func() {
|
||||
tok := b.String()
|
||||
b.Reset()
|
||||
if len(tok) < 3 {
|
||||
return
|
||||
}
|
||||
if _, stop := weakDescStopTokens[tok]; stop {
|
||||
return
|
||||
}
|
||||
out[tok] = struct{}{}
|
||||
}
|
||||
for _, r := range s {
|
||||
if (r >= 'a' && r <= 'z') || (r >= '0' && r <= '9') || r == '_' {
|
||||
b.WriteRune(r)
|
||||
continue
|
||||
}
|
||||
// Keep Latin-extended letters (Slovenian čšž etc.) via unicode letter-ish: non-space punctuation splits.
|
||||
if r > 127 && ((r >= 'à' && r <= 'ÿ') || r == 'č' || r == 'š' || r == 'ž' || r == 'ć' || r == 'đ') {
|
||||
b.WriteRune(r)
|
||||
continue
|
||||
}
|
||||
flush()
|
||||
}
|
||||
flush()
|
||||
return out
|
||||
}
|
||||
|
||||
// descriptionOverlapsProductFacts is true when desc shares significant tokens with
|
||||
// any fact source (title, brand, model, …).
|
||||
func descriptionOverlapsProductFacts(desc string, factSources ...string) bool {
|
||||
descToks := significantDescTokens(desc)
|
||||
if len(descToks) == 0 {
|
||||
return false
|
||||
}
|
||||
overlap := 0
|
||||
seen := map[string]struct{}{}
|
||||
for _, src := range factSources {
|
||||
for tok := range significantDescTokens(src) {
|
||||
if _, ok := descToks[tok]; !ok {
|
||||
continue
|
||||
}
|
||||
if _, dup := seen[tok]; dup {
|
||||
continue
|
||||
}
|
||||
seen[tok] = struct{}{}
|
||||
overlap++
|
||||
if overlap >= 2 || len(tok) >= 5 {
|
||||
return true
|
||||
}
|
||||
}
|
||||
}
|
||||
return overlap >= 1 && len(seen) >= 1 && len([]rune(desc)) >= minUsableProductDescRunes+20
|
||||
}
|
||||
|
||||
func normalizeForEchoCompare(s string) string {
|
||||
return strings.Join(strings.Fields(strings.ToLower(strings.TrimSpace(s))), " ")
|
||||
}
|
||||
|
||||
// DescriptionEchoesTitle is true when desc is EqualFold or near-equal to title
|
||||
// (whitespace-normalized, or one contains the other with only a tiny length delta).
|
||||
func DescriptionEchoesTitle(desc, title string) bool {
|
||||
d := normalizeForEchoCompare(desc)
|
||||
t := normalizeForEchoCompare(title)
|
||||
if d == "" || t == "" {
|
||||
return false
|
||||
}
|
||||
if d == t {
|
||||
return true
|
||||
}
|
||||
shorter, longer := d, t
|
||||
if len(d) > len(t) {
|
||||
shorter, longer = t, d
|
||||
}
|
||||
if !strings.Contains(longer, shorter) {
|
||||
return false
|
||||
}
|
||||
delta := len(longer) - len(shorter)
|
||||
return delta <= 8 && float64(len(shorter))/float64(len(longer)) >= 0.85
|
||||
}
|
||||
Reference in New Issue
Block a user