package company import "strings" // minUsableProductDescRunes is the floor for a prior description that may hash-skip // the enhance LLM. Shorter copy is treated as thin and forced through enhance. const minUsableProductDescRunes = 40 // shortBoilerplateDescRunes: short copy without product-fact token overlap is weak // even when longer than minUsableProductDescRunes (generic one-liners). const shortBoilerplateDescRunes = 96 // weakFillerPhrases are known heuristic / placeholder snippets that must never // hash-skip enhance (mock-llm / invent fallbacks historically produced these). var weakFillerPhrases = []string{ "ready for retail listing", "quality product ready", "product description", "based on available specifications", "with available specifications", "available catalog details", "pripravljeno za prodajo", "na podlagi razpoložljivih specifikacij", } // IsWeakPriorEnhanceDescription reports empty, too-short, title-echo, known filler, // or short boilerplate without overlapping tokens from title/fact sources. // Heuristic synthesize (invent) is intentionally excluded — use ShouldRefuseEnhanceHashSkip. func IsWeakPriorEnhanceDescription(priorDesc string, titles ...string) bool { reason := EnhanceHashSkipBlockReason(priorDesc, titles...) return reason == "weak" || reason == "title-echo" } // heuristicSynthesizePhrases are distinctive invent snippets from // synthesizeDescriptionFromTitle / synthesizeDescriptionFromFormula / factualDescriptionIntro. // These may look "strong" enough to pass IsWeakPriorEnhanceDescription but must never // hash-skip enhance (would leave fallback copy forever on reprocess). // Only whole sentences that no copywriter would produce belong here — anything a // real category formula could legitimately emit goes in synthSkeletonPhrases. var heuristicSynthesizePhrases = []string{ "is a catalog product with the known attributes", "is listed in the ", ". key specs:", "je katalogski izdelek z znanimi atributi", "je izdelek v kategoriji", "je izdelek znamke", ". ključne specifikacije:", " — katalogski izdelek", " — catalog product", } // synthSkeletonPhrases also appear in synthesize output but are ordinary words a // real description may use ("Ključne lastnosti" as a heading, "znamka X" in a spec // line, "Key features" as a section title). They are suggestive, not proof, so only // the hash-skip gate consults them — see IsInventFallbackDescription. var synthSkeletonPhrases = []string{ // factualDescriptionIntro (post-phrase-avoidance invent) " — znamka ", ", znamka ", " — from ", // synthesizeDescriptionFromFormula heading fallbacks "ključne lastnosti", "key features", } // maxSynthSkeletonRunes bounds the suggestive signals above. Every synthesize helper // emits at most a title heading, one intro sentence, and a short "Label: value" // list, so it stays well under this; real category-formula prose (multiple 100-word // paragraphs plus a spec list) runs several times longer. const maxSynthSkeletonRunes = 600 // maxSynthInventRunes bounds the generic " is a … product from …" / // "<title> je … znamke …" shapes. Those describe a single invent sentence, so they // may only match copy that is itself about one sentence long. Unbounded, " je " // plus "znamk" matched almost any Slovenian description. const maxSynthInventRunes = 2 * shortBoilerplateDescRunes // IsInventFallbackDescription reports copy that is unmistakably machine invent — // whole sentences emitted by the synthesize helpers, or the one-sentence // "<title> is a <category> product from <brand>" shape. // // This is the strict test, safe for discarding a description outright. The looser // LooksLikeHeuristicSynthesize adds ambiguous wording and is only used where a // false positive costs one extra enhance call rather than the copy itself. func IsInventFallbackDescription(desc string) bool { plain := strings.ToLower(plainTextForWeakCheck(desc)) if plain == "" { return false } for _, p := range heuristicSynthesizePhrases { if p != "" && strings.Contains(plain, p) { return true } } if len([]rune(plain)) > maxSynthInventRunes { return false } // EN invent: "<title> is a <category> product from <brand>" if strings.Contains(plain, " is a ") && strings.Contains(plain, " product from ") { return true } // SL/CS short invent: "<title> je … znamk|kategor|produkt …" return strings.Contains(plain, " je ") && (strings.Contains(plain, "znamk") || strings.Contains(plain, "kategor") || strings.Contains(plain, "produkt") || strings.Contains(plain, "televiz") || strings.Contains(plain, "monitor")) } // LooksLikeHeuristicSynthesize reports invent / formula-skeleton fallback copy for // the enhance hash-skip gate: IsInventFallbackDescription plus wording that merely // suggests a skeleton, inside a body short enough to be one. A false positive here // forces one re-enhance, which is cheap; never use it to drop a description. func LooksLikeHeuristicSynthesize(desc string) bool { if IsInventFallbackDescription(desc) { return true } plain := strings.ToLower(plainTextForWeakCheck(desc)) if plain == "" || len([]rune(plain)) > maxSynthSkeletonRunes { return false } for _, p := range synthSkeletonPhrases { if p != "" && strings.Contains(plain, p) { return true } } return false } // plainTextForWeakCheck strips HTML tags so <p>thin invent</p> is judged on visible copy. func plainTextForWeakCheck(s string) string { s = strings.TrimSpace(s) if s == "" { return "" } var b strings.Builder b.Grow(len(s)) inTag := false for _, r := range s { switch { case r == '<': inTag = true case r == '>': inTag = false case !inTag: b.WriteRune(r) } } return strings.TrimSpace(b.String()) } // EnhanceHashSkipBlockReason returns a stable reason when prior description must // not hash-skip the enhance LLM: "weak", "title-echo", "synth", or "". func EnhanceHashSkipBlockReason(priorDesc string, titles ...string) string { priorDesc = strings.TrimSpace(priorDesc) if priorDesc == "" || priorDesc == "<nil>" { return "weak" } plain := plainTextForWeakCheck(priorDesc) if plain == "" || plain == "<nil>" { return "weak" } if len([]rune(plain)) < minUsableProductDescRunes { return "weak" } for _, title := range titles { if DescriptionEchoesTitle(plain, title) || DescriptionEchoesTitle(priorDesc, title) { return "title-echo" } } if ContainsWeakFillerPhrase(plain) || ContainsWeakFillerPhrase(priorDesc) { return "weak" } if LooksLikeHeuristicSynthesize(priorDesc) { return "synth" } if len([]rune(plain)) <= shortBoilerplateDescRunes && !descriptionOverlapsProductFacts(plain, titles...) { return "weak" } return "" } // ShouldRefuseEnhanceHashSkip is true when a stored enhance_input_hash must be // ignored / cleared (weak, title-echo, or heuristic synthesize). func ShouldRefuseEnhanceHashSkip(priorDesc string, titles ...string) bool { return EnhanceHashSkipBlockReason(priorDesc, titles...) != "" } // DescriptionMissingFormulaHTMLTags is true when sectionTypes require HTML tags // (h1/h2/h3/h4, p, ul) that are absent from desc — formula-mismatch for skip/clear. // Counts matter: a formula with two h2 sections needs at least two <h2 opens. func DescriptionMissingFormulaHTMLTags(desc string, sectionTypes []string) bool { if len(sectionTypes) == 0 { return false } need := map[string]int{} for _, raw := range sectionTypes { typ := strings.ToLower(strings.TrimSpace(raw)) switch typ { case "h1", "h2", "h3", "h4", "ul": need[typ]++ case "p", "": need["p"]++ } } if len(need) == 0 { return false } lower := strings.ToLower(desc) for typ, n := range need { needle := "<" + typ if strings.Count(lower, needle) < n { return true } } return false } // ContainsWeakFillerPhrase reports known placeholder / invent-fallback snippets. func ContainsWeakFillerPhrase(desc string) bool { lower := strings.ToLower(desc) for _, p := range weakFillerPhrases { if p != "" && strings.Contains(lower, p) { return true } } return false } var weakDescStopTokens = map[string]struct{}{ "a": {}, "an": {}, "the": {}, "and": {}, "or": {}, "of": {}, "in": {}, "on": {}, "for": {}, "to": {}, "with": {}, "from": {}, "by": {}, "is": {}, "are": {}, "this": {}, "that": {}, "product": {}, "products": {}, "category": {}, "description": {}, "specs": {}, "spec": {}, "key": {}, "je": {}, "v": {}, "za": {}, "z": {}, "iz": {}, "ter": {}, "ali": {}, } func significantDescTokens(s string) map[string]struct{} { s = strings.ToLower(s) out := map[string]struct{}{} var b strings.Builder flush := func() { tok := b.String() b.Reset() if len(tok) < 3 { return } if _, stop := weakDescStopTokens[tok]; stop { return } out[tok] = struct{}{} } for _, r := range s { if (r >= 'a' && r <= 'z') || (r >= '0' && r <= '9') || r == '_' { b.WriteRune(r) continue } // Keep Latin-extended letters (Slovenian čšž etc.) via unicode letter-ish: non-space punctuation splits. if r > 127 && ((r >= 'à' && r <= 'ÿ') || r == 'č' || r == 'š' || r == 'ž' || r == 'ć' || r == 'đ') { b.WriteRune(r) continue } flush() } flush() return out } // descriptionOverlapsProductFacts is true when desc shares significant tokens with // any fact source (title, brand, model, …). func descriptionOverlapsProductFacts(desc string, factSources ...string) bool { descToks := significantDescTokens(desc) if len(descToks) == 0 { return false } overlap := 0 seen := map[string]struct{}{} for _, src := range factSources { for tok := range significantDescTokens(src) { if _, ok := descToks[tok]; !ok { continue } if _, dup := seen[tok]; dup { continue } seen[tok] = struct{}{} overlap++ if overlap >= 2 || len(tok) >= 5 { return true } } } return overlap >= 1 && len(seen) >= 1 && len([]rune(desc)) >= minUsableProductDescRunes+20 } func normalizeForEchoCompare(s string) string { return strings.Join(strings.Fields(strings.ToLower(strings.TrimSpace(s))), " ") } // DescriptionEchoesTitle is true when desc is EqualFold or near-equal to title // (whitespace-normalized, or one contains the other with only a tiny length delta). func DescriptionEchoesTitle(desc, title string) bool { d := normalizeForEchoCompare(desc) t := normalizeForEchoCompare(title) if d == "" || t == "" { return false } if d == t { return true } shorter, longer := d, t if len(d) > len(t) { shorter, longer = t, d } if !strings.Contains(longer, shorter) { return false } delta := len(longer) - len(shorter) return delta <= 8 && float64(len(shorter))/float64(len(longer)) >= 0.85 }