package company import "strings" // minUsableProductDescRunes is the floor for a prior description that may hash-skip // the enhance LLM. Shorter copy is treated as thin and forced through enhance. const minUsableProductDescRunes = 40 // shortBoilerplateDescRunes: short copy without product-fact token overlap is weak // even when longer than minUsableProductDescRunes (generic one-liners). const shortBoilerplateDescRunes = 96 // weakFillerPhrases are known heuristic / placeholder snippets that must never // hash-skip enhance (mock-llm / invent fallbacks historically produced these). var weakFillerPhrases = []string{ "ready for retail listing", "quality product ready", "product description", "based on available specifications", "with available specifications", "available catalog details", "pripravljeno za prodajo", "na podlagi razpoložljivih specifikacij", } // IsWeakPriorEnhanceDescription reports empty, too-short, title-echo, known filler, // or short boilerplate without overlapping tokens from title/fact sources. // Heuristic synthesize (invent) is intentionally excluded — use ShouldRefuseEnhanceHashSkip. func IsWeakPriorEnhanceDescription(priorDesc string, titles ...string) bool { reason := EnhanceHashSkipBlockReason(priorDesc, titles...) return reason == "weak" || reason == "title-echo" } // heuristicSynthesizePhrases are distinctive invent / formula-skeleton snippets from // synthesizeDescriptionFromTitle / synthesizeDescriptionFromFormula / factualDescriptionIntro. // These may look "strong" enough to pass IsWeakPriorEnhanceDescription but must never // hash-skip enhance (would leave fallback copy forever on reprocess). var heuristicSynthesizePhrases = []string{ "is a catalog product with the known attributes", "is listed in the ", ". key specs:", "je katalogski izdelek z znanimi atributi", "je izdelek v kategoriji", "je izdelek znamke", ". ključne specifikacije:", // factualDescriptionIntro (post-phrase-avoidance invent) " — znamka ", ", znamka ", " — katalogski izdelek", " — from ", " — catalog product", } // LooksLikeHeuristicSynthesize reports invent / formula-skeleton fallback copy. func LooksLikeHeuristicSynthesize(desc string) bool { plain := strings.ToLower(plainTextForWeakCheck(desc)) if plain == "" { return false } for _, p := range heuristicSynthesizePhrases { if p != "" && strings.Contains(plain, p) { return true } } // EN invent: " is a <category> product from <brand>" if strings.Contains(plain, " is a ") && strings.Contains(plain, " product from ") { return true } // SL/CS short invent: "<title> je … znamk|kategor|produkt …" if strings.Contains(plain, " je ") && (strings.Contains(plain, "znamk") || strings.Contains(plain, "kategor") || strings.Contains(plain, "produkt") || strings.Contains(plain, "televiz") || strings.Contains(plain, "monitor")) { return true } return false } // plainTextForWeakCheck strips HTML tags so <p>thin invent</p> is judged on visible copy. func plainTextForWeakCheck(s string) string { s = strings.TrimSpace(s) if s == "" { return "" } var b strings.Builder b.Grow(len(s)) inTag := false for _, r := range s { switch { case r == '<': inTag = true case r == '>': inTag = false case !inTag: b.WriteRune(r) } } return strings.TrimSpace(b.String()) } // EnhanceHashSkipBlockReason returns a stable reason when prior description must // not hash-skip the enhance LLM: "weak", "title-echo", "synth", or "". func EnhanceHashSkipBlockReason(priorDesc string, titles ...string) string { priorDesc = strings.TrimSpace(priorDesc) if priorDesc == "" || priorDesc == "<nil>" { return "weak" } plain := plainTextForWeakCheck(priorDesc) if plain == "" || plain == "<nil>" { return "weak" } if len([]rune(plain)) < minUsableProductDescRunes { return "weak" } for _, title := range titles { if DescriptionEchoesTitle(plain, title) || DescriptionEchoesTitle(priorDesc, title) { return "title-echo" } } if ContainsWeakFillerPhrase(plain) || ContainsWeakFillerPhrase(priorDesc) { return "weak" } if LooksLikeHeuristicSynthesize(priorDesc) { return "synth" } if len([]rune(plain)) <= shortBoilerplateDescRunes && !descriptionOverlapsProductFacts(plain, titles...) { return "weak" } return "" } // ShouldRefuseEnhanceHashSkip is true when a stored enhance_input_hash must be // ignored / cleared (weak, title-echo, or heuristic synthesize). func ShouldRefuseEnhanceHashSkip(priorDesc string, titles ...string) bool { return EnhanceHashSkipBlockReason(priorDesc, titles...) != "" } // DescriptionMissingFormulaHTMLTags is true when sectionTypes require HTML tags // (h1/h2/h3/h4, p, ul) that are absent from desc — formula-mismatch for skip/clear. func DescriptionMissingFormulaHTMLTags(desc string, sectionTypes []string) bool { if len(sectionTypes) == 0 { return false } lower := strings.ToLower(desc) for _, raw := range sectionTypes { typ := strings.ToLower(strings.TrimSpace(raw)) switch typ { case "h1", "h2", "h3", "h4": if !strings.Contains(lower, "<"+typ) { return true } case "ul": if !strings.Contains(lower, "<ul") { return true } case "p", "": if !strings.Contains(lower, "<p") { return true } } } return false } // ContainsWeakFillerPhrase reports known placeholder / invent-fallback snippets. func ContainsWeakFillerPhrase(desc string) bool { lower := strings.ToLower(desc) for _, p := range weakFillerPhrases { if p != "" && strings.Contains(lower, p) { return true } } return false } var weakDescStopTokens = map[string]struct{}{ "a": {}, "an": {}, "the": {}, "and": {}, "or": {}, "of": {}, "in": {}, "on": {}, "for": {}, "to": {}, "with": {}, "from": {}, "by": {}, "is": {}, "are": {}, "this": {}, "that": {}, "product": {}, "products": {}, "category": {}, "description": {}, "specs": {}, "spec": {}, "key": {}, "je": {}, "v": {}, "za": {}, "z": {}, "iz": {}, "ter": {}, "ali": {}, } func significantDescTokens(s string) map[string]struct{} { s = strings.ToLower(s) out := map[string]struct{}{} var b strings.Builder flush := func() { tok := b.String() b.Reset() if len(tok) < 3 { return } if _, stop := weakDescStopTokens[tok]; stop { return } out[tok] = struct{}{} } for _, r := range s { if (r >= 'a' && r <= 'z') || (r >= '0' && r <= '9') || r == '_' { b.WriteRune(r) continue } // Keep Latin-extended letters (Slovenian čšž etc.) via unicode letter-ish: non-space punctuation splits. if r > 127 && ((r >= 'à' && r <= 'ÿ') || r == 'č' || r == 'š' || r == 'ž' || r == 'ć' || r == 'đ') { b.WriteRune(r) continue } flush() } flush() return out } // descriptionOverlapsProductFacts is true when desc shares significant tokens with // any fact source (title, brand, model, …). func descriptionOverlapsProductFacts(desc string, factSources ...string) bool { descToks := significantDescTokens(desc) if len(descToks) == 0 { return false } overlap := 0 seen := map[string]struct{}{} for _, src := range factSources { for tok := range significantDescTokens(src) { if _, ok := descToks[tok]; !ok { continue } if _, dup := seen[tok]; dup { continue } seen[tok] = struct{}{} overlap++ if overlap >= 2 || len(tok) >= 5 { return true } } } return overlap >= 1 && len(seen) >= 1 && len([]rune(desc)) >= minUsableProductDescRunes+20 } func normalizeForEchoCompare(s string) string { return strings.Join(strings.Fields(strings.ToLower(strings.TrimSpace(s))), " ") } // DescriptionEchoesTitle is true when desc is EqualFold or near-equal to title // (whitespace-normalized, or one contains the other with only a tiny length delta). func DescriptionEchoesTitle(desc, title string) bool { d := normalizeForEchoCompare(desc) t := normalizeForEchoCompare(title) if d == "" || t == "" { return false } if d == t { return true } shorter, longer := d, t if len(d) > len(t) { shorter, longer = t, d } if !strings.Contains(longer, shorter) { return false } delta := len(longer) - len(shorter) return delta <= 8 && float64(len(shorter))/float64(len(longer)) >= 0.85 }