descrybe/apps/api/internal/company/weak_desc.go

package company

import "strings"

// minUsableProductDescRunes is the floor for a prior description that may hash-skip
// the enhance LLM. Shorter copy is treated as thin and forced through enhance.
const minUsableProductDescRunes = 40

// shortBoilerplateDescRunes: short copy without product-fact token overlap is weak
// even when longer than minUsableProductDescRunes (generic one-liners).
const shortBoilerplateDescRunes = 96

// weakFillerPhrases are known heuristic / placeholder snippets that must never
// hash-skip enhance (mock-llm / invent fallbacks historically produced these).
var weakFillerPhrases = []string{
	"ready for retail listing",
	"quality product ready",
	"product description",
	"based on available specifications",
	"with available specifications",
	"available catalog details",
	"pripravljeno za prodajo",
	"na podlagi razpoložljivih specifikacij",
}

// IsWeakPriorEnhanceDescription reports empty, too-short, title-echo, known filler,
// or short boilerplate without overlapping tokens from title/fact sources.
// Heuristic synthesize (invent) is intentionally excluded — use ShouldRefuseEnhanceHashSkip.
func IsWeakPriorEnhanceDescription(priorDesc string, titles ...string) bool {
	reason := EnhanceHashSkipBlockReason(priorDesc, titles...)
	return reason == "weak" || reason == "title-echo"
}

// heuristicSynthesizePhrases are distinctive invent / formula-skeleton snippets from
// synthesizeDescriptionFromTitle / synthesizeDescriptionFromFormula / factualDescriptionIntro.
// These may look "strong" enough to pass IsWeakPriorEnhanceDescription but must never
// hash-skip enhance (would leave fallback copy forever on reprocess).
var heuristicSynthesizePhrases = []string{
	"is a catalog product with the known attributes",
	"is listed in the ",
	". key specs:",
	"je katalogski izdelek z znanimi atributi",
	"je izdelek v kategoriji",
	"je izdelek znamke",
	". ključne specifikacije:",
	// factualDescriptionIntro (post-phrase-avoidance invent)
	" — znamka ",
	", znamka ",
	" — katalogski izdelek",
	" — from ",
	" — catalog product",
}

// LooksLikeHeuristicSynthesize reports invent / formula-skeleton fallback copy.
func LooksLikeHeuristicSynthesize(desc string) bool {
	plain := strings.ToLower(plainTextForWeakCheck(desc))
	if plain == "" {
		return false
	}
	for _, p := range heuristicSynthesizePhrases {
		if p != "" && strings.Contains(plain, p) {
			return true
		}
	}
	// EN invent: "<title> is a <category> product from <brand>"
	if strings.Contains(plain, " is a ") && strings.Contains(plain, " product from ") {
		return true
	}
	// SL/CS short invent: "<title> je … znamk|kategor|produkt …"
	if strings.Contains(plain, " je ") &&
		(strings.Contains(plain, "znamk") ||
			strings.Contains(plain, "kategor") ||
			strings.Contains(plain, "produkt") ||
			strings.Contains(plain, "televiz") ||
			strings.Contains(plain, "monitor")) {
		return true
	}
	return false
}

// plainTextForWeakCheck strips HTML tags so <p>thin invent</p> is judged on visible copy.
func plainTextForWeakCheck(s string) string {
	s = strings.TrimSpace(s)
	if s == "" {
		return ""
	}
	var b strings.Builder
	b.Grow(len(s))
	inTag := false
	for _, r := range s {
		switch {
		case r == '<':
			inTag = true
		case r == '>':
			inTag = false
		case !inTag:
			b.WriteRune(r)
		}
	}
	return strings.TrimSpace(b.String())
}

// EnhanceHashSkipBlockReason returns a stable reason when prior description must
// not hash-skip the enhance LLM: "weak", "title-echo", "synth", or "".
func EnhanceHashSkipBlockReason(priorDesc string, titles ...string) string {
	priorDesc = strings.TrimSpace(priorDesc)
	if priorDesc == "" || priorDesc == "<nil>" {
		return "weak"
	}
	plain := plainTextForWeakCheck(priorDesc)
	if plain == "" || plain == "<nil>" {
		return "weak"
	}
	if len([]rune(plain)) < minUsableProductDescRunes {
		return "weak"
	}
	for _, title := range titles {
		if DescriptionEchoesTitle(plain, title) || DescriptionEchoesTitle(priorDesc, title) {
			return "title-echo"
		}
	}
	if ContainsWeakFillerPhrase(plain) || ContainsWeakFillerPhrase(priorDesc) {
		return "weak"
	}
	if LooksLikeHeuristicSynthesize(priorDesc) {
		return "synth"
	}
	if len([]rune(plain)) <= shortBoilerplateDescRunes &&
		!descriptionOverlapsProductFacts(plain, titles...) {
		return "weak"
	}
	return ""
}

// ShouldRefuseEnhanceHashSkip is true when a stored enhance_input_hash must be
// ignored / cleared (weak, title-echo, or heuristic synthesize).
func ShouldRefuseEnhanceHashSkip(priorDesc string, titles ...string) bool {
	return EnhanceHashSkipBlockReason(priorDesc, titles...) != ""
}

// DescriptionMissingFormulaHTMLTags is true when sectionTypes require HTML tags
// (h1/h2/h3/h4, p, ul) that are absent from desc — formula-mismatch for skip/clear.
// Counts matter: a formula with two h2 sections needs at least two <h2 opens.
func DescriptionMissingFormulaHTMLTags(desc string, sectionTypes []string) bool {
	if len(sectionTypes) == 0 {
		return false
	}
	need := map[string]int{}
	for _, raw := range sectionTypes {
		typ := strings.ToLower(strings.TrimSpace(raw))
		switch typ {
		case "h1", "h2", "h3", "h4", "ul":
			need[typ]++
		case "p", "":
			need["p"]++
		}
	}
	if len(need) == 0 {
		return false
	}
	lower := strings.ToLower(desc)
	for typ, n := range need {
		needle := "<" + typ
		if strings.Count(lower, needle) < n {
			return true
		}
	}
	return false
}

// ContainsWeakFillerPhrase reports known placeholder / invent-fallback snippets.
func ContainsWeakFillerPhrase(desc string) bool {
	lower := strings.ToLower(desc)
	for _, p := range weakFillerPhrases {
		if p != "" && strings.Contains(lower, p) {
			return true
		}
	}
	return false
}

var weakDescStopTokens = map[string]struct{}{
	"a": {}, "an": {}, "the": {}, "and": {}, "or": {}, "of": {}, "in": {}, "on": {}, "for": {},
	"to": {}, "with": {}, "from": {}, "by": {}, "is": {}, "are": {}, "this": {}, "that": {},
	"product": {}, "products": {}, "category": {}, "description": {}, "specs": {}, "spec": {},
	"key": {}, "je": {}, "v": {}, "za": {}, "z": {}, "iz": {}, "ter": {}, "ali": {},
}

func significantDescTokens(s string) map[string]struct{} {
	s = strings.ToLower(s)
	out := map[string]struct{}{}
	var b strings.Builder
	flush := func() {
		tok := b.String()
		b.Reset()
		if len(tok) < 3 {
			return
		}
		if _, stop := weakDescStopTokens[tok]; stop {
			return
		}
		out[tok] = struct{}{}
	}
	for _, r := range s {
		if (r >= 'a' && r <= 'z') || (r >= '0' && r <= '9') || r == '_' {
			b.WriteRune(r)
			continue
		}
		// Keep Latin-extended letters (Slovenian čšž etc.) via unicode letter-ish: non-space punctuation splits.
		if r > 127 && ((r >= 'à' && r <= 'ÿ') || r == 'č' || r == 'š' || r == 'ž' || r == 'ć' || r == 'đ') {
			b.WriteRune(r)
			continue
		}
		flush()
	}
	flush()
	return out
}

// descriptionOverlapsProductFacts is true when desc shares significant tokens with
// any fact source (title, brand, model, …).
func descriptionOverlapsProductFacts(desc string, factSources ...string) bool {
	descToks := significantDescTokens(desc)
	if len(descToks) == 0 {
		return false
	}
	overlap := 0
	seen := map[string]struct{}{}
	for _, src := range factSources {
		for tok := range significantDescTokens(src) {
			if _, ok := descToks[tok]; !ok {
				continue
			}
			if _, dup := seen[tok]; dup {
				continue
			}
			seen[tok] = struct{}{}
			overlap++
			if overlap >= 2 || len(tok) >= 5 {
				return true
			}
		}
	}
	return overlap >= 1 && len(seen) >= 1 && len([]rune(desc)) >= minUsableProductDescRunes+20
}

func normalizeForEchoCompare(s string) string {
	return strings.Join(strings.Fields(strings.ToLower(strings.TrimSpace(s))), " ")
}

// DescriptionEchoesTitle is true when desc is EqualFold or near-equal to title
// (whitespace-normalized, or one contains the other with only a tiny length delta).
func DescriptionEchoesTitle(desc, title string) bool {
	d := normalizeForEchoCompare(desc)
	t := normalizeForEchoCompare(title)
	if d == "" || t == "" {
		return false
	}
	if d == t {
		return true
	}
	shorter, longer := d, t
	if len(d) > len(t) {
		shorter, longer = t, d
	}
	if !strings.Contains(longer, shorter) {
		return false
	}
	delta := len(longer) - len(shorter)
	return delta <= 8 && float64(len(shorter))/float64(len(longer)) >= 0.85
}