descrybe/apps/api/internal/company/weak_desc.go

package company

import "strings"

// minUsableProductDescRunes is the floor for a prior description that may hash-skip
// the enhance LLM. Shorter copy is treated as thin and forced through enhance.
const minUsableProductDescRunes = 40

// shortBoilerplateDescRunes: short copy without product-fact token overlap is weak
// even when longer than minUsableProductDescRunes (generic one-liners).
const shortBoilerplateDescRunes = 96

// weakFillerPhrases are known heuristic / placeholder snippets that must never
// hash-skip enhance (mock-llm / invent fallbacks historically produced these).
var weakFillerPhrases = []string{
	"ready for retail listing",
	"quality product ready",
	"product description",
	"based on available specifications",
	"with available specifications",
	"available catalog details",
	"pripravljeno za prodajo",
	"na podlagi razpoložljivih specifikacij",
}

// IsWeakPriorEnhanceDescription reports empty, too-short, title-echo, known filler,
// or short boilerplate without overlapping tokens from title/fact sources.
func IsWeakPriorEnhanceDescription(priorDesc string, titles ...string) bool {
	priorDesc = strings.TrimSpace(priorDesc)
	if priorDesc == "" || priorDesc == "<nil>" {
		return true
	}
	if len([]rune(priorDesc)) < minUsableProductDescRunes {
		return true
	}
	for _, title := range titles {
		if DescriptionEchoesTitle(priorDesc, title) {
			return true
		}
	}
	if ContainsWeakFillerPhrase(priorDesc) {
		return true
	}
	if len([]rune(priorDesc)) <= shortBoilerplateDescRunes &&
		!descriptionOverlapsProductFacts(priorDesc, titles...) {
		return true
	}
	return false
}

// ContainsWeakFillerPhrase reports known placeholder / invent-fallback snippets.
func ContainsWeakFillerPhrase(desc string) bool {
	lower := strings.ToLower(desc)
	for _, p := range weakFillerPhrases {
		if p != "" && strings.Contains(lower, p) {
			return true
		}
	}
	return false
}

var weakDescStopTokens = map[string]struct{}{
	"a": {}, "an": {}, "the": {}, "and": {}, "or": {}, "of": {}, "in": {}, "on": {}, "for": {},
	"to": {}, "with": {}, "from": {}, "by": {}, "is": {}, "are": {}, "this": {}, "that": {},
	"product": {}, "products": {}, "category": {}, "description": {}, "specs": {}, "spec": {},
	"key": {}, "je": {}, "v": {}, "za": {}, "z": {}, "iz": {}, "ter": {}, "ali": {},
}

func significantDescTokens(s string) map[string]struct{} {
	s = strings.ToLower(s)
	out := map[string]struct{}{}
	var b strings.Builder
	flush := func() {
		tok := b.String()
		b.Reset()
		if len(tok) < 3 {
			return
		}
		if _, stop := weakDescStopTokens[tok]; stop {
			return
		}
		out[tok] = struct{}{}
	}
	for _, r := range s {
		if (r >= 'a' && r <= 'z') || (r >= '0' && r <= '9') || r == '_' {
			b.WriteRune(r)
			continue
		}
		// Keep Latin-extended letters (Slovenian čšž etc.) via unicode letter-ish: non-space punctuation splits.
		if r > 127 && ((r >= 'à' && r <= 'ÿ') || r == 'č' || r == 'š' || r == 'ž' || r == 'ć' || r == 'đ') {
			b.WriteRune(r)
			continue
		}
		flush()
	}
	flush()
	return out
}

// descriptionOverlapsProductFacts is true when desc shares significant tokens with
// any fact source (title, brand, model, …).
func descriptionOverlapsProductFacts(desc string, factSources ...string) bool {
	descToks := significantDescTokens(desc)
	if len(descToks) == 0 {
		return false
	}
	overlap := 0
	seen := map[string]struct{}{}
	for _, src := range factSources {
		for tok := range significantDescTokens(src) {
			if _, ok := descToks[tok]; !ok {
				continue
			}
			if _, dup := seen[tok]; dup {
				continue
			}
			seen[tok] = struct{}{}
			overlap++
			if overlap >= 2 || len(tok) >= 5 {
				return true
			}
		}
	}
	return overlap >= 1 && len(seen) >= 1 && len([]rune(desc)) >= minUsableProductDescRunes+20
}

func normalizeForEchoCompare(s string) string {
	return strings.Join(strings.Fields(strings.ToLower(strings.TrimSpace(s))), " ")
}

// DescriptionEchoesTitle is true when desc is EqualFold or near-equal to title
// (whitespace-normalized, or one contains the other with only a tiny length delta).
func DescriptionEchoesTitle(desc, title string) bool {
	d := normalizeForEchoCompare(desc)
	t := normalizeForEchoCompare(title)
	if d == "" || t == "" {
		return false
	}
	if d == t {
		return true
	}
	shorter, longer := d, t
	if len(d) > len(t) {
		shorter, longer = t, d
	}
	if !strings.Contains(longer, shorter) {
		return false
	}
	delta := len(longer) - len(shorter)
	return delta <= 8 && float64(len(shorter))/float64(len(longer)) >= 0.85
}