This commit is contained in:
2026-08-23 20:49:40 +02:00
parent 3b678ca857
commit f3d4fb56ed
31 changed files with 3608 additions and 111 deletions
+54 -7
View File
@@ -31,10 +31,12 @@ func IsWeakPriorEnhanceDescription(priorDesc string, titles ...string) bool {
return reason == "weak" || reason == "title-echo"
}
// heuristicSynthesizePhrases are distinctive invent / formula-skeleton snippets from
// heuristicSynthesizePhrases are distinctive invent snippets from
// synthesizeDescriptionFromTitle / synthesizeDescriptionFromFormula / factualDescriptionIntro.
// These may look "strong" enough to pass IsWeakPriorEnhanceDescription but must never
// hash-skip enhance (would leave fallback copy forever on reprocess).
// Only whole sentences that no copywriter would produce belong here — anything a
// real category formula could legitimately emit goes in synthSkeletonPhrases.
var heuristicSynthesizePhrases = []string{
"is a catalog product with the known attributes",
"is listed in the ",
@@ -43,19 +45,44 @@ var heuristicSynthesizePhrases = []string{
"je izdelek v kategoriji",
"je izdelek znamke",
". ključne specifikacije:",
" — katalogski izdelek",
" — catalog product",
}
// synthSkeletonPhrases also appear in synthesize output but are ordinary words a
// real description may use ("Ključne lastnosti" as a heading, "znamka X" in a spec
// line, "Key features" as a section title). They are suggestive, not proof, so only
// the hash-skip gate consults them — see IsInventFallbackDescription.
var synthSkeletonPhrases = []string{
// factualDescriptionIntro (post-phrase-avoidance invent)
" — znamka ",
", znamka ",
" — katalogski izdelek",
" — from ",
" — catalog product",
// synthesizeDescriptionFromFormula heading fallbacks
"ključne lastnosti",
"key features",
}
// LooksLikeHeuristicSynthesize reports invent / formula-skeleton fallback copy.
func LooksLikeHeuristicSynthesize(desc string) bool {
// maxSynthSkeletonRunes bounds the suggestive signals above. Every synthesize helper
// emits at most a title heading, one intro sentence, and a short "Label: value"
// list, so it stays well under this; real category-formula prose (multiple 100-word
// paragraphs plus a spec list) runs several times longer.
const maxSynthSkeletonRunes = 600
// maxSynthInventRunes bounds the generic "<title> is a … product from …" /
// "<title> je … znamke …" shapes. Those describe a single invent sentence, so they
// may only match copy that is itself about one sentence long. Unbounded, " je "
// plus "znamk" matched almost any Slovenian description.
const maxSynthInventRunes = 2 * shortBoilerplateDescRunes
// IsInventFallbackDescription reports copy that is unmistakably machine invent —
// whole sentences emitted by the synthesize helpers, or the one-sentence
// "<title> is a <category> product from <brand>" shape.
//
// This is the strict test, safe for discarding a description outright. The looser
// LooksLikeHeuristicSynthesize adds ambiguous wording and is only used where a
// false positive costs one extra enhance call rather than the copy itself.
func IsInventFallbackDescription(desc string) bool {
plain := strings.ToLower(plainTextForWeakCheck(desc))
if plain == "" {
return false
@@ -65,19 +92,39 @@ func LooksLikeHeuristicSynthesize(desc string) bool {
return true
}
}
if len([]rune(plain)) > maxSynthInventRunes {
return false
}
// EN invent: "<title> is a <category> product from <brand>"
if strings.Contains(plain, " is a ") && strings.Contains(plain, " product from ") {
return true
}
// SL/CS short invent: "<title> je … znamk|kategor|produkt …"
if strings.Contains(plain, " je ") &&
return strings.Contains(plain, " je ") &&
(strings.Contains(plain, "znamk") ||
strings.Contains(plain, "kategor") ||
strings.Contains(plain, "produkt") ||
strings.Contains(plain, "televiz") ||
strings.Contains(plain, "monitor")) {
strings.Contains(plain, "monitor"))
}
// LooksLikeHeuristicSynthesize reports invent / formula-skeleton fallback copy for
// the enhance hash-skip gate: IsInventFallbackDescription plus wording that merely
// suggests a skeleton, inside a body short enough to be one. A false positive here
// forces one re-enhance, which is cheap; never use it to drop a description.
func LooksLikeHeuristicSynthesize(desc string) bool {
if IsInventFallbackDescription(desc) {
return true
}
plain := strings.ToLower(plainTextForWeakCheck(desc))
if plain == "" || len([]rune(plain)) > maxSynthSkeletonRunes {
return false
}
for _, p := range synthSkeletonPhrases {
if p != "" && strings.Contains(plain, p) {
return true
}
}
return false
}