fix
This commit is contained in:
@@ -25,25 +25,98 @@ var weakFillerPhrases = []string{
|
||||
|
||||
// IsWeakPriorEnhanceDescription reports empty, too-short, title-echo, known filler,
|
||||
// or short boilerplate without overlapping tokens from title/fact sources.
|
||||
// Heuristic synthesize (invent) is intentionally excluded — use ShouldRefuseEnhanceHashSkip.
|
||||
func IsWeakPriorEnhanceDescription(priorDesc string, titles ...string) bool {
|
||||
priorDesc = strings.TrimSpace(priorDesc)
|
||||
if priorDesc == "" || priorDesc == "<nil>" {
|
||||
return true
|
||||
}
|
||||
if len([]rune(priorDesc)) < minUsableProductDescRunes {
|
||||
return true
|
||||
}
|
||||
for _, title := range titles {
|
||||
if DescriptionEchoesTitle(priorDesc, title) {
|
||||
reason := EnhanceHashSkipBlockReason(priorDesc, titles...)
|
||||
return reason == "weak" || reason == "title-echo"
|
||||
}
|
||||
|
||||
// heuristicSynthesizePhrases are distinctive invent / formula-skeleton snippets from
|
||||
// synthesizeDescriptionFromTitle / synthesizeDescriptionFromFormula. These may look
|
||||
// "strong" enough to pass IsWeakPriorEnhanceDescription but must never hash-skip
|
||||
// enhance (would leave fallback copy forever on reprocess).
|
||||
var heuristicSynthesizePhrases = []string{
|
||||
"is a catalog product with the known attributes",
|
||||
"is listed in the ",
|
||||
". key specs:",
|
||||
"je katalogski izdelek z znanimi atributi",
|
||||
"je izdelek v kategoriji",
|
||||
"je izdelek znamke",
|
||||
". ključne specifikacije:",
|
||||
}
|
||||
|
||||
// LooksLikeHeuristicSynthesize reports invent / formula-skeleton fallback copy.
|
||||
func LooksLikeHeuristicSynthesize(desc string) bool {
|
||||
lower := strings.ToLower(desc)
|
||||
for _, p := range heuristicSynthesizePhrases {
|
||||
if p != "" && strings.Contains(lower, p) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
if ContainsWeakFillerPhrase(priorDesc) {
|
||||
// EN invent: "<title> is a <category> product from <brand>"
|
||||
if strings.Contains(lower, " is a ") && strings.Contains(lower, " product from ") {
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// EnhanceHashSkipBlockReason returns a stable reason when prior description must
|
||||
// not hash-skip the enhance LLM: "weak", "title-echo", "synth", or "".
|
||||
func EnhanceHashSkipBlockReason(priorDesc string, titles ...string) string {
|
||||
priorDesc = strings.TrimSpace(priorDesc)
|
||||
if priorDesc == "" || priorDesc == "<nil>" {
|
||||
return "weak"
|
||||
}
|
||||
if len([]rune(priorDesc)) < minUsableProductDescRunes {
|
||||
return "weak"
|
||||
}
|
||||
for _, title := range titles {
|
||||
if DescriptionEchoesTitle(priorDesc, title) {
|
||||
return "title-echo"
|
||||
}
|
||||
}
|
||||
if ContainsWeakFillerPhrase(priorDesc) {
|
||||
return "weak"
|
||||
}
|
||||
if LooksLikeHeuristicSynthesize(priorDesc) {
|
||||
return "synth"
|
||||
}
|
||||
if len([]rune(priorDesc)) <= shortBoilerplateDescRunes &&
|
||||
!descriptionOverlapsProductFacts(priorDesc, titles...) {
|
||||
return true
|
||||
return "weak"
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// ShouldRefuseEnhanceHashSkip is true when a stored enhance_input_hash must be
|
||||
// ignored / cleared (weak, title-echo, or heuristic synthesize).
|
||||
func ShouldRefuseEnhanceHashSkip(priorDesc string, titles ...string) bool {
|
||||
return EnhanceHashSkipBlockReason(priorDesc, titles...) != ""
|
||||
}
|
||||
|
||||
// DescriptionMissingFormulaHTMLTags is true when sectionTypes require HTML tags
|
||||
// (h1/h2/h3/h4, p, ul) that are absent from desc — formula-mismatch for skip/clear.
|
||||
func DescriptionMissingFormulaHTMLTags(desc string, sectionTypes []string) bool {
|
||||
if len(sectionTypes) == 0 {
|
||||
return false
|
||||
}
|
||||
lower := strings.ToLower(desc)
|
||||
for _, raw := range sectionTypes {
|
||||
typ := strings.ToLower(strings.TrimSpace(raw))
|
||||
switch typ {
|
||||
case "h1", "h2", "h3", "h4":
|
||||
if !strings.Contains(lower, "<"+typ) {
|
||||
return true
|
||||
}
|
||||
case "ul":
|
||||
if !strings.Contains(lower, "<ul") {
|
||||
return true
|
||||
}
|
||||
case "p", "":
|
||||
if !strings.Contains(lower, "<p") {
|
||||
return true
|
||||
}
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
@@ -0,0 +1,40 @@
|
||||
package company
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestLooksLikeHeuristicSynthesize(t *testing.T) {
|
||||
t.Parallel()
|
||||
title := "Vogel's WALL 3245 TV Wall Mount"
|
||||
en := title + " is a TV Mounts product from Vogel's. Key specs: width 45 cm, max_load 40 kg."
|
||||
if !LooksLikeHeuristicSynthesize(en) {
|
||||
t.Fatalf("expected EN invent detected: %q", en)
|
||||
}
|
||||
sl := title + " je izdelek v kategoriji TV Mounts znamke Vogel's. Ključne specifikacije: width 45 cm."
|
||||
if !LooksLikeHeuristicSynthesize(sl) {
|
||||
t.Fatalf("expected SL invent detected: %q", sl)
|
||||
}
|
||||
good := "Vogel's WALL 3245 is a sturdy TV mount with clear load ratings and install guidance for wall displays."
|
||||
if LooksLikeHeuristicSynthesize(good) {
|
||||
t.Fatalf("retail copy must not look like invent: %q", good)
|
||||
}
|
||||
if !ShouldRefuseEnhanceHashSkip(en, title) {
|
||||
t.Fatal("invent must refuse hash skip")
|
||||
}
|
||||
if ShouldRefuseEnhanceHashSkip(good, title) {
|
||||
t.Fatal("good retail copy must allow hash skip")
|
||||
}
|
||||
if !ShouldRefuseEnhanceHashSkip(title, title) {
|
||||
t.Fatal("title-echo must refuse hash skip")
|
||||
}
|
||||
}
|
||||
|
||||
func TestDescriptionMissingFormulaHTMLTags(t *testing.T) {
|
||||
t.Parallel()
|
||||
types := []string{"h1", "p", "ul"}
|
||||
if !DescriptionMissingFormulaHTMLTags("plain prose without tags", types) {
|
||||
t.Fatal("plain prose must mismatch")
|
||||
}
|
||||
if DescriptionMissingFormulaHTMLTags("<h1>T</h1><p>Body</p><ul><li>x</li></ul>", types) {
|
||||
t.Fatal("full HTML must match")
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user