greeneclipse/descrybe
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fix

Browse Source
This commit is contained in:
greeneclipse
2026-08-17 01:30:28 +02:00
parent c2429873b3
commit 0dceb3a404
18 changed files with 1021 additions and 219 deletions
Download Patch File Download Diff File Expand all files Collapse all files
apps/api/internal/processing/v1_process_item.go
+166 -86
View File
@@ -4,6 +4,7 @@ import (
"fmt"
"strings"
"github.com/descrybe/descrybe-v2/apps/api/internal/company"
"github.com/descrybe/descrybe-v2/apps/api/internal/eprel"
)
@@ -32,11 +33,13 @@ type V1ProcessItemScorecard struct {
// ScoreV1ProcessItemOptions tunes structure checks for a poll item.
type ScoreV1ProcessItemOptions struct {
// MappedCategory, when non-empty, requires item.category to equal it
// (projection must surface mapped unique_id).
// MappedCategory, when non-empty, requires item.category_id (or legacy
// item.category unique_id) to equal it.
MappedCategory string
// Language is used only for documentation of synthesize paths in tests.
Language string
// OmitSEOMeta skips meta_title / meta_description presence checks (A1 cohort).
OmitSEOMeta bool
}
// ScoreV1ProcessCompletedItem scores a successful (or terminal) V1 process item.
@@ -89,28 +92,35 @@ func ScoreV1ProcessCompletedItem(item V1ProcessJobItem, opts ScoreV1ProcessItemO
sc.HasMetaTitle = stringFromItem(item, "meta_title") != ""
sc.HasMetaDescription = stringFromItem(item, "meta_description") != ""
if sc.HasTitle && !sc.HasMetaTitle {
sc.FailFlags = append(sc.FailFlags, "meta_title_missing")
}
if sc.HasTitle && !sc.HasMetaDescription {
sc.FailFlags = append(sc.FailFlags, "meta_description_missing")
}
if md := stringFromItem(item, "meta_description"); md != "" && containsWeakFillerPhrase(md) {
sc.FailFlags = append(sc.FailFlags, "meta_description_weak_filler")
if !opts.OmitSEOMeta {
if sc.HasTitle && !sc.HasMetaTitle {
sc.FailFlags = append(sc.FailFlags, "meta_title_missing")
}
if sc.HasTitle && !sc.HasMetaDescription {
sc.FailFlags = append(sc.FailFlags, "meta_description_missing")
}
if md := stringFromItem(item, "meta_description"); md != "" && containsWeakFillerPhrase(md) {
sc.FailFlags = append(sc.FailFlags, "meta_description_weak_filler")
}
}
cat := stringFromItem(item, "category")
catID := stringFromItem(item, "category_id")
catName := stringFromItem(item, "category_name")
sc.HasCategory = cat != ""
sc.HasCategoryName = catName != ""
sc.HasCategory = cat != "" || catID != ""
sc.HasCategoryName = catName != "" || (cat != "" && catID != "" && cat != catID)
mappedCat := strings.TrimSpace(opts.MappedCategory)
if mappedCat != "" {
if cat == "" {
gotUID := catID
if gotUID == "" {
gotUID = cat // legacy: category held unique_id
}
if gotUID == "" {
sc.FailFlags = append(sc.FailFlags, "category_missing_though_mapped")
} else if cat != mappedCat {
} else if gotUID != mappedCat && cat != mappedCat {
sc.FailFlags = append(sc.FailFlags, "category_mismatch_mapped")
}
if cat != "" && catName == "" {
if catName == "" && (cat == "" || cat == gotUID) {
sc.FailFlags = append(sc.FailFlags, "category_name_missing")
}
}
@@ -182,14 +192,28 @@ func ScoreV1ProcessCompletedItem(item V1ProcessJobItem, opts ScoreV1ProcessItemO
return sc
}
// EnforceV1ProcessCompletedItem fills LegacyProcessItem projection gaps for a
// successful item: nonempty description when title exists (formula HTML
// preserved), meta_*, clean attributes, eprel object|null, and image key shapes.
// Category must already be set by the caller when mapped provides a unique_id.
//
// EnforceV1ProcessCompletedItem fills LegacyProcessItem projection gaps for a successful item:
// nonempty description when title exists (formula HTML preserved), readable title spacing,
// category display name as primary, optional SEO meta (unless omitSEOMeta), clean attrs.
// allowed is the company attribute_key set (canonicalized). When nil, only
// coreCharacteristicAttrKeys are kept (never leak feed junk like zavora).
func EnforceV1ProcessCompletedItem(item V1ProcessJobItem, language string, allowed map[string]struct{}) V1ProcessJobItem {
return EnforceV1ProcessCompletedItemOpts(item, EnforceV1Opts{
Language: language,
Allowed: allowed,
})
}
// EnforceV1Opts configures V1 completed-item projection.
type EnforceV1Opts struct {
Language string
Allowed map[string]struct{}
OmitSEOMeta bool
DescriptionTemplate any
}
// EnforceV1ProcessCompletedItemOpts is the options-aware EnforceV1 entrypoint.
func EnforceV1ProcessCompletedItemOpts(item V1ProcessJobItem, opts EnforceV1Opts) V1ProcessJobItem {
if item == nil {
return item
}
@@ -198,10 +222,11 @@ func EnforceV1ProcessCompletedItem(item V1ProcessJobItem, language string, allow
return item
}
attrs, _ := attrsMapFromItem(item)
allowed := opts.Allowed
if allowed == nil {
allowed = map[string]struct{}{}
}
attrs, _ := attrsMapFromItem(item)
attrs = SanitizeV1ProcessAttributesAllowed(attrs, allowed)
if len(attrs) > 0 {
item["attributes"] = attrs
@@ -209,7 +234,7 @@ func EnforceV1ProcessCompletedItem(item V1ProcessJobItem, language string, allow
item["attributes"] = nil
}
title := stringFromItem(item, "title")
title := ensureReadableTitleSpacing(stringFromItem(item, "title"))
if title != "" {
item["title"] = title
item["name"] = title
@@ -217,18 +242,26 @@ func EnforceV1ProcessCompletedItem(item V1ProcessJobItem, language string, allow
item["title"] = nil
item["name"] = nil
}
cat := stringFromItem(item, "category")
catName := stringFromItem(item, "category_name")
catID, catName := projectV1CategoryFields(item)
catLabel := catName
if catLabel == "" {
catLabel = cat
catLabel = catID
}
desc, _ := descriptionFromItem(item)
// Empty, weak, or title-echo copy must be replaced — never leave description==title.
// Formula HTML that satisfies multi-section templates is kept as-is.
if title != "" && (desc == "" || isWeakPriorEnhanceDescription(desc, title) || descriptionEchoesTitle(desc, title)) {
if synth := synthesizeDescriptionFromTitle(title, catLabel, language, attrs); synth != "" {
needsDesc := title != "" && (desc == "" ||
isWeakPriorEnhanceDescription(desc, title) ||
descriptionEchoesTitle(desc, title) ||
company.LooksLikeHeuristicSynthesize(desc))
if needsDesc {
tpl := opts.DescriptionTemplate
if tpl == nil {
tpl = inferDescriptionTemplateFromHTML(desc)
}
if synth := synthesizeProductDescription(title, catLabel, opts.Language, attrs, tpl); synth != "" {
desc = synth
} else if synth := synthesizeDescriptionFromTitle(title, catLabel, opts.Language, attrs); synth != "" {
desc = synth
}
}
@@ -238,66 +271,58 @@ func EnforceV1ProcessCompletedItem(item V1ProcessJobItem, language string, allow
item["description"] = nil
}
metaTitle := stringFromItem(item, "meta_title")
metaDesc := stringFromItem(item, "meta_description")
needMetaTitle := metaTitle == "" || isPoisonedMetaTitle(metaTitle)
// Empty, weak stub (Ready for retail…), title-echo, or prompt-leakage — refresh.
// When meta_title is poisoned, also force refresh of weak/leakage meta_description.
needMetaDesc := metaDesc == "" ||
isWeakPriorEnhanceDescription(metaDesc, title) ||
containsWeakFillerPhrase(metaDesc) ||
isPromptLeakageTitle(metaDesc) ||
(title != "" && descriptionEchoesTitle(metaDesc, title))
if needMetaTitle && isPoisonedMetaTitle(metaTitle) &&
(metaDesc == "" || isWeakPriorEnhanceDescription(metaDesc, title) || isPromptLeakageTitle(metaDesc)) {
needMetaDesc = true
}
if title != "" || desc != "" || cat != "" || catName != "" {
synthTitle, synthDesc := fillMetaFromResult(StepResult{
Name: title,
ProcessedName: title,
Category: cat,
CategoryName: catName,
Description: desc,
ProcessedDescription: desc,
Attributes: attrs,
ProcessedAttributes: attrs,
})
if needMetaTitle {
if synthTitle != "" {
metaTitle = synthTitle
} else {
metaTitle = title
if opts.OmitSEOMeta {
delete(item, "meta_title")
delete(item, "meta_description")
} else {
metaTitle := stringFromItem(item, "meta_title")
metaDesc := stringFromItem(item, "meta_description")
needMetaTitle := metaTitle == "" || isPoisonedMetaTitle(metaTitle)
needMetaDesc := metaDesc == "" ||
isWeakPriorEnhanceDescription(metaDesc, title) ||
containsWeakFillerPhrase(metaDesc) ||
isPromptLeakageTitle(metaDesc) ||
(title != "" && descriptionEchoesTitle(metaDesc, title))
if needMetaTitle && isPoisonedMetaTitle(metaTitle) &&
(metaDesc == "" || isWeakPriorEnhanceDescription(metaDesc, title) || isPromptLeakageTitle(metaDesc)) {
needMetaDesc = true
}
if title != "" || desc != "" || catID != "" || catName != "" {
synthTitle, synthDesc := fillMetaFromResult(StepResult{
Name: title,
ProcessedName: title,
Category: catID,
CategoryName: catName,
Description: desc,
ProcessedDescription: desc,
Attributes: attrs,
ProcessedAttributes: attrs,
})
if needMetaTitle {
if synthTitle != "" {
metaTitle = synthTitle
} else {
metaTitle = title
}
}
if needMetaDesc {
if synthDesc != "" {
metaDesc = synthDesc
} else if desc != "" {
metaDesc = truncateMetaDescription(desc, v1MetaDescriptionMaxChars)
}
}
}
if needMetaDesc {
if synthDesc != "" {
metaDesc = synthDesc
} else if desc != "" {
metaDesc = truncateMetaDescription(desc, v1MetaDescriptionMaxChars)
}
if metaTitle != "" {
item["meta_title"] = metaTitle
} else {
item["meta_title"] = nil
}
if metaDesc != "" {
item["meta_description"] = metaDesc
} else {
item["meta_description"] = nil
}
}
if metaTitle != "" {
item["meta_title"] = metaTitle
} else {
item["meta_title"] = nil
}
if metaDesc != "" {
item["meta_description"] = metaDesc
} else {
item["meta_description"] = nil
}
if cat != "" {
item["category"] = cat
} else {
item["category"] = nil
}
if catName != "" {
item["category_name"] = catName
} else {
item["category_name"] = nil
}
item["eprel"] = normalizeEPRELValue(item["eprel"])
@@ -318,6 +343,61 @@ func EnforceV1ProcessCompletedItem(item V1ProcessJobItem, language string, allow
return item
}
// projectV1CategoryFields sets category=display name, category_id=unique_id,
// category_name=display name. Returns (unique_id, display_name).
func projectV1CategoryFields(item V1ProcessJobItem) (catID, catName string) {
catID = stringFromItem(item, "category_id")
catName = stringFromItem(item, "category_name")
cat := stringFromItem(item, "category")
if catID == "" {
// Legacy: category held unique_id when category_name differed.
if catName != "" && cat != "" && cat != catName {
catID = cat
} else if catName == "" && cat != "" {
catID = cat
}
}
if catName == "" && cat != "" && cat != catID {
catName = cat
}
if catName != "" {
item["category"] = catName
item["category_name"] = catName
} else {
item["category"] = nil
item["category_name"] = nil
}
if catID != "" {
item["category_id"] = catID
} else {
delete(item, "category_id")
}
return catID, catName
}
// inferDescriptionTemplateFromHTML rebuilds a minimal description_template from
// tags already present so invent-garbage HTML can be re-synthesized without a DB lookup.
func inferDescriptionTemplateFromHTML(html string) any {
lower := strings.ToLower(html)
var sections []map[string]any
for _, typ := range []string{"h1", "h2", "h3", "h4", "p", "ul"} {
if strings.Contains(lower, "<"+typ) {
sections = append(sections, map[string]any{"type": typ})
}
}
if len(sections) == 0 {
return map[string]any{
"sections": []map[string]any{
{"type": "h1"},
{"type": "p"},
{"type": "h2"},
{"type": "ul"},
},
}
}
return map[string]any{"sections": sections}
}
func stringFromItem(item V1ProcessJobItem, key string) string {
if item == nil {
return ""