567 lines
15 KiB
Go
567 lines
15 KiB
Go
package processing
|
||
|
||
import (
|
||
"fmt"
|
||
"strings"
|
||
|
||
"github.com/descrybe/descrybe-v2/apps/api/internal/company"
|
||
"github.com/descrybe/descrybe-v2/apps/api/internal/eprel"
|
||
)
|
||
|
||
// V1ProcessItemScorecard scores one COMPLETED legacy process item against
|
||
// OpenAPI LegacyProcessItem + A1 poll expectations.
|
||
type V1ProcessItemScorecard struct {
|
||
EAN string `json:"ean"`
|
||
Status string `json:"status"`
|
||
OK bool `json:"ok"`
|
||
Score int `json:"score"` // 0–100
|
||
FailFlags []string `json:"fail_flags,omitempty"`
|
||
|
||
HasTitle bool `json:"has_title"`
|
||
HasName bool `json:"has_name"`
|
||
HasDescription bool `json:"has_description"`
|
||
DescriptionIsPlain bool `json:"description_is_plain"`
|
||
HasMetaTitle bool `json:"has_meta_title"`
|
||
HasMetaDescription bool `json:"has_meta_description"`
|
||
HasCategory bool `json:"has_category"`
|
||
HasCategoryName bool `json:"has_category_name"`
|
||
AttrsClean bool `json:"attrs_clean"`
|
||
EPRELOK bool `json:"eprel_ok"`
|
||
HasIDs bool `json:"has_ids"`
|
||
ImagesOK bool `json:"images_ok"`
|
||
}
|
||
|
||
// ScoreV1ProcessItemOptions tunes structure checks for a poll item.
|
||
type ScoreV1ProcessItemOptions struct {
|
||
// MappedCategory, when non-empty, requires item.category_id (or legacy
|
||
// item.category unique_id) to equal it.
|
||
MappedCategory string
|
||
// Language is used only for documentation of synthesize paths in tests.
|
||
Language string
|
||
// OmitSEOMeta skips meta_title / meta_description presence checks (A1 cohort).
|
||
OmitSEOMeta bool
|
||
}
|
||
|
||
// ScoreV1ProcessCompletedItem scores a successful (or terminal) V1 process item.
|
||
// For status != processed it only checks ean/status shape and returns OK when those exist.
|
||
func ScoreV1ProcessCompletedItem(item V1ProcessJobItem, opts ScoreV1ProcessItemOptions) V1ProcessItemScorecard {
|
||
sc := V1ProcessItemScorecard{
|
||
EAN: strings.TrimSpace(fmt.Sprint(item["ean"])),
|
||
Status: strings.TrimSpace(fmt.Sprint(item["status"])),
|
||
}
|
||
if sc.EAN == "" || sc.EAN == "<nil>" {
|
||
sc.FailFlags = append(sc.FailFlags, "missing_ean")
|
||
}
|
||
if sc.Status == "" || sc.Status == "<nil>" {
|
||
sc.FailFlags = append(sc.FailFlags, "missing_status")
|
||
}
|
||
if sc.Status != "processed" {
|
||
sc.OK = len(sc.FailFlags) == 0
|
||
if sc.OK {
|
||
sc.Score = 100
|
||
}
|
||
return sc
|
||
}
|
||
|
||
title := stringFromItem(item, "name")
|
||
if title == "" {
|
||
title = stringFromItem(item, "title")
|
||
}
|
||
name := stringFromItem(item, "name")
|
||
if name == "" {
|
||
name = title
|
||
}
|
||
sc.HasTitle = title != ""
|
||
sc.HasName = name != ""
|
||
if !sc.HasName && sc.HasTitle {
|
||
sc.FailFlags = append(sc.FailFlags, "missing_name")
|
||
}
|
||
if sc.HasTitle && sc.HasName && title != name {
|
||
sc.FailFlags = append(sc.FailFlags, "title_name_mismatch")
|
||
}
|
||
|
||
desc, descOK := plainDescriptionFromItem(item)
|
||
sc.HasDescription = desc != ""
|
||
sc.DescriptionIsPlain = descOK
|
||
if sc.HasTitle && !sc.HasDescription {
|
||
sc.FailFlags = append(sc.FailFlags, "description_empty_with_title")
|
||
}
|
||
if !descOK && item["description"] != nil {
|
||
sc.FailFlags = append(sc.FailFlags, "description_not_plain_string")
|
||
}
|
||
if sc.HasDescription && descriptionEchoesTitle(desc, title) {
|
||
sc.FailFlags = append(sc.FailFlags, "description_echoes_title")
|
||
}
|
||
if sc.HasDescription && containsWeakFillerPhrase(desc) {
|
||
sc.FailFlags = append(sc.FailFlags, "description_weak_filler")
|
||
}
|
||
|
||
sc.HasMetaTitle = stringFromItem(item, "meta_title") != ""
|
||
sc.HasMetaDescription = stringFromItem(item, "meta_description") != ""
|
||
if !opts.OmitSEOMeta {
|
||
if sc.HasTitle && !sc.HasMetaTitle {
|
||
sc.FailFlags = append(sc.FailFlags, "meta_title_missing")
|
||
}
|
||
if sc.HasTitle && !sc.HasMetaDescription {
|
||
sc.FailFlags = append(sc.FailFlags, "meta_description_missing")
|
||
}
|
||
if md := stringFromItem(item, "meta_description"); md != "" && containsWeakFillerPhrase(md) {
|
||
sc.FailFlags = append(sc.FailFlags, "meta_description_weak_filler")
|
||
}
|
||
}
|
||
|
||
cat := stringFromItem(item, "category")
|
||
catID := stringFromItem(item, "category_id")
|
||
catName := stringFromItem(item, "category_name")
|
||
sc.HasCategory = cat != "" || catID != ""
|
||
sc.HasCategoryName = catName != "" || (cat != "" && catID != "" && cat != catID)
|
||
mappedCat := strings.TrimSpace(opts.MappedCategory)
|
||
if mappedCat != "" {
|
||
gotUID := catID
|
||
if gotUID == "" {
|
||
gotUID = cat // legacy: category held unique_id
|
||
}
|
||
if gotUID == "" {
|
||
sc.FailFlags = append(sc.FailFlags, "category_missing_though_mapped")
|
||
} else if gotUID != mappedCat && cat != mappedCat {
|
||
sc.FailFlags = append(sc.FailFlags, "category_mismatch_mapped")
|
||
}
|
||
if catName == "" && (cat == "" || cat == gotUID) {
|
||
sc.FailFlags = append(sc.FailFlags, "category_name_missing")
|
||
}
|
||
}
|
||
|
||
attrs, attrsOK := attrsMapFromItem(item)
|
||
dirty := dirtyV1AttrKeys(attrs)
|
||
sc.AttrsClean = attrsOK && len(dirty) == 0
|
||
if !attrsOK {
|
||
sc.FailFlags = append(sc.FailFlags, "attributes_invalid_shape")
|
||
}
|
||
if len(dirty) > 0 {
|
||
sc.FailFlags = append(sc.FailFlags, "attributes_dirty:"+strings.Join(dirty, ","))
|
||
}
|
||
|
||
sc.EPRELOK = eprelShapeOK(item["eprel"])
|
||
if !sc.EPRELOK {
|
||
sc.FailFlags = append(sc.FailFlags, "eprel_invalid_shape")
|
||
}
|
||
|
||
// Internal UUIDs are omitted from the public V1 process item shape.
|
||
sc.HasIDs = true
|
||
|
||
sc.ImagesOK = imageFieldsOK(item)
|
||
if !sc.ImagesOK {
|
||
sc.FailFlags = append(sc.FailFlags, "images_invalid_shape")
|
||
}
|
||
|
||
// Weighted score: structure gates, not content quality beyond filler/echo.
|
||
checks := []bool{
|
||
sc.EAN != "",
|
||
sc.Status == "processed",
|
||
sc.HasTitle,
|
||
sc.HasName || !sc.HasTitle,
|
||
sc.HasDescription || !sc.HasTitle,
|
||
sc.DescriptionIsPlain || item["description"] == nil,
|
||
sc.HasMetaTitle || !sc.HasTitle,
|
||
sc.HasMetaDescription || !sc.HasTitle,
|
||
mappedCat == "" || sc.HasCategory,
|
||
mappedCat == "" || sc.HasCategoryName,
|
||
sc.AttrsClean,
|
||
sc.EPRELOK,
|
||
sc.HasIDs,
|
||
sc.ImagesOK,
|
||
!containsWeakFillerPhrase(desc),
|
||
!descriptionEchoesTitle(desc, title),
|
||
}
|
||
pass := 0
|
||
for _, ok := range checks {
|
||
if ok {
|
||
pass++
|
||
}
|
||
}
|
||
sc.Score = (pass * 100) / len(checks)
|
||
sc.OK = len(sc.FailFlags) == 0
|
||
return sc
|
||
}
|
||
|
||
// EnforceV1ProcessCompletedItem fills LegacyProcessItem projection gaps for a successful item:
|
||
// nonempty description when title exists (formula HTML preserved), readable title spacing,
|
||
// category display name as primary, optional SEO meta (unless omitSEOMeta), clean attrs.
|
||
// allowed is the company attribute_key set (canonicalized). When nil, only
|
||
// coreCharacteristicAttrKeys are kept (never leak feed junk like zavora).
|
||
func EnforceV1ProcessCompletedItem(item V1ProcessJobItem, language string, allowed map[string]struct{}) V1ProcessJobItem {
|
||
return EnforceV1ProcessCompletedItemOpts(item, EnforceV1Opts{
|
||
Language: language,
|
||
Allowed: allowed,
|
||
})
|
||
}
|
||
|
||
// EnforceV1Opts configures V1 completed-item projection.
|
||
type EnforceV1Opts struct {
|
||
Language string
|
||
Allowed map[string]struct{}
|
||
OmitSEOMeta bool
|
||
DescriptionTemplate any
|
||
}
|
||
|
||
// EnforceV1ProcessCompletedItemOpts is the options-aware EnforceV1 entrypoint.
|
||
func EnforceV1ProcessCompletedItemOpts(item V1ProcessJobItem, opts EnforceV1Opts) V1ProcessJobItem {
|
||
if item == nil {
|
||
return item
|
||
}
|
||
status, _ := item["status"].(string)
|
||
if status != "processed" {
|
||
return item
|
||
}
|
||
|
||
allowed := opts.Allowed
|
||
if allowed == nil {
|
||
allowed = map[string]struct{}{}
|
||
}
|
||
attrs, _ := attrsMapFromItem(item)
|
||
attrs = SanitizeV1ProcessAttributesAllowed(attrs, allowed)
|
||
if len(attrs) > 0 {
|
||
item["attributes"] = attrs
|
||
} else {
|
||
item["attributes"] = nil
|
||
}
|
||
|
||
title := ensureReadableTitleSpacing(stringFromItem(item, "name"))
|
||
if title == "" {
|
||
title = ensureReadableTitleSpacing(stringFromItem(item, "title"))
|
||
}
|
||
if title != "" {
|
||
item["name"] = title
|
||
delete(item, "title")
|
||
} else {
|
||
item["name"] = nil
|
||
delete(item, "title")
|
||
}
|
||
delete(item, "id")
|
||
delete(item, "processed_product_id")
|
||
delete(item, "raw_product_id")
|
||
|
||
catID, catName := projectV1CategoryFields(item)
|
||
catLabel := catName
|
||
if catLabel == "" {
|
||
catLabel = catID
|
||
}
|
||
|
||
desc, _ := descriptionFromItem(item)
|
||
needsDesc := title != "" && (desc == "" ||
|
||
isWeakPriorEnhanceDescription(desc, title) ||
|
||
descriptionEchoesTitle(desc, title) ||
|
||
company.LooksLikeHeuristicSynthesize(desc))
|
||
if needsDesc {
|
||
tpl := opts.DescriptionTemplate
|
||
if tpl == nil {
|
||
tpl = inferDescriptionTemplateFromHTML(desc)
|
||
}
|
||
if synth := synthesizeProductDescription(title, catLabel, opts.Language, attrs, tpl); synth != "" {
|
||
desc = synth
|
||
} else if synth := synthesizeDescriptionFromTitle(title, catLabel, opts.Language, attrs); synth != "" {
|
||
desc = synth
|
||
}
|
||
}
|
||
if desc != "" {
|
||
item["description"] = desc
|
||
} else {
|
||
item["description"] = nil
|
||
}
|
||
|
||
if opts.OmitSEOMeta {
|
||
delete(item, "meta_title")
|
||
delete(item, "meta_description")
|
||
} else {
|
||
metaTitle := stringFromItem(item, "meta_title")
|
||
metaDesc := stringFromItem(item, "meta_description")
|
||
needMetaTitle := metaTitle == "" || isPoisonedMetaTitle(metaTitle)
|
||
needMetaDesc := metaDesc == "" ||
|
||
isWeakPriorEnhanceDescription(metaDesc, title) ||
|
||
containsWeakFillerPhrase(metaDesc) ||
|
||
isPromptLeakageTitle(metaDesc) ||
|
||
(title != "" && descriptionEchoesTitle(metaDesc, title))
|
||
if needMetaTitle && isPoisonedMetaTitle(metaTitle) &&
|
||
(metaDesc == "" || isWeakPriorEnhanceDescription(metaDesc, title) || isPromptLeakageTitle(metaDesc)) {
|
||
needMetaDesc = true
|
||
}
|
||
if title != "" || desc != "" || catID != "" || catName != "" {
|
||
synthTitle, synthDesc := fillMetaFromResult(StepResult{
|
||
Name: title,
|
||
ProcessedName: title,
|
||
Category: catID,
|
||
CategoryName: catName,
|
||
Description: desc,
|
||
ProcessedDescription: desc,
|
||
Attributes: attrs,
|
||
ProcessedAttributes: attrs,
|
||
})
|
||
if needMetaTitle {
|
||
if synthTitle != "" {
|
||
metaTitle = synthTitle
|
||
} else {
|
||
metaTitle = title
|
||
}
|
||
}
|
||
if needMetaDesc {
|
||
if synthDesc != "" {
|
||
metaDesc = synthDesc
|
||
} else if desc != "" {
|
||
metaDesc = truncateMetaDescription(desc, v1MetaDescriptionMaxChars)
|
||
}
|
||
}
|
||
}
|
||
if metaTitle != "" {
|
||
item["meta_title"] = metaTitle
|
||
} else {
|
||
item["meta_title"] = nil
|
||
}
|
||
if metaDesc != "" {
|
||
item["meta_description"] = metaDesc
|
||
} else {
|
||
item["meta_description"] = nil
|
||
}
|
||
}
|
||
|
||
item["eprel"] = normalizeEPRELValue(item["eprel"])
|
||
|
||
main := stringFromItem(item, "main_image")
|
||
if main != "" {
|
||
item["main_image"] = main
|
||
} else {
|
||
item["main_image"] = nil
|
||
}
|
||
more := stringSliceFromItem(item, "more_images")
|
||
if len(more) > 0 {
|
||
item["more_images"] = more
|
||
} else {
|
||
item["more_images"] = nil
|
||
}
|
||
|
||
return item
|
||
}
|
||
|
||
// projectV1CategoryFields sets category=display name, category_id=unique_id,
|
||
// category_name=display name. Returns (unique_id, display_name).
|
||
func projectV1CategoryFields(item V1ProcessJobItem) (catID, catName string) {
|
||
catID = stringFromItem(item, "category_id")
|
||
catName = stringFromItem(item, "category_name")
|
||
cat := stringFromItem(item, "category")
|
||
if catID == "" {
|
||
// Legacy: category held unique_id when category_name differed.
|
||
if catName != "" && cat != "" && cat != catName {
|
||
catID = cat
|
||
} else if catName == "" && cat != "" {
|
||
catID = cat
|
||
}
|
||
}
|
||
if catName == "" && cat != "" && cat != catID {
|
||
catName = cat
|
||
}
|
||
if catName != "" {
|
||
item["category"] = catName
|
||
item["category_name"] = catName
|
||
} else {
|
||
item["category"] = nil
|
||
item["category_name"] = nil
|
||
}
|
||
if catID != "" {
|
||
item["category_id"] = catID
|
||
} else {
|
||
delete(item, "category_id")
|
||
}
|
||
return catID, catName
|
||
}
|
||
|
||
// inferDescriptionTemplateFromHTML rebuilds a minimal description_template from
|
||
// tags already present so invent-garbage HTML can be re-synthesized without a DB lookup.
|
||
func inferDescriptionTemplateFromHTML(html string) any {
|
||
lower := strings.ToLower(html)
|
||
var sections []map[string]any
|
||
for _, typ := range []string{"h1", "h2", "h3", "h4", "p", "ul"} {
|
||
if strings.Contains(lower, "<"+typ) {
|
||
sections = append(sections, map[string]any{"type": typ})
|
||
}
|
||
}
|
||
if len(sections) == 0 {
|
||
return map[string]any{
|
||
"sections": []map[string]any{
|
||
{"type": "h1"},
|
||
{"type": "p"},
|
||
{"type": "h2"},
|
||
{"type": "ul"},
|
||
},
|
||
}
|
||
}
|
||
return map[string]any{"sections": sections}
|
||
}
|
||
|
||
func stringFromItem(item V1ProcessJobItem, key string) string {
|
||
if item == nil {
|
||
return ""
|
||
}
|
||
return strings.TrimSpace(stringFromAny(item[key]))
|
||
}
|
||
|
||
func descriptionFromItem(item V1ProcessJobItem) (string, bool) {
|
||
if item == nil {
|
||
return "", true
|
||
}
|
||
raw := item["description"]
|
||
if raw == nil {
|
||
return "", true
|
||
}
|
||
switch raw.(type) {
|
||
case string:
|
||
return DescriptionFromAny(raw), true
|
||
case []any, []string:
|
||
// Legacy mistake: description as array — coerce to string, keep HTML.
|
||
if s := DescriptionFromAny(raw); s != "" {
|
||
return s, true
|
||
}
|
||
return "", false
|
||
case map[string]any:
|
||
if s := DescriptionFromAny(raw); s != "" {
|
||
return s, true
|
||
}
|
||
return "", false
|
||
default:
|
||
s := strings.TrimSpace(fmt.Sprint(raw))
|
||
if s == "" || s == "<nil>" {
|
||
return "", true
|
||
}
|
||
return "", false
|
||
}
|
||
}
|
||
|
||
// plainDescriptionFromItem is kept for callers that need SEO/plain text (meta).
|
||
func plainDescriptionFromItem(item V1ProcessJobItem) (string, bool) {
|
||
if item == nil {
|
||
return "", true
|
||
}
|
||
raw := item["description"]
|
||
if raw == nil {
|
||
return "", true
|
||
}
|
||
switch raw.(type) {
|
||
case string:
|
||
return PlainDescriptionFromAny(raw), true
|
||
case []any, []string:
|
||
if s := PlainDescriptionFromAny(raw); s != "" {
|
||
return s, true
|
||
}
|
||
return "", false
|
||
case map[string]any:
|
||
if s := PlainDescriptionFromAny(raw); s != "" {
|
||
return s, true
|
||
}
|
||
return "", false
|
||
default:
|
||
s := strings.TrimSpace(fmt.Sprint(raw))
|
||
if s == "" || s == "<nil>" {
|
||
return "", true
|
||
}
|
||
return "", false
|
||
}
|
||
}
|
||
|
||
func attrsMapFromItem(item V1ProcessJobItem) (map[string]any, bool) {
|
||
if item == nil {
|
||
return map[string]any{}, true
|
||
}
|
||
raw := item["attributes"]
|
||
if raw == nil {
|
||
return map[string]any{}, true
|
||
}
|
||
switch t := raw.(type) {
|
||
case map[string]any:
|
||
return t, true
|
||
default:
|
||
return map[string]any{}, false
|
||
}
|
||
}
|
||
|
||
func dirtyV1AttrKeys(attrs map[string]any) []string {
|
||
if len(attrs) == 0 {
|
||
return nil
|
||
}
|
||
var dirty []string
|
||
for k := range attrs {
|
||
key := strings.TrimSpace(k)
|
||
lk := strings.ToLower(key)
|
||
if key == "" || isReservedProductKey(key) || isInvalidAttributeKey(key) ||
|
||
strings.HasPrefix(lk, "eprel") {
|
||
dirty = append(dirty, key)
|
||
continue
|
||
}
|
||
// Re-run sanitize: if key disappears, it was dirty.
|
||
trial := SanitizeV1ProcessAttributes(map[string]any{key: attrs[k]})
|
||
if len(trial) == 0 {
|
||
dirty = append(dirty, key)
|
||
}
|
||
}
|
||
return dirty
|
||
}
|
||
|
||
func eprelShapeOK(v any) bool {
|
||
if v == nil {
|
||
return true
|
||
}
|
||
_, ok := v.(map[string]any)
|
||
return ok
|
||
}
|
||
|
||
func normalizeEPRELValue(v any) any {
|
||
return eprel.NormalizeShape(v)
|
||
}
|
||
|
||
func imageFieldsOK(item V1ProcessJobItem) bool {
|
||
if item == nil {
|
||
return true
|
||
}
|
||
if v := item["main_image"]; v != nil {
|
||
if _, ok := v.(string); !ok {
|
||
return false
|
||
}
|
||
}
|
||
if v := item["more_images"]; v != nil {
|
||
switch v.(type) {
|
||
case []string, []any:
|
||
return true
|
||
default:
|
||
return false
|
||
}
|
||
}
|
||
return true
|
||
}
|
||
|
||
func stringSliceFromItem(item V1ProcessJobItem, key string) []string {
|
||
if item == nil {
|
||
return nil
|
||
}
|
||
switch t := item[key].(type) {
|
||
case []string:
|
||
out := make([]string, 0, len(t))
|
||
for _, s := range t {
|
||
s = strings.TrimSpace(s)
|
||
if s != "" {
|
||
out = append(out, s)
|
||
}
|
||
}
|
||
return out
|
||
case []any:
|
||
out := make([]string, 0, len(t))
|
||
for _, e := range t {
|
||
if s, ok := e.(string); ok {
|
||
s = strings.TrimSpace(s)
|
||
if s != "" {
|
||
out = append(out, s)
|
||
}
|
||
}
|
||
}
|
||
return out
|
||
default:
|
||
return nil
|
||
}
|
||
}
|