This commit is contained in:
2026-08-16 16:57:36 +02:00
parent 96f8c1115c
commit 532d439c41
106 changed files with 10147 additions and 494 deletions
+13 -1
View File
@@ -9,6 +9,10 @@ import (
"github.com/jackc/pgx/v5/pgxpool"
)
// cloneProcessedFieldSourcesSQL copies field_sources but drops enhance_input_hash
// so the destination's first process cannot hash-skip a thin prior description.
const cloneProcessedFieldSourcesSQL = "COALESCE(pp.field_sources, '{}'::jsonb) - 'enhance_input_hash'"
// CloneResult summarizes a company catalog clone (source unchanged).
type CloneResult struct {
SourceCompanyID uuid.UUID `json:"source_company_id"`
@@ -382,7 +386,7 @@ func cloneCompanyCatalog(ctx context.Context, pool *pgxpool.Pool, src, dest uuid
pp.meta_description,
pp.last_transition_at,
pp.structured_description,
pp.field_sources,
` + cloneProcessedFieldSourcesSQL + `,
pp.created_at,
now(),
COALESCE(pp.ai_provider_mode, 'internal'),
@@ -452,6 +456,14 @@ func cloneCompanyCatalog(ctx context.Context, pool *pgxpool.Pool, src, dest uuid
return nil, fmt.Errorf("reset raw processing flags: %w", err)
}
// Drop skip hashes still present on weak / title-echo localized descriptions
// (field_sources hash is already stripped in the INSERT above).
hashRepair, err := repairWeakEnhanceHashesTx(ctx, tx, dest)
if err != nil {
return nil, fmt.Errorf("repair weak enhance hashes: %w", err)
}
add("weak_enhance_hashes_cleared", hashRepair.Cleared)
if err := tx.Commit(ctx); err != nil {
return nil, err
}
+12
View File
@@ -1,6 +1,7 @@
package catalog
import (
"strings"
"testing"
"github.com/google/uuid"
@@ -24,3 +25,14 @@ func TestValidateCloneCompanies(t *testing.T) {
t.Fatalf("unexpected error: %v", err)
}
}
func TestCloneFieldSourcesSQLClearsEnhanceHash(t *testing.T) {
t.Parallel()
const want = "enhance_input_hash"
if !strings.Contains(cloneProcessedFieldSourcesSQL, want) {
t.Fatalf("clone field_sources SQL must remove %q; got %q", want, cloneProcessedFieldSourcesSQL)
}
if !strings.Contains(cloneProcessedFieldSourcesSQL, "-") {
t.Fatalf("expected jsonb key removal operator in %q", cloneProcessedFieldSourcesSQL)
}
}
+76
View File
@@ -0,0 +1,76 @@
package catalog
import (
"context"
"fmt"
"github.com/descrybe/descrybe-v2/apps/api/internal/billing"
"github.com/google/uuid"
"github.com/jackc/pgx/v5/pgxpool"
)
// EnsureCategoryAttributeLinks removes orphan category_attributes rows (missing
// category or attribute) without wiping taxonomy or valid links. Returns orphans
// removed and remaining link count.
func EnsureCategoryAttributeLinks(ctx context.Context, pool *pgxpool.Pool, companyID uuid.UUID) (orphansRemoved int, links int, err error) {
if pool == nil {
return 0, 0, fmt.Errorf("nil pool")
}
ct, err := pool.Exec(ctx, `
DELETE FROM category_attributes ca
WHERE ca.company_id = $1
AND (
NOT EXISTS (
SELECT 1 FROM categories c
WHERE c.company_id = ca.company_id AND c.unique_id = ca.category_unique_id
)
OR NOT EXISTS (
SELECT 1 FROM attributes a
WHERE a.id = ca.attribute_id AND a.company_id = ca.company_id
)
)`, companyID)
if err != nil {
return 0, 0, fmt.Errorf("purge orphan category_attributes: %w", err)
}
orphansRemoved = int(ct.RowsAffected())
if err := pool.QueryRow(ctx, `
SELECT count(*)::int FROM category_attributes WHERE company_id = $1`, companyID).Scan(&links); err != nil {
return orphansRemoved, 0, err
}
return orphansRemoved, links, nil
}
// RepairCompanyCategoryEnhancePrompts is the company-scoped variant of
// RepairA1DemoCategoryEnhancePrompts: same repairedCategoryEnhancePromptMap
// (sl + "*" with CategoryEnhanceUserTemplate / {{attrs}}), applied to one company.
// Idempotent — already-OK categories count as already_ok, not updated.
func RepairCompanyCategoryEnhancePrompts(ctx context.Context, pool *pgxpool.Pool, companyID uuid.UUID) (updated, alreadyOK, emptySkipped int, err error) {
if pool == nil {
return 0, 0, 0, fmt.Errorf("nil pool")
}
want, err := repairedCategoryEnhancePromptMap()
if err != nil {
return 0, 0, 0, err
}
summary := &RepairCategoryEnhancePromptsResult{ByCompany: map[string]int{}}
if _, err := repairCompanyCategoryEnhancePrompts(ctx, pool, companyID, companyID.String(), want, false, summary); err != nil {
return 0, 0, 0, err
}
return summary.Updated, summary.AlreadyOK, summary.EmptySkipped, nil
}
// ResolvePlatformDemoCompanyID finds the Platform Demo sandbox (never A1 cohort).
func ResolvePlatformDemoCompanyID(ctx context.Context, pool *pgxpool.Pool) (uuid.UUID, error) {
var id uuid.UUID
err := pool.QueryRow(ctx, `
SELECT c.id
FROM companies c
WHERE c.name = $1
AND COALESCE(c.legacy_company_id, '') <> $2
ORDER BY c.created_at ASC
LIMIT 1`, platformDemoCompanyName, billing.A1LegacyCompanyID).Scan(&id)
if err != nil {
return uuid.Nil, fmt.Errorf("platform demo company not found: %w", err)
}
return id, nil
}
@@ -0,0 +1,45 @@
package catalog
import (
"strings"
"testing"
"github.com/descrybe/descrybe-v2/apps/api/internal/aiprompts"
"github.com/descrybe/descrybe-v2/apps/api/internal/company"
)
func TestCategoryEnhanceTemplateHasAttrs(t *testing.T) {
t.Parallel()
tpl := aiprompts.CategoryEnhanceUserTemplate
if !strings.Contains(tpl, "{{attrs}}") {
t.Fatalf("template missing {{attrs}}: %q", tpl)
}
if !aiprompts.CategoryEnhancePromptNeedsRepair("legacy HTML prompt without attrs") {
t.Fatal("expected legacy prompt to need repair")
}
if aiprompts.CategoryEnhancePromptNeedsRepair(tpl) {
t.Fatal("canonical template should not need repair")
}
}
func TestPlatformDemoNameConstant(t *testing.T) {
t.Parallel()
if platformDemoCompanyName != "Platform Demo" {
t.Fatalf("got %q", platformDemoCompanyName)
}
}
func TestRepairedCategoryEnhancePromptMapMatchesA1Demo(t *testing.T) {
t.Parallel()
want, err := repairedCategoryEnhancePromptMap()
if err != nil {
t.Fatal(err)
}
tpl := strings.TrimSpace(aiprompts.CategoryEnhanceUserTemplate)
if want["sl"] != tpl || want[company.LangPromptAny] != tpl {
t.Fatalf("want sl+* template, got %#v", want)
}
if !strings.Contains(tpl, "{{attrs}}") {
t.Fatal("template must include {{attrs}}")
}
}
@@ -0,0 +1,66 @@
package catalog
import (
"strings"
"github.com/descrybe/descrybe-v2/apps/api/internal/eprel"
)
// alignProductFieldsWithV1 mirrors V1 process-item field semantics on dashboard
// product payloads for the same EAN: title/name, preferred description, and
// structured eprel. Mutates item in place after SQL scan / feed-spec linking.
func alignProductFieldsWithV1(item map[string]any) {
if item == nil {
return
}
name := firstNonEmptyString(
item["processed_name"],
item["name"],
item["title"],
)
if name != "" {
item["name"] = name
item["title"] = name
}
desc := firstNonEmptyString(
item["processed_description"],
item["description"],
)
if desc != "" {
item["description"] = desc
}
eprelVal := eprel.ExtractFromAttrs(asStringAnyMap(item["processed_attributes"]))
if eprelVal == nil {
eprelVal = eprel.ExtractFromAttrs(asStringAnyMap(item["attributes"]))
}
if eprelVal == nil {
if mapped, ok := item["mapped_data"].(map[string]any); ok {
eprelVal = eprel.ExtractFromAttrs(mapped)
}
}
item["eprel"] = eprelVal
if eprelVal != nil {
item["has_eprel"] = true
}
}
func firstNonEmptyString(vals ...any) string {
for _, v := range vals {
switch t := v.(type) {
case string:
if s := strings.TrimSpace(t); s != "" {
return s
}
case *string:
if t != nil {
if s := strings.TrimSpace(*t); s != "" {
return s
}
}
}
}
return ""
}
@@ -0,0 +1,123 @@
package catalog
import (
"testing"
"github.com/descrybe/descrybe-v2/apps/api/internal/eprel"
)
func TestAlignProductFieldsWithV1_prefersProcessedAndEprel(t *testing.T) {
item := map[string]any{
"name": "RAW ALLCAPS TITLE",
"processed_name": "Retail Title",
"description": "raw feed description",
"processed_description": "AI retail description",
"attributes": map[string]any{
"vzmetenje": "junk",
"brand": "Ostalo",
},
"processed_attributes": map[string]any{
"brand": "Ostalo",
"product_model": "W53070",
"eprel_id": "1632113",
"eprel_label": "https://eprel.example/label",
"eprel_energy_class": "A",
},
"category_name": "Nosilci za TV",
}
alignProductFieldsWithV1(item)
if item["name"] != "Retail Title" {
t.Fatalf("name=%v want Retail Title", item["name"])
}
if item["title"] != "Retail Title" {
t.Fatalf("title=%v want Retail Title", item["title"])
}
if item["description"] != "AI retail description" {
t.Fatalf("description=%v", item["description"])
}
ep, ok := item["eprel"].(map[string]any)
if !ok {
t.Fatalf("eprel type=%T", item["eprel"])
}
if ep["id"] != "1632113" || ep["energy_class"] != "A" {
t.Fatalf("eprel=%v", ep)
}
if item["has_eprel"] != true {
t.Fatalf("has_eprel=%v", item["has_eprel"])
}
// Feed attrs stay available for dual-pane UI; V1 poll uses processed_attributes.
attrs, _ := item["attributes"].(map[string]any)
if attrs["vzmetenje"] != "junk" {
t.Fatalf("feed attributes should remain: %v", attrs)
}
if item["category_name"] != "Nosilci za TV" {
t.Fatalf("category_name mutated")
}
}
func TestAlignProductFieldsWithV1_matchesProcessItemSemantics(t *testing.T) {
// Same underlying row as a V1 process item would project for one EAN.
processedName := "Pralni stroj Samsung WW90CGC04DAELE, 9kg"
processedDesc := "Tehnologija Ecobubble…"
processedAttrs := map[string]any{
"brand": "Samsung",
"product_model": "WW90CGC04DAELE",
"eprel_id": "1632113",
"eprel_label": "https://eprel.example/label",
"eprel_pdf": "https://eprel.example/fiche.pdf",
"eprel_energy_class": "A",
"eprel_energy_scale": "A_G",
}
dashboard := map[string]any{
"name": "RAW FEED TITLE",
"processed_name": processedName,
"description": "raw description",
"processed_description": processedDesc,
"category": "40",
"category_name": "Pralni stroji",
"attributes": map[string]any{"dodatne-informacije": "AI", "brand": "Samsung"},
"processed_attributes": processedAttrs,
"gtin": "8806095210711",
}
alignProductFieldsWithV1(dashboard)
// V1 process item projection (subset used by poll clients).
v1Title := processedName
v1Desc := processedDesc
v1Eprel := eprel.ExtractFromAttrs(processedAttrs)
if dashboard["name"] != v1Title || dashboard["title"] != v1Title {
t.Fatalf("title/name mismatch: dash=%v/%v v1=%v", dashboard["title"], dashboard["name"], v1Title)
}
if dashboard["description"] != v1Desc {
t.Fatalf("description mismatch: dash=%v v1=%v", dashboard["description"], v1Desc)
}
if dashboard["category_name"] != "Pralni stroji" {
t.Fatalf("category_name=%v", dashboard["category_name"])
}
dashEprel, _ := dashboard["eprel"].(map[string]any)
v1Map, _ := v1Eprel.(map[string]any)
if dashEprel["id"] != v1Map["id"] || dashEprel["energy_class"] != v1Map["energy_class"] {
t.Fatalf("eprel mismatch dash=%v v1=%v", dashEprel, v1Map)
}
// V1 attributes ≈ processed_attributes (allowlist/sanitize aside).
pa, _ := dashboard["processed_attributes"].(map[string]any)
if pa["brand"] != "Samsung" || pa["product_model"] != "WW90CGC04DAELE" {
t.Fatalf("processed_attributes=%v", pa)
}
}
func TestAlignProductFieldsWithV1_eprelFromMappedFallback(t *testing.T) {
item := map[string]any{
"processed_name": "Washer",
"mapped_data": map[string]any{
"eprel_id": "1632113",
},
}
alignProductFieldsWithV1(item)
ep, ok := item["eprel"].(map[string]any)
if !ok || ep["id"] != "1632113" {
t.Fatalf("eprel=%v", item["eprel"])
}
}
+220
View File
@@ -0,0 +1,220 @@
package catalog
import (
"context"
"fmt"
"strings"
"github.com/descrybe/descrybe-v2/apps/api/internal/aiprompts"
"github.com/descrybe/descrybe-v2/apps/api/internal/billing"
"github.com/descrybe/descrybe-v2/apps/api/internal/company"
"github.com/google/uuid"
"github.com/jackc/pgx/v5/pgxpool"
)
// Canonical Postgres id for the migrated A1 Slovenija tenant (seed-a1 default).
const a1CompanyID = "604f23a8-b66e-4b21-8b45-0d72b68f4790"
// platformDemoCompanyName matches migrator / seed-demo standalone demo tenant.
const platformDemoCompanyName = "Platform Demo"
// RepairCategoryEnhancePromptsResult is the dry-run / apply summary for
// RepairA1DemoCategoryEnhancePrompts.
type RepairCategoryEnhancePromptsResult struct {
CompaniesScanned int `json:"companies_scanned"`
CategoriesSeen int `json:"categories_seen"`
WouldUpdate int `json:"would_update"`
Updated int `json:"updated"`
AlreadyOK int `json:"already_ok"`
EmptySkipped int `json:"empty_skipped"`
ByCompany map[string]int `json:"by_company"`
DryRun bool `json:"dry_run"`
}
// RepairA1DemoCategoryEnhancePrompts replaces legacy HTML marketing categories.prompt
// values for A1 Slovenija + Platform Demo with aiprompts.CategoryEnhanceUserTemplate.
//
// LOCAL repair only (idempotent):
// - Writes JSON-compatible user overlays under both "sl" (prompt->>'sl') and "*"
// (LangPromptAny) so language stays via {{language}}, not hardcoded-only copy.
// - Touches ONLY categories.prompt — never title_template / description_template
// (unique name/description formulas stay intact; AppendFormulaConstraints encodes
// them as plain-text instructions at enhance render time).
// - dryRun=true: count WouldUpdate only; dryRun=false: apply and set Updated.
//
// Entrypoint: go run ./cmd/repair-category-prompts (-dry-run | -apply).
func RepairA1DemoCategoryEnhancePrompts(ctx context.Context, pool *pgxpool.Pool, dryRun bool) (RepairCategoryEnhancePromptsResult, error) {
out := RepairCategoryEnhancePromptsResult{
DryRun: dryRun,
ByCompany: map[string]int{},
}
if pool == nil {
return out, fmt.Errorf("postgres pool is required")
}
a1ID, err := uuid.Parse(a1CompanyID)
if err != nil {
return out, fmt.Errorf("a1 company id: %w", err)
}
companyRows, err := pool.Query(ctx, `
SELECT id, name
FROM companies
WHERE id = $1
OR name = $2
OR COALESCE(legacy_company_id, '') = $3
ORDER BY name`, a1ID, platformDemoCompanyName, billing.A1LegacyCompanyID)
if err != nil {
return out, fmt.Errorf("list target companies: %w", err)
}
defer companyRows.Close()
type co struct {
id uuid.UUID
name string
}
companies := make([]co, 0, 2)
for companyRows.Next() {
var c co
if err := companyRows.Scan(&c.id, &c.name); err != nil {
return out, err
}
companies = append(companies, c)
}
if err := companyRows.Err(); err != nil {
return out, err
}
out.CompaniesScanned = len(companies)
if len(companies) == 0 {
return out, fmt.Errorf("no A1 / Platform Demo companies found")
}
want, err := repairedCategoryEnhancePromptMap()
if err != nil {
return out, err
}
for _, c := range companies {
n, err := repairCompanyCategoryEnhancePrompts(ctx, pool, c.id, c.name, want, dryRun, &out)
if err != nil {
return out, err
}
if n > 0 {
out.ByCompany[c.name] = n
}
}
return out, nil
}
// repairedCategoryEnhancePromptMap is the canonical stored shape: "sl" (prompt->>'sl'
// for A1/Demo) plus LangPromptAny ("*") so PromptForLanguage resolves for any content
// language. Copy stays language-agnostic via {{language}} — not hardcoded Slovenian.
func repairedCategoryEnhancePromptMap() (company.LangPromptMap, error) {
tpl := strings.TrimSpace(aiprompts.CategoryEnhanceUserTemplate)
if tpl == "" {
return nil, fmt.Errorf("CategoryEnhanceUserTemplate is empty")
}
return company.LangPromptMap{
"sl": tpl,
company.LangPromptAny: tpl,
}, nil
}
func categoryEnhancePromptMapOK(m company.LangPromptMap, want company.LangPromptMap) bool {
if !company.HasAnyPrompt(m) || !company.HasAnyPrompt(want) {
return false
}
tpl := strings.TrimSpace(want[company.LangPromptAny])
if tpl == "" {
tpl = strings.TrimSpace(want["sl"])
}
if tpl == "" {
return false
}
// Accept already-repaired maps: every non-empty value equals the shared template
// and LangPromptAny (or legacy sl-only) is present.
hasKey := false
for lang, p := range m {
p = strings.TrimSpace(p)
if p == "" {
continue
}
if p != tpl {
return false
}
if lang == company.LangPromptAny || lang == "sl" {
hasKey = true
}
}
return hasKey
}
func repairCompanyCategoryEnhancePrompts(
ctx context.Context,
pool *pgxpool.Pool,
companyID uuid.UUID,
companyName string,
want company.LangPromptMap,
dryRun bool,
out *RepairCategoryEnhancePromptsResult,
) (int, error) {
rows, err := pool.Query(ctx, `
SELECT id, COALESCE(prompt, '{}'::jsonb)
FROM categories
WHERE company_id = $1`, companyID)
if err != nil {
return 0, fmt.Errorf("list categories for %s: %w", companyName, err)
}
defer rows.Close()
updatedHere := 0
for rows.Next() {
var id uuid.UUID
var raw []byte
if err := rows.Scan(&id, &raw); err != nil {
return updatedHere, err
}
out.CategoriesSeen++
m, err := company.DecodeLangPromptMap(raw)
if err != nil {
return updatedHere, fmt.Errorf("decode prompt category=%s company=%s: %w", id, companyName, err)
}
if !company.HasAnyPrompt(m) {
out.EmptySkipped++
continue
}
if categoryEnhancePromptMapOK(m, want) {
out.AlreadyOK++
continue
}
out.WouldUpdate++
if dryRun {
updatedHere++
continue
}
cleaned, err := company.SanitizeLangPromptMap(want, MaxCategoryPromptRunes)
if err != nil {
return updatedHere, fmt.Errorf("sanitize prompt category=%s: %w", id, err)
}
encoded, err := company.EncodeLangPromptMap(cleaned)
if err != nil {
return updatedHere, err
}
ct, err := pool.Exec(ctx, `
UPDATE categories
SET prompt = $3::jsonb, updated_at = now()
WHERE id = $1 AND company_id = $2`, id, companyID, string(encoded))
if err != nil {
return updatedHere, fmt.Errorf("update prompt category=%s: %w", id, err)
}
if ct.RowsAffected() == 0 {
return updatedHere, fmt.Errorf("category %s not updated (company mismatch?)", id)
}
out.Updated++
updatedHere++
}
return updatedHere, rows.Err()
}
@@ -0,0 +1,52 @@
package catalog
import (
"strings"
"testing"
"github.com/descrybe/descrybe-v2/apps/api/internal/aiprompts"
"github.com/descrybe/descrybe-v2/apps/api/internal/company"
)
func TestRepairedCategoryEnhancePromptMap(t *testing.T) {
t.Parallel()
want, err := repairedCategoryEnhancePromptMap()
if err != nil {
t.Fatal(err)
}
tpl := strings.TrimSpace(aiprompts.CategoryEnhanceUserTemplate)
if want["sl"] != tpl {
t.Fatalf("want prompt->>'sl' = shared template")
}
if want[company.LangPromptAny] != tpl {
t.Fatalf("want * = shared template")
}
if !categoryEnhancePromptMapOK(want, want) {
t.Fatal("canonical map should be OK")
}
legacy := company.LangPromptMap{"sl": "<H2>legacy HTML marketing</H2>"}
if categoryEnhancePromptMapOK(legacy, want) {
t.Fatal("legacy HTML must need repair")
}
slOnly := company.LangPromptMap{"sl": tpl}
if !categoryEnhancePromptMapOK(slOnly, want) {
t.Fatal("sl-only repaired map should be OK (idempotent)")
}
starOnly := company.LangPromptMap{company.LangPromptAny: tpl}
if !categoryEnhancePromptMapOK(starOnly, want) {
t.Fatal("*-only repaired map should be OK (idempotent)")
}
}
func TestRepairTargetsUseSharedTemplate(t *testing.T) {
t.Parallel()
tpl := strings.TrimSpace(aiprompts.CategoryEnhanceUserTemplate)
for _, v := range []string{"name", "description", "attrs", "category", "language"} {
if !strings.Contains(tpl, "{{"+v+"}}") {
t.Fatalf("shared template missing {{%s}}", v)
}
}
if a1CompanyID == "" || platformDemoCompanyName == "" {
t.Fatal("missing company target constants")
}
}
+9 -5
View File
@@ -161,12 +161,11 @@ func ExtractProductImages(mapped, raw map[string]any) (main string, more []strin
break
}
}
if imgs, ok := merged["images"].([]any); ok && len(imgs) > 0 {
urls := coerceToURLList(imgs)
if main == "" && len(urls) > 0 {
if urls := coerceToURLList(merged["images"]); len(urls) > 0 {
if main == "" {
main = urls[0]
urls = urls[1:]
} else if len(urls) > 0 && urls[0] == main {
} else if urls[0] == main {
urls = urls[1:]
}
for _, u := range urls {
@@ -175,6 +174,11 @@ func ExtractProductImages(mapped, raw map[string]any) (main string, more []strin
}
}
}
// Feeds often leave main_image empty while moreimages/images hold URLs.
if main == "" && len(more) > 0 {
main = more[0]
more = more[1:]
}
if main != "" {
filtered := more[:0]
for _, u := range more {
@@ -211,7 +215,7 @@ func coerceToURLString(value any) string {
}
return ""
case map[string]any:
for _, k := range []string{"#text", "__cdata", "@_href", "@_url", "href", "url"} {
for _, k := range []string{"#text", "__cdata", "@_href", "@_url", "href", "url", "src", "link", "image"} {
if u := coerceToURLString(v[k]); u != "" {
return u
}
+35
View File
@@ -54,3 +54,38 @@ func TestBuildMappedDataFromV1ItemOmitsEmptyNulls(t *testing.T) {
t.Fatal("EAN-only must not count as content")
}
}
func TestExtractProductImages_promotesMoreimagesWhenMainEmpty(t *testing.T) {
main, more := ExtractProductImages(map[string]any{
"main_image": "",
"moreimages": "https://cdn.example.com/a.jpg,https://cdn.example.com/b.jpg",
}, nil)
if main != "https://cdn.example.com/a.jpg" {
t.Fatalf("main=%q", main)
}
if len(more) != 1 || more[0] != "https://cdn.example.com/b.jpg" {
t.Fatalf("more=%v", more)
}
}
func TestExtractProductImages_imagesStringSliceAndSrcObjects(t *testing.T) {
main, more := ExtractProductImages(map[string]any{
"images": []string{
"https://cdn.example.com/main.jpg",
"https://cdn.example.com/2.jpg",
},
}, nil)
if main != "https://cdn.example.com/main.jpg" || len(more) != 1 || more[0] != "https://cdn.example.com/2.jpg" {
t.Fatalf("string slice main=%q more=%v", main, more)
}
main, more = ExtractProductImages(map[string]any{
"images": []any{
map[string]any{"src": "https://cdn.example.com/from-src.jpg"},
map[string]any{"src": "https://cdn.example.com/from-src-2.jpg"},
},
}, nil)
if main != "https://cdn.example.com/from-src.jpg" || len(more) != 1 || more[0] != "https://cdn.example.com/from-src-2.jpg" {
t.Fatalf("src objects main=%q more=%v", main, more)
}
}
@@ -0,0 +1,188 @@
package catalog
import (
"context"
"encoding/json"
"fmt"
"strings"
"github.com/descrybe/descrybe-v2/apps/api/internal/company"
"github.com/google/uuid"
"github.com/jackc/pgx/v5"
"github.com/jackc/pgx/v5/pgxpool"
)
const enhanceInputHashKey = "enhance_input_hash"
// RepairWeakEnhanceHashesResult is the detailed report for weak-hash clearing.
type RepairWeakEnhanceHashesResult struct {
Cleared int64
Scanned int
ClearedRawIDs []uuid.UUID
}
// RepairWeakEnhanceHashes clears enhance_input_hash from field_sources and
// localized_content for processed products whose descriptions are weak /
// title-echo / filler (so the next enhance cannot hash-skip thin priors).
// Returns the number of products updated.
func RepairWeakEnhanceHashes(ctx context.Context, pool *pgxpool.Pool, companyID uuid.UUID) (int64, error) {
res, err := RepairWeakEnhanceHashesDetailed(ctx, pool, companyID)
return res.Cleared, err
}
// RepairWeakEnhanceHashesDetailed is RepairWeakEnhanceHashes plus scan/raw-id detail
// for admin Fix A1 hygiene reporting.
func RepairWeakEnhanceHashesDetailed(ctx context.Context, pool *pgxpool.Pool, companyID uuid.UUID) (RepairWeakEnhanceHashesResult, error) {
var out RepairWeakEnhanceHashesResult
if pool == nil {
return out, fmt.Errorf("catalog pool not configured")
}
if companyID == uuid.Nil {
return out, ClientMsg("company_id is required")
}
tx, err := pool.Begin(ctx)
if err != nil {
return out, err
}
defer tx.Rollback(ctx)
out, err = repairWeakEnhanceHashesTx(ctx, tx, companyID)
if err != nil {
return out, err
}
if err := tx.Commit(ctx); err != nil {
return out, err
}
return out, nil
}
// RepairWeakEnhanceHashes clears weak enhance skip hashes for the company.
func (s *Service) RepairWeakEnhanceHashes(ctx context.Context, companyID uuid.UUID) (int64, error) {
if s == nil || s.Pool == nil {
return 0, fmt.Errorf("catalog service not configured")
}
return RepairWeakEnhanceHashes(ctx, s.Pool, companyID)
}
func repairWeakEnhanceHashesTx(ctx context.Context, tx pgx.Tx, companyID uuid.UUID) (RepairWeakEnhanceHashesResult, error) {
var out RepairWeakEnhanceHashesResult
rows, err := tx.Query(ctx, `
SELECT id, raw_product_id,
COALESCE(name, ''),
COALESCE(description, ''),
COALESCE(processed_name, ''),
COALESCE(processed_description, ''),
COALESCE(field_sources, '{}'::jsonb),
COALESCE(localized_content, '{}'::jsonb)
FROM processed_products
WHERE company_id = $1`, companyID)
if err != nil {
return out, fmt.Errorf("list processed products: %w", err)
}
defer rows.Close()
type pending struct {
id uuid.UUID
rawID uuid.UUID
fs []byte
loc []byte
}
var updates []pending
for rows.Next() {
out.Scanned++
var (
id, rawID uuid.UUID
name, desc, processedName, processedDesc string
fsRaw, locRaw []byte
)
if err := rows.Scan(&id, &rawID, &name, &desc, &processedName, &processedDesc, &fsRaw, &locRaw); err != nil {
return out, err
}
fs := map[string]any{}
if len(fsRaw) > 0 {
if err := json.Unmarshal(fsRaw, &fs); err != nil {
return out, err
}
}
if fs == nil {
fs = map[string]any{}
}
loc, err := company.DecodeLocalizedContent(locRaw)
if err != nil {
return out, err
}
changed := false
primaryDesc := strings.TrimSpace(processedDesc)
if primaryDesc == "" {
primaryDesc = desc
}
primaryName := strings.TrimSpace(processedName)
if primaryName == "" {
primaryName = name
}
if _, has := fs[enhanceInputHashKey]; has {
if company.IsWeakPriorEnhanceDescription(primaryDesc, primaryName, name) {
delete(fs, enhanceInputHashKey)
changed = true
}
}
for lang, fields := range loc {
d := strings.TrimSpace(fields.ProcessedDescription)
n := strings.TrimSpace(fields.ProcessedName)
if n == "" {
n = primaryName
}
if strings.TrimSpace(fields.EnhanceInputHash) == "" {
continue
}
if company.IsWeakPriorEnhanceDescription(d, n, primaryName, name) {
fields.EnhanceInputHash = ""
loc[lang] = fields
changed = true
}
}
if !changed {
continue
}
fsBytes, err := json.Marshal(fs)
if err != nil {
return out, err
}
locBytes, err := company.EncodeLocalizedContent(loc)
if err != nil {
return out, err
}
updates = append(updates, pending{id: id, rawID: rawID, fs: fsBytes, loc: locBytes})
}
if err := rows.Err(); err != nil {
return out, err
}
rows.Close()
seenRaw := map[uuid.UUID]struct{}{}
for _, u := range updates {
ct, err := tx.Exec(ctx, `
UPDATE processed_products
SET field_sources = $2::jsonb,
localized_content = $3::jsonb,
updated_at = now()
WHERE id = $1 AND company_id = $4`,
u.id, u.fs, u.loc, companyID)
if err != nil {
return out, fmt.Errorf("clear enhance hashes product=%s: %w", u.id, err)
}
if ct.RowsAffected() > 0 {
out.Cleared++
if _, ok := seenRaw[u.rawID]; !ok {
seenRaw[u.rawID] = struct{}{}
out.ClearedRawIDs = append(out.ClearedRawIDs, u.rawID)
}
}
}
return out, nil
}
@@ -0,0 +1,137 @@
package catalog
import (
"context"
"encoding/json"
"os"
"strings"
"testing"
"github.com/descrybe/descrybe-v2/apps/api/internal/company"
"github.com/google/uuid"
"github.com/jackc/pgx/v5/pgxpool"
)
func TestClearWeakEnhanceHashesDecision(t *testing.T) {
t.Parallel()
title := "Acme Widget Pro 24"
weak := title
strong := "Acme Widget Pro 24 is a durable retail widget built for everyday warehouse use with a reinforced frame."
if !company.IsWeakPriorEnhanceDescription(weak, title) {
t.Fatal("title-echo must be weak")
}
if company.IsWeakPriorEnhanceDescription(strong, title) {
t.Fatal("strong description must not be weak")
}
}
func TestRepairWeakEnhanceHashes_nilPool(t *testing.T) {
t.Parallel()
_, err := RepairWeakEnhanceHashes(context.Background(), nil, uuid.New())
if err == nil {
t.Fatal("expected error for nil pool")
}
_, err = RepairWeakEnhanceHashes(context.Background(), &pgxpool.Pool{}, uuid.Nil)
if err == nil {
t.Fatal("expected error for nil company")
}
}
func TestRepairWeakEnhanceHashes_integration(t *testing.T) {
dsn := strings.TrimSpace(os.Getenv("DATABASE_URL"))
if dsn == "" {
t.Skip("DATABASE_URL not set")
}
ctx := context.Background()
pool, err := pgxpool.New(ctx, dsn)
if err != nil {
t.Fatalf("connect: %v", err)
}
defer pool.Close()
var companyID uuid.UUID
err = pool.QueryRow(ctx, `
SELECT id FROM companies
WHERE lower(name) IN ('platform demo', 'demo')
ORDER BY CASE WHEN lower(name) = 'platform demo' THEN 0 ELSE 1 END
LIMIT 1`).Scan(&companyID)
if err != nil {
t.Skipf("Platform Demo company not found: %v", err)
}
title := "Repair Hash Probe Widget X"
weakDesc := title
hash := "deadbeefcafebabe0123456789abcdef0123456789abcdef0123456789abcdef"
loc := company.LocalizedContent{
"en": {
ProcessedName: title,
ProcessedDescription: weakDesc,
EnhanceInputHash: hash,
},
}
locJSON, err := company.EncodeLocalizedContent(loc)
if err != nil {
t.Fatalf("encode loc: %v", err)
}
fs, _ := json.Marshal(map[string]any{enhanceInputHashKey: hash, "name": "ai_enhance"})
tx, err := pool.Begin(ctx)
if err != nil {
t.Fatalf("begin: %v", err)
}
defer tx.Rollback(ctx)
var rawID, ppID uuid.UUID
gtin := "8700999111222"
err = tx.QueryRow(ctx, `
INSERT INTO raw_products (company_id, gtin, raw_data, mapped_data, is_processed, processing_status)
VALUES ($1, $2, '{}'::jsonb, '{}'::jsonb, true, 'processed')
RETURNING id`, companyID, gtin).Scan(&rawID)
if err != nil {
t.Fatalf("insert raw: %v", err)
}
err = tx.QueryRow(ctx, `
INSERT INTO processed_products (
company_id, raw_product_id, product_id, name, processed_name, description, processed_description,
status, attributes, processed_attributes, gpt_response, field_sources, localized_content, ai_provider_mode
) VALUES (
$1, $2, $3, $4, $4, $5, $5, 'needs_review', '{}'::jsonb, '{}'::jsonb, '{}'::jsonb, $6::jsonb, $7::jsonb, 'internal'
) RETURNING id`, companyID, rawID, gtin, title, weakDesc, fs, locJSON).Scan(&ppID)
if err != nil {
t.Fatalf("insert processed: %v", err)
}
res, err := repairWeakEnhanceHashesTx(ctx, tx, companyID)
if err != nil {
t.Fatalf("repair: %v", err)
}
if res.Cleared < 1 {
t.Fatalf("expected at least 1 cleared, got %d (scanned=%d)", res.Cleared, res.Scanned)
}
var fsOut, locOut []byte
err = tx.QueryRow(ctx, `
SELECT COALESCE(field_sources, '{}'::jsonb), COALESCE(localized_content, '{}'::jsonb)
FROM processed_products WHERE id = $1`, ppID).Scan(&fsOut, &locOut)
if err != nil {
t.Fatalf("reload: %v", err)
}
var fsMap map[string]any
_ = json.Unmarshal(fsOut, &fsMap)
if _, ok := fsMap[enhanceInputHashKey]; ok {
t.Fatalf("field_sources hash still present: %v", fsMap[enhanceInputHashKey])
}
gotLoc, err := company.DecodeLocalizedContent(locOut)
if err != nil {
t.Fatalf("decode loc: %v", err)
}
if h := gotLoc["en"].EnhanceInputHash; h != "" {
t.Fatalf("localized hash still present: %q", h)
}
if gotLoc["en"].ProcessedDescription != weakDesc {
t.Fatalf("description should be preserved, got %q", gotLoc["en"].ProcessedDescription)
}
// Roll back probe rows — do not leave smoke data on Platform Demo.
}
+33 -12
View File
@@ -260,7 +260,7 @@ func enrichCategoryPrompts(ctx context.Context, pool *pgxpool.Pool, companyID uu
}
primary := company.LoadLanguage(ctx, pool, companyID)
item["prompts"] = prompts
item["prompt"] = company.PromptForLanguage(prompts, primary)
item["prompt"] = company.PromptForLanguage(prompts, primary, primary)
item["has_prompt"] = company.HasAnyPrompt(prompts)
return item, nil
}
@@ -668,8 +668,11 @@ func processedProductsCountFromSQL(needsRawJoin bool) string {
// Enrichment coverage SQL predicates (alias p = processed_products, r = raw_products).
const (
processedHasNameSQL = `(COALESCE(NULLIF(p.processed_name, ''), NULLIF(p.name, ''), NULLIF(r.mapped_data->>'name', ''), NULLIF(r.mapped_data->>'title', ''), '') <> '')`
processedHasDescriptionSQL = `(COALESCE(NULLIF(p.processed_description, ''), NULLIF(p.description, ''), NULLIF(r.mapped_data->>'description', ''), '') <> '')`
// Prefer processed_* so dashboard product fields match V1 process items for the same EAN.
processedPreferredNameSQL = `COALESCE(NULLIF(p.processed_name, ''), NULLIF(p.name, ''), NULLIF(r.mapped_data->>'name', ''), NULLIF(r.mapped_data->>'title', ''), '')`
processedPreferredDescriptionSQL = `COALESCE(NULLIF(p.processed_description, ''), NULLIF(p.description, ''), NULLIF(r.mapped_data->>'description', ''), '')`
processedHasNameSQL = `(` + processedPreferredNameSQL + ` <> '')`
processedHasDescriptionSQL = `(` + processedPreferredDescriptionSQL + ` <> '')`
processedHasCategorySQL = `(COALESCE(NULLIF(p.category, ''), '') <> '' AND lower(p.category) <> 'none')`
// Resolve display name / canonical unique_id when products store unique_id, UUID id, or name.
processedCategoryResolveJoin = `
@@ -1093,7 +1096,7 @@ func (s *Service) ListProcessedProducts(ctx context.Context, companyID uuid.UUID
func(ctx context.Context) ([]map[string]any, error) {
rows, err := s.Pool.Query(ctx, fmt.Sprintf(`
SELECT p.id, p.product_id,
COALESCE(NULLIF(p.name, ''), NULLIF(r.mapped_data->>'name', ''), NULLIF(r.mapped_data->>'title', ''), '') AS name,
`+processedPreferredNameSQL+` AS name,
COALESCE(NULLIF(p.processed_name, ''), '') AS processed_name,
p.category,
COALESCE(cat.name, NULLIF(BTRIM(p.category), '')) AS category_name,
@@ -1102,9 +1105,9 @@ func (s *Service) ListProcessedProducts(ctx context.Context, companyID uuid.UUID
f.name AS feed_name,
f.last_synced_at AS feed_last_synced_at,
r.updated_at AS raw_updated_at,
(COALESCE(NULLIF(p.processed_name, ''), NULLIF(p.name, ''), NULLIF(r.mapped_data->>'name', ''), NULLIF(r.mapped_data->>'title', ''), '') <> '') AS has_name,
(`+processedPreferredNameSQL+` <> '') AS has_name,
(COALESCE(NULLIF(p.processed_name, ''), '') <> '') AS has_processed_name,
(COALESCE(NULLIF(p.processed_description, ''), NULLIF(p.description, ''), NULLIF(r.mapped_data->>'description', ''), '') <> '') AS has_description,
(`+processedPreferredDescriptionSQL+` <> '') AS has_description,
(COALESCE(NULLIF(p.processed_description, ''), '') <> '') AS has_processed_description,
(COALESCE(NULLIF(p.category, ''), '') <> '' AND lower(p.category) <> 'none') AS has_category,
`+processedHasAttributesSQL+` AS has_attributes,
@@ -1120,7 +1123,7 @@ func (s *Service) ListProcessedProducts(ctx context.Context, companyID uuid.UUID
return nil, err
}
defer rows.Close()
return scanMaps(rows, []string{
items, err := scanMaps(rows, []string{
"id", "product_id", "name", "processed_name", "category", "category_name", "category_unique_id",
"status", "raw_product_id", "feed_id", "gtin",
"feed_name", "feed_last_synced_at", "raw_updated_at",
@@ -1128,6 +1131,15 @@ func (s *Service) ListProcessedProducts(ctx context.Context, companyID uuid.UUID
"has_eprel",
"created_at", "updated_at",
})
if err != nil {
return nil, err
}
for _, item := range items {
if name, ok := item["name"].(string); ok && strings.TrimSpace(name) != "" {
item["title"] = name
}
}
return items, nil
},
)
}
@@ -1172,12 +1184,12 @@ func (s *Service) ListProcessedProductsDetailed(ctx context.Context, companyID u
func(ctx context.Context) ([]map[string]any, error) {
rows, err := s.Pool.Query(ctx, fmt.Sprintf(`
SELECT p.id, p.product_id,
COALESCE(NULLIF(p.name, ''), NULLIF(r.mapped_data->>'name', ''), NULLIF(r.mapped_data->>'title', ''), '') AS name,
`+processedPreferredNameSQL+` AS name,
p.category,
COALESCE(cat.name, NULLIF(BTRIM(p.category), '')) AS category_name,
COALESCE(cat.unique_id, NULLIF(BTRIM(p.category), '')) AS category_unique_id,
p.status, p.raw_product_id, p.feed_id, r.gtin,
COALESCE(NULLIF(p.description, ''), NULLIF(r.mapped_data->>'description', ''), '') AS description,
`+processedPreferredDescriptionSQL+` AS description,
p.processed_name, p.processed_description,
COALESCE(p.meta_title, ''), COALESCE(p.meta_description, ''),
p.attributes, p.processed_attributes, r.mapped_data,
@@ -1190,7 +1202,7 @@ func (s *Service) ListProcessedProductsDetailed(ctx context.Context, companyID u
return nil, err
}
defer rows.Close()
return scanMaps(rows, []string{
items, err := scanMaps(rows, []string{
"id", "product_id", "name", "category", "category_name", "category_unique_id",
"status", "raw_product_id", "feed_id", "gtin",
"description", "processed_name", "processed_description",
@@ -1198,6 +1210,13 @@ func (s *Service) ListProcessedProductsDetailed(ctx context.Context, companyID u
"attributes", "processed_attributes", "mapped_data",
"created_at", "updated_at",
})
if err != nil {
return nil, err
}
for _, item := range items {
alignProductFieldsWithV1(item)
}
return items, nil
},
)
}
@@ -1205,11 +1224,11 @@ func (s *Service) ListProcessedProductsDetailed(ctx context.Context, companyID u
func (s *Service) GetProcessedProduct(ctx context.Context, companyID, id uuid.UUID) (map[string]any, error) {
row := s.Pool.QueryRow(ctx, `
SELECT p.id, p.product_id,
COALESCE(NULLIF(p.name, ''), NULLIF(r.mapped_data->>'name', ''), NULLIF(r.mapped_data->>'title', ''), '') AS name,
`+processedPreferredNameSQL+` AS name,
p.category,
COALESCE(cat.name, NULLIF(BTRIM(p.category), '')) AS category_name,
COALESCE(cat.unique_id, NULLIF(BTRIM(p.category), '')) AS category_unique_id,
COALESCE(NULLIF(p.description, ''), NULLIF(r.mapped_data->>'description', ''), '') AS description,
`+processedPreferredDescriptionSQL+` AS description,
p.processed_name, p.processed_description,
p.status, p.attributes, p.processed_attributes, r.gtin, r.mapped_data,
COALESCE(p.feed_id, r.feed_id) AS feed_id,
@@ -1241,6 +1260,7 @@ func (s *Service) GetProcessedProduct(ctx context.Context, companyID, id uuid.UU
return nil, err
}
linkFeedSpecificationsIntoProduct(item)
alignProductFieldsWithV1(item)
primary := company.LoadLanguage(ctx, s.Pool, companyID)
item["content_language"] = primary
item["content_languages"] = company.LoadContentLanguages(ctx, s.Pool, companyID)
@@ -1303,6 +1323,7 @@ func (s *Service) GetRawProduct(ctx context.Context, companyID, id uuid.UUID) (m
return nil, err
}
linkFeedSpecificationsIntoProduct(item)
alignProductFieldsWithV1(item)
primary := company.LoadLanguage(ctx, s.Pool, companyID)
item["content_language"] = primary
item["content_languages"] = company.LoadContentLanguages(ctx, s.Pool, companyID)