greeneclipse/descrybe
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fixes

Browse Source
This commit is contained in:
greeneclipse
2026-08-16 16:57:36 +02:00
parent 96f8c1115c
commit 532d439c41
106 changed files with 10147 additions and 494 deletions
Download Patch File Download Diff File Expand all files Collapse all files
apps/api/cmd/_electronics_probe_tmp/main.go
+188
View File
@@ -0,0 +1,188 @@
package main
import (
"context"
"fmt"
"log"
"os"
"os/signal"
"strings"
"syscall"
"time"
"github.com/descrybe/descrybe-v2/apps/api/internal/aiprompts"
"github.com/descrybe/descrybe-v2/apps/api/internal/config"
"github.com/descrybe/descrybe-v2/apps/api/internal/db"
"github.com/descrybe/descrybe-v2/apps/api/internal/logredact"
"github.com/descrybe/descrybe-v2/apps/api/internal/processing"
"github.com/google/uuid"
)
func main() {
log.SetOutput(logredact.Writer(os.Stderr))
eans := []string{
"4548736132597",
"194644207427",
"194644034894",
"195949822384",
"3830061960930",
}
if v := strings.TrimSpace(os.Getenv("PROBE_EANS")); v != "" {
eans = strings.Split(v, ",")
for i := range eans {
eans[i] = strings.TrimSpace(eans[i])
}
}
companyID, err := uuid.Parse("2b3159b0-fc08-415b-b248-35ed02a6baab")
if err != nil {
log.Fatal(err)
}
userID, err := uuid.Parse("0ce3305c-d810-4b56-b1d4-3c1ed510db76")
if err != nil {
log.Fatal(err)
}
cfg, err := config.Load()
if err != nil {
log.Fatalf("config: %v", err)
}
ctx, cancel := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
defer cancel()
pool, err := db.NewPool(ctx, cfg.DatabaseURL, db.PoolOptions{
MaxConns: int32(cfg.DBMaxConns),
MinConns: int32(cfg.DBMinConns),
MaxConnLifetime: cfg.DBMaxConnLifetime,
MaxConnLifetimeJitter: cfg.DBMaxConnLifetimeJitter,
MaxConnIdleTime: cfg.DBMaxConnIdleTime,
HealthCheckPeriod: cfg.DBHealthCheckPeriod,
StatementTimeout: cfg.DBStatementTimeout,
})
if err != nil {
log.Fatalf("db: %v", err)
}
defer pool.Close()
pipeline := processing.NewPipeline(pool)
pipeline.BatchSize = cfg.ProcessingBatchSize
// Force mock-llm: platform DB OpenAI binding would otherwise override env.
pipeline.AI = nil
pipeline.Prompts = aiprompts.NewService(pool)
mockKey := strings.TrimSpace(cfg.OpenAIAPIKey)
if mockKey == "" {
mockKey = "local-test"
}
mockBase := strings.TrimSpace(cfg.OpenAIBaseURL)
if mockBase == "" {
mockBase = "http://127.0.0.1:18767/v1"
}
mockModel := strings.TrimSpace(cfg.OpenAIModel)
if mockModel == "" {
mockModel = "mock-llm"
}
log.Printf("forcing completer base=%s model=%s", mockBase, mockModel)
pipeline.Engine = &processing.Engine{
Completer: processing.NewOpenAIClient(mockKey, mockBase, mockModel, cfg.ProcessingRPM, cfg.ProcessingMaxRetries),
Vector: processing.NoopVectorCategorizer{},
EPREL: nil, // skip remote EPREL hangs during local scorecard
ProviderMode: processing.AIProviderInternal,
}
rawIDs := make([]uuid.UUID, 0, len(eans))
for _, ean := range eans {
var id uuid.UUID
err := pool.QueryRow(ctx, `
SELECT id FROM raw_products
WHERE company_id = $1 AND gtin = $2`, companyID, ean).Scan(&id)
if err != nil {
log.Fatalf("raw product %s: %v", ean, err)
}
rawIDs = append(rawIDs, id)
log.Printf("ean=%s raw_id=%s", ean, id)
}
// Clear competing queue noise.
_, _ = pool.Exec(ctx, `
UPDATE processing_jobs
SET status = 'failed', error = 'yielded to electronics in-process probe', updated_at = now()
WHERE status IN ('pending','running','processing')`)
jobs, err := pipeline.StartJob(ctx, companyID, userID, rawIDs, "full")
if err != nil {
log.Fatalf("StartJob: %v", err)
}
if len(jobs) == 0 {
log.Fatal("StartJob returned no jobs")
}
jobID := jobs[0].ID
log.Printf("job_id=%s chunks=%d", jobID, len(jobs))
defend := make(chan struct{})
go func() {
t := time.NewTicker(400 * time.Millisecond)
defer t.Stop()
for {
select {
case <-defend:
return
case <-ctx.Done():
return
case <-t.C:
_, _ = pool.Exec(context.Background(), `
UPDATE processing_jobs
SET status = 'failed', error = 'yielded to electronics in-process probe', updated_at = now()
WHERE status IN ('pending','running','processing') AND id <> $1`, jobID)
_, _ = pool.Exec(context.Background(), `
UPDATE processing_jobs
SET status = 'running', error = NULL, updated_at = now()
WHERE id = $1 AND status IN ('failed','cancelled')
AND (error ILIKE '%P0%' OR error ILIKE '%parked%' OR error ILIKE '%yield%' OR error ILIKE '%reclaim%' OR error ILIKE '%cleared%' OR error ILIKE '%superseded%' OR error ILIKE '%abandoned%')`, jobID)
_, _ = pool.Exec(context.Background(), `
UPDATE processing_job_products
SET status = 'pending', error = NULL, updated_at = now()
WHERE job_id = $1 AND status IN ('failed','cancelled')
AND processed_product_id IS NULL
AND (error ILIKE '%P0%' OR error ILIKE '%parked%' OR error ILIKE '%yield%' OR error ILIKE '%reclaim%' OR error ILIKE '%cleared%' OR error ILIKE '%superseded%' OR error ILIKE '%abandoned%')`, jobID)
}
}
}()
_, _ = pool.Exec(ctx, `
UPDATE processing_jobs
SET status = 'running', started_at = COALESCE(started_at, now()), updated_at = now()
WHERE id = $1`, jobID)
runCtx, runCancel := context.WithTimeout(ctx, 3*time.Minute)
defer runCancel()
if err := pipeline.ProcessJob(runCtx, jobID); err != nil {
close(defend)
log.Fatalf("ProcessJob: %v", err)
}
close(defend)
rows, err := pool.Query(ctx, `
SELECT rp.gtin, pjp.status, COALESCE(pjp.error,''), pjp.processed_product_id IS NOT NULL
FROM processing_job_products pjp
JOIN raw_products rp ON rp.id = pjp.raw_product_id
WHERE pjp.job_id = $1
ORDER BY rp.gtin`, jobID)
if err != nil {
log.Fatalf("query items: %v", err)
}
defer rows.Close()
fmt.Printf("JOB %s\n", jobID)
for rows.Next() {
var gtin, status, errMsg string
var hasPP bool
if err := rows.Scan(&gtin, &status, &errMsg, &hasPP); err != nil {
log.Fatal(err)
}
fmt.Printf("ITEM %s status=%s has_pp=%v err=%s\n", gtin, status, hasPP, errMsg)
}
out := os.Getenv("PROBE_OUT_DIR")
if out == "" {
out = `F:\laragon\www\_MY\descrybe-v2\.codehelper\_process_probe_electronics`
}
_ = os.MkdirAll(out, 0o755)
_ = os.WriteFile(out+string(os.PathSeparator)+"process_id.txt", []byte(jobID.String()+"\n"), 0o644)
}
apps/api/cmd/_probe_batch_a_tmp/main.go
+192
View File
@@ -0,0 +1,192 @@
package main
import (
"context"
"encoding/json"
"fmt"
"log"
"os"
"os/signal"
"strings"
"syscall"
"time"
"github.com/descrybe/descrybe-v2/apps/api/internal/aiprompts"
"github.com/descrybe/descrybe-v2/apps/api/internal/config"
"github.com/descrybe/descrybe-v2/apps/api/internal/db"
"github.com/descrybe/descrybe-v2/apps/api/internal/eprel"
"github.com/descrybe/descrybe-v2/apps/api/internal/logredact"
"github.com/descrybe/descrybe-v2/apps/api/internal/processing"
"github.com/google/uuid"
)
func main() {
log.SetOutput(logredact.Writer(os.Stderr))
eans := []string{
"4242005342488",
"4242005479948",
"4242002996967",
"4242005327065",
"4242005382330",
}
companyID := uuid.MustParse("2b3159b0-fc08-415b-b248-35ed02a6baab")
userID := uuid.MustParse("0ce3305c-d810-4b56-b1d4-3c1ed510db76")
cfg, err := config.Load()
if err != nil {
log.Fatalf("config: %v", err)
}
ctx, cancel := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
defer cancel()
pool, err := db.NewPool(ctx, cfg.DatabaseURL, db.PoolOptions{
MaxConns: int32(cfg.DBMaxConns),
MinConns: int32(cfg.DBMinConns),
MaxConnLifetime: cfg.DBMaxConnLifetime,
MaxConnLifetimeJitter: cfg.DBMaxConnLifetimeJitter,
MaxConnIdleTime: cfg.DBMaxConnIdleTime,
HealthCheckPeriod: cfg.DBHealthCheckPeriod,
StatementTimeout: cfg.DBStatementTimeout,
})
if err != nil {
log.Fatalf("db: %v", err)
}
defer pool.Close()
out := os.Getenv("PROBE_OUT_DIR")
if out == "" {
out = `F:\laragon\www\_MY\descrybe-v2\.codehelper\_probe_batch_a`
}
_ = os.MkdirAll(out, 0o755)
pipeline := processing.NewPipeline(pool)
pipeline.BatchSize = cfg.ProcessingBatchSize
pipeline.AI = nil
pipeline.Prompts = aiprompts.NewService(pool)
mockKey := strings.TrimSpace(cfg.OpenAIAPIKey)
if mockKey == "" {
mockKey = "local-test"
}
mockBase := "http://127.0.0.1:18767/v1"
mockModel := "mock-llm"
log.Printf("forcing completer base=%s model=%s (ModeLabel=internal)", mockBase, mockModel)
client := processing.NewOpenAIClient(mockKey, mockBase, mockModel, cfg.ProcessingRPM, cfg.ProcessingMaxRetries)
client.ModeLabel = processing.AIProviderInternal
eprelClient := eprel.NewClient(eprel.Options{Enabled: true, Timeout: 20 * time.Second})
pipeline.Engine = &processing.Engine{
Completer: client,
Vector: processing.NoopVectorCategorizer{},
EPREL: eprelClient,
ProviderMode: processing.AIProviderInternal,
}
rawIDs := make([]uuid.UUID, 0, len(eans))
for _, ean := range eans {
var id uuid.UUID
err := pool.QueryRow(ctx, `
SELECT id FROM raw_products
WHERE company_id = $1 AND gtin = $2`, companyID, ean).Scan(&id)
if err != nil {
log.Fatalf("raw product %s: %v", ean, err)
}
rawIDs = append(rawIDs, id)
log.Printf("ean=%s raw_id=%s", ean, id)
}
// Self-heal only: do NOT fail/reclaim other jobs.
if n, err := processing.BackfillMissingMeta(ctx, pool, companyID); err != nil {
log.Printf("BackfillMissingMeta: %v", err)
} else {
log.Printf("BackfillMissingMeta updated=%d", n)
}
jobs, err := pipeline.StartJob(ctx, companyID, userID, rawIDs, "full")
if err != nil {
log.Fatalf("StartJob: %v", err)
}
if len(jobs) == 0 {
log.Fatal("StartJob returned no jobs")
}
jobID := jobs[0].ID
log.Printf("job_id=%s", jobID)
_ = os.WriteFile(out+string(os.PathSeparator)+"process_id.txt", []byte(jobID.String()+"\n"), 0o644)
defend := make(chan struct{})
go func() {
t := time.NewTicker(500 * time.Millisecond)
defer t.Stop()
for {
select {
case <-defend:
return
case <-ctx.Done():
return
case <-t.C:
// Restore ONLY our job if a sibling parked/yielded it. Never touch other jobs.
_, _ = pool.Exec(context.Background(), `
UPDATE processing_jobs
SET status = 'running', error = NULL, updated_at = now()
WHERE id = $1 AND status IN ('failed','cancelled')
AND (error ILIKE '%P0%' OR error ILIKE '%parked%' OR error ILIKE '%yield%'
OR error ILIKE '%reclaim%' OR error ILIKE '%cleared%' OR error ILIKE '%superseded%'
OR error ILIKE '%exclusive%' OR error ILIKE '%batch_%' OR error ILIKE '%inproc%')`, jobID)
_, _ = pool.Exec(context.Background(), `
UPDATE processing_job_products
SET status = 'pending', error = NULL, updated_at = now()
WHERE job_id = $1 AND status IN ('failed','cancelled')
AND processed_product_id IS NULL
AND (error ILIKE '%P0%' OR error ILIKE '%parked%' OR error ILIKE '%yield%'
OR error ILIKE '%reclaim%' OR error ILIKE '%cleared%' OR error ILIKE '%superseded%'
OR error ILIKE '%exclusive%' OR error ILIKE '%batch_%' OR error ILIKE '%inproc%')`, jobID)
}
}
}()
_, _ = pool.Exec(ctx, `
UPDATE processing_jobs
SET status = 'running', started_at = COALESCE(started_at, now()), updated_at = now()
WHERE id = $1`, jobID)
runCtx, runCancel := context.WithTimeout(ctx, 12*time.Minute)
defer runCancel()
if err := pipeline.ProcessJob(runCtx, jobID); err != nil {
close(defend)
log.Fatalf("ProcessJob: %v", err)
}
close(defend)
items, err := pipeline.LoadV1ProcessJobItems(ctx, companyID, jobID, "full")
if err != nil {
log.Fatalf("LoadV1ProcessJobItems: %v", err)
}
payload := map[string]any{
"data": map[string]any{
"process_id": jobID.String(),
"status": "COMPLETED",
"processing_type": "full",
"total_items": len(items),
"items": items,
},
}
raw, err := json.MarshalIndent(payload, "", " ")
if err != nil {
log.Fatalf("marshal: %v", err)
}
if err := os.WriteFile(out+string(os.PathSeparator)+"final.json", raw, 0o644); err != nil {
log.Fatalf("write final: %v", err)
}
fmt.Printf("JOB %s items=%d\n", jobID, len(items))
for _, it := range items {
ean, _ := it["ean"].(string)
st, _ := it["status"].(string)
title, _ := it["title"].(string)
if len(title) > 50 {
title = title[:50]
}
_, hasEprel := it["eprel"].(map[string]any)
ai, _ := it["ai_provider_mode"].(string)
fmt.Printf("ITEM %s status=%s eprel=%v ai=%s title=%s\n", ean, st, hasEprel, ai, title)
}
}
apps/api/cmd/_thin_synth_probe_tmp/main.go
+173
View File
@@ -0,0 +1,173 @@
package main
import (
"context"
"fmt"
"log"
"os"
"os/signal"
"strings"
"syscall"
"time"
"github.com/descrybe/descrybe-v2/apps/api/internal/aiprompts"
"github.com/descrybe/descrybe-v2/apps/api/internal/config"
"github.com/descrybe/descrybe-v2/apps/api/internal/db"
"github.com/descrybe/descrybe-v2/apps/api/internal/logredact"
"github.com/descrybe/descrybe-v2/apps/api/internal/processing"
"github.com/google/uuid"
)
// timeoutCompleter forces enhance deadline so synthesis path is exercised live.
type timeoutCompleter struct{}
func (timeoutCompleter) Complete(context.Context, string, string) (processing.Completion, error) {
return processing.Completion{}, context.DeadlineExceeded
}
func main() {
log.SetOutput(logredact.Writer(os.Stderr))
ean := strings.TrimSpace(os.Getenv("PROBE_EAN"))
if ean == "" {
ean = "4719331854218"
}
companyID := uuid.MustParse("2b3159b0-fc08-415b-b248-35ed02a6baab")
userID := uuid.MustParse("0ce3305c-d810-4b56-b1d4-3c1ed510db76")
cfg, err := config.Load()
if err != nil {
log.Fatalf("config: %v", err)
}
ctx, cancel := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
defer cancel()
pool, err := db.NewPool(ctx, cfg.DatabaseURL, db.PoolOptions{
MaxConns: int32(cfg.DBMaxConns),
MinConns: int32(cfg.DBMinConns),
MaxConnLifetime: cfg.DBMaxConnLifetime,
MaxConnLifetimeJitter: cfg.DBMaxConnLifetimeJitter,
MaxConnIdleTime: cfg.DBMaxConnIdleTime,
HealthCheckPeriod: cfg.DBHealthCheckPeriod,
StatementTimeout: cfg.DBStatementTimeout,
})
if err != nil {
log.Fatalf("db: %v", err)
}
defer pool.Close()
pipeline := processing.NewPipeline(pool)
pipeline.BatchSize = cfg.ProcessingBatchSize
pipeline.AI = nil
pipeline.Prompts = aiprompts.NewService(pool)
pipeline.Engine = &processing.Engine{
Completer: timeoutCompleter{},
Vector: processing.NoopVectorCategorizer{},
EPREL: nil,
ProviderMode: processing.AIProviderInternal,
}
var rawID uuid.UUID
if err := pool.QueryRow(ctx, `
SELECT id FROM raw_products
WHERE company_id = $1 AND gtin = $2`, companyID, ean).Scan(&rawID); err != nil {
log.Fatalf("raw product %s: %v", ean, err)
}
log.Printf("ean=%s raw_id=%s", ean, rawID)
_, _ = pool.Exec(ctx, `
UPDATE processed_products
SET processed_description = '', description = '',
field_sources = COALESCE(field_sources, '{}'::jsonb) - 'enhance_input_hash',
updated_at = now()
WHERE company_id = $1 AND product_id = $2`, companyID, ean)
_, _ = pool.Exec(ctx, `
UPDATE processing_jobs
SET status = 'failed', error = 'yielded to thin-synth probe', updated_at = now()
WHERE status IN ('pending','running','processing')`)
jobs, err := pipeline.StartJob(ctx, companyID, userID, []uuid.UUID{rawID}, "full")
if err != nil {
log.Fatalf("StartJob: %v", err)
}
if len(jobs) == 0 {
log.Fatal("StartJob returned no jobs")
}
jobID := jobs[0].ID
log.Printf("job_id=%s", jobID)
defend := make(chan struct{})
go func() {
t := time.NewTicker(300 * time.Millisecond)
defer t.Stop()
for {
select {
case <-defend:
return
case <-ctx.Done():
return
case <-t.C:
_, _ = pool.Exec(context.Background(), `
UPDATE processing_jobs
SET status = 'failed', error = 'yielded to thin-synth probe', updated_at = now()
WHERE status IN ('pending','running','processing') AND id <> $1`, jobID)
_, _ = pool.Exec(context.Background(), `
UPDATE processing_jobs
SET status = 'running', error = NULL, updated_at = now()
WHERE id = $1 AND status IN ('failed','cancelled')
AND (error ILIKE '%parked%' OR error ILIKE '%yield%' OR error ILIKE '%reclaim%'
OR error ILIKE '%exclusive%' OR error ILIKE '%cleared%' OR error ILIKE '%P0%')`, jobID)
_, _ = pool.Exec(context.Background(), `
UPDATE processing_job_products
SET status = 'pending', error = NULL, updated_at = now()
WHERE job_id = $1 AND status IN ('failed','cancelled')
AND processed_product_id IS NULL`, jobID)
}
}
}()
_, _ = pool.Exec(ctx, `
UPDATE processing_jobs
SET status = 'running', started_at = COALESCE(started_at, now()), updated_at = now()
WHERE id = $1`, jobID)
runCtx, runCancel := context.WithTimeout(ctx, 2*time.Minute)
defer runCancel()
if err := pipeline.ProcessJob(runCtx, jobID); err != nil {
close(defend)
log.Fatalf("ProcessJob: %v", err)
}
close(defend)
var name, desc, mode string
var descLen int
if err := pool.QueryRow(ctx, `
SELECT COALESCE(name,''), COALESCE(processed_description,''), length(COALESCE(processed_description,'')),
COALESCE(ai_provider_mode,'')
FROM processed_products
WHERE company_id = $1 AND product_id = $2`, companyID, ean).
Scan(&name, &desc, &descLen, &mode); err != nil {
log.Fatalf("load processed: %v", err)
}
fmt.Printf("EAN %s\n", ean)
fmt.Printf("name=%q\n", name)
fmt.Printf("processed_description_len=%d\n", descLen)
fmt.Printf("processed_description=%q\n", desc)
fmt.Printf("ai_provider_mode=%q\n", mode)
echo := strings.EqualFold(strings.TrimSpace(desc), strings.TrimSpace(name))
pass := descLen > 0 && !echo && !strings.Contains(strings.ToLower(desc), "ready for retail listing")
fmt.Printf("PASS_DESC_NE_TITLE=%v\n", pass)
if !pass {
os.Exit(1)
}
out := os.Getenv("PROBE_OUT_DIR")
if out == "" {
out = `F:\laragon\www\_MY\descrybe-v2\.codehelper\_process_probe_thin_synth`
}
_ = os.MkdirAll(out, 0o755)
_ = os.WriteFile(out+string(os.PathSeparator)+"process_id.txt", []byte(jobID.String()+"\n"), 0o644)
_ = os.WriteFile(out+string(os.PathSeparator)+"score.txt", []byte(fmt.Sprintf(
"ean=%s\nname=%s\npd_len=%d\npd=%s\nmode=%s\nPASS=%v\n", ean, name, descLen, desc, mode, pass,
)), 0o644)
}
apps/api/cmd/migrator/catalog_feeds.go
+36
View File
@@ -9,6 +9,7 @@ import (
"strconv"
"strings"
"github.com/descrybe/descrybe-v2/apps/api/internal/processing"
"github.com/google/uuid"
"github.com/jackc/pgx/v5"
"github.com/jackc/pgx/v5/pgxpool"
@@ -48,6 +49,7 @@ func migrateCatalogAndFeeds(
migrateProcessedProducts(ctx, mysqlDB, pg, companyMap, feedMap, rawMap, allow, report, dryRun)
backfillProcessedDescriptionsFromMapped(ctx, pg, companyMap, allow, report, dryRun)
backfillProcessedNamesFromMapped(ctx, pg, companyMap, allow, report, dryRun)
backfillProcessedCategoriesFromMapped(ctx, pg, companyMap, allow, report, dryRun)
}
if domains.has("feeds") {
migrateExportFeeds(ctx, mysqlDB, pg, companyMap, feedMap, allow, report, dryRun)
@@ -956,6 +958,40 @@ func backfillProcessedNamesFromMapped(
}
}
// backfillProcessedCategoriesFromMapped fills empty processed.category from mapped unique_ids.
func backfillProcessedCategoriesFromMapped(
ctx context.Context,
pg *pgxpool.Pool,
companyMap map[string]string,
allow map[string]bool,
report map[string]int,
dryRun bool,
) {
if dryRun || pg == nil {
return
}
total := 0
for legacy, cid := range companyMap {
if len(allow) > 0 && !allow[legacy] {
continue
}
u, err := uuid.Parse(cid)
if err != nil {
continue
}
res, err := processing.BackfillProcessedCategoriesFromMapped(ctx, pg, u)
if err != nil {
log.Printf("processed category backfill company=%s: %v", cid, err)
continue
}
total += int(res.Updated)
}
if total > 0 {
report["processed_categories_backfilled"] = total
log.Printf("backfilled %d processed_products.category from mapped_data", total)
}
}
func migrateExportFeeds(
ctx context.Context,
mysqlDB *sql.DB,
apps/api/cmd/repair-category-prompts/main.go
+77
View File
@@ -0,0 +1,77 @@
// Command repair-category-prompts replaces legacy A1 / Platform Demo categories.prompt
// HTML marketing formulas with aiprompts.CategoryEnhanceUserTemplate (JSON-compatible
// user overlay: {{name}} {{description}} {{attrs}} {{category}} {{language}}; stores
// both prompt->>'sl' and "*"). Does not touch title_template / description_template
// (formulas are appended as plain-text instructions at enhance render time).
//
// Usage (from apps/api):
//
// go run ./cmd/repair-category-prompts -dry-run
// go run ./cmd/repair-category-prompts -apply
//
// DATABASE_URL / -postgres required. Default is dry-run (count only).
package main
import (
"context"
"encoding/json"
"flag"
"fmt"
"log"
"os"
"strings"
"time"
"github.com/descrybe/descrybe-v2/apps/api/internal/catalog"
"github.com/jackc/pgx/v5/pgxpool"
)
func main() {
postgresURL := flag.String("postgres", os.Getenv("DATABASE_URL"), "Postgres URL (DATABASE_URL)")
dryRun := flag.Bool("dry-run", true, "count rows that would update (default true)")
apply := flag.Bool("apply", false, "write updates (implies not dry-run)")
flag.Parse()
if strings.TrimSpace(*postgresURL) == "" {
log.Fatal("-postgres / DATABASE_URL is required")
}
doDryRun := *dryRun
if *apply {
doDryRun = false
}
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute)
defer cancel()
pg, err := pgxpool.New(ctx, *postgresURL)
if err != nil {
log.Fatalf("postgres: %v", err)
}
defer pg.Close()
// Always print dry-run counts first when applying, so operators see the delta.
preview, err := catalog.RepairA1DemoCategoryEnhancePrompts(ctx, pg, true)
if err != nil {
log.Fatalf("dry-run: %v", err)
}
printResult("dry-run", preview)
if doDryRun {
return
}
res, err := catalog.RepairA1DemoCategoryEnhancePrompts(ctx, pg, false)
if err != nil {
log.Fatalf("apply: %v", err)
}
printResult("apply", res)
}
func printResult(phase string, res catalog.RepairCategoryEnhancePromptsResult) {
b, err := json.MarshalIndent(res, "", " ")
if err != nil {
log.Printf("%s: %+v", phase, res)
return
}
fmt.Printf("%s:\n%s\n", phase, string(b))
}
apps/api/cmd/seed-a1/attributes_backfill.go
+27
View File
@@ -0,0 +1,27 @@
package main
import (
"context"
"fmt"
"log"
"github.com/descrybe/descrybe-v2/apps/api/internal/processing"
"github.com/google/uuid"
"github.com/jackc/pgx/v5/pgxpool"
)
// backfillA1ProcessedAttributes rewrites A1 processed_products.attributes and
// processed_attributes with AttrsForPersist + category_attributes allowlists
// (same rules as processOne). Use -mode backfill-attributes for polluted local DBs
// where poll already looked clean but the catalog row still stored feed junk.
func backfillA1ProcessedAttributes(ctx context.Context, pg *pgxpool.Pool, companyID uuid.UUID) (int64, error) {
n, err := processing.BackfillCompanyProcessedAttributes(ctx, pg, companyID)
if err != nil {
return 0, fmt.Errorf("A1 attributes backfill: %w", err)
}
return n, nil
}
func logAttributesBackfillResult(updated int64) {
log.Printf("attributes backfill (A1): processed_updated=%d", updated)
}
apps/api/cmd/seed-a1/category_prompts.go
+68 -31
View File
@@ -12,6 +12,7 @@ import (
"strings"
"unicode"
"github.com/descrybe/descrybe-v2/apps/api/internal/aiprompts"
"github.com/descrybe/descrybe-v2/apps/api/internal/security"
"github.com/google/uuid"
"github.com/jackc/pgx/v5/pgxpool"
@@ -28,8 +29,9 @@ type categoryPromptsFile struct {
}
type categoryPromptEntry struct {
Name string `json:"name"`
Prompt string `json:"prompt"`
Name string `json:"name"`
UniqueID string `json:"unique_id,omitempty"`
Prompt string `json:"prompt"`
}
var (
@@ -145,26 +147,43 @@ func applyCategoryPrompts(ctx context.Context, pg *pgxpool.Pool, companyID uuid.
}
byNorm := make(map[string]string, len(file.Entries))
for _, e := range file.Entries {
name := strings.TrimSpace(e.Name)
prompt := prepareCategoryPrompt(e.Prompt)
if name == "" || prompt == "" {
out.Skipped++
continue
}
key := normalizeCategoryName(name)
if key == "" {
out.Skipped++
continue
}
byNorm[key] = prompt
byUnique := make(map[string]string, len(file.Entries))
// Overlay sets the shared JSON-compatible enhance user template (not legacy HTML).
// Seed JSON selects which categories get a prompt (prefer unique_id, else name).
// Unique title/description formulas stay in title_template / description_template.
// Stored under "sl" + "*" so prompt->>'sl' and LangPromptAny both resolve;
// language stays via {{language}}.
canonical := prepareCategoryPrompt(aiprompts.CategoryEnhanceUserTemplate)
if canonical == "" {
return out, fmt.Errorf("CategoryEnhanceUserTemplate sanitized to empty")
}
if len(byNorm) == 0 {
for _, e := range file.Entries {
uid := strings.TrimSpace(e.UniqueID)
name := strings.TrimSpace(e.Name)
if uid == "" && name == "" {
out.Skipped++
continue
}
if uid != "" {
byUnique[strings.ToLower(uid)] = canonical
}
if name != "" {
key := normalizeCategoryName(name)
if key == "" {
if uid == "" {
out.Skipped++
}
continue
}
byNorm[key] = canonical
}
}
if len(byNorm) == 0 && len(byUnique) == 0 {
return out, fmt.Errorf("no usable category prompts in %s", promptsPath)
}
rows, err := pg.Query(ctx, `
SELECT id, name
SELECT id, COALESCE(unique_id, ''), name
FROM categories
WHERE company_id = $1`, companyID)
if err != nil {
@@ -172,22 +191,34 @@ func applyCategoryPrompts(ctx context.Context, pg *pgxpool.Pool, companyID uuid.
}
defer rows.Close()
ids := make([]uuid.UUID, 0, len(byNorm))
prompts := make([]string, 0, len(byNorm))
matchedKeys := make(map[string]struct{}, len(byNorm))
ids := make([]uuid.UUID, 0, len(byNorm)+len(byUnique))
prompts := make([]string, 0, len(byNorm)+len(byUnique))
matchedNorm := make(map[string]struct{}, len(byNorm))
matchedUID := make(map[string]struct{}, len(byUnique))
for rows.Next() {
var id uuid.UUID
var name string
if err := rows.Scan(&id, &name); err != nil {
var uniqueID, name string
if err := rows.Scan(&id, &uniqueID, &name); err != nil {
return out, err
}
key := normalizeCategoryName(name)
prompt, ok := byNorm[key]
if !ok {
prompt := ""
if uidKey := strings.ToLower(strings.TrimSpace(uniqueID)); uidKey != "" {
if p, ok := byUnique[uidKey]; ok {
prompt = p
matchedUID[uidKey] = struct{}{}
}
}
if prompt == "" {
key := normalizeCategoryName(name)
if p, ok := byNorm[key]; ok {
prompt = p
matchedNorm[key] = struct{}{}
}
}
if prompt == "" {
continue
}
matchedKeys[key] = struct{}{}
ids = append(ids, id)
prompts = append(prompts, prompt)
}
@@ -195,22 +226,28 @@ func applyCategoryPrompts(ctx context.Context, pg *pgxpool.Pool, companyID uuid.
return out, err
}
for key := range byUnique {
if _, ok := matchedUID[key]; !ok {
out.Unmatched = append(out.Unmatched, "uid:"+key)
}
}
for key := range byNorm {
if _, ok := matchedKeys[key]; !ok {
out.Unmatched = append(out.Unmatched, key)
if _, ok := matchedNorm[key]; !ok {
// Name unmatched is noise when unique_id matched the same row; still report for seed hygiene.
out.Unmatched = append(out.Unmatched, "name:"+key)
}
}
sort.Strings(out.Unmatched)
if len(ids) == 0 {
return out, fmt.Errorf("no A1 categories matched any of %d seed prompts (check names)", len(byNorm))
return out, fmt.Errorf("no A1 categories matched any of %d seed prompts (check unique_id/names)", len(byNorm)+len(byUnique))
}
// Single parameterized batch update — company_id gate prevents cross-tenant writes.
// ASSUMPTION: A1 seed company content language is Slovenian ("sl").
// sl + * = prompt->>'sl' and company.LangPromptAny ({{language}} at render).
tag, err := pg.Exec(ctx, `
UPDATE categories AS c
SET prompt = jsonb_build_object('sl', v.prompt), updated_at = now()
SET prompt = jsonb_build_object('sl', v.prompt, '*', v.prompt), updated_at = now()
FROM (
SELECT * FROM unnest($1::uuid[], $2::text[]) AS t(id, prompt)
) AS v
apps/api/cmd/seed-a1/main.go
+21 -8
View File
@@ -7,11 +7,13 @@
// part of npm run seed:a1 (skipped on import + cleared after). Use
// -mode recover-jobs only when restoring legacy job history from a MySQL dump.
//
// After reimport, legacy per-category GPT prompts are overlaid from
// scripts/seed/a1-category-prompts.json (Name→Prompt export), matched by
// normalized category name, with v1 placeholders rewritten to {{name}} /
// {{description}}. Use -skip-category-prompts to skip, or
// -mode apply-category-prompts to overlay without a full wipe/reimport.
// After reimport, per-category enhance prompts are overlaid from
// scripts/seed/a1-category-prompts.json (Name list), matched by normalized
// category name, writing aiprompts.CategoryEnhanceUserTemplate (JSON-compatible;
// includes {{attrs}}/{{language}}). Legacy HTML bodies in that JSON are ignored.
// Use -skip-category-prompts to skip, or -mode apply-category-prompts to overlay
// without a full wipe/reimport. For polluted A1+Platform Demo prompts without
// reimport, run: go run ./cmd/repair-category-prompts -apply
//
// Categories: dump/archive store assignment on processed_products.category
// (category unique_id). Feed mappings do not map category. After reimport,
@@ -26,6 +28,9 @@
// exported/imported as-is (skip-processed must not strip them). For a
// polluted local DB without a full wipe, use -mode backfill-categories.
// Fixture EANs are purged from non-A1 tenants automatically.
// Processed attribute junk (poll clean, DB still has zavora/vzmetenje, …): use
// -mode backfill-attributes — same SanitizeProductAttributes + category allowlist
// as processOne/enhance.
//
// Usage:
//
@@ -34,6 +39,7 @@
// go run ./cmd/seed-a1 -mode apply-category-prompts -file ../../scripts/seed/a1-demo-data.sql.gz
// go run ./cmd/seed-a1 -mode recover-jobs -mysql-dump path/to/descrybe_new.sql
// go run ./cmd/seed-a1 -mode backfill-categories -mysql-dump path/to/descrybe_new.sql
// go run ./cmd/seed-a1 -mode backfill-attributes
//
// DATABASE_URL / -postgres required.
// Day-to-day: `npm run seed:a1` (reimport + category prompts + mapped category backfill). Maintainer snapshot: `npm run seed:a1:export`.
@@ -77,7 +83,7 @@ type tableSpec struct {
func main() {
postgresURL := flag.String("postgres", os.Getenv("DATABASE_URL"), "Postgres URL (DATABASE_URL)")
mode := flag.String("mode", "reimport", "export | reimport | recover-jobs | apply-category-prompts | backfill-categories")
mode := flag.String("mode", "reimport", "export | reimport | recover-jobs | apply-category-prompts | backfill-categories | backfill-attributes")
file := flag.String("file", "", "gzipped seed archive path (required for export/reimport)")
mysqlDump := flag.String("mysql-dump", os.Getenv("SEED_A1_MYSQL_DUMP"), "mysqldump path for recover-jobs / backfill-categories (or SEED_A1_MYSQL_DUMP)")
company := flag.String("company", defaultA1CompanyID, "Postgres companies.id for A1")
@@ -95,7 +101,8 @@ func main() {
}
modeVal := strings.ToLower(strings.TrimSpace(*mode))
if modeVal != "recover-jobs" && modeVal != "apply-category-prompts" && modeVal != "backfill-categories" && strings.TrimSpace(*file) == "" {
needsFile := modeVal != "recover-jobs" && modeVal != "apply-category-prompts" && modeVal != "backfill-categories" && modeVal != "backfill-attributes"
if needsFile && strings.TrimSpace(*file) == "" {
log.Fatal("-file is required (e.g. ../../scripts/seed/a1-demo-data.sql.gz)")
}
dumpPath := resolveMySQLDumpPath(*mysqlDump)
@@ -184,13 +191,19 @@ func main() {
}
res.PurgedOtherRaw, res.PurgedOtherPP = rawN, ppN
logCategoryBackfillResult(res, dumpPath)
case "backfill-attributes":
n, err := backfillA1ProcessedAttributes(ctx, pg, companyID)
if err != nil {
log.Fatalf("backfill-attributes: %v", err)
}
logAttributesBackfillResult(n)
case "recover-jobs":
if err := recoverJobsFromMySQLDump(ctx, pg, companyID, dumpPath); err != nil {
log.Fatalf("recover-jobs: %v", err)
}
log.Printf("recovered A1 processing jobs into company %s from %s", companyID, dumpPath)
default:
log.Fatalf("unknown -mode %q (want export|reimport|recover-jobs|apply-category-prompts|backfill-categories)", *mode)
log.Fatalf("unknown -mode %q (want export|reimport|recover-jobs|apply-category-prompts|backfill-categories|backfill-attributes)", *mode)
}
}
apps/api/internal/aiprompts/apply_defaults.go
+74
View File
@@ -0,0 +1,74 @@
package aiprompts
import (
"context"
"github.com/descrybe/descrybe-v2/apps/api/internal/company"
"github.com/google/uuid"
)
// productEnhanceUpsertLanguages returns ISO 639-1 codes to store product_enhance
// under. Never returns LangPromptAny ("*") — ai_prompt_templates.language has
// CHECK (language ~ '^[a-z]{2}$'). Categories.prompt JSON may still use "*".
func productEnhanceUpsertLanguages(contentLangs []string, primary string) []string {
primary = company.NormalizeLanguage(primary)
if primary == "" || primary == company.LangPromptAny || !company.IsAllowedLanguage(primary) {
primary = company.DefaultLanguage
}
out := make([]string, 0, len(contentLangs)+1)
seen := map[string]struct{}{}
add := func(code string) {
code = company.NormalizeLanguage(code)
if code == "" || code == company.LangPromptAny || !company.IsAllowedLanguage(code) {
return
}
if _, ok := seen[code]; ok {
return
}
seen[code] = struct{}{}
out = append(out, code)
}
add(primary)
for _, lang := range contentLangs {
add(lang)
}
if len(out) == 0 {
return []string{company.DefaultLanguage}
}
return out
}
// ApplyBuiltInProductEnhance upserts BuiltInDefaults for product_enhance under
// the company primary language and each content language. Templates still use
// {{language}} at render time; Resolve falls back requested → primary → built-in.
// Do not store LangPromptAny ("*") in ai_prompt_templates (DB language CHECK).
func (s *Service) ApplyBuiltInProductEnhance(ctx context.Context, companyID uuid.UUID) (languagesApplied int, err error) {
def, ok := DefaultFor(KeyProductEnhance)
if !ok {
return 0, ErrInvalidKey
}
primary := company.LoadLanguage(ctx, s.Pool, companyID)
langs := productEnhanceUpsertLanguages(
company.LoadContentLanguages(ctx, s.Pool, companyID),
primary,
)
enabled := true
items := make([]UpdateItem, 0, len(langs))
for _, lang := range langs {
items = append(items, UpdateItem{
Key: KeyProductEnhance,
Language: lang,
SystemTemplate: def.SystemTemplate,
UserTemplate: def.UserTemplate,
IsEnabled: &enabled,
})
}
_, err = s.Update(ctx, companyID, UpdateInput{
Language: langs[0],
Prompts: items,
})
if err != nil {
return 0, err
}
return len(langs), nil
}
apps/api/internal/aiprompts/apply_defaults_test.go
+71
View File
@@ -0,0 +1,71 @@
package aiprompts
import (
"regexp"
"testing"
"github.com/descrybe/descrybe-v2/apps/api/internal/company"
)
var languageFmt = regexp.MustCompile(`^[a-z]{2}$`)
func TestProductEnhanceUpsertLanguages_neverStar(t *testing.T) {
t.Parallel()
cases := []struct {
name string
contentLangs []string
primary string
want []string
}{
{
name: "primary_only",
contentLangs: nil,
primary: "sl",
want: []string{"sl"},
},
{
name: "primary_plus_content",
contentLangs: []string{"sl", "en", "de"},
primary: "sl",
want: []string{"sl", "en", "de"},
},
{
name: "drops_star_and_invalid",
contentLangs: []string{"*", "sl", "xx", "", "EN"},
primary: "*",
want: []string{"en", "sl"},
},
{
name: "dedupes_primary_in_content",
contentLangs: []string{"sl", "sl", "en"},
primary: "sl",
want: []string{"sl", "en"},
},
{
name: "empty_falls_back_default",
contentLangs: []string{"*", ""},
primary: "",
want: []string{company.DefaultLanguage},
},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
t.Parallel()
got := productEnhanceUpsertLanguages(tc.contentLangs, tc.primary)
if len(got) != len(tc.want) {
t.Fatalf("len=%d want %d (%v)", len(got), len(tc.want), got)
}
for i := range got {
if got[i] != tc.want[i] {
t.Fatalf("got %v want %v", got, tc.want)
}
if !languageFmt.MatchString(got[i]) {
t.Fatalf("language %q fails ai_prompt_templates_language_fmt", got[i])
}
if got[i] == company.LangPromptAny {
t.Fatalf("must not upsert LangPromptAny")
}
}
})
}
}
apps/api/internal/aiprompts/kinds.go
+31 -5
View File
@@ -1,5 +1,7 @@
package aiprompts
import "strings"
// Prompt keys stored in ai_prompt_templates.prompt_key.
const (
KeyProductEnhance = "product_enhance"
@@ -45,6 +47,32 @@ type DefaultTemplate struct {
UserTemplate string `json:"user_template"`
}
// CategoryEnhanceUserTemplate is the shared per-category (and built-in) enhance USER
// message. Includes {{name}} {{description}} {{attrs}} {{category}} {{language}}.
// Compatible with the enhance system JSON schema {"name","description"} — not a
// competing HTML marketing document. Title/description formulas stay in
// title_template / description_template and are appended at render time as plain
// text instructions (see processing.AppendFormulaConstraints).
// Used by local A1/Demo prompt repair and seed-a1 overlays.
const CategoryEnhanceUserTemplate = `Your reply is parsed as JSON {"name":"string","description":"string"} only (system schema). Write name and description in {{language}} (do not hardcode a language).
- name: short retail title; follow any Title formula constraints that follow; use Attrs
- description: prefer 1-3 factual paragraphs as ONE string; limited HTML (<h2><p><ul><li>) is allowed if needed — do NOT emit a competing full HTML document, <name>/<metaDescription> blocks, or separate schema
Category: {{category}}
Name: {{name}}
Desc: {{description}}
Attrs: {{attrs}}`
// CategoryEnhancePromptNeedsRepair reports whether a stored categories.prompt value
// should be replaced by CategoryEnhanceUserTemplate (idempotent equality check).
func CategoryEnhancePromptNeedsRepair(prompt string) bool {
p := strings.TrimSpace(prompt)
if p == "" {
return false
}
return p != strings.TrimSpace(CategoryEnhanceUserTemplate)
}
// BuiltInDefaults match the previous hardcoded system prompts, with structured user templates.
var BuiltInDefaults = []DefaultTemplate{
{
@@ -56,15 +84,13 @@ Rules:
- Reply with ONLY JSON (no markdown)
- Schema: {"name":"string","description":"string"}
- name: short retail title
- description: 1-2 factual sentences
- description: 1-2 factual sentences; never copy name as description
- When Desc is empty or the same as Name, write 1-2 factual sentences from Category and Attrs only (do not invent specs)
- Write name and description in {{language}}
Example:
{"name":"Acme Widget Pro","description":"Durable widget for everyday use. Clear specs, ready to ship."}
{{brand_voice}}`,
UserTemplate: `Category: {{category}}
Name: {{name}}
Desc: {{description}}
Attrs: {{attrs}}`,
UserTemplate: CategoryEnhanceUserTemplate,
},
{
Key: KeySEOMeta,
apps/api/internal/aiprompts/kinds_test.go
+59
View File
@@ -0,0 +1,59 @@
package aiprompts
import (
"strings"
"testing"
)
func TestCategoryEnhanceUserTemplateHasRequiredVars(t *testing.T) {
t.Parallel()
vars := ExtractVariables(CategoryEnhanceUserTemplate)
want := []string{"language", "category", "name", "description", "attrs"}
got := map[string]struct{}{}
for _, v := range vars {
got[v] = struct{}{}
}
for _, w := range want {
if _, ok := got[w]; !ok {
t.Fatalf("CategoryEnhanceUserTemplate missing {{%s}}; vars=%v", w, vars)
}
}
lower := strings.ToLower(CategoryEnhanceUserTemplate)
if strings.Contains(lower, "100 besed") || strings.Contains(lower, "gpt predloga") {
t.Fatal("template must not demand legacy long HTML marketing docs")
}
if !strings.Contains(CategoryEnhanceUserTemplate, `{"name":"string","description":"string"}`) {
t.Fatal("template should reference JSON schema shape")
}
if !strings.Contains(lower, "{{language}}") {
t.Fatal("language must come from {{language}}")
}
}
func TestCategoryEnhancePromptNeedsRepair(t *testing.T) {
t.Parallel()
if CategoryEnhancePromptNeedsRepair("") {
t.Fatal("empty should not need repair")
}
if CategoryEnhancePromptNeedsRepair(CategoryEnhanceUserTemplate) {
t.Fatal("canonical template should be idempotent")
}
if CategoryEnhancePromptNeedsRepair(" " + CategoryEnhanceUserTemplate + "\n") {
t.Fatal("whitespace-trimmed canonical should be idempotent")
}
legacy := `Ustvari nov opis\n<H2>foo</H2>\nAttrs missing`
if !CategoryEnhancePromptNeedsRepair(legacy) {
t.Fatal("legacy HTML prompt should need repair")
}
}
func TestBuiltInProductEnhanceUsesSharedUserTemplate(t *testing.T) {
t.Parallel()
def, ok := DefaultFor(KeyProductEnhance)
if !ok {
t.Fatal("missing product_enhance default")
}
if def.UserTemplate != CategoryEnhanceUserTemplate {
t.Fatalf("UserTemplate must be CategoryEnhanceUserTemplate")
}
}
apps/api/internal/aiprompts/service.go
+31 -10
View File
@@ -87,8 +87,11 @@ func (s *Service) GetBundle(ctx context.Context, companyID uuid.UUID, language s
}, nil
}
// Resolve returns the effective templates for one key + language
// (custom if enabled for lang, else built-in). No cross-language company fallback.
// Resolve returns the effective templates for one key + language.
// Fallback: requested lang → "*" → company primary language → built-in.
// Language on the result is always the requested (normalized) lang so {{language}}
// still resolves to the content language being generated — templates are shared,
// not duplicated per language.
func (s *Service) Resolve(ctx context.Context, companyID uuid.UUID, key, language string) (Resolved, error) {
if !ValidPromptKey(key) {
return Resolved{}, ErrInvalidKey
@@ -101,11 +104,19 @@ func (s *Service) Resolve(ctx context.Context, companyID uuid.UUID, key, languag
if err != nil {
lang = company.DefaultLanguage
}
st, err := s.loadOne(ctx, companyID, key, lang)
if err != nil && !errors.Is(err, pgx.ErrNoRows) {
return Resolved{}, err
primary := company.LoadLanguage(ctx, s.Pool, companyID)
candidates := []string{lang, company.LangPromptAny}
if primary != "" && primary != lang && primary != company.LangPromptAny {
candidates = append(candidates, primary)
}
if err == nil && st.isEnabled {
for _, cand := range candidates {
st, err := s.loadOne(ctx, companyID, key, cand)
if err != nil && !errors.Is(err, pgx.ErrNoRows) {
return Resolved{}, err
}
if err != nil || !st.isEnabled {
continue
}
sys := strings.TrimSpace(st.systemTemplate)
user := strings.TrimSpace(st.userTemplate)
if sys == "" {
@@ -155,9 +166,15 @@ func (s *Service) Update(ctx context.Context, companyID uuid.UUID, in UpdateInpu
if langRaw == "" {
langRaw = defaultLang
}
lang, err := company.ParseLanguage(langRaw, false)
if err != nil {
return Bundle{}, fmt.Errorf("%w: language %q", ErrInvalidInput, langRaw)
var lang string
if langRaw == company.LangPromptAny {
lang = company.LangPromptAny
} else {
parsed, err := company.ParseLanguage(langRaw, false)
if err != nil {
return Bundle{}, fmt.Errorf("%w: language %q", ErrInvalidInput, langRaw)
}
lang = parsed
}
lastLang = lang
if item.Reset {
@@ -196,7 +213,11 @@ func (s *Service) Update(ctx context.Context, companyID uuid.UUID, in UpdateInpu
if err := tx.Commit(ctx); err != nil {
return Bundle{}, err
}
return s.GetBundle(ctx, companyID, lastLang)
bundleLang := lastLang
if bundleLang == company.LangPromptAny {
bundleLang = defaultLang
}
return s.GetBundle(ctx, companyID, bundleLang)
}
func (s *Service) loadAll(ctx context.Context, companyID uuid.UUID) ([]stored, error) {
apps/api/internal/aiprovider/resolve_platform_test.go
+12
View File
@@ -61,3 +61,15 @@ func TestResolvePlatformOpenAI_viaPlatformService(t *testing.T) {
t.Fatalf("source=%q", oi.Source)
}
}
func TestPlatformModeLabel(t *testing.T) {
if got := platformModeLabel(platformsettings.SourceDB, "https://llm.overloadedbot.org/v1"); got != ModeInternalLabel {
t.Fatalf("db remote: %q", got)
}
if got := platformModeLabel(platformsettings.SourceEnv, "https://api.openai.com/v1"); got != ModeCustomLabel {
t.Fatalf("env remote: %q", got)
}
if got := platformModeLabel(platformsettings.SourceDB, "http://127.0.0.1:18767/v1"); got != ModeCustomLabel {
t.Fatalf("mock loopback: %q", got)
}
}
apps/api/internal/aiprovider/roles.go
+5 -3
View File
@@ -4,6 +4,7 @@ import (
"context"
"strings"
"github.com/descrybe/descrybe-v2/apps/api/internal/platformsettings"
"github.com/descrybe/descrybe-v2/apps/api/internal/processing"
"github.com/google/uuid"
)
@@ -134,7 +135,7 @@ func (s *Service) resolvePlatformRoleCompleter(ctx context.Context, role string)
BaseURL: cfg.BaseURL,
Model: model,
UsingBYOK: false,
ModeLabel: ModeInternalLabel,
ModeLabel: platformModeLabel(cfg.Source, cfg.BaseURL),
})
}
}
@@ -143,13 +144,14 @@ func (s *Service) resolvePlatformRoleCompleter(ctx context.Context, role string)
if role == RoleProcessing {
key := strings.TrimSpace(s.Env.OpenAIAPIKey)
model := strings.TrimSpace(s.Env.OpenAIModel)
baseURL := strings.TrimSpace(s.Env.OpenAIBaseURL)
if key != "" && model != "" {
return s.completerFromEndpoint(RoleEndpoint{
APIKey: key,
BaseURL: strings.TrimSpace(s.Env.OpenAIBaseURL),
BaseURL: baseURL,
Model: model,
UsingBYOK: false,
ModeLabel: ModeInternalLabel,
ModeLabel: platformModeLabel(platformsettings.SourceEnv, baseURL),
})
}
}
apps/api/internal/aiprovider/service.go
+15 -2
View File
@@ -277,6 +277,7 @@ func (s *Service) Resolve(ctx context.Context, companyID uuid.UUID) (Resolved, e
if strings.TrimSpace(platform.APIKey) == "" {
return Resolved{ModeLabel: ModeInternalLabel, UsingBYOK: false}, nil
}
modeLabel := platformModeLabel(platform.Source, platform.BaseURL)
client := processing.NewOpenAIClient(
platform.APIKey,
platform.BaseURL,
@@ -284,17 +285,29 @@ func (s *Service) Resolve(ctx context.Context, companyID uuid.UUID) (Resolved, e
rpm,
retries,
)
client.ModeLabel = ModeInternalLabel
client.ModeLabel = modeLabel
if s.HTTPClient != nil {
client.HTTPClient = s.HTTPClient
}
return Resolved{
Completer: client,
ModeLabel: ModeInternalLabel,
ModeLabel: modeLabel,
UsingBYOK: false,
}, nil
}
// platformModeLabel maps platform OpenAI source/base to analytics ModeLabel.
// DB-backed admin settings → internal; env bootstrap or mock/loopback → custom.
func platformModeLabel(source, baseURL string) string {
if processing.IsMockOrLoopbackBaseURL(baseURL) {
return ModeCustomLabel
}
if strings.TrimSpace(source) == platformsettings.SourceEnv {
return ModeCustomLabel
}
return ModeInternalLabel
}
func (s *Service) platformConfigured(ctx context.Context) (bool, error) {
oi, err := s.resolvePlatformOpenAI(ctx)
if err != nil {
apps/api/internal/catalog/clone.go
+13 -1
View File
@@ -9,6 +9,10 @@ import (
"github.com/jackc/pgx/v5/pgxpool"
)
// cloneProcessedFieldSourcesSQL copies field_sources but drops enhance_input_hash
// so the destination's first process cannot hash-skip a thin prior description.
const cloneProcessedFieldSourcesSQL = "COALESCE(pp.field_sources, '{}'::jsonb) - 'enhance_input_hash'"
// CloneResult summarizes a company catalog clone (source unchanged).
type CloneResult struct {
SourceCompanyID uuid.UUID `json:"source_company_id"`
@@ -382,7 +386,7 @@ func cloneCompanyCatalog(ctx context.Context, pool *pgxpool.Pool, src, dest uuid
pp.meta_description,
pp.last_transition_at,
pp.structured_description,
pp.field_sources,
` + cloneProcessedFieldSourcesSQL + `,
pp.created_at,
now(),
COALESCE(pp.ai_provider_mode, 'internal'),
@@ -452,6 +456,14 @@ func cloneCompanyCatalog(ctx context.Context, pool *pgxpool.Pool, src, dest uuid
return nil, fmt.Errorf("reset raw processing flags: %w", err)
}
// Drop skip hashes still present on weak / title-echo localized descriptions
// (field_sources hash is already stripped in the INSERT above).
hashRepair, err := repairWeakEnhanceHashesTx(ctx, tx, dest)
if err != nil {
return nil, fmt.Errorf("repair weak enhance hashes: %w", err)
}
add("weak_enhance_hashes_cleared", hashRepair.Cleared)
if err := tx.Commit(ctx); err != nil {
return nil, err
}
apps/api/internal/catalog/clone_test.go
+12
View File
@@ -1,6 +1,7 @@
package catalog
import (
"strings"
"testing"
"github.com/google/uuid"
@@ -24,3 +25,14 @@ func TestValidateCloneCompanies(t *testing.T) {
t.Fatalf("unexpected error: %v", err)
}
}
func TestCloneFieldSourcesSQLClearsEnhanceHash(t *testing.T) {
t.Parallel()
const want = "enhance_input_hash"
if !strings.Contains(cloneProcessedFieldSourcesSQL, want) {
t.Fatalf("clone field_sources SQL must remove %q; got %q", want, cloneProcessedFieldSourcesSQL)
}
if !strings.Contains(cloneProcessedFieldSourcesSQL, "-") {
t.Fatalf("expected jsonb key removal operator in %q", cloneProcessedFieldSourcesSQL)
}
}
apps/api/internal/catalog/fix_catalog.go
+76
View File
@@ -0,0 +1,76 @@
package catalog
import (
"context"
"fmt"
"github.com/descrybe/descrybe-v2/apps/api/internal/billing"
"github.com/google/uuid"
"github.com/jackc/pgx/v5/pgxpool"
)
// EnsureCategoryAttributeLinks removes orphan category_attributes rows (missing
// category or attribute) without wiping taxonomy or valid links. Returns orphans
// removed and remaining link count.
func EnsureCategoryAttributeLinks(ctx context.Context, pool *pgxpool.Pool, companyID uuid.UUID) (orphansRemoved int, links int, err error) {
if pool == nil {
return 0, 0, fmt.Errorf("nil pool")
}
ct, err := pool.Exec(ctx, `
DELETE FROM category_attributes ca
WHERE ca.company_id = $1
AND (
NOT EXISTS (
SELECT 1 FROM categories c
WHERE c.company_id = ca.company_id AND c.unique_id = ca.category_unique_id
)
OR NOT EXISTS (
SELECT 1 FROM attributes a
WHERE a.id = ca.attribute_id AND a.company_id = ca.company_id
)
)`, companyID)
if err != nil {
return 0, 0, fmt.Errorf("purge orphan category_attributes: %w", err)
}
orphansRemoved = int(ct.RowsAffected())
if err := pool.QueryRow(ctx, `
SELECT count(*)::int FROM category_attributes WHERE company_id = $1`, companyID).Scan(&links); err != nil {
return orphansRemoved, 0, err
}
return orphansRemoved, links, nil
}
// RepairCompanyCategoryEnhancePrompts is the company-scoped variant of
// RepairA1DemoCategoryEnhancePrompts: same repairedCategoryEnhancePromptMap
// (sl + "*" with CategoryEnhanceUserTemplate / {{attrs}}), applied to one company.
// Idempotent — already-OK categories count as already_ok, not updated.
func RepairCompanyCategoryEnhancePrompts(ctx context.Context, pool *pgxpool.Pool, companyID uuid.UUID) (updated, alreadyOK, emptySkipped int, err error) {
if pool == nil {
return 0, 0, 0, fmt.Errorf("nil pool")
}
want, err := repairedCategoryEnhancePromptMap()
if err != nil {
return 0, 0, 0, err
}
summary := &RepairCategoryEnhancePromptsResult{ByCompany: map[string]int{}}
if _, err := repairCompanyCategoryEnhancePrompts(ctx, pool, companyID, companyID.String(), want, false, summary); err != nil {
return 0, 0, 0, err
}
return summary.Updated, summary.AlreadyOK, summary.EmptySkipped, nil
}
// ResolvePlatformDemoCompanyID finds the Platform Demo sandbox (never A1 cohort).
func ResolvePlatformDemoCompanyID(ctx context.Context, pool *pgxpool.Pool) (uuid.UUID, error) {
var id uuid.UUID
err := pool.QueryRow(ctx, `
SELECT c.id
FROM companies c
WHERE c.name = $1
AND COALESCE(c.legacy_company_id, '') <> $2
ORDER BY c.created_at ASC
LIMIT 1`, platformDemoCompanyName, billing.A1LegacyCompanyID).Scan(&id)
if err != nil {
return uuid.Nil, fmt.Errorf("platform demo company not found: %w", err)
}
return id, nil
}
apps/api/internal/catalog/fix_catalog_test.go
+45
View File
@@ -0,0 +1,45 @@
package catalog
import (
"strings"
"testing"
"github.com/descrybe/descrybe-v2/apps/api/internal/aiprompts"
"github.com/descrybe/descrybe-v2/apps/api/internal/company"
)
func TestCategoryEnhanceTemplateHasAttrs(t *testing.T) {
t.Parallel()
tpl := aiprompts.CategoryEnhanceUserTemplate
if !strings.Contains(tpl, "{{attrs}}") {
t.Fatalf("template missing {{attrs}}: %q", tpl)
}
if !aiprompts.CategoryEnhancePromptNeedsRepair("legacy HTML prompt without attrs") {
t.Fatal("expected legacy prompt to need repair")
}
if aiprompts.CategoryEnhancePromptNeedsRepair(tpl) {
t.Fatal("canonical template should not need repair")
}
}
func TestPlatformDemoNameConstant(t *testing.T) {
t.Parallel()
if platformDemoCompanyName != "Platform Demo" {
t.Fatalf("got %q", platformDemoCompanyName)
}
}
func TestRepairedCategoryEnhancePromptMapMatchesA1Demo(t *testing.T) {
t.Parallel()
want, err := repairedCategoryEnhancePromptMap()
if err != nil {
t.Fatal(err)
}
tpl := strings.TrimSpace(aiprompts.CategoryEnhanceUserTemplate)
if want["sl"] != tpl || want[company.LangPromptAny] != tpl {
t.Fatalf("want sl+* template, got %#v", want)
}
if !strings.Contains(tpl, "{{attrs}}") {
t.Fatal("template must include {{attrs}}")
}
}
apps/api/internal/catalog/product_v1_align.go
+66
View File
@@ -0,0 +1,66 @@
package catalog
import (
"strings"
"github.com/descrybe/descrybe-v2/apps/api/internal/eprel"
)
// alignProductFieldsWithV1 mirrors V1 process-item field semantics on dashboard
// product payloads for the same EAN: title/name, preferred description, and
// structured eprel. Mutates item in place after SQL scan / feed-spec linking.
func alignProductFieldsWithV1(item map[string]any) {
if item == nil {
return
}
name := firstNonEmptyString(
item["processed_name"],
item["name"],
item["title"],
)
if name != "" {
item["name"] = name
item["title"] = name
}
desc := firstNonEmptyString(
item["processed_description"],
item["description"],
)
if desc != "" {
item["description"] = desc
}
eprelVal := eprel.ExtractFromAttrs(asStringAnyMap(item["processed_attributes"]))
if eprelVal == nil {
eprelVal = eprel.ExtractFromAttrs(asStringAnyMap(item["attributes"]))
}
if eprelVal == nil {
if mapped, ok := item["mapped_data"].(map[string]any); ok {
eprelVal = eprel.ExtractFromAttrs(mapped)
}
}
item["eprel"] = eprelVal
if eprelVal != nil {
item["has_eprel"] = true
}
}
func firstNonEmptyString(vals ...any) string {
for _, v := range vals {
switch t := v.(type) {
case string:
if s := strings.TrimSpace(t); s != "" {
return s
}
case *string:
if t != nil {
if s := strings.TrimSpace(*t); s != "" {
return s
}
}
}
}
return ""
}
apps/api/internal/catalog/product_v1_align_test.go
+123
View File
@@ -0,0 +1,123 @@
package catalog
import (
"testing"
"github.com/descrybe/descrybe-v2/apps/api/internal/eprel"
)
func TestAlignProductFieldsWithV1_prefersProcessedAndEprel(t *testing.T) {
item := map[string]any{
"name": "RAW ALLCAPS TITLE",
"processed_name": "Retail Title",
"description": "raw feed description",
"processed_description": "AI retail description",
"attributes": map[string]any{
"vzmetenje": "junk",
"brand": "Ostalo",
},
"processed_attributes": map[string]any{
"brand": "Ostalo",
"product_model": "W53070",
"eprel_id": "1632113",
"eprel_label": "https://eprel.example/label",
"eprel_energy_class": "A",
},
"category_name": "Nosilci za TV",
}
alignProductFieldsWithV1(item)
if item["name"] != "Retail Title" {
t.Fatalf("name=%v want Retail Title", item["name"])
}
if item["title"] != "Retail Title" {
t.Fatalf("title=%v want Retail Title", item["title"])
}
if item["description"] != "AI retail description" {
t.Fatalf("description=%v", item["description"])
}
ep, ok := item["eprel"].(map[string]any)
if !ok {
t.Fatalf("eprel type=%T", item["eprel"])
}
if ep["id"] != "1632113" || ep["energy_class"] != "A" {
t.Fatalf("eprel=%v", ep)
}
if item["has_eprel"] != true {
t.Fatalf("has_eprel=%v", item["has_eprel"])
}
// Feed attrs stay available for dual-pane UI; V1 poll uses processed_attributes.
attrs, _ := item["attributes"].(map[string]any)
if attrs["vzmetenje"] != "junk" {
t.Fatalf("feed attributes should remain: %v", attrs)
}
if item["category_name"] != "Nosilci za TV" {
t.Fatalf("category_name mutated")
}
}
func TestAlignProductFieldsWithV1_matchesProcessItemSemantics(t *testing.T) {
// Same underlying row as a V1 process item would project for one EAN.
processedName := "Pralni stroj Samsung WW90CGC04DAELE, 9kg"
processedDesc := "Tehnologija Ecobubble…"
processedAttrs := map[string]any{
"brand": "Samsung",
"product_model": "WW90CGC04DAELE",
"eprel_id": "1632113",
"eprel_label": "https://eprel.example/label",
"eprel_pdf": "https://eprel.example/fiche.pdf",
"eprel_energy_class": "A",
"eprel_energy_scale": "A_G",
}
dashboard := map[string]any{
"name": "RAW FEED TITLE",
"processed_name": processedName,
"description": "raw description",
"processed_description": processedDesc,
"category": "40",
"category_name": "Pralni stroji",
"attributes": map[string]any{"dodatne-informacije": "AI", "brand": "Samsung"},
"processed_attributes": processedAttrs,
"gtin": "8806095210711",
}
alignProductFieldsWithV1(dashboard)
// V1 process item projection (subset used by poll clients).
v1Title := processedName
v1Desc := processedDesc
v1Eprel := eprel.ExtractFromAttrs(processedAttrs)
if dashboard["name"] != v1Title || dashboard["title"] != v1Title {
t.Fatalf("title/name mismatch: dash=%v/%v v1=%v", dashboard["title"], dashboard["name"], v1Title)
}
if dashboard["description"] != v1Desc {
t.Fatalf("description mismatch: dash=%v v1=%v", dashboard["description"], v1Desc)
}
if dashboard["category_name"] != "Pralni stroji" {
t.Fatalf("category_name=%v", dashboard["category_name"])
}
dashEprel, _ := dashboard["eprel"].(map[string]any)
v1Map, _ := v1Eprel.(map[string]any)
if dashEprel["id"] != v1Map["id"] || dashEprel["energy_class"] != v1Map["energy_class"] {
t.Fatalf("eprel mismatch dash=%v v1=%v", dashEprel, v1Map)
}
// V1 attributes ≈ processed_attributes (allowlist/sanitize aside).
pa, _ := dashboard["processed_attributes"].(map[string]any)
if pa["brand"] != "Samsung" || pa["product_model"] != "WW90CGC04DAELE" {
t.Fatalf("processed_attributes=%v", pa)
}
}
func TestAlignProductFieldsWithV1_eprelFromMappedFallback(t *testing.T) {
item := map[string]any{
"processed_name": "Washer",
"mapped_data": map[string]any{
"eprel_id": "1632113",
},
}
alignProductFieldsWithV1(item)
ep, ok := item["eprel"].(map[string]any)
if !ok || ep["id"] != "1632113" {
t.Fatalf("eprel=%v", item["eprel"])
}
}
apps/api/internal/catalog/prompt_repair.go
+220
View File
@@ -0,0 +1,220 @@
package catalog
import (
"context"
"fmt"
"strings"
"github.com/descrybe/descrybe-v2/apps/api/internal/aiprompts"
"github.com/descrybe/descrybe-v2/apps/api/internal/billing"
"github.com/descrybe/descrybe-v2/apps/api/internal/company"
"github.com/google/uuid"
"github.com/jackc/pgx/v5/pgxpool"
)
// Canonical Postgres id for the migrated A1 Slovenija tenant (seed-a1 default).
const a1CompanyID = "604f23a8-b66e-4b21-8b45-0d72b68f4790"
// platformDemoCompanyName matches migrator / seed-demo standalone demo tenant.
const platformDemoCompanyName = "Platform Demo"
// RepairCategoryEnhancePromptsResult is the dry-run / apply summary for
// RepairA1DemoCategoryEnhancePrompts.
type RepairCategoryEnhancePromptsResult struct {
CompaniesScanned int `json:"companies_scanned"`
CategoriesSeen int `json:"categories_seen"`
WouldUpdate int `json:"would_update"`
Updated int `json:"updated"`
AlreadyOK int `json:"already_ok"`
EmptySkipped int `json:"empty_skipped"`
ByCompany map[string]int `json:"by_company"`
DryRun bool `json:"dry_run"`
}
// RepairA1DemoCategoryEnhancePrompts replaces legacy HTML marketing categories.prompt
// values for A1 Slovenija + Platform Demo with aiprompts.CategoryEnhanceUserTemplate.
//
// LOCAL repair only (idempotent):
// - Writes JSON-compatible user overlays under both "sl" (prompt->>'sl') and "*"
// (LangPromptAny) so language stays via {{language}}, not hardcoded-only copy.
// - Touches ONLY categories.prompt — never title_template / description_template
// (unique name/description formulas stay intact; AppendFormulaConstraints encodes
// them as plain-text instructions at enhance render time).
// - dryRun=true: count WouldUpdate only; dryRun=false: apply and set Updated.
//
// Entrypoint: go run ./cmd/repair-category-prompts (-dry-run | -apply).
func RepairA1DemoCategoryEnhancePrompts(ctx context.Context, pool *pgxpool.Pool, dryRun bool) (RepairCategoryEnhancePromptsResult, error) {
out := RepairCategoryEnhancePromptsResult{
DryRun: dryRun,
ByCompany: map[string]int{},
}
if pool == nil {
return out, fmt.Errorf("postgres pool is required")
}
a1ID, err := uuid.Parse(a1CompanyID)
if err != nil {
return out, fmt.Errorf("a1 company id: %w", err)
}
companyRows, err := pool.Query(ctx, `
SELECT id, name
FROM companies
WHERE id = $1
OR name = $2
OR COALESCE(legacy_company_id, '') = $3
ORDER BY name`, a1ID, platformDemoCompanyName, billing.A1LegacyCompanyID)
if err != nil {
return out, fmt.Errorf("list target companies: %w", err)
}
defer companyRows.Close()
type co struct {
id uuid.UUID
name string
}
companies := make([]co, 0, 2)
for companyRows.Next() {
var c co
if err := companyRows.Scan(&c.id, &c.name); err != nil {
return out, err
}
companies = append(companies, c)
}
if err := companyRows.Err(); err != nil {
return out, err
}
out.CompaniesScanned = len(companies)
if len(companies) == 0 {
return out, fmt.Errorf("no A1 / Platform Demo companies found")
}
want, err := repairedCategoryEnhancePromptMap()
if err != nil {
return out, err
}
for _, c := range companies {
n, err := repairCompanyCategoryEnhancePrompts(ctx, pool, c.id, c.name, want, dryRun, &out)
if err != nil {
return out, err
}
if n > 0 {
out.ByCompany[c.name] = n
}
}
return out, nil
}
// repairedCategoryEnhancePromptMap is the canonical stored shape: "sl" (prompt->>'sl'
// for A1/Demo) plus LangPromptAny ("*") so PromptForLanguage resolves for any content
// language. Copy stays language-agnostic via {{language}} — not hardcoded Slovenian.
func repairedCategoryEnhancePromptMap() (company.LangPromptMap, error) {
tpl := strings.TrimSpace(aiprompts.CategoryEnhanceUserTemplate)
if tpl == "" {
return nil, fmt.Errorf("CategoryEnhanceUserTemplate is empty")
}
return company.LangPromptMap{
"sl": tpl,
company.LangPromptAny: tpl,
}, nil
}
func categoryEnhancePromptMapOK(m company.LangPromptMap, want company.LangPromptMap) bool {
if !company.HasAnyPrompt(m) || !company.HasAnyPrompt(want) {
return false
}
tpl := strings.TrimSpace(want[company.LangPromptAny])
if tpl == "" {
tpl = strings.TrimSpace(want["sl"])
}
if tpl == "" {
return false
}
// Accept already-repaired maps: every non-empty value equals the shared template
// and LangPromptAny (or legacy sl-only) is present.
hasKey := false
for lang, p := range m {
p = strings.TrimSpace(p)
if p == "" {
continue
}
if p != tpl {
return false
}
if lang == company.LangPromptAny || lang == "sl" {
hasKey = true
}
}
return hasKey
}
func repairCompanyCategoryEnhancePrompts(
ctx context.Context,
pool *pgxpool.Pool,
companyID uuid.UUID,
companyName string,
want company.LangPromptMap,
dryRun bool,
out *RepairCategoryEnhancePromptsResult,
) (int, error) {
rows, err := pool.Query(ctx, `
SELECT id, COALESCE(prompt, '{}'::jsonb)
FROM categories
WHERE company_id = $1`, companyID)
if err != nil {
return 0, fmt.Errorf("list categories for %s: %w", companyName, err)
}
defer rows.Close()
updatedHere := 0
for rows.Next() {
var id uuid.UUID
var raw []byte
if err := rows.Scan(&id, &raw); err != nil {
return updatedHere, err
}
out.CategoriesSeen++
m, err := company.DecodeLangPromptMap(raw)
if err != nil {
return updatedHere, fmt.Errorf("decode prompt category=%s company=%s: %w", id, companyName, err)
}
if !company.HasAnyPrompt(m) {
out.EmptySkipped++
continue
}
if categoryEnhancePromptMapOK(m, want) {
out.AlreadyOK++
continue
}
out.WouldUpdate++
if dryRun {
updatedHere++
continue
}
cleaned, err := company.SanitizeLangPromptMap(want, MaxCategoryPromptRunes)
if err != nil {
return updatedHere, fmt.Errorf("sanitize prompt category=%s: %w", id, err)
}
encoded, err := company.EncodeLangPromptMap(cleaned)
if err != nil {
return updatedHere, err
}
ct, err := pool.Exec(ctx, `
UPDATE categories
SET prompt = $3::jsonb, updated_at = now()
WHERE id = $1 AND company_id = $2`, id, companyID, string(encoded))
if err != nil {
return updatedHere, fmt.Errorf("update prompt category=%s: %w", id, err)
}
if ct.RowsAffected() == 0 {
return updatedHere, fmt.Errorf("category %s not updated (company mismatch?)", id)
}
out.Updated++
updatedHere++
}
return updatedHere, rows.Err()
}
apps/api/internal/catalog/prompt_repair_test.go
+52
View File
@@ -0,0 +1,52 @@
package catalog
import (
"strings"
"testing"
"github.com/descrybe/descrybe-v2/apps/api/internal/aiprompts"
"github.com/descrybe/descrybe-v2/apps/api/internal/company"
)
func TestRepairedCategoryEnhancePromptMap(t *testing.T) {
t.Parallel()
want, err := repairedCategoryEnhancePromptMap()
if err != nil {
t.Fatal(err)
}
tpl := strings.TrimSpace(aiprompts.CategoryEnhanceUserTemplate)
if want["sl"] != tpl {
t.Fatalf("want prompt->>'sl' = shared template")
}
if want[company.LangPromptAny] != tpl {
t.Fatalf("want * = shared template")
}
if !categoryEnhancePromptMapOK(want, want) {
t.Fatal("canonical map should be OK")
}
legacy := company.LangPromptMap{"sl": "<H2>legacy HTML marketing</H2>"}
if categoryEnhancePromptMapOK(legacy, want) {
t.Fatal("legacy HTML must need repair")
}
slOnly := company.LangPromptMap{"sl": tpl}
if !categoryEnhancePromptMapOK(slOnly, want) {
t.Fatal("sl-only repaired map should be OK (idempotent)")
}
starOnly := company.LangPromptMap{company.LangPromptAny: tpl}
if !categoryEnhancePromptMapOK(starOnly, want) {
t.Fatal("*-only repaired map should be OK (idempotent)")
}
}
func TestRepairTargetsUseSharedTemplate(t *testing.T) {
t.Parallel()
tpl := strings.TrimSpace(aiprompts.CategoryEnhanceUserTemplate)
for _, v := range []string{"name", "description", "attrs", "category", "language"} {
if !strings.Contains(tpl, "{{"+v+"}}") {
t.Fatalf("shared template missing {{%s}}", v)
}
}
if a1CompanyID == "" || platformDemoCompanyName == "" {
t.Fatal("missing company target constants")
}
}
apps/api/internal/catalog/raw_v1.go
+9 -5
View File
@@ -161,12 +161,11 @@ func ExtractProductImages(mapped, raw map[string]any) (main string, more []strin
break
}
}
if imgs, ok := merged["images"].([]any); ok && len(imgs) > 0 {
urls := coerceToURLList(imgs)
if main == "" && len(urls) > 0 {
if urls := coerceToURLList(merged["images"]); len(urls) > 0 {
if main == "" {
main = urls[0]
urls = urls[1:]
} else if len(urls) > 0 && urls[0] == main {
} else if urls[0] == main {
urls = urls[1:]
}
for _, u := range urls {
@@ -175,6 +174,11 @@ func ExtractProductImages(mapped, raw map[string]any) (main string, more []strin
}
}
}
// Feeds often leave main_image empty while moreimages/images hold URLs.
if main == "" && len(more) > 0 {
main = more[0]
more = more[1:]
}
if main != "" {
filtered := more[:0]
for _, u := range more {
@@ -211,7 +215,7 @@ func coerceToURLString(value any) string {
}
return ""
case map[string]any:
for _, k := range []string{"#text", "__cdata", "@_href", "@_url", "href", "url"} {
for _, k := range []string{"#text", "__cdata", "@_href", "@_url", "href", "url", "src", "link", "image"} {
if u := coerceToURLString(v[k]); u != "" {
return u
}
apps/api/internal/catalog/raw_v1_test.go
+35
View File
@@ -54,3 +54,38 @@ func TestBuildMappedDataFromV1ItemOmitsEmptyNulls(t *testing.T) {
t.Fatal("EAN-only must not count as content")
}
}
func TestExtractProductImages_promotesMoreimagesWhenMainEmpty(t *testing.T) {
main, more := ExtractProductImages(map[string]any{
"main_image": "",
"moreimages": "https://cdn.example.com/a.jpg,https://cdn.example.com/b.jpg",
}, nil)
if main != "https://cdn.example.com/a.jpg" {
t.Fatalf("main=%q", main)
}
if len(more) != 1 || more[0] != "https://cdn.example.com/b.jpg" {
t.Fatalf("more=%v", more)
}
}
func TestExtractProductImages_imagesStringSliceAndSrcObjects(t *testing.T) {
main, more := ExtractProductImages(map[string]any{
"images": []string{
"https://cdn.example.com/main.jpg",
"https://cdn.example.com/2.jpg",
},
}, nil)
if main != "https://cdn.example.com/main.jpg" || len(more) != 1 || more[0] != "https://cdn.example.com/2.jpg" {
t.Fatalf("string slice main=%q more=%v", main, more)
}
main, more = ExtractProductImages(map[string]any{
"images": []any{
map[string]any{"src": "https://cdn.example.com/from-src.jpg"},
map[string]any{"src": "https://cdn.example.com/from-src-2.jpg"},
},
}, nil)
if main != "https://cdn.example.com/from-src.jpg" || len(more) != 1 || more[0] != "https://cdn.example.com/from-src-2.jpg" {
t.Fatalf("src objects main=%q more=%v", main, more)
}
}
apps/api/internal/catalog/repair_enhance_hashes.go
+188
View File
@@ -0,0 +1,188 @@
package catalog
import (
"context"
"encoding/json"
"fmt"
"strings"
"github.com/descrybe/descrybe-v2/apps/api/internal/company"
"github.com/google/uuid"
"github.com/jackc/pgx/v5"
"github.com/jackc/pgx/v5/pgxpool"
)
const enhanceInputHashKey = "enhance_input_hash"
// RepairWeakEnhanceHashesResult is the detailed report for weak-hash clearing.
type RepairWeakEnhanceHashesResult struct {
Cleared int64
Scanned int
ClearedRawIDs []uuid.UUID
}
// RepairWeakEnhanceHashes clears enhance_input_hash from field_sources and
// localized_content for processed products whose descriptions are weak /
// title-echo / filler (so the next enhance cannot hash-skip thin priors).
// Returns the number of products updated.
func RepairWeakEnhanceHashes(ctx context.Context, pool *pgxpool.Pool, companyID uuid.UUID) (int64, error) {
res, err := RepairWeakEnhanceHashesDetailed(ctx, pool, companyID)
return res.Cleared, err
}
// RepairWeakEnhanceHashesDetailed is RepairWeakEnhanceHashes plus scan/raw-id detail
// for admin Fix A1 hygiene reporting.
func RepairWeakEnhanceHashesDetailed(ctx context.Context, pool *pgxpool.Pool, companyID uuid.UUID) (RepairWeakEnhanceHashesResult, error) {
var out RepairWeakEnhanceHashesResult
if pool == nil {
return out, fmt.Errorf("catalog pool not configured")
}
if companyID == uuid.Nil {
return out, ClientMsg("company_id is required")
}
tx, err := pool.Begin(ctx)
if err != nil {
return out, err
}
defer tx.Rollback(ctx)
out, err = repairWeakEnhanceHashesTx(ctx, tx, companyID)
if err != nil {
return out, err
}
if err := tx.Commit(ctx); err != nil {
return out, err
}
return out, nil
}
// RepairWeakEnhanceHashes clears weak enhance skip hashes for the company.
func (s *Service) RepairWeakEnhanceHashes(ctx context.Context, companyID uuid.UUID) (int64, error) {
if s == nil || s.Pool == nil {
return 0, fmt.Errorf("catalog service not configured")
}
return RepairWeakEnhanceHashes(ctx, s.Pool, companyID)
}
func repairWeakEnhanceHashesTx(ctx context.Context, tx pgx.Tx, companyID uuid.UUID) (RepairWeakEnhanceHashesResult, error) {
var out RepairWeakEnhanceHashesResult
rows, err := tx.Query(ctx, `
SELECT id, raw_product_id,
COALESCE(name, ''),
COALESCE(description, ''),
COALESCE(processed_name, ''),
COALESCE(processed_description, ''),
COALESCE(field_sources, '{}'::jsonb),
COALESCE(localized_content, '{}'::jsonb)
FROM processed_products
WHERE company_id = $1`, companyID)
if err != nil {
return out, fmt.Errorf("list processed products: %w", err)
}
defer rows.Close()
type pending struct {
id uuid.UUID
rawID uuid.UUID
fs []byte
loc []byte
}
var updates []pending
for rows.Next() {
out.Scanned++
var (
id, rawID uuid.UUID
name, desc, processedName, processedDesc string
fsRaw, locRaw []byte
)
if err := rows.Scan(&id, &rawID, &name, &desc, &processedName, &processedDesc, &fsRaw, &locRaw); err != nil {
return out, err
}
fs := map[string]any{}
if len(fsRaw) > 0 {
if err := json.Unmarshal(fsRaw, &fs); err != nil {
return out, err
}
}
if fs == nil {
fs = map[string]any{}
}
loc, err := company.DecodeLocalizedContent(locRaw)
if err != nil {
return out, err
}
changed := false
primaryDesc := strings.TrimSpace(processedDesc)
if primaryDesc == "" {
primaryDesc = desc
}
primaryName := strings.TrimSpace(processedName)
if primaryName == "" {
primaryName = name
}
if _, has := fs[enhanceInputHashKey]; has {
if company.IsWeakPriorEnhanceDescription(primaryDesc, primaryName, name) {
delete(fs, enhanceInputHashKey)
changed = true
}
}
for lang, fields := range loc {
d := strings.TrimSpace(fields.ProcessedDescription)
n := strings.TrimSpace(fields.ProcessedName)
if n == "" {
n = primaryName
}
if strings.TrimSpace(fields.EnhanceInputHash) == "" {
continue
}
if company.IsWeakPriorEnhanceDescription(d, n, primaryName, name) {
fields.EnhanceInputHash = ""
loc[lang] = fields
changed = true
}
}
if !changed {
continue
}
fsBytes, err := json.Marshal(fs)
if err != nil {
return out, err
}
locBytes, err := company.EncodeLocalizedContent(loc)
if err != nil {
return out, err
}
updates = append(updates, pending{id: id, rawID: rawID, fs: fsBytes, loc: locBytes})
}
if err := rows.Err(); err != nil {
return out, err
}
rows.Close()
seenRaw := map[uuid.UUID]struct{}{}
for _, u := range updates {
ct, err := tx.Exec(ctx, `
UPDATE processed_products
SET field_sources = $2::jsonb,
localized_content = $3::jsonb,
updated_at = now()
WHERE id = $1 AND company_id = $4`,
u.id, u.fs, u.loc, companyID)
if err != nil {
return out, fmt.Errorf("clear enhance hashes product=%s: %w", u.id, err)
}
if ct.RowsAffected() > 0 {
out.Cleared++
if _, ok := seenRaw[u.rawID]; !ok {
seenRaw[u.rawID] = struct{}{}
out.ClearedRawIDs = append(out.ClearedRawIDs, u.rawID)
}
}
}
return out, nil
}
apps/api/internal/catalog/repair_enhance_hashes_test.go
+137
View File
@@ -0,0 +1,137 @@
package catalog
import (
"context"
"encoding/json"
"os"
"strings"
"testing"
"github.com/descrybe/descrybe-v2/apps/api/internal/company"
"github.com/google/uuid"
"github.com/jackc/pgx/v5/pgxpool"
)
func TestClearWeakEnhanceHashesDecision(t *testing.T) {
t.Parallel()
title := "Acme Widget Pro 24"
weak := title
strong := "Acme Widget Pro 24 is a durable retail widget built for everyday warehouse use with a reinforced frame."
if !company.IsWeakPriorEnhanceDescription(weak, title) {
t.Fatal("title-echo must be weak")
}
if company.IsWeakPriorEnhanceDescription(strong, title) {
t.Fatal("strong description must not be weak")
}
}
func TestRepairWeakEnhanceHashes_nilPool(t *testing.T) {
t.Parallel()
_, err := RepairWeakEnhanceHashes(context.Background(), nil, uuid.New())
if err == nil {
t.Fatal("expected error for nil pool")
}
_, err = RepairWeakEnhanceHashes(context.Background(), &pgxpool.Pool{}, uuid.Nil)
if err == nil {
t.Fatal("expected error for nil company")
}
}
func TestRepairWeakEnhanceHashes_integration(t *testing.T) {
dsn := strings.TrimSpace(os.Getenv("DATABASE_URL"))
if dsn == "" {
t.Skip("DATABASE_URL not set")
}
ctx := context.Background()
pool, err := pgxpool.New(ctx, dsn)
if err != nil {
t.Fatalf("connect: %v", err)
}
defer pool.Close()
var companyID uuid.UUID
err = pool.QueryRow(ctx, `
SELECT id FROM companies
WHERE lower(name) IN ('platform demo', 'demo')
ORDER BY CASE WHEN lower(name) = 'platform demo' THEN 0 ELSE 1 END
LIMIT 1`).Scan(&companyID)
if err != nil {
t.Skipf("Platform Demo company not found: %v", err)
}
title := "Repair Hash Probe Widget X"
weakDesc := title
hash := "deadbeefcafebabe0123456789abcdef0123456789abcdef0123456789abcdef"
loc := company.LocalizedContent{
"en": {
ProcessedName: title,
ProcessedDescription: weakDesc,
EnhanceInputHash: hash,
},
}
locJSON, err := company.EncodeLocalizedContent(loc)
if err != nil {
t.Fatalf("encode loc: %v", err)
}
fs, _ := json.Marshal(map[string]any{enhanceInputHashKey: hash, "name": "ai_enhance"})
tx, err := pool.Begin(ctx)
if err != nil {
t.Fatalf("begin: %v", err)
}
defer tx.Rollback(ctx)
var rawID, ppID uuid.UUID
gtin := "8700999111222"
err = tx.QueryRow(ctx, `
INSERT INTO raw_products (company_id, gtin, raw_data, mapped_data, is_processed, processing_status)
VALUES ($1, $2, '{}'::jsonb, '{}'::jsonb, true, 'processed')
RETURNING id`, companyID, gtin).Scan(&rawID)
if err != nil {
t.Fatalf("insert raw: %v", err)
}
err = tx.QueryRow(ctx, `
INSERT INTO processed_products (
company_id, raw_product_id, product_id, name, processed_name, description, processed_description,
status, attributes, processed_attributes, gpt_response, field_sources, localized_content, ai_provider_mode
) VALUES (
$1, $2, $3, $4, $4, $5, $5, 'needs_review', '{}'::jsonb, '{}'::jsonb, '{}'::jsonb, $6::jsonb, $7::jsonb, 'internal'
) RETURNING id`, companyID, rawID, gtin, title, weakDesc, fs, locJSON).Scan(&ppID)
if err != nil {
t.Fatalf("insert processed: %v", err)
}
res, err := repairWeakEnhanceHashesTx(ctx, tx, companyID)
if err != nil {
t.Fatalf("repair: %v", err)
}
if res.Cleared < 1 {
t.Fatalf("expected at least 1 cleared, got %d (scanned=%d)", res.Cleared, res.Scanned)
}
var fsOut, locOut []byte
err = tx.QueryRow(ctx, `
SELECT COALESCE(field_sources, '{}'::jsonb), COALESCE(localized_content, '{}'::jsonb)
FROM processed_products WHERE id = $1`, ppID).Scan(&fsOut, &locOut)
if err != nil {
t.Fatalf("reload: %v", err)
}
var fsMap map[string]any
_ = json.Unmarshal(fsOut, &fsMap)
if _, ok := fsMap[enhanceInputHashKey]; ok {
t.Fatalf("field_sources hash still present: %v", fsMap[enhanceInputHashKey])
}
gotLoc, err := company.DecodeLocalizedContent(locOut)
if err != nil {
t.Fatalf("decode loc: %v", err)
}
if h := gotLoc["en"].EnhanceInputHash; h != "" {
t.Fatalf("localized hash still present: %q", h)
}
if gotLoc["en"].ProcessedDescription != weakDesc {
t.Fatalf("description should be preserved, got %q", gotLoc["en"].ProcessedDescription)
}
// Roll back probe rows — do not leave smoke data on Platform Demo.
}
apps/api/internal/catalog/service.go
+33 -12
View File
@@ -260,7 +260,7 @@ func enrichCategoryPrompts(ctx context.Context, pool *pgxpool.Pool, companyID uu
}
primary := company.LoadLanguage(ctx, pool, companyID)
item["prompts"] = prompts
item["prompt"] = company.PromptForLanguage(prompts, primary)
item["prompt"] = company.PromptForLanguage(prompts, primary, primary)
item["has_prompt"] = company.HasAnyPrompt(prompts)
return item, nil
}
@@ -668,8 +668,11 @@ func processedProductsCountFromSQL(needsRawJoin bool) string {
// Enrichment coverage SQL predicates (alias p = processed_products, r = raw_products).
const (
processedHasNameSQL = `(COALESCE(NULLIF(p.processed_name, ''), NULLIF(p.name, ''), NULLIF(r.mapped_data->>'name', ''), NULLIF(r.mapped_data->>'title', ''), '') <> '')`
processedHasDescriptionSQL = `(COALESCE(NULLIF(p.processed_description, ''), NULLIF(p.description, ''), NULLIF(r.mapped_data->>'description', ''), '') <> '')`
// Prefer processed_* so dashboard product fields match V1 process items for the same EAN.
processedPreferredNameSQL = `COALESCE(NULLIF(p.processed_name, ''), NULLIF(p.name, ''), NULLIF(r.mapped_data->>'name', ''), NULLIF(r.mapped_data->>'title', ''), '')`
processedPreferredDescriptionSQL = `COALESCE(NULLIF(p.processed_description, ''), NULLIF(p.description, ''), NULLIF(r.mapped_data->>'description', ''), '')`
processedHasNameSQL = `(` + processedPreferredNameSQL + ` <> '')`
processedHasDescriptionSQL = `(` + processedPreferredDescriptionSQL + ` <> '')`
processedHasCategorySQL = `(COALESCE(NULLIF(p.category, ''), '') <> '' AND lower(p.category) <> 'none')`
// Resolve display name / canonical unique_id when products store unique_id, UUID id, or name.
processedCategoryResolveJoin = `
@@ -1093,7 +1096,7 @@ func (s *Service) ListProcessedProducts(ctx context.Context, companyID uuid.UUID
func(ctx context.Context) ([]map[string]any, error) {
rows, err := s.Pool.Query(ctx, fmt.Sprintf(`
SELECT p.id, p.product_id,
COALESCE(NULLIF(p.name, ''), NULLIF(r.mapped_data->>'name', ''), NULLIF(r.mapped_data->>'title', ''), '') AS name,
`+processedPreferredNameSQL+` AS name,
COALESCE(NULLIF(p.processed_name, ''), '') AS processed_name,
p.category,
COALESCE(cat.name, NULLIF(BTRIM(p.category), '')) AS category_name,
@@ -1102,9 +1105,9 @@ func (s *Service) ListProcessedProducts(ctx context.Context, companyID uuid.UUID
f.name AS feed_name,
f.last_synced_at AS feed_last_synced_at,
r.updated_at AS raw_updated_at,
(COALESCE(NULLIF(p.processed_name, ''), NULLIF(p.name, ''), NULLIF(r.mapped_data->>'name', ''), NULLIF(r.mapped_data->>'title', ''), '') <> '') AS has_name,
(`+processedPreferredNameSQL+` <> '') AS has_name,
(COALESCE(NULLIF(p.processed_name, ''), '') <> '') AS has_processed_name,
(COALESCE(NULLIF(p.processed_description, ''), NULLIF(p.description, ''), NULLIF(r.mapped_data->>'description', ''), '') <> '') AS has_description,
(`+processedPreferredDescriptionSQL+` <> '') AS has_description,
(COALESCE(NULLIF(p.processed_description, ''), '') <> '') AS has_processed_description,
(COALESCE(NULLIF(p.category, ''), '') <> '' AND lower(p.category) <> 'none') AS has_category,
`+processedHasAttributesSQL+` AS has_attributes,
@@ -1120,7 +1123,7 @@ func (s *Service) ListProcessedProducts(ctx context.Context, companyID uuid.UUID
return nil, err
}
defer rows.Close()
return scanMaps(rows, []string{
items, err := scanMaps(rows, []string{
"id", "product_id", "name", "processed_name", "category", "category_name", "category_unique_id",
"status", "raw_product_id", "feed_id", "gtin",
"feed_name", "feed_last_synced_at", "raw_updated_at",
@@ -1128,6 +1131,15 @@ func (s *Service) ListProcessedProducts(ctx context.Context, companyID uuid.UUID
"has_eprel",
"created_at", "updated_at",
})
if err != nil {
return nil, err
}
for _, item := range items {
if name, ok := item["name"].(string); ok && strings.TrimSpace(name) != "" {
item["title"] = name
}
}
return items, nil
},
)
}
@@ -1172,12 +1184,12 @@ func (s *Service) ListProcessedProductsDetailed(ctx context.Context, companyID u
func(ctx context.Context) ([]map[string]any, error) {
rows, err := s.Pool.Query(ctx, fmt.Sprintf(`
SELECT p.id, p.product_id,
COALESCE(NULLIF(p.name, ''), NULLIF(r.mapped_data->>'name', ''), NULLIF(r.mapped_data->>'title', ''), '') AS name,
`+processedPreferredNameSQL+` AS name,
p.category,
COALESCE(cat.name, NULLIF(BTRIM(p.category), '')) AS category_name,
COALESCE(cat.unique_id, NULLIF(BTRIM(p.category), '')) AS category_unique_id,
p.status, p.raw_product_id, p.feed_id, r.gtin,
COALESCE(NULLIF(p.description, ''), NULLIF(r.mapped_data->>'description', ''), '') AS description,
`+processedPreferredDescriptionSQL+` AS description,
p.processed_name, p.processed_description,
COALESCE(p.meta_title, ''), COALESCE(p.meta_description, ''),
p.attributes, p.processed_attributes, r.mapped_data,
@@ -1190,7 +1202,7 @@ func (s *Service) ListProcessedProductsDetailed(ctx context.Context, companyID u
return nil, err
}
defer rows.Close()
return scanMaps(rows, []string{
items, err := scanMaps(rows, []string{
"id", "product_id", "name", "category", "category_name", "category_unique_id",
"status", "raw_product_id", "feed_id", "gtin",
"description", "processed_name", "processed_description",
@@ -1198,6 +1210,13 @@ func (s *Service) ListProcessedProductsDetailed(ctx context.Context, companyID u
"attributes", "processed_attributes", "mapped_data",
"created_at", "updated_at",
})
if err != nil {
return nil, err
}
for _, item := range items {
alignProductFieldsWithV1(item)
}
return items, nil
},
)
}
@@ -1205,11 +1224,11 @@ func (s *Service) ListProcessedProductsDetailed(ctx context.Context, companyID u
func (s *Service) GetProcessedProduct(ctx context.Context, companyID, id uuid.UUID) (map[string]any, error) {
row := s.Pool.QueryRow(ctx, `
SELECT p.id, p.product_id,
COALESCE(NULLIF(p.name, ''), NULLIF(r.mapped_data->>'name', ''), NULLIF(r.mapped_data->>'title', ''), '') AS name,
`+processedPreferredNameSQL+` AS name,
p.category,
COALESCE(cat.name, NULLIF(BTRIM(p.category), '')) AS category_name,
COALESCE(cat.unique_id, NULLIF(BTRIM(p.category), '')) AS category_unique_id,
COALESCE(NULLIF(p.description, ''), NULLIF(r.mapped_data->>'description', ''), '') AS description,
`+processedPreferredDescriptionSQL+` AS description,
p.processed_name, p.processed_description,
p.status, p.attributes, p.processed_attributes, r.gtin, r.mapped_data,
COALESCE(p.feed_id, r.feed_id) AS feed_id,
@@ -1241,6 +1260,7 @@ func (s *Service) GetProcessedProduct(ctx context.Context, companyID, id uuid.UU
return nil, err
}
linkFeedSpecificationsIntoProduct(item)
alignProductFieldsWithV1(item)
primary := company.LoadLanguage(ctx, s.Pool, companyID)
item["content_language"] = primary
item["content_languages"] = company.LoadContentLanguages(ctx, s.Pool, companyID)
@@ -1303,6 +1323,7 @@ func (s *Service) GetRawProduct(ctx context.Context, companyID, id uuid.UUID) (m
return nil, err
}
linkFeedSpecificationsIntoProduct(item)
alignProductFieldsWithV1(item)
primary := company.LoadLanguage(ctx, s.Pool, companyID)
item["content_language"] = primary
item["content_languages"] = company.LoadContentLanguages(ctx, s.Pool, companyID)
apps/api/internal/company/lang_content.go
+44 -8
View File
@@ -11,7 +11,12 @@ import (
"github.com/jackc/pgx/v5/pgxpool"
)
// LangPromptAny is the wildcard key for a single shared prompt used for every
// content language (avoid duplicating the same text per lang).
const LangPromptAny = "*"
// LangPromptMap is language-code → prompt text for category / template overrides.
// Optional key LangPromptAny ("*") is a shared overlay for all languages.
type LangPromptMap map[string]string
// LocalizedFields holds AI/output fields for one content language.
@@ -27,15 +32,16 @@ type LocalizedFields struct {
type LocalizedContent map[string]LocalizedFields
// SanitizeLangPromptMap validates language codes, sanitizes prompts, and drops empties.
// Accepts LangPromptAny ("*") as a shared any-language prompt key.
func SanitizeLangPromptMap(in map[string]string, maxRunes int) (LangPromptMap, error) {
out := make(LangPromptMap)
if len(in) == 0 {
return out, nil
}
for lang, prompt := range in {
code, err := ParseLanguage(lang, false)
code, err := parseLangPromptKey(lang)
if err != nil {
return nil, fmt.Errorf("unsupported language %q", lang)
return nil, err
}
p := strings.TrimSpace(security.SanitizePrompt(prompt, maxRunes))
if p == "" {
@@ -46,16 +52,46 @@ func SanitizeLangPromptMap(in map[string]string, maxRunes int) (LangPromptMap, e
return out, nil
}
// PromptForLanguage returns the prompt for lang, or empty if unset.
func PromptForLanguage(m LangPromptMap, lang string) string {
func parseLangPromptKey(raw string) (string, error) {
key := strings.TrimSpace(raw)
if key == LangPromptAny {
return LangPromptAny, nil
}
code, err := ParseLanguage(key, false)
if err != nil {
return "", fmt.Errorf("unsupported language %q", raw)
}
return code, nil
}
// PromptForLanguage resolves a prompt with fallback:
// requested lang → LangPromptAny ("*") → primary → "" (caller may then use built-in).
// Empty primary skips the primary step. Does not invent cross-lang text beyond this chain.
func PromptForLanguage(m LangPromptMap, lang, primary string) string {
if len(m) == 0 {
return ""
}
code, err := ParseLanguage(lang, true)
if err != nil {
code = DefaultLanguage
try := func(code string) string {
code = strings.TrimSpace(code)
if code == "" {
return ""
}
return strings.TrimSpace(m[code])
}
return strings.TrimSpace(m[code])
if code, err := ParseLanguage(lang, true); err == nil {
if p := try(code); p != "" {
return p
}
}
if p := try(LangPromptAny); p != "" {
return p
}
if prim, err := ParseLanguage(primary, false); err == nil {
if p := try(prim); p != "" {
return p
}
}
return ""
}
// HasAnyPrompt reports whether any language has a non-empty prompt.
apps/api/internal/company/lang_content_test.go
+25 -3
View File
@@ -17,6 +17,7 @@ func TestSanitizeLangPromptMap(t *testing.T) {
m, err := SanitizeLangPromptMap(map[string]string{
"SL": " hello {{name}} ",
"en": "",
"*": " shared ",
}, 100)
if err != nil {
t.Fatal(err)
@@ -27,16 +28,37 @@ func TestSanitizeLangPromptMap(t *testing.T) {
if _, ok := m["en"]; ok {
t.Fatalf("empty en should be dropped: %#v", m)
}
if m[LangPromptAny] != "shared" {
t.Fatalf("wildcard missing: %#v", m)
}
}
func TestPromptForLanguage(t *testing.T) {
t.Parallel()
m := LangPromptMap{"sl": "slo", "en": "eng"}
if got := PromptForLanguage(m, "SL"); got != "slo" {
if got := PromptForLanguage(m, "SL", ""); got != "slo" {
t.Fatalf("got %q", got)
}
if got := PromptForLanguage(m, "de"); got != "" {
t.Fatalf("expected empty, got %q", got)
if got := PromptForLanguage(m, "de", ""); got != "" {
t.Fatalf("expected empty without primary, got %q", got)
}
// sl-only map: exact hit for sl
slOnly := LangPromptMap{"sl": "slo-only"}
if got := PromptForLanguage(slOnly, "sl", "sl"); got != "slo-only" {
t.Fatalf("sl exact: got %q", got)
}
// sl-only map: en request falls back to primary=sl
if got := PromptForLanguage(slOnly, "en", "sl"); got != "slo-only" {
t.Fatalf("en→primary sl: got %q", got)
}
// wildcard before primary
anyMap := LangPromptMap{"*": "shared", "sl": "slo"}
if got := PromptForLanguage(anyMap, "de", "sl"); got != "shared" {
t.Fatalf("wildcard before primary: got %q", got)
}
// explicit lang beats wildcard
if got := PromptForLanguage(anyMap, "sl", "en"); got != "slo" {
t.Fatalf("explicit beats *: got %q", got)
}
}
apps/api/internal/company/weak_desc.go
+151
View File
@@ -0,0 +1,151 @@
package company
import "strings"
// minUsableProductDescRunes is the floor for a prior description that may hash-skip
// the enhance LLM. Shorter copy is treated as thin and forced through enhance.
const minUsableProductDescRunes = 40
// shortBoilerplateDescRunes: short copy without product-fact token overlap is weak
// even when longer than minUsableProductDescRunes (generic one-liners).
const shortBoilerplateDescRunes = 96
// weakFillerPhrases are known heuristic / placeholder snippets that must never
// hash-skip enhance (mock-llm / invent fallbacks historically produced these).
var weakFillerPhrases = []string{
"ready for retail listing",
"quality product ready",
"product description",
"based on available specifications",
"with available specifications",
"available catalog details",
"pripravljeno za prodajo",
"na podlagi razpoložljivih specifikacij",
}
// IsWeakPriorEnhanceDescription reports empty, too-short, title-echo, known filler,
// or short boilerplate without overlapping tokens from title/fact sources.
func IsWeakPriorEnhanceDescription(priorDesc string, titles ...string) bool {
priorDesc = strings.TrimSpace(priorDesc)
if priorDesc == "" || priorDesc == "<nil>" {
return true
}
if len([]rune(priorDesc)) < minUsableProductDescRunes {
return true
}
for _, title := range titles {
if DescriptionEchoesTitle(priorDesc, title) {
return true
}
}
if ContainsWeakFillerPhrase(priorDesc) {
return true
}
if len([]rune(priorDesc)) <= shortBoilerplateDescRunes &&
!descriptionOverlapsProductFacts(priorDesc, titles...) {
return true
}
return false
}
// ContainsWeakFillerPhrase reports known placeholder / invent-fallback snippets.
func ContainsWeakFillerPhrase(desc string) bool {
lower := strings.ToLower(desc)
for _, p := range weakFillerPhrases {
if p != "" && strings.Contains(lower, p) {
return true
}
}
return false
}
var weakDescStopTokens = map[string]struct{}{
"a": {}, "an": {}, "the": {}, "and": {}, "or": {}, "of": {}, "in": {}, "on": {}, "for": {},
"to": {}, "with": {}, "from": {}, "by": {}, "is": {}, "are": {}, "this": {}, "that": {},
"product": {}, "products": {}, "category": {}, "description": {}, "specs": {}, "spec": {},
"key": {}, "je": {}, "v": {}, "za": {}, "z": {}, "iz": {}, "ter": {}, "ali": {},
}
func significantDescTokens(s string) map[string]struct{} {
s = strings.ToLower(s)
out := map[string]struct{}{}
var b strings.Builder
flush := func() {
tok := b.String()
b.Reset()
if len(tok) < 3 {
return
}
if _, stop := weakDescStopTokens[tok]; stop {
return
}
out[tok] = struct{}{}
}
for _, r := range s {
if (r >= 'a' && r <= 'z') || (r >= '0' && r <= '9') || r == '_' {
b.WriteRune(r)
continue
}
// Keep Latin-extended letters (Slovenian čšž etc.) via unicode letter-ish: non-space punctuation splits.
if r > 127 && ((r >= 'à' && r <= 'ÿ') || r == 'č' || r == 'š' || r == 'ž' || r == 'ć' || r == 'đ') {
b.WriteRune(r)
continue
}
flush()
}
flush()
return out
}
// descriptionOverlapsProductFacts is true when desc shares significant tokens with
// any fact source (title, brand, model, …).
func descriptionOverlapsProductFacts(desc string, factSources ...string) bool {
descToks := significantDescTokens(desc)
if len(descToks) == 0 {
return false
}
overlap := 0
seen := map[string]struct{}{}
for _, src := range factSources {
for tok := range significantDescTokens(src) {
if _, ok := descToks[tok]; !ok {
continue
}
if _, dup := seen[tok]; dup {
continue
}
seen[tok] = struct{}{}
overlap++
if overlap >= 2 || len(tok) >= 5 {
return true
}
}
}
return overlap >= 1 && len(seen) >= 1 && len([]rune(desc)) >= minUsableProductDescRunes+20
}
func normalizeForEchoCompare(s string) string {
return strings.Join(strings.Fields(strings.ToLower(strings.TrimSpace(s))), " ")
}
// DescriptionEchoesTitle is true when desc is EqualFold or near-equal to title
// (whitespace-normalized, or one contains the other with only a tiny length delta).
func DescriptionEchoesTitle(desc, title string) bool {
d := normalizeForEchoCompare(desc)
t := normalizeForEchoCompare(title)
if d == "" || t == "" {
return false
}
if d == t {
return true
}
shorter, longer := d, t
if len(d) > len(t) {
shorter, longer = t, d
}
if !strings.Contains(longer, shorter) {
return false
}
delta := len(longer) - len(shorter)
return delta <= 8 && float64(len(shorter))/float64(len(longer)) >= 0.85
}
apps/api/internal/eprel/attrs.go
+145
View File
@@ -0,0 +1,145 @@
package eprel
import (
"fmt"
"strings"
)
// ExtractFromAttrs builds the nested EPREL object used by V1 process items and
// the dashboard product API (id / label / pdf / energy_class / energy_scale).
// Returns nil when no usable EPREL fields are present.
//
// Nested attrs["eprel"] may use flat export aliases (eprel_id, eprel_label, …);
// those are normalized onto the public keys. Flat eprel_* keys fill any gaps.
func ExtractFromAttrs(attrs map[string]any) any {
if attrs == nil {
return nil
}
out := map[string]any{}
if e, ok := attrs["eprel"]; ok && e != nil {
switch t := e.(type) {
case map[string]any:
for k, v := range t {
if s := attrString(v); s == "" {
continue
} else {
out[strings.TrimSpace(k)] = s
}
}
default:
if s := attrString(e); s != "" {
return map[string]any{"id": s}
}
return nil
}
}
put := func(dstKey string, keys ...string) {
if attrString(out[dstKey]) != "" {
return
}
for _, k := range keys {
if s := attrString(attrs[k]); s != "" {
out[dstKey] = s
return
}
// Also accept aliases already copied from a nested object.
if s := attrString(out[k]); s != "" {
out[dstKey] = s
return
}
}
}
put("id", "eprel_id", "EPRELID", "eprelId")
put("label", "eprel_label", "eprel_label_url")
put("pdf", "eprel_pdf", "eprel_pdf_url")
put("energy_class", "eprel_energy_class", "energyClass")
put("energy_scale", "eprel_energy_scale", "energyClassRange", "energyScale")
return NormalizeShape(out)
}
// NormalizeShape keeps only the public EPREL keys with non-empty string values.
// Alias keys (eprel_id, …) are folded onto id/label/pdf/energy_class/energy_scale.
func NormalizeShape(v any) any {
if v == nil {
return nil
}
m, ok := v.(map[string]any)
if !ok {
return nil
}
if len(m) == 0 {
return nil
}
out := map[string]any{}
for k, val := range m {
if s := attrString(val); s != "" {
out[strings.TrimSpace(k)] = s
}
}
fold := func(dst string, aliases ...string) {
if attrString(out[dst]) != "" {
for _, a := range aliases {
delete(out, a)
}
return
}
for _, a := range aliases {
if s := attrString(out[a]); s != "" {
out[dst] = s
break
}
}
for _, a := range aliases {
delete(out, a)
}
}
fold("id", "eprel_id", "EPRELID", "eprelId")
fold("label", "eprel_label", "eprel_label_url")
fold("pdf", "eprel_pdf", "eprel_pdf_url")
fold("energy_class", "eprel_energy_class", "energyClass")
fold("energy_scale", "eprel_energy_scale", "energyClassRange", "energyScale")
canonical := map[string]any{}
for _, k := range []string{"id", "label", "pdf", "energy_class", "energy_scale"} {
if s := attrString(out[k]); s != "" {
canonical[k] = s
}
}
if len(canonical) == 0 {
return nil
}
return canonical
}
func attrString(v any) string {
if v == nil {
return ""
}
switch t := v.(type) {
case string:
return strings.TrimSpace(t)
case float64:
// JSON numbers for numeric eprel ids.
if t == float64(int64(t)) {
return strings.TrimSpace(fmt.Sprintf("%.0f", t))
}
return strings.TrimSpace(fmt.Sprint(t))
case int:
return fmt.Sprintf("%d", t)
case int64:
return fmt.Sprintf("%d", t)
case map[string]any:
for _, k := range []string{"value", "#text", "text", "id"} {
if s := attrString(t[k]); s != "" {
return s
}
}
return ""
default:
s := strings.TrimSpace(fmt.Sprint(v))
if s == "" || s == "<nil>" {
return ""
}
return s
}
}
apps/api/internal/eprel/attrs_test.go
+84
View File
@@ -0,0 +1,84 @@
package eprel
import "testing"
func TestExtractFromAttrs_flatKeys(t *testing.T) {
got := ExtractFromAttrs(map[string]any{
"eprel_id": "1632113",
"eprel_label": "https://eprel.ec.europa.eu/api/product/1632113/labels?format=png",
"eprel_pdf_url": "https://eprel.ec.europa.eu/fiches/x.pdf",
"eprel_energy_class": "A",
"brand": "Samsung",
})
m, ok := got.(map[string]any)
if !ok {
t.Fatalf("type=%T", got)
}
if m["id"] != "1632113" || m["label"] == nil || m["pdf"] == nil || m["energy_class"] != "A" {
t.Fatalf("got=%v", m)
}
if _, leak := m["eprel_id"]; leak {
t.Fatalf("alias leaked: %v", m)
}
}
func TestExtractFromAttrs_nestedAliasesNormalized(t *testing.T) {
got := ExtractFromAttrs(map[string]any{
"eprel": map[string]any{
"eprel_id": "2208111",
"eprel_label": "https://eprel.ec.europa.eu/api/product/2208111/labels?format=png",
"eprel_pdf": "https://eprel.ec.europa.eu/fiches/y.pdf",
"eprel_energy_class": "E",
"source": "eprel_api",
},
})
m, ok := got.(map[string]any)
if !ok {
t.Fatalf("type=%T", got)
}
if m["id"] != "2208111" {
t.Fatalf("id=%v", m["id"])
}
if m["label"] == nil || m["pdf"] == nil {
t.Fatalf("missing label/pdf: %v", m)
}
if m["energy_class"] != "E" {
t.Fatalf("energy_class=%v", m["energy_class"])
}
if _, ok := m["eprel_id"]; ok {
t.Fatalf("eprel_id alias must be folded: %v", m)
}
if _, ok := m["source"]; ok {
t.Fatalf("non-canonical keys must be dropped: %v", m)
}
}
func TestExtractFromAttrs_nestedPartialFilledFromFlat(t *testing.T) {
got := ExtractFromAttrs(map[string]any{
"eprel": map[string]any{
"id": "1632113",
"label": "https://eprel.ec.europa.eu/api/product/1632113/labels?format=png",
},
"eprel_pdf": "https://eprel.ec.europa.eu/fiches/z.pdf",
"eprel_energy_class": "A",
})
m, ok := got.(map[string]any)
if !ok {
t.Fatalf("type=%T", got)
}
if m["id"] != "1632113" || m["pdf"] == nil || m["energy_class"] != "A" {
t.Fatalf("got=%v", m)
}
}
func TestNormalizeShape_empty(t *testing.T) {
if NormalizeShape(nil) != nil {
t.Fatal("nil")
}
if NormalizeShape(map[string]any{}) != nil {
t.Fatal("empty")
}
if NormalizeShape(map[string]any{"eprel_id": ""}) != nil {
t.Fatal("blank")
}
}
apps/api/internal/eprel/client_test.go
+10
View File
@@ -32,6 +32,16 @@ func TestNormalizeAndExtractID(t *testing.T) {
if got := ExtractID(mapped2); got != "777" {
t.Fatalf("mapped key=%q", got)
}
attrsOnly := map[string]any{"eprel_id": "888"}
if got := ExtractID(nil, nil, attrsOnly); got != "888" {
t.Fatalf("attrs extract=%q", got)
}
if got := ExtractID(map[string]any{"eprel_id": "0000"}); got != "" {
t.Fatalf("placeholder must be empty, got %q", got)
}
if got := NormalizeID("0000"); got != "" {
t.Fatalf("NormalizeID placeholder=%q", got)
}
}
func TestClientFetch_httptest(t *testing.T) {
apps/api/internal/eprel/id.go
+45 -13
View File
@@ -16,32 +16,34 @@ var eprelIDKeys = []string{
}
// NormalizeID coerces XML/API values (string, number, {"#text": ...}) to a trimmed ID.
// Placeholder feed values like "0" / "0000" become empty so the EPREL step is not
// falsely "skipped" after a doomed fetch attempt.
func NormalizeID(v any) string {
if v == nil {
return ""
}
var s string
switch t := v.(type) {
case string:
return strings.TrimSpace(t)
s = strings.TrimSpace(t)
case json.Number:
s := strings.TrimSpace(t.String())
s = strings.TrimSpace(t.String())
if i := strings.IndexByte(s, '.'); i >= 0 {
s = s[:i]
}
return s
case float64:
if t != t { // NaN
return ""
}
return strconv.FormatInt(int64(t), 10)
s = strconv.FormatInt(int64(t), 10)
case float32:
return strconv.FormatInt(int64(t), 10)
s = strconv.FormatInt(int64(t), 10)
case int:
return strconv.Itoa(t)
s = strconv.Itoa(t)
case int64:
return strconv.FormatInt(t, 10)
s = strconv.FormatInt(t, 10)
case int32:
return strconv.FormatInt(int64(t), 10)
s = strconv.FormatInt(int64(t), 10)
case json.RawMessage:
var decoded any
if err := json.Unmarshal(t, &decoded); err != nil {
@@ -55,14 +57,44 @@ func NormalizeID(v any) string {
if text, ok := t["text"]; ok {
return NormalizeID(text)
}
return ""
case []any:
for _, item := range t {
if id := NormalizeID(item); id != "" {
return id
}
}
return ""
case []string:
for _, item := range t {
if id := NormalizeID(item); id != "" {
return id
}
}
return ""
default:
s := strings.TrimSpace(fmt.Sprint(t))
s = strings.TrimSpace(fmt.Sprint(t))
if s == "" || s == "<nil>" {
return ""
}
return s
}
return ""
if isPlaceholderEPRELID(s) {
return ""
}
return s
}
func isPlaceholderEPRELID(id string) bool {
id = strings.TrimSpace(id)
if id == "" {
return true
}
for _, r := range id {
if r != '0' {
return false
}
}
return true
}
// IsValidID reports whether v normalizes to a non-empty EPREL registration id.
@@ -70,8 +102,8 @@ func IsValidID(v any) bool {
return NormalizeID(v) != ""
}
// ExtractID finds an EPREL ID in mapped and/or raw product field maps
// (vendor feeds often use <EPRELID/> / eprel_id).
// ExtractID finds an EPREL ID in mapped, raw, and/or attribute field maps
// (vendor feeds often use <EPRELID/> / eprel_id; specs parse may put it only in attrs).
func ExtractID(sources ...map[string]any) string {
for _, src := range sources {
if src == nil {
apps/api/internal/httpapi/admin_fix_catalog_handlers.go
+105
View File
@@ -0,0 +1,105 @@
package httpapi
import (
"net/http"
"strings"
"github.com/descrybe/descrybe-v2/apps/api/internal/billing"
"github.com/descrybe/descrybe-v2/apps/api/internal/catalog"
"github.com/descrybe/descrybe-v2/apps/api/internal/platformsettings"
"github.com/descrybe/descrybe-v2/apps/api/internal/processing"
"github.com/go-chi/chi/v5"
"github.com/google/uuid"
)
// handleAdminFixCompanyCatalog runs in-place Fix A1 / catalog hygiene for one company.
// Single route: POST /api/admin/companies/{id}/fix-catalog (no separate fix-a1).
// Delegates to processing.FixCompanyCatalog, which:
// 1. Ensures category_attributes links (orphan purge only)
// 2. RepairCompanyCategoryEnhancePrompts (same map as RepairA1DemoCategoryEnhancePrompts)
// 3. Re-applies product_enhance BuiltInDefaults when AIPrompts is set
// 4–7. FixCatalogHygieneWithIDs + attribute sanitize + reprocess sample
//
// Body: confirm=true required; backfill_categories (default true);
// reprocess_sample_limit (default 25, max 200, 0 = counts only).
// May target A1 in place with confirm — prefer Platform Demo when unsure.
// Refuses system company. Does not use A1 as a clone destination.
//
// Flash (UI): result.prompts / hashes / categories → flash.admin.fixA1Success.
func (s *Server) handleAdminFixCompanyCatalog(w http.ResponseWriter, r *http.Request) {
if s.Pool == nil {
Error(w, http.StatusServiceUnavailable, "database unavailable")
return
}
companyID, err := uuid.Parse(strings.TrimSpace(chi.URLParam(r, "id")))
if err != nil {
Error(w, http.StatusBadRequest, "invalid company id")
return
}
var body struct {
Confirm bool `json:"confirm"`
BackfillCategories *bool `json:"backfill_categories"`
ReprocessSampleLimit *int `json:"reprocess_sample_limit"`
}
if err := DecodeJSON(r, &body); err != nil {
Error(w, http.StatusBadRequest, "invalid json")
return
}
if !body.Confirm {
Error(w, http.StatusBadRequest, "confirm must be true")
return
}
if platformsettings.IsSystemCompany(companyID) {
Error(w, http.StatusBadRequest, "cannot fix the platform settings company")
return
}
var legacy, name string
if err := s.Pool.QueryRow(r.Context(), `
SELECT COALESCE(legacy_company_id, ''), name FROM companies WHERE id = $1`, companyID).
Scan(&legacy, &name); err != nil {
Error(w, http.StatusBadRequest, "company not found")
return
}
backfill := true
if body.BackfillCategories != nil {
backfill = *body.BackfillCategories
}
sampleLimit := 25
if body.ReprocessSampleLimit != nil {
sampleLimit = *body.ReprocessSampleLimit
}
if sampleLimit < 0 {
sampleLimit = 0
}
if sampleLimit > 200 {
sampleLimit = 200
}
result, err := processing.FixCompanyCatalog(
r.Context(),
s.Pool,
companyID,
name,
billing.IsA1CohortCompany(legacy, name),
processing.FixCompanyCatalogOpts{
BackfillCategories: backfill,
ReprocessSampleLimit: sampleLimit,
AIPrompts: s.AIPrompts,
},
)
if err != nil {
ClientOrLog(w, http.StatusBadRequest, "could not fix catalog", err, catalog.ClientError)
return
}
JSON(w, http.StatusOK, map[string]any{
"status": "ok",
"result": result,
"note": "Catalog was repaired in place; reprocess recommended products manually (no mass reprocess).",
})
}
apps/api/internal/httpapi/admin_fix_catalog_handlers_test.go
+91
View File
@@ -0,0 +1,91 @@
package httpapi
import (
"context"
"net/http"
"net/http/httptest"
"strings"
"testing"
"github.com/alexedwards/scs/v2"
"github.com/descrybe/descrybe-v2/apps/api/internal/auth"
"github.com/descrybe/descrybe-v2/apps/api/internal/config"
"github.com/google/uuid"
)
func TestHandleAdminFixCompanyCatalogNilPool(t *testing.T) {
t.Parallel()
s := &Server{}
req := httptest.NewRequest(http.MethodPost, "/api/admin/companies/"+uuid.NewString()+"/fix-catalog", strings.NewReader(`{"confirm":true}`))
rec := httptest.NewRecorder()
s.handleAdminFixCompanyCatalog(rec, req)
if rec.Code != http.StatusServiceUnavailable {
t.Fatalf("status=%d want 503 body=%s", rec.Code, rec.Body.String())
}
}
func TestHandleAdminFixCompanyCatalogRequiresConfirm(t *testing.T) {
t.Parallel()
// Pool nil short-circuits before confirm — use non-nil Pool stub via missing company path is harder.
// Confirm gate is covered when Pool is set; here we only assert invalid uuid.
s := &Server{Pool: nil}
req := httptest.NewRequest(http.MethodPost, "/api/admin/companies/not-a-uuid/fix-catalog", strings.NewReader(`{"confirm":false}`))
rec := httptest.NewRecorder()
s.handleAdminFixCompanyCatalog(rec, req)
if rec.Code != http.StatusServiceUnavailable {
t.Fatalf("status=%d want 503 (nil pool) body=%s", rec.Code, rec.Body.String())
}
}
// TestRouterAdminFixCatalogMounted locks POST /api/admin/companies/{id}/fix-catalog
// after session + CSRF + platform-admin (503 with nil Pool), not chi 404.
func TestRouterAdminFixCatalogMounted(t *testing.T) {
t.Parallel()
sm := scs.New()
sm.Cookie.Name = "descrybe_session"
uid := uuid.MustParse("aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa")
s := &Server{
Config: config.Config{
CSRFCookieName: "descrybe_csrf",
WebOrigin: "http://localhost:5173",
},
Sessions: sm,
Auth: &auth.Service{},
testPlatformAdmin: func(_ context.Context, got uuid.UUID) (bool, error) {
return got == uid, nil
},
}
var token string
seed := LoadSession(sm)(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
sm.Put(r.Context(), auth.SessionUserIDKey, uid.String())
w.WriteHeader(http.StatusNoContent)
}))
seedRec := httptest.NewRecorder()
seed.ServeHTTP(seedRec, httptest.NewRequest(http.MethodGet, "/seed", nil))
for _, c := range seedRec.Result().Cookies() {
if c.Name == sm.Cookie.Name {
token = c.Value
}
}
if token == "" {
t.Fatal("expected session cookie from seed request")
}
h := s.Router()
path := "/api/admin/companies/" + uuid.NewString() + "/fix-catalog"
csrf := csrfCookieForSession(t, h, sm, token)
mounted := httptest.NewRecorder()
req := httptest.NewRequest(http.MethodPost, path, strings.NewReader(`{"confirm":true}`))
req.AddCookie(&http.Cookie{Name: sm.Cookie.Name, Value: token})
req.AddCookie(csrf)
req.Header.Set("X-CSRF-Token", csrf.Value)
h.ServeHTTP(mounted, req)
if mounted.Code == http.StatusNotFound {
t.Fatalf("route not mounted: status=404 body=%s", mounted.Body.String())
}
if mounted.Code != http.StatusServiceUnavailable {
t.Fatalf("status=%d want 503 (nil pool) body=%s", mounted.Code, mounted.Body.String())
}
}
apps/api/internal/httpapi/server.go
+1
View File
@@ -391,6 +391,7 @@ func (s *Server) Router() http.Handler {
r.Post("/users/{id}/dev-password", s.handleAdminDevSetPassword)
r.Get("/companies", s.handleAdminListCompanies)
r.Post("/companies/{id}/clone-catalog", s.handleAdminCloneCompanyCatalog)
r.Post("/companies/{id}/fix-catalog", s.handleAdminFixCompanyCatalog)
r.Get("/readiness", s.handleAdminReadiness)
r.Get("/diagnostics", s.handleAdminDiagnostics)
r.Get("/analytics", s.handleAdminAnalytics)
apps/api/internal/httpapi/v1_openapi.go
+142 -74
View File
@@ -43,7 +43,10 @@ info:
4. Store it securely. Later list/revoke shows only key_prefix (first 10
characters). Revoked keys fail auth immediately.
API keys are created in Settings -> API keys
### Cutover / migration (reissue)
API keys from the previous Descrybe platform were not migrated. After
cutover, integrations must create a new dk_ key in Settings -> API keys
(or Use my API key on /docs). Pre-cutover secrets return the same HTTP 401
Unauthorized as unknown keys — there is no separate “legacy key” error.
@@ -893,8 +896,8 @@ paths:
jobs omit items. Not the same shape as GET /process/{id} (flat ProcessingJob).
On COMPLETED items, id is the processed_products UUID (legacy). Additive aliases:
processed_product_id (same value as id) and raw_product_id (raw_products.id)
for dual-mode clients that also call raw_product_ids surfaces.
processed_product_id (same value as id), raw_product_id (raw_products.id), and
name (same value as title) for dual-mode clients / scorecards.
parameters:
- name: id
in: path
@@ -914,20 +917,36 @@ paths:
$ref: "#/components/schemas/LegacyProcessStatusEnvelope"
examples:
completed:
summary: Completed
summary: Completed A1-shaped item
value:
data:
status: COMPLETED
process_id: aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa
processing_type: full
items:
- ean: 0123456789012
- ean: '8606019604493'
id: bbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb
processed_product_id: bbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb
raw_product_id: cccccccc-cccc-cccc-cccc-cccccccccccc
status: processed
category: electronics
title: Acme Wireless Earbuds ANC Black
category: '50'
category_name: Cookers
title: VOX electric cooker EHT 6020 WG
name: VOX electric cooker EHT 6020 WG
meta_title: VOX electric cooker EHT 6020 WG | 50
meta_description: Affordable electric cooker with four plates and a 65 L fan oven.
description: "Vox Electronics electric cooker EHT 6020 WG offers strong value with four electric hobs and a 65 L fan oven."
attributes:
brand: Vox
product_model: EHT6020WG
main_image: https://images.example.com/products/vox-eht6020wg.jpg
more_images: null
eprel:
id: "1234567"
label: https://eprel.ec.europa.eu/label/Example
pdf: https://eprel.ec.europa.eu/fiches/Example.pdf
energy_class: A
energy_scale: A-G
total_items: 1
processed_at: '2026-08-04T10:04:12Z'
processing:
@@ -6256,78 +6275,117 @@ components:
format: date-time
nullable: true
LegacyProcessItem:
type: object
required:
- ean
properties:
type: object
description: |
One COMPLETED legacy process line (A1 / public contract). Successful items
expose category as categories.unique_id, a plain-text description string
(never a JSON array; HTML stripped), SEO meta_title / meta_description,
optional eprel object or null, clean attributes, images, and dual-mode ids.
Product display name is title; additive name mirrors the same processed title
(dual-mode for scorecards / legacy clients that read name).
required:
- ean
properties:
ean:
type: string
type: string
id:
type: string
format: uuid
description: |
Legacy field: processed_products.id when enrichment succeeded.
Do not treat as raw_products.id. Same value as processed_product_id.
type: string
format: uuid
description: |
Legacy field: processed_products.id when enrichment succeeded.
Do not treat as raw_products.id. Same value as processed_product_id.
processed_product_id:
type: string
format: uuid
description: |
Explicit alias of id (processed_products.id). Prefer this name in new
dual-mode clients; id remains for backward compatibility.
type: string
format: uuid
description: |
Explicit alias of id (processed_products.id). Prefer this name in new
dual-mode clients; id remains for backward compatibility.
raw_product_id:
type: string
format: uuid
description: |
raw_products.id for this job line. Use with POST /process raw_product_ids
or dashboard catalog APIs. Present whenever the job product row exists.
type: string
format: uuid
description: |
raw_products.id for this job line. Use with POST /process raw_product_ids
or dashboard catalog APIs. Present whenever the job product row exists.
category:
type: string
nullable: true
type: string
nullable: true
description: |
categories.unique_id for the assigned category. Opaque string — may be
numeric (e.g. "28" or "50") or slug-like. Not a URL path and not a UUID.
category_name:
type: string
nullable: true
type: string
nullable: true
description: Human-readable category display name (not the unique_id).
title:
type: string
nullable: true
type: string
nullable: true
description: |
Product display name (processed title). Primary legacy field; same value
as name when present.
name:
type: string
nullable: true
description: |
Additive alias of title (same processed display name). Prefer title in
new clients; name remains for scorecards and legacy readers.
meta_title:
type: string
nullable: true
type: string
nullable: true
description: |
SEO title. Filled from processing meta or synthesized from title / category
when empty so successful items are not left with null meta.
meta_description:
type: string
nullable: true
type: string
nullable: true
description: |
SEO description (word-safe truncate). Distinct from body description when
possible; synthesized from plain description when DB meta is empty.
description:
type: string
nullable: true
description: Plain-text product description (HTML stripped). attributes:
type: object
additionalProperties: true
nullable: true
type: string
nullable: true
description: |
Plain-text product body description. Always a string — never a one-element
JSON array. Feed HTML tags are stripped; newlines may remain between paragraphs.
attributes:
type: object
additionalProperties: true
nullable: true
description: |
Characteristic catalog attributes only (brand, model, dims, warranty, …).
Core fields and eprel_* keys are not duplicated here.
main_image:
type: string
nullable: true
type: string
nullable: true
more_images:
type: array
items:
type: string
nullable: true
type: array
items:
type: string
nullable: true
eprel:
nullable: true
type: object
properties:
nullable: true
type: object
description: |
EU energy label payload when EPREL data is available; otherwise null.
Prefer this object over raw eprel_* keys inside attributes.
Live shape keys: id, label, pdf, energy_class, energy_scale.
properties:
id:
type: string
description: EPREL product registration id (same as eprel_id).
label:
type: string
type: string
pdf:
type: string
type: string
energy_class:
type: string
type: string
energy_scale:
type: string
type: string
status:
type: string
description: Per-item outcome — processed on success; not_found / failed / cancelled otherwise
example: processed
type: string
description: Per-item outcome — processed on success; not_found / failed / cancelled otherwise
example: processed
error:
type: string
type: string
examples:
ProcessingJobAccepted:
summary: Single processing job accepted (POST /process)
@@ -6393,32 +6451,42 @@ components:
started_at: '2026-08-04T10:00:00Z'
created_at: '2026-08-04T09:59:50Z'
LegacyProcessCompleted:
summary: "Completed legacy poll (GET /products/process/{id})"
summary: "Completed legacy poll with A1-shaped item (unique_id category, plain description, meta, eprel)"
value:
data:
status: COMPLETED
process_id: 8f14e45f-ceea-467a-9e5d-9c6b0e8f3a21
processing_type: full
items:
- ean: '4548736132174'
- ean: '8606019604493'
id: bbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb
processed_product_id: bbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb
raw_product_id: cccccccc-cccc-cccc-cccc-cccccccccccc
status: processed
category: electronics
category_name: Electronics
title: Sony WH-1000XM5 Wireless Noise Cancelling Headphones Black
meta_title: Sony WH-1000XM5 | Noise Cancelling Headphones
meta_description: Industry-leading noise cancellation with up to 30 hours battery life.
description: Industry-leading noise cancellation with up to 30 hours battery life.
category: '50'
category_name: Cookers
title: VOX electric cooker EHT 6020 WG
name: VOX electric cooker EHT 6020 WG
meta_title: VOX electric cooker EHT 6020 WG | 50
meta_description: Affordable electric cooker with four plates and a 65 L fan oven.
description: "Vox Electronics electric cooker EHT 6020 WG offers strong value with four electric hobs and a 65 L fan oven. Energy class A with practical everyday capacity."
attributes:
color: Black
brand: Sony
battery_life_hours: '30'
main_image: https://images.example.com/products/wh1000xm5-black.jpg
brand: Vox
product_model: EHT6020WG
width: 0.6 m
height: 0.85 m
depth: 0.6 m
weight: 42.81 kg
warranty: 60 months
main_image: https://images.example.com/products/vox-eht6020wg.jpg
more_images:
- https://images.example.com/products/wh1000xm5-black-side.jpg
eprel: null
- https://images.example.com/products/vox-eht6020wg-side.jpg
eprel:
id: "1234567"
label: https://eprel.ec.europa.eu/label/Example
pdf: https://eprel.ec.europa.eu/fiches/Example.pdf
energy_class: A
energy_scale: A-G
total_items: 1
processed_at: '2026-08-04T10:04:12Z'
LegacyProcessInProgress:
apps/api/internal/httpapi/v1_openapi_test.go
+30
View File
@@ -415,6 +415,36 @@ func TestV1OpenAPIYAMLProcessDualIDsAndGatesDocumentsContracts(t *testing.T) {
}
}
// TestV1OpenAPIYAMLLegacyProcessItemEprelShape locks LegacyProcessItem.eprel
// to the live nested object from eprel.MergeInto / extractEPRELFromAttrs.
func TestV1OpenAPIYAMLLegacyProcessItemEprelShape(t *testing.T) {
t.Parallel()
root := mustParseOpenAPIRoot(t, v1OpenAPIYAML)
comps, _ := root["components"].(map[string]any)
schemas, _ := comps["schemas"].(map[string]any)
item, _ := schemas["LegacyProcessItem"].(map[string]any)
itemProps, _ := item["properties"].(map[string]any)
eprel, _ := itemProps["eprel"].(map[string]any)
eprelProps, _ := eprel["properties"].(map[string]any)
if eprelProps == nil {
t.Fatal("LegacyProcessItem.eprel.properties missing")
}
want := []string{"id", "label", "pdf", "energy_class", "energy_scale"}
for _, key := range want {
if _, ok := eprelProps[key]; !ok {
t.Fatalf("LegacyProcessItem.eprel missing property %q (live shape)", key)
}
}
if len(eprelProps) != len(want) {
keys := make([]string, 0, len(eprelProps))
for k := range eprelProps {
keys = append(keys, k)
}
t.Fatalf("LegacyProcessItem.eprel properties = %v, want exactly %v", keys, want)
}
}
// TestV1OpenAPIYAMLDocumentsAPIKeyCutoverReissue locks cutover honesty: legacy
// API keys were not migrated and clients must create new keys.
func TestV1OpenAPIYAMLDocumentsAPIKeyCutoverReissue(t *testing.T) {
apps/api/internal/platformsettings/ai_configs.go
+30 -12
View File
@@ -312,11 +312,13 @@ func (s *Service) ResolveAIConfig(ctx context.Context, role string) (ResolvedAIC
Source: SourceDB,
}
if role == AIRoleProcessing {
// Prefer legacy openai JSON fields before OPENAI_* env so a mock .env
// cannot override a configured platform DB endpoint.
if out.BaseURL == "" {
out.BaseURL = strings.TrimSpace(s.Env.OpenAIBaseURL)
out.BaseURL = firstNonEmpty(strings.TrimSpace(doc.OpenAI.BaseURL), strings.TrimSpace(s.Env.OpenAIBaseURL))
}
if out.Model == "" {
out.Model = strings.TrimSpace(s.Env.OpenAIModel)
out.Model = firstNonEmpty(strings.TrimSpace(doc.OpenAI.Model), strings.TrimSpace(s.Env.OpenAIModel))
}
}
if role == AIRoleVectorization {
@@ -330,20 +332,36 @@ func (s *Service) ResolveAIConfig(ctx context.Context, role string) (ResolvedAIC
return out, nil
}
if role == AIRoleProcessing && !hasRoleMeta {
// processing: prefer legacy openai (DB) then env when the role slot has no key,
// even if provider/model/base meta was partially written.
if role == AIRoleProcessing {
legacy, err := s.resolveLegacyOpenAI(doc)
if err != nil {
return ResolvedAIConfig{}, err
}
return ResolvedAIConfig{
Role: role,
Provider: defaultAIProvider,
APIKey: legacy.APIKey,
BaseURL: legacy.BaseURL,
Model: legacy.Model,
Enabled: legacy.APIKey != "",
Source: legacy.Source,
}, nil
if strings.TrimSpace(legacy.APIKey) != "" {
return ResolvedAIConfig{
Role: role,
Provider: firstNonEmpty(strings.TrimSpace(st.Provider), defaultAIProvider),
APIKey: legacy.APIKey,
BaseURL: firstNonEmpty(strings.TrimSpace(st.BaseURL), legacy.BaseURL),
Model: firstNonEmpty(strings.TrimSpace(st.Model), legacy.Model),
Enabled: true,
Extras: copyStringMap(st.Extras),
Source: legacy.Source,
}, nil
}
if !hasRoleMeta {
return ResolvedAIConfig{
Role: role,
Provider: defaultAIProvider,
APIKey: legacy.APIKey,
BaseURL: legacy.BaseURL,
Model: legacy.Model,
Enabled: legacy.APIKey != "",
Source: legacy.Source,
}, nil
}
}
if role == AIRoleVectorization && !hasRoleMeta {
apps/api/internal/platformsettings/ai_configs_test.go
+32
View File
@@ -115,6 +115,38 @@ func TestResolveAIConfig_envFallback(t *testing.T) {
}
}
func TestResolveAIConfig_dbPreferredOverMockEnv(t *testing.T) {
key := DeriveKey("test-ai-config-secret-material", "fallback")
plain := "sk-db-overloaded-key"
enc, err := EncryptSecret(key, plain)
if err != nil {
t.Fatal(err)
}
svc := &Service{
Key: key,
Env: EnvConfig{
OpenAIAPIKey: "sk-mock-env",
OpenAIBaseURL: "http://127.0.0.1:18767/v1",
OpenAIModel: "mock-llm",
},
}
doc := storedDoc{
OpenAI: openaiStored{
BaseURL: "https://llm.overloadedbot.org/v1",
Model: "code-fast",
APIKeyEnc: enc,
APIKeyLast4: last4(plain),
},
}
got, err := svc.resolveLegacyOpenAI(doc)
if err != nil {
t.Fatal(err)
}
if got.Source != SourceDB || got.APIKey != plain || got.BaseURL != "https://llm.overloadedbot.org/v1" || got.Model != "code-fast" {
t.Fatalf("legacy resolve should prefer DB over mock env: %+v", got)
}
}
func TestPatchAIExtras(t *testing.T) {
var extras map[string]string
val := "1536"
apps/api/internal/processing/ai.go
+37 -6
View File
@@ -87,8 +87,25 @@ type ProductInput struct {
// CategoryEnhancePrompt overrides EnhanceUserTemplate when non-empty
// (resolved for the active language before enhance).
CategoryEnhancePrompt string
// CategoryPromptsByLang maps lower(name) → lang → category override prompt.
// CategoryPromptsByLang maps lower(name|unique_id) → lang → category override prompt
// (JSON/overlay text with {{attrs}}/{{category}}; works with repaired A1 overlays).
CategoryPromptsByLang map[string]company.LangPromptMap
// TitleTemplate / DescriptionTemplate are language-agnostic category formulas
// (categories.title_template / description_template). Injected into the enhance
// user prompt as structure constraints; not per-lang text maps.
TitleTemplate any
DescriptionTemplate any
// CategoryFormulasByKey maps lower(name|unique_id) → title/description templates.
CategoryFormulasByKey map[string]CategoryFormulas
// CategoryNamesByUID maps category unique_id → display name (e.g. "50"→"Štedilniki").
CategoryNamesByUID map[string]string
// AllowedAttrKeys are company category_attributes keys for this product's
// category unique_id (plus coreCharacteristicAttrKeys via AttrsForEnhance).
// nil = sanitize only (unit tests); empty non-nil = core keys only.
AllowedAttrKeys map[string]struct{}
// CategoryAttrKeys maps category unique_id → attribute keys (job cache).
// When set, enhance resolves allowlist from the step result category.
CategoryAttrKeys map[string]map[string]struct{}
// Prior* are loaded from the last processed_products row for this raw product.
PriorEnhanceHash string
PriorProcessedName string
@@ -97,6 +114,12 @@ type ProductInput struct {
PriorLocalized company.LocalizedContent
}
// CategoryFormulas holds optional title/description templates for one category key.
type CategoryFormulas struct {
TitleTemplate any
DescriptionTemplate any
}
// PromptTemplates is a system+user pair for one language.
type PromptTemplates struct {
System string
@@ -105,7 +128,11 @@ type PromptTemplates struct {
// StepResult is the cumulative output for one product.
type StepResult struct {
Category string
// Category is the company taxonomy unique_id (DB/API). Prefer CategoryName
// for human-facing copy (meta, synthesize, {{category}}).
Category string
// CategoryName is the display label resolved from categories.name.
CategoryName string
Name string
Description string
ProcessedName string
@@ -126,6 +153,10 @@ type StepResult struct {
// enhance input hash matched (ai_enhance_unchanged). processOne must not
// ConsumeCredits in that case — no LLM and no meaningful rework.
SkipCreditDebit bool
// MetaTitle / MetaDescription are free template SEO fields written by
// processOne via fillMetaFromResult (never AI-enhanced by default).
MetaTitle string
MetaDescription string
}
// StepProgress is a job-level snapshot of pipeline step status.
@@ -153,8 +184,8 @@ func (e *Engine) CompleterEnabled() bool {
if c, ok := e.Completer.(EnableChecker); ok {
return c.Enabled()
}
// HeuristicCompleter has no Enabled — treat as enabled only if explicitly set.
// Worker sets Completer=nil when platform OpenAI (admin settings / env) is unset.
_, isHeuristic := e.Completer.(HeuristicCompleter)
return !isHeuristic
// HeuristicCompleter (local/dev fallback) and other Completers without
// EnableChecker are treated as enabled when explicitly wired on the Engine.
// Worker still sets Completer=nil when no provider is configured.
return true
}
apps/api/internal/processing/ai_provider_mode.go
+23
View File
@@ -56,3 +56,26 @@ func normalizeProviderMode(label string) string {
return AIProviderUnknown
}
}
// isUnknownProviderMode reports empty or the sentinel "unknown" analytics label.
func isUnknownProviderMode(label string) bool {
m := strings.ToLower(strings.TrimSpace(label))
return m == "" || m == AIProviderUnknown
}
// preferKnownProviderMode returns the first non-unknown label, else unknown.
func preferKnownProviderMode(candidates ...string) string {
for _, c := range candidates {
if !isUnknownProviderMode(c) {
return normalizeProviderMode(c)
}
}
return AIProviderUnknown
}
func lastResultAIProviderMode(r *StepResult) string {
if r == nil {
return ""
}
return r.AIProviderMode
}
apps/api/internal/processing/ai_provider_mode_test.go
+22
View File
@@ -2,6 +2,28 @@ package processing
import "testing"
func TestPreferKnownProviderMode(t *testing.T) {
t.Parallel()
if got := preferKnownProviderMode(AIProviderUnknown, AIProviderInternal); got != AIProviderInternal {
t.Fatalf("unknown+internal=%q", got)
}
if got := preferKnownProviderMode("", "custom"); got != AIProviderCustom {
t.Fatalf("empty+custom=%q", got)
}
if got := preferKnownProviderMode(AIProviderInternal, AIProviderCustom); got != AIProviderInternal {
t.Fatalf("internal must win first=%q", got)
}
if got := preferKnownProviderMode(AIProviderUnknown, ""); got != AIProviderUnknown {
t.Fatalf("all unknown=%q", got)
}
if isUnknownProviderMode("") != true || isUnknownProviderMode(AIProviderUnknown) != true {
t.Fatal("expected empty/unknown to be unknown")
}
if isUnknownProviderMode(AIProviderInternal) {
t.Fatal("internal must not be unknown")
}
}
func TestNormalizeProviderMode(t *testing.T) {
t.Parallel()
cases := []struct {
apps/api/internal/processing/attr_map.go
+269
View File
@@ -0,0 +1,269 @@
package processing
import (
"fmt"
"strings"
"unicode"
)
// knownAttrKeyAliases maps compact feed/spec tokens onto compact category
// attribute_key tokens (Slovenian catalog keys + common English synonyms).
var knownAttrKeyAliases = map[string]string{
"velikostzaslona": "diagonalazaslona",
"screensize": "diagonalazaslona",
"screen": "diagonalazaslona",
"diagonal": "diagonalazaslona",
"diagonala": "diagonalazaslona",
"nastavljivpovisini": "nastavljivopovisini",
"heightadjustable": "nastavljivopovisini",
"zaslonnadotik": "zaslonobcutljivnadotik",
"touchscreen": "zaslonobcutljivnadotik",
"tipmatrike": "vrstapanela",
"tippanela": "vrstapanela",
"paneltype": "vrstapanela",
"panel": "vrstapanela",
"zvocniki": "vgrajenizvocniki",
"speakers": "vgrajenizvocniki",
"ukrivljenzaslon": "oblikazaslona",
"curved": "oblikazaslona",
"slusalkezmikrofonom": "vgrajenmikrofon",
"mikrofon": "vgrajenmikrofon",
"builtinmic": "vgrajenmikrofon",
"microphone": "vgrajenmikrofon",
"brezzicne": "brezzicnapovezava",
"brezzicna": "brezzicnapovezava",
"wireless": "brezzicnapovezava",
"bluetooth": "brezzicnatehnologija",
"bt": "brezzicnatehnologija",
"anc": "odpravljanjehrupa",
"noisecancelling": "odpravljanjehrupa",
"noisecanceling": "odpravljanjehrupa",
"odpravljanjehrupa": "odpravljanjehrupa",
"resolution": "locljivost",
"responsetime": "odzivnicas",
}
// MapAttrsOntoAllowedKeys remaps feed/spec keys onto category_attributes keys
// when a confident match exists (exact compact, known alias, or unique token
// subset). Existing allowed/core values are never overwritten. When allowed is
// nil or empty, attrs are returned unchanged (caller still filters).
func MapAttrsOntoAllowedKeys(attrs map[string]any, allowed map[string]struct{}) map[string]any {
if len(attrs) == 0 || len(allowed) == 0 {
return attrs
}
prefer := preferredAllowedKeys(allowed)
if len(prefer) == 0 {
return attrs
}
out := make(map[string]any, len(attrs)+len(prefer))
for k, v := range attrs {
out[k] = v
}
for k, v := range attrs {
if !attrValuePresent(v) {
continue
}
canon := canonicalizeAttrKey(k)
if canon != "" {
if _, ok := coreCharacteristicAttrKeys[canon]; ok {
continue
}
if _, ok := allowed[canon]; ok {
continue
}
if _, ok := allowed[strings.ToLower(strings.TrimSpace(k))]; ok {
continue
}
}
target := resolveAllowedAttrKey(k, prefer)
if target == "" || target == canon || target == k {
continue
}
if attrValuePresent(out[target]) {
continue
}
out[target] = v
}
return out
}
func preferredAllowedKeys(allowed map[string]struct{}) map[string]string {
// compact → preferred output key (canonical snake_case when possible)
prefer := make(map[string]string, len(allowed))
for k := range allowed {
c := compactAttrToken(k)
if c == "" {
continue
}
canon := canonicalizeAttrKey(k)
if canon == "" {
canon = strings.ToLower(strings.TrimSpace(k))
}
if prev, ok := prefer[c]; ok {
// Prefer underscore form over hyphen duplicates from loadCategoryAttributeKeySets.
if strings.Contains(canon, "_") && !strings.Contains(prev, "_") {
prefer[c] = canon
}
continue
}
prefer[c] = canon
}
return prefer
}
func resolveAllowedAttrKey(feedKey string, prefer map[string]string) string {
ck := compactAttrToken(feedKey)
if ck == "" {
return ""
}
if target, ok := prefer[ck]; ok {
return target
}
if alias, ok := knownAttrKeyAliases[ck]; ok {
if target, ok := prefer[alias]; ok {
return target
}
}
return softMatchAllowedKey(feedKey, ck, prefer)
}
func softMatchAllowedKey(feedKey, feedCompact string, prefer map[string]string) string {
feedTokens := attrKeyTokens(feedKey)
if len(feedTokens) == 0 && feedCompact == "" {
return ""
}
var hits []string
for compact, target := range prefer {
if feedCompact != "" && (strings.Contains(compact, feedCompact) || strings.Contains(feedCompact, compact)) {
// Require meaningful overlap (avoid tiny tokens matching everything).
shorter, longer := feedCompact, compact
if len(shorter) > len(longer) {
shorter, longer = longer, shorter
}
if len(shorter) >= 6 && len(shorter)*10 >= len(longer)*6 {
hits = append(hits, target)
continue
}
}
if len(feedTokens) == 0 {
continue
}
allowedTokens := attrKeyTokens(target)
if len(allowedTokens) == 0 {
allowedTokens = attrKeyTokens(compact)
}
if tokensSubsetMatch(feedTokens, allowedTokens) {
hits = append(hits, target)
}
}
if len(hits) != 1 {
return ""
}
return hits[0]
}
func tokensSubsetMatch(feed, allowed []string) bool {
significant := 0
for _, ft := range feed {
if len(ft) < 3 {
continue
}
significant++
ok := false
for _, at := range allowed {
if at == ft || strings.HasPrefix(at, ft) || strings.HasPrefix(ft, at) {
ok = true
break
}
}
if !ok {
return false
}
}
return significant >= 2
}
func attrKeyTokens(k string) []string {
k = strings.ToLower(strings.TrimSpace(k))
if k == "" {
return nil
}
var b strings.Builder
var out []string
flush := func() {
if b.Len() == 0 {
return
}
tok := b.String()
b.Reset()
tok = compactAttrToken(tok)
if len(tok) >= 2 {
out = append(out, tok)
}
}
for _, r := range k {
r = foldAttrRune(r)
switch {
case r >= 'a' && r <= 'z', r >= '0' && r <= '9':
b.WriteRune(r)
default:
flush()
}
}
flush()
return out
}
func compactAttrToken(k string) string {
var b strings.Builder
b.Grow(len(k))
for _, r := range strings.ToLower(k) {
r = foldAttrRune(r)
if (r >= 'a' && r <= 'z') || (r >= '0' && r <= '9') {
b.WriteRune(r)
}
}
return b.String()
}
func foldAttrRune(r rune) rune {
switch r {
case 'š', 'ś', 'ş':
return 's'
case 'č', 'ć', 'ç':
return 'c'
case 'ž', 'ź', 'ż':
return 'z'
case 'đ':
return 'd'
case 'ä', 'á', 'à', 'â', 'ã', 'å':
return 'a'
case 'ë', 'é', 'è', 'ê':
return 'e'
case 'ï', 'í', 'ì', 'î':
return 'i'
case 'ö', 'ó', 'ò', 'ô', 'õ':
return 'o'
case 'ü', 'ú', 'ù', 'û':
return 'u'
case 'ý', 'ÿ':
return 'y'
}
if unicode.IsLetter(r) {
return unicode.ToLower(r)
}
return r
}
func attrValuePresent(v any) bool {
if v == nil {
return false
}
s := strings.TrimSpace(fmt.Sprint(v))
return s != "" && s != "<nil>"
}
apps/api/internal/processing/attr_map_test.go
+220
View File
@@ -0,0 +1,220 @@
package processing
import (
"context"
"testing"
)
func TestMapAttrsOntoAllowedKeys_monitorAliases(t *testing.T) {
allowed := map[string]struct{}{
"barva": {},
"diagonala_zaslona": {},
"diagonala-zaslona": {},
"locljivost": {},
"nastavljivo_po_visini": {},
"nastavljivo-po-visini": {},
"odzivni_cas": {},
"odzivni-cas": {},
"zaslon_obcutljiv_na_dotik": {},
"zaslon-obcutljiv-na-dotik": {},
"vrsta_panela": {},
"vrsta-panela": {},
"vgrajeni_zvocniki": {},
"vgrajeni-zvocniki": {},
}
in := map[string]any{
"brand": "Xiaomi",
"barva": "Črna",
"locljivost": "1920 x 1080",
"odzivni-cas": "6 ms",
"velikost-zaslona": `68,58 cm (27,0")`,
"nastavljiv-po-visini": "Ne",
"zaslon-na-dotik": "Ne",
"tip-matrike": "IPS",
"zvocniki": "Ne",
"kontrast": "1000:1", // not on allowlist
}
got := MapAttrsOntoAllowedKeys(in, allowed)
if got["diagonala_zaslona"] != `68,58 cm (27,0")` {
t.Fatalf("diagonala_zaslona=%v keys=%v", got["diagonala_zaslona"], got)
}
if got["nastavljivo_po_visini"] != "Ne" {
t.Fatalf("nastavljivo_po_visini=%v", got["nastavljivo_po_visini"])
}
if got["zaslon_obcutljiv_na_dotik"] != "Ne" {
t.Fatalf("zaslon_obcutljiv_na_dotik=%v", got["zaslon_obcutljiv_na_dotik"])
}
if got["vrsta_panela"] != "IPS" {
t.Fatalf("vrsta_panela=%v", got["vrsta_panela"])
}
if got["vgrajeni_zvocniki"] != "Ne" {
t.Fatalf("vgrajeni_zvocniki=%v", got["vgrajeni_zvocniki"])
}
// Exact keys untouched.
if got["locljivost"] != "1920 x 1080" || got["barva"] != "Črna" {
t.Fatalf("exact keys: %v", got)
}
}
func TestMapAttrsOntoAllowedKeys_headphonesAliases(t *testing.T) {
allowed := map[string]struct{}{
"brezzicna_povezava": {},
"brezzicna-povezava": {},
"brezzicna_tehnologija": {},
"brezzicna-tehnologija": {},
"odpravljanje_hrupa": {},
"odpravljanje-hrupa": {},
"vgrajen_mikrofon": {},
"vgrajen-mikrofon": {},
}
in := map[string]any{
"slusalke-z-mikrofonom": "Da",
"brezzicne": "Da",
"bluetooth": "5.3",
"vrsta-slusalk": "Naglavne",
}
got := MapAttrsOntoAllowedKeys(in, allowed)
if got["vgrajen_mikrofon"] != "Da" {
t.Fatalf("vgrajen_mikrofon=%v got=%v", got["vgrajen_mikrofon"], got)
}
if got["brezzicna_povezava"] != "Da" {
t.Fatalf("brezzicna_povezava=%v", got["brezzicna_povezava"])
}
if got["brezzicna_tehnologija"] != "5.3" {
t.Fatalf("brezzicna_tehnologija=%v", got["brezzicna_tehnologija"])
}
}
func TestAttrsForPersist_keepsMappedFormulaKeys(t *testing.T) {
attrs := map[string]any{
"brand": "Xiaomi",
"velikost-zaslona": "27\"",
"locljivost": "1920 x 1080",
"tip-matrike": "IPS",
"kontrast": "1000:1",
}
allowed := map[string]struct{}{
"diagonala_zaslona": {},
"diagonala-zaslona": {},
"locljivost": {},
"vrsta_panela": {},
"vrsta-panela": {},
"barva": {},
}
got := AttrsForPersist(attrs, allowed)
if got["brand"] != "Xiaomi" {
t.Fatalf("brand=%v", got)
}
if got["diagonala_zaslona"] != "27\"" {
t.Fatalf("formula diagonala_zaslona missing: %v", got)
}
if got["locljivost"] != "1920 x 1080" {
t.Fatalf("locljivost missing: %v", got)
}
if got["vrsta_panela"] != "IPS" {
t.Fatalf("vrsta_panela missing: %v", got)
}
if _, ok := got["kontrast"]; ok {
t.Fatalf("non-formula junk must drop: %v", got)
}
if _, ok := got["velikost_zaslona"]; ok {
t.Fatalf("unmapped alias key should not persist after filter: %v", got)
}
}
func TestRunSteps_parseFillMapsMonitorFormulaAttrs(t *testing.T) {
e := &Engine{}
in := ProductInput{
GTIN: "6941948701199",
Name: "Monitor Xiaomi A27i",
Mapped: map[string]any{
"name": "Monitor Xiaomi A27i",
"category": "25",
"brand": "Xiaomi",
"specifications": map[string]any{
"barva": "Črna",
"locljivost": "1920 x 1080",
"odzivni-cas": "6 ms",
"velikost-zaslona": "27\"",
"nastavljiv-po-visini": "Ne",
"zaslon-na-dotik": "Ne",
"kontrast": "1000:1",
},
},
CategoryAttrKeys: map[string]map[string]struct{}{
"25": {
"barva": {},
"diagonala_zaslona": {},
"diagonala-zaslona": {},
"locljivost": {},
"nastavljivo_po_visini": {},
"nastavljivo-po-visini": {},
"odzivni_cas": {},
"odzivni-cas": {},
"zaslon_obcutljiv_na_dotik": {},
"zaslon-obcutljiv-na-dotik": {},
},
},
}
out, err := e.RunSteps(context.Background(), "co", in, "attributes", nil, StepPolicy{})
if err != nil {
t.Fatal(err)
}
pa := out.ProcessedAttributes
if pa["barva"] != "Črna" || pa["locljivost"] != "1920 x 1080" {
t.Fatalf("exact formula keys: %v", pa)
}
if pa["diagonala_zaslona"] != "27\"" {
t.Fatalf("diagonala_zaslona=%v pa=%v", pa["diagonala_zaslona"], pa)
}
if pa["nastavljivo_po_visini"] != "Ne" {
t.Fatalf("nastavljivo_po_visini=%v", pa["nastavljivo_po_visini"])
}
if pa["zaslon_obcutljiv_na_dotik"] != "Ne" {
t.Fatalf("zaslon_obcutljiv_na_dotik=%v", pa["zaslon_obcutljiv_na_dotik"])
}
if _, ok := pa["kontrast"]; ok {
t.Fatalf("kontrast must not persist: %v", pa)
}
}
func TestRunSteps_parseFillMapsHeadphonesFormulaAttrs(t *testing.T) {
e := &Engine{}
in := ProductInput{
GTIN: "650311612227",
Name: "UVI Gear WRATH V2",
Mapped: map[string]any{
"name": "UVI Gear WRATH V2",
"category": "48",
"brand": "UVI Gear",
"specifications": map[string]any{
"barva": "Črna",
"slusalke-z-mikrofonom": "Da",
"brezzicne": "Ne",
"vrsta-slusalk": "Naglavne",
},
},
CategoryAttrKeys: map[string]map[string]struct{}{
"48": {
"brezzicna_povezava": {},
"brezzicna-povezava": {},
"vgrajen_mikrofon": {},
"vgrajen-mikrofon": {},
},
},
}
out, err := e.RunSteps(context.Background(), "co", in, "attributes", nil, StepPolicy{})
if err != nil {
t.Fatal(err)
}
pa := out.ProcessedAttributes
if pa["vgrajen_mikrofon"] != "Da" {
t.Fatalf("vgrajen_mikrofon=%v pa=%v", pa["vgrajen_mikrofon"], pa)
}
if pa["brezzicna_povezava"] != "Ne" {
t.Fatalf("brezzicna_povezava=%v", pa["brezzicna_povezava"])
}
if _, ok := pa["vrsta_slusalk"]; ok {
t.Fatalf("unlinked feed key must drop: %v", pa)
}
}
apps/api/internal/processing/attrs_backfill.go
+110
View File
@@ -0,0 +1,110 @@
package processing
import (
"bytes"
"context"
"encoding/json"
"fmt"
"github.com/google/uuid"
"github.com/jackc/pgx/v5/pgxpool"
)
// BackfillCompanyProcessedAttributes rewrites attributes and processed_attributes
// for every processed row in a company using AttrsForPersist + category_attributes
// allowlists (same rules as processOne / enhance). Returns rows updated.
func BackfillCompanyProcessedAttributes(ctx context.Context, pool *pgxpool.Pool, companyID uuid.UUID) (int64, error) {
if pool == nil {
return 0, fmt.Errorf("backfill attributes: nil pool")
}
p := &Pipeline{Pool: pool}
sets := p.loadCategoryAttributeKeySets(ctx, companyID, uuid.Nil)
rows, err := pool.Query(ctx, `
SELECT id, COALESCE(category, ''), COALESCE(attributes, '{}'::jsonb), COALESCE(processed_attributes, '{}'::jsonb)
FROM processed_products
WHERE company_id = $1`, companyID)
if err != nil {
return 0, fmt.Errorf("backfill attributes query: %w", err)
}
defer rows.Close()
var updated int64
for rows.Next() {
var (
id uuid.UUID
category string
attrsJSON []byte
procAttrsJSON []byte
)
if err := rows.Scan(&id, &category, &attrsJSON, &procAttrsJSON); err != nil {
return updated, fmt.Errorf("backfill attributes scan: %w", err)
}
attrs := decodeAttrMap(attrsJSON)
procAttrs := decodeAttrMap(procAttrsJSON)
allowed := allowedAttrKeysFromSets(sets, category)
cleanAttrs := AttrsForPersist(attrs, allowed)
cleanProc := AttrsForPersist(procAttrs, allowed)
if len(cleanProc) == 0 {
cleanProc = cleanAttrs
}
newAttrsJSON, err := json.Marshal(cleanAttrs)
if err != nil {
return updated, fmt.Errorf("backfill attributes marshal attrs %s: %w", id, err)
}
newProcJSON, err := json.Marshal(cleanProc)
if err != nil {
return updated, fmt.Errorf("backfill attributes marshal processed %s: %w", id, err)
}
if bytes.Equal(compactJSON(attrsJSON), compactJSON(newAttrsJSON)) &&
bytes.Equal(compactJSON(procAttrsJSON), compactJSON(newProcJSON)) {
continue
}
ct, err := pool.Exec(ctx, `
UPDATE processed_products
SET attributes = $2::jsonb,
processed_attributes = $3::jsonb,
updated_at = now()
WHERE id = $1 AND company_id = $4`, id, string(newAttrsJSON), string(newProcJSON), companyID)
if err != nil {
return updated, fmt.Errorf("backfill attributes update %s: %w", id, err)
}
updated += ct.RowsAffected()
}
if err := rows.Err(); err != nil {
return updated, fmt.Errorf("backfill attributes rows: %w", err)
}
return updated, nil
}
func decodeAttrMap(raw []byte) map[string]any {
out := map[string]any{}
if len(raw) == 0 {
return out
}
var parsed any
if err := json.Unmarshal(raw, &parsed); err != nil {
return out
}
switch t := parsed.(type) {
case map[string]any:
return t
case []any:
for _, entry := range t {
if m, ok := entry.(map[string]any); ok {
if k, ok := m["key"].(string); ok && k != "" {
out[k] = m
}
}
}
}
return out
}
func compactJSON(b []byte) []byte {
var buf bytes.Buffer
if err := json.Compact(&buf, b); err != nil {
return b
}
return buf.Bytes()
}
apps/api/internal/processing/attrs_backfill_test.go
+48
View File
@@ -0,0 +1,48 @@
package processing
import (
"encoding/json"
"testing"
)
func TestAttrsForPersist_nilAllowlistSanitizeOnly(t *testing.T) {
got := AttrsForPersist(map[string]any{
"name": "TV Mount",
"brand": "Ostalo",
"zavora": "Mehanska",
}, nil)
if _, ok := got["name"]; ok {
t.Fatalf("name must be stripped: %v", got)
}
if got["zavora"] != "Mehanska" {
t.Fatalf("nil allowlist keeps non-reserved: %v", got)
}
}
func TestCompactJSON_stable(t *testing.T) {
a := []byte(`{"b":1,"a":2}`)
b := []byte(`{ "a" : 2 , "b" : 1 }`)
ca := compactJSON(a)
cb := compactJSON(b)
var ma, mb map[string]any
if err := json.Unmarshal(ca, &ma); err != nil {
t.Fatal(err)
}
if err := json.Unmarshal(cb, &mb); err != nil {
t.Fatal(err)
}
if ma["a"] != mb["a"] || ma["b"] != mb["b"] {
t.Fatalf("compact mismatch %s vs %s", ca, cb)
}
}
func TestDecodeAttrMap_objectAndKeyedArray(t *testing.T) {
obj := decodeAttrMap([]byte(`{"brand":"Ostalo","zavora":"x"}`))
if obj["brand"] != "Ostalo" {
t.Fatalf("obj=%v", obj)
}
arr := decodeAttrMap([]byte(`[{"key":"brand","value":"Ostalo"},{"key":"zavora","value":"x"}]`))
if _, ok := arr["brand"]; !ok {
t.Fatalf("arr=%v", arr)
}
}
apps/api/internal/processing/catalog_fix.go
+386
View File
@@ -0,0 +1,386 @@
package processing
import (
"context"
"encoding/json"
"fmt"
"strings"
"github.com/descrybe/descrybe-v2/apps/api/internal/catalog"
"github.com/descrybe/descrybe-v2/apps/api/internal/company"
"github.com/google/uuid"
"github.com/jackc/pgx/v5/pgxpool"
)
// CatalogFixResult summarizes in-place catalog hygiene (never clears the catalog).
type CatalogFixResult struct {
WeakHashesCleared int `json:"weak_hashes_cleared"`
CategoriesBackfilled int `json:"categories_backfilled"`
DescriptionsNormalized int `json:"descriptions_normalized"`
DescriptionsBackfilled int `json:"descriptions_backfilled"`
NamesBackfilled int `json:"names_backfilled"`
MappedDescriptionsFlat int `json:"mapped_descriptions_flattened"`
MetaBackfilled int `json:"meta_backfilled"`
ProductsScanned int `json:"products_scanned"`
}
// ClearWeakEnhanceHashes removes enhance_input_hash from field_sources and
// localized_content when the product description is weak (poisoned skip hashes).
// Delegates to catalog.RepairWeakEnhanceHashesDetailed (canonical implementation).
func ClearWeakEnhanceHashes(ctx context.Context, pool *pgxpool.Pool, companyID uuid.UUID) (cleared int, scanned int, clearedRawIDs []uuid.UUID, err error) {
res, err := catalog.RepairWeakEnhanceHashesDetailed(ctx, pool, companyID)
if err != nil {
return 0, 0, nil, err
}
return int(res.Cleared), res.Scanned, res.ClearedRawIDs, nil
}
// BackfillCategoriesFromMapped copies mapped_data category unique_ids onto
// processed_products.category when the processed category is empty/"none".
// Uses Go-side categoryUniqueIDFromMaps (nested/array shapes) and validates
// against the company taxonomy when categories exist.
func BackfillCategoriesFromMapped(ctx context.Context, pool *pgxpool.Pool, companyID uuid.UUID) (updated int, err error) {
if pool == nil {
return 0, fmt.Errorf("nil pool")
}
valid, err := loadCompanyCategoryUniqueIDs(ctx, pool, companyID)
if err != nil {
return 0, err
}
rows, err := pool.Query(ctx, `
SELECT pp.id,
COALESCE(pp.category, ''),
COALESCE(rp.mapped_data, '{}'::jsonb),
COALESCE(rp.raw_data, '{}'::jsonb)
FROM processed_products pp
JOIN raw_products rp ON rp.id = pp.raw_product_id AND rp.company_id = pp.company_id
WHERE pp.company_id = $1
AND (
pp.category IS NULL
OR btrim(pp.category) = ''
OR lower(btrim(pp.category)) = 'none'
)`, companyID)
if err != nil {
return 0, err
}
defer rows.Close()
for rows.Next() {
var (
id uuid.UUID
prior string
mappedB []byte
rawB []byte
)
if err := rows.Scan(&id, &prior, &mappedB, &rawB); err != nil {
return updated, err
}
mapped, err := decodeJSONObject(mappedB)
if err != nil {
return updated, err
}
raw, err := decodeJSONObject(rawB)
if err != nil {
return updated, err
}
cat := categoryUniqueIDFromMaps(mapped, raw)
if cat == "" {
continue
}
if len(valid) > 0 {
if _, ok := valid[cat]; !ok {
continue
}
}
ct, err := pool.Exec(ctx, `
UPDATE processed_products
SET category = $2,
field_sources = jsonb_set(
COALESCE(field_sources, '{}'::jsonb),
'{category}',
'"mapped_backfill"'::jsonb,
true
),
updated_at = now()
WHERE id = $1 AND company_id = $3
AND (
category IS NULL
OR btrim(category) = ''
OR lower(btrim(category)) = 'none'
)`,
id, cat, companyID)
if err != nil {
return updated, err
}
if ct.RowsAffected() > 0 {
updated++
}
_ = prior
}
return updated, rows.Err()
}
func loadCompanyCategoryUniqueIDs(ctx context.Context, pool *pgxpool.Pool, companyID uuid.UUID) (map[string]struct{}, error) {
out := map[string]struct{}{}
rows, err := pool.Query(ctx, `
SELECT unique_id FROM categories
WHERE company_id = $1 AND COALESCE(NULLIF(btrim(unique_id), ''), '') <> ''`, companyID)
if err != nil {
return nil, err
}
defer rows.Close()
for rows.Next() {
var uid string
if err := rows.Scan(&uid); err != nil {
return nil, err
}
uid = strings.TrimSpace(uid)
if uid != "" {
out[uid] = struct{}{}
}
}
return out, rows.Err()
}
// FixCatalogHygiene clears poisoned enhance hashes and optionally backfills categories.
func FixCatalogHygiene(ctx context.Context, pool *pgxpool.Pool, companyID uuid.UUID) (CatalogFixResult, error) {
out, _, err := FixCatalogHygieneWithIDs(ctx, pool, companyID, true)
return out, err
}
// FixCatalogHygieneWithIDs is FixCatalogHygiene plus raw_product_ids that had weak hashes cleared.
func FixCatalogHygieneWithIDs(ctx context.Context, pool *pgxpool.Pool, companyID uuid.UUID, backfillCategories bool) (CatalogFixResult, []uuid.UUID, error) {
var out CatalogFixResult
cleared, scanned, ids, err := ClearWeakEnhanceHashes(ctx, pool, companyID)
if err != nil {
return out, nil, err
}
out.WeakHashesCleared = cleared
out.ProductsScanned = scanned
flat, err := FlattenMappedDescriptionArrays(ctx, pool, companyID)
if err != nil {
return out, ids, err
}
out.MappedDescriptionsFlat = flat
if backfillCategories {
// Go-side unique_id extraction (nested/array) + taxonomy validation.
cats, err := BackfillCategoriesFromMapped(ctx, pool, companyID)
if err != nil {
return out, ids, err
}
out.CategoriesBackfilled = cats
descN, err := BackfillDescriptionsFromMapped(ctx, pool, companyID)
if err != nil {
return out, ids, err
}
out.DescriptionsBackfilled = descN
nameN, err := BackfillPollutedNamesFromMapped(ctx, pool, companyID)
if err != nil {
return out, ids, err
}
out.NamesBackfilled = nameN
metaN, err := BackfillMissingMeta(ctx, pool, companyID)
if err != nil {
return out, ids, err
}
out.MetaBackfilled = metaN
}
descs, err := NormalizeProcessedDescriptions(ctx, pool, companyID)
if err != nil {
return out, ids, err
}
out.DescriptionsNormalized = descs
return out, ids, nil
}
// NormalizeProcessedDescriptions rewrites processed_description to the plain-text
// form produced by PlainDescriptionFromAny (unwraps JSON arrays, strips HTML). When
// processed_description is empty, normalizes from description. Returns rows updated.
func NormalizeProcessedDescriptions(ctx context.Context, pool *pgxpool.Pool, companyID uuid.UUID) (int, error) {
if pool == nil {
return 0, fmt.Errorf("normalize descriptions: nil pool")
}
rows, err := pool.Query(ctx, `
SELECT id,
COALESCE(description, ''),
COALESCE(processed_description, '')
FROM processed_products
WHERE company_id = $1`, companyID)
if err != nil {
return 0, fmt.Errorf("normalize descriptions query: %w", err)
}
defer rows.Close()
updated := 0
for rows.Next() {
var (
id uuid.UUID
desc, processedDesc string
)
if err := rows.Scan(&id, &desc, &processedDesc); err != nil {
return updated, fmt.Errorf("normalize descriptions scan: %w", err)
}
raw := strings.TrimSpace(processedDesc)
if raw == "" {
raw = strings.TrimSpace(desc)
}
if raw == "" {
continue
}
plain := plainDescriptionFromStored(raw)
if plain == "" || plain == strings.TrimSpace(processedDesc) {
continue
}
ct, err := pool.Exec(ctx, `
UPDATE processed_products
SET processed_description = $2,
updated_at = now()
WHERE id = $1 AND company_id = $3`, id, plain, companyID)
if err != nil {
return updated, fmt.Errorf("normalize descriptions update %s: %w", id, err)
}
updated += int(ct.RowsAffected())
}
if err := rows.Err(); err != nil {
return updated, fmt.Errorf("normalize descriptions rows: %w", err)
}
return updated, nil
}
// plainDescriptionFromStored handles DB text that may itself be a JSON array/string.
func plainDescriptionFromStored(raw string) string {
raw = strings.TrimSpace(raw)
if raw == "" {
return ""
}
if strings.HasPrefix(raw, "[") || strings.HasPrefix(raw, "{") {
var decoded any
if err := json.Unmarshal([]byte(raw), &decoded); err == nil {
if s := PlainDescriptionFromAny(decoded); s != "" {
return s
}
}
}
return PlainDescriptionFromAny(raw)
}
// RecommendReprocessRawProductIDs returns how many products should be reprocessed
// after Fix A1 (weak desc and/or missing enhance hash and/or empty category), plus
// an optional sample of raw_product_ids (limit 0 = no sample).
func RecommendReprocessRawProductIDs(ctx context.Context, pool *pgxpool.Pool, companyID uuid.UUID, prefer []uuid.UUID, sampleLimit int) (needed int, sample []uuid.UUID, err error) {
if pool == nil {
return 0, nil, fmt.Errorf("nil pool")
}
seen := map[uuid.UUID]struct{}{}
add := func(id uuid.UUID) {
if id == uuid.Nil {
return
}
if _, ok := seen[id]; ok {
return
}
seen[id] = struct{}{}
needed++
if sampleLimit > 0 && len(sample) < sampleLimit {
sample = append(sample, id)
}
}
for _, id := range prefer {
add(id)
}
rows, err := pool.Query(ctx, `
SELECT raw_product_id,
COALESCE(name, ''),
COALESCE(description, ''),
COALESCE(processed_name, ''),
COALESCE(processed_description, ''),
COALESCE(category, ''),
COALESCE(field_sources, '{}'::jsonb),
COALESCE(localized_content, '{}'::jsonb)
FROM processed_products
WHERE company_id = $1`, companyID)
if err != nil {
return 0, nil, err
}
defer rows.Close()
for rows.Next() {
var (
rawID uuid.UUID
name, desc, processedName, processedDesc, category string
fsRaw, locRaw []byte
)
if err := rows.Scan(&rawID, &name, &desc, &processedName, &processedDesc, &category, &fsRaw, &locRaw); err != nil {
return needed, sample, err
}
primaryDesc := strings.TrimSpace(processedDesc)
if primaryDesc == "" {
primaryDesc = desc
}
primaryName := strings.TrimSpace(processedName)
if primaryName == "" {
primaryName = name
}
fs, err := decodeJSONObject(fsRaw)
if err != nil {
return needed, sample, err
}
loc, err := company.DecodeLocalizedContent(locRaw)
if err != nil {
return needed, sample, err
}
needs := false
if strings.TrimSpace(category) == "" {
needs = true
}
if isPromptLabelTitle(primaryName) || isPromptLabelTitle(processedName) || isPromptLabelTitle(name) {
needs = true
}
if isWeakPriorEnhanceDescription(primaryDesc, primaryName, name) {
needs = true
}
if _, has := fs[FieldEnhanceInputHash]; !has {
// Missing hash after weak clear / never enhanced — recommend when desc weak or empty category.
if isWeakPriorEnhanceDescription(primaryDesc, primaryName, name) || strings.TrimSpace(category) == "" {
needs = true
}
}
for _, fields := range loc {
d := strings.TrimSpace(fields.ProcessedDescription)
n := strings.TrimSpace(fields.ProcessedName)
if n == "" {
n = primaryName
}
if fields.EnhanceInputHash == "" && isWeakPriorEnhanceDescription(d, n, primaryName, name) {
needs = true
break
}
}
if needs {
add(rawID)
}
}
return needed, sample, rows.Err()
}
func decodeJSONObject(raw []byte) (map[string]any, error) {
out := map[string]any{}
if len(raw) == 0 {
return out, nil
}
if err := json.Unmarshal(raw, &out); err != nil {
return nil, err
}
if out == nil {
out = map[string]any{}
}
return out, nil
}
apps/api/internal/processing/catalog_fix_test.go
+20
View File
@@ -0,0 +1,20 @@
package processing
import (
"testing"
)
func TestIsWeakPriorEnhanceDescription_forFixCatalog(t *testing.T) {
t.Parallel()
title := "Acme Widget Pro 12"
if !isWeakPriorEnhanceDescription("", title) {
t.Fatal("empty desc should be weak")
}
if !isWeakPriorEnhanceDescription(title, title) {
t.Fatal("title echo should be weak")
}
good := "Durable widget for everyday kitchen use with clear specs."
if isWeakPriorEnhanceDescription(good, title) {
t.Fatal("factual desc should not be weak")
}
}
apps/api/internal/processing/category.go
+184
View File
@@ -0,0 +1,184 @@
package processing
import (
"strconv"
"strings"
"unicode"
)
// categoryDisplayLabel returns the human category name for meta / synthesize /
// {{category}}. Prefers CategoryName; never falls back to a digits-only unique_id.
func categoryDisplayLabel(result StepResult) string {
if n := strings.TrimSpace(result.CategoryName); n != "" {
return n
}
cat := strings.TrimSpace(result.Category)
if cat == "" || isDigitsOnlyCategoryID(cat) {
return ""
}
return cat
}
// resolveCategoryDisplayName looks up unique_id → name; returns "" when unknown
// so callers do not treat "50" as a human label.
func resolveCategoryDisplayName(uid string, namesByUID map[string]string) string {
uid = strings.TrimSpace(uid)
if uid == "" || len(namesByUID) == 0 {
return ""
}
return strings.TrimSpace(namesByUID[uid])
}
func syncCategoryName(out *StepResult, namesByUID map[string]string) {
if out == nil {
return
}
if n := resolveCategoryDisplayName(out.Category, namesByUID); n != "" {
out.CategoryName = n
}
}
func isDigitsOnlyCategoryID(s string) bool {
if s == "" {
return false
}
for _, r := range s {
if !unicode.IsDigit(r) {
return false
}
}
return true
}
// categoryUniqueIDFromAny extracts a company category unique_id from feed/mapped
// shapes: plain string/number, nested {unique_id|category_unique_id|#text|…}, or
// a one-element array of those. Returns "" for empty/"none"/unusable values.
func categoryUniqueIDFromAny(v any) string {
if v == nil {
return ""
}
switch t := v.(type) {
case string:
return normalizeCategoryUniqueID(t)
case float64:
if t == float64(int64(t)) {
return normalizeCategoryUniqueID(strconv.FormatInt(int64(t), 10))
}
return normalizeCategoryUniqueID(strconv.FormatFloat(t, 'f', -1, 64))
case float32:
return categoryUniqueIDFromAny(float64(t))
case int:
return normalizeCategoryUniqueID(strconv.Itoa(t))
case int64:
return normalizeCategoryUniqueID(strconv.FormatInt(t, 10))
case int32:
return normalizeCategoryUniqueID(strconv.FormatInt(int64(t), 10))
case jsonNumberStringer:
return normalizeCategoryUniqueID(t.String())
case map[string]any:
for _, k := range []string{
"unique_id", "category_unique_id", "categoryUniqueId",
"#text", "text", "id", "value", "code",
} {
if s := categoryUniqueIDFromAny(t[k]); s != "" {
return s
}
}
return ""
case []any:
for _, item := range t {
if s := categoryUniqueIDFromAny(item); s != "" {
return s
}
}
return ""
default:
return ""
}
}
// jsonNumberStringer matches encoding/json.Number without importing encoding/json here.
type jsonNumberStringer interface {
String() string
}
func normalizeCategoryUniqueID(s string) string {
s = strings.TrimSpace(s)
if s == "" || s == "<nil>" || strings.EqualFold(s, "none") {
return ""
}
return s
}
// categoryUniqueIDFromMaps returns the first usable category unique_id from maps
// (normalized/mapped/raw). Prefer canonical keys; mapped should win by call order.
func categoryUniqueIDFromMaps(maps ...map[string]any) string {
for _, m := range maps {
if m == nil {
continue
}
for _, k := range []string{
"category", "category_unique_id", "categoryUniqueId",
"Category", "product_type", "productType",
} {
if s := categoryUniqueIDFromAny(m[k]); s != "" {
return s
}
}
}
return ""
}
// coerceMappedCategoryUniqueID writes a string unique_id onto m["category"] when
// category (or category_unique_id) can be resolved — so RunSteps string paths stay deterministic.
func coerceMappedCategoryUniqueID(m map[string]any) string {
if m == nil {
return ""
}
cat := categoryUniqueIDFromMaps(m)
if cat == "" {
return ""
}
m["category"] = cat
if _, ok := m["category_unique_id"]; !ok {
m["category_unique_id"] = cat
}
return cat
}
// applyCategoryFromMapped sets StepResult.Category from feed/mapped unique_id when present.
// Does not clear an existing Category when maps lack a usable value (caller may preserve prior).
func applyCategoryFromMapped(out *StepResult, maps ...map[string]any) {
if out == nil {
return
}
cat := categoryUniqueIDFromMaps(maps...)
if cat == "" {
return
}
out.Category = SanitizeText(cat)
if out.FieldSources == nil {
out.FieldSources = map[string]any{}
}
out.FieldSources["category"] = "mapped"
}
// filterCategoryIfInvalid clears Category when the company has a known unique_id set
// and the value is not in that set. Empty valid set means "skip validation" (tests / no taxonomy).
func filterCategoryIfInvalid(out *StepResult, valid map[string]struct{}) {
if out == nil || len(valid) == 0 {
return
}
cat := strings.TrimSpace(out.Category)
if cat == "" {
return
}
if _, ok := valid[cat]; ok {
return
}
out.Category = ""
if out.FieldSources != nil {
delete(out.FieldSources, "category")
}
out.Notes = append(out.Notes, "category: ignored (unknown unique_id for company)")
}
apps/api/internal/processing/category_backfill.go
+117
View File
@@ -0,0 +1,117 @@
package processing
import (
"context"
"fmt"
"github.com/google/uuid"
"github.com/jackc/pgx/v5/pgxpool"
)
// CategoryBackfillResult is the outcome of BackfillProcessedCategoriesFromMapped.
type CategoryBackfillResult struct {
Updated int64 `json:"updated"`
}
// BackfillProcessedCategoriesFromMapped fills empty processed_products.category from
// raw_products.mapped_data when a usable unique_id is present. When the company has
// categories rows, only known unique_ids are applied (same rule as processOne filter).
// Safe for admin "Fix catalog" and migrator reuse. Does not invent categories via vector.
func BackfillProcessedCategoriesFromMapped(ctx context.Context, pool *pgxpool.Pool, companyID uuid.UUID) (CategoryBackfillResult, error) {
var out CategoryBackfillResult
if pool == nil {
return out, fmt.Errorf("category backfill: nil pool")
}
if companyID == uuid.Nil {
return out, fmt.Errorf("category backfill: empty company id")
}
ct, err := pool.Exec(ctx, `
WITH src AS (
SELECT
p.id AS processed_id,
BTRIM(COALESCE(
CASE
WHEN jsonb_typeof(r.mapped_data->'category') IN ('string', 'number')
THEN NULLIF(BTRIM(r.mapped_data->>'category'), '')
ELSE NULL
END,
NULLIF(BTRIM(r.mapped_data->>'category_unique_id'), ''),
NULLIF(BTRIM(r.mapped_data->'category'->>'unique_id'), ''),
NULLIF(BTRIM(r.mapped_data->'category'->>'category_unique_id'), ''),
NULLIF(BTRIM(r.mapped_data->'category'->>'#text'), ''),
NULLIF(BTRIM(r.mapped_data->'category'->>'id'), ''),
NULLIF(BTRIM(r.mapped_data->'category'->>'value'), ''),
NULLIF(BTRIM(r.mapped_data->'category'->>'code'), '')
)) AS cat_raw
FROM processed_products p
INNER JOIN raw_products r ON r.id = p.raw_product_id AND r.company_id = p.company_id
WHERE p.company_id = $1
AND (
COALESCE(NULLIF(BTRIM(p.category), ''), '') = ''
OR LOWER(BTRIM(p.category)) = 'none'
)
),
mapped AS (
SELECT
processed_id,
CASE
WHEN cat_raw IS NULL OR cat_raw = '' OR LOWER(cat_raw) = 'none' THEN NULL
ELSE cat_raw
END AS cat_uid
FROM src
),
eligible AS (
SELECT m.processed_id, m.cat_uid
FROM mapped m
WHERE m.cat_uid IS NOT NULL
AND (
NOT EXISTS (SELECT 1 FROM categories c WHERE c.company_id = $1)
OR EXISTS (
SELECT 1 FROM categories c
WHERE c.company_id = $1
AND BTRIM(c.unique_id) = m.cat_uid
)
)
)
UPDATE processed_products p
SET category = e.cat_uid,
field_sources = jsonb_set(
COALESCE(p.field_sources, '{}'::jsonb),
'{category}',
'"mapped_backfill"'::jsonb,
true
),
updated_at = now()
FROM eligible e
WHERE p.id = e.processed_id`, companyID)
if err != nil {
return out, fmt.Errorf("category backfill: %w", err)
}
out.Updated = ct.RowsAffected()
return out, nil
}
// CountProcessedCategoryCoverage returns how many processed rows for companyID have
// a non-empty category vs empty/"none".
func CountProcessedCategoryCoverage(ctx context.Context, pool *pgxpool.Pool, companyID uuid.UUID) (withCat, withoutCat int64, err error) {
if pool == nil {
return 0, 0, fmt.Errorf("category coverage: nil pool")
}
err = pool.QueryRow(ctx, `
SELECT
COUNT(*) FILTER (
WHERE COALESCE(NULLIF(BTRIM(category), ''), '') <> ''
AND LOWER(BTRIM(category)) <> 'none'
),
COUNT(*) FILTER (
WHERE COALESCE(NULLIF(BTRIM(category), ''), '') = ''
OR LOWER(BTRIM(category)) = 'none'
)
FROM processed_products
WHERE company_id = $1`, companyID).Scan(&withCat, &withoutCat)
if err != nil {
return 0, 0, fmt.Errorf("category coverage: %w", err)
}
return withCat, withoutCat, nil
}
apps/api/internal/processing/category_test.go
+404
View File
@@ -0,0 +1,404 @@
package processing
import (
"context"
"fmt"
"os"
"strings"
"testing"
"time"
"github.com/google/uuid"
"github.com/jackc/pgx/v5/pgxpool"
)
func TestCategoryUniqueIDFromAny(t *testing.T) {
t.Parallel()
cases := []struct {
in any
want string
}{
{nil, ""},
{"", ""},
{"none", ""},
{"NONE", ""},
{"50", "50"},
{50, "50"},
{float64(50), "50"},
{map[string]any{"unique_id": "28"}, "28"},
{map[string]any{"category_unique_id": 48}, "48"},
{map[string]any{"#text": "120"}, "120"},
{[]any{map[string]any{"unique_id": "7"}}, "7"},
{map[string]any{"name": "OnlyName"}, ""},
}
for i, tc := range cases {
if got := categoryUniqueIDFromAny(tc.in); got != tc.want {
t.Fatalf("case %d: got %q want %q", i, got, tc.want)
}
}
}
func TestRunSteps_full_mappedUniqueIDCategory(t *testing.T) {
e := &Engine{
Completer: stubCompleter{fn: func(system, user string) (Completion, error) {
return Completion{Text: `{"name":"N","description":"D"}`, TotalTokens: 1}, nil
}},
Vector: NoopVectorCategorizer{},
}
out, err := e.RunSteps(context.Background(), "co", ProductInput{
GTIN: "8606019604493",
Mapped: map[string]any{
"name": "Cooker",
"description": "stove",
"category": "50",
},
}, "full", nil, StepPolicy{AllowAI: true, AllowEPREL: false})
if err != nil {
t.Fatal(err)
}
if out.Category != "50" {
t.Fatalf("category=%q want 50", out.Category)
}
if src, _ := out.FieldSources["category"].(string); src != "mapped" {
t.Fatalf("field_sources.category=%v want mapped", out.FieldSources["category"])
}
}
func TestRunSteps_full_categoryUniqueIDKey(t *testing.T) {
e := &Engine{Vector: NoopVectorCategorizer{}}
out, err := e.RunSteps(context.Background(), "co", ProductInput{
Mapped: map[string]any{
"name": "Headphones",
"category_unique_id": "48",
},
}, "normalize_only", nil, StepPolicy{})
if err != nil {
t.Fatal(err)
}
if out.Category != "48" {
t.Fatalf("category=%q want 48", out.Category)
}
}
func TestRunSteps_full_nestedCategoryUniqueID(t *testing.T) {
e := &Engine{Vector: NoopVectorCategorizer{}}
out, err := e.RunSteps(context.Background(), "co", ProductInput{
Mapped: map[string]any{
"name": "Dryer",
"category": map[string]any{
"unique_id": "52",
"name": "Sušilni stroji",
},
},
}, "normalize_only", nil, StepPolicy{})
if err != nil {
t.Fatal(err)
}
if out.Category != "52" {
t.Fatalf("category=%q want 52", out.Category)
}
}
func TestCategoryDisplayLabel_andSync(t *testing.T) {
t.Parallel()
out := StepResult{Category: "50"}
syncCategoryName(&out, map[string]string{"50": "Štedilniki"})
if out.CategoryName != "Štedilniki" {
t.Fatalf("CategoryName=%q", out.CategoryName)
}
if out.Category != "50" {
t.Fatalf("Category unique_id mutated: %q", out.Category)
}
if got := categoryDisplayLabel(out); got != "Štedilniki" {
t.Fatalf("display=%q", got)
}
synth := synthesizeDescriptionFromTitle("Cooker X", categoryDisplayLabel(out), "sl", nil)
if strings.Contains(synth, "kategoriji 50") || strings.Contains(synth, " 50") {
t.Fatalf("synth used unique_id: %q", synth)
}
if !strings.Contains(synth, "Štedilniki") {
t.Fatalf("synth missing display name: %q", synth)
}
}
func TestFilterCategoryIfInvalid(t *testing.T) {
t.Parallel()
out := StepResult{Category: "50", FieldSources: map[string]any{"category": "mapped"}}
filterCategoryIfInvalid(&out, map[string]struct{}{"50": {}})
if out.Category != "50" {
t.Fatalf("valid unique_id cleared: %q", out.Category)
}
filterCategoryIfInvalid(&out, map[string]struct{}{"99": {}})
if out.Category != "" {
t.Fatalf("invalid unique_id kept: %q", out.Category)
}
out.Category = "50"
filterCategoryIfInvalid(&out, nil)
if out.Category != "50" {
t.Fatalf("empty valid set should skip filter: %q", out.Category)
}
}
func TestLoadV1ProcessJobItems_resolvesCategoryName(t *testing.T) {
dsn := os.Getenv("DATABASE_URL")
if dsn == "" {
t.Skip("DATABASE_URL not set")
}
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
pg, err := pgxpool.New(ctx, dsn)
if err != nil {
t.Fatal(err)
}
defer pg.Close()
companyID := uuid.New()
userID := uuid.New()
rawID := uuid.New()
jobID := uuid.New()
ppID := uuid.New()
gtin := "cat-v1-" + companyID.String()[:8]
if _, err := pg.Exec(ctx, `INSERT INTO companies (id, name) VALUES ($1, $2)`, companyID, "cat-v1-test"); err != nil {
t.Fatal(err)
}
// users.id may be required for processing_jobs — reuse an existing user when possible.
err = pg.QueryRow(ctx, `SELECT id FROM users ORDER BY created_at DESC LIMIT 1`).Scan(&userID)
if err != nil {
t.Skip("no users rows available")
}
defer func() {
_, _ = pg.Exec(context.Background(), `DELETE FROM processing_job_products WHERE job_id = $1`, jobID)
_, _ = pg.Exec(context.Background(), `DELETE FROM processing_jobs WHERE id = $1`, jobID)
_, _ = pg.Exec(context.Background(), `DELETE FROM processed_products WHERE company_id = $1`, companyID)
_, _ = pg.Exec(context.Background(), `DELETE FROM raw_products WHERE company_id = $1`, companyID)
_, _ = pg.Exec(context.Background(), `DELETE FROM categories WHERE company_id = $1`, companyID)
_, _ = pg.Exec(context.Background(), `DELETE FROM companies WHERE id = $1`, companyID)
}()
if _, err := pg.Exec(ctx, `
INSERT INTO categories (id, company_id, name, unique_id, is_active)
VALUES (gen_random_uuid(), $1, 'Štedilniki', '50', true)`, companyID); err != nil {
t.Fatal(err)
}
if _, err := pg.Exec(ctx, `
INSERT INTO raw_products (id, company_id, gtin, raw_data, mapped_data, is_processed, processing_status)
VALUES ($1, $2, $3, '{}'::jsonb, '{"category":"50","name":"Cooker"}'::jsonb, true, 'processed')`,
rawID, companyID, gtin); err != nil {
t.Fatal(err)
}
if _, err := pg.Exec(ctx, `
INSERT INTO processed_products (
id, company_id, product_id, name, category, description, processed_name, processed_description,
raw_product_id, status, attributes, processed_attributes, field_sources
) VALUES (
$1, $2, $3, 'Cooker', '50', 'stove', 'Cooker', 'stove',
$4, 'needs_review', '{}'::jsonb, '{}'::jsonb, '{"category":"mapped"}'::jsonb
)`, ppID, companyID, gtin, rawID); err != nil {
t.Fatal(err)
}
if _, err := pg.Exec(ctx, `
INSERT INTO processing_jobs (id, company_id, user_id, status, processing_type, total_products, processed_products)
VALUES ($1, $2, $3, 'completed', 'full', 1, 1)`, jobID, companyID, userID); err != nil {
t.Fatal(err)
}
if _, err := pg.Exec(ctx, `
INSERT INTO processing_job_products (id, job_id, raw_product_id, processed_product_id, status)
VALUES (gen_random_uuid(), $1, $2, $3, 'processed')`, jobID, rawID, ppID); err != nil {
t.Fatal(err)
}
p := NewPipeline(pg)
items, err := p.LoadV1ProcessJobItems(ctx, companyID, jobID, "full")
if err != nil {
t.Fatal(err)
}
if len(items) != 1 {
t.Fatalf("items=%d want 1", len(items))
}
if got := fmt.Sprint(items[0]["category"]); got != "50" {
t.Fatalf("category=%v want 50", items[0]["category"])
}
if got := fmt.Sprint(items[0]["category_name"]); got != "Štedilniki" {
t.Fatalf("category_name=%v want Štedilniki", items[0]["category_name"])
}
}
type stubVectorCategorizer struct {
enabled bool
cat string
err error
calls int
}
func (s *stubVectorCategorizer) Enabled() bool { return s.enabled }
func (s *stubVectorCategorizer) SuggestCategory(context.Context, string, string, []string) (string, error) {
s.calls++
return s.cat, s.err
}
func TestRunSteps_vectorCategoryRequiresAllowAI(t *testing.T) {
vec := &stubVectorCategorizer{enabled: true, cat: "28"}
e := &Engine{Vector: vec}
out, err := e.RunSteps(context.Background(), "co", ProductInput{
Name: "TV mount",
Mapped: map[string]any{
"name": "TV mount",
},
}, "normalize_only", nil, StepPolicy{AllowAI: false})
if err != nil {
t.Fatal(err)
}
if out.Category != "" {
t.Fatalf("category=%q want empty when AllowAI=false", out.Category)
}
if vec.calls != 0 {
t.Fatalf("vector calls=%d want 0", vec.calls)
}
found := false
for _, n := range out.Notes {
if strings.Contains(n, "category: unset") && strings.Contains(n, "AI/vector not allowed") {
found = true
break
}
}
if !found {
t.Fatalf("expected unset note, got %v", out.Notes)
}
}
func TestRunSteps_vectorCategoryWhenAllowAI(t *testing.T) {
vec := &stubVectorCategorizer{enabled: true, cat: "28"}
e := &Engine{Vector: vec}
out, err := e.RunSteps(context.Background(), "co", ProductInput{
Name: "TV mount",
Mapped: map[string]any{
"name": "TV mount",
},
}, "normalize_only", nil, StepPolicy{AllowAI: true})
if err != nil {
t.Fatal(err)
}
if out.Category != "28" {
t.Fatalf("category=%q want 28", out.Category)
}
if src, _ := out.FieldSources["category"].(string); src != "vector" {
t.Fatalf("field_sources.category=%v want vector", out.FieldSources["category"])
}
if vec.calls != 1 {
t.Fatalf("vector calls=%d want 1", vec.calls)
}
}
func TestRunSteps_mappedCategorySkipsVector(t *testing.T) {
vec := &stubVectorCategorizer{enabled: true, cat: "99"}
e := &Engine{Vector: vec}
out, err := e.RunSteps(context.Background(), "co", ProductInput{
Mapped: map[string]any{
"name": "Cooker",
"category": "50",
},
}, "normalize_only", nil, StepPolicy{AllowAI: true})
if err != nil {
t.Fatal(err)
}
if out.Category != "50" {
t.Fatalf("category=%q want 50", out.Category)
}
if vec.calls != 0 {
t.Fatalf("vector should not run when mapped unique_id present: calls=%d", vec.calls)
}
}
func TestBackfillProcessedCategoriesFromMapped(t *testing.T) {
dsn := os.Getenv("DATABASE_URL")
if dsn == "" {
t.Skip("DATABASE_URL not set")
}
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
pg, err := pgxpool.New(ctx, dsn)
if err != nil {
t.Fatal(err)
}
defer pg.Close()
companyID := uuid.New()
rawID := uuid.New()
ppID := uuid.New()
gtin := "cat-bf-" + companyID.String()[:8]
if _, err := pg.Exec(ctx, `INSERT INTO companies (id, name) VALUES ($1, $2)`, companyID, "cat-backfill-test"); err != nil {
t.Fatal(err)
}
defer func() {
_, _ = pg.Exec(context.Background(), `DELETE FROM processed_products WHERE company_id = $1`, companyID)
_, _ = pg.Exec(context.Background(), `DELETE FROM raw_products WHERE company_id = $1`, companyID)
_, _ = pg.Exec(context.Background(), `DELETE FROM categories WHERE company_id = $1`, companyID)
_, _ = pg.Exec(context.Background(), `DELETE FROM companies WHERE id = $1`, companyID)
}()
if _, err := pg.Exec(ctx, `
INSERT INTO categories (id, company_id, name, unique_id, is_active)
VALUES (gen_random_uuid(), $1, 'TV mounts', '28', true)`, companyID); err != nil {
t.Fatal(err)
}
if _, err := pg.Exec(ctx, `
INSERT INTO raw_products (id, company_id, gtin, raw_data, mapped_data, is_processed, processing_status)
VALUES ($1, $2, $3, '{}'::jsonb, '{"name":"Mount","category":{"unique_id":"28","name":"TV"}}'::jsonb, true, 'processed')`,
rawID, companyID, gtin); err != nil {
t.Fatal(err)
}
if _, err := pg.Exec(ctx, `
INSERT INTO processed_products (
id, company_id, product_id, name, category, description, processed_name, processed_description,
raw_product_id, status, attributes, processed_attributes, field_sources
) VALUES (
$1, $2, $3, 'Mount', '', 'x', 'Mount', 'x',
$4, 'needs_review', '{}'::jsonb, '{}'::jsonb, '{}'::jsonb
)`, ppID, companyID, gtin, rawID); err != nil {
t.Fatal(err)
}
res, err := BackfillProcessedCategoriesFromMapped(ctx, pg, companyID)
if err != nil {
t.Fatal(err)
}
if res.Updated != 1 {
t.Fatalf("updated=%d want 1", res.Updated)
}
var cat, src string
if err := pg.QueryRow(ctx, `
SELECT COALESCE(category, ''), COALESCE(field_sources->>'category', '')
FROM processed_products WHERE id = $1`, ppID).Scan(&cat, &src); err != nil {
t.Fatal(err)
}
if cat != "28" {
t.Fatalf("category=%q want 28", cat)
}
if src != "mapped_backfill" {
t.Fatalf("field_sources.category=%q want mapped_backfill", src)
}
// Idempotent.
res2, err := BackfillProcessedCategoriesFromMapped(ctx, pg, companyID)
if err != nil {
t.Fatal(err)
}
if res2.Updated != 0 {
t.Fatalf("second backfill updated=%d want 0", res2.Updated)
}
withCat, withoutCat, err := CountProcessedCategoryCoverage(ctx, pg, companyID)
if err != nil {
t.Fatal(err)
}
if withCat != 1 || withoutCat != 0 {
t.Fatalf("coverage with=%d without=%d", withCat, withoutCat)
}
}
apps/api/internal/processing/enhance_hash_test.go
+185 -5
View File
@@ -4,6 +4,8 @@ import (
"context"
"strings"
"testing"
"github.com/descrybe/descrybe-v2/apps/api/internal/company"
)
func TestHashEnhanceInput_stableAndSensitive(t *testing.T) {
@@ -41,18 +43,19 @@ func TestRunSteps_skipsEnhanceWhenInputHashUnchanged(t *testing.T) {
}},
Vector: NoopVectorCategorizer{},
}
priorDesc := "Cached description with enough retail detail for listing copy."
in := ProductInput{
Name: "Widget",
Description: "A widget",
Mapped: map[string]any{"name": "Widget", "description": "A widget"},
Description: "A widget with enough mapped detail for hashing.",
Mapped: map[string]any{"name": "Widget", "description": "A widget with enough mapped detail for hashing."},
PriorProcessedName: "Cached Widget",
PriorProcessedDescription: "Cached description.",
PriorProcessedDescription: priorDesc,
}
// First pass without prior hash to compute the hash shape via enhance path is awkward;
// compute the same hash RunSteps will see after normalize (name/desc from mapped).
// enhance_only: normalize then enhance with out.Name from normalized.
normName := "Widget"
normDesc := "A widget"
normDesc := "A widget with enough mapped detail for hashing."
sysTpl, userTpl := resolveProductPromptTemplates(ProductInput{})
in.PriorEnhanceHash = HashEnhanceInput("", normName, normDesc, "", "", sysTpl, userTpl, map[string]any{})
@@ -66,7 +69,7 @@ func TestRunSteps_skipsEnhanceWhenInputHashUnchanged(t *testing.T) {
if out.ProcessedName != "Cached Widget" {
t.Fatalf("name=%q", out.ProcessedName)
}
if out.ProcessedDescription != "Cached description." {
if out.ProcessedDescription != priorDesc {
t.Fatalf("desc=%q", out.ProcessedDescription)
}
if out.TotalTokens != 0 {
@@ -84,6 +87,15 @@ func TestRunSteps_skipsEnhanceWhenInputHashUnchanged(t *testing.T) {
if shouldDebitProductProcessing(false, out) {
t.Fatal("processOne must not debit when enhance unchanged")
}
// Hash-skip (0 tokens) must still persist known engine mode, not "unknown".
if out.AIProviderMode != AIProviderInternal {
t.Fatalf("ai_provider_mode=%q want %q on hash-skip", out.AIProviderMode, AIProviderInternal)
}
// processOne treats unknown as empty and prefers job modeLabel.
got := preferKnownProviderMode(AIProviderUnknown, AIProviderInternal)
if got != AIProviderInternal {
t.Fatalf("preferKnownProviderMode(unknown, internal)=%q", got)
}
joined := strings.Join(out.Notes, ";")
if !strings.Contains(joined, "unchanged") {
t.Fatalf("notes=%v", out.Notes)
@@ -128,3 +140,171 @@ func TestRunSteps_callsEnhanceWhenInputHashDiffers(t *testing.T) {
t.Fatalf("expected new hash in field_sources, got %q", h)
}
}
func TestRunSteps_reenhancesWhenPriorDescEqualsTitle(t *testing.T) {
calls := 0
e := &Engine{
Completer: stubCompleter{fn: func(system, user string) (Completion, error) {
calls++
return Completion{Text: `{"name":"Widget Pro","description":"A durable retail widget for everyday use with clear specs."}`, TotalTokens: 5}, nil
}},
Vector: NoopVectorCategorizer{},
}
in := ProductInput{
Name: "Widget",
Description: "Widget",
Mapped: map[string]any{"name": "Widget", "description": "Widget"},
PriorProcessedName: "Widget",
PriorProcessedDescription: "Widget",
}
sysTpl, userTpl := resolveProductPromptTemplates(ProductInput{})
in.PriorEnhanceHash = HashEnhanceInput("", "Widget", "Widget", "", "", sysTpl, userTpl, map[string]any{})
out, err := e.RunSteps(context.Background(), "co", in, "enhance_only", nil, StepPolicy{AllowAI: true, AllowEPREL: true})
if err != nil {
t.Fatal(err)
}
if calls != 1 {
t.Fatalf("expected LLM re-enhance when prior desc equals title, calls=%d", calls)
}
if out.ProcessedDescription == "Widget" || out.ProcessedDescription == "" {
t.Fatalf("desc=%q", out.ProcessedDescription)
}
if out.SkipCreditDebit {
t.Fatal("title-echo prior must still debit")
}
}
func TestPreferredProductDescription_skipsTitleEcho(t *testing.T) {
title := "Acme Widget Pro"
if got := preferredProductDescription(title, title, "Acme Widget Pro.", "A durable retail widget for everyday use."); got != "A durable retail widget for everyday use." {
t.Fatalf("got %q", got)
}
if got := preferredProductDescription(title, title, strings.ToUpper(title)); got != "" {
t.Fatalf("expected empty when all candidates echo title, got %q", got)
}
if got := preferredProductDescription(title, "", "<nil>", "Category:"); got != "" {
t.Fatalf("expected empty for unusable candidates, got %q", got)
}
}
func TestRunSteps_emptyDescriptionGetsNonemptyProcessedDesc(t *testing.T) {
calls := 0
e := &Engine{
Completer: stubCompleter{fn: func(system, user string) (Completion, error) {
calls++
return Completion{Text: `{"name":"Gigabyte Monitor","description":""}`, TotalTokens: 4}, nil
}},
Vector: NoopVectorCategorizer{},
}
out, err := e.RunSteps(context.Background(), "co", ProductInput{
Name: "Gigabyte Monitor",
Mapped: map[string]any{"name": "Gigabyte Monitor", "brand": "Gigabyte"},
}, "enhance_only", nil, StepPolicy{AllowAI: true, AllowEPREL: true})
if err != nil {
t.Fatal(err)
}
if calls != 1 {
t.Fatalf("expected enhance call, calls=%d", calls)
}
if strings.TrimSpace(out.ProcessedDescription) == "" {
t.Fatal("expected nonempty ProcessedDescription when title set")
}
if descriptionEchoesTitle(out.ProcessedDescription, out.ProcessedName) {
t.Fatalf("ProcessedDescription must not echo title: name=%q desc=%q", out.ProcessedName, out.ProcessedDescription)
}
if descriptionEchoesTitle(out.ProcessedDescription, "Gigabyte Monitor") {
t.Fatalf("ProcessedDescription must not echo mapped title: desc=%q", out.ProcessedDescription)
}
}
func TestRunSteps_thinOKDoesNotPersistEnhanceHash(t *testing.T) {
e := &Engine{
Completer: stubCompleter{fn: func(system, user string) (Completion, error) {
return Completion{Text: `{"name":"Monitor","description":"ok"}`, TotalTokens: 2}, nil
}},
Vector: NoopVectorCategorizer{},
}
out, err := e.RunSteps(context.Background(), "co", ProductInput{
Name: "Monitor",
Description: "Monitor",
Mapped: map[string]any{"name": "Monitor", "description": "Monitor"},
}, "enhance_only", nil, StepPolicy{AllowAI: true, AllowEPREL: true})
if err != nil {
t.Fatal(err)
}
if h, _ := out.FieldSources[FieldEnhanceInputHash].(string); h != "" && isWeakPriorEnhanceDescription(out.ProcessedDescription, out.ProcessedName) {
t.Fatalf("must not persist hash with weak desc: hash=%q desc=%q", h, out.ProcessedDescription)
}
for _, lf := range out.LocalizedContent {
if lf.EnhanceInputHash != "" && isWeakPriorEnhanceDescription(lf.ProcessedDescription, lf.ProcessedName) {
t.Fatalf("localized hash poisoned with weak desc: %+v", lf)
}
}
}
func TestRunSteps_weakPriorHashDoesNotSkipReprocess(t *testing.T) {
calls := 0
e := &Engine{
Completer: stubCompleter{fn: func(system, user string) (Completion, error) {
calls++
return Completion{Text: `{"name":"Widget Pro","description":"A durable retail widget for everyday use with clear specs."}`, TotalTokens: 5}, nil
}},
Vector: NoopVectorCategorizer{},
}
normName := "Widget"
normDesc := "Widget"
sysTpl, userTpl := resolveProductPromptTemplates(ProductInput{})
priorHash := HashEnhanceInput("", normName, normDesc, "", "", sysTpl, userTpl, map[string]any{})
out, err := e.RunSteps(context.Background(), "co", ProductInput{
Name: "Widget",
Description: "Widget",
Mapped: map[string]any{"name": "Widget", "description": "Widget"},
PriorEnhanceHash: priorHash,
PriorProcessedName: "Widget",
PriorProcessedDescription: "ok",
PriorLocalized: company.LocalizedContent{
"en": {ProcessedName: "Widget", ProcessedDescription: "ok", EnhanceInputHash: priorHash},
},
}, "enhance_only", nil, StepPolicy{AllowAI: true, AllowEPREL: true})
if err != nil {
t.Fatal(err)
}
if calls != 1 {
t.Fatalf("weak prior desc must force completer, calls=%d", calls)
}
if out.SkipCreditDebit {
t.Fatal("weak prior must not SkipCreditDebit")
}
if out.ProcessedDescription == "ok" || out.ProcessedDescription == "Widget" {
t.Fatalf("desc=%q", out.ProcessedDescription)
}
}
func TestRunSteps_failedEnhanceDoesNotPoisonLocalizedHash(t *testing.T) {
e := &Engine{
Completer: stubCompleter{fn: func(system, user string) (Completion, error) {
return Completion{}, context.DeadlineExceeded
}},
Vector: NoopVectorCategorizer{},
}
out, err := e.RunSteps(context.Background(), "co", ProductInput{
Name: "Widget",
Mapped: map[string]any{"name": "Widget", "description": "A widget with enough mapped detail for hashing."},
PriorEnhanceHash: "should-not-survive",
PriorLocalized: company.LocalizedContent{
"en": {ProcessedName: "Old", ProcessedDescription: "Old desc that is long enough to look real.", EnhanceInputHash: "poison"},
},
}, "enhance_only", nil, StepPolicy{AllowAI: true, AllowEPREL: true})
if err != nil {
t.Fatal(err)
}
if _, ok := out.FieldSources[FieldEnhanceInputHash]; ok {
t.Fatalf("failed enhance must not keep field_sources hash, got %v", out.FieldSources[FieldEnhanceInputHash])
}
for lang, lf := range out.LocalizedContent {
if lf.EnhanceInputHash != "" {
t.Fatalf("lang %s: failed enhance must clear localized hash, got %q", lang, lf.EnhanceInputHash)
}
}
}
apps/api/internal/processing/eprel_test.go
+169
View File
@@ -2,7 +2,10 @@ package processing
import (
"context"
"encoding/json"
"strings"
"testing"
"time"
"github.com/descrybe/descrybe-v2/apps/api/internal/eprel"
)
@@ -51,6 +54,12 @@ func TestRunSteps_EPREL_mergesAttributes(t *testing.T) {
if out.ProcessedAttributes["eprel_energy_class"] != "C" {
t.Fatalf("class missing: %v", out.ProcessedAttributes)
}
if out.ProcessedAttributes["energy_class"] != "C" {
t.Fatalf("energy_class not promoted: %v", out.ProcessedAttributes)
}
if out.EPREL["id"] != "246834" || out.EPREL["label"] == "" {
t.Fatalf("out.EPREL=%v", out.EPREL)
}
}
func TestRunSteps_EPREL_disabledOrMissingID(t *testing.T) {
@@ -76,3 +85,163 @@ func TestRunSteps_EPREL_disabledOrMissingID(t *testing.T) {
t.Fatal("should not call fetch without id")
}
}
func TestRunSteps_EPREL_readsIDFromParsedAttrs(t *testing.T) {
st := &stubEprel{
enabled: true,
data: &eprel.Data{
ID: "999001",
Label: "https://eprel.ec.europa.eu/api/product/999001/labels?format=png",
PDF: "https://eprel.ec.europa.eu/fiches/y.pdf",
EnergyClass: "B",
EnergyScale: "A-G",
},
}
e := &Engine{EPREL: st, Completer: HeuristicCompleter{}, Vector: NoopVectorCategorizer{}}
out, err := e.RunSteps(context.Background(), "co", ProductInput{
Mapped: map[string]any{
"name": "Washer",
"specifications": map[string]any{
"eprel_id": "999001",
"brand": "Samsung",
},
},
}, "eprel_only", nil, StepPolicy{AllowAI: false, AllowEPREL: true})
if err != nil {
t.Fatal(err)
}
if st.calls != 1 || st.lastID != "999001" {
t.Fatalf("calls=%d id=%q", st.calls, st.lastID)
}
if out.EPREL["id"] != "999001" || out.EPREL["energy_class"] != "B" {
t.Fatalf("eprel=%v", out.EPREL)
}
if out.ProcessedAttributes["energy_class"] != "B" {
t.Fatalf("energy_class not promoted: %v", out.ProcessedAttributes)
}
if out.ProcessedAttributes["eprel_id"] != "999001" {
t.Fatalf("eprel_id missing: %v", out.ProcessedAttributes)
}
// Nested eprel object must survive sanitize/persist for V1 extract.
nested, _ := out.ProcessedAttributes["eprel"].(map[string]any)
if nested == nil || nested["label"] == nil {
t.Fatalf("nested eprel missing: %v", out.ProcessedAttributes["eprel"])
}
extracted := extractEPRELFromAttrs(out.ProcessedAttributes)
em, ok := extracted.(map[string]any)
if !ok || em["id"] != "999001" || em["label"] == nil || em["pdf"] == nil || em["energy_class"] != "B" {
t.Fatalf("V1 extract=%v", extracted)
}
}
func TestRunSteps_EPREL_skipsPlaceholderID(t *testing.T) {
st := &stubEprel{enabled: true}
e := &Engine{EPREL: st, Completer: HeuristicCompleter{}, Vector: NoopVectorCategorizer{}}
_, err := e.RunSteps(context.Background(), "co", ProductInput{
Mapped: map[string]any{"eprel_id": "0000"},
}, "eprel_only", nil, StepPolicy{AllowEPREL: true})
if err != nil {
t.Fatal(err)
}
if st.calls != 0 {
t.Fatalf("placeholder id must not fetch, calls=%d", st.calls)
}
}
func TestExtractEPRELFromAttrs_flatAndNested(t *testing.T) {
got := extractEPRELFromAttrs(map[string]any{
"eprel_id": "1632113",
"eprel_label": "https://eprel.ec.europa.eu/api/product/1632113/labels?format=png",
"eprel_pdf_url": "https://eprel.ec.europa.eu/fiches/x.pdf",
"eprel_energy_class": "A",
"brand": "Samsung",
})
m, ok := got.(map[string]any)
if !ok {
t.Fatalf("type=%T", got)
}
if m["id"] != "1632113" || m["label"] == nil || m["pdf"] == nil || m["energy_class"] != "A" {
t.Fatalf("got=%v", m)
}
// Nested legacy aliases must fold onto public keys.
got2 := extractEPRELFromAttrs(map[string]any{
"eprel": map[string]any{
"eprel_id": "999",
"eprel_label": "https://eprel.example/label",
"eprel_pdf": "https://eprel.example/pdf",
"eprel_energy_class": "B",
},
})
m2, ok := got2.(map[string]any)
if !ok || m2["id"] != "999" || m2["energy_class"] != "B" || m2["label"] == nil || m2["pdf"] == nil {
t.Fatalf("nested aliases: %v", got2)
}
// V1 poll must strip eprel_* from attributes after extract.
clean := SanitizeV1ProcessAttributesAllowed(map[string]any{
"eprel_id": "1632113",
"eprel_label": "https://eprel.example/label",
"brand": "Samsung",
"eprel_energy_class": "A",
}, map[string]struct{}{})
if _, ok := clean["eprel_id"]; ok {
t.Fatalf("eprel leaked into attributes: %v", clean)
}
if clean["brand"] != "Samsung" {
t.Fatalf("brand missing: %v", clean)
}
}
func TestRunSteps_EPREL_liveFetchIfNetwork(t *testing.T) {
if testing.Short() {
t.Skip("short")
}
client := eprel.NewClient(eprel.Options{Enabled: true, Timeout: 20 * time.Second})
if !client.Enabled() {
t.Fatal("client disabled")
}
e := &Engine{EPREL: client, Completer: HeuristicCompleter{}, Vector: NoopVectorCategorizer{}}
out, err := e.RunSteps(context.Background(), "co", ProductInput{
Mapped: map[string]any{
"name": "Samsung WW90CGC04DAELE",
"eprel_id": "1632113",
},
}, "eprel_only", nil, StepPolicy{AllowEPREL: true})
if err != nil {
t.Fatal(err)
}
if out.EPREL["id"] != "1632113" {
t.Fatalf("eprel=%v", out.EPREL)
}
if out.EPREL["label"] == nil || out.EPREL["label"] == "" {
t.Fatalf("missing label: %v", out.EPREL)
}
if out.EPREL["pdf"] == nil || out.EPREL["pdf"] == "" {
t.Fatalf("missing pdf: %v", out.EPREL)
}
if out.EPREL["energy_class"] == nil || out.EPREL["energy_class"] == "" {
t.Fatalf("missing energy_class: %v", out.EPREL)
}
if out.ProcessedAttributes["eprel_id"] != "1632113" {
t.Fatalf("attrs=%v", out.ProcessedAttributes)
}
eprelVal := extractEPRELFromAttrs(out.ProcessedAttributes)
m, ok := eprelVal.(map[string]any)
if !ok || m["id"] != "1632113" {
t.Fatalf("extract=%v", eprelVal)
}
if m["label"] == nil || m["label"] == "" || m["pdf"] == nil || m["pdf"] == "" || m["energy_class"] == nil || m["energy_class"] == "" {
t.Fatalf("extract missing fields: %v", m)
}
clean := SanitizeV1ProcessAttributesAllowed(out.ProcessedAttributes, map[string]struct{}{})
for k := range clean {
if strings.HasPrefix(strings.ToLower(k), "eprel") {
t.Fatalf("eprel leaked into V1 attributes: %v", clean)
}
}
sample, _ := json.MarshalIndent(map[string]any{
"eprel": m,
"attributes": clean,
"step_eprel": out.EPREL,
}, "", " ")
t.Logf("sample eprel JSON:\n%s", sample)
}
apps/api/internal/processing/fill.go
+9 -20
View File
@@ -1,7 +1,6 @@
package processing
import (
"fmt"
"regexp"
"strings"
)
@@ -67,10 +66,12 @@ func FillMissingFields(mapped map[string]any, attrs map[string]any) map[string]a
}
}
if stringFromAny(out["category"]) == "" {
if c := stringFromAny(attrs["category"]); c != "" {
out["category"] = c
}
if c := categoryUniqueIDFromAny(out["category"]); c != "" {
out["category"] = c
} else if c := categoryUniqueIDFromAny(attrs["category"]); c != "" {
out["category"] = c
} else if c := categoryUniqueIDFromAny(out["category_unique_id"]); c != "" {
out["category"] = c
}
if stringFromAny(out["stock_status"]) == "" {
@@ -142,19 +143,7 @@ func normalizeStockStatusLabel(s string) string {
}
func stringFromAny(v any) string {
if v == nil {
return ""
}
switch t := v.(type) {
case string:
return strings.TrimSpace(t)
case float64, float32, int, int64, bool:
s := strings.TrimSpace(fmt.Sprint(t))
if s == "<nil>" {
return ""
}
return s
default:
return ""
}
// Flatten arrays / #text wrappers; callers that need HTML-stripped descriptions
// should use PlainDescriptionFromAny (normalize + V1 enforce do that).
return coerceScalarText(v)
}
apps/api/internal/processing/fix_company_catalog.go
+127
View File
@@ -0,0 +1,127 @@
package processing
import (
"context"
"fmt"
"github.com/descrybe/descrybe-v2/apps/api/internal/aiprompts"
"github.com/descrybe/descrybe-v2/apps/api/internal/catalog"
"github.com/google/uuid"
"github.com/jackc/pgx/v5/pgxpool"
)
// FixCompanyCatalogOpts controls optional Fix A1 / catalog hygiene steps.
type FixCompanyCatalogOpts struct {
// BackfillCategories copies mapped_data category unique_ids onto empty
// processed_products.category.
BackfillCategories bool
// ReprocessSampleLimit caps recommended raw_product_ids in the response
// (0 = counts only, no sample ids).
ReprocessSampleLimit int
// AIPrompts optionally re-applies BuiltInDefaults product_enhance per content language.
AIPrompts *aiprompts.Service
}
// FixCompanyCatalogResult is the idempotent admin Fix A1 report.
type FixCompanyCatalogResult struct {
CompanyID uuid.UUID `json:"company_id"`
CompanyName string `json:"company_name"`
A1Cohort bool `json:"a1_cohort"`
CategoryAttributeOrphansRemoved int `json:"category_attribute_orphans_removed"`
CategoryAttributeLinks int `json:"category_attribute_links"`
CategoryPromptsUpdated int `json:"category_prompts_updated"`
CategoryPromptsAlreadyOK int `json:"category_prompts_already_ok"`
CategoryPromptsEmpty int `json:"category_prompts_empty_skipped"`
// Prompts aliases category_prompts_updated for flash.admin.fixA1Success {prompts}.
Prompts int `json:"prompts"`
ProductEnhanceLanguages int `json:"product_enhance_languages"`
WeakHashesCleared int `json:"weak_hashes_cleared"`
CategoriesBackfilled int `json:"categories_backfilled"`
// Hashes / Categories alias the long names for flash.admin.fixA1Success.
Hashes int `json:"hashes"`
Categories int `json:"categories"`
DescriptionsNormalized int `json:"descriptions_normalized"`
DescriptionsBackfilled int `json:"descriptions_backfilled"`
MappedDescriptionsFlat int `json:"mapped_descriptions_flattened"`
MetaBackfilled int `json:"meta_backfilled"`
AttributesSanitized int64 `json:"attributes_sanitized"`
ProductsScanned int `json:"products_scanned"`
ReprocessNeededCount int `json:"reprocess_needed_count"`
ReprocessSampleRawProductIDs []uuid.UUID `json:"reprocess_sample_raw_product_ids,omitempty"`
}
// FixCompanyCatalog runs the admin Fix A1 / catalog hygiene sequence in place.
// Never clears the catalog or uses A1 as a clone destination.
// Step 2 applies RepairCompanyCategoryEnhancePrompts (same repaired map as
// RepairA1DemoCategoryEnhancePrompts / cmd/repair-category-prompts -apply).
// Legacy converters + weak-hash clear live in FixCatalogHygieneWithIDs.
func FixCompanyCatalog(ctx context.Context, pool *pgxpool.Pool, companyID uuid.UUID, companyName string, a1Cohort bool, opts FixCompanyCatalogOpts) (FixCompanyCatalogResult, error) {
out := FixCompanyCatalogResult{
CompanyID: companyID,
CompanyName: companyName,
A1Cohort: a1Cohort,
}
if pool == nil {
return out, fmt.Errorf("nil pool")
}
orphans, links, err := catalog.EnsureCategoryAttributeLinks(ctx, pool, companyID)
if err != nil {
return out, err
}
out.CategoryAttributeOrphansRemoved = orphans
out.CategoryAttributeLinks = links
updated, alreadyOK, emptySkipped, err := catalog.RepairCompanyCategoryEnhancePrompts(ctx, pool, companyID)
if err != nil {
return out, err
}
out.CategoryPromptsUpdated = updated
out.CategoryPromptsAlreadyOK = alreadyOK
out.CategoryPromptsEmpty = emptySkipped
if opts.AIPrompts != nil {
n, err := opts.AIPrompts.ApplyBuiltInProductEnhance(ctx, companyID)
if err != nil {
return out, fmt.Errorf("product_enhance templates: %w", err)
}
out.ProductEnhanceLanguages = n
}
hygiene, clearedIDs, err := FixCatalogHygieneWithIDs(ctx, pool, companyID, opts.BackfillCategories)
if err != nil {
return out, err
}
out.WeakHashesCleared = hygiene.WeakHashesCleared
out.CategoriesBackfilled = hygiene.CategoriesBackfilled
out.DescriptionsNormalized = hygiene.DescriptionsNormalized
out.DescriptionsBackfilled = hygiene.DescriptionsBackfilled
out.MappedDescriptionsFlat = hygiene.MappedDescriptionsFlat
out.MetaBackfilled = hygiene.MetaBackfilled
out.ProductsScanned = hygiene.ProductsScanned
sanitized, err := BackfillCompanyProcessedAttributes(ctx, pool, companyID)
if err != nil {
return out, fmt.Errorf("sanitize attributes: %w", err)
}
out.AttributesSanitized = sanitized
needed, sample, err := RecommendReprocessRawProductIDs(ctx, pool, companyID, clearedIDs, opts.ReprocessSampleLimit)
if err != nil {
return out, err
}
out.ReprocessNeededCount = needed
out.ReprocessSampleRawProductIDs = sample
// Short keys for flash.admin.fixA1Success {prompts,hashes,categories}.
out.Prompts = out.CategoryPromptsUpdated
out.Hashes = out.WeakHashesCleared
out.Categories = out.CategoriesBackfilled
return out, nil
}
apps/api/internal/processing/fix_company_catalog_test.go
+23
View File
@@ -0,0 +1,23 @@
package processing
import (
"testing"
"github.com/google/uuid"
)
func TestFixCompanyCatalogResultFlashAliases(t *testing.T) {
t.Parallel()
out := FixCompanyCatalogResult{
CompanyID: uuid.New(),
CategoryPromptsUpdated: 3,
WeakHashesCleared: 7,
CategoriesBackfilled: 11,
}
out.Prompts = out.CategoryPromptsUpdated
out.Hashes = out.WeakHashesCleared
out.Categories = out.CategoriesBackfilled
if out.Prompts != 3 || out.Hashes != 7 || out.Categories != 11 {
t.Fatalf("flash aliases mismatch: %+v", out)
}
}
apps/api/internal/processing/formula_prompt.go
+216
View File
@@ -0,0 +1,216 @@
package processing
import (
"encoding/json"
"fmt"
"strconv"
"strings"
)
// AppendFormulaConstraints appends language-agnostic title/description formula
// guidance to the enhance user template (before {{var}} render). Empty templates
// are no-ops. Shared skeleton stays in company/built-in prompts; formulas only
// constrain structure for the active language via {{language}} elsewhere.
func AppendFormulaConstraints(userTpl string, titleTemplate, descriptionTemplate any) string {
userTpl = strings.TrimSpace(userTpl)
titleBlock := FormatTitleFormulaConstraint(titleTemplate)
descBlock := FormatDescriptionFormulaConstraint(descriptionTemplate)
if titleBlock == "" && descBlock == "" {
return userTpl
}
var b strings.Builder
if userTpl != "" {
b.WriteString(userTpl)
b.WriteString("\n\n")
}
if titleBlock != "" {
b.WriteString(titleBlock)
if descBlock != "" {
b.WriteString("\n\n")
}
}
if descBlock != "" {
b.WriteString(descBlock)
}
return b.String()
}
// FormatTitleFormulaConstraint turns categories.title_template into enhance-prompt
// constraints (element order: literal text + attribute variable keys).
func FormatTitleFormulaConstraint(template any) string {
elements, separator, ok := parseTitleFormula(template)
if !ok || len(elements) == 0 {
return ""
}
sep := separator
if sep == "" {
sep = " "
}
var b strings.Builder
b.WriteString("Title formula (order matters; join with ")
b.WriteString(strconv.Quote(sep))
b.WriteString("). Build name from Attrs using this structure:\n")
for i, el := range elements {
switch el.Type {
case "text":
fmt.Fprintf(&b, "%d. text %s\n", i+1, strconv.Quote(el.Value))
case "variable":
key := strings.TrimSpace(el.Value)
if key == "" {
continue
}
fmt.Fprintf(&b, "%d. attr [%s]\n", i+1, key)
default:
continue
}
}
b.WriteString("Prefer Attrs values for [attr] slots; keep literal text as written; write name in {{language}}.")
return strings.TrimSpace(b.String())
}
// FormatDescriptionFormulaConstraint turns categories.description_template sections
// into bullet instructions for the enhance user prompt.
func FormatDescriptionFormulaConstraint(template any) string {
sections, ok := parseDescriptionFormulaSections(template)
if !ok || len(sections) == 0 {
return ""
}
var b strings.Builder
b.WriteString("Description formula (cover each section in order; write in {{language}}):\n")
for _, s := range sections {
typ := strings.TrimSpace(s.Type)
instr := strings.TrimSpace(s.Instructions)
if typ == "" && instr == "" {
continue
}
if typ == "" {
typ = "section"
}
if instr == "" {
fmt.Fprintf(&b, "- %s\n", typ)
continue
}
fmt.Fprintf(&b, "- %s: %s\n", typ, instr)
}
return strings.TrimSpace(b.String())
}
type titleFormulaElement struct {
Type string `json:"type"`
Value string `json:"value"`
}
type descriptionFormulaSection struct {
Type string `json:"type"`
Instructions string `json:"instructions"`
}
func parseTitleFormula(template any) (elements []titleFormulaElement, separator string, ok bool) {
if template == nil {
return nil, "", false
}
obj, err := asObjectMap(template)
if err != nil || obj == nil {
return nil, "", false
}
sep, _ := obj["separator"].(string)
rawEls, exists := obj["elements"]
if !exists || rawEls == nil {
return nil, "", false
}
b, err := json.Marshal(rawEls)
if err != nil {
return nil, "", false
}
var parsed []titleFormulaElement
if err := json.Unmarshal(b, &parsed); err != nil {
return nil, "", false
}
out := make([]titleFormulaElement, 0, len(parsed))
for _, el := range parsed {
t := strings.ToLower(strings.TrimSpace(el.Type))
v := strings.TrimSpace(el.Value)
if t != "text" && t != "variable" {
continue
}
if v == "" {
continue
}
out = append(out, titleFormulaElement{Type: t, Value: v})
}
if len(out) == 0 {
return nil, "", false
}
return out, sep, true
}
func parseDescriptionFormulaSections(template any) ([]descriptionFormulaSection, bool) {
if template == nil {
return nil, false
}
obj, err := asObjectMap(template)
if err != nil || obj == nil {
return nil, false
}
raw, exists := obj["sections"]
if !exists || raw == nil {
return nil, false
}
b, err := json.Marshal(raw)
if err != nil {
return nil, false
}
var parsed []descriptionFormulaSection
if err := json.Unmarshal(b, &parsed); err != nil {
return nil, false
}
out := make([]descriptionFormulaSection, 0, len(parsed))
for _, s := range parsed {
t := strings.ToLower(strings.TrimSpace(s.Type))
instr := strings.TrimSpace(s.Instructions)
if t == "" && instr == "" {
continue
}
out = append(out, descriptionFormulaSection{Type: t, Instructions: instr})
}
return out, len(out) > 0
}
// categoryFormulasFor resolves title/description templates for a category key
// (unique_id or name). Explicit ProductInput fields win over the job cache map.
func categoryFormulasFor(in ProductInput, category string) (title, description any) {
if in.TitleTemplate != nil || in.DescriptionTemplate != nil {
return in.TitleTemplate, in.DescriptionTemplate
}
key := strings.ToLower(strings.TrimSpace(category))
if key == "" || len(in.CategoryFormulasByKey) == 0 {
return nil, nil
}
f, ok := in.CategoryFormulasByKey[key]
if !ok {
return nil, nil
}
return f.TitleTemplate, f.DescriptionTemplate
}
func asObjectMap(v any) (map[string]any, error) {
switch t := v.(type) {
case map[string]any:
return t, nil
case nil:
return nil, nil
default:
b, err := json.Marshal(t)
if err != nil {
return nil, err
}
if len(b) == 0 || string(b) == "null" {
return nil, nil
}
var obj map[string]any
if err := json.Unmarshal(b, &obj); err != nil {
return nil, err
}
return obj, nil
}
}
apps/api/internal/processing/formula_prompt_test.go
+118
View File
@@ -0,0 +1,118 @@
package processing
import (
"strings"
"testing"
)
func TestFormatTitleFormulaConstraint(t *testing.T) {
t.Parallel()
got := FormatTitleFormulaConstraint(map[string]any{
"separator": " ",
"elements": []any{
map[string]any{"type": "variable", "value": "brand"},
map[string]any{"type": "text", "value": "TV"},
map[string]any{"type": "variable", "value": "product_model"},
},
})
if !strings.Contains(got, `join with " "`) {
t.Fatalf("missing separator: %s", got)
}
if !strings.Contains(got, "1. attr [brand]") || !strings.Contains(got, `2. text "TV"`) || !strings.Contains(got, "3. attr [product_model]") {
t.Fatalf("missing ordered elements: %s", got)
}
if !strings.Contains(got, "{{language}}") {
t.Fatalf("missing language placeholder: %s", got)
}
if FormatTitleFormulaConstraint(nil) != "" || FormatTitleFormulaConstraint(map[string]any{}) != "" {
t.Fatal("empty templates should yield empty constraint")
}
}
func TestFormatDescriptionFormulaConstraint(t *testing.T) {
t.Parallel()
got := FormatDescriptionFormulaConstraint(map[string]any{
"sections": []any{
map[string]any{"type": "h1", "instructions": "Main product heading"},
map[string]any{"type": "p", "instructions": "Two factual sentences"},
map[string]any{"type": "ul", "instructions": "3 bullets of specs"},
},
})
if !strings.Contains(got, "- h1: Main product heading") {
t.Fatalf("missing h1: %s", got)
}
if !strings.Contains(got, "- p: Two factual sentences") || !strings.Contains(got, "- ul: 3 bullets of specs") {
t.Fatalf("missing sections: %s", got)
}
if !strings.Contains(got, "{{language}}") {
t.Fatalf("missing language placeholder: %s", got)
}
}
func TestAppendFormulaConstraints_injectedIntoResolve(t *testing.T) {
t.Parallel()
title := map[string]any{
"separator": "-",
"elements": []any{
map[string]any{"type": "variable", "value": "brand"},
map[string]any{"type": "variable", "value": "product_model"},
},
}
desc := map[string]any{
"sections": []any{
map[string]any{"type": "p", "instructions": "Factual summary"},
},
}
_, user := resolveProductPromptTemplates(ProductInput{
EnhanceUserTemplate: "Name: {{name}}\nAttrs: {{attrs}}",
TitleTemplate: title,
DescriptionTemplate: desc,
})
if !strings.Contains(user, "Name: {{name}}") {
t.Fatalf("lost base template: %s", user)
}
if !strings.Contains(user, "Title formula") || !strings.Contains(user, "attr [brand]") {
t.Fatalf("missing title formula: %s", user)
}
if !strings.Contains(user, "Description formula") || !strings.Contains(user, "- p: Factual summary") {
t.Fatalf("missing description formula: %s", user)
}
// Render substitutes {{language}} in formula blocks.
_, rendered := RenderProductEnhancePrompts(
"Write in {{language}}.", user,
"50", "Old Name", "", "", "", "sl",
map[string]any{"brand": "Vox", "product_model": "EHT6020"},
)
if !strings.Contains(rendered, "Slovenian") {
t.Fatalf("formula language not rendered: %s", rendered)
}
if strings.Contains(rendered, "{{language}}") {
t.Fatalf("unrendered language placeholder left: %s", rendered)
}
}
func TestCategoryFormulasFor_explicitWins(t *testing.T) {
t.Parallel()
in := ProductInput{
TitleTemplate: map[string]any{"elements": []any{map[string]any{"type": "text", "value": "X"}}},
CategoryFormulasByKey: map[string]CategoryFormulas{
"50": {TitleTemplate: map[string]any{"elements": []any{map[string]any{"type": "text", "value": "Y"}}}},
},
}
title, _ := categoryFormulasFor(in, "50")
m, _ := title.(map[string]any)
els, _ := m["elements"].([]any)
first, _ := els[0].(map[string]any)
if first["value"] != "X" {
t.Fatalf("explicit should win, got %#v", title)
}
in2 := ProductInput{
CategoryFormulasByKey: map[string]CategoryFormulas{
"50": {DescriptionTemplate: map[string]any{"sections": []any{map[string]any{"type": "p", "instructions": "Hi"}}}},
},
}
_, desc := categoryFormulasFor(in2, "50")
if FormatDescriptionFormulaConstraint(desc) == "" {
t.Fatal("expected cache formula")
}
}
apps/api/internal/processing/legacy_backfill.go
+382
View File
@@ -0,0 +1,382 @@
package processing
import (
"context"
"encoding/json"
"fmt"
"strings"
"github.com/google/uuid"
"github.com/jackc/pgx/v5/pgxpool"
)
// BackfillDescriptionsFromMapped fills empty processed_products.description from
// linked raw mapped_data (mirrors migrator backfillProcessedDescriptionsFromMapped)
// and coerces array/HTML shapes via PlainDescriptionFromAny.
func BackfillDescriptionsFromMapped(ctx context.Context, pool *pgxpool.Pool, companyID uuid.UUID) (updated int, err error) {
if pool == nil {
return 0, fmt.Errorf("description backfill: nil pool")
}
if companyID == uuid.Nil {
return 0, fmt.Errorf("description backfill: empty company id")
}
rows, err := pool.Query(ctx, `
SELECT p.id,
COALESCE(p.description, ''),
COALESCE(r.mapped_data, '{}'::jsonb),
COALESCE(r.raw_data, '{}'::jsonb)
FROM processed_products p
INNER JOIN raw_products r ON r.id = p.raw_product_id AND r.company_id = p.company_id
WHERE p.company_id = $1
AND COALESCE(NULLIF(BTRIM(p.description), ''), '') = ''`, companyID)
if err != nil {
return 0, fmt.Errorf("description backfill query: %w", err)
}
defer rows.Close()
for rows.Next() {
var (
id uuid.UUID
prior string
mappedB []byte
rawB []byte
)
if err := rows.Scan(&id, &prior, &mappedB, &rawB); err != nil {
return updated, fmt.Errorf("description backfill scan: %w", err)
}
mapped, err := decodeJSONObject(mappedB)
if err != nil {
return updated, err
}
raw, err := decodeJSONObject(rawB)
if err != nil {
return updated, err
}
desc := PlainDescriptionFromAny(mapped["description"])
if desc == "" {
desc = PlainDescriptionFromAny(mapped["desc"])
}
if desc == "" {
desc = PlainDescriptionFromAny(mapped["body"])
}
if desc == "" {
desc = PlainDescriptionFromAny(raw["description"])
}
if desc == "" {
continue
}
ct, err := pool.Exec(ctx, `
UPDATE processed_products
SET description = $2,
updated_at = now()
WHERE id = $1 AND company_id = $3
AND COALESCE(NULLIF(BTRIM(description), ''), '') = ''`,
id, desc, companyID)
if err != nil {
return updated, fmt.Errorf("description backfill update %s: %w", id, err)
}
if ct.RowsAffected() > 0 {
updated++
}
_ = prior
}
return updated, rows.Err()
}
// mappedFeedTitle picks a clean name/title from mapped or raw feed maps.
func mappedFeedTitle(mapped, raw map[string]any) string {
for _, src := range []map[string]any{mapped, raw} {
if src == nil {
continue
}
for _, key := range []string{"name", "title", "product_name", "product_title"} {
v := strings.TrimSpace(stringFromAny(src[key]))
if v == "" || v == "<nil>" || isPromptLabelTitle(v) {
continue
}
return SanitizeOutput(v)
}
}
return ""
}
// BackfillPollutedNamesFromMapped clears processed_name / name when they match
// prompt-leakage patterns (Title formula / short retail title / …) and replaces
// them with mapped_data name/title when available. Next Load/FixCatalog or
// reprocess then shows the feed title instead of instruction text.
func BackfillPollutedNamesFromMapped(ctx context.Context, pool *pgxpool.Pool, companyID uuid.UUID) (updated int, err error) {
if pool == nil {
return 0, fmt.Errorf("name pollution backfill: nil pool")
}
if companyID == uuid.Nil {
return 0, fmt.Errorf("name pollution backfill: empty company id")
}
rows, err := pool.Query(ctx, `
SELECT p.id,
COALESCE(p.name, ''),
COALESCE(p.processed_name, ''),
COALESCE(r.mapped_data, '{}'::jsonb),
COALESCE(r.raw_data, '{}'::jsonb)
FROM processed_products p
INNER JOIN raw_products r ON r.id = p.raw_product_id AND r.company_id = p.company_id
WHERE p.company_id = $1`, companyID)
if err != nil {
return 0, fmt.Errorf("name pollution backfill query: %w", err)
}
defer rows.Close()
for rows.Next() {
var (
id uuid.UUID
name, processedName string
mappedB, rawB []byte
)
if err := rows.Scan(&id, &name, &processedName, &mappedB, &rawB); err != nil {
return updated, fmt.Errorf("name pollution backfill scan: %w", err)
}
namePolluted := isPromptLabelTitle(name)
procPolluted := isPromptLabelTitle(processedName)
if !namePolluted && !procPolluted {
continue
}
mapped, err := decodeJSONObject(mappedB)
if err != nil {
return updated, err
}
raw, err := decodeJSONObject(rawB)
if err != nil {
return updated, err
}
feed := mappedFeedTitle(mapped, raw)
if feed == "" {
continue
}
newName := name
newProcessed := processedName
if namePolluted {
newName = feed
}
if procPolluted {
newProcessed = feed
}
if newName == name && newProcessed == processedName {
continue
}
ct, err := pool.Exec(ctx, `
UPDATE processed_products
SET name = $2,
processed_name = $3,
field_sources = jsonb_set(
COALESCE(field_sources, '{}'::jsonb),
'{name}',
'"mapped_pollution_backfill"'::jsonb,
true
),
updated_at = now()
WHERE id = $1 AND company_id = $4`,
id, newName, newProcessed, companyID)
if err != nil {
return updated, fmt.Errorf("name pollution backfill update %s: %w", id, err)
}
if ct.RowsAffected() > 0 {
updated++
}
}
return updated, rows.Err()
}
// BackfillMissingMeta fills empty meta_title / meta_description using the same
// free template fillMetaFromResult uses at process time (no AI / no credits).
func BackfillMissingMeta(ctx context.Context, pool *pgxpool.Pool, companyID uuid.UUID) (updated int, err error) {
if pool == nil {
return 0, fmt.Errorf("meta backfill: nil pool")
}
if companyID == uuid.Nil {
return 0, fmt.Errorf("meta backfill: empty company id")
}
rows, err := pool.Query(ctx, `
SELECT id,
COALESCE(name, ''),
COALESCE(processed_name, ''),
COALESCE(category, ''),
COALESCE(description, ''),
COALESCE(processed_description, ''),
COALESCE(meta_title, ''),
COALESCE(meta_description, ''),
COALESCE(attributes, '{}'::jsonb),
COALESCE(processed_attributes, '{}'::jsonb)
FROM processed_products
WHERE company_id = $1
AND (
COALESCE(NULLIF(BTRIM(meta_title), ''), '') = ''
OR COALESCE(NULLIF(BTRIM(meta_description), ''), '') = ''
OR BTRIM(meta_title) ~ '\|\s+[0-9]+$'
OR LOWER(meta_title) LIKE '%short retail title%'
OR LOWER(meta_title) LIKE '%title formula%'
OR LOWER(meta_title) LIKE '%follow any%'
OR LOWER(meta_title) LIKE '%constraints that follow%'
OR LOWER(meta_title) LIKE '%prefer 1-3%'
OR LOWER(meta_title) LIKE '%reply with only json%'
OR LOWER(meta_title) LIKE '%your reply is parsed as json%'
OR LOWER(meta_description) LIKE '%prefer 1-3%'
OR LOWER(meta_description) LIKE '%short retail title%'
OR LOWER(meta_description) LIKE '%title formula%'
OR LOWER(meta_description) LIKE '%do not emit%'
)`, companyID)
if err != nil {
return 0, fmt.Errorf("meta backfill query: %w", err)
}
defer rows.Close()
for rows.Next() {
var (
id uuid.UUID
name, processedName, category string
desc, processedDesc, metaTitle, metaDesc string
attrsJSON, procAttrsJSON []byte
)
if err := rows.Scan(&id, &name, &processedName, &category, &desc, &processedDesc,
&metaTitle, &metaDesc, &attrsJSON, &procAttrsJSON); err != nil {
return updated, fmt.Errorf("meta backfill scan: %w", err)
}
attrs := decodeAttrMap(attrsJSON)
procAttrs := decodeAttrMap(procAttrsJSON)
synthTitle, synthDesc := fillMetaFromResult(StepResult{
Name: name,
ProcessedName: processedName,
Category: category,
Description: PlainDescriptionFromAny(desc),
ProcessedDescription: PlainDescriptionFromAny(processedDesc),
Attributes: attrs,
ProcessedAttributes: procAttrs,
})
newTitle := strings.TrimSpace(metaTitle)
newDesc := strings.TrimSpace(metaDesc)
if newTitle == "" || isPoisonedMetaTitle(newTitle) {
newTitle = synthTitle
}
if newDesc == "" || isPromptLeakageTitle(newDesc) ||
(isPoisonedMetaTitle(strings.TrimSpace(metaTitle)) &&
(isWeakPriorEnhanceDescription(newDesc, name, processedName) || isPromptLeakageTitle(newDesc))) {
newDesc = synthDesc
}
if newTitle == strings.TrimSpace(metaTitle) && newDesc == strings.TrimSpace(metaDesc) {
continue
}
if newTitle == "" && newDesc == "" {
continue
}
ct, err := pool.Exec(ctx, `
UPDATE processed_products
SET meta_title = CASE
WHEN COALESCE(NULLIF(BTRIM(meta_title), ''), '') = ''
OR BTRIM(meta_title) ~ '\|\s+[0-9]+$'
OR LOWER(meta_title) LIKE '%short retail title%'
OR LOWER(meta_title) LIKE '%title formula%'
OR LOWER(meta_title) LIKE '%follow any%'
OR LOWER(meta_title) LIKE '%constraints that follow%'
OR LOWER(meta_title) LIKE '%prefer 1-3%'
OR LOWER(meta_title) LIKE '%reply with only json%'
OR LOWER(meta_title) LIKE '%your reply is parsed as json%'
THEN NULLIF(BTRIM($2), '')
ELSE meta_title
END,
meta_description = CASE
WHEN COALESCE(NULLIF(BTRIM(meta_description), ''), '') = '' THEN NULLIF(BTRIM($3), '')
WHEN LOWER(meta_description) LIKE '%prefer 1-3%'
OR LOWER(meta_description) LIKE '%short retail title%'
OR LOWER(meta_description) LIKE '%title formula%'
OR LOWER(meta_description) LIKE '%do not emit%'
THEN NULLIF(BTRIM($3), '')
WHEN (
BTRIM(meta_title) ~ '\|\s+[0-9]+$'
OR LOWER(meta_title) LIKE '%short retail title%'
OR LOWER(meta_title) LIKE '%title formula%'
OR LOWER(meta_title) LIKE '%follow any%'
OR LOWER(meta_title) LIKE '%constraints that follow%'
OR LOWER(meta_title) LIKE '%prefer 1-3%'
OR LOWER(meta_title) LIKE '%reply with only json%'
OR LOWER(meta_title) LIKE '%your reply is parsed as json%'
)
AND (
char_length(BTRIM(meta_description)) < 40
OR LOWER(meta_description) LIKE '%ready for retail listing%'
OR LOWER(meta_description) LIKE '%quality product ready%'
OR LOWER(meta_description) LIKE '%product description%'
OR LOWER(meta_description) LIKE '%based on available specifications%'
OR LOWER(meta_description) LIKE '%with available specifications%'
OR LOWER(meta_description) LIKE '%available catalog details%'
OR LOWER(meta_description) LIKE '%pripravljeno za prodajo%'
OR LOWER(meta_description) LIKE '%na podlagi razpoložljivih specifikacij%'
OR LOWER(meta_description) LIKE '%prefer 1-3%'
)
THEN NULLIF(BTRIM($3), '')
ELSE meta_description
END,
updated_at = now()
WHERE id = $1 AND company_id = $4`,
id, newTitle, newDesc, companyID)
if err != nil {
return updated, fmt.Errorf("meta backfill update %s: %w", id, err)
}
if ct.RowsAffected() > 0 {
updated++
}
}
return updated, rows.Err()
}
// FlattenMappedDescriptionArrays rewrites mapped_data.description when it is a
// JSON array / HTML blob into a plain string (local admin Fix A1 hygiene).
func FlattenMappedDescriptionArrays(ctx context.Context, pool *pgxpool.Pool, companyID uuid.UUID) (updated int, err error) {
if pool == nil {
return 0, fmt.Errorf("flatten mapped descriptions: nil pool")
}
rows, err := pool.Query(ctx, `
SELECT id, COALESCE(mapped_data, '{}'::jsonb)
FROM raw_products
WHERE company_id = $1
AND mapped_data ? 'description'
AND jsonb_typeof(mapped_data->'description') <> 'string'`, companyID)
if err != nil {
return 0, fmt.Errorf("flatten mapped descriptions query: %w", err)
}
defer rows.Close()
for rows.Next() {
var id uuid.UUID
var mappedB []byte
if err := rows.Scan(&id, &mappedB); err != nil {
return updated, err
}
mapped, err := decodeJSONObject(mappedB)
if err != nil {
return updated, err
}
rawDesc := mapped["description"]
plain := PlainDescriptionFromAny(rawDesc)
if plain == "" {
continue
}
// Skip when already a plain string equal to conversion (shouldn't hit query).
if s, ok := rawDesc.(string); ok && v1PlainDescription(s) == plain {
continue
}
mapped["description"] = plain
encoded, err := json.Marshal(mapped)
if err != nil {
return updated, err
}
ct, err := pool.Exec(ctx, `
UPDATE raw_products
SET mapped_data = $2::jsonb, updated_at = now()
WHERE id = $1 AND company_id = $3`, id, string(encoded), companyID)
if err != nil {
return updated, fmt.Errorf("flatten mapped description %s: %w", id, err)
}
if ct.RowsAffected() > 0 {
updated++
}
}
return updated, rows.Err()
}
apps/api/internal/processing/legacy_convert.go
+111
View File
@@ -0,0 +1,111 @@
package processing
import (
"fmt"
"strings"
)
// isDescriptionFieldKey reports keys whose values should be plain text scalars
// (not JSON arrays / nested HTML wrappers) for process + V1 poll contracts.
func isDescriptionFieldKey(key string) bool {
switch strings.ToLower(strings.TrimSpace(key)) {
case "description", "desc", "body", "short_description", "shortdescription",
"processed_description", "long_description", "longdescription":
return true
default:
return false
}
}
// PlainDescriptionFromAny coerces legacy description shapes to a single plain
// string: string, {#text}, or arrays of those (joined with newlines). HTML is
// stripped via v1PlainDescription so poll/store never keep ["…"] or markup blobs.
func PlainDescriptionFromAny(v any) string {
if v == nil {
return ""
}
switch t := v.(type) {
case string:
return v1PlainDescription(t)
case []string:
parts := make([]string, 0, len(t))
for _, s := range t {
if p := v1PlainDescription(s); p != "" {
parts = append(parts, p)
}
}
return strings.Join(parts, "\n")
case []any:
parts := make([]string, 0, len(t))
for _, item := range t {
if p := PlainDescriptionFromAny(item); p != "" {
parts = append(parts, p)
}
}
return strings.Join(parts, "\n")
case map[string]any:
for _, k := range []string{"#text", "text", "value", "description", "body"} {
if p := PlainDescriptionFromAny(t[k]); p != "" {
return p
}
}
return ""
default:
s := strings.TrimSpace(fmt.Sprint(t))
if s == "" || s == "<nil>" {
return ""
}
return v1PlainDescription(s)
}
}
// coerceScalarText flattens arrays / #text wrappers into a trimmed string without
// HTML stripping (names, brands, stock labels, …).
func coerceScalarText(v any) string {
if v == nil {
return ""
}
switch t := v.(type) {
case string:
return strings.TrimSpace(t)
case []string:
parts := make([]string, 0, len(t))
for _, s := range t {
s = strings.TrimSpace(s)
if s != "" {
parts = append(parts, s)
}
}
return strings.Join(parts, " ")
case []any:
parts := make([]string, 0, len(t))
for _, item := range t {
if s := coerceScalarText(item); s != "" {
parts = append(parts, s)
}
}
return strings.Join(parts, " ")
case map[string]any:
for _, k := range []string{"#text", "text", "value"} {
if s := coerceScalarText(t[k]); s != "" {
return s
}
}
return ""
case float64, float32, int, int64, int32, bool:
s := strings.TrimSpace(fmt.Sprint(t))
if s == "<nil>" {
return ""
}
return s
default:
if n, ok := t.(jsonNumberStringer); ok {
return strings.TrimSpace(n.String())
}
s := strings.TrimSpace(fmt.Sprint(t))
if s == "" || s == "<nil>" {
return ""
}
return s
}
}
apps/api/internal/processing/legacy_convert_test.go
+61
View File
@@ -0,0 +1,61 @@
package processing
import "testing"
func TestPlainDescriptionFromAny_arrayAndHTML(t *testing.T) {
got := PlainDescriptionFromAny([]any{"Hello<br>World", "Second"})
if got != "Hello\nWorld\nSecond" {
t.Fatalf("array+html: %q", got)
}
got = PlainDescriptionFromAny(map[string]any{"#text": "Plain &amp; ok"})
if got != "Plain & ok" {
t.Fatalf("#text: %q", got)
}
if PlainDescriptionFromAny(nil) != "" {
t.Fatal("nil should be empty")
}
}
func TestNormalizeMapped_descriptionArrayBecomesPlainString(t *testing.T) {
got := NormalizeMapped(map[string]any{
"description": []any{"Line1<br/>A", "Line2"},
"name": "Widget",
}, nil)
desc, ok := got["description"].(string)
if !ok {
t.Fatalf("description should be string, got %T %v", got["description"], got["description"])
}
if desc != "Line1\nA\nLine2" {
t.Fatalf("description=%q", desc)
}
}
func TestNormalizeMapped_eprelIDAlias(t *testing.T) {
got := NormalizeMapped(map[string]any{"EPRELID": "246834"}, nil)
if got["eprel_id"] != "246834" {
t.Fatalf("eprel_id=%v", got["eprel_id"])
}
}
func TestStringFromAny_flattensArray(t *testing.T) {
got := stringFromAny([]any{"a", "b"})
if got != "a b" {
t.Fatalf("got %q", got)
}
}
func TestCanonicalizeAttrKey_eprel(t *testing.T) {
if canonicalizeAttrKey("EPRELID") != "eprel_id" {
t.Fatalf("got %q", canonicalizeAttrKey("EPRELID"))
}
if canonicalizeAttrKey("eprel") != "eprel_id" {
t.Fatalf("got %q", canonicalizeAttrKey("eprel"))
}
}
func TestPlainDescriptionFromStored_jsonArrayString(t *testing.T) {
got := plainDescriptionFromStored(`["Hello<br>World"]`)
if got != "Hello\nWorld" {
t.Fatalf("got %q", got)
}
}
apps/api/internal/processing/live_mock_llm_title_test.go
+67
View File
@@ -0,0 +1,67 @@
package processing_test
import (
"context"
"os"
"strings"
"testing"
"time"
"github.com/descrybe/descrybe-v2/apps/api/internal/aiprompts"
"github.com/descrybe/descrybe-v2/apps/api/internal/processing"
)
// Live proof against local mock-llm (OPENAI_BASE_URL). Skipped unless LIVE_MOCK_LLM=1.
func TestLive_MockLLM_rejectsCategoryEnhanceInstructionTitle(t *testing.T) {
if strings.TrimSpace(os.Getenv("LIVE_MOCK_LLM")) == "" {
t.Skip("set LIVE_MOCK_LLM=1 with mock-llm on OPENAI_BASE_URL")
}
base := strings.TrimSpace(os.Getenv("OPENAI_BASE_URL"))
key := strings.TrimSpace(os.Getenv("OPENAI_API_KEY"))
model := strings.TrimSpace(os.Getenv("OPENAI_MODEL"))
if base == "" || key == "" {
t.Fatal("OPENAI_BASE_URL and OPENAI_API_KEY required")
}
if model == "" {
model = "mock-llm"
}
client := processing.NewOpenAIClient(key, base, model, 0, 2)
eng := &processing.Engine{
Completer: client,
Vector: processing.NoopVectorCategorizer{},
}
ctx, cancel := context.WithTimeout(context.Background(), 45*time.Second)
defer cancel()
out, err := eng.RunSteps(ctx, "live-probe", processing.ProductInput{
GTIN: "8712285326882",
Name: "Vox SWA-8000W",
Description: "Washer dryer combo",
Mapped: map[string]any{"name": "Vox SWA-8000W", "description": "Washer dryer combo", "category": "demo-electronics", "brand": "Vox"},
Language: "en",
EnhanceUserTemplate: aiprompts.CategoryEnhanceUserTemplate,
EnhanceSystemTemplate: `Retail product copywriter. Reply with ONLY JSON. Schema: {"name":"string","description":"string"}`,
TitleTemplate: map[string]any{
"separator": " ",
"elements": []any{
map[string]any{"type": "variable", "value": "brand"},
map[string]any{"type": "text", "value": " "},
map[string]any{"type": "variable", "value": "product_model"},
},
},
}, "enhance_only", nil, processing.StepPolicy{AllowAI: true})
if err != nil {
t.Fatal(err)
}
t.Logf("ProcessedName=%q AIProviderMode=%q", out.ProcessedName, out.AIProviderMode)
lower := strings.ToLower(out.ProcessedName)
for _, p := range []string{"title formula", "follow any", "short retail title", "use attrs", "schema:", "reply with only json"} {
if strings.Contains(lower, p) {
t.Fatalf("polluted title %q contains %q", out.ProcessedName, p)
}
}
if !strings.Contains(lower, "vox") {
t.Fatalf("ProcessedName=%q want Vox feed title", out.ProcessedName)
}
}
apps/api/internal/processing/llm_json.go
+14 -11
View File
@@ -11,15 +11,18 @@ import (
// Local / weak-model defaults (8k-class context). See docs/local-llm-tuning.md.
const (
DefaultStructuredTemp = 0.2
MaxTokensEnhance = 350
MaxTokensSEO = 180
MaxTokensCampaign = 650
MaxProductDescRunes = 400
MaxAttrKeys = 10
MaxAttrValueRunes = 60
MaxBrandInjectRunes = 500
MaxCampaignProducts = 8
MaxCampaignNameRunes = 80
// Reasoning-capable OpenAI-compatible models (e.g. code-fast / Qwen3) spend
// hundreds–thousands of tokens in reasoning_content before writing message.content.
// 350 capped mid-thought → empty content → "empty model response".
MaxTokensEnhance = 4096
MaxTokensSEO = 180
MaxTokensCampaign = 650
MaxProductDescRunes = 400
MaxAttrKeys = 10
MaxAttrValueRunes = 60
MaxBrandInjectRunes = 500
MaxCampaignProducts = 8
MaxCampaignNameRunes = 80
)
// CompleteOptions tunes a single chat completion for structured tasks.
@@ -152,14 +155,14 @@ func CompactAttrs(attrs map[string]any, maxKeys int) map[string]any {
keys := make([]string, 0, len(attrs))
for k := range attrs {
k = strings.TrimSpace(k)
if k == "" {
if k == "" || isInvalidAttributeKey(k) {
continue
}
keys = append(keys, k)
}
sort.Strings(keys)
// Prefer common retail keys first.
priority := []string{"brand", "Brand", "color", "Color", "material", "Material", "size", "Size", "model", "Model", "gtin", "GTIN", "ean", "EAN"}
priority := []string{"brand", "Brand", "product_model", "color", "Color", "material", "Material", "size", "Size", "model", "Model", "gtin", "GTIN", "ean", "EAN"}
ordered := make([]string, 0, len(keys))
seen := map[string]bool{}
for _, p := range priority {
apps/api/internal/processing/meta.go
+150
View File
@@ -0,0 +1,150 @@
package processing
import (
"regexp"
"strings"
"unicode"
)
// Meta title/description caps mirror seo.MetaTitleMaxChars / MetaDescriptionMaxChars.
// Duplicated here so processing can fill meta without importing seo (seo imports processing).
const (
metaTitleMaxChars = 60
metaDescriptionMaxChars = 155
)
// Bare "| <digits>" suffix = legacy unique_id leak (e.g. "VOX EBR700 | 50").
// Matches upsertProcessedProductSQL refresh predicate.
var poisonedMetaTitleUniqueIDRe = regexp.MustCompile(`\|\s+[0-9]+$`)
// isPoisonedMetaTitle reports meta_title that must be refreshed: bare unique_id
// suffix ("… | 50") or enhance-prompt instruction leakage ("short retail title",
// "Title formula", …). Matches upsert / BackfillMissingMeta SQL predicates.
func isPoisonedMetaTitle(s string) bool {
s = strings.TrimSpace(s)
if s == "" {
return false
}
if poisonedMetaTitleUniqueIDRe.MatchString(s) {
return true
}
return isPromptLeakageTitle(s)
}
// fillMetaFromResult builds free template meta from a StepResult (FillMetaTemplate-equivalent).
// Does not call AI / does not spend credits.
func fillMetaFromResult(result StepResult) (title, description string) {
name := strings.TrimSpace(result.ProcessedName)
if name == "" {
name = strings.TrimSpace(result.Name)
}
if name == "" {
name = "Product"
}
cat := categoryDisplayLabel(result)
brand := metaBrandFromAttrs(result.ProcessedAttributes, result.Attributes)
parts := make([]string, 0, 3)
if brand != "" && !strings.Contains(strings.ToLower(name), strings.ToLower(brand)) {
parts = append(parts, brand)
}
parts = append(parts, name)
if cat != "" && !strings.Contains(strings.ToLower(name), strings.ToLower(cat)) {
parts = append(parts, cat)
}
title = truncateMetaRunes(strings.Join(parts, " | "), metaTitleMaxChars)
body := strings.TrimSpace(result.ProcessedDescription)
if body == "" {
body = strings.TrimSpace(result.Description)
}
body = stripMetaTags(body)
if body == "" {
var bits []string
if brand != "" {
bits = append(bits, brand)
}
bits = append(bits, name)
if cat != "" {
bits = append(bits, "in "+cat)
}
body = strings.Join(bits, " ") + ". Shop quality products with clear specs and fast delivery."
}
description = truncateMetaDescription(body, metaDescriptionMaxChars)
return title, description
}
// truncateMetaDescription collapses whitespace and truncates at a word boundary when possible.
func truncateMetaDescription(s string, max int) string {
s = collapseMetaSpace(s)
if max <= 0 {
return ""
}
r := []rune(s)
if len(r) <= max {
return s
}
cut := r[:max]
lastSpace := -1
for i := len(cut) - 1; i >= 0; i-- {
if unicode.IsSpace(cut[i]) {
lastSpace = i
break
}
}
if lastSpace > max/2 {
return strings.TrimSpace(string(cut[:lastSpace]))
}
return string(cut)
}
func truncateMetaRunes(s string, max int) string {
s = collapseMetaSpace(s)
r := []rune(s)
if max <= 0 || len(r) <= max {
return s
}
if max <= 3 {
return string(r[:max])
}
return string(r[:max-1]) + "…"
}
func collapseMetaSpace(s string) string {
return strings.Join(strings.Fields(s), " ")
}
func stripMetaTags(s string) string {
var b strings.Builder
inTag := false
for _, r := range s {
switch {
case r == '<':
inTag = true
case r == '>':
inTag = false
case !inTag:
b.WriteRune(r)
}
}
return b.String()
}
func metaBrandFromAttrs(bags ...map[string]any) string {
keys := []string{"brand", "Brand", "manufacturer"}
for _, m := range bags {
if m == nil {
continue
}
for _, k := range keys {
if v, ok := m[k]; ok {
if s, ok := v.(string); ok {
if t := strings.TrimSpace(s); t != "" {
return t
}
}
}
}
}
return ""
}
apps/api/internal/processing/meta_test.go
+113
View File
@@ -0,0 +1,113 @@
package processing
import (
"strings"
"testing"
"unicode/utf8"
)
func TestFillMetaFromResult_buildsTitleAndDesc(t *testing.T) {
t.Parallel()
title, desc := fillMetaFromResult(StepResult{
ProcessedName: "Drill 18V",
Category: "Tools",
ProcessedDescription: "Cordless drill with battery pack for DIY projects.",
ProcessedAttributes: map[string]any{"brand": "Bosch"},
})
if title == "" || desc == "" {
t.Fatal("expected non-empty meta")
}
if !strings.Contains(title, "Drill 18V") {
t.Fatalf("title missing name: %q", title)
}
if !strings.Contains(title, "Bosch") {
t.Fatalf("title missing brand: %q", title)
}
if utf8.RuneCountInString(title) > metaTitleMaxChars {
t.Fatalf("title too long: %d", utf8.RuneCountInString(title))
}
if utf8.RuneCountInString(desc) > metaDescriptionMaxChars {
t.Fatalf("desc too long: %d", utf8.RuneCountInString(desc))
}
}
func TestFillMetaFromResult_usesCategoryNameNotUniqueID(t *testing.T) {
t.Parallel()
title, _ := fillMetaFromResult(StepResult{
ProcessedName: "Gorenje GEC5A21WG",
Category: "50",
CategoryName: "Štedilniki",
ProcessedAttributes: map[string]any{
"brand": "Gorenje",
},
})
if strings.Contains(title, "| 50") || strings.HasSuffix(title, "50") {
t.Fatalf("meta title must not use unique_id: %q", title)
}
if !strings.Contains(title, "Štedilniki") {
t.Fatalf("meta title missing display name: %q", title)
}
// Digits-only Category without CategoryName must omit category from title.
title2, _ := fillMetaFromResult(StepResult{
ProcessedName: "Cooker",
Category: "50",
})
if strings.Contains(title2, "50") {
t.Fatalf("digits-only unique_id must not appear in meta: %q", title2)
}
}
func TestTruncateMetaDescription_wordBoundary(t *testing.T) {
t.Parallel()
// 160+ runes with a clear word break near 155.
words := strings.Repeat("word ", 40) // 200 chars with spaces
out := truncateMetaDescription(words, v1MetaDescriptionMaxChars)
n := utf8.RuneCountInString(out)
if n > v1MetaDescriptionMaxChars {
t.Fatalf("len=%d want <= %d (%q)", n, v1MetaDescriptionMaxChars, out)
}
if strings.HasSuffix(out, "wor") || strings.HasSuffix(out, "wo") || strings.HasSuffix(out, "w") {
t.Fatalf("mid-word cut: %q", out)
}
if !strings.HasSuffix(out, "word") {
t.Fatalf("expected last full word, got %q", out)
}
}
func TestTruncateMetaDescription_collapsesSpace(t *testing.T) {
t.Parallel()
out := truncateMetaDescription(" hello \n\t world ", 155)
if out != "hello world" {
t.Fatalf("got %q", out)
}
}
func TestV1PollMetaFallback_usesTemplateWhenEmpty(t *testing.T) {
t.Parallel()
title := "Cordless Drill"
cat := "Tools"
plain := strings.Repeat("quality cordless drill for home and workshop use. ", 10)
synthTitle, synthDesc := fillMetaFromResult(StepResult{
Name: title,
ProcessedName: title,
Category: cat,
Description: plain,
ProcessedDescription: plain,
Attributes: map[string]any{"brand": "Makita"},
})
if synthTitle == "" || synthDesc == "" {
t.Fatal("expected synth meta")
}
if utf8.RuneCountInString(synthDesc) > v1MetaDescriptionMaxChars {
t.Fatalf("poll meta_description too long: %d", utf8.RuneCountInString(synthDesc))
}
// Word-safe: last rune should not be mid-word fragment from a hard cut.
r := []rune(synthDesc)
if len(r) == v1MetaDescriptionMaxChars {
// hard-cut path only when no good space; otherwise last char is letter from a word end
last := r[len(r)-1]
if last == ' ' {
t.Fatalf("trailing space in meta_description: %q", synthDesc)
}
}
}
apps/api/internal/processing/normalize.go
+57
View File
@@ -85,6 +85,13 @@ func normalizeValue(key string, v any) any {
if v == nil {
return nil
}
// Descriptions must be plain scalars (never leftover ["…"] / HTML arrays).
if isDescriptionFieldKey(key) {
if s := PlainDescriptionFromAny(v); s != "" {
return SanitizeText(s)
}
return nil
}
switch t := v.(type) {
case string:
s := strings.TrimSpace(t)
@@ -124,6 +131,13 @@ func normalizeValue(key string, v any) any {
if text, ok := t["text"]; ok {
return normalizeValue(key, text)
}
// Feed/XML category objects often carry unique_id — flatten to a scalar
// so StepResult.Category / processed_products.category get the code.
if key == "category" {
if id := categoryUniqueIDFromAny(t); id != "" {
return id
}
}
nested := make(map[string]any, len(t))
for nk, nv := range t {
if nn := normalizeValue(canonicalizeKey(nk), nv); nn != nil {
@@ -138,6 +152,14 @@ func normalizeValue(key string, v any) any {
if len(t) == 0 {
return nil
}
// Scalar-ish fields (name, brand, eprel_id, …): collapse one-element /
// joinable arrays so process steps never see []any where a string is expected.
if flat := coerceScalarText(t); flat != "" && looksLikeScalarFieldArray(t) {
if isDimensionKey(key) && isZeroishString(flat) {
return nil
}
return SanitizeText(flat)
}
out := make([]any, 0, len(t))
for _, item := range t {
if nn := normalizeValue(key, item); nn != nil {
@@ -160,6 +182,41 @@ func normalizeValue(key string, v any) any {
}
}
// looksLikeScalarFieldArray is true when every element coerces to a short scalar
// (string/number/#text) — not nested attribute/spec objects.
func looksLikeScalarFieldArray(items []any) bool {
if len(items) == 0 {
return false
}
for _, item := range items {
switch t := item.(type) {
case nil:
continue
case string, float64, float32, int, int64, int32, bool:
continue
case map[string]any:
if _, ok := t["#text"]; ok {
continue
}
if _, ok := t["text"]; ok {
continue
}
if _, ok := t["value"]; ok {
continue
}
return false
case []any, []string:
return false
default:
if _, ok := item.(jsonNumberStringer); ok {
continue
}
return false
}
}
return true
}
func isDimensionKey(key string) bool {
switch key {
case "width", "height", "depth", "weight", "length", "net_width", "net_height", "net_depth", "net_mass":
apps/api/internal/processing/openai.go
+166 -12
View File
@@ -16,6 +16,7 @@ import (
"sync"
"time"
"github.com/descrybe/descrybe-v2/apps/api/internal/company"
"github.com/descrybe/descrybe-v2/apps/api/internal/config"
"github.com/descrybe/descrybe-v2/apps/api/internal/security"
)
@@ -56,11 +57,14 @@ func NewOpenAIClient(apiKey, baseURL, model string, rpm, maxRetries int) *OpenAI
}
// Dial-time SSRF. Loopback when base URL is local; RFC1918 only in non-prod.
policy := openAIDialPolicy(baseURL)
// Header/body timeout: code-fast often exceeds 180s; 240s reduces false timeouts.
// Tradeoff: slower fail on hung upstream. Prefer synthesize fallback over infinite wait
// (RunSteps always synthesizes on timeout/error/empty — do not raise unboundedly).
return &OpenAIClient{
APIKey: apiKey,
BaseURL: strings.TrimRight(baseURL, "/"),
Model: model,
HTTPClient: security.SafeHTTPClientPolicy(60*time.Second, policy),
HTTPClient: security.SafeHTTPClientPolicy(240*time.Second, policy),
MinInterval: interval,
MaxRetries: maxRetries,
ModeLabel: AIProviderInternal,
@@ -112,16 +116,31 @@ func (c *OpenAIClient) Enabled() bool {
}
// ProviderModeLabel implements ProviderLabeler for analytics writes.
// Loopback / mock-llm base URLs must not report as managed "internal".
func (c *OpenAIClient) ProviderModeLabel() string {
if c == nil {
return AIProviderUnknown
}
if label := strings.TrimSpace(c.ModeLabel); label != "" {
label := strings.TrimSpace(c.ModeLabel)
if IsMockOrLoopbackBaseURL(c.BaseURL) && (label == "" || label == AIProviderInternal) {
return AIProviderCustom
}
if label != "" {
return label
}
return AIProviderInternal
}
// IsMockOrLoopbackBaseURL reports local mock / loopback OpenAI-compatible bases
// (e.g. mock-llm on 127.0.0.1:18767). Used for analytics ModeLabel accuracy.
func IsMockOrLoopbackBaseURL(baseURL string) bool {
if openAIBaseAllowsLoopback(baseURL) {
return true
}
lower := strings.ToLower(strings.TrimSpace(baseURL))
return strings.Contains(lower, "mock-llm")
}
type chatRequest struct {
Model string `json:"model"`
Messages []chatMessage `json:"messages"`
@@ -137,8 +156,11 @@ type chatMessage struct {
type chatResponse struct {
Model string `json:"model"`
Choices []struct {
Message struct {
Content string `json:"content"`
FinishReason string `json:"finish_reason"`
Message struct {
Content json.RawMessage `json:"content"`
ReasoningContent string `json:"reasoning_content"`
Reasoning string `json:"reasoning"`
} `json:"message"`
} `json:"choices"`
Usage struct {
@@ -152,6 +174,12 @@ type chatResponse struct {
} `json:"error"`
}
var errEmptyModelResponse = errors.New("empty model response")
// maxTokensReasoningBudget is used when a capped completion returns empty
// content with finish_reason=length (reasoning models).
const maxTokensReasoningBudget = 4096
func (c *OpenAIClient) Complete(ctx context.Context, system, user string) (Completion, error) {
return c.CompleteWithOptions(ctx, system, user, CompleteOptions{})
}
@@ -169,6 +197,7 @@ func (c *OpenAIClient) CompleteWithOptions(ctx context.Context, system, user str
if temp > 0.3 {
temp = 0.3
}
maxTok := opts.MaxTokens
var lastErr error
for attempt := 0; attempt <= c.MaxRetries; attempt++ {
@@ -184,11 +213,15 @@ func (c *OpenAIClient) CompleteWithOptions(ctx context.Context, system, user str
if err := c.waitRate(ctx); err != nil {
return Completion{}, err
}
comp, retryable, err := c.doComplete(ctx, system, user, temp, opts.MaxTokens)
comp, retryable, err := c.doComplete(ctx, system, user, temp, maxTok)
if err == nil {
return comp, nil
}
lastErr = err
if errors.Is(err, errEmptyModelResponse) && maxTok > 0 && maxTok < maxTokensReasoningBudget {
maxTok = maxTokensReasoningBudget
retryable = true
}
if !retryable {
return Completion{}, err
}
@@ -386,11 +419,15 @@ func (c *OpenAIClient) doComplete(ctx context.Context, system, user string, temp
return Completion{}, false, errors.New(msg)
}
text := ""
finishReason := ""
if len(parsed.Choices) > 0 {
text = SanitizeOutput(parsed.Choices[0].Message.Content)
finishReason = strings.TrimSpace(parsed.Choices[0].FinishReason)
text = SanitizeOutput(choiceMessageText(parsed.Choices[0].Message.Content, parsed.Choices[0].Message.ReasoningContent, parsed.Choices[0].Message.Reasoning))
}
if text == "" {
return Completion{}, false, errors.New("empty model response")
// Reasoning models often return empty content when max_tokens cuts mid-thought.
retryable := strings.EqualFold(finishReason, "length")
return Completion{}, retryable, errEmptyModelResponse
}
return Completion{
Text: text,
@@ -399,13 +436,61 @@ func (c *OpenAIClient) doComplete(ctx context.Context, system, user string, temp
TotalTokens: parsed.Usage.TotalTokens,
Model: parsed.Model,
Raw: map[string]any{
"model": parsed.Model,
"usage": parsed.Usage,
"status": res.StatusCode,
"model": parsed.Model,
"usage": parsed.Usage,
"status": res.StatusCode,
"finish_reason": finishReason,
},
}, false, nil
}
// choiceMessageText reads assistant text from OpenAI-compatible chat responses.
// Prefer message.content (string or multipart text parts). When empty — common for
// reasoning models that only fill reasoning_content under a tight max_tokens —
// fall back to reasoning fields and prefer an embedded JSON object when present.
func choiceMessageText(content json.RawMessage, reasoningContent, reasoning string) string {
if text := decodeChatContent(content); text != "" {
return text
}
for _, alt := range []string{reasoningContent, reasoning} {
alt = strings.TrimSpace(alt)
if alt == "" {
continue
}
if obj := StripJSONFences(alt); strings.HasPrefix(obj, "{") {
if _, err := ParseJSONObject(obj); err == nil {
return obj
}
}
}
return ""
}
func decodeChatContent(raw json.RawMessage) string {
raw = bytes.TrimSpace(raw)
if len(raw) == 0 || string(raw) == "null" {
return ""
}
var s string
if err := json.Unmarshal(raw, &s); err == nil {
return strings.TrimSpace(s)
}
var parts []struct {
Type string `json:"type"`
Text string `json:"text"`
}
if err := json.Unmarshal(raw, &parts); err == nil {
var b strings.Builder
for _, p := range parts {
if strings.TrimSpace(p.Text) != "" {
b.WriteString(p.Text)
}
}
return strings.TrimSpace(b.String())
}
return ""
}
// HeuristicCompleter is used when OpenAI is not configured (local/dev fallback).
type HeuristicCompleter struct{}
@@ -427,8 +512,8 @@ func (h HeuristicCompleter) Complete(_ context.Context, system, user string) (Co
name = "Product"
}
desc := labeledPromptValue(user, "desc:", "description:", "current description:")
if desc == "" {
desc = "Product description"
if desc == "" || descriptionEchoesTitle(desc, name) || isWeakPriorEnhanceDescription(desc, name) {
desc = inventHeuristicDescription(system, user, name)
}
b, _ := json.Marshal(map[string]string{"name": name, "description": desc})
text = string(b)
@@ -450,3 +535,72 @@ func (h HeuristicCompleter) Complete(_ context.Context, system, user string) (Co
Raw: map[string]any{"provider": "heuristic"},
}, nil
}
// parseAttrsFromPrompt extracts the Attrs:/Attributes: JSON object from an enhance user message.
func parseAttrsFromPrompt(user string) map[string]any {
lower := strings.ToLower(user)
label := "attrs:"
at := strings.Index(lower, label)
if at < 0 {
label = "attributes:"
at = strings.Index(lower, label)
if at < 0 {
return nil
}
}
rest := strings.TrimSpace(user[at+len(label):])
if i := strings.Index(rest, "{"); i >= 0 {
rest = rest[i:]
if end := strings.Index(rest, "}"); end >= 0 {
rest = rest[:end+1]
}
} else {
rest = firstLine(rest)
}
var m map[string]any
if err := json.Unmarshal([]byte(rest), &m); err != nil || len(m) == 0 {
return nil
}
return m
}
// inventHeuristicDescription builds a short factual description from Category + Attrs
// (+ brand / model / dims) for thin inputs. Matches Slovenian vs English from the
// enhance system/user prompt language hint. Never returns sole retail-filler copy
// when a title or attributes exist.
func inventHeuristicDescription(system, user, name string) string {
name = strings.TrimSpace(name)
if name == "" || name == "<nil>" || isPromptLabelTitle(name) {
name = labeledPromptValue(user, "name:", "current name:")
}
if name == "" || isPromptLabelTitle(name) {
name = "Product"
}
cat := labeledPromptValue(user, "category:")
if isPromptLabelTitle(cat) {
cat = ""
}
lang := languageCodeFromEnhancePrompt(system, user)
attrs := CompactAttrs(parseAttrsFromPrompt(user), MaxAttrKeys)
if out := synthesizeDescriptionFromTitle(name, cat, lang, attrs); out != "" {
return out
}
return SanitizeOutput(fmt.Sprintf("%s is a catalog product with the known attributes.", name))
}
func languageCodeFromEnhancePrompt(system, user string) string {
blob := strings.ToLower(system + "\n" + user)
for _, code := range company.ContentLanguages {
label := strings.ToLower(company.LanguageLabel(code))
if label == "" {
continue
}
if strings.Contains(blob, "in "+label) || strings.Contains(blob, "language: "+label) {
return code
}
}
if code, err := company.ParseLanguage(labeledPromptValue(user, "language:", "content language:"), false); err == nil {
return code
}
return company.DefaultLanguage
}
apps/api/internal/processing/openai_test.go
+119
View File
@@ -2,12 +2,39 @@ package processing
import (
"context"
"encoding/json"
"net/http"
"net/http/httptest"
"strings"
"testing"
"time"
)
func TestHeuristicCompleter_thinProductInventFromAttrs(t *testing.T) {
t.Parallel()
h := HeuristicCompleter{}
system := `Retail product copywriter. Schema: {"name":"string","description":"string"}`
user := "Category: Monitors\nName: UltraView 27\nDesc: UltraView 27\nAttrs: {\"brand\":\"Acme\",\"size\":\"27 inch\",\"panel\":\"IPS\"}"
comp, err := h.Complete(context.Background(), system, user)
if err != nil {
t.Fatal(err)
}
var out map[string]string
if err := json.Unmarshal([]byte(comp.Text), &out); err != nil {
t.Fatalf("json: %v text=%q", err, comp.Text)
}
if out["name"] != "UltraView 27" {
t.Fatalf("name=%q", out["name"])
}
desc := out["description"]
if desc == "" || strings.EqualFold(desc, "UltraView 27") || desc == "Product description" {
t.Fatalf("desc=%q want invented non-title copy", desc)
}
if !strings.Contains(strings.ToLower(desc), "acme") && !strings.Contains(strings.ToLower(desc), "ips") && !strings.Contains(strings.ToLower(desc), "27") {
t.Fatalf("desc=%q want attrs-derived facts", desc)
}
}
func TestNewOpenAIClient_capsRetries(t *testing.T) {
c := NewOpenAIClient("k", "https://api.openai.com/v1", "m", 0, 99)
if c.MaxRetries != maxOpenAIRetries {
@@ -107,3 +134,95 @@ func TestNewOpenAIClient_blocksPrivateDialInProduction(t *testing.T) {
t.Fatalf("expected host is not allowed, got: %v", err)
}
}
func TestChoiceMessageText_prefersContentThenReasoningJSON(t *testing.T) {
t.Parallel()
if got := choiceMessageText(json.RawMessage(`"{\"name\":\"A\",\"description\":\"B\"}"`), "", ""); got == "" {
t.Fatal("expected content string")
}
reasoning := `Thinking…\nDraft JSON:\n{"name":"NOSILEC W53070","description":"Stenski nosilec za TV."}\nVerify…`
got := choiceMessageText(json.RawMessage(`""`), reasoning, "")
if !strings.Contains(got, `"name"`) || !strings.Contains(got, "NOSILEC") {
t.Fatalf("got=%q want JSON from reasoning_content", got)
}
if got := choiceMessageText(json.RawMessage(`null`), "no json here", ""); got != "" {
t.Fatalf("expected empty without JSON, got %q", got)
}
}
func TestOpenAIClient_doComplete_emptyContentWithReasoningJSON(t *testing.T) {
t.Parallel()
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
_ = json.NewEncoder(w).Encode(map[string]any{
"model": "code-fast",
"choices": []map[string]any{{
"finish_reason": "length",
"message": map[string]any{
"role": "assistant",
"content": "",
"reasoning_content": `steps… {"name":"TV Mount","description":"A wall mount for TVs."} more`,
},
}},
"usage": map[string]int{"prompt_tokens": 10, "completion_tokens": 50, "total_tokens": 60},
})
}))
defer srv.Close()
c := NewOpenAIClient("test-key", srv.URL, "code-fast", 0, 1)
c.HTTPClient = srv.Client()
comp, err := c.CompleteWithOptions(context.Background(), "sys", "user", CompleteOptions{MaxTokens: 350, Temperature: 0.2})
if err != nil {
t.Fatal(err)
}
if !strings.Contains(comp.Text, "TV Mount") {
t.Fatalf("text=%q", comp.Text)
}
}
func TestOpenAIClient_doComplete_emptyContentLengthRetriesWithBudget(t *testing.T) {
t.Parallel()
calls := 0
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
calls++
var req map[string]any
_ = json.NewDecoder(r.Body).Decode(&req)
maxTok, _ := req["max_tokens"].(float64)
if calls == 1 {
if maxTok != 350 {
t.Errorf("first max_tokens=%v want 350", maxTok)
}
_ = json.NewEncoder(w).Encode(map[string]any{
"model": "code-fast",
"choices": []map[string]any{{
"finish_reason": "length",
"message": map[string]any{"role": "assistant", "content": "", "reasoning_content": "still thinking"},
}},
"usage": map[string]int{"prompt_tokens": 1, "completion_tokens": 350, "total_tokens": 351},
})
return
}
if maxTok != float64(maxTokensReasoningBudget) {
t.Errorf("retry max_tokens=%v want %d", maxTok, maxTokensReasoningBudget)
}
_ = json.NewEncoder(w).Encode(map[string]any{
"model": "code-fast",
"choices": []map[string]any{{
"finish_reason": "stop",
"message": map[string]any{"role": "assistant", "content": `{"name":"X","description":"Y product description text here."}`},
}},
"usage": map[string]int{"prompt_tokens": 1, "completion_tokens": 40, "total_tokens": 41},
})
}))
defer srv.Close()
c := NewOpenAIClient("test-key", srv.URL, "code-fast", 0, 2)
c.HTTPClient = srv.Client()
comp, err := c.CompleteWithOptions(context.Background(), "sys", "user", CompleteOptions{MaxTokens: 350, Temperature: 0.2})
if err != nil {
t.Fatal(err)
}
if calls != 2 {
t.Fatalf("calls=%d want 2", calls)
}
if !strings.Contains(comp.Text, "\"name\"") {
t.Fatalf("text=%q", comp.Text)
}
}
apps/api/internal/processing/pipeline.go
+352 -49
View File
@@ -657,10 +657,7 @@ func (p *Pipeline) ProcessJob(ctx context.Context, jobID uuid.UUID) error {
if !shouldFlushJobProgress(sinceFlush, progressEvery, batchDone) {
return
}
mode := modeLabel
if lastResult != nil && lastResult.AIProviderMode != "" {
mode = lastResult.AIProviderMode
}
mode := preferKnownProviderMode(lastResultAIProviderMode(lastResult), modeLabel)
if err := p.flushJobCountersAndProgress(ctx, jobID, processingType, lastResult, processed, tokensSinceFlush, mode); err != nil {
log.Printf("processing: flush progress job=%s err=%s", jobID, TruncateError(err))
}
@@ -672,6 +669,7 @@ func (p *Pipeline) ProcessJob(ctx context.Context, jobID uuid.UUID) error {
if err := stopOnCancel(p.jobCancelled(ctx, jobID)); err != nil {
flushProgress(true)
if errors.Is(err, errJobCancelled) {
_ = p.reclaimOrphanedProcessingItems(ctx, jobID)
return nil
}
return fmt.Errorf("processing: check cancel job=%s: %w", jobID, err)
@@ -682,6 +680,27 @@ func (p *Pipeline) ProcessJob(ctx context.Context, jobID uuid.UUID) error {
return err
}
if len(items) == 0 {
// loadPendingItems only reclaims processing rows older than StuckAgeInterval.
// A crashed peer can leave fresh processing rows; do not mark the job
// completed while open work remains — reset them and keep going.
var open int
if err := p.Pool.QueryRow(ctx, `
SELECT COUNT(*) FROM processing_job_products
WHERE job_id = $1 AND processed_product_id IS NULL
AND status IN ('pending', 'processing')`, jobID).Scan(&open); err != nil {
flushProgress(true)
return fmt.Errorf("processing: count open items job=%s: %w", jobID, err)
}
if open > 0 {
if _, err := p.Pool.Exec(ctx, `
UPDATE processing_job_products
SET status = 'pending', error = NULL, updated_at = now()
WHERE job_id = $1 AND status = 'processing' AND processed_product_id IS NULL`, jobID); err != nil {
flushProgress(true)
return fmt.Errorf("processing: reclaim fresh processing job=%s: %w", jobID, err)
}
continue
}
break
}
if err := p.hydrateJobItems(ctx, companyID, items); err != nil {
@@ -695,6 +714,7 @@ func (p *Pipeline) ProcessJob(ctx context.Context, jobID uuid.UUID) error {
if err := stopOnCancel(p.jobCancelled(ctx, jobID)); err != nil {
flushProgress(true)
if errors.Is(err, errJobCancelled) {
_ = p.reclaimOrphanedProcessingItems(ctx, jobID)
return nil
}
return fmt.Errorf("processing: check cancel job=%s: %w", jobID, err)
@@ -766,10 +786,7 @@ func (p *Pipeline) ProcessJob(ctx context.Context, jobID uuid.UUID) error {
if finalStatus == "failed" {
finalStep = "failed"
}
finalMode := modeLabel
if lastResult != nil && lastResult.AIProviderMode != "" {
finalMode = lastResult.AIProviderMode
}
finalMode := preferKnownProviderMode(lastResultAIProviderMode(lastResult), modeLabel)
_, err = p.Pool.Exec(ctx, `
UPDATE processing_jobs
SET status = $2, processed_products = $3, error = $4, completed_at = now(), updated_at = now(),
@@ -818,9 +835,18 @@ type jobScopedCache struct {
enhanceSystemTemplate string
enhanceUserTemplate string
enhanceByLang map[string]PromptTemplates
// categoryPromptsByLang maps lower(trim(name)) → lang → sanitized prompt.
// categoryPromptsByLang maps lower(trim(name|unique_id)) → lang → sanitized prompt.
categoryPromptsByLang map[string]company.LangPromptMap
contentLanguages []string
// categoryFormulasByKey maps lower(trim(name|unique_id)) → title/description templates.
categoryFormulasByKey map[string]CategoryFormulas
// categoryNamesByUID maps unique_id → display name for meta/enhance/synthesize.
categoryNamesByUID map[string]string
// categoryUniqueIDs is the company taxonomy unique_id set (empty = skip validation).
categoryUniqueIDs map[string]struct{}
// categoryAttrKeys maps category unique_id → canonicalized attribute_key set
// from category_attributes (for AI enhance allowlisting).
categoryAttrKeys map[string]map[string]struct{}
contentLanguages []string
// Entitlements snapshot — avoids EntitlementsForCompany N+1 per product.
billingEnabled bool
canUseAI bool
@@ -902,56 +928,224 @@ func (p *Pipeline) loadJobScopedCache(ctx context.Context, companyID, jobID uuid
}
}
}
cache.categoryPromptsByLang = p.loadCategoryEnhancePrompts(ctx, companyID, jobID)
cache.categoryPromptsByLang, cache.categoryFormulasByKey, cache.categoryNamesByUID = p.loadCategoryEnhanceOverlays(ctx, companyID, jobID)
cache.categoryUniqueIDs = p.loadCategoryUniqueIDs(ctx, companyID, jobID)
cache.categoryAttrKeys = p.loadCategoryAttributeKeySets(ctx, companyID, jobID)
return cache
}
func (p *Pipeline) loadCategoryEnhancePrompts(ctx context.Context, companyID, jobID uuid.UUID) map[string]company.LangPromptMap {
func (p *Pipeline) loadCategoryUniqueIDs(ctx context.Context, companyID, jobID uuid.UUID) map[string]struct{} {
rows, err := p.Pool.Query(ctx, `
SELECT name, COALESCE(prompt, '{}'::jsonb)
SELECT unique_id
FROM categories
WHERE company_id = $1 AND prompt <> '{}'::jsonb`, companyID)
WHERE company_id = $1
AND COALESCE(NULLIF(BTRIM(unique_id), ''), '') <> ''`, companyID)
if err != nil {
log.Printf("processing: load category prompts job=%s company=%s err=%s", jobID, companyID, TruncateError(err))
log.Printf("processing: load category unique_ids job=%s company=%s err=%s", jobID, companyID, TruncateError(err))
return nil
}
defer rows.Close()
out := make(map[string]company.LangPromptMap)
out := make(map[string]struct{})
for rows.Next() {
var name string
var raw []byte
if err := rows.Scan(&name, &raw); err != nil {
log.Printf("processing: scan category prompt job=%s err=%s", jobID, TruncateError(err))
var id string
if err := rows.Scan(&id); err != nil {
log.Printf("processing: scan category unique_id job=%s err=%s", jobID, TruncateError(err))
continue
}
key := strings.ToLower(strings.TrimSpace(name))
if key == "" {
id = strings.TrimSpace(id)
if id == "" {
continue
}
m, err := company.DecodeLangPromptMap(raw)
if err != nil || !company.HasAnyPrompt(m) {
continue
}
// Re-sanitize with category rune cap.
cleaned := company.LangPromptMap{}
for lang, prompt := range m {
p := strings.TrimSpace(security.SanitizePrompt(prompt, catalog.MaxCategoryPromptRunes))
if p == "" {
continue
}
cleaned[lang] = p
}
if company.HasAnyPrompt(cleaned) {
out[key] = cleaned
}
out[id] = struct{}{}
}
if err := rows.Err(); err != nil {
log.Printf("processing: category prompts rows job=%s err=%s", jobID, TruncateError(err))
log.Printf("processing: category unique_ids rows job=%s err=%s", jobID, TruncateError(err))
}
return out
}
func categoryEnhancePromptFor(prompts map[string]company.LangPromptMap, category, language string) string {
// loadCategoryAttributeKeySets returns category unique_id → attribute_key set from
// category_attributes. Empty map on failure (callers still fall back to core keys).
func (p *Pipeline) loadCategoryAttributeKeySets(ctx context.Context, companyID, jobID uuid.UUID) map[string]map[string]struct{} {
out := map[string]map[string]struct{}{}
if p == nil || p.Pool == nil {
return out
}
rows, err := p.Pool.Query(ctx, `
SELECT ca.category_unique_id, a.attribute_key
FROM category_attributes ca
INNER JOIN attributes a ON a.id = ca.attribute_id AND a.company_id = ca.company_id
WHERE ca.company_id = $1
AND COALESCE(NULLIF(BTRIM(a.attribute_key), ''), '') <> ''`, companyID)
if err != nil {
log.Printf("processing: load category attribute keys job=%s company=%s err=%s", jobID, companyID, TruncateError(err))
return out
}
defer rows.Close()
for rows.Next() {
var catUID, key string
if err := rows.Scan(&catUID, &key); err != nil {
log.Printf("processing: scan category attribute key job=%s err=%s", jobID, TruncateError(err))
continue
}
catUID = strings.TrimSpace(catUID)
if catUID == "" {
continue
}
canon := canonicalizeAttrKey(key)
if canon == "" {
continue
}
set := out[catUID]
if set == nil {
set = map[string]struct{}{}
out[catUID] = set
}
set[canon] = struct{}{}
set[strings.ToLower(strings.TrimSpace(key))] = struct{}{}
}
if err := rows.Err(); err != nil {
log.Printf("processing: category attribute keys rows job=%s err=%s", jobID, TruncateError(err))
}
return out
}
// allowedAttrKeysForCategory returns category_attributes keys when known; empty
// non-nil map means core characteristics only. nil cache → nil (sanitize-only).
func allowedAttrKeysForCategory(cache *jobScopedCache, categoryUID string) map[string]struct{} {
if cache == nil {
return nil
}
return allowedAttrKeysFromSets(cache.categoryAttrKeys, categoryUID)
}
func categoryAttrKeysFromCache(cache *jobScopedCache) map[string]map[string]struct{} {
if cache == nil {
return nil
}
return cache.categoryAttrKeys
}
// allowedAttrKeysFromSets picks category keys when present; empty non-nil → core only.
// When sets is nil, returns nil (sanitize-only / unit tests).
func allowedAttrKeysFromSets(sets map[string]map[string]struct{}, categoryUID string) map[string]struct{} {
if sets == nil {
return nil
}
categoryUID = strings.TrimSpace(categoryUID)
if categoryUID != "" {
if keys, ok := sets[categoryUID]; ok && len(keys) > 0 {
return keys
}
}
return map[string]struct{}{}
}
func (p *Pipeline) loadCategoryEnhanceOverlays(ctx context.Context, companyID, jobID uuid.UUID) (map[string]company.LangPromptMap, map[string]CategoryFormulas, map[string]string) {
rows, err := p.Pool.Query(ctx, `
SELECT COALESCE(unique_id, ''), name,
COALESCE(prompt, '{}'::jsonb),
title_template,
description_template
FROM categories
WHERE company_id = $1`, companyID)
if err != nil {
log.Printf("processing: load category overlays job=%s company=%s err=%s", jobID, companyID, TruncateError(err))
return nil, nil, nil
}
defer rows.Close()
prompts := make(map[string]company.LangPromptMap)
formulas := make(map[string]CategoryFormulas)
namesByUID := make(map[string]string)
for rows.Next() {
var uniqueID, name string
var promptRaw []byte
var titleRaw, descRaw []byte
if err := rows.Scan(&uniqueID, &name, &promptRaw, &titleRaw, &descRaw); err != nil {
log.Printf("processing: scan category overlay job=%s err=%s", jobID, TruncateError(err))
continue
}
uid := strings.TrimSpace(uniqueID)
display := strings.TrimSpace(name)
if uid != "" && display != "" {
namesByUID[uid] = display
}
keys := categoryOverlayKeys(uniqueID, name)
if len(keys) == 0 {
continue
}
if m, err := company.DecodeLangPromptMap(promptRaw); err == nil && company.HasAnyPrompt(m) {
cleaned := company.LangPromptMap{}
for lang, prompt := range m {
p := strings.TrimSpace(security.SanitizePrompt(prompt, catalog.MaxCategoryPromptRunes))
if p == "" {
continue
}
cleaned[lang] = p
}
if company.HasAnyPrompt(cleaned) {
for _, key := range keys {
prompts[key] = cleaned
}
}
}
f := CategoryFormulas{}
if t := decodeOptionalJSONObject(titleRaw); t != nil {
f.TitleTemplate = t
}
if d := decodeOptionalJSONObject(descRaw); d != nil {
f.DescriptionTemplate = d
}
if f.TitleTemplate != nil || f.DescriptionTemplate != nil {
for _, key := range keys {
formulas[key] = f
}
}
}
if err := rows.Err(); err != nil {
log.Printf("processing: category overlays rows job=%s err=%s", jobID, TruncateError(err))
}
return prompts, formulas, namesByUID
}
func categoryOverlayKeys(uniqueID, name string) []string {
seen := map[string]struct{}{}
var out []string
add := func(raw string) {
key := strings.ToLower(strings.TrimSpace(raw))
if key == "" {
return
}
if _, ok := seen[key]; ok {
return
}
seen[key] = struct{}{}
out = append(out, key)
}
add(uniqueID)
add(name)
return out
}
func decodeOptionalJSONObject(raw []byte) map[string]any {
raw = []byte(strings.TrimSpace(string(raw)))
if len(raw) == 0 || string(raw) == "null" || string(raw) == "{}" {
return nil
}
var obj map[string]any
if err := json.Unmarshal(raw, &obj); err != nil || len(obj) == 0 {
return nil
}
return obj
}
// loadCategoryEnhancePrompts is retained for tests / callers that only need prompts.
func (p *Pipeline) loadCategoryEnhancePrompts(ctx context.Context, companyID, jobID uuid.UUID) map[string]company.LangPromptMap {
prompts, _, _ := p.loadCategoryEnhanceOverlays(ctx, companyID, jobID)
return prompts
}
func categoryEnhancePromptFor(prompts map[string]company.LangPromptMap, category, language, primary string) string {
if len(prompts) == 0 {
return ""
}
@@ -959,7 +1153,7 @@ func categoryEnhancePromptFor(prompts map[string]company.LangPromptMap, category
if key == "" {
return ""
}
return company.PromptForLanguage(prompts[key], language)
return company.PromptForLanguage(prompts[key], language, primary)
}
func progressFromResult(processingType string, result *StepResult) []StepProgress {
@@ -1180,7 +1374,35 @@ func (p *Pipeline) jobCancelled(ctx context.Context, jobID uuid.UUID) (bool, err
if err != nil {
return false, err
}
return status == "cancelled", nil
return isTerminalJobStatus(status), nil
}
// isTerminalJobStatus reports statuses that should stop an in-flight ProcessJob
// so worker JobSlots are released (failed/completed parkers included).
func isTerminalJobStatus(status string) bool {
switch strings.ToLower(strings.TrimSpace(status)) {
case "cancelled", "failed", "completed":
return true
default:
return false
}
}
// reclaimOrphanedProcessingItems returns in-flight items to pending when the job
// was marked failed/completed externally so a later ClaimNext can finish them.
func (p *Pipeline) reclaimOrphanedProcessingItems(ctx context.Context, jobID uuid.UUID) error {
var status string
if err := p.Pool.QueryRow(ctx, `SELECT status FROM processing_jobs WHERE id = $1`, jobID).Scan(&status); err != nil {
return err
}
if strings.EqualFold(strings.TrimSpace(status), "cancelled") {
return nil
}
_, err := p.Pool.Exec(ctx, `
UPDATE processing_job_products
SET status = 'pending', error = NULL, updated_at = now()
WHERE job_id = $1 AND status = 'processing' AND processed_product_id IS NULL`, jobID)
return err
}
func (p *Pipeline) processOne(ctx context.Context, companyID, jobID uuid.UUID, it *jobItem, processingType string, engine *Engine, modeLabel string, usingBYOK bool, cache *jobScopedCache) (bool, int, StepResult, error) {
@@ -1215,6 +1437,8 @@ func (p *Pipeline) processOne(ctx context.Context, companyID, jobID uuid.UUID, i
var stdDefs []StandardFieldDef
var brandPrompt, language, enhanceSystemTemplate, enhanceUserTemplate string
var categoryPromptsByLang map[string]company.LangPromptMap
var categoryFormulasByKey map[string]CategoryFormulas
var categoryNamesByUID map[string]string
var contentLanguages []string
var enhanceByLang map[string]PromptTemplates
if cache != nil {
@@ -1224,6 +1448,8 @@ func (p *Pipeline) processOne(ctx context.Context, companyID, jobID uuid.UUID, i
enhanceSystemTemplate = cache.enhanceSystemTemplate
enhanceUserTemplate = cache.enhanceUserTemplate
categoryPromptsByLang = cache.categoryPromptsByLang
categoryFormulasByKey = cache.categoryFormulasByKey
categoryNamesByUID = cache.categoryNamesByUID
contentLanguages = cache.contentLanguages
enhanceByLang = cache.enhanceByLang
}
@@ -1231,6 +1457,11 @@ func (p *Pipeline) processOne(ctx context.Context, companyID, jobID uuid.UUID, i
if len(stdDefs) > 0 {
enriched = FillMissingStandardFields(enriched, raw, stdDefs)
}
// Deterministic category unique_id from feed/mapped (no LLM required).
coerceMappedCategoryUniqueID(enriched)
if cat := categoryUniqueIDFromMaps(enriched, raw); cat != "" {
enriched["category"] = cat
}
if gtin == "" {
gtin = stringFromMap(enriched, "gtin", "ean", "EAN", "upc")
}
@@ -1249,6 +1480,10 @@ func (p *Pipeline) processOne(ctx context.Context, companyID, jobID uuid.UUID, i
EnhanceSystemTemplate: enhanceSystemTemplate,
EnhanceUserTemplate: enhanceUserTemplate,
CategoryPromptsByLang: categoryPromptsByLang,
CategoryFormulasByKey: categoryFormulasByKey,
CategoryNamesByUID: categoryNamesByUID,
AllowedAttrKeys: allowedAttrKeysForCategory(cache, stringFromAny(enriched["category"])),
CategoryAttrKeys: categoryAttrKeysFromCache(cache),
}
if it.hydrated {
if it.hasPrior {
@@ -1292,16 +1527,31 @@ func (p *Pipeline) processOne(ctx context.Context, companyID, jobID uuid.UUID, i
if err != nil {
return false, 0, StepResult{}, err
}
// Persist mapped unique_id only when it exists in the company taxonomy.
// Empty taxonomy set skips filtering (unit tests / companies without categories).
if cache != nil {
filterCategoryIfInvalid(&result, cache.categoryUniqueIDs)
syncCategoryName(&result, cache.categoryNamesByUID)
}
// Re-apply after category validation so allowlist matches the persisted category.
allowed := allowedAttrKeysForCategory(cache, result.Category)
result.Attributes = AttrsForPersist(result.Attributes, allowed)
result.ProcessedAttributes = AttrsForPersist(result.ProcessedAttributes, allowed)
if len(result.ProcessedAttributes) == 0 {
result.ProcessedAttributes = result.Attributes
}
// Free template SEO meta (no FillMetaAI / no extra credits).
result.MetaTitle, result.MetaDescription = fillMetaFromResult(result)
attrsJSON, procAttrsJSON, gptJSON, sourcesJSON, err := marshalProcessOnePayload(result)
if err != nil {
return false, 0, result, err
}
providerMode := result.AIProviderMode
if providerMode == "" {
if modeLabel != "" {
providerMode = modeLabel
} else if result.TotalTokens > 0 {
// Treat "unknown" like empty so job modeLabel / EngineProviderMode wins on
// hash-skip (TotalTokens==0) paths that previously stamped unknown in RunSteps.
providerMode := preferKnownProviderMode(result.AIProviderMode, modeLabel)
if isUnknownProviderMode(providerMode) {
if result.TotalTokens > 0 {
providerMode = AIProviderInternal
} else {
providerMode = AIProviderUnknown
@@ -1375,8 +1625,9 @@ const upsertProcessedProductSQL = `
INSERT INTO processed_products (
company_id, raw_product_id, product_id, name, category, description,
processed_name, processed_description, status, attributes, processed_attributes,
gpt_response, total_tokens, field_sources, ai_provider_mode, localized_content
) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,'needs_review',$9,$10,$11,$12,$13,$14,$15::jsonb)
gpt_response, total_tokens, field_sources, ai_provider_mode, localized_content,
meta_title, meta_description
) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,'needs_review',$9,$10,$11,$12,$13,$14,$15::jsonb,$16,$17)
ON CONFLICT (company_id, raw_product_id) DO UPDATE SET
product_id = EXCLUDED.product_id,
name = EXCLUDED.name,
@@ -1392,6 +1643,57 @@ const upsertProcessedProductSQL = `
field_sources = EXCLUDED.field_sources,
ai_provider_mode = EXCLUDED.ai_provider_mode,
localized_content = EXCLUDED.localized_content,
meta_title = CASE
WHEN NULLIF(BTRIM(processed_products.meta_title), '') IS NULL
THEN NULLIF(BTRIM(EXCLUDED.meta_title), '')
WHEN BTRIM(processed_products.meta_title) ~ '\|\s+[0-9]+$'
THEN NULLIF(BTRIM(EXCLUDED.meta_title), '')
WHEN LOWER(processed_products.meta_title) LIKE '%short retail title%'
OR LOWER(processed_products.meta_title) LIKE '%title formula%'
OR LOWER(processed_products.meta_title) LIKE '%follow any%'
OR LOWER(processed_products.meta_title) LIKE '%constraints that follow%'
OR LOWER(processed_products.meta_title) LIKE '%prefer 1-3%'
OR LOWER(processed_products.meta_title) LIKE '%reply with only json%'
OR LOWER(processed_products.meta_title) LIKE '%your reply is parsed as json%'
THEN NULLIF(BTRIM(EXCLUDED.meta_title), '')
ELSE processed_products.meta_title
END,
meta_description = CASE
WHEN NULLIF(BTRIM(processed_products.meta_description), '') IS NULL
THEN NULLIF(BTRIM(EXCLUDED.meta_description), '')
WHEN LOWER(processed_products.meta_description) LIKE '%prefer 1-3%'
OR LOWER(processed_products.meta_description) LIKE '%short retail title%'
OR LOWER(processed_products.meta_description) LIKE '%title formula%'
OR LOWER(processed_products.meta_description) LIKE '%do not emit%'
OR LOWER(processed_products.meta_description) LIKE '%reply with only json%'
THEN NULLIF(BTRIM(EXCLUDED.meta_description), '')
-- When refreshing a poisoned meta_title (| digits or prompt leakage), also refresh empty/weak stub desc.
WHEN (
BTRIM(processed_products.meta_title) ~ '\|\s+[0-9]+$'
OR LOWER(processed_products.meta_title) LIKE '%short retail title%'
OR LOWER(processed_products.meta_title) LIKE '%title formula%'
OR LOWER(processed_products.meta_title) LIKE '%follow any%'
OR LOWER(processed_products.meta_title) LIKE '%constraints that follow%'
OR LOWER(processed_products.meta_title) LIKE '%prefer 1-3%'
OR LOWER(processed_products.meta_title) LIKE '%reply with only json%'
OR LOWER(processed_products.meta_title) LIKE '%your reply is parsed as json%'
)
AND (
NULLIF(BTRIM(processed_products.meta_description), '') IS NULL
OR char_length(BTRIM(processed_products.meta_description)) < 40
OR LOWER(processed_products.meta_description) LIKE '%ready for retail listing%'
OR LOWER(processed_products.meta_description) LIKE '%quality product ready%'
OR LOWER(processed_products.meta_description) LIKE '%product description%'
OR LOWER(processed_products.meta_description) LIKE '%based on available specifications%'
OR LOWER(processed_products.meta_description) LIKE '%with available specifications%'
OR LOWER(processed_products.meta_description) LIKE '%available catalog details%'
OR LOWER(processed_products.meta_description) LIKE '%pripravljeno za prodajo%'
OR LOWER(processed_products.meta_description) LIKE '%na podlagi razpoložljivih specifikacij%'
OR LOWER(processed_products.meta_description) LIKE '%prefer 1-3%'
)
THEN NULLIF(BTRIM(EXCLUDED.meta_description), '')
ELSE processed_products.meta_description
END,
updated_at = now()
RETURNING id`
@@ -1426,6 +1728,7 @@ func (p *Pipeline) upsertProcessedProduct(
err = queryRow(ctx, upsertProcessedProductSQL,
companyID, rawID, gtin, result.Name, result.Category, result.Description,
result.ProcessedName, result.ProcessedDescription, attrsJSON, procAttrsJSON, gptJSON, result.TotalTokens, sourcesJSON, providerMode, string(locJSON),
result.MetaTitle, result.MetaDescription,
).Scan(&processedID)
return processedID, err
}
apps/api/internal/processing/pipeline_llm_mock_test.go
+53
View File
@@ -119,6 +119,12 @@ func TestRunSteps_enhanceMockProviderFailure(t *testing.T) {
if !foundFailed {
t.Fatalf("expected ai_enhance failed in progress=%v", prog)
}
if strings.TrimSpace(out.ProcessedDescription) == "" {
t.Fatal("provider failure must leave nonempty ProcessedDescription when title exists")
}
if descriptionEchoesTitle(out.ProcessedDescription, out.ProcessedName) {
t.Fatalf("failure synth must not echo title: name=%q desc=%q", out.ProcessedName, out.ProcessedDescription)
}
}
func TestRunSteps_enhanceMockTimeout(t *testing.T) {
@@ -150,6 +156,53 @@ func TestRunSteps_enhanceMockTimeout(t *testing.T) {
if !ok {
t.Fatalf("expected ai_enhance failed, progress=%v", prog)
}
if strings.TrimSpace(out.ProcessedDescription) == "" {
t.Fatal("timeout must leave nonempty ProcessedDescription when title exists")
}
if descriptionEchoesTitle(out.ProcessedDescription, out.ProcessedName) {
t.Fatalf("timeout synth must not echo title: name=%q desc=%q", out.ProcessedName, out.ProcessedDescription)
}
}
func TestRunSteps_enhanceTimeoutSynthesizesDescriptionAndInternalMode(t *testing.T) {
t.Parallel()
e := &Engine{
Completer: stubCompleter{fn: func(_, _ string) (Completion, error) {
return Completion{}, context.DeadlineExceeded
}},
ProviderMode: AIProviderInternal,
Vector: NoopVectorCategorizer{},
}
out, err := e.RunSteps(context.Background(), "co-timeout-synth", ProductInput{
Mapped: map[string]any{
"name": "VOX EBR700",
"description": "",
"brand": "VOX",
"category": "50",
},
Language: "sl",
CategoryNamesByUID: map[string]string{
"50": "Štedilniki",
},
}, "enhance_only", nil, StepPolicy{AllowAI: true})
if err != nil {
t.Fatalf("RunSteps should swallow timeout: %v", err)
}
if strings.TrimSpace(out.ProcessedName) == "" {
t.Fatal("expected nonempty ProcessedName")
}
if strings.TrimSpace(out.ProcessedDescription) == "" {
t.Fatal("timeout must synthesize nonempty ProcessedDescription when title exists")
}
if descriptionEchoesTitle(out.ProcessedDescription, out.ProcessedName) {
t.Fatalf("synth must not echo title: name=%q desc=%q", out.ProcessedName, out.ProcessedDescription)
}
if out.AIProviderMode != AIProviderInternal {
t.Fatalf("ai_provider_mode=%q want %q (attempted enhance, not unknown)", out.AIProviderMode, AIProviderInternal)
}
if _, ok := out.FieldSources[FieldEnhanceInputHash]; ok {
t.Fatalf("timeout must clear enhance hash, got %v", out.FieldSources[FieldEnhanceInputHash])
}
}
func TestOpenAIClient_Complete_httptestHappyPath(t *testing.T) {
apps/api/internal/processing/pipeline_process_test.go
+17
View File
@@ -28,6 +28,23 @@ func TestStopOnCancel(t *testing.T) {
}
}
func TestIsTerminalJobStatus(t *testing.T) {
cases := map[string]bool{
"pending": false,
"running": false,
"cancelled": true,
"failed": true,
"completed": true,
" FAILED ": true,
"": false,
}
for in, want := range cases {
if got := isTerminalJobStatus(in); got != want {
t.Fatalf("isTerminalJobStatus(%q)=%v want %v", in, got, want)
}
}
}
func TestMarshalStepProgress_roundTrip(t *testing.T) {
progress := InitialStepProgress("full")
if len(progress) == 0 {
apps/api/internal/processing/pipeline_steps_test.go
+161 -3
View File
@@ -88,15 +88,25 @@ func TestSanitizeProductAttributes(t *testing.T) {
if _, ok := got["netwidth"]; ok {
t.Fatalf("alias netwidth should collapse to width: %v", got)
}
dropped := SanitizeProductAttributes(map[string]any{
"brand": "XIAOMI",
"product_model": "6941948703193",
})
if _, ok := dropped["product_model"]; ok {
t.Fatalf("barcode-like product_model must be dropped: %v", dropped)
}
if dropped["brand"] != "XIAOMI" {
t.Fatalf("brand=%v", dropped["brand"])
}
}
func TestFilterAttributesByAllowed(t *testing.T) {
in := map[string]any{
"brand": "Ostalo",
"zavora": "Mehanska",
"brand": "Ostalo",
"zavora": "Mehanska",
"vzmetenje": "Spredaj",
"nosilnost": "40 kg",
"width": "0.6 m",
"width": "0.6 m",
}
allowed := map[string]struct{}{
"nosilnost": {},
@@ -121,6 +131,154 @@ func TestFilterAttributesByAllowed(t *testing.T) {
}
}
func TestAttrsForEnhance_cat28DropsZavora(t *testing.T) {
attrs := map[string]any{
"brand": "Ostalo",
"product_model": "W53070",
"zavora": "Mehanska in elektricna",
"vzmetenje": "Spredaj in zadaj",
"nosilnost": "40 kg",
}
// Cat 28 TV mounts: barva/nosilnost — not motorcycle specs.
allowed := map[string]struct{}{
"barva": {},
"nosilnost": {},
}
got := AttrsForEnhance(attrs, allowed)
if got["brand"] != "Ostalo" || got["product_model"] != "W53070" {
t.Fatalf("core keys missing: %v", got)
}
if got["nosilnost"] != "40 kg" {
t.Fatalf("category key missing: %v", got)
}
if _, ok := got["zavora"]; ok {
t.Fatalf("zavora must not reach enhance: %v", got)
}
if _, ok := got["vzmetenje"]; ok {
t.Fatalf("vzmetenje must not reach enhance: %v", got)
}
user := ProductEnhanceUser("Nosilci za TV", "NOSILEC W53070", "", got)
if strings.Contains(strings.ToLower(user), "zavora") {
t.Fatalf("enhance user must not mention zavora:\n%s", user)
}
if !strings.Contains(user, "Ostalo") || !strings.Contains(user, "W53070") {
t.Fatalf("enhance user should keep brand/model:\n%s", user)
}
}
func TestAttrsForEnhance_emptyAllowlistCoreOnly(t *testing.T) {
got := AttrsForEnhance(map[string]any{
"brand": "Ostalo",
"zavora": "Mehanska",
}, map[string]struct{}{})
if got["brand"] != "Ostalo" {
t.Fatalf("brand=%v", got["brand"])
}
if _, ok := got["zavora"]; ok {
t.Fatalf("empty allowlist should drop junk: %v", got)
}
}
func TestAttrsForEnhance_copiesEPRELEnergyClass(t *testing.T) {
t.Parallel()
attrs := map[string]any{
"brand": "Bosch",
"eprel_energy_class": "C",
}
ensureEnergyClassFromEPREL(attrs)
got := AttrsForEnhance(attrs, map[string]struct{}{})
if got["energy_class"] != "C" {
t.Fatalf("energy_class=%v want C (from eprel_energy_class); attrs=%v", got["energy_class"], got)
}
if _, ok := got["eprel_energy_class"]; ok {
t.Fatalf("eprel_energy_class must be stripped by AttrsForEnhance: %v", got)
}
// Do not overwrite an existing energy_class.
attrs2 := map[string]any{
"energy_class": "A",
"eprel_energy_class": "C",
}
ensureEnergyClassFromEPREL(attrs2)
if attrs2["energy_class"] != "A" {
t.Fatalf("existing energy_class overwritten: %v", attrs2["energy_class"])
}
}
func TestAttrsForPersist_tvMountDropsJunkKeepsEPREL(t *testing.T) {
attrs := map[string]any{
"name": "TV Mount",
"description": "html",
"brand": "Ostalo",
"product_model": "W53070",
"zavora": "Mehanska in elektricna",
"vzmetenje": "Spredaj in zadaj",
"nosilnost": "40 kg",
"eprel_id": "12345",
"eprel_label": "https://eprel.example/label",
}
allowed := map[string]struct{}{
"barva": {},
"nosilnost": {},
}
got := AttrsForPersist(attrs, allowed)
if _, ok := got["name"]; ok {
t.Fatalf("reserved name must be stripped: %v", got)
}
if got["brand"] != "Ostalo" || got["product_model"] != "W53070" {
t.Fatalf("core keys missing: %v", got)
}
if got["nosilnost"] != "40 kg" {
t.Fatalf("category key missing: %v", got)
}
if _, ok := got["zavora"]; ok {
t.Fatalf("zavora must not be stored: %v", got)
}
if _, ok := got["vzmetenje"]; ok {
t.Fatalf("vzmetenje must not be stored: %v", got)
}
if got["eprel_id"] != "12345" || got["eprel_label"] != "https://eprel.example/label" {
t.Fatalf("eprel keys must persist for poll projection: %v", got)
}
}
func TestRunSteps_persistsCategoryAllowlist(t *testing.T) {
e := &Engine{}
in := ProductInput{
GTIN: "8712285326882",
Name: "NOSILEC W53070",
Mapped: map[string]any{
"name": "NOSILEC W53070",
"category": "28",
"brand": "Ostalo",
"specifications": map[string]any{
"productmodel": "W53070",
"zavora": "Mehanska",
"vzmetenje": "Spredaj",
"nosilnost": "40 kg",
},
},
CategoryAttrKeys: map[string]map[string]struct{}{
"28": {"nosilnost": {}, "barva": {}},
},
}
out, err := e.RunSteps(context.Background(), "co", in, "attributes", nil, StepPolicy{})
if err != nil {
t.Fatal(err)
}
if out.ProcessedAttributes["brand"] != "Ostalo" {
t.Fatalf("brand=%v", out.ProcessedAttributes["brand"])
}
if out.ProcessedAttributes["nosilnost"] != "40 kg" {
t.Fatalf("nosilnost=%v", out.ProcessedAttributes)
}
if _, ok := out.ProcessedAttributes["zavora"]; ok {
t.Fatalf("zavora must not be stored: %v", out.ProcessedAttributes)
}
if _, ok := out.ProcessedAttributes["vzmetenje"]; ok {
t.Fatalf("vzmetenje must not be stored: %v", out.ProcessedAttributes)
}
}
func TestV1PlainDescription(t *testing.T) {
got := v1PlainDescription("Hello<br>World<br/>&amp; more</p><b>bold</b>")
if strings.Contains(got, "<") {
apps/api/internal/processing/prompt_fallback_test.go
+20 -7
View File
@@ -1,6 +1,7 @@
package processing
import (
"strings"
"testing"
"github.com/descrybe/descrybe-v2/apps/api/internal/company"
@@ -9,14 +10,18 @@ import (
func TestResolvePromptFallbackChain(t *testing.T) {
t.Parallel()
// Category override for language wins over company user template.
// Overrides without {{attrs}} get standard context placeholders injected.
sys, user := resolveProductPromptTemplates(ProductInput{
EnhanceSystemTemplate: "sys",
EnhanceUserTemplate: "company-en",
CategoryEnhancePrompt: "cat-sl",
Language: "sl",
})
if sys != "sys" || user != "cat-sl" {
t.Fatalf("sys=%q user=%q", sys, user)
if sys != "sys" {
t.Fatalf("sys=%q", sys)
}
if !strings.HasPrefix(user, "cat-sl") || !strings.Contains(user, "{{attrs}}") {
t.Fatalf("sys=%q user=%q want cat-sl + attrs", sys, user)
}
// Empty category → company template.
@@ -28,12 +33,20 @@ func TestResolvePromptFallbackChain(t *testing.T) {
t.Fatalf("user=%q", user)
}
// categoryEnhancePromptFor: lang-specific only (no cross-lang fallback).
// sl-only category prompt: exact for sl; en falls back to primary=sl.
m := map[string]company.LangPromptMap{"audio": {"sl": "slo-prompt"}}
if got := categoryEnhancePromptFor(m, "audio", "sl"); got != "slo-prompt" {
t.Fatalf("got %q", got)
if got := categoryEnhancePromptFor(m, "audio", "sl", "sl"); got != "slo-prompt" {
t.Fatalf("sl exact: got %q", got)
}
if got := categoryEnhancePromptFor(m, "audio", "en"); got != "" {
t.Fatalf("cross-lang should be empty, got %q", got)
if got := categoryEnhancePromptFor(m, "audio", "en", "sl"); got != "slo-prompt" {
t.Fatalf("en→primary sl: got %q", got)
}
if got := categoryEnhancePromptFor(m, "audio", "en", "en"); got != "" {
t.Fatalf("en with primary=en and no en/* prompt should be empty, got %q", got)
}
// wildcard shared overlay
m2 := map[string]company.LangPromptMap{"audio": {"*": "shared-overlay"}}
if got := categoryEnhancePromptFor(m2, "audio", "en", "sl"); got != "shared-overlay" {
t.Fatalf("wildcard: got %q", got)
}
}
apps/api/internal/processing/prompt_render.go
+67 -9
View File
@@ -12,21 +12,76 @@ func resolveProductPromptTemplates(in ProductInput) (systemTpl, userTpl string)
userTpl = strings.TrimSpace(in.EnhanceUserTemplate)
// Per-category prompt wins for the user message (company system keeps JSON schema / brand).
if cat := strings.TrimSpace(in.CategoryEnhancePrompt); cat != "" {
userTpl = cat
userTpl = ensureCategoryEnhanceUserContext(cat)
}
def, ok := aiprompts.DefaultFor(aiprompts.KeyProductEnhance)
if !ok {
return systemTpl, userTpl
}
if systemTpl == "" {
systemTpl = def.SystemTemplate
}
if userTpl == "" {
userTpl = def.UserTemplate
if ok {
if systemTpl == "" {
systemTpl = def.SystemTemplate
}
if userTpl == "" {
userTpl = def.UserTemplate
}
}
// Category formulas are language-agnostic; inject once into the shared user skeleton.
userTpl = AppendFormulaConstraints(userTpl, in.TitleTemplate, in.DescriptionTemplate)
return systemTpl, userTpl
}
// thinEnhanceUserInstruction is prepended when Desc is empty or ≈ Name so the model
// builds copy from Attrs+Category instead of echoing the title.
const thinEnhanceUserInstruction = "Build the description from Attrs and Category only; never copy Name."
// templateHasVar reports whether tpl contains {{name}} (via aiprompts.ExtractVariables).
func templateHasVar(tpl, name string) bool {
for _, v := range aiprompts.ExtractVariables(tpl) {
if v == name {
return true
}
}
return false
}
// ensureCategoryEnhanceUserContext appends standard product context placeholders when a
// category override omits {{attrs}} (common in DB category prompts). Does not rewrite
// category copy that already includes attrs.
func ensureCategoryEnhanceUserContext(userTpl string) string {
userTpl = strings.TrimSpace(userTpl)
if userTpl == "" || templateHasVar(userTpl, "attrs") {
return userTpl
}
var b strings.Builder
b.WriteString(userTpl)
b.WriteString("\n\n")
first := true
appendLine := func(line string) {
if !first {
b.WriteByte('\n')
}
first = false
b.WriteString(line)
}
if !templateHasVar(userTpl, "category") {
appendLine("Category: {{category}}")
}
if !templateHasVar(userTpl, "name") {
appendLine("Name: {{name}}")
}
if !templateHasVar(userTpl, "description") {
appendLine("Desc: {{description}}")
}
appendLine("Attrs: {{attrs}}")
return b.String()
}
func isThinEnhanceDescription(desc, name string) bool {
desc = strings.TrimSpace(desc)
if desc == "" || desc == "<nil>" {
return true
}
return descriptionEchoesTitle(desc, name)
}
// RenderProductEnhancePrompts fills company/built-in templates with product variables.
func RenderProductEnhancePrompts(systemTpl, userTpl, category, name, description, gtin, brandPrompt, language string, attrs map[string]any) (system, user string) {
attrsJSON := ""
@@ -49,5 +104,8 @@ func RenderProductEnhancePrompts(systemTpl, userTpl, category, name, description
// Safety net if a custom user template renders empty.
user = ProductEnhanceUser(category, name, description, attrs)
}
if isThinEnhanceDescription(description, name) {
user = thinEnhanceUserInstruction + "\n\n" + user
}
return system, user
}
apps/api/internal/processing/prompt_render_test.go
+108 -7
View File
@@ -1,8 +1,10 @@
package processing
import (
"strings"
"testing"
"github.com/descrybe/descrybe-v2/apps/api/internal/aiprompts"
"github.com/descrybe/descrybe-v2/apps/api/internal/company"
)
@@ -16,8 +18,24 @@ func TestResolveProductPromptTemplates_categoryOverridesUser(t *testing.T) {
if sys != "sys {{brand_voice}}" {
t.Fatalf("system=%q", sys)
}
if user != "category user {{description}}" {
t.Fatalf("user=%q want category override", user)
if !strings.HasPrefix(user, "category user {{description}}") {
t.Fatalf("user=%q want category override prefix", user)
}
if !strings.Contains(user, "{{attrs}}") {
t.Fatalf("user=%q want injected {{attrs}}", user)
}
if !strings.Contains(user, "{{name}}") || !strings.Contains(user, "{{category}}") {
t.Fatalf("user=%q want name/category placeholders", user)
}
}
func TestResolveProductPromptTemplates_categoryWithAttrsUnchanged(t *testing.T) {
t.Parallel()
_, user := resolveProductPromptTemplates(ProductInput{
CategoryEnhancePrompt: "Write copy.\nAttrs: {{attrs}}\nName: {{name}}",
})
if user != "Write copy.\nAttrs: {{attrs}}\nName: {{name}}" {
t.Fatalf("user=%q want unchanged when attrs present", user)
}
}
@@ -31,22 +49,105 @@ func TestResolveProductPromptTemplates_fallsBackToCompany(t *testing.T) {
}
}
func TestRenderProductEnhancePrompts_categoryOverrideGetsAttrs(t *testing.T) {
t.Parallel()
sysTpl, userTpl := resolveProductPromptTemplates(ProductInput{
CategoryEnhancePrompt: "Write a monitor listing. Focus on panel tech.",
})
_, user := RenderProductEnhancePrompts(
sysTpl, userTpl,
"Monitors", "UltraView 27", "", "", "", "en",
map[string]any{"brand": "Acme", "size": "27 inch"},
)
if !strings.Contains(user, "Attrs:") {
t.Fatalf("user missing Attrs block:\n%s", user)
}
if !strings.Contains(user, "Acme") || !strings.Contains(user, "27") {
t.Fatalf("user missing attr values:\n%s", user)
}
if !strings.Contains(user, thinEnhanceUserInstruction) {
t.Fatalf("thin input should prepend instruction:\n%s", user)
}
if !strings.Contains(user, "UltraView 27") {
t.Fatalf("user missing name:\n%s", user)
}
}
func TestCategoryEnhancePromptFor(t *testing.T) {
t.Parallel()
m := map[string]company.LangPromptMap{
"monitorji": {"sl": "prompt-a", "en": "prompt-a-en"},
"televizorji": {"sl": "prompt-b"},
}
if got := categoryEnhancePromptFor(m, " Monitorji ", "sl"); got != "prompt-a" {
if got := categoryEnhancePromptFor(m, " Monitorji ", "sl", "sl"); got != "prompt-a" {
t.Fatalf("got %q", got)
}
if got := categoryEnhancePromptFor(m, " Monitorji ", "en"); got != "prompt-a-en" {
if got := categoryEnhancePromptFor(m, " Monitorji ", "en", "sl"); got != "prompt-a-en" {
t.Fatalf("got %q", got)
}
if got := categoryEnhancePromptFor(m, "missing", "sl"); got != "" {
if got := categoryEnhancePromptFor(m, "missing", "sl", "sl"); got != "" {
t.Fatalf("expected empty, got %q", got)
}
if got := categoryEnhancePromptFor(m, "televizorji", "en"); got != "" {
t.Fatalf("expected empty fallback, got %q", got)
// sl-only category: en falls back to primary=sl
if got := categoryEnhancePromptFor(m, "televizorji", "en", "sl"); got != "prompt-b" {
t.Fatalf("en→primary sl: got %q", got)
}
if got := categoryEnhancePromptFor(m, "televizorji", "en", "en"); got != "" {
t.Fatalf("en with primary=en should stay empty, got %q", got)
}
}
func TestCategoryEnhancePromptFor_uniqueIDAndNameKeys(t *testing.T) {
t.Parallel()
overlay := company.LangPromptMap{"*": aiprompts.CategoryEnhanceUserTemplate}
byName := map[string]company.LangPromptMap{
"štedilniki": overlay,
}
byUID := map[string]company.LangPromptMap{
"50": overlay,
}
if got := categoryEnhancePromptFor(byName, "50", "sl", "sl"); got != "" {
t.Fatalf("name-only map must miss unique_id lookup, got %q", got)
}
if got := categoryEnhancePromptFor(byUID, "50", "sl", "sl"); !strings.Contains(got, "{{attrs}}") {
t.Fatalf("unique_id key should resolve overlay: %q", got)
}
// Dual-index map (as loadCategoryEnhanceOverlays builds): unique_id OR name works.
dual := map[string]company.LangPromptMap{}
for _, k := range categoryOverlayKeys("50", "Štedilniki") {
dual[k] = overlay
}
if got := categoryEnhancePromptFor(dual, "50", "en", "sl"); !strings.Contains(got, "{{category}}") {
t.Fatalf("dual unique_id lookup: %q", got)
}
if got := categoryEnhancePromptFor(dual, "Štedilniki", "en", "sl"); !strings.Contains(got, "{{attrs}}") {
t.Fatalf("dual name lookup: %q", got)
}
}
func TestCategoryOverlayKeys_indexesUIDAndName(t *testing.T) {
t.Parallel()
keys := categoryOverlayKeys("50", "Štedilniki")
want := map[string]bool{"50": true, "štedilniki": true}
if len(keys) != 2 {
t.Fatalf("keys=%v want 2", keys)
}
for _, k := range keys {
if !want[k] {
t.Fatalf("unexpected key %q in %v", k, keys)
}
}
}
func TestRenderProductEnhancePrompts_languageLabelsSLEN(t *testing.T) {
t.Parallel()
sysTpl, userTpl := resolveProductPromptTemplates(ProductInput{})
sysSL, _ := RenderProductEnhancePrompts(sysTpl, userTpl, "Cat", "Name", "Desc", "", "", "sl", nil)
if !strings.Contains(sysSL, "Slovenian") {
t.Fatalf("sl system missing Slovenian label:\n%s", sysSL)
}
sysEN, _ := RenderProductEnhancePrompts(sysTpl, userTpl, "Cat", "Name", "Desc", "", "", "en", nil)
if !strings.Contains(sysEN, "English") {
t.Fatalf("en system missing English label:\n%s", sysEN)
}
}
apps/api/internal/processing/specs.go
+101 -1
View File
@@ -13,6 +13,8 @@ var (
specsHTMLTagRe = regexp.MustCompile(`(?is)<[^>]+>`)
csvLikeRe = regexp.MustCompile(`(?m)^\s*([^:=\n\r]{1,120})\s*[:=]\s*(.+?)\s*$`)
bulletLineRe = regexp.MustCompile(`(?m)^\s*[-•*]\s*([^:=\n\r]{1,120})\s*[:=]\s*(.+?)\s*$`)
// GTIN/EAN/UPC masquerading as product_model (feed mapping mistakes).
barcodeLikeModelRe = regexp.MustCompile(`^\d{8,14}$`)
)
// Caps for locale/spec parsing — unbounded FindAll on huge CDATA can OOM the worker.
@@ -173,6 +175,73 @@ func SanitizeV1ProcessAttributesAllowed(attrs map[string]any, allowed map[string
return FilterAttributesByAllowed(sanitizeProductAttributes(attrs, true), allowed)
}
// AttrsForEnhance prepares attributes for AI enhance prompts / hashes.
// Always drops reserved and invalid keys. When allowed is non-nil (including empty),
// keeps only coreCharacteristicAttrKeys plus the allowlist (category_attributes).
// When allowed is nil, returns sanitized attrs unchanged (unit-test fallback).
func AttrsForEnhance(attrs map[string]any, allowed map[string]struct{}) map[string]any {
cleaned := sanitizeProductAttributes(attrs, true)
if allowed == nil {
return cleaned
}
cleaned = MapAttrsOntoAllowedKeys(cleaned, allowed)
return FilterAttributesByAllowed(cleaned, allowed)
}
// ensureEnergyClassFromEPREL copies eprel_energy_class → energy_class when the
// core energy_class key is empty so AttrsForEnhance (which dropEPREL strips
// eprel_*) still surfaces the label class to the model.
func ensureEnergyClassFromEPREL(attrs map[string]any) {
if attrs == nil {
return
}
if v, ok := attrs["energy_class"]; ok {
if s := strings.TrimSpace(fmt.Sprint(v)); s != "" && s != "<nil>" {
return
}
}
for _, k := range []string{"eprel_energy_class", "eprelEnergyClass"} {
if v, ok := attrs[k]; ok {
s := strings.TrimSpace(fmt.Sprint(v))
if s != "" && s != "<nil>" {
attrs["energy_class"] = s
return
}
}
}
}
// AttrsForPersist prepares attributes for DB storage (attributes / processed_attributes).
// Same category allowlist semantics as AttrsForEnhance, but uses SanitizeProductAttributes
// (keeps eprel_* for poll extractEPRELFromAttrs) then FilterAttributesByAllowed.
// Feed/spec keys are remapped onto category_attributes (formula) keys when possible so
// values under near-miss labels (velikost-zaslona → diagonala_zaslona) are kept.
// When allowed is nil, returns sanitized attrs unchanged (unit-test fallback).
func AttrsForPersist(attrs map[string]any, allowed map[string]struct{}) map[string]any {
cleaned := SanitizeProductAttributes(attrs)
if allowed == nil {
return cleaned
}
cleaned = MapAttrsOntoAllowedKeys(cleaned, allowed)
filtered := FilterAttributesByAllowed(cleaned, allowed)
// Preserve eprel_* that SanitizeProductAttributes kept (not on category allowlists).
for k, v := range cleaned {
if strings.HasPrefix(strings.ToLower(strings.TrimSpace(k)), "eprel") {
filtered[k] = v
}
}
return filtered
}
// enhanceAllowedAttrKeys prefers category_attributes for the resolved category;
// falls back to AllowedAttrKeys (nil = sanitize-only for unit tests).
func enhanceAllowedAttrKeys(in ProductInput, categoryUID string) map[string]struct{} {
if in.CategoryAttrKeys != nil {
return allowedAttrKeysFromSets(in.CategoryAttrKeys, categoryUID)
}
return in.AllowedAttrKeys
}
// FilterAttributesByAllowed keeps coreCharacteristicAttrKeys plus keys present in
// allowed (after canonicalizeAttrKey). When allowed is nil, returns attrs unchanged.
func FilterAttributesByAllowed(attrs map[string]any, allowed map[string]struct{}) map[string]any {
@@ -216,6 +285,31 @@ func sanitizeProductAttributes(attrs map[string]any, dropEPREL bool) map[string]
if dropEPREL && strings.HasPrefix(strings.ToLower(key), "eprel") {
continue
}
// Keep nested eprel object for storage / extractEPRELFromAttrs (stringify would drop maps).
if !dropEPREL && strings.EqualFold(key, "eprel") {
if m, ok := v.(map[string]any); ok && len(m) > 0 {
nested := make(map[string]any, len(m))
for nk, nv := range m {
nk = strings.TrimSpace(nk)
if nk == "" || nv == nil {
continue
}
if s, ok := nv.(string); ok {
s = strings.TrimSpace(SanitizeOutput(s))
if s == "" {
continue
}
nested[nk] = s
continue
}
nested[nk] = nv
}
if len(nested) > 0 {
out["eprel"] = nested
}
}
continue
}
s := stringifySpecValue(v)
if s == "" || s == "<nil>" {
continue
@@ -231,6 +325,10 @@ func sanitizeProductAttributes(attrs map[string]any, dropEPREL bool) map[string]
if isDimensionKey(canon) && isZeroishString(s) {
continue
}
// Drop barcode-as-model (e.g. mapped product_model = GTIN).
if canon == "product_model" && barcodeLikeModelRe.MatchString(strings.TrimSpace(s)) {
continue
}
if _, exists := out[canon]; exists && (canon != key) {
continue
}
@@ -257,6 +355,8 @@ func canonicalizeAttrKey(k string) string {
return "product_model"
case "energijskirazred", "energyclass":
return "energy_class"
case "eprelid", "eprel":
return "eprel_id"
default:
return compact
}
@@ -459,4 +559,4 @@ func attributeKeyFromLabel(label string) string {
return ""
}
return slug
}
}
apps/api/internal/processing/steps.go
+499 -110
View File
@@ -4,6 +4,7 @@ import (
"context"
"encoding/json"
"fmt"
"sort"
"strings"
"github.com/descrybe/descrybe-v2/apps/api/internal/company"
@@ -43,14 +44,15 @@ func (e *Engine) RunSteps(ctx context.Context, companyID string, in ProductInput
in.Name,
in.PriorProcessedName,
)
out.Description = preferredProductDescription(
out.Description = preferredProductDescription(out.Name,
stringFromAny(normalized["description"]),
in.Description,
in.PriorProcessedDescription,
)
out.Category = stringFromAny(normalized["category"])
// mapped_data.category wins; otherwise keep existing processed category
// so enhance_only / reprocess cannot blank A1 legacy categories.
// mapped_data.category / category_unique_id (unique_id codes) win.
applyCategoryFromMapped(&out, normalized, in.Mapped, in.Raw)
// otherwise keep existing processed category so enhance_only / reprocess
// cannot blank A1 legacy categories.
preserveCategoryIfEmpty(&out, in.PriorCategory)
out.FieldSources["normalize"] = "mapped+raw"
appendStepLog(out.GPTResponse, StepNormalize, map[string]any{
@@ -74,10 +76,13 @@ func (e *Engine) RunSteps(ctx context.Context, companyID string, in ProductInput
}
}
}
attrs = parsed
out.Attributes = SanitizeProductAttributes(attrs)
out.ProcessedAttributes = out.Attributes
attrs = out.Attributes
attrs = SanitizeProductAttributes(parsed)
// Map feed/spec labels onto category_attributes (formula) keys early.
if allowed := enhanceAllowedAttrKeys(in, out.Category); allowed != nil {
attrs = MapAttrsOntoAllowedKeys(attrs, allowed)
}
out.Attributes = attrs
out.ProcessedAttributes = attrs
out.FieldSources["attributes"] = "specifications"
appendStepLog(out.GPTResponse, StepParseSpecs, map[string]any{
"count": len(attrs),
@@ -95,12 +100,12 @@ func (e *Engine) RunSteps(ctx context.Context, companyID string, in ProductInput
in.Name,
in.PriorProcessedName,
)
out.Description = preferredProductDescription(
out.Description = preferredProductDescription(out.Name,
stringFromAny(normalized["description"]),
out.Description,
in.Description,
)
out.Category = stringFromAny(normalized["category"])
applyCategoryFromMapped(&out, normalized, in.Mapped, in.Raw)
// Promote characteristic fields only — never core product identity/content
// (those belong on the V1 item root: title, description, ean, images, …).
promote := []string{"brand", "width", "height", "depth", "weight", "product_model", "warranty"}
@@ -125,31 +130,15 @@ func (e *Engine) RunSteps(ctx context.Context, companyID string, in ProductInput
}
}
attrs = SanitizeProductAttributes(attrs)
// Remap again after category may have resolved in applyCategoryFromMapped.
if allowed := enhanceAllowedAttrKeys(in, out.Category); allowed != nil {
attrs = MapAttrsOntoAllowedKeys(attrs, allowed)
}
out.Attributes = attrs
out.ProcessedAttributes = attrs
appendStepLog(out.GPTResponse, StepFillFields, map[string]any{
"brand": stringFromAny(normalized["brand"]),
})
if out.Category == "" && e != nil && e.Vector != nil && e.Vector.Enabled() {
text := strings.TrimSpace(out.Name + " " + out.Description)
if text != "" {
if cat, err := e.Vector.SuggestCategory(ctx, companyID, text, categoryNames); err == nil && strings.TrimSpace(cat) != "" {
out.Category = SanitizeOutput(cat)
out.FieldSources["category"] = "vector"
out.Notes = append(out.Notes, "category: vector")
appendStepLog(out.GPTResponse, "vector_categorize", map[string]any{
"status": "ok",
"category": out.Category,
})
} else if err != nil {
out.Notes = append(out.Notes, "vector_categorize: "+TruncateError(err))
appendStepLog(out.GPTResponse, "vector_categorize", map[string]any{
"status": "failed",
"error": TruncateError(err),
})
}
}
}
preserveCategoryIfEmpty(&out, in.PriorCategory)
case StepEPREL:
@@ -161,7 +150,8 @@ func (e *Engine) RunSteps(ctx context.Context, companyID string, in ProductInput
})
break
}
id := eprel.ExtractID(normalized, in.Mapped, in.Raw)
// Also scan parsed attrs — eprel_id may only appear after parse_specs.
id := eprel.ExtractID(normalized, in.Mapped, in.Raw, attrs)
if id == "" {
out.Notes = append(out.Notes, "eprel: no id")
appendStepLog(out.GPTResponse, StepEPREL, map[string]any{"status": "skipped", "reason": "no_id"})
@@ -173,7 +163,7 @@ func (e *Engine) RunSteps(ctx context.Context, companyID string, in ProductInput
}
if !enricher.Enabled() {
out.Notes = append(out.Notes, "eprel: enricher disabled")
out.EPREL = map[string]any{"eprel_id": id, "status": "skipped"}
out.EPREL = map[string]any{"id": id, "status": "skipped"}
appendStepLog(out.GPTResponse, StepEPREL, map[string]any{"status": "skipped", "eprel_id": id, "reason": "disabled"})
break
}
@@ -191,15 +181,22 @@ func (e *Engine) RunSteps(ctx context.Context, companyID string, in ProductInput
attrs["eprel_id"] = id
out.Attributes = attrs
out.ProcessedAttributes = attrs
out.EPREL = map[string]any{"eprel_id": id, "status": "empty"}
out.EPREL = map[string]any{"id": id, "status": "empty"}
appendStepLog(out.GPTResponse, StepEPREL, map[string]any{"status": "empty", "eprel_id": id})
break
}
attrs = eprel.MergeInto(attrs, data)
// Promote energy class onto the characteristic attr key when missing
// (feeds often omit it; EPREL API is the source of truth).
if data.EnergyClass != "" {
if cur := strings.TrimSpace(fmt.Sprint(attrs["energy_class"])); cur == "" || cur == "<nil>" {
attrs["energy_class"] = data.EnergyClass
}
}
out.Attributes = attrs
out.ProcessedAttributes = attrs
out.EPREL = map[string]any{
"eprel_id": data.ID,
"id": data.ID,
"label": data.Label,
"pdf": data.PDF,
"energy_class": data.EnergyClass,
@@ -247,6 +244,10 @@ func (e *Engine) RunSteps(ctx context.Context, companyID string, in ProductInput
if primary == "" {
primary = langs[0]
}
syncCategoryName(&out, in.CategoryNamesByUID)
displayCat := categoryDisplayLabel(out)
ensureEnergyClassFromEPREL(attrs)
enhanceAttrs := AttrsForEnhance(attrs, enhanceAllowedAttrKeys(in, out.Category))
localized := company.LocalizedContent{}
if in.PriorLocalized != nil {
for k, v := range in.PriorLocalized {
@@ -264,8 +265,9 @@ func (e *Engine) RunSteps(ctx context.Context, companyID string, in ProductInput
}
catPrompt := in.CategoryEnhancePrompt
if lang != primary || catPrompt == "" {
catPrompt = categoryEnhancePromptFor(in.CategoryPromptsByLang, out.Category, lang)
catPrompt = categoryEnhancePromptFor(in.CategoryPromptsByLang, out.Category, lang, primary)
}
titleTpl, descTpl := categoryFormulasFor(in, out.Category)
priorFields := company.FieldsForLanguage(in.PriorLocalized, lang)
priorHash := priorFields.EnhanceInputHash
priorName := priorFields.ProcessedName
@@ -291,10 +293,12 @@ func (e *Engine) RunSteps(ctx context.Context, companyID string, in ProductInput
EnhanceSystemTemplate: tpl.System,
EnhanceUserTemplate: tpl.User,
CategoryEnhancePrompt: catPrompt,
TitleTemplate: titleTpl,
DescriptionTemplate: descTpl,
PriorEnhanceHash: priorHash,
PriorProcessedName: priorName,
PriorProcessedDescription: priorDesc,
}, out.Category, attrs)
}, displayCat, enhanceAttrs)
out.TotalTokens += tokens
status := enhanceStatusFromMeta(raw)
meta := map[string]any{"language": lang, "raw": raw}
@@ -319,29 +323,59 @@ func (e *Engine) RunSteps(ctx context.Context, companyID string, in ProductInput
anyOK = true
meta["status"] = status
}
hash := enhanceHashFromMeta(raw)
// Prefer title-aware selection + synthesize before deciding hash persistence.
name = preferredProductTitle(in.GTIN, name, out.Name, priorName, in.Name)
desc = preferredProductDescription(name, desc, out.Description, priorDesc)
if isWeakPriorEnhanceDescription(desc, name) {
if synth := synthesizeDescriptionFromTitle(name, displayCat, lang, enhanceAttrs); synth != "" {
desc = synth
}
}
weakDesc := isWeakPriorEnhanceDescription(desc, name)
// Only persist enhance_input_hash for quality ok / hash-skip unchanged.
// Never copy input_hash from error/passthrough/thin meta into localized.
persistHash := ""
rawHash := enhanceHashFromMeta(raw)
switch {
case err != nil:
persistHash = ""
case status == "unchanged" && !weakDesc:
persistHash = rawHash
case status == "ok" && !weakDesc:
persistHash = rawHash
default:
// thin ok / parse_failed / skipped — do not poison reprocess skip
persistHash = ""
if status == "ok" && weakDesc {
allUnchanged = false
}
}
localized[lang] = company.LocalizedFields{
ProcessedName: name,
ProcessedDescription: desc,
EnhanceInputHash: hash,
EnhanceInputHash: persistHash,
MetaTitle: company.FieldsForLanguage(localized, lang).MetaTitle,
MetaDescription: company.FieldsForLanguage(localized, lang).MetaDescription,
}
// Preserve existing meta when re-enhancing titles only.
// Never keep a bare "| <unique_id>" poisoned meta_title.
// When dropping poisoned title, also drop empty/weak stub meta_description.
if prev := company.FieldsForLanguage(in.PriorLocalized, lang); prev.MetaTitle != "" || prev.MetaDescription != "" {
f := localized[lang]
if f.MetaTitle == "" {
poisonedTitle := isPoisonedMetaTitle(prev.MetaTitle)
if f.MetaTitle == "" && !poisonedTitle {
f.MetaTitle = prev.MetaTitle
}
if f.MetaDescription == "" {
f.MetaDescription = prev.MetaDescription
weakStub := poisonedTitle && isWeakPriorEnhanceDescription(prev.MetaDescription, name, priorName)
if !weakStub {
f.MetaDescription = prev.MetaDescription
}
}
localized[lang] = f
}
langMetas = append(langMetas, meta)
if lang == primary {
name = preferredProductTitle(in.GTIN, name, out.Name, in.PriorProcessedName)
desc = preferredProductDescription(desc, out.Description, in.PriorProcessedDescription)
out.ProcessedName = name
out.ProcessedDescription = desc
if name != "" {
@@ -350,10 +384,10 @@ func (e *Engine) RunSteps(ctx context.Context, companyID string, in ProductInput
if desc != "" {
out.Description = desc
}
if hash != "" && (status == "ok" || status == "unchanged") {
out.FieldSources[FieldEnhanceInputHash] = hash
} else if err != nil {
preservePriorEnhanceHash()
if persistHash != "" {
out.FieldSources[FieldEnhanceInputHash] = persistHash
} else {
delete(out.FieldSources, FieldEnhanceInputHash)
}
}
}
@@ -365,7 +399,34 @@ func (e *Engine) RunSteps(ctx context.Context, companyID string, in ProductInput
if out.ProcessedDescription == "" {
out.ProcessedDescription = out.Description
}
preservePriorEnhanceHash()
// Timeout/error/empty: always synthesize a factual fallback when a title exists.
if out.ProcessedName != "" && (out.ProcessedDescription == "" ||
isWeakPriorEnhanceDescription(out.ProcessedDescription, out.ProcessedName, out.Name)) {
if synth := synthesizeDescriptionFromTitle(out.ProcessedName, displayCat, primary, enhanceAttrs); synth != "" {
out.ProcessedDescription = synth
if out.Description == "" || isWeakPriorEnhanceDescription(out.Description, out.Name, out.ProcessedName) {
out.Description = synth
}
if lf, ok := localized[primary]; ok {
lf.ProcessedDescription = synth
if lf.ProcessedName == "" {
lf.ProcessedName = out.ProcessedName
}
localized[primary] = lf
}
}
}
// Attempted enhance (even on timeout) — record provider mode, not misleading unknown.
out.AIProviderMode = e.EngineProviderMode()
out.FieldSources["name"] = "ai_enhance_failed"
out.FieldSources["description"] = "ai_enhance_failed"
// Failed enhance must not leave a skippable hash behind.
delete(out.FieldSources, FieldEnhanceInputHash)
for lang, lf := range localized {
lf.EnhanceInputHash = ""
localized[lang] = lf
}
out.LocalizedContent = localized
errNote := ""
for _, m := range langMetas {
if mm, ok := m.(map[string]any); ok {
@@ -402,31 +463,50 @@ func (e *Engine) RunSteps(ctx context.Context, companyID string, in ProductInput
}
}
// Paths without fill_fields (enhance_only / normalize_only) still get vector
// categorize when AllowAI + embeddings are available.
tryVectorCategorize(ctx, e, companyID, &out, categoryNames, policy)
preserveCategoryIfEmpty(&out, in.PriorCategory)
noteMissingCategory(&out, policy, e != nil && e.Vector != nil && e.Vector.Enabled())
syncCategoryName(&out, in.CategoryNamesByUID)
displayCat := categoryDisplayLabel(out)
out.Name = preferredProductTitle(in.GTIN, out.Name, out.ProcessedName, in.Name, in.PriorProcessedName)
out.ProcessedName = preferredProductTitle(in.GTIN, out.ProcessedName, out.Name, in.PriorProcessedName, in.Name)
out.Description = preferredProductDescription(out.Description, out.ProcessedDescription, in.Description)
out.ProcessedDescription = preferredProductDescription(out.ProcessedDescription, out.Description, in.PriorProcessedDescription)
out.Description = preferredProductDescription(out.Name, out.Description, out.ProcessedDescription, in.Description)
out.ProcessedDescription = preferredProductDescription(out.ProcessedName, out.ProcessedDescription, out.Description, in.PriorProcessedDescription)
if out.ProcessedName == "" {
out.ProcessedName = out.Name
}
if out.ProcessedDescription == "" {
out.ProcessedDescription = out.Description
if out.ProcessedDescription == "" || isWeakPriorEnhanceDescription(out.ProcessedDescription, out.ProcessedName, out.Name) {
if synth := synthesizeDescriptionFromTitle(out.ProcessedName, displayCat, in.Language, out.Attributes); synth != "" {
out.ProcessedDescription = synth
}
}
if out.Description == "" || isWeakPriorEnhanceDescription(out.Description, out.Name, out.ProcessedName) {
if out.ProcessedDescription != "" && !isWeakPriorEnhanceDescription(out.ProcessedDescription, out.Name, out.ProcessedName) {
out.Description = out.ProcessedDescription
} else if synth := synthesizeDescriptionFromTitle(out.Name, displayCat, in.Language, out.Attributes); synth != "" {
out.Description = synth
if out.ProcessedDescription == "" || isWeakPriorEnhanceDescription(out.ProcessedDescription, out.ProcessedName, out.Name) {
out.ProcessedDescription = synth
}
}
}
if out.Attributes == nil {
out.Attributes = map[string]any{}
}
out.Attributes = SanitizeProductAttributes(out.Attributes)
out.ProcessedAttributes = SanitizeProductAttributes(out.ProcessedAttributes)
// Persist the same sanitize + category allowlist used by enhance (poll already
// projects clean attrs; DB must not keep feed junk like zavora/vzmetenje).
allowed := enhanceAllowedAttrKeys(in, out.Category)
out.Attributes = AttrsForPersist(out.Attributes, allowed)
out.ProcessedAttributes = AttrsForPersist(out.ProcessedAttributes, allowed)
if len(out.ProcessedAttributes) == 0 {
out.ProcessedAttributes = out.Attributes
}
if out.AIProviderMode == "" {
if out.TotalTokens > 0 {
out.AIProviderMode = e.EngineProviderMode()
} else {
out.AIProviderMode = AIProviderUnknown
}
// Prefer EngineProviderMode (job/completer label) over stamping "unknown" on
// 0-token paths (hash-skip / AI skipped). processOne also treats unknown as empty.
if isUnknownProviderMode(out.AIProviderMode) {
out.AIProviderMode = preferKnownProviderMode(e.EngineProviderMode(), out.AIProviderMode)
}
if len(out.Notes) > 0 {
out.GPTResponse["notes"] = out.Notes
@@ -451,6 +531,66 @@ func preserveCategoryIfEmpty(out *StepResult, prior string) {
out.FieldSources["category"] = "prior_processed"
}
// tryVectorCategorize sets Category from embeddings when mapped unique_id is absent.
// Requires policy.AllowAI and a configured/enabled VectorCategorizer.
func tryVectorCategorize(ctx context.Context, e *Engine, companyID string, out *StepResult, categoryNames []string, policy StepPolicy) {
if out == nil || strings.TrimSpace(out.Category) != "" {
return
}
if !policy.AllowAI {
return
}
if e == nil || e.Vector == nil || !e.Vector.Enabled() {
return
}
text := strings.TrimSpace(out.Name + " " + out.Description)
if text == "" {
return
}
cat, err := e.Vector.SuggestCategory(ctx, companyID, text, categoryNames)
if err != nil {
out.Notes = append(out.Notes, "vector_categorize: "+TruncateError(err))
appendStepLog(out.GPTResponse, "vector_categorize", map[string]any{
"status": "failed",
"error": TruncateError(err),
})
return
}
if strings.TrimSpace(cat) == "" {
return
}
out.Category = SanitizeOutput(cat)
if out.FieldSources == nil {
out.FieldSources = map[string]any{}
}
out.FieldSources["category"] = "vector"
out.Notes = append(out.Notes, "category: vector")
appendStepLog(out.GPTResponse, "vector_categorize", map[string]any{
"status": "ok",
"category": out.Category,
})
}
// noteMissingCategory records why Category stayed empty (mapped absent; vector skipped or failed).
func noteMissingCategory(out *StepResult, policy StepPolicy, vectorEnabled bool) {
if out == nil || strings.TrimSpace(out.Category) != "" {
return
}
for _, n := range out.Notes {
if strings.HasPrefix(n, "category: unset") || strings.HasPrefix(n, "category: vector") {
return
}
}
switch {
case !policy.AllowAI:
out.Notes = append(out.Notes, "category: unset (no mapped unique_id; AI/vector not allowed)")
case !vectorEnabled:
out.Notes = append(out.Notes, "category: unset (no mapped unique_id; vector embeddings unavailable)")
default:
out.Notes = append(out.Notes, "category: unset (no mapped unique_id; vector did not match)")
}
}
func resolveSteps(processingType string) []string {
switch strings.ToLower(strings.TrimSpace(processingType)) {
case "enhance", "enhance_only", "enhance-only", "title", "description":
@@ -458,7 +598,8 @@ func resolveSteps(processingType string) []string {
case "attributes", "attributes_only", "specs", "specifications":
return []string{StepNormalize, StepParseSpecs, StepFillFields}
case "eprel", "eprel_only":
return []string{StepNormalize, StepEPREL}
// parse_specs first so eprel_id buried in specifications/attributes is visible.
return []string{StepNormalize, StepParseSpecs, StepEPREL}
case "normalize_only":
return []string{StepNormalize}
case "categorize", "categorize_only", "categorize_enhance":
@@ -488,15 +629,19 @@ func (e *Engine) enhance(ctx context.Context, in ProductInput, category string,
sysTpl, userTpl := resolveProductPromptTemplates(in)
hash := HashEnhanceInput(category, in.Name, in.Description, in.BrandPrompt, in.Language, sysTpl, userTpl, attrs)
if e == nil || e.Completer == nil {
return preferredProductTitle(in.GTIN, in.Name, in.PriorProcessedName),
preferredProductDescription(in.Description, in.PriorProcessedDescription),
name := preferredProductTitle(in.GTIN, in.Name, in.PriorProcessedName)
return name,
preferredProductDescription(name, in.Description, in.PriorProcessedDescription),
0, map[string]any{"status": "skipped", "input_hash": hash}, nil
}
// Skip LLM when inputs match the last successful enhance (before any credit debit).
// Skip LLM when inputs match the last successful enhance (before any credit debit),
// but never reuse a thin / title-echo prior description, or a prompt-leakage title.
if in.PriorEnhanceHash != "" && in.PriorEnhanceHash == hash &&
(in.PriorProcessedName != "" || in.PriorProcessedDescription != "") {
(in.PriorProcessedName != "" || in.PriorProcessedDescription != "") &&
!isPromptLabelTitle(in.PriorProcessedName) &&
!isWeakPriorEnhanceDescription(in.PriorProcessedDescription, in.PriorProcessedName, in.Name) {
name := preferredProductTitle(in.GTIN, in.PriorProcessedName, in.Name)
desc := preferredProductDescription(in.PriorProcessedDescription, in.Description)
desc := preferredProductDescription(name, in.PriorProcessedDescription, in.Description)
return name, desc, 0, map[string]any{
"status": "unchanged",
"input_hash": hash,
@@ -509,32 +654,43 @@ func (e *Engine) enhance(ctx context.Context, in ProductInput, category string,
})
if err != nil {
// Network/provider failure vs parse failure after retry
if obj == nil && comp.Text == "" {
return preferredProductTitle(in.GTIN, in.Name, in.PriorProcessedName),
preferredProductDescription(in.Description, in.PriorProcessedDescription),
0, map[string]any{
"provider": "passthrough",
"error": TruncateError(err),
"input_hash": hash,
}, err
name := preferredProductTitle(in.GTIN, in.Name, in.PriorProcessedName)
desc := preferredProductDescription(name, in.Description, in.PriorProcessedDescription)
if isWeakPriorEnhanceDescription(desc, name) {
if synth := synthesizeDescriptionFromTitle(name, category, in.Language, attrs); synth != "" {
desc = synth
}
}
// Parse failed after retry — keep original copy (avoid garbage titles)
return preferredProductTitle(in.GTIN, in.Name, in.PriorProcessedName),
preferredProductDescription(in.Description, in.PriorProcessedDescription),
comp.TotalTokens, map[string]any{
"status": "parse_failed",
"error": "AI returned invalid JSON; kept original title/description",
"raw": truncateRunes(comp.Text, 200),
"input_hash": hash,
}, nil
if obj == nil && comp.Text == "" {
return name, desc, 0, map[string]any{
"provider": "passthrough",
"error": TruncateError(err),
}, err
}
// Parse failed after retry — keep usable copy; synthesize when empty/weak.
return name, desc, comp.TotalTokens, map[string]any{
"status": "parse_failed",
"error": "AI returned invalid JSON; kept original title/description",
"raw": truncateRunes(comp.Text, 200),
}, nil
}
name := preferredProductTitle(in.GTIN, SanitizeOutput(fmt.Sprint(obj["name"])), in.Name, in.PriorProcessedName)
desc := preferredProductDescription(SanitizeOutput(fmt.Sprint(obj["description"])), in.Description, in.PriorProcessedDescription)
return name, desc, comp.TotalTokens, map[string]any{
"status": "ok",
"input_hash": hash,
"raw": comp.Raw,
}, nil
desc := preferredProductDescription(name, SanitizeOutput(fmt.Sprint(obj["description"])), in.Description, in.PriorProcessedDescription)
if isWeakPriorEnhanceDescription(desc, name) {
if synth := synthesizeDescriptionFromTitle(name, category, in.Language, attrs); synth != "" {
desc = synth
}
}
meta := map[string]any{
"status": "ok",
"raw": comp.Raw,
}
// Only attach input_hash when output description is quality-worthy; thin
// title-echo / heuristic filler must not poison field_sources / localized skip hashes.
if !isWeakPriorEnhanceDescription(desc, name) {
meta["input_hash"] = hash
}
return name, desc, comp.TotalTokens, meta, nil
}
func sanitizeJSON(v any) string {
@@ -556,46 +712,67 @@ func firstLine(s string) string {
return SanitizeOutput(strings.Trim(s, "\"'` "))
}
// labeledPromptValue returns the first line after any of the given labels
// labeledPromptValue returns the first usable line after any of the given labels
// (case-insensitive), e.g. "Name:" / "Desc:" from ProductEnhanceUser.
// Skips matches whose value is prompt-label / formula-leakage text so instruction
// bullets like "- name: short retail title; follow any Title formula…" do not
// win over the later "Name: <product>" line in CategoryEnhanceUserTemplate.
func labeledPromptValue(user string, labels ...string) string {
lower := strings.ToLower(user)
bestAt := -1
bestLabel := ""
type hit struct {
at int
label string
}
var hits []hit
for _, label := range labels {
label = strings.ToLower(strings.TrimSpace(label))
if label == "" {
continue
}
at := strings.Index(lower, label)
if at < 0 {
continue
}
if bestAt < 0 || at < bestAt {
bestAt = at
bestLabel = label
searchFrom := 0
for {
rel := strings.Index(lower[searchFrom:], label)
if rel < 0 {
break
}
at := searchFrom + rel
hits = append(hits, hit{at: at, label: label})
searchFrom = at + len(label)
}
}
if bestAt < 0 {
if len(hits) == 0 {
return ""
}
rest := user[bestAt+len(bestLabel):]
if j := strings.Index(strings.ToLower(rest), "attrs:"); j >= 0 {
rest = rest[:j]
sort.Slice(hits, func(i, j int) bool { return hits[i].at < hits[j].at })
for _, h := range hits {
rest := user[h.at+len(h.label):]
if j := strings.Index(strings.ToLower(rest), "attrs:"); j >= 0 {
rest = rest[:j]
}
if j := strings.Index(strings.ToLower(rest), "attributes:"); j >= 0 {
rest = rest[:j]
}
val := firstLine(rest)
if val == "" || isPromptLabelTitle(val) {
continue
}
return val
}
if j := strings.Index(strings.ToLower(rest), "attributes:"); j >= 0 {
rest = rest[:j]
}
return firstLine(rest)
return ""
}
// isPromptLabelTitle detects enhance pollution where the model echoed a
// prompt header ("Category:" / "Category: 120") as the product title.
// prompt header ("Category:" / "Category: 120") as the product title, or
// leaked instruction scaffolding from CategoryEnhanceUserTemplate /
// AppendFormulaConstraints into name.
func isPromptLabelTitle(s string) bool {
s = strings.TrimSpace(s)
if s == "" || s == "<nil>" {
return false
}
if isPromptLeakageTitle(s) {
return true
}
lower := strings.ToLower(s)
for _, label := range []string{
"category", "name", "desc", "description",
@@ -611,8 +788,61 @@ func isPromptLabelTitle(s string) bool {
return false
}
// preferredProductTitle picks the first usable title, skipping empty values and
// prompt-label echoes like "Category:" (seen on A1 Elkotex reprocess).
// isPromptLeakageTitle detects when an LLM echoed enhance-prompt instructions
// (Title formula / short retail title / Schema / Reply with ONLY JSON / …)
// as the product name instead of a real title.
func isPromptLeakageTitle(s string) bool {
s = strings.TrimSpace(s)
if s == "" || s == "<nil>" {
return false
}
lower := strings.ToLower(s)
for _, phrase := range promptLeakagePhrases {
if strings.Contains(lower, phrase) {
return true
}
}
// Long dumps of the enhance template: any formula/schema keyword is enough.
if len([]rune(s)) > 120 {
for _, kw := range promptLeakageLongKeywords {
if strings.Contains(lower, kw) {
return true
}
}
}
return false
}
// Phrases copied from aiprompts.CategoryEnhanceUserTemplate, BuiltInDefaults,
// and processing.AppendFormulaConstraints / FormatTitleFormulaConstraint.
var promptLeakagePhrases = []string{
"title formula",
"follow any",
"constraints that follow",
"short retail title",
"use attrs",
"write name in",
"schema:",
"reply with only json",
"your reply is parsed as json",
"description formula",
"build name from attrs",
"prefer attrs values",
"order matters; join with",
"do not hardcode a language",
}
var promptLeakageLongKeywords = []string{
"formula",
"constraints",
"schema",
"json",
"attrs",
"retail title",
}
// preferredProductTitle picks the first usable title, skipping empty values,
// prompt-label echoes like "Category:", and instruction-text leakage.
func preferredProductTitle(gtin string, candidates ...string) string {
for _, c := range candidates {
c = strings.TrimSpace(c)
@@ -627,13 +857,172 @@ func preferredProductTitle(gtin string, candidates ...string) string {
return "Product"
}
func preferredProductDescription(candidates ...string) string {
// Thin wrappers keep processing call sites stable; logic lives in company so
// catalog.RepairWeakEnhanceHashes can reuse it without an import cycle.
func isWeakPriorEnhanceDescription(priorDesc string, titles ...string) bool {
return company.IsWeakPriorEnhanceDescription(priorDesc, titles...)
}
func containsWeakFillerPhrase(desc string) bool {
return company.ContainsWeakFillerPhrase(desc)
}
func descriptionEchoesTitle(desc, title string) bool {
return company.DescriptionEchoesTitle(desc, title)
}
// preferredProductDescription picks the first usable description, skipping empty
// values, prompt-label echoes, weak filler phrases, and copy that merely repeats
// the product title.
func preferredProductDescription(title string, candidates ...string) string {
for _, c := range candidates {
c = strings.TrimSpace(c)
if c == "" || c == "<nil>" || isPromptLabelTitle(c) {
continue
}
if descriptionEchoesTitle(c, title) {
continue
}
if containsWeakFillerPhrase(c) {
continue
}
return SanitizeOutput(c)
}
return ""
}
func attrLookupCI(attrs map[string]any, keys ...string) string {
if attrs == nil {
return ""
}
for _, want := range keys {
want = strings.TrimSpace(want)
if want == "" {
continue
}
if v := strings.TrimSpace(stringFromAny(attrs[want])); v != "" {
return v
}
for k, raw := range attrs {
if strings.EqualFold(strings.TrimSpace(k), want) {
if v := strings.TrimSpace(stringFromAny(raw)); v != "" {
return v
}
}
}
}
return ""
}
func formatAttrDimParts(attrs map[string]any, maxParts int) []string {
if attrs == nil || maxParts <= 0 {
return nil
}
prefer := []string{
"width", "height", "depth", "weight", "max_load", "max load", "load_capacity",
"vesa", "screen_size", "diagonal", "color", "material", "size",
}
parts := make([]string, 0, maxParts)
seen := map[string]struct{}{}
add := func(k, v string) {
k = strings.TrimSpace(k)
v = strings.TrimSpace(v)
if k == "" || v == "" {
return
}
lk := strings.ToLower(k)
if _, ok := seen[lk]; ok {
return
}
seen[lk] = struct{}{}
parts = append(parts, fmt.Sprintf("%s %s", k, v))
}
for _, k := range prefer {
if len(parts) >= maxParts {
break
}
if v := attrLookupCI(attrs, k); v != "" {
add(k, v)
}
}
return parts
}
// synthesizeDescriptionFromTitle builds a short factual fallback when enhance
// returns empty/title-echo/filler copy. Uses title, category, brand, model, and
// key dims. language is a content-language code (en/sl/…) or English label.
func synthesizeDescriptionFromTitle(title, category, language string, attrs map[string]any) string {
title = strings.TrimSpace(title)
if title == "" || title == "<nil>" || isPromptLabelTitle(title) {
return ""
}
cat := strings.TrimSpace(category)
if strings.EqualFold(cat, "general") {
cat = ""
}
brand := attrLookupCI(attrs, "brand")
model := attrLookupCI(attrs, "product_model", "model", "sku")
dims := formatAttrDimParts(attrs, 3)
sl := isSlovenianContentLanguage(language)
var b strings.Builder
if sl {
b.WriteString(title)
switch {
case cat != "" && brand != "":
fmt.Fprintf(&b, " je izdelek v kategoriji %s znamke %s", cat, brand)
case cat != "":
fmt.Fprintf(&b, " je izdelek v kategoriji %s", cat)
case brand != "":
fmt.Fprintf(&b, " je izdelek znamke %s", brand)
default:
b.WriteString(" je katalogski izdelek z znanimi atributi")
}
if model != "" && !strings.Contains(strings.ToLower(title), strings.ToLower(model)) {
fmt.Fprintf(&b, " (model %s)", model)
}
if len(dims) > 0 {
fmt.Fprintf(&b, ". Ključne specifikacije: %s", strings.Join(dims, ", "))
}
b.WriteByte('.')
} else {
b.WriteString(title)
switch {
case cat != "" && brand != "":
fmt.Fprintf(&b, " is a %s product from %s", cat, brand)
case cat != "":
fmt.Fprintf(&b, " is listed in the %s category", cat)
case brand != "":
fmt.Fprintf(&b, " is a product from %s", brand)
default:
b.WriteString(" is a catalog product with the known attributes")
}
if model != "" && !strings.Contains(strings.ToLower(title), strings.ToLower(model)) {
fmt.Fprintf(&b, " (model %s)", model)
}
if len(dims) > 0 {
fmt.Fprintf(&b, ". Key specs: %s", strings.Join(dims, ", "))
}
b.WriteByte('.')
}
out := SanitizeOutput(b.String())
// Never emit sole retail-filler when title/attrs exist — strip legacy phrase if any helper reintroduces it.
if containsWeakFillerPhrase(out) {
out = strings.TrimSpace(strings.ReplaceAll(out, "Ready for retail listing.", ""))
out = strings.TrimSpace(strings.ReplaceAll(out, "ready for retail listing.", ""))
out = strings.TrimSpace(strings.Trim(out, ".")) + "."
}
return out
}
func isSlovenianContentLanguage(raw string) bool {
raw = strings.TrimSpace(raw)
if raw == "" {
return false
}
if code, err := company.ParseLanguage(raw, false); err == nil && code == "sl" {
return true
}
lower := strings.ToLower(raw)
return lower == "slovenian" || strings.Contains(lower, "slovenian")
}
apps/api/internal/processing/steps_test.go
+107 -1
View File
@@ -22,7 +22,7 @@ func TestResolveSteps(t *testing.T) {
"full": {StepNormalize, StepParseSpecs, StepFillFields, StepEPREL, StepAIEnhance},
"enhance_only": {StepNormalize, StepAIEnhance},
"attributes_only": {StepNormalize, StepParseSpecs, StepFillFields},
"eprel_only": {StepNormalize, StepEPREL},
"eprel_only": {StepNormalize, StepParseSpecs, StepEPREL},
"normalize_only": {StepNormalize},
}
for in, want := range cases {
@@ -187,6 +187,18 @@ func TestPreferredProductTitle_skipsPromptLabels(t *testing.T) {
}
}
func TestPreferredProductTitle_skipsFormulaLeakage(t *testing.T) {
leak := "short retail title; follow any Title formula constraints that follow; use Attrs"
got := preferredProductTitle("1", leak, "follow any Title formula constraints that follow; use Attrs", "Vox SWA-8000W")
if got != "Vox SWA-8000W" {
t.Fatalf("got %q want feed title", got)
}
got = preferredProductTitle("", leak, "Schema: {\"name\":\"string\"}", "Reply with ONLY JSON")
if got != "Product" {
t.Fatalf("all-leakage fallback got %q", got)
}
}
func TestIsPromptLabelTitle(t *testing.T) {
cases := map[string]bool{
"Category:": true,
@@ -204,6 +216,100 @@ func TestIsPromptLabelTitle(t *testing.T) {
}
}
func TestIsPromptLeakageTitle(t *testing.T) {
cases := map[string]bool{
"follow any Title formula constraints that follow; use Attrs": true,
"short retail title; follow any Title formula constraints…": true,
"write name in {{language}}": true,
"Schema: {\"name\":\"string\",\"description\":\"string\"}": true,
"Reply with ONLY JSON (no markdown)": true,
"Vox SWA-8000W dishwasher": false,
"Acme Widget Pro": false,
"": false,
}
for in, want := range cases {
if got := isPromptLeakageTitle(in); got != want {
t.Fatalf("%q: leakage got %v want %v", in, got, want)
}
if want && !isPromptLabelTitle(in) {
t.Fatalf("%q: isPromptLabelTitle should include leakage", in)
}
}
// Long dump with formula keyword (>120 runes).
long := strings.Repeat("x", 100) + " Title formula scaffolding " + strings.Repeat("y", 30)
if !isPromptLeakageTitle(long) {
t.Fatalf("long formula dump should leak: len=%d", len([]rune(long)))
}
}
func TestEnhance_rejectsFormulaLeakageTitle(t *testing.T) {
leak := "short retail title; follow any Title formula constraints that follow; use Attrs"
e := &Engine{
Completer: stubCompleter{fn: func(system, user string) (Completion, error) {
return Completion{Text: `{"name":"` + leak + `","description":"A solid washer."}`, TotalTokens: 2}, nil
}},
Vector: NoopVectorCategorizer{},
}
out, err := e.RunSteps(context.Background(), "co", ProductInput{
GTIN: "8712285326882",
Name: "Vox SWA-8000W",
Mapped: map[string]any{"name": "Vox SWA-8000W", "description": "Washer", "category": "demo-electronics"},
}, "enhance_only", nil, StepPolicy{AllowAI: true})
if err != nil {
t.Fatal(err)
}
if out.ProcessedName != "Vox SWA-8000W" {
t.Fatalf("ProcessedName=%q want Vox SWA-8000W (reject formula leakage)", out.ProcessedName)
}
if isPromptLeakageTitle(out.Name) || isPromptLabelTitle(out.Name) {
t.Fatalf("Name polluted: %q", out.Name)
}
}
func TestHeuristicCompleter_keepsNameWhenFormulaConstraintsPresent(t *testing.T) {
h := HeuristicCompleter{}
// Shape matches aiprompts.CategoryEnhanceUserTemplate: instruction "- name:"
// appears BEFORE the real "Name:" field — labeledPromptValue must skip leakage.
user := `Your reply is parsed as JSON {"name":"string","description":"string"} only (system schema). Write name and description in English (do not hardcode a language).
- name: short retail title; follow any Title formula constraints that follow; use Attrs
- description: prefer 1-3 factual paragraphs as ONE string
Category: demo-electronics
Name: Vox SWA-8000W
Desc: Washer
Attrs: {"brand":"Vox"}
Title formula (order matters; join with " "). Build name from Attrs using this structure:
1. attr [brand]
Prefer Attrs values for [attr] slots; write name in English.`
comp, err := h.Complete(context.Background(), `Return JSON with "name" and "description".`, user)
if err != nil {
t.Fatal(err)
}
obj, err := ParseJSONObject(comp.Text)
if err != nil {
t.Fatal(err)
}
name := SanitizeOutput(fmt.Sprint(obj["name"]))
if isPromptLeakageTitle(name) || isPromptLabelTitle(name) {
t.Fatalf("heuristic leaked formula into name: %q", name)
}
if !strings.Contains(strings.ToLower(name), "vox") {
t.Fatalf("name=%q want Vox from Name: line", name)
}
}
func TestLabeledPromptValue_skipsInstructionNameBullet(t *testing.T) {
user := `- name: short retail title; follow any Title formula constraints that follow; use Attrs
Name: Vox SWA-8000W
Desc: Washer
Attrs: {}`
got := labeledPromptValue(user, "name:", "current name:")
if got != "Vox SWA-8000W" {
t.Fatalf("got %q want Vox SWA-8000W", got)
}
}
func TestRunSteps_full_mappedCategoryWinsOverPrior(t *testing.T) {
e := &Engine{
Completer: stubCompleter{fn: func(system, user string) (Completion, error) {
apps/api/internal/processing/upsert_processed_product_test.go
+16
View File
@@ -28,6 +28,22 @@ func TestUpsertProcessedProductSQL_usesOnConflict(t *testing.T) {
if !strings.Contains(upsertProcessedProductSQL, "COALESCE(NULLIF(BTRIM(EXCLUDED.category), ''), processed_products.category)") {
t.Fatalf("expected category preserve on conflict: got SQL without COALESCE preserve")
}
// Free template meta must persist; existing non-empty meta must be preserved on reprocess.
if !strings.Contains(upsertProcessedProductSQL, "meta_title") || !strings.Contains(upsertProcessedProductSQL, "meta_description") {
t.Fatalf("expected meta_title/meta_description columns in upsert")
}
if !strings.Contains(upsertProcessedProductSQL, "WHEN NULLIF(BTRIM(processed_products.meta_title), '') IS NULL") {
t.Fatalf("expected meta_title preserve-when-set CASE on conflict")
}
if !strings.Contains(upsertProcessedProductSQL, `~ '\|\s+[0-9]+$'`) {
t.Fatalf("expected meta_title refresh when trailing | <digits-only unique_id>")
}
if !strings.Contains(upsertProcessedProductSQL, "WHEN NULLIF(BTRIM(processed_products.meta_description), '') IS NULL") {
t.Fatalf("expected meta_description preserve-when-set CASE on conflict")
}
if !strings.Contains(upsertProcessedProductSQL, "ready for retail listing") {
t.Fatalf("expected meta_description refresh of weak stubs when meta_title is poisoned")
}
}
func TestUpsertProcessedProduct_concurrentIdempotent(t *testing.T) {
apps/api/internal/processing/v1_images_test.go
+98
View File
@@ -0,0 +1,98 @@
package processing
import (
"context"
"os"
"testing"
"time"
"github.com/google/uuid"
"github.com/jackc/pgx/v5/pgxpool"
)
func TestLoadV1ProcessJobItems_promotesMoreimagesWhenMainEmpty(t *testing.T) {
dsn := os.Getenv("DATABASE_URL")
if dsn == "" {
t.Skip("DATABASE_URL not set")
}
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
pg, err := pgxpool.New(ctx, dsn)
if err != nil {
t.Fatal(err)
}
defer pg.Close()
companyID := uuid.New()
userID := uuid.New()
rawID := uuid.New()
jobID := uuid.New()
ppID := uuid.New()
gtin := "img-v1-" + companyID.String()[:8]
if _, err := pg.Exec(ctx, `INSERT INTO companies (id, name) VALUES ($1, $2)`, companyID, "img-v1-test"); err != nil {
t.Fatal(err)
}
err = pg.QueryRow(ctx, `SELECT id FROM users ORDER BY created_at DESC LIMIT 1`).Scan(&userID)
if err != nil {
t.Skip("no users rows available")
}
defer func() {
_, _ = pg.Exec(context.Background(), `DELETE FROM processing_job_products WHERE job_id = $1`, jobID)
_, _ = pg.Exec(context.Background(), `DELETE FROM processing_jobs WHERE id = $1`, jobID)
_, _ = pg.Exec(context.Background(), `DELETE FROM processed_products WHERE company_id = $1`, companyID)
_, _ = pg.Exec(context.Background(), `DELETE FROM raw_products WHERE company_id = $1`, companyID)
_, _ = pg.Exec(context.Background(), `DELETE FROM companies WHERE id = $1`, companyID)
}()
mapped := `{"main_image":"","moreimages":"https://cdn.example.com/a.jpg,https://cdn.example.com/b.jpg","name":"Widget"}`
if _, err := pg.Exec(ctx, `
INSERT INTO raw_products (id, company_id, gtin, raw_data, mapped_data, is_processed, processing_status)
VALUES ($1, $2, $3, '{}'::jsonb, $4::jsonb, true, 'processed')`,
rawID, companyID, gtin, mapped); err != nil {
t.Fatal(err)
}
if _, err := pg.Exec(ctx, `
INSERT INTO processed_products (
id, company_id, product_id, name, category, description, processed_name, processed_description,
raw_product_id, status, attributes, processed_attributes, field_sources
) VALUES (
$1, $2, $3, 'Widget', NULL, 'desc', 'Widget', 'desc',
$4, 'needs_review', '{}'::jsonb, '{}'::jsonb, '{}'::jsonb
)`, ppID, companyID, gtin, rawID); err != nil {
t.Fatal(err)
}
if _, err := pg.Exec(ctx, `
INSERT INTO processing_jobs (id, company_id, user_id, status, processing_type, total_products, processed_products)
VALUES ($1, $2, $3, 'completed', 'full', 1, 1)`, jobID, companyID, userID); err != nil {
t.Fatal(err)
}
if _, err := pg.Exec(ctx, `
INSERT INTO processing_job_products (id, job_id, raw_product_id, processed_product_id, status)
VALUES (gen_random_uuid(), $1, $2, $3, 'processed')`, jobID, rawID, ppID); err != nil {
t.Fatal(err)
}
p := NewPipeline(pg)
items, err := p.LoadV1ProcessJobItems(ctx, companyID, jobID, "full")
if err != nil {
t.Fatal(err)
}
if len(items) != 1 {
t.Fatalf("items=%d want 1", len(items))
}
main, _ := items[0]["main_image"].(string)
if main != "https://cdn.example.com/a.jpg" {
t.Fatalf("main_image=%v want promoted moreimages[0]", items[0]["main_image"])
}
switch more := items[0]["more_images"].(type) {
case []string:
if len(more) != 1 || more[0] != "https://cdn.example.com/b.jpg" {
t.Fatalf("more_images=%v", more)
}
default:
t.Fatalf("more_images type %T = %v", items[0]["more_images"], items[0]["more_images"])
}
}
apps/api/internal/processing/v1_legacy.go
+193 -27
View File
@@ -9,6 +9,7 @@ import (
"strings"
"github.com/descrybe/descrybe-v2/apps/api/internal/catalog"
"github.com/descrybe/descrybe-v2/apps/api/internal/eprel"
"github.com/google/uuid"
)
@@ -206,6 +207,7 @@ func (p *Pipeline) LoadV1ProcessJobItems(ctx context.Context, companyID, jobID u
return nil, fmt.Errorf("pipeline not configured")
}
allowedAttrs := loadCompanyAttributeKeySet(ctx, p, companyID)
categoryNames := loadCompanyCategoryNameMap(ctx, p, companyID)
rows, err := p.Pool.Query(ctx, `
SELECT
COALESCE(r.gtin, p.product_id, '') AS ean,
@@ -297,31 +299,98 @@ func (p *Pipeline) LoadV1ProcessJobItems(ctx context.Context, companyID, jobID u
if descTxt != nil {
plainDesc = v1PlainDescription(*descTxt)
}
eprelVal := extractEPRELFromAttrs(attrs)
attrs = SanitizeV1ProcessAttributesAllowed(attrs, allowedAttrs)
titleStr := derefStringPtr(title)
catStr := derefStringPtr(category)
catNameStr := derefStringPtr(categoryName)
// Projection gap: when processed.category is empty but mapped/raw carries a
// unique_id, surface it (and resolve category_name from company taxonomy).
if catStr == "" {
catStr = categoryUniqueIDFromMaps(mapped, rawData)
}
if catNameStr == "" && catStr != "" {
if name, ok := categoryNames[catStr]; ok && name != "" {
catNameStr = name
} else if nested, ok := mapped["category"].(map[string]any); ok {
for _, k := range []string{"name", "category_name", "title"} {
if s := strings.TrimSpace(stringFromAny(nested[k])); s != "" {
catNameStr = s
break
}
}
}
}
metaTitleOut := nullIfEmptyPtr(metaTitle)
metaDescOut := nullIfEmptyPtr(metaDesc)
if s, ok := metaTitleOut.(string); ok && isPoisonedMetaTitle(s) {
metaTitleOut = nil
// Poisoned title refresh: also drop empty/weak/leakage stub meta_description.
if ds, ok := metaDescOut.(string); ok && (ds == "" || isWeakPriorEnhanceDescription(ds, titleStr) || isPromptLeakageTitle(ds)) {
metaDescOut = nil
}
}
if ds, ok := metaDescOut.(string); ok && isPromptLeakageTitle(ds) {
metaDescOut = nil
}
catLabel := catNameStr
if catLabel == "" {
catLabel = catStr
}
if (metaTitleOut == nil || metaDescOut == nil) && (titleStr != "" || plainDesc != "" || catLabel != "") {
synthTitle, synthDesc := fillMetaFromResult(StepResult{
Name: titleStr,
ProcessedName: titleStr,
Category: catStr,
CategoryName: catNameStr,
Description: plainDesc,
ProcessedDescription: plainDesc,
Attributes: attrs,
ProcessedAttributes: attrs,
})
if metaTitleOut == nil {
if synthTitle != "" {
metaTitleOut = synthTitle
} else {
metaTitleOut = nullIfEmptyPtr(title)
}
}
if metaDescOut == nil {
if synthDesc != "" {
metaDescOut = synthDesc
} else if plainDesc != "" {
metaDescOut = truncateMetaDescription(plainDesc, v1MetaDescriptionMaxChars)
}
}
} else if metaTitleOut == nil {
metaTitleOut = nullIfEmptyPtr(title)
}
var description any
if plainDesc != "" {
description = plainDesc
} else {
description = nil
}
eprelVal := extractEPRELFromAttrs(attrs)
attrs = SanitizeV1ProcessAttributesAllowed(attrs, allowedAttrs)
metaTitleOut := nullIfEmptyPtr(metaTitle)
if metaTitleOut == nil {
metaTitleOut = nullIfEmptyPtr(title)
var catOut, catNameOut any
if catStr != "" {
catOut = catStr
}
metaDescOut := nullIfEmptyPtr(metaDesc)
if metaDescOut == nil && plainDesc != "" {
metaDescOut = truncateRunes(plainDesc, v1MetaDescriptionMaxChars)
if catNameStr != "" {
catNameOut = catNameStr
}
titleOut := nullIfEmptyPtr(title)
item := V1ProcessJobItem{
"ean": ean,
"status": MapV1JobItemStatus(itemStatus, true),
"category": nullIfEmptyPtr(category),
"category_name": nullIfEmptyPtr(categoryName),
"title": nullIfEmptyPtr(title),
"category": catOut,
"category_name": catNameOut,
"title": titleOut,
"name": titleOut,
"meta_title": metaTitleOut,
"meta_description": metaDescOut,
"description": description,
@@ -343,6 +412,7 @@ func (p *Pipeline) LoadV1ProcessJobItems(ctx context.Context, companyID, jobID u
if len(more) > 0 {
item["more_images"] = more
}
item = EnforceV1ProcessCompletedItem(item, "", allowedAttrs)
full = append(full, item)
}
if err := rows.Err(); err != nil {
@@ -358,6 +428,13 @@ func nullIfEmptyPtr(s *string) any {
return *s
}
func derefStringPtr(s *string) string {
if s == nil {
return ""
}
return strings.TrimSpace(*s)
}
// loadCompanyAttributeKeySet returns canonicalized attribute_key values for the company.
// On query failure returns an empty (non-nil) set so FilterAttributesByAllowed still
// restricts to coreCharacteristicAttrKeys only.
@@ -389,15 +466,52 @@ func loadCompanyAttributeKeySet(ctx context.Context, p *Pipeline, companyID uuid
return out
}
// v1PlainDescription normalizes feed HTML into a single plain-text string for the
// legacy process poll (description is a string, not a one-element array).
// loadCompanyCategoryNameMap returns unique_id → display name for the company.
func loadCompanyCategoryNameMap(ctx context.Context, p *Pipeline, companyID uuid.UUID) map[string]string {
out := map[string]string{}
if p == nil || p.Pool == nil {
return out
}
rows, err := p.Pool.Query(ctx, `
SELECT unique_id, name
FROM categories
WHERE company_id = $1 AND COALESCE(unique_id, '') <> ''`, companyID)
if err != nil {
return out
}
defer rows.Close()
for rows.Next() {
var uid, name string
if err := rows.Scan(&uid, &name); err != nil {
continue
}
uid = strings.TrimSpace(uid)
name = strings.TrimSpace(name)
if uid == "" || name == "" {
continue
}
out[uid] = name
}
return out
}
// v1PlainDescription normalizes legacy stored descriptions into a single plain-text
// string for the process poll (description is a string, not a one-element array).
// Handles JSON array/string encodings (e.g. mapped_data->>'description' when the
// feed value was an array) and HTML/<br>/entity markup.
func v1PlainDescription(s string) string {
s = strings.TrimSpace(s)
if s == "" {
return ""
}
s = unwrapLegacyDescriptionStored(s)
s = strings.TrimSpace(s)
if s == "" {
return ""
}
s = html.UnescapeString(s)
s = strings.ReplaceAll(s, "\u00a0", " ")
s = strings.ReplaceAll(s, `\u00a0`, " ")
s = v1BreakTagRe.ReplaceAllString(s, "\n")
s = v1BlockEndRe.ReplaceAllString(s, "\n")
s = specsHTMLTagRe.ReplaceAllString(s, " ")
@@ -406,6 +520,10 @@ func v1PlainDescription(s string) string {
kept := make([]string, 0, len(lines))
for _, line := range lines {
line = strings.Join(strings.Fields(line), " ")
line = strings.ReplaceAll(line, " .", ".")
line = strings.ReplaceAll(line, " ,", ",")
line = strings.ReplaceAll(line, " ;", ";")
line = strings.ReplaceAll(line, " :", ":")
if line != "" {
kept = append(kept, line)
}
@@ -413,23 +531,67 @@ func v1PlainDescription(s string) string {
return strings.TrimSpace(strings.Join(kept, "\n"))
}
func extractEPRELFromAttrs(attrs map[string]any) any {
if attrs == nil {
return nil
const maxLegacyDescriptionUnwrapDepth = 4
// unwrapLegacyDescriptionStored flattens JSON-encoded legacy description payloads
// (one-element arrays, multi-part arrays, nested arrays, JSON-quoted strings).
// Non-JSON text is returned unchanged.
func unwrapLegacyDescriptionStored(s string) string {
return unwrapLegacyDescriptionStoredN(s, 0)
}
func unwrapLegacyDescriptionStoredN(s string, depth int) string {
s = strings.TrimSpace(s)
if s == "" || depth > maxLegacyDescriptionUnwrapDepth {
return s
}
if e, ok := attrs["eprel"]; ok && e != nil {
return e
if !looksLikeLegacyDescriptionJSON(s) {
return s
}
out := map[string]any{}
for _, k := range []string{"label", "pdf", "energy_class", "energy_scale"} {
if v, ok := attrs["eprel_"+k]; ok && v != nil {
out[k] = v
var raw any
if err := json.Unmarshal([]byte(s), &raw); err != nil {
return s
}
flat := flattenLegacyDescriptionAny(raw, depth)
if flat == "" {
if _, ok := raw.([]any); ok {
return ""
}
return s
}
if len(out) == 0 {
return nil
return flat
}
func looksLikeLegacyDescriptionJSON(s string) bool {
if len(s) >= 2 && s[0] == '[' && s[len(s)-1] == ']' {
return true
}
return out
return len(s) >= 2 && s[0] == '"' && s[len(s)-1] == '"'
}
func flattenLegacyDescriptionAny(v any, depth int) string {
switch t := v.(type) {
case string:
return unwrapLegacyDescriptionStoredN(t, depth+1)
case []any:
parts := make([]string, 0, len(t))
for _, el := range t {
if p := flattenLegacyDescriptionAny(el, depth+1); p != "" {
parts = append(parts, p)
}
}
return strings.Join(parts, "\n")
case float64:
return strings.TrimSpace(fmt.Sprint(t))
case bool:
return fmt.Sprint(t)
default:
return ""
}
}
func extractEPRELFromAttrs(attrs map[string]any) any {
return eprel.ExtractFromAttrs(attrs)
}
// ProjectV1ProcessJobItems applies legacy partial-type field projection.
@@ -471,6 +633,10 @@ func ProjectV1ProcessJobItems(storedType string, items []V1ProcessJobItem) []V1P
projected["category_name"] = item["category_name"]
case "title":
projected["title"] = item["title"]
projected["name"] = item["name"]
if projected["name"] == nil {
projected["name"] = item["title"]
}
projected["meta_title"] = item["meta_title"]
case "description":
projected["description"] = item["description"]
apps/api/internal/processing/v1_legacy_test.go
+58 -1
View File
@@ -76,7 +76,7 @@ func TestProjectV1ProcessJobItemsPartial(t *testing.T) {
"processed_product_id": "bbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb",
"raw_product_id": "cccccccc-cccc-cccc-cccc-cccccccccccc",
"status": "processed",
"title": "T", "meta_title": "MT",
"title": "T", "name": "T", "meta_title": "MT",
"description": "D", "attributes": map[string]any{"brand": "X"},
"main_image": "https://example.com/a.jpg", "more_images": []string{"https://example.com/b.jpg"},
"eprel": nil, "category": "cat", "category_name": "Cat",
@@ -91,6 +91,9 @@ func TestProjectV1ProcessJobItemsPartial(t *testing.T) {
if got["title"] != "T" || got["ean"] != "123" {
t.Fatalf("got=%v", got)
}
if got["name"] != "T" {
t.Fatalf("name dual-mode alias missing or mismatched: %v", got)
}
if _, ok := got["attributes"]; ok {
t.Fatalf("attributes should be projected out: %v", got)
}
@@ -108,6 +111,23 @@ func TestProjectV1ProcessJobItemsPartial(t *testing.T) {
}
}
func TestProjectV1ProcessJobItemsTitleDerivesName(t *testing.T) {
items := []V1ProcessJobItem{{
"ean": "123", "id": "bbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb",
"status": "processed", "title": "OnlyTitle", "meta_title": "MT",
}}
out := ProjectV1ProcessJobItems("title", items)
if len(out) != 1 {
t.Fatalf("len=%d", len(out))
}
if out[0]["title"] != "OnlyTitle" {
t.Fatalf("title=%v", out[0]["title"])
}
if out[0]["name"] != "OnlyTitle" {
t.Fatalf("name should mirror title when absent: %v", out[0]["name"])
}
}
func TestV1PlainDescriptionStandalone(t *testing.T) {
got := v1PlainDescription("Line1<br>Line2 &amp; ok")
if strings.Contains(got, "<") {
@@ -118,6 +138,43 @@ func TestV1PlainDescriptionStandalone(t *testing.T) {
}
}
func TestV1PlainDescriptionEdgeCases(t *testing.T) {
nbsp := "\u00a0"
cases := []struct {
name string
in string
want string
}{
{name: "empty", in: "", want: ""},
{name: "whitespace", in: " \n\t ", want: ""},
{name: "plain", in: "Hello world", want: "Hello world"},
{name: "html_br", in: "Hello<br>World<br/>&amp; more</p><b>bold</b>", want: "Hello\nWorld\n& more\nbold"},
{name: "one_element_json_array_html", in: `["Hello<br>World"]`, want: "Hello\nWorld"},
{name: "multi_element_array", in: `["Para one.", "Para <b>two</b>."]`, want: "Para one.\nPara two."},
{name: "nested_array", in: `[["inner<br>x"]]`, want: "inner\nx"},
{name: "json_quoted_string", in: `"Plain &amp; quoted"`, want: "Plain & quoted"},
{name: "empty_array", in: `[]`, want: ""},
{name: "array_with_nulls", in: `[null, "keep", null, ""]`, want: "keep"},
{name: "array_with_object_ignored", in: `[{"text":"x"}, "ok"]`, want: "ok"},
{name: "invalid_bracket_text", in: `[not json`, want: "[not json"},
{name: "nbsp_and_entities", in: "A" + nbsp + "B&nbsp;C", want: "A B C"},
{name: "literal_unicode_escape", in: `A\u00a0B`, want: "A B"},
{name: "double_encoded_array", in: `"[\"Hello<br>World\"]"`, want: "Hello\nWorld"},
{name: "whitespace_around_array", in: " [\" spaced \"] ", want: "spaced"},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
got := v1PlainDescription(tc.in)
if got != tc.want {
t.Fatalf("in=%q got=%q want=%q", tc.in, got, tc.want)
}
if strings.Contains(got, "<") {
t.Fatalf("html left: %q", got)
}
})
}
}
func TestApplyV1ProcessItemIDsDualMode(t *testing.T) {
processed := mustParseTestUUID(t, "bbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb")
raw := mustParseTestUUID(t, "cccccccc-cccc-cccc-cccc-cccccccccccc")
apps/api/internal/processing/v1_process_item.go
+456
View File
@@ -0,0 +1,456 @@
package processing
import (
"fmt"
"strings"
"github.com/descrybe/descrybe-v2/apps/api/internal/eprel"
)
// V1ProcessItemScorecard scores one COMPLETED legacy process item against
// OpenAPI LegacyProcessItem + A1 poll expectations.
type V1ProcessItemScorecard struct {
EAN string `json:"ean"`
Status string `json:"status"`
OK bool `json:"ok"`
Score int `json:"score"` // 0–100
FailFlags []string `json:"fail_flags,omitempty"`
HasTitle bool `json:"has_title"`
HasName bool `json:"has_name"`
HasDescription bool `json:"has_description"`
DescriptionIsPlain bool `json:"description_is_plain"`
HasMetaTitle bool `json:"has_meta_title"`
HasMetaDescription bool `json:"has_meta_description"`
HasCategory bool `json:"has_category"`
HasCategoryName bool `json:"has_category_name"`
AttrsClean bool `json:"attrs_clean"`
EPRELOK bool `json:"eprel_ok"`
HasIDs bool `json:"has_ids"`
ImagesOK bool `json:"images_ok"`
}
// ScoreV1ProcessItemOptions tunes structure checks for a poll item.
type ScoreV1ProcessItemOptions struct {
// MappedCategory, when non-empty, requires item.category to equal it
// (projection must surface mapped unique_id).
MappedCategory string
// Language is used only for documentation of synthesize paths in tests.
Language string
}
// ScoreV1ProcessCompletedItem scores a successful (or terminal) V1 process item.
// For status != processed it only checks ean/status shape and returns OK when those exist.
func ScoreV1ProcessCompletedItem(item V1ProcessJobItem, opts ScoreV1ProcessItemOptions) V1ProcessItemScorecard {
sc := V1ProcessItemScorecard{
EAN: strings.TrimSpace(fmt.Sprint(item["ean"])),
Status: strings.TrimSpace(fmt.Sprint(item["status"])),
}
if sc.EAN == "" || sc.EAN == "<nil>" {
sc.FailFlags = append(sc.FailFlags, "missing_ean")
}
if sc.Status == "" || sc.Status == "<nil>" {
sc.FailFlags = append(sc.FailFlags, "missing_status")
}
if sc.Status != "processed" {
sc.OK = len(sc.FailFlags) == 0
if sc.OK {
sc.Score = 100
}
return sc
}
title := stringFromItem(item, "title")
name := stringFromItem(item, "name")
sc.HasTitle = title != ""
sc.HasName = name != ""
if sc.HasTitle && !sc.HasName {
sc.FailFlags = append(sc.FailFlags, "missing_name")
}
if sc.HasTitle && sc.HasName && title != name {
sc.FailFlags = append(sc.FailFlags, "title_name_mismatch")
}
desc, descOK := plainDescriptionFromItem(item)
sc.HasDescription = desc != ""
sc.DescriptionIsPlain = descOK
if sc.HasTitle && !sc.HasDescription {
sc.FailFlags = append(sc.FailFlags, "description_empty_with_title")
}
if !descOK && item["description"] != nil {
sc.FailFlags = append(sc.FailFlags, "description_not_plain_string")
}
if sc.HasDescription && descriptionEchoesTitle(desc, title) {
sc.FailFlags = append(sc.FailFlags, "description_echoes_title")
}
if sc.HasDescription && containsWeakFillerPhrase(desc) {
sc.FailFlags = append(sc.FailFlags, "description_weak_filler")
}
sc.HasMetaTitle = stringFromItem(item, "meta_title") != ""
sc.HasMetaDescription = stringFromItem(item, "meta_description") != ""
if sc.HasTitle && !sc.HasMetaTitle {
sc.FailFlags = append(sc.FailFlags, "meta_title_missing")
}
if sc.HasTitle && !sc.HasMetaDescription {
sc.FailFlags = append(sc.FailFlags, "meta_description_missing")
}
if md := stringFromItem(item, "meta_description"); md != "" && containsWeakFillerPhrase(md) {
sc.FailFlags = append(sc.FailFlags, "meta_description_weak_filler")
}
cat := stringFromItem(item, "category")
catName := stringFromItem(item, "category_name")
sc.HasCategory = cat != ""
sc.HasCategoryName = catName != ""
mappedCat := strings.TrimSpace(opts.MappedCategory)
if mappedCat != "" {
if cat == "" {
sc.FailFlags = append(sc.FailFlags, "category_missing_though_mapped")
} else if cat != mappedCat {
sc.FailFlags = append(sc.FailFlags, "category_mismatch_mapped")
}
if cat != "" && catName == "" {
sc.FailFlags = append(sc.FailFlags, "category_name_missing")
}
}
attrs, attrsOK := attrsMapFromItem(item)
dirty := dirtyV1AttrKeys(attrs)
sc.AttrsClean = attrsOK && len(dirty) == 0
if !attrsOK {
sc.FailFlags = append(sc.FailFlags, "attributes_invalid_shape")
}
if len(dirty) > 0 {
sc.FailFlags = append(sc.FailFlags, "attributes_dirty:"+strings.Join(dirty, ","))
}
sc.EPRELOK = eprelShapeOK(item["eprel"])
if !sc.EPRELOK {
sc.FailFlags = append(sc.FailFlags, "eprel_invalid_shape")
}
id := stringFromItem(item, "id")
ppID := stringFromItem(item, "processed_product_id")
rawID := stringFromItem(item, "raw_product_id")
sc.HasIDs = id != "" && ppID != "" && rawID != ""
if id == "" {
sc.FailFlags = append(sc.FailFlags, "missing_id")
}
if ppID == "" {
sc.FailFlags = append(sc.FailFlags, "missing_processed_product_id")
}
if rawID == "" {
sc.FailFlags = append(sc.FailFlags, "missing_raw_product_id")
}
if id != "" && ppID != "" && id != ppID {
sc.FailFlags = append(sc.FailFlags, "id_processed_product_id_mismatch")
}
sc.ImagesOK = imageFieldsOK(item)
if !sc.ImagesOK {
sc.FailFlags = append(sc.FailFlags, "images_invalid_shape")
}
// Weighted score: structure gates, not content quality beyond filler/echo.
checks := []bool{
sc.EAN != "",
sc.Status == "processed",
sc.HasTitle,
sc.HasName || !sc.HasTitle,
sc.HasDescription || !sc.HasTitle,
sc.DescriptionIsPlain || item["description"] == nil,
sc.HasMetaTitle || !sc.HasTitle,
sc.HasMetaDescription || !sc.HasTitle,
mappedCat == "" || sc.HasCategory,
mappedCat == "" || sc.HasCategoryName,
sc.AttrsClean,
sc.EPRELOK,
sc.HasIDs,
sc.ImagesOK,
!containsWeakFillerPhrase(desc),
!descriptionEchoesTitle(desc, title),
}
pass := 0
for _, ok := range checks {
if ok {
pass++
}
}
sc.Score = (pass * 100) / len(checks)
sc.OK = len(sc.FailFlags) == 0
return sc
}
// EnforceV1ProcessCompletedItem fills LegacyProcessItem projection gaps for a
// successful item: plain nonempty description when title exists, meta_*, clean
// attributes, eprel object|null, and image key shapes. Category must already be
// set by the caller when mapped provides a unique_id.
//
// allowed is the company attribute_key set (canonicalized). When nil, only
// coreCharacteristicAttrKeys are kept (never leak feed junk like zavora).
func EnforceV1ProcessCompletedItem(item V1ProcessJobItem, language string, allowed map[string]struct{}) V1ProcessJobItem {
if item == nil {
return item
}
status, _ := item["status"].(string)
if status != "processed" {
return item
}
attrs, _ := attrsMapFromItem(item)
if allowed == nil {
allowed = map[string]struct{}{}
}
attrs = SanitizeV1ProcessAttributesAllowed(attrs, allowed)
if len(attrs) > 0 {
item["attributes"] = attrs
} else {
item["attributes"] = nil
}
title := stringFromItem(item, "title")
if title != "" {
item["title"] = title
item["name"] = title
} else {
item["title"] = nil
item["name"] = nil
}
cat := stringFromItem(item, "category")
catName := stringFromItem(item, "category_name")
catLabel := catName
if catLabel == "" {
catLabel = cat
}
desc, _ := plainDescriptionFromItem(item)
// Empty, weak, or title-echo copy must be replaced — never leave description==title.
if title != "" && (desc == "" || isWeakPriorEnhanceDescription(desc, title) || descriptionEchoesTitle(desc, title)) {
if synth := synthesizeDescriptionFromTitle(title, catLabel, language, attrs); synth != "" {
desc = synth
}
}
if desc != "" {
item["description"] = desc
} else {
item["description"] = nil
}
metaTitle := stringFromItem(item, "meta_title")
metaDesc := stringFromItem(item, "meta_description")
needMetaTitle := metaTitle == "" || isPoisonedMetaTitle(metaTitle)
// Empty, weak stub (Ready for retail…), title-echo, or prompt-leakage — refresh.
// When meta_title is poisoned, also force refresh of weak/leakage meta_description.
needMetaDesc := metaDesc == "" ||
isWeakPriorEnhanceDescription(metaDesc, title) ||
containsWeakFillerPhrase(metaDesc) ||
isPromptLeakageTitle(metaDesc) ||
(title != "" && descriptionEchoesTitle(metaDesc, title))
if needMetaTitle && isPoisonedMetaTitle(metaTitle) &&
(metaDesc == "" || isWeakPriorEnhanceDescription(metaDesc, title) || isPromptLeakageTitle(metaDesc)) {
needMetaDesc = true
}
if title != "" || desc != "" || cat != "" || catName != "" {
synthTitle, synthDesc := fillMetaFromResult(StepResult{
Name: title,
ProcessedName: title,
Category: cat,
CategoryName: catName,
Description: desc,
ProcessedDescription: desc,
Attributes: attrs,
ProcessedAttributes: attrs,
})
if needMetaTitle {
if synthTitle != "" {
metaTitle = synthTitle
} else {
metaTitle = title
}
}
if needMetaDesc {
if synthDesc != "" {
metaDesc = synthDesc
} else if desc != "" {
metaDesc = truncateMetaDescription(desc, v1MetaDescriptionMaxChars)
}
}
}
if metaTitle != "" {
item["meta_title"] = metaTitle
} else {
item["meta_title"] = nil
}
if metaDesc != "" {
item["meta_description"] = metaDesc
} else {
item["meta_description"] = nil
}
if cat != "" {
item["category"] = cat
} else {
item["category"] = nil
}
if catName != "" {
item["category_name"] = catName
} else {
item["category_name"] = nil
}
item["eprel"] = normalizeEPRELValue(item["eprel"])
main := stringFromItem(item, "main_image")
if main != "" {
item["main_image"] = main
} else {
item["main_image"] = nil
}
more := stringSliceFromItem(item, "more_images")
if len(more) > 0 {
item["more_images"] = more
} else {
item["more_images"] = nil
}
return item
}
func stringFromItem(item V1ProcessJobItem, key string) string {
if item == nil {
return ""
}
return strings.TrimSpace(stringFromAny(item[key]))
}
func plainDescriptionFromItem(item V1ProcessJobItem) (string, bool) {
if item == nil {
return "", true
}
raw := item["description"]
if raw == nil {
return "", true
}
switch raw.(type) {
case string:
return PlainDescriptionFromAny(raw), true
case []any, []string:
// Legacy mistake: description as array — convert to plain string.
if s := PlainDescriptionFromAny(raw); s != "" {
return s, true
}
return "", false
case map[string]any:
if s := PlainDescriptionFromAny(raw); s != "" {
return s, true
}
return "", false
default:
s := strings.TrimSpace(fmt.Sprint(raw))
if s == "" || s == "<nil>" {
return "", true
}
return "", false
}
}
func attrsMapFromItem(item V1ProcessJobItem) (map[string]any, bool) {
if item == nil {
return map[string]any{}, true
}
raw := item["attributes"]
if raw == nil {
return map[string]any{}, true
}
switch t := raw.(type) {
case map[string]any:
return t, true
default:
return map[string]any{}, false
}
}
func dirtyV1AttrKeys(attrs map[string]any) []string {
if len(attrs) == 0 {
return nil
}
var dirty []string
for k := range attrs {
key := strings.TrimSpace(k)
lk := strings.ToLower(key)
if key == "" || isReservedProductKey(key) || isInvalidAttributeKey(key) ||
strings.HasPrefix(lk, "eprel") {
dirty = append(dirty, key)
continue
}
// Re-run sanitize: if key disappears, it was dirty.
trial := SanitizeV1ProcessAttributes(map[string]any{key: attrs[k]})
if len(trial) == 0 {
dirty = append(dirty, key)
}
}
return dirty
}
func eprelShapeOK(v any) bool {
if v == nil {
return true
}
_, ok := v.(map[string]any)
return ok
}
func normalizeEPRELValue(v any) any {
return eprel.NormalizeShape(v)
}
func imageFieldsOK(item V1ProcessJobItem) bool {
if item == nil {
return true
}
if v := item["main_image"]; v != nil {
if _, ok := v.(string); !ok {
return false
}
}
if v := item["more_images"]; v != nil {
switch v.(type) {
case []string, []any:
return true
default:
return false
}
}
return true
}
func stringSliceFromItem(item V1ProcessJobItem, key string) []string {
if item == nil {
return nil
}
switch t := item[key].(type) {
case []string:
out := make([]string, 0, len(t))
for _, s := range t {
s = strings.TrimSpace(s)
if s != "" {
out = append(out, s)
}
}
return out
case []any:
out := make([]string, 0, len(t))
for _, e := range t {
if s, ok := e.(string); ok {
s = strings.TrimSpace(s)
if s != "" {
out = append(out, s)
}
}
}
return out
default:
return nil
}
}
apps/api/internal/processing/v1_process_item_test.go
+319
View File
@@ -0,0 +1,319 @@
package processing
import (
"fmt"
"strings"
"testing"
)
func TestScoreV1ProcessCompletedItem_processedStructure(t *testing.T) {
t.Parallel()
item := V1ProcessJobItem{
"ean": "123",
"status": "processed",
"id": "bbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb",
"processed_product_id": "bbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb",
"raw_product_id": "cccccccc-cccc-cccc-cccc-cccccccccccc",
"title": "Sony Headphones",
"name": "Sony Headphones",
"description": "Sony Headphones deliver clear audio with noise cancelling for daily commuting.",
"meta_title": "Sony | Sony Headphones",
"meta_description": "Sony Headphones deliver clear audio with noise cancelling for daily commuting.",
"category": "48",
"category_name": "Slušalke",
"attributes": map[string]any{"brand": "Sony", "color": "Black"},
"main_image": "https://example.com/a.jpg",
"more_images": []string{"https://example.com/b.jpg"},
"eprel": nil,
}
sc := ScoreV1ProcessCompletedItem(item, ScoreV1ProcessItemOptions{MappedCategory: "48"})
if !sc.OK {
t.Fatalf("expected OK, flags=%v score=%d", sc.FailFlags, sc.Score)
}
if sc.Score < 90 {
t.Fatalf("score=%d want >= 90", sc.Score)
}
}
func TestScoreV1ProcessCompletedItem_flagsGaps(t *testing.T) {
t.Parallel()
item := V1ProcessJobItem{
"ean": "999",
"status": "processed",
"id": "bbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb",
"processed_product_id": "bbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb",
"raw_product_id": "cccccccc-cccc-cccc-cccc-cccccccccccc",
"title": "Mount",
"description": nil,
"meta_title": nil,
"meta_description": nil,
"category": nil,
"category_name": nil,
"attributes": map[string]any{
"brand": "X",
"eprel_id": "1",
":": "true",
},
"eprel": "not-an-object",
}
sc := ScoreV1ProcessCompletedItem(item, ScoreV1ProcessItemOptions{MappedCategory: "28"})
if sc.OK {
t.Fatal("expected failures")
}
joined := strings.Join(sc.FailFlags, ",")
for _, want := range []string{
"missing_name",
"description_empty_with_title",
"meta_title_missing",
"meta_description_missing",
"category_missing_though_mapped",
"eprel_invalid_shape",
} {
if !strings.Contains(joined, want) {
t.Fatalf("missing flag %q in %v", want, sc.FailFlags)
}
}
}
func TestEnforceV1ProcessCompletedItem_fillsDescriptionMetaSanitizes(t *testing.T) {
t.Parallel()
item := V1ProcessJobItem{
"ean": "1",
"status": "processed",
"id": "bbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb",
"processed_product_id": "bbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb",
"raw_product_id": "cccccccc-cccc-cccc-cccc-cccccccccccc",
"title": "Fiksni stenski TV nosilec 32''-55'' MANHATTAN",
"description": "Fiksni stenski TV nosilec 32''-55'' MANHATTAN from Ostalo. Ready for retail listing.",
"meta_title": nil,
"meta_description": nil,
"category": "28",
"category_name": "Nosilci za TV",
"attributes": map[string]any{
"brand": "Ostalo",
"product_model": "460934",
"eprel_label": "A",
"name": "should-drop",
":": "true",
},
"eprel": nil,
}
out := EnforceV1ProcessCompletedItem(item, "sl", nil)
desc, _ := out["description"].(string)
if desc == "" {
t.Fatal("expected nonempty description")
}
if descriptionEchoesTitle(desc, fmt.Sprint(out["title"])) {
t.Fatalf("EnforceV1 must replace weak/echo desc, got title-echo: %q", desc)
}
if containsWeakFillerPhrase(desc) {
t.Fatalf("filler left in description: %q", desc)
}
if strings.TrimSpace(fmt.Sprint(out["meta_title"])) == "" || out["meta_title"] == nil {
t.Fatalf("meta_title empty: %v", out["meta_title"])
}
if strings.TrimSpace(fmt.Sprint(out["meta_description"])) == "" || out["meta_description"] == nil {
t.Fatalf("meta_description empty: %v", out["meta_description"])
}
if containsWeakFillerPhrase(fmt.Sprint(out["meta_description"])) {
t.Fatalf("meta_description still filler: %v", out["meta_description"])
}
attrs, ok := out["attributes"].(map[string]any)
if !ok {
t.Fatalf("attributes type=%T", out["attributes"])
}
if _, bad := attrs["eprel_label"]; bad {
t.Fatalf("eprel_* should be stripped: %v", attrs)
}
if _, bad := attrs[":"]; bad {
t.Fatalf("invalid key kept: %v", attrs)
}
if _, bad := attrs["name"]; bad {
t.Fatalf("reserved name kept: %v", attrs)
}
if out["name"] != out["title"] {
t.Fatalf("name should mirror title: name=%v title=%v", out["name"], out["title"])
}
sc := ScoreV1ProcessCompletedItem(out, ScoreV1ProcessItemOptions{MappedCategory: "28"})
if !sc.OK {
t.Fatalf("enforced item should score OK, flags=%v", sc.FailFlags)
}
}
func TestEnforceV1ProcessCompletedItem_skipsTerminalStatuses(t *testing.T) {
t.Parallel()
item := V1ProcessJobItem{"ean": "1", "status": "not_found", "error": "missing"}
out := EnforceV1ProcessCompletedItem(item, "", nil)
if _, ok := out["title"]; ok {
t.Fatalf("should not invent fields for not_found: %v", out)
}
}
func TestEnforceV1ProcessCompletedItem_categoryFromCallerPreserved(t *testing.T) {
t.Parallel()
item := V1ProcessJobItem{
"ean": "2",
"status": "processed",
"id": "bbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb",
"processed_product_id": "bbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb",
"raw_product_id": "cccccccc-cccc-cccc-cccc-cccccccccccc",
"title": "Cooker",
"description": "",
"category": "50",
"category_name": "Štedilniki",
"attributes": map[string]any{"brand": "Vox"},
"eprel": nil,
}
out := EnforceV1ProcessCompletedItem(item, "", nil)
if out["category"] != "50" {
t.Fatalf("category=%v", out["category"])
}
if out["category_name"] != "Štedilniki" {
t.Fatalf("category_name=%v", out["category_name"])
}
desc, _ := out["description"].(string)
if desc == "" {
t.Fatal("expected synthesized description when title set")
}
if descriptionEchoesTitle(desc, "Cooker") {
t.Fatalf("synthesized description must not equal title: %q", desc)
}
sc := ScoreV1ProcessCompletedItem(out, ScoreV1ProcessItemOptions{MappedCategory: "50"})
if !sc.OK {
t.Fatalf("flags=%v", sc.FailFlags)
}
}
func TestEnforceV1ProcessCompletedItem_replacesTitleEchoDescription(t *testing.T) {
t.Parallel()
title := "VOX EBR700"
item := V1ProcessJobItem{
"ean": "8806095210711",
"status": "processed",
"id": "bbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb",
"processed_product_id": "bbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb",
"raw_product_id": "cccccccc-cccc-cccc-cccc-cccccccccccc",
"title": title,
"description": title,
"category": "50",
"category_name": "Štedilniki",
"attributes": map[string]any{"brand": "VOX"},
"eprel": nil,
}
out := EnforceV1ProcessCompletedItem(item, "sl", nil)
desc, _ := out["description"].(string)
if desc == "" {
t.Fatal("expected synthesized description")
}
if descriptionEchoesTitle(desc, title) {
t.Fatalf("must not leave title-echo description: %q", desc)
}
if !strings.Contains(desc, "Štedilniki") && !strings.Contains(desc, "VOX") {
t.Fatalf("expected factual synth with category/brand: %q", desc)
}
}
func TestEnforceV1ProcessCompletedItem_refreshesPoisonedMetaTitle(t *testing.T) {
t.Parallel()
item := V1ProcessJobItem{
"ean": "8806095210711",
"status": "processed",
"id": "bbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb",
"processed_product_id": "bbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb",
"raw_product_id": "cccccccc-cccc-cccc-cccc-cccccccccccc",
"title": "VOX EBR700",
"description": "VOX EBR700 is a Štedilniki product from VOX.",
"meta_title": "VOX EBR700 | 50",
"meta_description": "Ready for retail listing.",
"category": "50",
"category_name": "Štedilniki",
"attributes": map[string]any{
"brand": "VOX",
"zavora": "junk",
},
"eprel": nil,
}
out := EnforceV1ProcessCompletedItem(item, "sl", nil)
mt := fmt.Sprint(out["meta_title"])
if isPoisonedMetaTitle(mt) || strings.Contains(mt, "| 50") {
t.Fatalf("poisoned meta_title preserved: %q", mt)
}
if !strings.Contains(mt, "Štedilniki") {
t.Fatalf("expected CategoryName in meta_title: %q", mt)
}
md := fmt.Sprint(out["meta_description"])
if md == "" || md == "<nil>" || out["meta_description"] == nil {
t.Fatalf("meta_description empty after poisoned title refresh: %v", out["meta_description"])
}
if containsWeakFillerPhrase(md) || isWeakPriorEnhanceDescription(md, "VOX EBR700") {
t.Fatalf("stub meta_description preserved with poisoned title: %q", md)
}
attrs, ok := out["attributes"].(map[string]any)
if !ok {
t.Fatalf("attributes type=%T", out["attributes"])
}
if _, bad := attrs["zavora"]; bad {
t.Fatalf("feed junk zavora leaked: %v", attrs)
}
if _, ok := attrs["brand"]; !ok {
t.Fatalf("brand should remain: %v", attrs)
}
}
func TestIsPoisonedMetaTitle(t *testing.T) {
t.Parallel()
if !isPoisonedMetaTitle("VOX EBR700 | 50") {
t.Fatal("expected poisoned")
}
if !isPoisonedMetaTitle("Name | 7") {
t.Fatal("expected poisoned with extra spaces")
}
if isPoisonedMetaTitle("VOX | Štedilniki") {
t.Fatal("display name suffix must not count as poisoned")
}
if isPoisonedMetaTitle("") {
t.Fatal("empty not poisoned")
}
if !isPoisonedMetaTitle("Sony | short retail title; follow any Title formula constr…") {
t.Fatal("expected prompt-leakage meta_title poisoned")
}
if !isPoisonedMetaTitle("ANKER | short retail title; follow any Title formula constraints that follow") {
t.Fatal("expected Title formula leakage poisoned")
}
}
func TestEnforceV1ProcessCompletedItem_refreshesPromptLeakageMeta(t *testing.T) {
t.Parallel()
item := V1ProcessJobItem{
"ean": "4548736132597",
"status": "processed",
"id": "dddddddd-dddd-dddd-dddd-dddddddddddd",
"processed_product_id": "dddddddd-dddd-dddd-dddd-dddddddddddd",
"raw_product_id": "eeeeeeee-eeee-eeee-eeee-eeeeeeeeeeee",
"title": "SONY BT slušalke WH1000XM5S sive",
"description": "SONY BT slušalke WH1000XM5S sive je izdelek v kategoriji Slušalke znamke Sony.",
"meta_title": "Sony | short retail title; follow any Title formula constr…",
"meta_description": "prefer 1-3 factual paragraphs as ONE string; limited HTML () is allowed if needed",
"category": "48",
"category_name": "Slušalke",
"attributes": map[string]any{
"brand": "Sony",
},
"eprel": nil,
}
out := EnforceV1ProcessCompletedItem(item, "sl", nil)
mt := fmt.Sprint(out["meta_title"])
if isPoisonedMetaTitle(mt) || strings.Contains(strings.ToLower(mt), "short retail title") {
t.Fatalf("prompt-leakage meta_title preserved: %q", mt)
}
if !strings.Contains(mt, "Slušalke") && !strings.Contains(mt, "SONY") {
t.Fatalf("expected synth meta_title with product/category: %q", mt)
}
md := fmt.Sprint(out["meta_description"])
if md == "" || md == "<nil>" || out["meta_description"] == nil {
t.Fatalf("meta_description empty after leakage refresh: %v", out["meta_description"])
}
if isPromptLeakageTitle(md) || strings.Contains(strings.ToLower(md), "prefer 1-3") {
t.Fatalf("prompt-leakage meta_description preserved: %q", md)
}
}
apps/api/internal/processing/weak_desc_test.go
+116
View File
@@ -0,0 +1,116 @@
package processing
import (
"strings"
"testing"
)
func TestIsWeakPriorEnhanceDescription_fillerPhrases(t *testing.T) {
title := "Vogel's WALL 3245 TV Wall Mount"
cases := []struct {
desc string
want bool
}{
{"Ready for retail listing with extra padding text here.", true},
{title + " in the Mounts category. Ready for retail listing.", true},
{"Product description placeholder text that is long enough to pass length.", true},
{"A product in the Mounts category based on available specifications.", true},
{"Product with available specifications ready for retail listing.", true},
{"Generic one-liner without product tokens and enough length!!!!!", true},
{
"Vogel's WALL 3245 is a TV Mounts product from Vogel's. Key specs: width 45 cm, max_load 40 kg.",
false,
},
{
"This durable retail mount includes mounting hardware, load ratings, and install guidance for wall displays.",
false,
},
}
for _, tc := range cases {
got := isWeakPriorEnhanceDescription(tc.desc, title)
if got != tc.want {
t.Fatalf("desc=%q got weak=%v want %v", tc.desc, got, tc.want)
}
}
// Long non-filler copy that does not share tokens with this title is still usable
// once past the short-boilerplate window (hash-skip quality floor).
longUnrelated := strings.Repeat("Detailed retail copy covering materials finishes warranty and care. ", 3)
if isWeakPriorEnhanceDescription(longUnrelated, title) {
t.Fatalf("long non-filler unrelated copy should not be weak solely for token miss: %q", longUnrelated)
}
if !isWeakPriorEnhanceDescription("ok", title) {
t.Fatal("too-short must be weak")
}
if !isWeakPriorEnhanceDescription(title, title) {
t.Fatal("title-echo must be weak")
}
}
func TestSynthesizeDescriptionFromTitle_brandModelCategory(t *testing.T) {
title := "Vogel's WALL 3245 TV Wall Mount"
attrs := map[string]any{
"brand": "Vogel's",
"product_model": "WALL 3245",
"width": "45 cm",
"max_load": "40 kg",
}
en := synthesizeDescriptionFromTitle(title, "TV Mounts", "en", attrs)
if en == "" {
t.Fatal("expected nonempty English invent")
}
if strings.Contains(strings.ToLower(en), "ready for retail listing") {
t.Fatalf("must not emit retail filler: %q", en)
}
for _, need := range []string{"Vogel", "TV Mounts", "45 cm", "40 kg"} {
if !strings.Contains(en, need) {
t.Fatalf("English invent missing %q: %q", need, en)
}
}
if isWeakPriorEnhanceDescription(en, title) {
t.Fatalf("factual invent must not be weak: %q", en)
}
sl := synthesizeDescriptionFromTitle(title, "TV Mounts", "sl", attrs)
if sl == "" || !strings.Contains(sl, "kategoriji") {
t.Fatalf("expected Slovenian invent, got %q", sl)
}
if strings.Contains(strings.ToLower(sl), "ready for retail listing") {
t.Fatalf("SL invent must not emit EN filler: %q", sl)
}
if isWeakPriorEnhanceDescription(sl, title) {
t.Fatalf("SL factual invent must not be weak: %q", sl)
}
}
func TestInventHeuristicDescription_fromBrandModelCategory(t *testing.T) {
title := "Vogel's WALL 3245 TV Wall Mount"
user := ProductEnhanceUser("TV Mounts", title, "", map[string]any{
"brand": "Vogel's",
"product_model": "WALL 3245",
"width": "45 cm",
"max_load": "40 kg",
})
system := "Write name and description in English. Return JSON with \"name\" and \"description\"."
got := inventHeuristicDescription(system, user, title)
if got == "" {
t.Fatal("expected invent output")
}
t.Logf("TV mount invent sample: %s", got)
if strings.Contains(strings.ToLower(got), "ready for retail listing") {
t.Fatalf("invent must not return retail filler: %q", got)
}
for _, need := range []string{"Vogel", "TV Mounts", "45 cm"} {
if !strings.Contains(got, need) {
t.Fatalf("invent missing %q: %q", need, got)
}
}
if isWeakPriorEnhanceDescription(got, title) {
t.Fatalf("invent output must not be classified weak: %q", got)
}
slSystem := "Write name and description in Slovenian. Return JSON with \"name\"."
sl := inventHeuristicDescription(slSystem, user, title)
if !strings.Contains(sl, "kategoriji") {
t.Fatalf("expected SL invent from system language, got %q", sl)
}
}
apps/api/internal/security/http_client.go
+6 -1
View File
@@ -26,9 +26,14 @@ func SafeHTTPClientPolicy(timeout time.Duration, policy DialPolicy) *http.Client
if timeout <= 0 {
timeout = 30 * time.Second
}
tr := SafeHTTPTransportPolicy(policy)
// Match header wait to overall timeout. A fixed 30s ResponseHeaderTimeout
// aborts OpenAI-compatible LLM calls that withhold headers until generation
// finishes (reasoning models often take 60–180s).
tr.ResponseHeaderTimeout = timeout
return &http.Client{
Timeout: timeout,
Transport: SafeHTTPTransportPolicy(policy),
Transport: tr,
}
}
apps/web/src/lib/admin-orgs.ts
+55 -1
View File
@@ -1,5 +1,7 @@
/**
* Admin orgs UI client — users + companies directory, staff roles, plan assign.
* Admin orgs UI client — users + companies directory, staff roles, plan assign,
* clone-catalog, and Fix A1 (`POST …/fix-catalog`).
* Flash Fix A1: result.prompts / hashes / categories → flash.admin.fixA1Success.
* Contract: docs/admin-roles-support/04-contract.md · Docs: 10-admin-orgs-ui.md
*/
import { api, ApiError } from "$lib/api";
@@ -18,6 +20,8 @@ export const ADMIN_USERS_PATH = "/api/admin/users";
export const ADMIN_COMPANIES_PATH = "/api/admin/companies";
export const ADMIN_CLONE_CATALOG_PATH = (companyId: string) =>
`${ADMIN_COMPANIES_PATH}/${encodeURIComponent(companyId)}/clone-catalog`;
export const ADMIN_FIX_CATALOG_PATH = (companyId: string) =>
`${ADMIN_COMPANIES_PATH}/${encodeURIComponent(companyId)}/fix-catalog`;
export const ADMIN_STAFF_ROLE_PATH = (userId: string) =>
`/api/admin/users/${encodeURIComponent(userId)}/staff-role`;
@@ -226,6 +230,56 @@ export async function cloneAdminCompanyCatalog(
});
}
export type FixCatalogResult = {
company_id: string;
company_name: string;
a1_cohort?: boolean;
category_attribute_orphans_removed?: number;
category_attribute_links?: number;
/** Alias of category_prompts_updated for flash.admin.fixA1Success {prompts}. */
prompts?: number;
category_prompts_updated?: number;
product_enhance_languages?: number;
/** Alias of weak_hashes_cleared for flash {hashes}. */
hashes?: number;
weak_hashes_cleared?: number;
/** Alias of categories_backfilled for flash {categories}. */
categories?: number;
categories_backfilled?: number;
attributes_sanitized?: number;
products_scanned?: number;
reprocess_needed_count?: number;
reprocess_sample_raw_product_ids?: string[];
};
export type FixCatalogResponse = {
status: string;
result: FixCatalogResult;
note?: string;
};
/** In-place Fix A1 / catalog hygiene. Never clears catalog; no mass reprocess. */
export async function fixAdminCompanyCatalog(
companyId: string,
opts?: { backfillCategories?: boolean; reprocessSampleLimit?: number }
): Promise<FixCatalogResponse> {
const body: {
confirm: true;
backfill_categories?: boolean;
reprocess_sample_limit?: number;
} = { confirm: true };
if (opts?.backfillCategories === false) {
body.backfill_categories = false;
}
if (typeof opts?.reprocessSampleLimit === "number") {
body.reprocess_sample_limit = opts.reprocessSampleLimit;
}
return api<FixCatalogResponse>(ADMIN_FIX_CATALOG_PATH(companyId), {
method: "POST",
body
});
}
export function isStaffRoleApiUnavailable(err: unknown): boolean {
return err instanceof ApiError && (err.status === 404 || err.status === 501);
}
apps/web/src/lib/components/products/ProductEditPanel.svelte
+57 -12
View File
@@ -33,6 +33,7 @@
productFeedSyncLabel,
productHasEprel,
productHasSpecs,
productImageUrls,
resolveOriginalDescription,
resolveOriginalName
} from "./types";
@@ -438,6 +439,7 @@
const coverageCount = $derived(
enrichment ? Object.values(enrichment).filter((s) => s !== "missing").length : 0
);
const imageUrls = $derived.by(() => productImageUrls(product));
function attrDisplayValue(key: string, value: string): string {
if (key.trim().toLowerCase() !== "category") return value;
@@ -571,8 +573,9 @@
primary={contentLanguages[0] || DEFAULT_CONTENT_LANGUAGE}
allowAdd={false}
hasOverride={(code) => {
const f = fieldsForLang(code);
return Boolean(f.processedName.trim() || f.processedDescription.trim());
const primary = contentLanguages[0] || DEFAULT_CONTENT_LANGUAGE;
if (code === primary) return false;
return Boolean(product?.localized_content?.[code]);
}}
onChange={(code) => applyContentLang(code)}
/>
@@ -890,8 +893,36 @@
</select>
</div>
</div>
</section>
</TabsContent>
</section>
{#if imageUrls.length > 0}
<section class="space-y-3" data-testid="product-images">
<h3 class="text-sm font-medium">{i18n.t("products.edit.images")}</h3>
<div class="flex flex-wrap gap-3">
{#each imageUrls as src, i}
<a
href={src}
target="_blank"
rel="noopener noreferrer"
class="block overflow-hidden rounded-md border border-border bg-muted/30"
title={src}
>
<img
src={src}
alt={i18n.t("products.edit.imageAlt", {
index: i + 1,
name: product?.processed_name || product?.name || productId
})}
class="h-28 w-28 object-contain"
loading="lazy"
referrerpolicy="no-referrer"
/>
</a>
{/each}
</div>
</section>
{/if}
</TabsContent>
<TabsContent value="feed" class="mt-4 space-y-4">
<section class="rounded-md border border-border px-3 py-3">
@@ -957,14 +988,28 @@
>{field.value}</pre>
{/if}
{:else if /^https?:\/\//i.test(field.value)}
<a
href={field.value}
target="_blank"
rel="noopener noreferrer"
class="break-all text-primary underline-offset-2 hover:underline"
>
{field.value}
</a>
{@const isImage =
/image|img|photo|picture|thumbnail/i.test(field.key) ||
/\.(jpe?g|png|gif|webp|svg|avif)(\?|$)/i.test(field.value)}
<div class="space-y-2">
{#if isImage}
<img
src={field.value}
alt={field.key}
class="max-h-40 max-w-full rounded border border-border object-contain"
loading="lazy"
referrerpolicy="no-referrer"
/>
{/if}
<a
href={field.value}
target="_blank"
rel="noopener noreferrer"
class="break-all text-primary underline-offset-2 hover:underline"
>
{field.value}
</a>
</div>
{:else}
<span class="break-words {field.ok ? '' : 'text-chart-amber'}">{field.value}</span>
{/if}
apps/web/src/lib/components/products/types.ts
+33
View File
@@ -847,6 +847,39 @@ function digAttrBag(
return undefined;
}
/** Collect http(s) image URLs from mapped_data (main_image + more_images). */
export function productImageUrls(product: ProductRow | null | undefined): string[] {
const mapped = asRecord(product?.mapped_data);
if (!mapped) return [];
const out: string[] = [];
const seen = new Set<string>();
const push = (raw: unknown) => {
if (typeof raw === "string") {
const url = raw.trim();
if (!/^https?:\/\//i.test(url) || seen.has(url)) return;
seen.add(url);
out.push(url);
return;
}
if (Array.isArray(raw)) {
for (const item of raw) push(item);
return;
}
const rec = asRecord(raw);
if (!rec) return;
for (const key of ["url", "src", "href", "link"]) {
if (rec[key] != null) push(rec[key]);
}
};
for (const key of ["main_image", "mainImage", "image", "image_url", "imageUrl"]) {
if (mapped[key] != null) push(mapped[key]);
}
for (const key of ["more_images", "moreImages", "images", "additional_images"]) {
if (mapped[key] != null) push(mapped[key]);
}
return out;
}
/** True when product carries an EPREL id or enriched energy-label payload. */
export function productHasEprel(product: ProductRow | null | undefined): boolean {
if (!product) return false;
apps/web/src/lib/i18n/fix-a1-keys.test.ts
+51
View File
@@ -0,0 +1,51 @@
/**
* Focused check: admin Fix A1 i18n keys exist in every UI_LOCALES pack.
*/
import assert from "node:assert/strict";
import { describe, it } from "node:test";
import { UI_LOCALES } from "./locales.ts";
import { en } from "./messages/en.ts";
import { loadAllMessages, messagesFor } from "./messages/catalog.ts";
import { sl } from "./messages/sl.ts";
const FIX_A1_KEYS = [
"admin.users.fixA1",
"admin.users.fixA1Aria",
"admin.users.fixA1Title",
"admin.users.fixA1Desc",
"admin.users.fixA1Company",
"admin.users.fixA1Warning",
"admin.users.fixA1Cancel",
"admin.users.fixA1Confirm",
"flash.admin.fixA1Success",
"flash.admin.fixA1Error"
] as const;
describe("admin Fix A1 i18n keys", () => {
it("English defines every Fix A1 key", () => {
for (const key of FIX_A1_KEYS) {
assert.equal(typeof en[key], "string", key);
assert.ok(en[key].trim().length > 0, key);
}
});
it("every registered UI locale has Fix A1 keys", async () => {
await loadAllMessages();
for (const { code } of UI_LOCALES) {
const pack = code === "en" ? en : messagesFor(code);
for (const key of FIX_A1_KEYS) {
const value = pack[key];
assert.equal(typeof value, "string", `${code}:${key}`);
assert.ok(String(value).trim().length > 0, `${code}:${key}`);
}
}
});
it("Slovenian ready pack has Fix A1 keys (not in UI_LOCALES)", () => {
for (const key of FIX_A1_KEYS) {
assert.equal(typeof sl[key], "string", key);
assert.ok(sl[key].trim().length > 0, key);
}
});
});
apps/web/src/lib/i18n/messages/de.ts
+10
View File
@@ -812,6 +812,14 @@ export const de: MessageDict = {
"admin.users.cloneCatalogCancel": "Abbrechen",
"admin.users.cloneCatalogConfirm": "Katalog kopieren",
"admin.users.cloneDestFallback": "Ihr Sandbox-Unternehmen",
"admin.users.fixA1": "A1-Katalog reparieren",
"admin.users.fixA1Aria": "KI-Prompts, Kategorien und Enhance-Hashes für {name} reparieren",
"admin.users.fixA1Title": "Unternehmenskatalog reparieren",
"admin.users.fixA1Desc": "Wendet korrigierte KI-Prompts erneut an, löst schwache Enhance-Hashes und füllt Kategorien aus mapped_data nach.",
"admin.users.fixA1Company": "Unternehmen: {name}",
"admin.users.fixA1Warning": "Bevorzugen Sie Platform Demo oder ein explizites Unternehmen. Löscht keine Feeds, Mappings oder Rohprodukte. Schützt A1-Kohorten-Überschreibregeln.",
"admin.users.fixA1Cancel": "Abbrechen",
"admin.users.fixA1Confirm": "Katalog reparieren",
"admin.users.noUsers": "Keine Benutzer entsprechen diesem Filter.",
"admin.users.noCompanies": "Keine Unternehmen entsprechen diesem Filter.",
"admin.users.assignRoleTitle": "Mitarbeiterrolle zuweisen",
@@ -2297,6 +2305,8 @@ export const de: MessageDict = {
"flash.admin.staffRoleUnavailable": "Mitarbeiterrollen-Updates sind auf diesem Server noch nicht verfügbar.",
"flash.admin.planAssigned": "Plan {name} zugewiesen.",
"flash.admin.catalogCloned": "{source} nach {dest} kopiert: {products} Produkte, {categories} Kategorien.",
"flash.admin.fixA1Success": "Katalogreparatur für {name} abgeschlossen: {prompts} Prompts aktualisiert, {hashes} Hashes gelöscht, {categories} Kategorien nachgefüllt.",
"flash.admin.fixA1Error": "Katalogreparatur für {name} fehlgeschlagen.",
"flash.admin.planAssignedShort": "Plan zugewiesen.",
"flash.admin.creditsUpdated": "Credits aktualisiert.",
"flash.admin.cyclesProcessed": "Abrechnungszyklen verarbeitet: {count}",
apps/web/src/lib/i18n/messages/en.ts
+16
View File
@@ -355,6 +355,10 @@ export const en: MessageDict = {
"dashboard.lowCreditsTitle": "Credits running low",
"dashboard.lowCreditsMessage": "{remaining} of {total} credits left. Top up or upgrade before jobs stall.",
"dashboard.latestJobs": "Latest processing jobs",
"dashboard.jobsRunning": "{count} job running",
"dashboard.jobsRunningPlural": "{count} jobs running",
"dashboard.activeJobs": "{count} active job",
"dashboard.activeJobsPlural": "{count} active jobs",
"dashboard.noJobsEmpty": "No jobs yet — import products first.",
"dashboard.noJobsReady": "No recent jobs. Start one from Products when you are ready.",
"dashboard.startJob": "Start a job",
@@ -835,6 +839,14 @@ export const en: MessageDict = {
"admin.users.cloneCatalogCancel": "Cancel",
"admin.users.cloneCatalogConfirm": "Copy catalog",
"admin.users.cloneDestFallback": "your sandbox company",
"admin.users.fixA1": "Fix A1 catalog",
"admin.users.fixA1Aria": "Repair AI prompts, categories, and enhance hashes for {name}",
"admin.users.fixA1Title": "Fix company catalog",
"admin.users.fixA1Desc": "In-place repair: category attribute links, category enhance prompts (with attrs), weak enhance hashes, optional category backfill, attribute sanitize. No mass reprocess.",
"admin.users.fixA1Company": "Company: {name}",
"admin.users.fixA1Warning": "Prefer Platform Demo; A1 may be targeted in place with confirm. Never clears the catalog or uses A1 as a clone destination.",
"admin.users.fixA1Cancel": "Cancel",
"admin.users.fixA1Confirm": "Fix catalog",
"admin.users.noUsers": "No users match this filter.",
"admin.users.noCompanies": "No companies match this filter.",
"admin.users.assignRoleTitle": "Assign staff role",
@@ -2322,6 +2334,8 @@ export const en: MessageDict = {
"flash.admin.staffRoleUnavailable": "Staff role updates are not available on this server yet.",
"flash.admin.planAssigned": "Plan assigned to {name}.",
"flash.admin.catalogCloned": "Copied {source} into {dest}: {products} products, {categories} categories. Open Products and process catalog GTINs (EAN-only API calls need a matching feed product).",
"flash.admin.fixA1Success": "Catalog repair finished for {name}: {prompts} category prompts updated, {hashes} hashes cleared, {categories} categories backfilled.",
"flash.admin.fixA1Error": "Catalog repair failed for {name}.",
"flash.admin.planAssignedShort": "Plan assigned.",
"flash.admin.creditsUpdated": "Credits updated.",
"flash.admin.cyclesProcessed": "Billing cycles processed: {count}",
@@ -4238,6 +4252,8 @@ export const en: MessageDict = {
"products.edit.updatedRelative": "· updated {when}",
"products.edit.viewAllFeedFields": "View all feed fields ({filled}/{total})",
"products.edit.productIdentity": "Product identity",
"products.edit.images": "Images",
"products.edit.imageAlt": "Product image {index}: {name}",
"products.edit.aiProcessedName": "AI-processed name",
"products.edit.originalName": "Original name",
"products.edit.productIdSku": "Product ID / SKU",
apps/web/src/lib/i18n/messages/es.ts
+10
View File
@@ -812,6 +812,14 @@ export const es: MessageDict = {
"admin.users.cloneCatalogCancel": "Cancelar",
"admin.users.cloneCatalogConfirm": "Copiar catálogo",
"admin.users.cloneDestFallback": "tu empresa sandbox",
"admin.users.fixA1": "Reparar catálogo A1",
"admin.users.fixA1Aria": "Reparar prompts de IA, categorías y hashes de enhance para {name}",
"admin.users.fixA1Title": "Reparar catálogo de la empresa",
"admin.users.fixA1Desc": "Vuelve a aplicar prompts de IA corregidos, limpia hashes de enhance débiles y completa categorías desde mapped_data.",
"admin.users.fixA1Company": "Empresa: {name}",
"admin.users.fixA1Warning": "Prefiera Platform Demo o una empresa explícita. No elimina feeds, mapeos ni productos en bruto. Protege las reglas de sobrescritura de la cohorte A1.",
"admin.users.fixA1Cancel": "Cancelar",
"admin.users.fixA1Confirm": "Reparar catálogo",
"admin.users.noUsers": "Ningún usuario coincide con este filtro.",
"admin.users.noCompanies": "Ninguna empresa coincide con este filtro.",
"admin.users.assignRoleTitle": "Asignar rol de personal",
@@ -2296,6 +2304,8 @@ export const es: MessageDict = {
"flash.admin.staffRoleUpdated": "Rol de personal actualizado para {email}.",
"flash.admin.staffRoleUnavailable": "Las actualizaciones de rol de personal aún no están disponibles en este servidor.",
"flash.admin.catalogCloned": "Se copió {source} en {dest}: {products} productos, {categories} categorías.",
"flash.admin.fixA1Success": "Reparación del catálogo de {name} finalizada: {prompts} prompts actualizados, {hashes} hashes borrados, {categories} categorías rellenadas.",
"flash.admin.fixA1Error": "Falló la reparación del catálogo de {name}.",
"flash.admin.planAssigned": "Plan asignado a {name}.",
"flash.admin.planAssignedShort": "Plan asignado.",
"flash.admin.creditsUpdated": "Créditos actualizados.",
apps/web/src/lib/i18n/messages/fr.ts
+10
View File
@@ -812,6 +812,14 @@ export const fr: MessageDict = {
"admin.users.cloneCatalogCancel": "Annuler",
"admin.users.cloneCatalogConfirm": "Copier le catalogue",
"admin.users.cloneDestFallback": "votre entreprise sandbox",
"admin.users.fixA1": "Réparer le catalogue A1",
"admin.users.fixA1Aria": "Réparer les prompts IA, catégories et hashes enhance pour {name}",
"admin.users.fixA1Title": "Réparer le catalogue entreprise",
"admin.users.fixA1Desc": "Réapplique les prompts IA corrigés, efface les hashes enhance faibles et complète les catégories depuis mapped_data.",
"admin.users.fixA1Company": "Entreprise : {name}",
"admin.users.fixA1Warning": "Préférer Platform Demo ou une entreprise explicite. Ne supprime ni feeds, ni mappings, ni produits bruts. Protège les règles d'écrasement de la cohorte A1.",
"admin.users.fixA1Cancel": "Annuler",
"admin.users.fixA1Confirm": "Réparer le catalogue",
"admin.users.noUsers": "Aucun utilisateur ne correspond à ce filtre.",
"admin.users.noCompanies": "Aucune entreprise ne correspond à ce filtre.",
"admin.users.assignRoleTitle": "Assigner un rôle du personnel",
@@ -2296,6 +2304,8 @@ export const fr: MessageDict = {
"flash.admin.staffRoleUpdated": "Rôle du personnel mis à jour pour {email}.",
"flash.admin.staffRoleUnavailable": "Les mises à jour de rôle personnel ne sont pas encore disponibles sur ce serveur.",
"flash.admin.catalogCloned": "{source} copié vers {dest} : {products} produits, {categories} catégories.",
"flash.admin.fixA1Success": "Réparation du catalogue pour {name} terminée : {prompts} prompts mis à jour, {hashes} hashes effacés, {categories} catégories complétées.",
"flash.admin.fixA1Error": "Échec de la réparation du catalogue pour {name}.",
"flash.admin.planAssigned": "Offre assignée à {name}.",
"flash.admin.planAssignedShort": "Offre assignée.",
"flash.admin.creditsUpdated": "Crédits mis à jour.",
apps/web/src/lib/i18n/messages/it.ts
+10
View File
@@ -812,6 +812,14 @@ export const it: MessageDict = {
"admin.users.cloneCatalogCancel": "Annulla",
"admin.users.cloneCatalogConfirm": "Copia catalogo",
"admin.users.cloneDestFallback": "la tua azienda sandbox",
"admin.users.fixA1": "Ripara catalogo A1",
"admin.users.fixA1Aria": "Ripara prompt IA, categorie e hash enhance per {name}",
"admin.users.fixA1Title": "Ripara catalogo azienda",
"admin.users.fixA1Desc": "Riapplica i prompt IA corretti, cancella hash enhance deboli e completa le categorie da mapped_data.",
"admin.users.fixA1Company": "Azienda: {name}",
"admin.users.fixA1Warning": "Preferisci Platform Demo o un'azienda esplicita. Non elimina feed, mapping o prodotti grezzi. Protegge le regole di sovrascrittura della coorte A1.",
"admin.users.fixA1Cancel": "Annulla",
"admin.users.fixA1Confirm": "Ripara catalogo",
"admin.users.noUsers": "Nessun utente corrisponde a questo filtro.",
"admin.users.noCompanies": "Nessuna azienda corrisponde a questo filtro.",
"admin.users.assignRoleTitle": "Assegna ruolo staff",
@@ -2296,6 +2304,8 @@ export const it: MessageDict = {
"flash.admin.staffRoleUpdated": "Ruolo staff aggiornato per {email}.",
"flash.admin.staffRoleUnavailable": "Gli aggiornamenti del ruolo staff non sono ancora disponibili su questo server.",
"flash.admin.catalogCloned": "Copiato {source} in {dest}: {products} prodotti, {categories} categorie.",
"flash.admin.fixA1Success": "Riparazione catalogo per {name} completata: {prompts} prompt aggiornati, {hashes} hash cancellati, {categories} categorie integrate.",
"flash.admin.fixA1Error": "Riparazione catalogo per {name} non riuscita.",
"flash.admin.planAssigned": "Piano assegnato a {name}.",
"flash.admin.planAssignedShort": "Piano assegnato.",
"flash.admin.creditsUpdated": "Crediti aggiornati.",
apps/web/src/lib/i18n/messages/ja.ts
+10
View File
@@ -812,6 +812,14 @@ export const ja: MessageDict = {
"admin.users.cloneCatalogCancel": "キャンセル",
"admin.users.cloneCatalogConfirm": "カタログをコピー",
"admin.users.cloneDestFallback": "サンドボックス会社",
"admin.users.fixA1": "A1カタログを修復",
"admin.users.fixA1Aria": "{name} のAIプロンプト、カテゴリ、enhanceハッシュを修復",
"admin.users.fixA1Title": "会社カタログを修復",
"admin.users.fixA1Desc": "修正済みAIプロンプトを再適用し、弱いenhanceハッシュを消去し、mapped_dataからカテゴリを補完します。",
"admin.users.fixA1Company": "会社: {name}",
"admin.users.fixA1Warning": "Platform Demoまたは明示的な会社を優先してください。フィード、マッピング、生製品は削除しません。A1コホートの上書きルールを保護します。",
"admin.users.fixA1Cancel": "キャンセル",
"admin.users.fixA1Confirm": "カタログを修復",
"admin.users.noUsers": "このフィルタに一致するユーザーはいません。",
"admin.users.noCompanies": "このフィルタに一致する会社はありません。",
"admin.users.assignRoleTitle": "スタッフロールを割り当て",
@@ -2296,6 +2304,8 @@ export const ja: MessageDict = {
"flash.admin.staffRoleUpdated": "{email} のスタッフロールを更新しました。",
"flash.admin.staffRoleUnavailable": "このサーバーではスタッフロールの更新はまだ利用できません。",
"flash.admin.catalogCloned": "{source} を {dest} にコピーしました: 商品 {products}、カテゴリ {categories}。",
"flash.admin.fixA1Success": "{name} のカタログ修復が完了: プロンプト {prompts} 件更新、ハッシュ {hashes} 件消去、カテゴリ {categories} 件補完。",
"flash.admin.fixA1Error": "{name} のカタログ修復に失敗しました。",
"flash.admin.planAssigned": "{name} にプランを割り当てました。",
"flash.admin.planAssignedShort": "プランを割り当てました。",
"flash.admin.creditsUpdated": "クレジットを更新しました。",

Some files were not shown because too many files have changed in this diff Show More