From 532d439c41a74ea03db422a8528c7e45970d0ffa Mon Sep 17 00:00:00 2001 From: GreenEclipse Date: Sun, 16 Aug 2026 16:57:36 +0200 Subject: [PATCH] fixes --- apps/api/cmd/_electronics_probe_tmp/main.go | 188 ++++++ apps/api/cmd/_probe_batch_a_tmp/main.go | 192 ++++++ apps/api/cmd/_thin_synth_probe_tmp/main.go | 173 +++++ apps/api/cmd/migrator/catalog_feeds.go | 36 ++ apps/api/cmd/repair-category-prompts/main.go | 77 +++ apps/api/cmd/seed-a1/attributes_backfill.go | 27 + apps/api/cmd/seed-a1/category_prompts.go | 99 ++- apps/api/cmd/seed-a1/main.go | 29 +- apps/api/internal/aiprompts/apply_defaults.go | 74 +++ .../internal/aiprompts/apply_defaults_test.go | 71 ++ apps/api/internal/aiprompts/kinds.go | 36 +- apps/api/internal/aiprompts/kinds_test.go | 59 ++ apps/api/internal/aiprompts/service.go | 41 +- .../aiprovider/resolve_platform_test.go | 12 + apps/api/internal/aiprovider/roles.go | 8 +- apps/api/internal/aiprovider/service.go | 17 +- apps/api/internal/catalog/clone.go | 14 +- apps/api/internal/catalog/clone_test.go | 12 + apps/api/internal/catalog/fix_catalog.go | 76 +++ apps/api/internal/catalog/fix_catalog_test.go | 45 ++ apps/api/internal/catalog/product_v1_align.go | 66 ++ .../internal/catalog/product_v1_align_test.go | 123 ++++ apps/api/internal/catalog/prompt_repair.go | 220 +++++++ .../internal/catalog/prompt_repair_test.go | 52 ++ apps/api/internal/catalog/raw_v1.go | 14 +- apps/api/internal/catalog/raw_v1_test.go | 35 + .../internal/catalog/repair_enhance_hashes.go | 188 ++++++ .../catalog/repair_enhance_hashes_test.go | 137 ++++ apps/api/internal/catalog/service.go | 45 +- apps/api/internal/company/lang_content.go | 52 +- .../api/internal/company/lang_content_test.go | 28 +- apps/api/internal/company/weak_desc.go | 151 +++++ apps/api/internal/eprel/attrs.go | 145 +++++ apps/api/internal/eprel/attrs_test.go | 84 +++ apps/api/internal/eprel/client_test.go | 10 + apps/api/internal/eprel/id.go | 58 +- .../httpapi/admin_fix_catalog_handlers.go | 105 +++ .../admin_fix_catalog_handlers_test.go | 91 +++ apps/api/internal/httpapi/server.go | 1 + apps/api/internal/httpapi/v1_openapi.go | 216 ++++--- apps/api/internal/httpapi/v1_openapi_test.go | 30 + .../internal/platformsettings/ai_configs.go | 42 +- .../platformsettings/ai_configs_test.go | 32 + apps/api/internal/processing/ai.go | 43 +- .../internal/processing/ai_provider_mode.go | 23 + .../processing/ai_provider_mode_test.go | 22 + apps/api/internal/processing/attr_map.go | 269 ++++++++ apps/api/internal/processing/attr_map_test.go | 220 +++++++ .../api/internal/processing/attrs_backfill.go | 110 ++++ .../processing/attrs_backfill_test.go | 48 ++ apps/api/internal/processing/catalog_fix.go | 386 +++++++++++ .../internal/processing/catalog_fix_test.go | 20 + apps/api/internal/processing/category.go | 184 ++++++ .../internal/processing/category_backfill.go | 117 ++++ apps/api/internal/processing/category_test.go | 404 ++++++++++++ .../internal/processing/enhance_hash_test.go | 190 +++++- apps/api/internal/processing/eprel_test.go | 169 +++++ apps/api/internal/processing/fill.go | 29 +- .../processing/fix_company_catalog.go | 127 ++++ .../processing/fix_company_catalog_test.go | 23 + .../api/internal/processing/formula_prompt.go | 216 +++++++ .../processing/formula_prompt_test.go | 118 ++++ .../internal/processing/legacy_backfill.go | 382 +++++++++++ .../api/internal/processing/legacy_convert.go | 111 ++++ .../processing/legacy_convert_test.go | 61 ++ .../processing/live_mock_llm_title_test.go | 67 ++ apps/api/internal/processing/llm_json.go | 25 +- apps/api/internal/processing/meta.go | 150 +++++ apps/api/internal/processing/meta_test.go | 113 ++++ apps/api/internal/processing/normalize.go | 57 ++ apps/api/internal/processing/openai.go | 178 ++++- apps/api/internal/processing/openai_test.go | 119 ++++ apps/api/internal/processing/pipeline.go | 401 ++++++++++-- .../processing/pipeline_llm_mock_test.go | 53 ++ .../processing/pipeline_process_test.go | 17 + .../processing/pipeline_steps_test.go | 164 ++++- .../processing/prompt_fallback_test.go | 27 +- apps/api/internal/processing/prompt_render.go | 76 ++- .../internal/processing/prompt_render_test.go | 115 +++- apps/api/internal/processing/specs.go | 102 ++- apps/api/internal/processing/steps.go | 609 ++++++++++++++---- apps/api/internal/processing/steps_test.go | 108 +++- .../upsert_processed_product_test.go | 16 + .../api/internal/processing/v1_images_test.go | 98 +++ apps/api/internal/processing/v1_legacy.go | 220 ++++++- .../api/internal/processing/v1_legacy_test.go | 59 +- .../internal/processing/v1_process_item.go | 456 +++++++++++++ .../processing/v1_process_item_test.go | 319 +++++++++ .../api/internal/processing/weak_desc_test.go | 116 ++++ apps/api/internal/security/http_client.go | 7 +- apps/web/src/lib/admin-orgs.ts | 56 +- .../products/ProductEditPanel.svelte | 69 +- apps/web/src/lib/components/products/types.ts | 33 + apps/web/src/lib/i18n/fix-a1-keys.test.ts | 51 ++ apps/web/src/lib/i18n/messages/de.ts | 10 + apps/web/src/lib/i18n/messages/en.ts | 16 + apps/web/src/lib/i18n/messages/es.ts | 10 + apps/web/src/lib/i18n/messages/fr.ts | 10 + apps/web/src/lib/i18n/messages/it.ts | 10 + apps/web/src/lib/i18n/messages/ja.ts | 10 + apps/web/src/lib/i18n/messages/nl.ts | 10 + apps/web/src/lib/i18n/messages/pl.ts | 10 + apps/web/src/lib/i18n/messages/pt.ts | 10 + apps/web/src/lib/i18n/messages/sl.ts | 15 + apps/web/src/routes/admin/users/+page.svelte | 131 +++- docs/admin-roles-support/10-admin-orgs-ui.md | 35 +- 106 files changed, 10147 insertions(+), 494 deletions(-) create mode 100644 apps/api/cmd/_electronics_probe_tmp/main.go create mode 100644 apps/api/cmd/_probe_batch_a_tmp/main.go create mode 100644 apps/api/cmd/_thin_synth_probe_tmp/main.go create mode 100644 apps/api/cmd/repair-category-prompts/main.go create mode 100644 apps/api/cmd/seed-a1/attributes_backfill.go create mode 100644 apps/api/internal/aiprompts/apply_defaults.go create mode 100644 apps/api/internal/aiprompts/apply_defaults_test.go create mode 100644 apps/api/internal/aiprompts/kinds_test.go create mode 100644 apps/api/internal/catalog/fix_catalog.go create mode 100644 apps/api/internal/catalog/fix_catalog_test.go create mode 100644 apps/api/internal/catalog/product_v1_align.go create mode 100644 apps/api/internal/catalog/product_v1_align_test.go create mode 100644 apps/api/internal/catalog/prompt_repair.go create mode 100644 apps/api/internal/catalog/prompt_repair_test.go create mode 100644 apps/api/internal/catalog/repair_enhance_hashes.go create mode 100644 apps/api/internal/catalog/repair_enhance_hashes_test.go create mode 100644 apps/api/internal/company/weak_desc.go create mode 100644 apps/api/internal/eprel/attrs.go create mode 100644 apps/api/internal/eprel/attrs_test.go create mode 100644 apps/api/internal/httpapi/admin_fix_catalog_handlers.go create mode 100644 apps/api/internal/httpapi/admin_fix_catalog_handlers_test.go create mode 100644 apps/api/internal/processing/attr_map.go create mode 100644 apps/api/internal/processing/attr_map_test.go create mode 100644 apps/api/internal/processing/attrs_backfill.go create mode 100644 apps/api/internal/processing/attrs_backfill_test.go create mode 100644 apps/api/internal/processing/catalog_fix.go create mode 100644 apps/api/internal/processing/catalog_fix_test.go create mode 100644 apps/api/internal/processing/category.go create mode 100644 apps/api/internal/processing/category_backfill.go create mode 100644 apps/api/internal/processing/category_test.go create mode 100644 apps/api/internal/processing/fix_company_catalog.go create mode 100644 apps/api/internal/processing/fix_company_catalog_test.go create mode 100644 apps/api/internal/processing/formula_prompt.go create mode 100644 apps/api/internal/processing/formula_prompt_test.go create mode 100644 apps/api/internal/processing/legacy_backfill.go create mode 100644 apps/api/internal/processing/legacy_convert.go create mode 100644 apps/api/internal/processing/legacy_convert_test.go create mode 100644 apps/api/internal/processing/live_mock_llm_title_test.go create mode 100644 apps/api/internal/processing/meta.go create mode 100644 apps/api/internal/processing/meta_test.go create mode 100644 apps/api/internal/processing/v1_images_test.go create mode 100644 apps/api/internal/processing/v1_process_item.go create mode 100644 apps/api/internal/processing/v1_process_item_test.go create mode 100644 apps/api/internal/processing/weak_desc_test.go create mode 100644 apps/web/src/lib/i18n/fix-a1-keys.test.ts create mode 100644 apps/web/src/lib/i18n/messages/sl.ts diff --git a/apps/api/cmd/_electronics_probe_tmp/main.go b/apps/api/cmd/_electronics_probe_tmp/main.go new file mode 100644 index 0000000..2bc7739 --- /dev/null +++ b/apps/api/cmd/_electronics_probe_tmp/main.go @@ -0,0 +1,188 @@ +package main + +import ( + "context" + "fmt" + "log" + "os" + "os/signal" + "strings" + "syscall" + "time" + + "github.com/descrybe/descrybe-v2/apps/api/internal/aiprompts" + "github.com/descrybe/descrybe-v2/apps/api/internal/config" + "github.com/descrybe/descrybe-v2/apps/api/internal/db" + "github.com/descrybe/descrybe-v2/apps/api/internal/logredact" + "github.com/descrybe/descrybe-v2/apps/api/internal/processing" + "github.com/google/uuid" +) + +func main() { + log.SetOutput(logredact.Writer(os.Stderr)) + eans := []string{ + "4548736132597", + "194644207427", + "194644034894", + "195949822384", + "3830061960930", + } + if v := strings.TrimSpace(os.Getenv("PROBE_EANS")); v != "" { + eans = strings.Split(v, ",") + for i := range eans { + eans[i] = strings.TrimSpace(eans[i]) + } + } + companyID, err := uuid.Parse("2b3159b0-fc08-415b-b248-35ed02a6baab") + if err != nil { + log.Fatal(err) + } + userID, err := uuid.Parse("0ce3305c-d810-4b56-b1d4-3c1ed510db76") + if err != nil { + log.Fatal(err) + } + + cfg, err := config.Load() + if err != nil { + log.Fatalf("config: %v", err) + } + ctx, cancel := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM) + defer cancel() + + pool, err := db.NewPool(ctx, cfg.DatabaseURL, db.PoolOptions{ + MaxConns: int32(cfg.DBMaxConns), + MinConns: int32(cfg.DBMinConns), + MaxConnLifetime: cfg.DBMaxConnLifetime, + MaxConnLifetimeJitter: cfg.DBMaxConnLifetimeJitter, + MaxConnIdleTime: cfg.DBMaxConnIdleTime, + HealthCheckPeriod: cfg.DBHealthCheckPeriod, + StatementTimeout: cfg.DBStatementTimeout, + }) + if err != nil { + log.Fatalf("db: %v", err) + } + defer pool.Close() + + pipeline := processing.NewPipeline(pool) + pipeline.BatchSize = cfg.ProcessingBatchSize + // Force mock-llm: platform DB OpenAI binding would otherwise override env. + pipeline.AI = nil + pipeline.Prompts = aiprompts.NewService(pool) + mockKey := strings.TrimSpace(cfg.OpenAIAPIKey) + if mockKey == "" { + mockKey = "local-test" + } + mockBase := strings.TrimSpace(cfg.OpenAIBaseURL) + if mockBase == "" { + mockBase = "http://127.0.0.1:18767/v1" + } + mockModel := strings.TrimSpace(cfg.OpenAIModel) + if mockModel == "" { + mockModel = "mock-llm" + } + log.Printf("forcing completer base=%s model=%s", mockBase, mockModel) + pipeline.Engine = &processing.Engine{ + Completer: processing.NewOpenAIClient(mockKey, mockBase, mockModel, cfg.ProcessingRPM, cfg.ProcessingMaxRetries), + Vector: processing.NoopVectorCategorizer{}, + EPREL: nil, // skip remote EPREL hangs during local scorecard + ProviderMode: processing.AIProviderInternal, + } + + rawIDs := make([]uuid.UUID, 0, len(eans)) + for _, ean := range eans { + var id uuid.UUID + err := pool.QueryRow(ctx, ` + SELECT id FROM raw_products + WHERE company_id = $1 AND gtin = $2`, companyID, ean).Scan(&id) + if err != nil { + log.Fatalf("raw product %s: %v", ean, err) + } + rawIDs = append(rawIDs, id) + log.Printf("ean=%s raw_id=%s", ean, id) + } + + // Clear competing queue noise. + _, _ = pool.Exec(ctx, ` + UPDATE processing_jobs + SET status = 'failed', error = 'yielded to electronics in-process probe', updated_at = now() + WHERE status IN ('pending','running','processing')`) + + jobs, err := pipeline.StartJob(ctx, companyID, userID, rawIDs, "full") + if err != nil { + log.Fatalf("StartJob: %v", err) + } + if len(jobs) == 0 { + log.Fatal("StartJob returned no jobs") + } + jobID := jobs[0].ID + log.Printf("job_id=%s chunks=%d", jobID, len(jobs)) + + defend := make(chan struct{}) + go func() { + t := time.NewTicker(400 * time.Millisecond) + defer t.Stop() + for { + select { + case <-defend: + return + case <-ctx.Done(): + return + case <-t.C: + _, _ = pool.Exec(context.Background(), ` + UPDATE processing_jobs + SET status = 'failed', error = 'yielded to electronics in-process probe', updated_at = now() + WHERE status IN ('pending','running','processing') AND id <> $1`, jobID) + _, _ = pool.Exec(context.Background(), ` + UPDATE processing_jobs + SET status = 'running', error = NULL, updated_at = now() + WHERE id = $1 AND status IN ('failed','cancelled') + AND (error ILIKE '%P0%' OR error ILIKE '%parked%' OR error ILIKE '%yield%' OR error ILIKE '%reclaim%' OR error ILIKE '%cleared%' OR error ILIKE '%superseded%' OR error ILIKE '%abandoned%')`, jobID) + _, _ = pool.Exec(context.Background(), ` + UPDATE processing_job_products + SET status = 'pending', error = NULL, updated_at = now() + WHERE job_id = $1 AND status IN ('failed','cancelled') + AND processed_product_id IS NULL + AND (error ILIKE '%P0%' OR error ILIKE '%parked%' OR error ILIKE '%yield%' OR error ILIKE '%reclaim%' OR error ILIKE '%cleared%' OR error ILIKE '%superseded%' OR error ILIKE '%abandoned%')`, jobID) + } + } + }() + + _, _ = pool.Exec(ctx, ` + UPDATE processing_jobs + SET status = 'running', started_at = COALESCE(started_at, now()), updated_at = now() + WHERE id = $1`, jobID) + + runCtx, runCancel := context.WithTimeout(ctx, 3*time.Minute) + defer runCancel() + if err := pipeline.ProcessJob(runCtx, jobID); err != nil { + close(defend) + log.Fatalf("ProcessJob: %v", err) + } + close(defend) + + rows, err := pool.Query(ctx, ` + SELECT rp.gtin, pjp.status, COALESCE(pjp.error,''), pjp.processed_product_id IS NOT NULL + FROM processing_job_products pjp + JOIN raw_products rp ON rp.id = pjp.raw_product_id + WHERE pjp.job_id = $1 + ORDER BY rp.gtin`, jobID) + if err != nil { + log.Fatalf("query items: %v", err) + } + defer rows.Close() + fmt.Printf("JOB %s\n", jobID) + for rows.Next() { + var gtin, status, errMsg string + var hasPP bool + if err := rows.Scan(>in, &status, &errMsg, &hasPP); err != nil { + log.Fatal(err) + } + fmt.Printf("ITEM %s status=%s has_pp=%v err=%s\n", gtin, status, hasPP, errMsg) + } + out := os.Getenv("PROBE_OUT_DIR") + if out == "" { + out = `F:\laragon\www\_MY\descrybe-v2\.codehelper\_process_probe_electronics` + } + _ = os.MkdirAll(out, 0o755) + _ = os.WriteFile(out+string(os.PathSeparator)+"process_id.txt", []byte(jobID.String()+"\n"), 0o644) +} diff --git a/apps/api/cmd/_probe_batch_a_tmp/main.go b/apps/api/cmd/_probe_batch_a_tmp/main.go new file mode 100644 index 0000000..186316c --- /dev/null +++ b/apps/api/cmd/_probe_batch_a_tmp/main.go @@ -0,0 +1,192 @@ +package main + +import ( + "context" + "encoding/json" + "fmt" + "log" + "os" + "os/signal" + "strings" + "syscall" + "time" + + "github.com/descrybe/descrybe-v2/apps/api/internal/aiprompts" + "github.com/descrybe/descrybe-v2/apps/api/internal/config" + "github.com/descrybe/descrybe-v2/apps/api/internal/db" + "github.com/descrybe/descrybe-v2/apps/api/internal/eprel" + "github.com/descrybe/descrybe-v2/apps/api/internal/logredact" + "github.com/descrybe/descrybe-v2/apps/api/internal/processing" + "github.com/google/uuid" +) + +func main() { + log.SetOutput(logredact.Writer(os.Stderr)) + eans := []string{ + "4242005342488", + "4242005479948", + "4242002996967", + "4242005327065", + "4242005382330", + } + companyID := uuid.MustParse("2b3159b0-fc08-415b-b248-35ed02a6baab") + userID := uuid.MustParse("0ce3305c-d810-4b56-b1d4-3c1ed510db76") + + cfg, err := config.Load() + if err != nil { + log.Fatalf("config: %v", err) + } + ctx, cancel := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM) + defer cancel() + + pool, err := db.NewPool(ctx, cfg.DatabaseURL, db.PoolOptions{ + MaxConns: int32(cfg.DBMaxConns), + MinConns: int32(cfg.DBMinConns), + MaxConnLifetime: cfg.DBMaxConnLifetime, + MaxConnLifetimeJitter: cfg.DBMaxConnLifetimeJitter, + MaxConnIdleTime: cfg.DBMaxConnIdleTime, + HealthCheckPeriod: cfg.DBHealthCheckPeriod, + StatementTimeout: cfg.DBStatementTimeout, + }) + if err != nil { + log.Fatalf("db: %v", err) + } + defer pool.Close() + + out := os.Getenv("PROBE_OUT_DIR") + if out == "" { + out = `F:\laragon\www\_MY\descrybe-v2\.codehelper\_probe_batch_a` + } + _ = os.MkdirAll(out, 0o755) + + pipeline := processing.NewPipeline(pool) + pipeline.BatchSize = cfg.ProcessingBatchSize + pipeline.AI = nil + pipeline.Prompts = aiprompts.NewService(pool) + + mockKey := strings.TrimSpace(cfg.OpenAIAPIKey) + if mockKey == "" { + mockKey = "local-test" + } + mockBase := "http://127.0.0.1:18767/v1" + mockModel := "mock-llm" + log.Printf("forcing completer base=%s model=%s (ModeLabel=internal)", mockBase, mockModel) + client := processing.NewOpenAIClient(mockKey, mockBase, mockModel, cfg.ProcessingRPM, cfg.ProcessingMaxRetries) + client.ModeLabel = processing.AIProviderInternal + + eprelClient := eprel.NewClient(eprel.Options{Enabled: true, Timeout: 20 * time.Second}) + pipeline.Engine = &processing.Engine{ + Completer: client, + Vector: processing.NoopVectorCategorizer{}, + EPREL: eprelClient, + ProviderMode: processing.AIProviderInternal, + } + + rawIDs := make([]uuid.UUID, 0, len(eans)) + for _, ean := range eans { + var id uuid.UUID + err := pool.QueryRow(ctx, ` + SELECT id FROM raw_products + WHERE company_id = $1 AND gtin = $2`, companyID, ean).Scan(&id) + if err != nil { + log.Fatalf("raw product %s: %v", ean, err) + } + rawIDs = append(rawIDs, id) + log.Printf("ean=%s raw_id=%s", ean, id) + } + + // Self-heal only: do NOT fail/reclaim other jobs. + if n, err := processing.BackfillMissingMeta(ctx, pool, companyID); err != nil { + log.Printf("BackfillMissingMeta: %v", err) + } else { + log.Printf("BackfillMissingMeta updated=%d", n) + } + + jobs, err := pipeline.StartJob(ctx, companyID, userID, rawIDs, "full") + if err != nil { + log.Fatalf("StartJob: %v", err) + } + if len(jobs) == 0 { + log.Fatal("StartJob returned no jobs") + } + jobID := jobs[0].ID + log.Printf("job_id=%s", jobID) + _ = os.WriteFile(out+string(os.PathSeparator)+"process_id.txt", []byte(jobID.String()+"\n"), 0o644) + + defend := make(chan struct{}) + go func() { + t := time.NewTicker(500 * time.Millisecond) + defer t.Stop() + for { + select { + case <-defend: + return + case <-ctx.Done(): + return + case <-t.C: + // Restore ONLY our job if a sibling parked/yielded it. Never touch other jobs. + _, _ = pool.Exec(context.Background(), ` + UPDATE processing_jobs + SET status = 'running', error = NULL, updated_at = now() + WHERE id = $1 AND status IN ('failed','cancelled') + AND (error ILIKE '%P0%' OR error ILIKE '%parked%' OR error ILIKE '%yield%' + OR error ILIKE '%reclaim%' OR error ILIKE '%cleared%' OR error ILIKE '%superseded%' + OR error ILIKE '%exclusive%' OR error ILIKE '%batch_%' OR error ILIKE '%inproc%')`, jobID) + _, _ = pool.Exec(context.Background(), ` + UPDATE processing_job_products + SET status = 'pending', error = NULL, updated_at = now() + WHERE job_id = $1 AND status IN ('failed','cancelled') + AND processed_product_id IS NULL + AND (error ILIKE '%P0%' OR error ILIKE '%parked%' OR error ILIKE '%yield%' + OR error ILIKE '%reclaim%' OR error ILIKE '%cleared%' OR error ILIKE '%superseded%' + OR error ILIKE '%exclusive%' OR error ILIKE '%batch_%' OR error ILIKE '%inproc%')`, jobID) + } + } + }() + + _, _ = pool.Exec(ctx, ` + UPDATE processing_jobs + SET status = 'running', started_at = COALESCE(started_at, now()), updated_at = now() + WHERE id = $1`, jobID) + + runCtx, runCancel := context.WithTimeout(ctx, 12*time.Minute) + defer runCancel() + if err := pipeline.ProcessJob(runCtx, jobID); err != nil { + close(defend) + log.Fatalf("ProcessJob: %v", err) + } + close(defend) + + items, err := pipeline.LoadV1ProcessJobItems(ctx, companyID, jobID, "full") + if err != nil { + log.Fatalf("LoadV1ProcessJobItems: %v", err) + } + payload := map[string]any{ + "data": map[string]any{ + "process_id": jobID.String(), + "status": "COMPLETED", + "processing_type": "full", + "total_items": len(items), + "items": items, + }, + } + raw, err := json.MarshalIndent(payload, "", " ") + if err != nil { + log.Fatalf("marshal: %v", err) + } + if err := os.WriteFile(out+string(os.PathSeparator)+"final.json", raw, 0o644); err != nil { + log.Fatalf("write final: %v", err) + } + fmt.Printf("JOB %s items=%d\n", jobID, len(items)) + for _, it := range items { + ean, _ := it["ean"].(string) + st, _ := it["status"].(string) + title, _ := it["title"].(string) + if len(title) > 50 { + title = title[:50] + } + _, hasEprel := it["eprel"].(map[string]any) + ai, _ := it["ai_provider_mode"].(string) + fmt.Printf("ITEM %s status=%s eprel=%v ai=%s title=%s\n", ean, st, hasEprel, ai, title) + } +} diff --git a/apps/api/cmd/_thin_synth_probe_tmp/main.go b/apps/api/cmd/_thin_synth_probe_tmp/main.go new file mode 100644 index 0000000..901230f --- /dev/null +++ b/apps/api/cmd/_thin_synth_probe_tmp/main.go @@ -0,0 +1,173 @@ +package main + +import ( + "context" + "fmt" + "log" + "os" + "os/signal" + "strings" + "syscall" + "time" + + "github.com/descrybe/descrybe-v2/apps/api/internal/aiprompts" + "github.com/descrybe/descrybe-v2/apps/api/internal/config" + "github.com/descrybe/descrybe-v2/apps/api/internal/db" + "github.com/descrybe/descrybe-v2/apps/api/internal/logredact" + "github.com/descrybe/descrybe-v2/apps/api/internal/processing" + "github.com/google/uuid" +) + +// timeoutCompleter forces enhance deadline so synthesis path is exercised live. +type timeoutCompleter struct{} + +func (timeoutCompleter) Complete(context.Context, string, string) (processing.Completion, error) { + return processing.Completion{}, context.DeadlineExceeded +} + +func main() { + log.SetOutput(logredact.Writer(os.Stderr)) + ean := strings.TrimSpace(os.Getenv("PROBE_EAN")) + if ean == "" { + ean = "4719331854218" + } + companyID := uuid.MustParse("2b3159b0-fc08-415b-b248-35ed02a6baab") + userID := uuid.MustParse("0ce3305c-d810-4b56-b1d4-3c1ed510db76") + + cfg, err := config.Load() + if err != nil { + log.Fatalf("config: %v", err) + } + ctx, cancel := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM) + defer cancel() + + pool, err := db.NewPool(ctx, cfg.DatabaseURL, db.PoolOptions{ + MaxConns: int32(cfg.DBMaxConns), + MinConns: int32(cfg.DBMinConns), + MaxConnLifetime: cfg.DBMaxConnLifetime, + MaxConnLifetimeJitter: cfg.DBMaxConnLifetimeJitter, + MaxConnIdleTime: cfg.DBMaxConnIdleTime, + HealthCheckPeriod: cfg.DBHealthCheckPeriod, + StatementTimeout: cfg.DBStatementTimeout, + }) + if err != nil { + log.Fatalf("db: %v", err) + } + defer pool.Close() + + pipeline := processing.NewPipeline(pool) + pipeline.BatchSize = cfg.ProcessingBatchSize + pipeline.AI = nil + pipeline.Prompts = aiprompts.NewService(pool) + pipeline.Engine = &processing.Engine{ + Completer: timeoutCompleter{}, + Vector: processing.NoopVectorCategorizer{}, + EPREL: nil, + ProviderMode: processing.AIProviderInternal, + } + + var rawID uuid.UUID + if err := pool.QueryRow(ctx, ` + SELECT id FROM raw_products + WHERE company_id = $1 AND gtin = $2`, companyID, ean).Scan(&rawID); err != nil { + log.Fatalf("raw product %s: %v", ean, err) + } + log.Printf("ean=%s raw_id=%s", ean, rawID) + + _, _ = pool.Exec(ctx, ` + UPDATE processed_products + SET processed_description = '', description = '', + field_sources = COALESCE(field_sources, '{}'::jsonb) - 'enhance_input_hash', + updated_at = now() + WHERE company_id = $1 AND product_id = $2`, companyID, ean) + + _, _ = pool.Exec(ctx, ` + UPDATE processing_jobs + SET status = 'failed', error = 'yielded to thin-synth probe', updated_at = now() + WHERE status IN ('pending','running','processing')`) + + jobs, err := pipeline.StartJob(ctx, companyID, userID, []uuid.UUID{rawID}, "full") + if err != nil { + log.Fatalf("StartJob: %v", err) + } + if len(jobs) == 0 { + log.Fatal("StartJob returned no jobs") + } + jobID := jobs[0].ID + log.Printf("job_id=%s", jobID) + + defend := make(chan struct{}) + go func() { + t := time.NewTicker(300 * time.Millisecond) + defer t.Stop() + for { + select { + case <-defend: + return + case <-ctx.Done(): + return + case <-t.C: + _, _ = pool.Exec(context.Background(), ` + UPDATE processing_jobs + SET status = 'failed', error = 'yielded to thin-synth probe', updated_at = now() + WHERE status IN ('pending','running','processing') AND id <> $1`, jobID) + _, _ = pool.Exec(context.Background(), ` + UPDATE processing_jobs + SET status = 'running', error = NULL, updated_at = now() + WHERE id = $1 AND status IN ('failed','cancelled') + AND (error ILIKE '%parked%' OR error ILIKE '%yield%' OR error ILIKE '%reclaim%' + OR error ILIKE '%exclusive%' OR error ILIKE '%cleared%' OR error ILIKE '%P0%')`, jobID) + _, _ = pool.Exec(context.Background(), ` + UPDATE processing_job_products + SET status = 'pending', error = NULL, updated_at = now() + WHERE job_id = $1 AND status IN ('failed','cancelled') + AND processed_product_id IS NULL`, jobID) + } + } + }() + + _, _ = pool.Exec(ctx, ` + UPDATE processing_jobs + SET status = 'running', started_at = COALESCE(started_at, now()), updated_at = now() + WHERE id = $1`, jobID) + + runCtx, runCancel := context.WithTimeout(ctx, 2*time.Minute) + defer runCancel() + if err := pipeline.ProcessJob(runCtx, jobID); err != nil { + close(defend) + log.Fatalf("ProcessJob: %v", err) + } + close(defend) + + var name, desc, mode string + var descLen int + if err := pool.QueryRow(ctx, ` + SELECT COALESCE(name,''), COALESCE(processed_description,''), length(COALESCE(processed_description,'')), + COALESCE(ai_provider_mode,'') + FROM processed_products + WHERE company_id = $1 AND product_id = $2`, companyID, ean). + Scan(&name, &desc, &descLen, &mode); err != nil { + log.Fatalf("load processed: %v", err) + } + fmt.Printf("EAN %s\n", ean) + fmt.Printf("name=%q\n", name) + fmt.Printf("processed_description_len=%d\n", descLen) + fmt.Printf("processed_description=%q\n", desc) + fmt.Printf("ai_provider_mode=%q\n", mode) + echo := strings.EqualFold(strings.TrimSpace(desc), strings.TrimSpace(name)) + pass := descLen > 0 && !echo && !strings.Contains(strings.ToLower(desc), "ready for retail listing") + fmt.Printf("PASS_DESC_NE_TITLE=%v\n", pass) + if !pass { + os.Exit(1) + } + + out := os.Getenv("PROBE_OUT_DIR") + if out == "" { + out = `F:\laragon\www\_MY\descrybe-v2\.codehelper\_process_probe_thin_synth` + } + _ = os.MkdirAll(out, 0o755) + _ = os.WriteFile(out+string(os.PathSeparator)+"process_id.txt", []byte(jobID.String()+"\n"), 0o644) + _ = os.WriteFile(out+string(os.PathSeparator)+"score.txt", []byte(fmt.Sprintf( + "ean=%s\nname=%s\npd_len=%d\npd=%s\nmode=%s\nPASS=%v\n", ean, name, descLen, desc, mode, pass, + )), 0o644) +} diff --git a/apps/api/cmd/migrator/catalog_feeds.go b/apps/api/cmd/migrator/catalog_feeds.go index 00fef98..304a0dc 100644 --- a/apps/api/cmd/migrator/catalog_feeds.go +++ b/apps/api/cmd/migrator/catalog_feeds.go @@ -9,6 +9,7 @@ import ( "strconv" "strings" + "github.com/descrybe/descrybe-v2/apps/api/internal/processing" "github.com/google/uuid" "github.com/jackc/pgx/v5" "github.com/jackc/pgx/v5/pgxpool" @@ -48,6 +49,7 @@ func migrateCatalogAndFeeds( migrateProcessedProducts(ctx, mysqlDB, pg, companyMap, feedMap, rawMap, allow, report, dryRun) backfillProcessedDescriptionsFromMapped(ctx, pg, companyMap, allow, report, dryRun) backfillProcessedNamesFromMapped(ctx, pg, companyMap, allow, report, dryRun) + backfillProcessedCategoriesFromMapped(ctx, pg, companyMap, allow, report, dryRun) } if domains.has("feeds") { migrateExportFeeds(ctx, mysqlDB, pg, companyMap, feedMap, allow, report, dryRun) @@ -956,6 +958,40 @@ func backfillProcessedNamesFromMapped( } } +// backfillProcessedCategoriesFromMapped fills empty processed.category from mapped unique_ids. +func backfillProcessedCategoriesFromMapped( + ctx context.Context, + pg *pgxpool.Pool, + companyMap map[string]string, + allow map[string]bool, + report map[string]int, + dryRun bool, +) { + if dryRun || pg == nil { + return + } + total := 0 + for legacy, cid := range companyMap { + if len(allow) > 0 && !allow[legacy] { + continue + } + u, err := uuid.Parse(cid) + if err != nil { + continue + } + res, err := processing.BackfillProcessedCategoriesFromMapped(ctx, pg, u) + if err != nil { + log.Printf("processed category backfill company=%s: %v", cid, err) + continue + } + total += int(res.Updated) + } + if total > 0 { + report["processed_categories_backfilled"] = total + log.Printf("backfilled %d processed_products.category from mapped_data", total) + } +} + func migrateExportFeeds( ctx context.Context, mysqlDB *sql.DB, diff --git a/apps/api/cmd/repair-category-prompts/main.go b/apps/api/cmd/repair-category-prompts/main.go new file mode 100644 index 0000000..6656124 --- /dev/null +++ b/apps/api/cmd/repair-category-prompts/main.go @@ -0,0 +1,77 @@ +// Command repair-category-prompts replaces legacy A1 / Platform Demo categories.prompt +// HTML marketing formulas with aiprompts.CategoryEnhanceUserTemplate (JSON-compatible +// user overlay: {{name}} {{description}} {{attrs}} {{category}} {{language}}; stores +// both prompt->>'sl' and "*"). Does not touch title_template / description_template +// (formulas are appended as plain-text instructions at enhance render time). +// +// Usage (from apps/api): +// +// go run ./cmd/repair-category-prompts -dry-run +// go run ./cmd/repair-category-prompts -apply +// +// DATABASE_URL / -postgres required. Default is dry-run (count only). +package main + +import ( + "context" + "encoding/json" + "flag" + "fmt" + "log" + "os" + "strings" + "time" + + "github.com/descrybe/descrybe-v2/apps/api/internal/catalog" + "github.com/jackc/pgx/v5/pgxpool" +) + +func main() { + postgresURL := flag.String("postgres", os.Getenv("DATABASE_URL"), "Postgres URL (DATABASE_URL)") + dryRun := flag.Bool("dry-run", true, "count rows that would update (default true)") + apply := flag.Bool("apply", false, "write updates (implies not dry-run)") + flag.Parse() + + if strings.TrimSpace(*postgresURL) == "" { + log.Fatal("-postgres / DATABASE_URL is required") + } + doDryRun := *dryRun + if *apply { + doDryRun = false + } + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute) + defer cancel() + + pg, err := pgxpool.New(ctx, *postgresURL) + if err != nil { + log.Fatalf("postgres: %v", err) + } + defer pg.Close() + + // Always print dry-run counts first when applying, so operators see the delta. + preview, err := catalog.RepairA1DemoCategoryEnhancePrompts(ctx, pg, true) + if err != nil { + log.Fatalf("dry-run: %v", err) + } + printResult("dry-run", preview) + + if doDryRun { + return + } + + res, err := catalog.RepairA1DemoCategoryEnhancePrompts(ctx, pg, false) + if err != nil { + log.Fatalf("apply: %v", err) + } + printResult("apply", res) +} + +func printResult(phase string, res catalog.RepairCategoryEnhancePromptsResult) { + b, err := json.MarshalIndent(res, "", " ") + if err != nil { + log.Printf("%s: %+v", phase, res) + return + } + fmt.Printf("%s:\n%s\n", phase, string(b)) +} diff --git a/apps/api/cmd/seed-a1/attributes_backfill.go b/apps/api/cmd/seed-a1/attributes_backfill.go new file mode 100644 index 0000000..b6e8c87 --- /dev/null +++ b/apps/api/cmd/seed-a1/attributes_backfill.go @@ -0,0 +1,27 @@ +package main + +import ( + "context" + "fmt" + "log" + + "github.com/descrybe/descrybe-v2/apps/api/internal/processing" + "github.com/google/uuid" + "github.com/jackc/pgx/v5/pgxpool" +) + +// backfillA1ProcessedAttributes rewrites A1 processed_products.attributes and +// processed_attributes with AttrsForPersist + category_attributes allowlists +// (same rules as processOne). Use -mode backfill-attributes for polluted local DBs +// where poll already looked clean but the catalog row still stored feed junk. +func backfillA1ProcessedAttributes(ctx context.Context, pg *pgxpool.Pool, companyID uuid.UUID) (int64, error) { + n, err := processing.BackfillCompanyProcessedAttributes(ctx, pg, companyID) + if err != nil { + return 0, fmt.Errorf("A1 attributes backfill: %w", err) + } + return n, nil +} + +func logAttributesBackfillResult(updated int64) { + log.Printf("attributes backfill (A1): processed_updated=%d", updated) +} diff --git a/apps/api/cmd/seed-a1/category_prompts.go b/apps/api/cmd/seed-a1/category_prompts.go index ad73176..f241827 100644 --- a/apps/api/cmd/seed-a1/category_prompts.go +++ b/apps/api/cmd/seed-a1/category_prompts.go @@ -12,6 +12,7 @@ import ( "strings" "unicode" + "github.com/descrybe/descrybe-v2/apps/api/internal/aiprompts" "github.com/descrybe/descrybe-v2/apps/api/internal/security" "github.com/google/uuid" "github.com/jackc/pgx/v5/pgxpool" @@ -28,8 +29,9 @@ type categoryPromptsFile struct { } type categoryPromptEntry struct { - Name string `json:"name"` - Prompt string `json:"prompt"` + Name string `json:"name"` + UniqueID string `json:"unique_id,omitempty"` + Prompt string `json:"prompt"` } var ( @@ -145,26 +147,43 @@ func applyCategoryPrompts(ctx context.Context, pg *pgxpool.Pool, companyID uuid. } byNorm := make(map[string]string, len(file.Entries)) - for _, e := range file.Entries { - name := strings.TrimSpace(e.Name) - prompt := prepareCategoryPrompt(e.Prompt) - if name == "" || prompt == "" { - out.Skipped++ - continue - } - key := normalizeCategoryName(name) - if key == "" { - out.Skipped++ - continue - } - byNorm[key] = prompt + byUnique := make(map[string]string, len(file.Entries)) + // Overlay sets the shared JSON-compatible enhance user template (not legacy HTML). + // Seed JSON selects which categories get a prompt (prefer unique_id, else name). + // Unique title/description formulas stay in title_template / description_template. + // Stored under "sl" + "*" so prompt->>'sl' and LangPromptAny both resolve; + // language stays via {{language}}. + canonical := prepareCategoryPrompt(aiprompts.CategoryEnhanceUserTemplate) + if canonical == "" { + return out, fmt.Errorf("CategoryEnhanceUserTemplate sanitized to empty") } - if len(byNorm) == 0 { + for _, e := range file.Entries { + uid := strings.TrimSpace(e.UniqueID) + name := strings.TrimSpace(e.Name) + if uid == "" && name == "" { + out.Skipped++ + continue + } + if uid != "" { + byUnique[strings.ToLower(uid)] = canonical + } + if name != "" { + key := normalizeCategoryName(name) + if key == "" { + if uid == "" { + out.Skipped++ + } + continue + } + byNorm[key] = canonical + } + } + if len(byNorm) == 0 && len(byUnique) == 0 { return out, fmt.Errorf("no usable category prompts in %s", promptsPath) } rows, err := pg.Query(ctx, ` - SELECT id, name + SELECT id, COALESCE(unique_id, ''), name FROM categories WHERE company_id = $1`, companyID) if err != nil { @@ -172,22 +191,34 @@ func applyCategoryPrompts(ctx context.Context, pg *pgxpool.Pool, companyID uuid. } defer rows.Close() - ids := make([]uuid.UUID, 0, len(byNorm)) - prompts := make([]string, 0, len(byNorm)) - matchedKeys := make(map[string]struct{}, len(byNorm)) + ids := make([]uuid.UUID, 0, len(byNorm)+len(byUnique)) + prompts := make([]string, 0, len(byNorm)+len(byUnique)) + matchedNorm := make(map[string]struct{}, len(byNorm)) + matchedUID := make(map[string]struct{}, len(byUnique)) for rows.Next() { var id uuid.UUID - var name string - if err := rows.Scan(&id, &name); err != nil { + var uniqueID, name string + if err := rows.Scan(&id, &uniqueID, &name); err != nil { return out, err } - key := normalizeCategoryName(name) - prompt, ok := byNorm[key] - if !ok { + prompt := "" + if uidKey := strings.ToLower(strings.TrimSpace(uniqueID)); uidKey != "" { + if p, ok := byUnique[uidKey]; ok { + prompt = p + matchedUID[uidKey] = struct{}{} + } + } + if prompt == "" { + key := normalizeCategoryName(name) + if p, ok := byNorm[key]; ok { + prompt = p + matchedNorm[key] = struct{}{} + } + } + if prompt == "" { continue } - matchedKeys[key] = struct{}{} ids = append(ids, id) prompts = append(prompts, prompt) } @@ -195,22 +226,28 @@ func applyCategoryPrompts(ctx context.Context, pg *pgxpool.Pool, companyID uuid. return out, err } + for key := range byUnique { + if _, ok := matchedUID[key]; !ok { + out.Unmatched = append(out.Unmatched, "uid:"+key) + } + } for key := range byNorm { - if _, ok := matchedKeys[key]; !ok { - out.Unmatched = append(out.Unmatched, key) + if _, ok := matchedNorm[key]; !ok { + // Name unmatched is noise when unique_id matched the same row; still report for seed hygiene. + out.Unmatched = append(out.Unmatched, "name:"+key) } } sort.Strings(out.Unmatched) if len(ids) == 0 { - return out, fmt.Errorf("no A1 categories matched any of %d seed prompts (check names)", len(byNorm)) + return out, fmt.Errorf("no A1 categories matched any of %d seed prompts (check unique_id/names)", len(byNorm)+len(byUnique)) } // Single parameterized batch update — company_id gate prevents cross-tenant writes. - // ASSUMPTION: A1 seed company content language is Slovenian ("sl"). + // sl + * = prompt->>'sl' and company.LangPromptAny ({{language}} at render). tag, err := pg.Exec(ctx, ` UPDATE categories AS c - SET prompt = jsonb_build_object('sl', v.prompt), updated_at = now() + SET prompt = jsonb_build_object('sl', v.prompt, '*', v.prompt), updated_at = now() FROM ( SELECT * FROM unnest($1::uuid[], $2::text[]) AS t(id, prompt) ) AS v diff --git a/apps/api/cmd/seed-a1/main.go b/apps/api/cmd/seed-a1/main.go index e74e5c3..c7e9133 100644 --- a/apps/api/cmd/seed-a1/main.go +++ b/apps/api/cmd/seed-a1/main.go @@ -7,11 +7,13 @@ // part of npm run seed:a1 (skipped on import + cleared after). Use // -mode recover-jobs only when restoring legacy job history from a MySQL dump. // -// After reimport, legacy per-category GPT prompts are overlaid from -// scripts/seed/a1-category-prompts.json (Name→Prompt export), matched by -// normalized category name, with v1 placeholders rewritten to {{name}} / -// {{description}}. Use -skip-category-prompts to skip, or -// -mode apply-category-prompts to overlay without a full wipe/reimport. +// After reimport, per-category enhance prompts are overlaid from +// scripts/seed/a1-category-prompts.json (Name list), matched by normalized +// category name, writing aiprompts.CategoryEnhanceUserTemplate (JSON-compatible; +// includes {{attrs}}/{{language}}). Legacy HTML bodies in that JSON are ignored. +// Use -skip-category-prompts to skip, or -mode apply-category-prompts to overlay +// without a full wipe/reimport. For polluted A1+Platform Demo prompts without +// reimport, run: go run ./cmd/repair-category-prompts -apply // // Categories: dump/archive store assignment on processed_products.category // (category unique_id). Feed mappings do not map category. After reimport, @@ -26,6 +28,9 @@ // exported/imported as-is (skip-processed must not strip them). For a // polluted local DB without a full wipe, use -mode backfill-categories. // Fixture EANs are purged from non-A1 tenants automatically. +// Processed attribute junk (poll clean, DB still has zavora/vzmetenje, …): use +// -mode backfill-attributes — same SanitizeProductAttributes + category allowlist +// as processOne/enhance. // // Usage: // @@ -34,6 +39,7 @@ // go run ./cmd/seed-a1 -mode apply-category-prompts -file ../../scripts/seed/a1-demo-data.sql.gz // go run ./cmd/seed-a1 -mode recover-jobs -mysql-dump path/to/descrybe_new.sql // go run ./cmd/seed-a1 -mode backfill-categories -mysql-dump path/to/descrybe_new.sql +// go run ./cmd/seed-a1 -mode backfill-attributes // // DATABASE_URL / -postgres required. // Day-to-day: `npm run seed:a1` (reimport + category prompts + mapped category backfill). Maintainer snapshot: `npm run seed:a1:export`. @@ -77,7 +83,7 @@ type tableSpec struct { func main() { postgresURL := flag.String("postgres", os.Getenv("DATABASE_URL"), "Postgres URL (DATABASE_URL)") - mode := flag.String("mode", "reimport", "export | reimport | recover-jobs | apply-category-prompts | backfill-categories") + mode := flag.String("mode", "reimport", "export | reimport | recover-jobs | apply-category-prompts | backfill-categories | backfill-attributes") file := flag.String("file", "", "gzipped seed archive path (required for export/reimport)") mysqlDump := flag.String("mysql-dump", os.Getenv("SEED_A1_MYSQL_DUMP"), "mysqldump path for recover-jobs / backfill-categories (or SEED_A1_MYSQL_DUMP)") company := flag.String("company", defaultA1CompanyID, "Postgres companies.id for A1") @@ -95,7 +101,8 @@ func main() { } modeVal := strings.ToLower(strings.TrimSpace(*mode)) - if modeVal != "recover-jobs" && modeVal != "apply-category-prompts" && modeVal != "backfill-categories" && strings.TrimSpace(*file) == "" { + needsFile := modeVal != "recover-jobs" && modeVal != "apply-category-prompts" && modeVal != "backfill-categories" && modeVal != "backfill-attributes" + if needsFile && strings.TrimSpace(*file) == "" { log.Fatal("-file is required (e.g. ../../scripts/seed/a1-demo-data.sql.gz)") } dumpPath := resolveMySQLDumpPath(*mysqlDump) @@ -184,13 +191,19 @@ func main() { } res.PurgedOtherRaw, res.PurgedOtherPP = rawN, ppN logCategoryBackfillResult(res, dumpPath) + case "backfill-attributes": + n, err := backfillA1ProcessedAttributes(ctx, pg, companyID) + if err != nil { + log.Fatalf("backfill-attributes: %v", err) + } + logAttributesBackfillResult(n) case "recover-jobs": if err := recoverJobsFromMySQLDump(ctx, pg, companyID, dumpPath); err != nil { log.Fatalf("recover-jobs: %v", err) } log.Printf("recovered A1 processing jobs into company %s from %s", companyID, dumpPath) default: - log.Fatalf("unknown -mode %q (want export|reimport|recover-jobs|apply-category-prompts|backfill-categories)", *mode) + log.Fatalf("unknown -mode %q (want export|reimport|recover-jobs|apply-category-prompts|backfill-categories|backfill-attributes)", *mode) } } diff --git a/apps/api/internal/aiprompts/apply_defaults.go b/apps/api/internal/aiprompts/apply_defaults.go new file mode 100644 index 0000000..24fb314 --- /dev/null +++ b/apps/api/internal/aiprompts/apply_defaults.go @@ -0,0 +1,74 @@ +package aiprompts + +import ( + "context" + + "github.com/descrybe/descrybe-v2/apps/api/internal/company" + "github.com/google/uuid" +) + +// productEnhanceUpsertLanguages returns ISO 639-1 codes to store product_enhance +// under. Never returns LangPromptAny ("*") — ai_prompt_templates.language has +// CHECK (language ~ '^[a-z]{2}$'). Categories.prompt JSON may still use "*". +func productEnhanceUpsertLanguages(contentLangs []string, primary string) []string { + primary = company.NormalizeLanguage(primary) + if primary == "" || primary == company.LangPromptAny || !company.IsAllowedLanguage(primary) { + primary = company.DefaultLanguage + } + out := make([]string, 0, len(contentLangs)+1) + seen := map[string]struct{}{} + add := func(code string) { + code = company.NormalizeLanguage(code) + if code == "" || code == company.LangPromptAny || !company.IsAllowedLanguage(code) { + return + } + if _, ok := seen[code]; ok { + return + } + seen[code] = struct{}{} + out = append(out, code) + } + add(primary) + for _, lang := range contentLangs { + add(lang) + } + if len(out) == 0 { + return []string{company.DefaultLanguage} + } + return out +} + +// ApplyBuiltInProductEnhance upserts BuiltInDefaults for product_enhance under +// the company primary language and each content language. Templates still use +// {{language}} at render time; Resolve falls back requested → primary → built-in. +// Do not store LangPromptAny ("*") in ai_prompt_templates (DB language CHECK). +func (s *Service) ApplyBuiltInProductEnhance(ctx context.Context, companyID uuid.UUID) (languagesApplied int, err error) { + def, ok := DefaultFor(KeyProductEnhance) + if !ok { + return 0, ErrInvalidKey + } + primary := company.LoadLanguage(ctx, s.Pool, companyID) + langs := productEnhanceUpsertLanguages( + company.LoadContentLanguages(ctx, s.Pool, companyID), + primary, + ) + enabled := true + items := make([]UpdateItem, 0, len(langs)) + for _, lang := range langs { + items = append(items, UpdateItem{ + Key: KeyProductEnhance, + Language: lang, + SystemTemplate: def.SystemTemplate, + UserTemplate: def.UserTemplate, + IsEnabled: &enabled, + }) + } + _, err = s.Update(ctx, companyID, UpdateInput{ + Language: langs[0], + Prompts: items, + }) + if err != nil { + return 0, err + } + return len(langs), nil +} diff --git a/apps/api/internal/aiprompts/apply_defaults_test.go b/apps/api/internal/aiprompts/apply_defaults_test.go new file mode 100644 index 0000000..01bb762 --- /dev/null +++ b/apps/api/internal/aiprompts/apply_defaults_test.go @@ -0,0 +1,71 @@ +package aiprompts + +import ( + "regexp" + "testing" + + "github.com/descrybe/descrybe-v2/apps/api/internal/company" +) + +var languageFmt = regexp.MustCompile(`^[a-z]{2}$`) + +func TestProductEnhanceUpsertLanguages_neverStar(t *testing.T) { + t.Parallel() + cases := []struct { + name string + contentLangs []string + primary string + want []string + }{ + { + name: "primary_only", + contentLangs: nil, + primary: "sl", + want: []string{"sl"}, + }, + { + name: "primary_plus_content", + contentLangs: []string{"sl", "en", "de"}, + primary: "sl", + want: []string{"sl", "en", "de"}, + }, + { + name: "drops_star_and_invalid", + contentLangs: []string{"*", "sl", "xx", "", "EN"}, + primary: "*", + want: []string{"en", "sl"}, + }, + { + name: "dedupes_primary_in_content", + contentLangs: []string{"sl", "sl", "en"}, + primary: "sl", + want: []string{"sl", "en"}, + }, + { + name: "empty_falls_back_default", + contentLangs: []string{"*", ""}, + primary: "", + want: []string{company.DefaultLanguage}, + }, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + got := productEnhanceUpsertLanguages(tc.contentLangs, tc.primary) + if len(got) != len(tc.want) { + t.Fatalf("len=%d want %d (%v)", len(got), len(tc.want), got) + } + for i := range got { + if got[i] != tc.want[i] { + t.Fatalf("got %v want %v", got, tc.want) + } + if !languageFmt.MatchString(got[i]) { + t.Fatalf("language %q fails ai_prompt_templates_language_fmt", got[i]) + } + if got[i] == company.LangPromptAny { + t.Fatalf("must not upsert LangPromptAny") + } + } + }) + } +} diff --git a/apps/api/internal/aiprompts/kinds.go b/apps/api/internal/aiprompts/kinds.go index 6bde54b..1dc5f12 100644 --- a/apps/api/internal/aiprompts/kinds.go +++ b/apps/api/internal/aiprompts/kinds.go @@ -1,5 +1,7 @@ package aiprompts +import "strings" + // Prompt keys stored in ai_prompt_templates.prompt_key. const ( KeyProductEnhance = "product_enhance" @@ -45,6 +47,32 @@ type DefaultTemplate struct { UserTemplate string `json:"user_template"` } +// CategoryEnhanceUserTemplate is the shared per-category (and built-in) enhance USER +// message. Includes {{name}} {{description}} {{attrs}} {{category}} {{language}}. +// Compatible with the enhance system JSON schema {"name","description"} — not a +// competing HTML marketing document. Title/description formulas stay in +// title_template / description_template and are appended at render time as plain +// text instructions (see processing.AppendFormulaConstraints). +// Used by local A1/Demo prompt repair and seed-a1 overlays. +const CategoryEnhanceUserTemplate = `Your reply is parsed as JSON {"name":"string","description":"string"} only (system schema). Write name and description in {{language}} (do not hardcode a language). +- name: short retail title; follow any Title formula constraints that follow; use Attrs +- description: prefer 1-3 factual paragraphs as ONE string; limited HTML (