greeneclipse/descrybe
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fixes

Browse Source
This commit is contained in:
greeneclipse
2026-08-16 16:57:36 +02:00
parent 96f8c1115c
commit 532d439c41
106 changed files with 10147 additions and 494 deletions
Download Patch File Download Diff File Expand all files Collapse all files
apps/api/cmd/seed-a1/attributes_backfill.go
+27
View File
@@ -0,0 +1,27 @@
package main
import (
"context"
"fmt"
"log"
"github.com/descrybe/descrybe-v2/apps/api/internal/processing"
"github.com/google/uuid"
"github.com/jackc/pgx/v5/pgxpool"
)
// backfillA1ProcessedAttributes rewrites A1 processed_products.attributes and
// processed_attributes with AttrsForPersist + category_attributes allowlists
// (same rules as processOne). Use -mode backfill-attributes for polluted local DBs
// where poll already looked clean but the catalog row still stored feed junk.
func backfillA1ProcessedAttributes(ctx context.Context, pg *pgxpool.Pool, companyID uuid.UUID) (int64, error) {
n, err := processing.BackfillCompanyProcessedAttributes(ctx, pg, companyID)
if err != nil {
return 0, fmt.Errorf("A1 attributes backfill: %w", err)
}
return n, nil
}
func logAttributesBackfillResult(updated int64) {
log.Printf("attributes backfill (A1): processed_updated=%d", updated)
}
apps/api/cmd/seed-a1/category_prompts.go
+68 -31
View File
@@ -12,6 +12,7 @@ import (
"strings"
"unicode"
"github.com/descrybe/descrybe-v2/apps/api/internal/aiprompts"
"github.com/descrybe/descrybe-v2/apps/api/internal/security"
"github.com/google/uuid"
"github.com/jackc/pgx/v5/pgxpool"
@@ -28,8 +29,9 @@ type categoryPromptsFile struct {
}
type categoryPromptEntry struct {
Name string `json:"name"`
Prompt string `json:"prompt"`
Name string `json:"name"`
UniqueID string `json:"unique_id,omitempty"`
Prompt string `json:"prompt"`
}
var (
@@ -145,26 +147,43 @@ func applyCategoryPrompts(ctx context.Context, pg *pgxpool.Pool, companyID uuid.
}
byNorm := make(map[string]string, len(file.Entries))
for _, e := range file.Entries {
name := strings.TrimSpace(e.Name)
prompt := prepareCategoryPrompt(e.Prompt)
if name == "" || prompt == "" {
out.Skipped++
continue
}
key := normalizeCategoryName(name)
if key == "" {
out.Skipped++
continue
}
byNorm[key] = prompt
byUnique := make(map[string]string, len(file.Entries))
// Overlay sets the shared JSON-compatible enhance user template (not legacy HTML).
// Seed JSON selects which categories get a prompt (prefer unique_id, else name).
// Unique title/description formulas stay in title_template / description_template.
// Stored under "sl" + "*" so prompt->>'sl' and LangPromptAny both resolve;
// language stays via {{language}}.
canonical := prepareCategoryPrompt(aiprompts.CategoryEnhanceUserTemplate)
if canonical == "" {
return out, fmt.Errorf("CategoryEnhanceUserTemplate sanitized to empty")
}
if len(byNorm) == 0 {
for _, e := range file.Entries {
uid := strings.TrimSpace(e.UniqueID)
name := strings.TrimSpace(e.Name)
if uid == "" && name == "" {
out.Skipped++
continue
}
if uid != "" {
byUnique[strings.ToLower(uid)] = canonical
}
if name != "" {
key := normalizeCategoryName(name)
if key == "" {
if uid == "" {
out.Skipped++
}
continue
}
byNorm[key] = canonical
}
}
if len(byNorm) == 0 && len(byUnique) == 0 {
return out, fmt.Errorf("no usable category prompts in %s", promptsPath)
}
rows, err := pg.Query(ctx, `
SELECT id, name
SELECT id, COALESCE(unique_id, ''), name
FROM categories
WHERE company_id = $1`, companyID)
if err != nil {
@@ -172,22 +191,34 @@ func applyCategoryPrompts(ctx context.Context, pg *pgxpool.Pool, companyID uuid.
}
defer rows.Close()
ids := make([]uuid.UUID, 0, len(byNorm))
prompts := make([]string, 0, len(byNorm))
matchedKeys := make(map[string]struct{}, len(byNorm))
ids := make([]uuid.UUID, 0, len(byNorm)+len(byUnique))
prompts := make([]string, 0, len(byNorm)+len(byUnique))
matchedNorm := make(map[string]struct{}, len(byNorm))
matchedUID := make(map[string]struct{}, len(byUnique))
for rows.Next() {
var id uuid.UUID
var name string
if err := rows.Scan(&id, &name); err != nil {
var uniqueID, name string
if err := rows.Scan(&id, &uniqueID, &name); err != nil {
return out, err
}
key := normalizeCategoryName(name)
prompt, ok := byNorm[key]
if !ok {
prompt := ""
if uidKey := strings.ToLower(strings.TrimSpace(uniqueID)); uidKey != "" {
if p, ok := byUnique[uidKey]; ok {
prompt = p
matchedUID[uidKey] = struct{}{}
}
}
if prompt == "" {
key := normalizeCategoryName(name)
if p, ok := byNorm[key]; ok {
prompt = p
matchedNorm[key] = struct{}{}
}
}
if prompt == "" {
continue
}
matchedKeys[key] = struct{}{}
ids = append(ids, id)
prompts = append(prompts, prompt)
}
@@ -195,22 +226,28 @@ func applyCategoryPrompts(ctx context.Context, pg *pgxpool.Pool, companyID uuid.
return out, err
}
for key := range byUnique {
if _, ok := matchedUID[key]; !ok {
out.Unmatched = append(out.Unmatched, "uid:"+key)
}
}
for key := range byNorm {
if _, ok := matchedKeys[key]; !ok {
out.Unmatched = append(out.Unmatched, key)
if _, ok := matchedNorm[key]; !ok {
// Name unmatched is noise when unique_id matched the same row; still report for seed hygiene.
out.Unmatched = append(out.Unmatched, "name:"+key)
}
}
sort.Strings(out.Unmatched)
if len(ids) == 0 {
return out, fmt.Errorf("no A1 categories matched any of %d seed prompts (check names)", len(byNorm))
return out, fmt.Errorf("no A1 categories matched any of %d seed prompts (check unique_id/names)", len(byNorm)+len(byUnique))
}
// Single parameterized batch update — company_id gate prevents cross-tenant writes.
// ASSUMPTION: A1 seed company content language is Slovenian ("sl").
// sl + * = prompt->>'sl' and company.LangPromptAny ({{language}} at render).
tag, err := pg.Exec(ctx, `
UPDATE categories AS c
SET prompt = jsonb_build_object('sl', v.prompt), updated_at = now()
SET prompt = jsonb_build_object('sl', v.prompt, '*', v.prompt), updated_at = now()
FROM (
SELECT * FROM unnest($1::uuid[], $2::text[]) AS t(id, prompt)
) AS v
apps/api/cmd/seed-a1/main.go
+21 -8
View File
@@ -7,11 +7,13 @@
// part of npm run seed:a1 (skipped on import + cleared after). Use
// -mode recover-jobs only when restoring legacy job history from a MySQL dump.
//
// After reimport, legacy per-category GPT prompts are overlaid from
// scripts/seed/a1-category-prompts.json (Name→Prompt export), matched by
// normalized category name, with v1 placeholders rewritten to {{name}} /
// {{description}}. Use -skip-category-prompts to skip, or
// -mode apply-category-prompts to overlay without a full wipe/reimport.
// After reimport, per-category enhance prompts are overlaid from
// scripts/seed/a1-category-prompts.json (Name list), matched by normalized
// category name, writing aiprompts.CategoryEnhanceUserTemplate (JSON-compatible;
// includes {{attrs}}/{{language}}). Legacy HTML bodies in that JSON are ignored.
// Use -skip-category-prompts to skip, or -mode apply-category-prompts to overlay
// without a full wipe/reimport. For polluted A1+Platform Demo prompts without
// reimport, run: go run ./cmd/repair-category-prompts -apply
//
// Categories: dump/archive store assignment on processed_products.category
// (category unique_id). Feed mappings do not map category. After reimport,
@@ -26,6 +28,9 @@
// exported/imported as-is (skip-processed must not strip them). For a
// polluted local DB without a full wipe, use -mode backfill-categories.
// Fixture EANs are purged from non-A1 tenants automatically.
// Processed attribute junk (poll clean, DB still has zavora/vzmetenje, …): use
// -mode backfill-attributes — same SanitizeProductAttributes + category allowlist
// as processOne/enhance.
//
// Usage:
//
@@ -34,6 +39,7 @@
// go run ./cmd/seed-a1 -mode apply-category-prompts -file ../../scripts/seed/a1-demo-data.sql.gz
// go run ./cmd/seed-a1 -mode recover-jobs -mysql-dump path/to/descrybe_new.sql
// go run ./cmd/seed-a1 -mode backfill-categories -mysql-dump path/to/descrybe_new.sql
// go run ./cmd/seed-a1 -mode backfill-attributes
//
// DATABASE_URL / -postgres required.
// Day-to-day: `npm run seed:a1` (reimport + category prompts + mapped category backfill). Maintainer snapshot: `npm run seed:a1:export`.
@@ -77,7 +83,7 @@ type tableSpec struct {
func main() {
postgresURL := flag.String("postgres", os.Getenv("DATABASE_URL"), "Postgres URL (DATABASE_URL)")
mode := flag.String("mode", "reimport", "export | reimport | recover-jobs | apply-category-prompts | backfill-categories")
mode := flag.String("mode", "reimport", "export | reimport | recover-jobs | apply-category-prompts | backfill-categories | backfill-attributes")
file := flag.String("file", "", "gzipped seed archive path (required for export/reimport)")
mysqlDump := flag.String("mysql-dump", os.Getenv("SEED_A1_MYSQL_DUMP"), "mysqldump path for recover-jobs / backfill-categories (or SEED_A1_MYSQL_DUMP)")
company := flag.String("company", defaultA1CompanyID, "Postgres companies.id for A1")
@@ -95,7 +101,8 @@ func main() {
}
modeVal := strings.ToLower(strings.TrimSpace(*mode))
if modeVal != "recover-jobs" && modeVal != "apply-category-prompts" && modeVal != "backfill-categories" && strings.TrimSpace(*file) == "" {
needsFile := modeVal != "recover-jobs" && modeVal != "apply-category-prompts" && modeVal != "backfill-categories" && modeVal != "backfill-attributes"
if needsFile && strings.TrimSpace(*file) == "" {
log.Fatal("-file is required (e.g. ../../scripts/seed/a1-demo-data.sql.gz)")
}
dumpPath := resolveMySQLDumpPath(*mysqlDump)
@@ -184,13 +191,19 @@ func main() {
}
res.PurgedOtherRaw, res.PurgedOtherPP = rawN, ppN
logCategoryBackfillResult(res, dumpPath)
case "backfill-attributes":
n, err := backfillA1ProcessedAttributes(ctx, pg, companyID)
if err != nil {
log.Fatalf("backfill-attributes: %v", err)
}
logAttributesBackfillResult(n)
case "recover-jobs":
if err := recoverJobsFromMySQLDump(ctx, pg, companyID, dumpPath); err != nil {
log.Fatalf("recover-jobs: %v", err)
}
log.Printf("recovered A1 processing jobs into company %s from %s", companyID, dumpPath)
default:
log.Fatalf("unknown -mode %q (want export|reimport|recover-jobs|apply-category-prompts|backfill-categories)", *mode)
log.Fatalf("unknown -mode %q (want export|reimport|recover-jobs|apply-category-prompts|backfill-categories|backfill-attributes)", *mode)
}
}