Initial commit of Descrybe v2 without local scratch artifacts.
Drop one-shot tmp/axe scripts and agent i18n scratch so the Gitea tree is deployable.
This commit is contained in:
@@ -0,0 +1,763 @@
|
||||
// Command seed-a1 exports or reimports the A1 Slovenija tenant (catalog, feeds,
|
||||
// mappings, raw products, settings) as a gzipped COPY archive.
|
||||
//
|
||||
// Source of truth for "correct mapped fields" is live Postgres (already migrated),
|
||||
// not the raw MySQL dump. Reimport wipes only the A1 company_id and preserves
|
||||
// other tenants. Processed products and processing jobs are intentionally NOT
|
||||
// part of npm run seed:a1 (skipped on import + cleared after). Use
|
||||
// -mode recover-jobs only when restoring legacy job history from a MySQL dump.
|
||||
//
|
||||
// After reimport, legacy per-category GPT prompts are overlaid from
|
||||
// scripts/seed/a1-category-prompts.json (Name→Prompt export), matched by
|
||||
// normalized category name, with v1 placeholders rewritten to {{name}} /
|
||||
// {{description}}. Use -skip-category-prompts to skip, or
|
||||
// -mode apply-category-prompts to overlay without a full wipe/reimport.
|
||||
//
|
||||
// Categories: dump/archive store assignment on processed_products.category
|
||||
// (category unique_id). Feed mappings do not map category. After reimport,
|
||||
// seed-a1 copies those codes into raw_products.mapped_data.category so the UI
|
||||
// coverage chip and later re-processing keep the assignment. Prefer
|
||||
// -mysql-dump / SEED_A1_MYSQL_DUMP; when unset, seed-a1 auto-detects common
|
||||
// dump paths (e.g. ~/Downloads/descrybe_new (1).sql). Dump backfill is
|
||||
// mapped-only — it does not create processed rows. Without a dump,
|
||||
// backfillMappedCategoriesFromProcessed is a no-op when processed was cleared
|
||||
// (archive mapped_data.category is still restored from COPY). Original
|
||||
// description + feed attributes live on raw_products.mapped_data and are
|
||||
// exported/imported as-is (skip-processed must not strip them). For a
|
||||
// polluted local DB without a full wipe, use -mode backfill-categories.
|
||||
// Fixture EANs are purged from non-A1 tenants automatically.
|
||||
//
|
||||
// Usage:
|
||||
//
|
||||
// go run ./cmd/seed-a1 -mode export -file ../../scripts/seed/a1-demo-data.sql.gz
|
||||
// go run ./cmd/seed-a1 -mode reimport -file ../../scripts/seed/a1-demo-data.sql.gz
|
||||
// go run ./cmd/seed-a1 -mode apply-category-prompts -file ../../scripts/seed/a1-demo-data.sql.gz
|
||||
// go run ./cmd/seed-a1 -mode recover-jobs -mysql-dump path/to/descrybe_new.sql
|
||||
// go run ./cmd/seed-a1 -mode backfill-categories -mysql-dump path/to/descrybe_new.sql
|
||||
//
|
||||
// DATABASE_URL / -postgres required.
|
||||
// Day-to-day: `npm run seed:a1` (reimport + category prompts + mapped category backfill). Maintainer snapshot: `npm run seed:a1:export`.
|
||||
// recover-jobs is opt-in only and will reintroduce job history.
|
||||
package main
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"context"
|
||||
"flag"
|
||||
"fmt"
|
||||
"io"
|
||||
"log"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"compress/gzip"
|
||||
|
||||
"github.com/descrybe/descrybe-v2/apps/api/internal/billing"
|
||||
"github.com/google/uuid"
|
||||
"github.com/jackc/pgx/v5"
|
||||
"github.com/jackc/pgx/v5/pgconn"
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
)
|
||||
|
||||
// Canonical Postgres id for the migrated A1 Slovenija tenant.
|
||||
const defaultA1CompanyID = "604f23a8-b66e-4b21-8b45-0d72b68f4790"
|
||||
|
||||
const archiveMagic = "# seed-a1 v1"
|
||||
|
||||
// tableSpec describes one archive section.
|
||||
// selectSQL must return columns matching cols (order matters for COPY).
|
||||
type tableSpec struct {
|
||||
name string
|
||||
cols string
|
||||
selectSQL string // may use $1 = company uuid
|
||||
scoped bool // true → DELETE WHERE company_id = $1 before load
|
||||
}
|
||||
|
||||
func main() {
|
||||
postgresURL := flag.String("postgres", os.Getenv("DATABASE_URL"), "Postgres URL (DATABASE_URL)")
|
||||
mode := flag.String("mode", "reimport", "export | reimport | recover-jobs | apply-category-prompts | backfill-categories")
|
||||
file := flag.String("file", "", "gzipped seed archive path (required for export/reimport)")
|
||||
mysqlDump := flag.String("mysql-dump", os.Getenv("SEED_A1_MYSQL_DUMP"), "mysqldump path for recover-jobs / backfill-categories (or SEED_A1_MYSQL_DUMP)")
|
||||
company := flag.String("company", defaultA1CompanyID, "Postgres companies.id for A1")
|
||||
categoryPrompts := flag.String("category-prompts", "", "A1 category prompts JSON (default: sibling a1-category-prompts.json next to -file)")
|
||||
skipCategoryPrompts := flag.Bool("skip-category-prompts", false, "skip overlay of legacy category prompts after reimport")
|
||||
skipCategoryBackfill := flag.Bool("skip-category-backfill", false, "skip mapped_data.category backfill after reimport")
|
||||
flag.Parse()
|
||||
|
||||
if strings.TrimSpace(*postgresURL) == "" {
|
||||
log.Fatal("-postgres / DATABASE_URL is required")
|
||||
}
|
||||
companyID, err := uuid.Parse(strings.TrimSpace(*company))
|
||||
if err != nil {
|
||||
log.Fatalf("-company: %v", err)
|
||||
}
|
||||
|
||||
modeVal := strings.ToLower(strings.TrimSpace(*mode))
|
||||
if modeVal != "recover-jobs" && modeVal != "apply-category-prompts" && modeVal != "backfill-categories" && strings.TrimSpace(*file) == "" {
|
||||
log.Fatal("-file is required (e.g. ../../scripts/seed/a1-demo-data.sql.gz)")
|
||||
}
|
||||
dumpPath := resolveMySQLDumpPath(*mysqlDump)
|
||||
if dumpPath != "" && strings.TrimSpace(*mysqlDump) == "" {
|
||||
log.Printf("auto-detected MySQL dump: %s", dumpPath)
|
||||
}
|
||||
if modeVal == "recover-jobs" && dumpPath == "" {
|
||||
log.Fatal("-mysql-dump or SEED_A1_MYSQL_DUMP is required for recover-jobs (or place descrybe_new.sql in Downloads)")
|
||||
}
|
||||
if modeVal == "backfill-categories" && dumpPath == "" {
|
||||
log.Fatal("-mysql-dump or SEED_A1_MYSQL_DUMP is required for backfill-categories (or place descrybe_new.sql in Downloads)")
|
||||
}
|
||||
promptsPath := strings.TrimSpace(*categoryPrompts)
|
||||
if promptsPath == "" {
|
||||
promptsPath = defaultCategoryPromptsPath(*file)
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 60*time.Minute)
|
||||
defer cancel()
|
||||
|
||||
pg, err := pgxpool.New(ctx, *postgresURL)
|
||||
if err != nil {
|
||||
log.Fatalf("postgres: %v", err)
|
||||
}
|
||||
defer pg.Close()
|
||||
|
||||
switch modeVal {
|
||||
case "export":
|
||||
if err := exportArchive(ctx, pg, companyID, *file); err != nil {
|
||||
log.Fatalf("export: %v", err)
|
||||
}
|
||||
log.Printf("exported A1 company %s → %s", companyID, *file)
|
||||
case "reimport", "import":
|
||||
if err := reimportArchive(ctx, pg, companyID, *file); err != nil {
|
||||
log.Fatalf("reimport: %v", err)
|
||||
}
|
||||
log.Printf("reimported A1 company %s from %s", companyID, *file)
|
||||
if !*skipCategoryPrompts {
|
||||
res, err := applyCategoryPrompts(ctx, pg, companyID, promptsPath)
|
||||
if err != nil {
|
||||
log.Fatalf("category prompts: %v", err)
|
||||
}
|
||||
logCategoryPromptResult(res, promptsPath)
|
||||
}
|
||||
if !*skipCategoryBackfill {
|
||||
if dumpPath != "" {
|
||||
res, err := backfillCategoriesFromMySQLDump(ctx, pg, companyID, dumpPath)
|
||||
if err != nil {
|
||||
log.Fatalf("category backfill: %v", err)
|
||||
}
|
||||
logCategoryBackfillResult(res, dumpPath)
|
||||
} else {
|
||||
n, err := backfillMappedCategoriesFromProcessed(ctx, pg, companyID)
|
||||
if err != nil {
|
||||
log.Fatalf("category backfill: %v", err)
|
||||
}
|
||||
log.Printf("mapped_data.category backfill from processed: updated=%d (A1 only; 0 if processed was cleared — place dump in Downloads or set SEED_A1_MYSQL_DUMP)", n)
|
||||
}
|
||||
}
|
||||
rawN, ppN, err := purgeA1FixtureEANsFromOtherTenants(ctx, pg, companyID)
|
||||
if err != nil {
|
||||
log.Fatalf("purge demo fixtures: %v", err)
|
||||
}
|
||||
if rawN > 0 || ppN > 0 {
|
||||
log.Printf("purged A1 fixture EANs from other tenants: raw=%d processed=%d", rawN, ppN)
|
||||
}
|
||||
if cov, err := measureMappedCoverage(ctx, pg, companyID); err != nil {
|
||||
log.Printf("mapped coverage: %v", err)
|
||||
} else {
|
||||
logMappedCoverage(cov, "post-reimport")
|
||||
}
|
||||
case "apply-category-prompts":
|
||||
res, err := applyCategoryPrompts(ctx, pg, companyID, promptsPath)
|
||||
if err != nil {
|
||||
log.Fatalf("category prompts: %v", err)
|
||||
}
|
||||
logCategoryPromptResult(res, promptsPath)
|
||||
case "backfill-categories":
|
||||
res, err := backfillCategoriesFromMySQLDump(ctx, pg, companyID, dumpPath)
|
||||
if err != nil {
|
||||
log.Fatalf("backfill-categories: %v", err)
|
||||
}
|
||||
rawN, ppN, err := purgeA1FixtureEANsFromOtherTenants(ctx, pg, companyID)
|
||||
if err != nil {
|
||||
log.Fatalf("purge demo fixtures: %v", err)
|
||||
}
|
||||
res.PurgedOtherRaw, res.PurgedOtherPP = rawN, ppN
|
||||
logCategoryBackfillResult(res, dumpPath)
|
||||
case "recover-jobs":
|
||||
if err := recoverJobsFromMySQLDump(ctx, pg, companyID, dumpPath); err != nil {
|
||||
log.Fatalf("recover-jobs: %v", err)
|
||||
}
|
||||
log.Printf("recovered A1 processing jobs into company %s from %s", companyID, dumpPath)
|
||||
default:
|
||||
log.Fatalf("unknown -mode %q (want export|reimport|recover-jobs|apply-category-prompts|backfill-categories)", *mode)
|
||||
}
|
||||
}
|
||||
|
||||
func tableSpecs(companyID uuid.UUID) []tableSpec {
|
||||
_ = companyID // reserved; SELECT SQLs bind $1 at export time
|
||||
return []tableSpec{
|
||||
{
|
||||
name: "plans",
|
||||
cols: "id,name,description,monthly_credits,yearly_credits,max_products,is_custom,term,created_at,updated_at,features,is_legacy",
|
||||
selectSQL: `SELECT p.id, p.name, p.description, p.monthly_credits, p.yearly_credits, p.max_products, p.is_custom, p.term, p.created_at, p.updated_at, p.features, p.is_legacy FROM plans p WHERE p.id IN (SELECT plan_id FROM company_plans WHERE company_id = $1)`,
|
||||
scoped: false,
|
||||
},
|
||||
{
|
||||
name: "companies",
|
||||
cols: "id,name,language,merge_products_by_gtin,legacy_company_id,created_at,updated_at,stripe_customer_id",
|
||||
selectSQL: `SELECT id, name, language, merge_products_by_gtin, legacy_company_id, created_at, updated_at, stripe_customer_id FROM companies WHERE id = $1`,
|
||||
scoped: false,
|
||||
},
|
||||
{
|
||||
name: "users",
|
||||
cols: "id,email,name,password_hash,must_set_password,email_verified_at,is_platform_admin,is_active,legacy_user_id,last_login_at,created_at,updated_at,staff_role",
|
||||
selectSQL: `SELECT u.id, u.email, u.name, u.password_hash, u.must_set_password, u.email_verified_at, u.is_platform_admin, u.is_active, u.legacy_user_id, u.last_login_at, u.created_at, u.updated_at, u.staff_role FROM users u WHERE u.id IN (SELECT user_id FROM memberships WHERE company_id = $1)`,
|
||||
scoped: false,
|
||||
},
|
||||
{
|
||||
name: "memberships",
|
||||
cols: "id,company_id,user_id,role,status,created_at,updated_at",
|
||||
selectSQL: `SELECT id, company_id, user_id, role, status, created_at, updated_at FROM memberships WHERE company_id = $1`,
|
||||
scoped: true,
|
||||
},
|
||||
{
|
||||
name: "company_settings",
|
||||
cols: "company_id,settings,updated_at",
|
||||
selectSQL: `SELECT company_id, settings, updated_at FROM company_settings WHERE company_id = $1`,
|
||||
scoped: true,
|
||||
},
|
||||
{
|
||||
name: "company_brand",
|
||||
cols: "company_id,voice_tone,dos,donts,primary_color,secondary_color,logo_url,preferred_terms,updated_at",
|
||||
selectSQL: `SELECT company_id, voice_tone, dos, donts, primary_color, secondary_color, logo_url, preferred_terms, updated_at FROM company_brand WHERE company_id = $1`,
|
||||
scoped: true,
|
||||
},
|
||||
{
|
||||
name: "company_plans",
|
||||
cols: "id,company_id,plan_id,is_active,billing_cycle_start,next_billing_date,contract_start_date,contract_end_date,custom_monthly_credits,total_credits_allocated,custom_max_products,contract_reference,notes,is_trial,trial_ends_at,trial_credits,created_at,updated_at,stripe_subscription_id,stripe_price_id",
|
||||
selectSQL: `SELECT id, company_id, plan_id, is_active, billing_cycle_start, next_billing_date, contract_start_date, contract_end_date, custom_monthly_credits, total_credits_allocated, custom_max_products, contract_reference, notes, is_trial, trial_ends_at, trial_credits, created_at, updated_at, stripe_subscription_id, stripe_price_id FROM company_plans WHERE company_id = $1`,
|
||||
scoped: true,
|
||||
},
|
||||
{
|
||||
name: "credit_balances",
|
||||
cols: "company_id,total_credits,used_credits,updated_at",
|
||||
selectSQL: `SELECT company_id, total_credits, used_credits, updated_at FROM credit_balances WHERE company_id = $1`,
|
||||
scoped: true,
|
||||
},
|
||||
{
|
||||
name: "api_keys",
|
||||
cols: "id,company_id,user_id,name,key_hash,key_prefix,last_used_at,revoked_at,created_at,updated_at",
|
||||
selectSQL: `SELECT id, company_id, user_id, name, key_hash, key_prefix, last_used_at, revoked_at, created_at, updated_at FROM api_keys WHERE company_id = $1`,
|
||||
scoped: true,
|
||||
},
|
||||
{
|
||||
name: "field_groups",
|
||||
cols: `id,company_id,name,description,"order",is_system,created_at,updated_at`,
|
||||
selectSQL: `SELECT id, company_id, name, description, "order", is_system, created_at, updated_at FROM field_groups WHERE company_id = $1`,
|
||||
scoped: true,
|
||||
},
|
||||
{
|
||||
name: "standard_fields",
|
||||
cols: "id,company_id,name,key,type,group_id,is_required,description,default_value,validation,is_system,created_at,updated_at,enabled,unit,sort_order,mapping_hints",
|
||||
selectSQL: `SELECT id, company_id, name, key, type, group_id, is_required, description, default_value, validation, is_system, created_at, updated_at, enabled, unit, sort_order, mapping_hints FROM standard_fields WHERE company_id = $1`,
|
||||
scoped: true,
|
||||
},
|
||||
{
|
||||
name: "structured_description_fields",
|
||||
cols: "id,company_id,field_key,type,created_at,updated_at",
|
||||
selectSQL: `SELECT id, company_id, field_key, type, created_at, updated_at FROM structured_description_fields WHERE company_id = $1`,
|
||||
scoped: true,
|
||||
},
|
||||
{
|
||||
name: "attributes",
|
||||
cols: "id,company_id,attribute_key,name,value_type,unit,example,parent_key,created_at,updated_at",
|
||||
selectSQL: `SELECT id, company_id, attribute_key, name, value_type, unit, example, parent_key, created_at, updated_at FROM attributes WHERE company_id = $1`,
|
||||
scoped: true,
|
||||
},
|
||||
{
|
||||
name: "categories",
|
||||
cols: "id,company_id,name,unique_id,parent_unique_id,path,level,position,is_active,description,prompt,metadata,config,title_template,description_template,created_at,updated_at",
|
||||
selectSQL: `SELECT id, company_id, name, unique_id, parent_unique_id, path, level, position, is_active, description, prompt, metadata, config, title_template, description_template, created_at, updated_at FROM categories WHERE company_id = $1`,
|
||||
scoped: true,
|
||||
},
|
||||
{
|
||||
name: "category_attributes",
|
||||
cols: "id,company_id,category_unique_id,attribute_id,required,created_at,updated_at",
|
||||
selectSQL: `SELECT id, company_id, category_unique_id, attribute_id, required, created_at, updated_at FROM category_attributes WHERE company_id = $1`,
|
||||
scoped: true,
|
||||
},
|
||||
{
|
||||
name: "custom_variables",
|
||||
cols: "id,company_id,name,value,description,created_at,updated_at",
|
||||
selectSQL: `SELECT id, company_id, name, value, description, created_at, updated_at FROM custom_variables WHERE company_id = $1`,
|
||||
scoped: true,
|
||||
},
|
||||
{
|
||||
name: "input_feeds",
|
||||
cols: "id,company_id,name,url,feed_type,status,sync_interval_minutes,last_synced_at,auth_config,options,created_at,updated_at",
|
||||
selectSQL: `SELECT id, company_id, name, url, feed_type, status, sync_interval_minutes, last_synced_at, auth_config, options, created_at, updated_at FROM input_feeds WHERE company_id = $1`,
|
||||
scoped: true,
|
||||
},
|
||||
{
|
||||
name: "feed_mappings",
|
||||
cols: "id,feed_id,company_id,version,mappings,is_active,created_at,updated_at",
|
||||
selectSQL: `SELECT id, feed_id, company_id, version, mappings, is_active, created_at, updated_at FROM feed_mappings WHERE company_id = $1`,
|
||||
scoped: true,
|
||||
},
|
||||
{
|
||||
name: "feed_tags",
|
||||
cols: "id,company_id,name,color,created_at",
|
||||
selectSQL: `SELECT id, company_id, name, color, created_at FROM feed_tags WHERE company_id = $1`,
|
||||
scoped: true,
|
||||
},
|
||||
{
|
||||
name: "feed_tag_mappings",
|
||||
cols: "feed_id,tag_id",
|
||||
selectSQL: `SELECT ftm.feed_id, ftm.tag_id FROM feed_tag_mappings ftm
|
||||
JOIN feed_tags ft ON ft.id = ftm.tag_id WHERE ft.company_id = $1`,
|
||||
scoped: false, // wiped via cascading / explicit join delete
|
||||
},
|
||||
{
|
||||
name: "files",
|
||||
cols: "id,company_id,user_id,name,path,content_type,size_bytes,status,metadata,created_at,updated_at",
|
||||
selectSQL: `SELECT id, company_id, user_id, name, path, content_type, size_bytes, status, metadata, created_at, updated_at FROM files WHERE company_id = $1`,
|
||||
scoped: true,
|
||||
},
|
||||
{
|
||||
name: "raw_products",
|
||||
cols: "id,company_id,gtin,feed_id,feed_ids,raw_data,mapped_data,sync_job_id,is_processed,processing_status,file_id,created_at,updated_at",
|
||||
// sync_job_id points at ephemeral feed_sync_jobs (not archived) — export as NULL.
|
||||
// A1 seed stays catalog-clean: never archive processing state on raw rows.
|
||||
selectSQL: `SELECT id, company_id, gtin, feed_id, feed_ids, raw_data, mapped_data, NULL::uuid AS sync_job_id, false AS is_processed, 'unprocessed' AS processing_status, file_id, created_at, updated_at FROM raw_products WHERE company_id = $1`,
|
||||
scoped: true,
|
||||
},
|
||||
// processed_products / processing_jobs / processing_job_products are intentionally
|
||||
// omitted from the A1 demo archive. Use -mode recover-jobs only when restoring
|
||||
// legacy job history from a MySQL dump (not part of npm run seed:a1).
|
||||
{
|
||||
name: "export_feeds",
|
||||
cols: "id,company_id,name,source_feed_id,format,public_token,template,filters,is_active,last_generated_at,created_at,updated_at",
|
||||
selectSQL: `SELECT id, company_id, name, source_feed_id, format, public_token, template, filters, is_active, last_generated_at, created_at, updated_at FROM export_feeds WHERE company_id = $1`,
|
||||
scoped: true,
|
||||
},
|
||||
{
|
||||
name: "woocommerce_configs",
|
||||
cols: "company_id,store_url,consumer_key,consumer_secret,is_enabled,sync_options,last_synced_at,last_test_at,last_test_status,created_at,updated_at",
|
||||
selectSQL: `SELECT company_id, store_url, consumer_key, consumer_secret, is_enabled, sync_options, last_synced_at, last_test_at, last_test_status, created_at, updated_at FROM woocommerce_configs WHERE company_id = $1`,
|
||||
scoped: true,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func exportTables(companyID uuid.UUID) []tableSpec {
|
||||
return tableSpecs(companyID)
|
||||
}
|
||||
|
||||
func exportArchive(ctx context.Context, pg *pgxpool.Pool, companyID uuid.UUID, outPath string) error {
|
||||
var name, legacy string
|
||||
err := pg.QueryRow(ctx, `
|
||||
SELECT name, COALESCE(legacy_company_id::text, '')
|
||||
FROM companies WHERE id = $1`, companyID).Scan(&name, &legacy)
|
||||
if err != nil {
|
||||
return fmt.Errorf("resolve company %s: %w (expected A1 Slovenija)", companyID, err)
|
||||
}
|
||||
if legacy != "" && !strings.EqualFold(legacy, billing.A1LegacyCompanyID) {
|
||||
log.Printf("warning: legacy_company_id=%s (canonical MySQL A1 is %s)", legacy, billing.A1LegacyCompanyID)
|
||||
}
|
||||
log.Printf("exporting company %s (%q legacy=%s)", companyID, name, legacy)
|
||||
|
||||
if err := os.MkdirAll(filepath.Dir(outPath), 0o755); err != nil {
|
||||
return err
|
||||
}
|
||||
tmp := outPath + ".tmp"
|
||||
f, err := os.Create(tmp)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
gz := gzip.NewWriter(f)
|
||||
bw := bufio.NewWriterSize(gz, 1<<20)
|
||||
|
||||
write := func(s string) error {
|
||||
_, err := bw.WriteString(s)
|
||||
return err
|
||||
}
|
||||
|
||||
if err := write(fmt.Sprintf("%s\n# company_id=%s\n# company_name=%s\n# legacy_company_id=%s\n# generated_at=%s\n",
|
||||
archiveMagic, companyID, sanitizeHeader(name), legacy, time.Now().UTC().Format(time.RFC3339))); err != nil {
|
||||
_ = f.Close()
|
||||
_ = os.Remove(tmp)
|
||||
return err
|
||||
}
|
||||
|
||||
conn, err := pg.Acquire(ctx)
|
||||
if err != nil {
|
||||
_ = f.Close()
|
||||
_ = os.Remove(tmp)
|
||||
return err
|
||||
}
|
||||
defer conn.Release()
|
||||
|
||||
for _, t := range exportTables(companyID) {
|
||||
log.Printf(" export %s…", t.name)
|
||||
if err := write(fmt.Sprintf("BEGIN_TABLE %s\nCOLUMNS %s\n", t.name, t.cols)); err != nil {
|
||||
_ = f.Close()
|
||||
_ = os.Remove(tmp)
|
||||
return err
|
||||
}
|
||||
// Bind company id into COPY subquery by substituting a validated uuid literal
|
||||
// (companyID already parsed). ReplaceAll so multi-$1 SELECTs stay correct.
|
||||
q := strings.ReplaceAll(t.selectSQL, "$1", "'"+companyID.String()+"'::uuid")
|
||||
copySQL := fmt.Sprintf("COPY (%s) TO STDOUT", q)
|
||||
tag, err := conn.Conn().PgConn().CopyTo(ctx, bw, copySQL)
|
||||
if err != nil {
|
||||
_ = f.Close()
|
||||
_ = os.Remove(tmp)
|
||||
return fmt.Errorf("copy %s: %w", t.name, err)
|
||||
}
|
||||
rows := tag.RowsAffected()
|
||||
if err := write(fmt.Sprintf("END_TABLE %s rows=%d\n", t.name, rows)); err != nil {
|
||||
_ = f.Close()
|
||||
_ = os.Remove(tmp)
|
||||
return err
|
||||
}
|
||||
log.Printf(" %s rows=%d", t.name, rows)
|
||||
}
|
||||
|
||||
if err := bw.Flush(); err != nil {
|
||||
_ = f.Close()
|
||||
_ = os.Remove(tmp)
|
||||
return err
|
||||
}
|
||||
if err := gz.Close(); err != nil {
|
||||
_ = f.Close()
|
||||
_ = os.Remove(tmp)
|
||||
return err
|
||||
}
|
||||
if err := f.Close(); err != nil {
|
||||
_ = os.Remove(tmp)
|
||||
return err
|
||||
}
|
||||
return os.Rename(tmp, outPath)
|
||||
}
|
||||
|
||||
func sanitizeHeader(s string) string {
|
||||
return strings.Map(func(r rune) rune {
|
||||
if r == '\n' || r == '\r' {
|
||||
return -1
|
||||
}
|
||||
return r
|
||||
}, s)
|
||||
}
|
||||
|
||||
func wipeA1Tenant(ctx context.Context, tx pgx.Tx, companyID uuid.UUID) error {
|
||||
// FK-safe deletes for A1 only. Users/plans/global rows are not deleted.
|
||||
stmts := []string{
|
||||
`DELETE FROM feed_tag_mappings WHERE tag_id IN (SELECT id FROM feed_tags WHERE company_id = $1)
|
||||
OR feed_id IN (SELECT id FROM input_feeds WHERE company_id = $1)`,
|
||||
// Jobs before products: job_products FK raw_products / processed_products.
|
||||
`DELETE FROM processing_job_products WHERE job_id IN (SELECT id FROM processing_jobs WHERE company_id = $1)`,
|
||||
`DELETE FROM processing_jobs WHERE company_id = $1`,
|
||||
`DELETE FROM processed_products WHERE company_id = $1`,
|
||||
`DELETE FROM raw_products WHERE company_id = $1`,
|
||||
`DELETE FROM export_feeds WHERE company_id = $1`,
|
||||
`DELETE FROM feed_mappings WHERE company_id = $1`,
|
||||
`DELETE FROM feed_sync_jobs WHERE company_id = $1`,
|
||||
`DELETE FROM input_feeds WHERE company_id = $1`,
|
||||
`DELETE FROM category_attributes WHERE company_id = $1`,
|
||||
`DELETE FROM categories WHERE company_id = $1`,
|
||||
`DELETE FROM attributes WHERE company_id = $1`,
|
||||
`DELETE FROM custom_variables WHERE company_id = $1`,
|
||||
`DELETE FROM standard_fields WHERE company_id = $1`,
|
||||
`DELETE FROM field_groups WHERE company_id = $1`,
|
||||
`DELETE FROM structured_description_fields WHERE company_id = $1`,
|
||||
`DELETE FROM feed_tags WHERE company_id = $1`,
|
||||
`DELETE FROM files WHERE company_id = $1`,
|
||||
`DELETE FROM schema_extraction_tasks WHERE company_id = $1`,
|
||||
`DELETE FROM tasks WHERE company_id = $1`,
|
||||
`DELETE FROM product_reviews WHERE company_id = $1`,
|
||||
`DELETE FROM woo_order_items WHERE company_id = $1`,
|
||||
`DELETE FROM woo_orders WHERE company_id = $1`,
|
||||
`DELETE FROM woocommerce_configs WHERE company_id = $1`,
|
||||
`DELETE FROM api_keys WHERE company_id = $1`,
|
||||
`DELETE FROM company_plans WHERE company_id = $1`,
|
||||
`DELETE FROM credit_balances WHERE company_id = $1`,
|
||||
`DELETE FROM company_brand WHERE company_id = $1`,
|
||||
`DELETE FROM company_settings WHERE company_id = $1`,
|
||||
`DELETE FROM memberships WHERE company_id = $1`,
|
||||
`DELETE FROM invites WHERE company_id = $1`,
|
||||
`DELETE FROM billing_cycles WHERE company_id = $1`,
|
||||
}
|
||||
for _, q := range stmts {
|
||||
if _, err := tx.Exec(ctx, q, companyID); err != nil {
|
||||
return fmt.Errorf("wipe: %s: %w", q, err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func reimportArchive(ctx context.Context, pg *pgxpool.Pool, companyID uuid.UUID, inPath string) error {
|
||||
f, err := os.Open(inPath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("open seed file: %w (expected committed scripts/seed/a1-demo-data.sql.gz; maintainers regenerate with npm run seed:a1:export)", err)
|
||||
}
|
||||
defer f.Close()
|
||||
|
||||
gz, err := gzip.NewReader(f)
|
||||
if err != nil {
|
||||
return fmt.Errorf("gzip: %w", err)
|
||||
}
|
||||
defer gz.Close()
|
||||
|
||||
sections, headerCompany, err := parseArchive(gz)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if headerCompany != "" && headerCompany != companyID.String() {
|
||||
return fmt.Errorf("archive company_id=%s does not match -company=%s", headerCompany, companyID)
|
||||
}
|
||||
|
||||
acq, err := pg.Acquire(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer acq.Release()
|
||||
|
||||
tx, err := acq.Begin(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer tx.Rollback(ctx)
|
||||
|
||||
log.Printf("wiping A1 tenant %s (other companies untouched)…", companyID)
|
||||
if err := wipeA1Tenant(ctx, tx, companyID); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
pgConn := acq.Conn().PgConn()
|
||||
|
||||
for _, sec := range sections {
|
||||
if skipA1ProcessingSeedTable(sec.name) {
|
||||
log.Printf(" skip %s (%d bytes) — A1 seed keeps zero processed/jobs", sec.name, len(sec.data))
|
||||
continue
|
||||
}
|
||||
log.Printf(" load %s (%d bytes)…", sec.name, len(sec.data))
|
||||
switch sec.name {
|
||||
case "plans":
|
||||
if err := upsertFromCopy(ctx, tx, pgConn, "plans", sec.cols, sec.data, `
|
||||
INSERT INTO plans AS p (`+sec.cols+`)
|
||||
SELECT `+sec.cols+` FROM tmp_seed_a1_plans
|
||||
ON CONFLICT (id) DO UPDATE SET
|
||||
name = EXCLUDED.name,
|
||||
description = EXCLUDED.description,
|
||||
monthly_credits = EXCLUDED.monthly_credits,
|
||||
yearly_credits = EXCLUDED.yearly_credits,
|
||||
max_products = EXCLUDED.max_products,
|
||||
is_custom = EXCLUDED.is_custom,
|
||||
term = EXCLUDED.term,
|
||||
features = EXCLUDED.features,
|
||||
is_legacy = EXCLUDED.is_legacy,
|
||||
updated_at = EXCLUDED.updated_at`); err != nil {
|
||||
return err
|
||||
}
|
||||
case "companies":
|
||||
if err := upsertFromCopy(ctx, tx, pgConn, "companies", sec.cols, sec.data, `
|
||||
INSERT INTO companies AS c (`+sec.cols+`)
|
||||
SELECT `+sec.cols+` FROM tmp_seed_a1_companies
|
||||
ON CONFLICT (id) DO UPDATE SET
|
||||
name = EXCLUDED.name,
|
||||
language = EXCLUDED.language,
|
||||
merge_products_by_gtin = EXCLUDED.merge_products_by_gtin,
|
||||
legacy_company_id = EXCLUDED.legacy_company_id,
|
||||
stripe_customer_id = EXCLUDED.stripe_customer_id,
|
||||
updated_at = EXCLUDED.updated_at`); err != nil {
|
||||
return err
|
||||
}
|
||||
case "users":
|
||||
if err := upsertFromCopy(ctx, tx, pgConn, "users", sec.cols, sec.data, `
|
||||
INSERT INTO users AS u (`+sec.cols+`)
|
||||
SELECT `+sec.cols+` FROM tmp_seed_a1_users
|
||||
ON CONFLICT (id) DO UPDATE SET
|
||||
email = EXCLUDED.email,
|
||||
name = EXCLUDED.name,
|
||||
password_hash = COALESCE(EXCLUDED.password_hash, u.password_hash),
|
||||
must_set_password = EXCLUDED.must_set_password,
|
||||
is_platform_admin = u.is_platform_admin OR EXCLUDED.is_platform_admin,
|
||||
is_active = EXCLUDED.is_active,
|
||||
staff_role = COALESCE(EXCLUDED.staff_role, u.staff_role),
|
||||
updated_at = EXCLUDED.updated_at`); err != nil {
|
||||
return err
|
||||
}
|
||||
case "feed_tag_mappings":
|
||||
if err := copyInto(ctx, pgConn, "feed_tag_mappings", sec.cols, sec.data); err != nil {
|
||||
return err
|
||||
}
|
||||
default:
|
||||
if err := copyInto(ctx, pgConn, sec.name, sec.cols, sec.data); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if err := tx.Commit(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err := clearA1ProcessingArtifacts(ctx, pg, companyID); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
var rawN, procN, feedN, mapN, jobN, jobProdN int
|
||||
_ = pg.QueryRow(ctx, `SELECT count(*) FROM raw_products WHERE company_id=$1`, companyID).Scan(&rawN)
|
||||
_ = pg.QueryRow(ctx, `SELECT count(*) FROM processed_products WHERE company_id=$1`, companyID).Scan(&procN)
|
||||
_ = pg.QueryRow(ctx, `SELECT count(*) FROM input_feeds WHERE company_id=$1`, companyID).Scan(&feedN)
|
||||
_ = pg.QueryRow(ctx, `SELECT count(*) FROM feed_mappings WHERE company_id=$1`, companyID).Scan(&mapN)
|
||||
_ = pg.QueryRow(ctx, `SELECT count(*) FROM processing_jobs WHERE company_id=$1`, companyID).Scan(&jobN)
|
||||
_ = pg.QueryRow(ctx, `SELECT count(*) FROM processing_job_products WHERE job_id IN (SELECT id FROM processing_jobs WHERE company_id=$1)`, companyID).Scan(&jobProdN)
|
||||
log.Printf("post-import counts: raw=%d processed=%d feeds=%d mappings=%d jobs=%d job_products=%d", rawN, procN, feedN, mapN, jobN, jobProdN)
|
||||
return nil
|
||||
}
|
||||
|
||||
// skipA1ProcessingSeedTable omits processed/job history from older archives so
|
||||
// npm run seed:a1 leaves A1 with a clean catalog (zero processed, zero jobs).
|
||||
func skipA1ProcessingSeedTable(name string) bool {
|
||||
switch name {
|
||||
case "processed_products", "processing_jobs", "processing_job_products", "tasks":
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
// clearA1ProcessingArtifacts deletes A1 processed products, jobs, and tasks and
|
||||
// resets raw processing flags. Safe to run after reimport (other companies untouched).
|
||||
func clearA1ProcessingArtifacts(ctx context.Context, pg *pgxpool.Pool, companyID uuid.UUID) error {
|
||||
tx, err := pg.Begin(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer tx.Rollback(ctx)
|
||||
|
||||
stmts := []string{
|
||||
`DELETE FROM processing_job_products WHERE job_id IN (SELECT id FROM processing_jobs WHERE company_id = $1)`,
|
||||
`DELETE FROM processing_jobs WHERE company_id = $1`,
|
||||
`DELETE FROM processed_products WHERE company_id = $1`,
|
||||
`DELETE FROM tasks WHERE company_id = $1`,
|
||||
`UPDATE raw_products
|
||||
SET processing_status = 'unprocessed', is_processed = false, updated_at = now()
|
||||
WHERE company_id = $1
|
||||
AND (COALESCE(processing_status, '') NOT IN ('', 'unprocessed') OR is_processed IS TRUE)`,
|
||||
}
|
||||
for _, q := range stmts {
|
||||
if _, err := tx.Exec(ctx, q, companyID); err != nil {
|
||||
return fmt.Errorf("clear processing artifacts: %w", err)
|
||||
}
|
||||
}
|
||||
if err := tx.Commit(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
log.Printf("A1 processing artifacts cleared for company %s (processed=0 jobs=0)", companyID)
|
||||
return nil
|
||||
}
|
||||
|
||||
type archiveSection struct {
|
||||
name string
|
||||
cols string
|
||||
data []byte
|
||||
}
|
||||
|
||||
func parseArchive(r io.Reader) ([]archiveSection, string, error) {
|
||||
br := bufio.NewReaderSize(r, 1<<20)
|
||||
var headerCompany string
|
||||
var sections []archiveSection
|
||||
var cur *archiveSection
|
||||
var buf strings.Builder
|
||||
|
||||
flush := func() {
|
||||
if cur == nil {
|
||||
return
|
||||
}
|
||||
cur.data = []byte(buf.String())
|
||||
sections = append(sections, *cur)
|
||||
cur = nil
|
||||
buf.Reset()
|
||||
}
|
||||
|
||||
lineNo := 0
|
||||
for {
|
||||
line, err := br.ReadString('\n')
|
||||
if len(line) > 0 {
|
||||
lineNo++
|
||||
trimmedRight := strings.TrimRight(line, "\r\n")
|
||||
if cur == nil {
|
||||
s := strings.TrimSpace(trimmedRight)
|
||||
if lineNo == 1 && !strings.HasPrefix(s, "# seed-a1") {
|
||||
return nil, "", fmt.Errorf("bad magic on line 1: %q", s)
|
||||
}
|
||||
if strings.HasPrefix(s, "# company_id=") {
|
||||
headerCompany = strings.TrimPrefix(s, "# company_id=")
|
||||
}
|
||||
if strings.HasPrefix(s, "BEGIN_TABLE ") {
|
||||
flush()
|
||||
cur = &archiveSection{name: strings.TrimSpace(strings.TrimPrefix(s, "BEGIN_TABLE "))}
|
||||
buf.Reset()
|
||||
}
|
||||
continue
|
||||
}
|
||||
// inside table
|
||||
if strings.HasPrefix(trimmedRight, "COLUMNS ") && cur.cols == "" {
|
||||
cur.cols = strings.TrimSpace(strings.TrimPrefix(trimmedRight, "COLUMNS "))
|
||||
continue
|
||||
}
|
||||
if strings.HasPrefix(trimmedRight, "END_TABLE ") {
|
||||
flush()
|
||||
continue
|
||||
}
|
||||
buf.WriteString(line)
|
||||
continue
|
||||
}
|
||||
if err == io.EOF {
|
||||
break
|
||||
}
|
||||
if err != nil {
|
||||
return nil, "", err
|
||||
}
|
||||
}
|
||||
flush()
|
||||
if len(sections) == 0 {
|
||||
return nil, "", fmt.Errorf("archive contained no tables")
|
||||
}
|
||||
return sections, headerCompany, nil
|
||||
}
|
||||
|
||||
func copyInto(ctx context.Context, pgConn *pgconn.PgConn, table, cols string, data []byte) error {
|
||||
if len(strings.TrimSpace(string(data))) == 0 {
|
||||
return nil
|
||||
}
|
||||
sql := fmt.Sprintf("COPY %s (%s) FROM STDIN", table, cols)
|
||||
_, err := pgConn.CopyFrom(ctx, strings.NewReader(string(data)), sql)
|
||||
if err != nil {
|
||||
return fmt.Errorf("COPY %s: %w", table, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func upsertFromCopy(ctx context.Context, tx pgx.Tx, pgConn *pgconn.PgConn, table, cols string, data []byte, mergeSQL string) error {
|
||||
tmp := "tmp_seed_a1_" + table
|
||||
if _, err := tx.Exec(ctx, fmt.Sprintf(`DROP TABLE IF EXISTS %s`, tmp)); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := tx.Exec(ctx, fmt.Sprintf(`CREATE TEMP TABLE %s (LIKE %s INCLUDING DEFAULTS) ON COMMIT DROP`, tmp, table)); err != nil {
|
||||
return err
|
||||
}
|
||||
if len(strings.TrimSpace(string(data))) > 0 {
|
||||
sql := fmt.Sprintf("COPY %s (%s) FROM STDIN", tmp, cols)
|
||||
if _, err := pgConn.CopyFrom(ctx, strings.NewReader(string(data)), sql); err != nil {
|
||||
return fmt.Errorf("COPY temp %s: %w", table, err)
|
||||
}
|
||||
}
|
||||
if _, err := tx.Exec(ctx, mergeSQL); err != nil {
|
||||
return fmt.Errorf("merge %s: %w", table, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
Reference in New Issue
Block a user