Initial commit of Descrybe v2 without local scratch artifacts.
Drop one-shot tmp/axe scripts and agent i18n scratch so the Gitea tree is deployable.
This commit is contained in:
@@ -0,0 +1,99 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"log"
|
||||
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
)
|
||||
|
||||
// applyPlatformAdmins maps legacy admin_users → users.is_platform_admin.
|
||||
// Matching prefers remapped Clerk/legacy user_id, then email. Never creates orphan admin rows.
|
||||
// Company-admin memberships (member→admin) are a separate post-load step:
|
||||
// see runMembershipRoleRepair (-list-member-memberships / -promote-company-admins).
|
||||
func applyPlatformAdmins(
|
||||
ctx context.Context,
|
||||
mysqlDB *sql.DB,
|
||||
pg *pgxpool.Pool,
|
||||
userMap map[string]string,
|
||||
report map[string]int,
|
||||
dryRun bool,
|
||||
) {
|
||||
if !mysqlTableExists(ctx, mysqlDB, "admin_users") {
|
||||
log.Printf("admin_users skipped: table missing")
|
||||
return
|
||||
}
|
||||
|
||||
// Legacy shape: user_id (Clerk text) + email. Column presence varies by dump age.
|
||||
hasUserID := mysqlColumnExists(ctx, mysqlDB, "admin_users", "user_id")
|
||||
hasEmail := mysqlColumnExists(ctx, mysqlDB, "admin_users", "email")
|
||||
if !hasUserID && !hasEmail {
|
||||
log.Printf("admin_users skipped: no user_id/email columns")
|
||||
return
|
||||
}
|
||||
|
||||
q := `SELECT `
|
||||
switch {
|
||||
case hasUserID && hasEmail:
|
||||
q += `COALESCE(user_id, ''), COALESCE(email, '') FROM admin_users`
|
||||
case hasUserID:
|
||||
q += `user_id, '' FROM admin_users`
|
||||
default:
|
||||
q += `'', email FROM admin_users`
|
||||
}
|
||||
|
||||
rows, err := mysqlDB.QueryContext(ctx, q)
|
||||
if err != nil {
|
||||
log.Printf("admin_users skipped: %v", err)
|
||||
return
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
for rows.Next() {
|
||||
var legacyUserID, email string
|
||||
if err := rows.Scan(&legacyUserID, &email); err != nil {
|
||||
report["admin_users_skipped"]++
|
||||
continue
|
||||
}
|
||||
pgUserID, ok := userMap[legacyUserID]
|
||||
if !ok && email != "" {
|
||||
// Resolve via email already loaded into Postgres (or dry-run map miss).
|
||||
if dryRun {
|
||||
report["admin_users_unmatched"]++
|
||||
continue
|
||||
}
|
||||
var id string
|
||||
err := pg.QueryRow(ctx, `SELECT id::text FROM users WHERE lower(email) = lower($1)`, email).Scan(&id)
|
||||
if err != nil {
|
||||
report["admin_users_unmatched"]++
|
||||
continue
|
||||
}
|
||||
pgUserID = id
|
||||
}
|
||||
if pgUserID == "" {
|
||||
report["admin_users_unmatched"]++
|
||||
continue
|
||||
}
|
||||
if dryRun {
|
||||
report["admin_users"]++
|
||||
continue
|
||||
}
|
||||
tag, err := pg.Exec(ctx, `
|
||||
UPDATE users
|
||||
SET is_platform_admin = true,
|
||||
staff_role = COALESCE(staff_role, 'admin'),
|
||||
updated_at = now()
|
||||
WHERE id = $1::uuid`, pgUserID)
|
||||
if err != nil {
|
||||
log.Printf("admin_users update %s: %v", legacyUserID, err)
|
||||
report["admin_users_skipped"]++
|
||||
continue
|
||||
}
|
||||
if tag.RowsAffected() == 0 {
|
||||
report["admin_users_unmatched"]++
|
||||
continue
|
||||
}
|
||||
report["admin_users"]++
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,158 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"log"
|
||||
"strings"
|
||||
|
||||
"github.com/descrybe/descrybe-v2/apps/api/internal/billing"
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
)
|
||||
|
||||
// runCompaniesWithoutPlansRepair lists and/or assigns plans for companies that
|
||||
// have no active company_plans row. Postgres-only; never deletes or overwrites
|
||||
// an existing active plan. Live writes require confirm=true (no blind assigns).
|
||||
func runCompaniesWithoutPlansRepair(postgresURL, planName string, listOnly, assign bool, dryRun, confirm bool) {
|
||||
if postgresURL == "" {
|
||||
log.Fatal("-postgres / DATABASE_URL is required for companies-without-plans tooling")
|
||||
}
|
||||
planName, err := normalizeAssignPlanName(planName, assign)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
if !listOnly && !assign {
|
||||
log.Fatal("pass -list-companies-without-plans and/or -assign-missing-plans")
|
||||
}
|
||||
if err := guardLiveMutation(assign, dryRun, confirm, "-assign-missing-plans"); err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
|
||||
ctx := context.Background()
|
||||
pg, err := pgxpool.New(ctx, postgresURL)
|
||||
if err != nil {
|
||||
log.Fatalf("postgres: %v", err)
|
||||
}
|
||||
defer pg.Close()
|
||||
|
||||
svc := &billing.Service{Pool: pg}
|
||||
if err := svc.EnsureDefaultPlans(ctx); err != nil {
|
||||
log.Fatalf("ensure default plans: %v", err)
|
||||
}
|
||||
|
||||
total, err := svc.CountCompaniesWithoutActivePlan(ctx)
|
||||
if err != nil {
|
||||
log.Fatalf("count companies without active plan: %v", err)
|
||||
}
|
||||
fmt.Printf("companies_without_active_plan: %d\n", total)
|
||||
|
||||
const page = 200
|
||||
offset := 0
|
||||
listed := 0
|
||||
assigned := 0
|
||||
skipped := 0
|
||||
a1Skipped := 0
|
||||
|
||||
for {
|
||||
rows, err := svc.ListCompaniesWithoutActivePlan(ctx, page, offset)
|
||||
if err != nil {
|
||||
log.Fatalf("list companies without active plan: %v", err)
|
||||
}
|
||||
if len(rows) == 0 {
|
||||
break
|
||||
}
|
||||
pageAssigned := 0
|
||||
for _, c := range rows {
|
||||
listed++
|
||||
a1 := billing.IsA1CohortCompany(c.LegacyCompanyID, c.Name)
|
||||
if listOnly || !assign {
|
||||
fmt.Printf(" %s\t%s\t%s\ta1=%v\n", c.ID, c.Name, c.Language, a1)
|
||||
}
|
||||
mutate, skipReason := decideAssignMissingPlan(assign, c.LegacyCompanyID, c.Name)
|
||||
if !mutate {
|
||||
if assign && skipReason != "" {
|
||||
fmt.Printf("skip\t%s\t%s\t%s\n", c.ID, c.Name, skipReason)
|
||||
skipped++
|
||||
a1Skipped++
|
||||
}
|
||||
continue
|
||||
}
|
||||
if dryRun {
|
||||
fmt.Printf("dry-run: would assign plan %q to company %s (%s)\n", planName, c.ID, c.Name)
|
||||
assigned++
|
||||
continue
|
||||
}
|
||||
ok, err := svc.AssignPlanByNameIfMissing(ctx, c.ID, planName)
|
||||
if err != nil {
|
||||
log.Printf("assign plan %q to company %s: %v", planName, c.ID, err)
|
||||
skipped++
|
||||
continue
|
||||
}
|
||||
if !ok {
|
||||
skipped++
|
||||
continue
|
||||
}
|
||||
fmt.Printf("assigned plan %q to company %s (%s)\n", planName, c.ID, c.Name)
|
||||
assigned++
|
||||
pageAssigned++
|
||||
}
|
||||
if len(rows) < page {
|
||||
break
|
||||
}
|
||||
if assign && !dryRun {
|
||||
// Live assigns shrink the result set, so restart from offset 0.
|
||||
// If this page assigned nothing (e.g. all A1 skips), advance offset
|
||||
// so we cannot spin forever on the same unassignable rows.
|
||||
if pageAssigned == 0 {
|
||||
offset += page
|
||||
} else {
|
||||
offset = 0
|
||||
}
|
||||
continue
|
||||
}
|
||||
offset += page
|
||||
}
|
||||
|
||||
if assign {
|
||||
fmt.Printf("listed=%d assigned=%d skipped=%d a1_skipped=%d dry_run=%v plan=%q\n", listed, assigned, skipped, a1Skipped, dryRun, planName)
|
||||
} else {
|
||||
fmt.Printf("listed=%d\n", listed)
|
||||
}
|
||||
}
|
||||
|
||||
// decideAssignMissingPlan is the assign gate used by dry-run and -confirm.
|
||||
// A1 cohort companies always skip (never get Free/other fallback), even when confirm=true.
|
||||
func decideAssignMissingPlan(assign bool, legacyCompanyID, companyName string) (mutate bool, skipReason string) {
|
||||
if !assign {
|
||||
return false, ""
|
||||
}
|
||||
if billing.IsA1CohortCompany(legacyCompanyID, companyName) {
|
||||
return false, "a1_cohort"
|
||||
}
|
||||
return true, ""
|
||||
}
|
||||
|
||||
func normalizeAssignPlanName(planName string, assign bool) (string, error) {
|
||||
planName = strings.TrimSpace(planName)
|
||||
if assign && planName == "" {
|
||||
return "", fmt.Errorf("-plan-name is required with -assign-missing-plans (e.g. Free)")
|
||||
}
|
||||
return planName, nil
|
||||
}
|
||||
|
||||
// guardLiveAssign is kept for tests; prefer guardLiveMutation for new call sites.
|
||||
func guardLiveAssign(assign, dryRun, confirm bool) error {
|
||||
return guardLiveMutation(assign, dryRun, confirm, "-assign-missing-plans")
|
||||
}
|
||||
|
||||
func resolveFallbackPlanID(ctx context.Context, pg *pgxpool.Pool, planName string) (int64, error) {
|
||||
planName = strings.TrimSpace(planName)
|
||||
if planName == "" {
|
||||
return 0, nil
|
||||
}
|
||||
svc := &billing.Service{Pool: pg}
|
||||
if err := svc.EnsureDefaultPlans(ctx); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return svc.PlanIDByName(ctx, planName)
|
||||
}
|
||||
@@ -0,0 +1,75 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/descrybe/descrybe-v2/apps/api/internal/billing"
|
||||
)
|
||||
|
||||
func TestDecideAssignMissingPlanSkipsA1(t *testing.T) {
|
||||
t.Parallel()
|
||||
mutate, reason := decideAssignMissingPlan(true, billing.A1LegacyCompanyID, "Anything")
|
||||
if mutate || reason != "a1_cohort" {
|
||||
t.Fatalf("A1 legacy id must never assign (even with -confirm): mutate=%v reason=%q", mutate, reason)
|
||||
}
|
||||
mutate, reason = decideAssignMissingPlan(true, "other-legacy-id", "A1 Slovenija")
|
||||
if !mutate || reason != "" {
|
||||
t.Fatalf("display name alone must not skip assign: mutate=%v reason=%q", mutate, reason)
|
||||
}
|
||||
mutate, reason = decideAssignMissingPlan(false, billing.A1LegacyCompanyID, "A1")
|
||||
if mutate || reason != "" {
|
||||
t.Fatalf("list-only: mutate=%v reason=%q", mutate, reason)
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolveFallbackPlanIDEmpty(t *testing.T) {
|
||||
t.Parallel()
|
||||
id, err := resolveFallbackPlanID(context.Background(), nil, " ")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if id != 0 {
|
||||
t.Fatalf("got %d, want 0", id)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNormalizeAssignPlanName(t *testing.T) {
|
||||
t.Parallel()
|
||||
got, err := normalizeAssignPlanName(" Free ", true)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got != "Free" {
|
||||
t.Fatalf("got %q", got)
|
||||
}
|
||||
_, err = normalizeAssignPlanName(" ", true)
|
||||
if err == nil || !strings.Contains(err.Error(), "-plan-name") {
|
||||
t.Fatalf("expected plan-name error, got %v", err)
|
||||
}
|
||||
got, err = normalizeAssignPlanName(" ", false)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got != "" {
|
||||
t.Fatalf("list-only allows empty plan name, got %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGuardLiveAssign(t *testing.T) {
|
||||
t.Parallel()
|
||||
if err := guardLiveAssign(false, false, false); err != nil {
|
||||
t.Fatalf("list-only: %v", err)
|
||||
}
|
||||
if err := guardLiveAssign(true, true, false); err != nil {
|
||||
t.Fatalf("dry-run: %v", err)
|
||||
}
|
||||
if err := guardLiveAssign(true, false, true); err != nil {
|
||||
t.Fatalf("confirm: %v", err)
|
||||
}
|
||||
err := guardLiveAssign(true, false, false)
|
||||
if err == nil || !strings.Contains(err.Error(), "no blind live writes") {
|
||||
t.Fatalf("expected blind-assign refusal, got %v", err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,181 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// MigratorConfig holds portable CLI options for MySQL → Postgres ETL.
|
||||
type MigratorConfig struct {
|
||||
MySQLDSN string
|
||||
PostgresURL string
|
||||
DryRun bool
|
||||
MapsDir string
|
||||
IDMapPath string
|
||||
ReportDir string
|
||||
FixturePath string
|
||||
Resume bool
|
||||
CompanyFilter []string // legacy company ids; empty = all
|
||||
Domains domainSet
|
||||
SkipPostImport bool
|
||||
EnsureDemo bool
|
||||
DemoEmail string
|
||||
DemoPassword string
|
||||
DemoName string
|
||||
LocalDemoCo string
|
||||
}
|
||||
|
||||
type domainSet map[string]bool
|
||||
|
||||
func parseDomains(raw string) domainSet {
|
||||
raw = strings.TrimSpace(strings.ToLower(raw))
|
||||
if raw == "" || raw == "all" {
|
||||
return domainSet{"all": true}
|
||||
}
|
||||
out := domainSet{}
|
||||
for _, p := range strings.Split(raw, ",") {
|
||||
p = strings.TrimSpace(p)
|
||||
if p == "" {
|
||||
continue
|
||||
}
|
||||
out[p] = true
|
||||
}
|
||||
if len(out) == 0 {
|
||||
return domainSet{"all": true}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func (d domainSet) has(name string) bool {
|
||||
if d == nil || d["all"] {
|
||||
return true
|
||||
}
|
||||
return d[name]
|
||||
}
|
||||
|
||||
func (d domainSet) String() string {
|
||||
if d == nil || d["all"] {
|
||||
return "all"
|
||||
}
|
||||
parts := make([]string, 0, len(d))
|
||||
for k := range d {
|
||||
parts = append(parts, k)
|
||||
}
|
||||
return strings.Join(parts, ",")
|
||||
}
|
||||
|
||||
func parseCompanyFilter(raw string) []string {
|
||||
raw = strings.TrimSpace(raw)
|
||||
if raw == "" {
|
||||
return nil
|
||||
}
|
||||
var out []string
|
||||
seen := map[string]bool{}
|
||||
for _, p := range strings.Split(raw, ",") {
|
||||
p = strings.TrimSpace(p)
|
||||
if p == "" || seen[p] {
|
||||
continue
|
||||
}
|
||||
seen[p] = true
|
||||
out = append(out, p)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func companyFilterSet(ids []string) map[string]bool {
|
||||
if len(ids) == 0 {
|
||||
return nil
|
||||
}
|
||||
m := make(map[string]bool, len(ids))
|
||||
for _, id := range ids {
|
||||
m[id] = true
|
||||
}
|
||||
return m
|
||||
}
|
||||
|
||||
func filterCompanies(rows []companyRow, allow map[string]bool) []companyRow {
|
||||
if allow == nil {
|
||||
return rows
|
||||
}
|
||||
out := make([]companyRow, 0, len(rows))
|
||||
for _, c := range rows {
|
||||
if allow[c.LegacyID] {
|
||||
out = append(out, c)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// mysqlCompanyFilter appends AND company_id IN (...) when a filter is set.
|
||||
// Values are bound as ? placeholders; the column path is validated and quoted.
|
||||
func mysqlCompanyFilter(column string, allow map[string]bool) (clause string, args []any) {
|
||||
if len(allow) == 0 {
|
||||
return "", nil
|
||||
}
|
||||
quotedCol, err := quoteMySQLIdentPath(column)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
ids := make([]string, 0, len(allow))
|
||||
for id := range allow {
|
||||
ids = append(ids, id)
|
||||
}
|
||||
placeholders := make([]string, len(ids))
|
||||
args = make([]any, len(ids))
|
||||
for i, id := range ids {
|
||||
placeholders[i] = "?"
|
||||
args[i] = id
|
||||
}
|
||||
return fmt.Sprintf(" AND %s IN (%s)", quotedCol, strings.Join(placeholders, ",")), args
|
||||
}
|
||||
|
||||
// MigrationRunReport is the portable JSON artifact written after each run.
|
||||
type MigrationRunReport struct {
|
||||
GeneratedAt string `json:"generated_at"`
|
||||
Mode string `json:"mode"`
|
||||
Domains string `json:"domains"`
|
||||
CompanyFilter []string `json:"company_filter,omitempty"`
|
||||
Resume bool `json:"resume"`
|
||||
Counts map[string]int `json:"counts"`
|
||||
Validation any `json:"validation,omitempty"`
|
||||
Demo *DemoReport `json:"demo,omitempty"`
|
||||
Notes []string `json:"notes,omitempty"`
|
||||
ElapsedMS int64 `json:"elapsed_ms"`
|
||||
}
|
||||
|
||||
// DemoReport documents the ensure-demo outcome (no password plaintext).
|
||||
type DemoReport struct {
|
||||
Email string `json:"email"`
|
||||
PasswordSet bool `json:"password_set"`
|
||||
UserID string `json:"user_id,omitempty"`
|
||||
PrimaryCompany string `json:"primary_company,omitempty"`
|
||||
PrimaryName string `json:"primary_company_name,omitempty"`
|
||||
Memberships int64 `json:"memberships_admin"`
|
||||
PlatformAdmin bool `json:"platform_admin"`
|
||||
Note string `json:"note,omitempty"`
|
||||
}
|
||||
|
||||
func newRunReport(cfg MigratorConfig, dryRun bool) *MigrationRunReport {
|
||||
mode := "live"
|
||||
if dryRun {
|
||||
mode = "dry-run"
|
||||
}
|
||||
return &MigrationRunReport{
|
||||
GeneratedAt: time.Now().UTC().Format(time.RFC3339),
|
||||
Mode: mode,
|
||||
Domains: cfg.Domains.String(),
|
||||
CompanyFilter: append([]string(nil), cfg.CompanyFilter...),
|
||||
Resume: cfg.Resume,
|
||||
Counts: map[string]int{},
|
||||
Notes: []string{
|
||||
"Clerk is excluded: users mapped by email only; no Clerk API.",
|
||||
"Legacy password hashes are never imported.",
|
||||
"API key secrets are not migrated; clients must mint new keys (seed-demo / ensure-demo for local).",
|
||||
"File blobs are metadata-only; resync object storage separately.",
|
||||
"Job history: domain jobs migrates processing_jobs (+ best-effort job_products) and tasks; tagged ai_provider_mode=migrated so retention keeps them.",
|
||||
"company_settings: language + merge_products only (domain settings); other legacy settings fields are not imported.",
|
||||
"woocommerce_configs: migrated from wc_* custom_fields when domain woo is enabled.",
|
||||
},
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,53 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestParseDomains(t *testing.T) {
|
||||
all := parseDomains("all")
|
||||
if !all.has("products") || !all.has("woo") {
|
||||
t.Fatalf("all should include every domain")
|
||||
}
|
||||
d := parseDomains("settings,formulas,tags")
|
||||
if d.has("products") {
|
||||
t.Fatalf("products should be excluded")
|
||||
}
|
||||
if !d.has("settings") || !d.has("formulas") || !d.has("tags") {
|
||||
t.Fatalf("expected settings/formulas/tags: %#v", d)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseCompanyFilter(t *testing.T) {
|
||||
ids := parseCompanyFilter(" a ,b, a ")
|
||||
if len(ids) != 2 || ids[0] != "a" || ids[1] != "b" {
|
||||
t.Fatalf("got %#v", ids)
|
||||
}
|
||||
set := companyFilterSet(ids)
|
||||
if !set["a"] || set["c"] {
|
||||
t.Fatalf("set %#v", set)
|
||||
}
|
||||
filtered := filterCompanies([]companyRow{{LegacyID: "a"}, {LegacyID: "c"}}, set)
|
||||
if len(filtered) != 1 || filtered[0].LegacyID != "a" {
|
||||
t.Fatalf("filtered %#v", filtered)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMysqlCompanyFilter(t *testing.T) {
|
||||
clause, args := mysqlCompanyFilter("company_id", map[string]bool{"x": true, "y": true})
|
||||
if clause == "" || len(args) != 2 {
|
||||
t.Fatalf("clause=%q args=%v", clause, args)
|
||||
}
|
||||
if !strings.Contains(clause, "`company_id`") || !strings.Contains(clause, "?") {
|
||||
t.Fatalf("expected quoted column and placeholders: %q", clause)
|
||||
}
|
||||
qual, qArgs := mysqlCompanyFilter("cf.company_id", map[string]bool{"a": true})
|
||||
if len(qArgs) != 1 || qual != " AND `cf`.`company_id` IN (?)" {
|
||||
t.Fatalf("qualified: clause=%q args=%v", qual, qArgs)
|
||||
}
|
||||
empty, emptyArgs := mysqlCompanyFilter("company_id", nil)
|
||||
if empty != "" || emptyArgs != nil {
|
||||
t.Fatalf("expected empty filter")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,150 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"github.com/descrybe/descrybe-v2/apps/api/internal/auth"
|
||||
"github.com/descrybe/descrybe-v2/apps/api/internal/billing"
|
||||
"github.com/google/uuid"
|
||||
"github.com/jackc/pgx/v5"
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
)
|
||||
|
||||
const defaultMigratorDemoCompany = "Platform Demo"
|
||||
|
||||
// ensureDemoUser upserts a local demo account (no Clerk), makes them platform admin,
|
||||
// and binds them only to a standalone Platform Demo company (never A1 / every tenant).
|
||||
func ensureDemoUser(
|
||||
ctx context.Context,
|
||||
pg *pgxpool.Pool,
|
||||
email, password, displayName, localDemoName string,
|
||||
dryRun bool,
|
||||
report map[string]int,
|
||||
) (*DemoReport, error) {
|
||||
emailNorm := strings.ToLower(strings.TrimSpace(email))
|
||||
if emailNorm == "" || password == "" {
|
||||
return nil, fmt.Errorf("demo email and password required")
|
||||
}
|
||||
if localDemoName == "" {
|
||||
localDemoName = defaultMigratorDemoCompany
|
||||
}
|
||||
if displayName == "" {
|
||||
displayName = "Demo User"
|
||||
}
|
||||
out := &DemoReport{
|
||||
Email: emailNorm,
|
||||
PasswordSet: true,
|
||||
PlatformAdmin: true,
|
||||
Note: "Password documented in docs/portable-mysql-pg-migration.md (not written to report JSON).",
|
||||
}
|
||||
if dryRun {
|
||||
report["demo_user"] = 1
|
||||
out.Note = "dry-run: demo user not written"
|
||||
out.PasswordSet = false
|
||||
return out, nil
|
||||
}
|
||||
|
||||
hash, err := auth.HashPassword(password)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
tx, err := pg.Begin(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer tx.Rollback(ctx)
|
||||
|
||||
var userID uuid.UUID
|
||||
err = tx.QueryRow(ctx, `
|
||||
INSERT INTO users (
|
||||
email, name, password_hash, must_set_password,
|
||||
is_platform_admin, is_active, updated_at
|
||||
) VALUES ($1, $2, $3, false, true, true, now())
|
||||
ON CONFLICT (email) DO UPDATE SET
|
||||
name = EXCLUDED.name,
|
||||
password_hash = EXCLUDED.password_hash,
|
||||
must_set_password = false,
|
||||
is_platform_admin = true,
|
||||
is_active = true,
|
||||
updated_at = now()
|
||||
RETURNING id`, emailNorm, displayName, hash).Scan(&userID)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("upsert demo user: %w", err)
|
||||
}
|
||||
out.UserID = userID.String()
|
||||
|
||||
demoCompanyID, demoName, err := ensureMigratorDemoCompany(ctx, tx, localDemoName)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out.PrimaryCompany = demoCompanyID.String()
|
||||
out.PrimaryName = demoName
|
||||
|
||||
ct, err := tx.Exec(ctx, `
|
||||
INSERT INTO memberships (company_id, user_id, role, status)
|
||||
VALUES ($1, $2, 'admin', 'active')
|
||||
ON CONFLICT (company_id, user_id) DO UPDATE
|
||||
SET role = 'admin', status = 'active', updated_at = now()`, demoCompanyID, userID)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("demo membership: %w", err)
|
||||
}
|
||||
if _, err := tx.Exec(ctx, `
|
||||
DELETE FROM memberships
|
||||
WHERE user_id = $1 AND company_id <> $2`, userID, demoCompanyID); err != nil {
|
||||
return nil, fmt.Errorf("remove non-demo memberships: %w", err)
|
||||
}
|
||||
out.Memberships = ct.RowsAffected()
|
||||
|
||||
if err := tx.Commit(ctx); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
report["demo_user"] = 1
|
||||
report["demo_memberships"] = int(out.Memberships)
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func ensureMigratorDemoCompany(ctx context.Context, tx pgx.Tx, name string) (uuid.UUID, string, error) {
|
||||
name = strings.TrimSpace(name)
|
||||
if name == "" {
|
||||
name = defaultMigratorDemoCompany
|
||||
}
|
||||
if strings.EqualFold(name, "A1 Slovenija") || strings.EqualFold(name, "A1") || strings.EqualFold(name, "Local Demo Co") {
|
||||
return uuid.Nil, "", fmt.Errorf("demo company name %q collides with A1 tenant — use %q", name, defaultMigratorDemoCompany)
|
||||
}
|
||||
|
||||
var id uuid.UUID
|
||||
err := tx.QueryRow(ctx, `
|
||||
SELECT c.id
|
||||
FROM companies c
|
||||
WHERE c.name = $1
|
||||
AND COALESCE(c.legacy_company_id, '') <> $2
|
||||
ORDER BY c.created_at ASC
|
||||
LIMIT 1`, name, billing.A1LegacyCompanyID).Scan(&id)
|
||||
if err == nil {
|
||||
if _, err := tx.Exec(ctx, `INSERT INTO company_settings (company_id) VALUES ($1) ON CONFLICT DO NOTHING`, id); err != nil {
|
||||
return uuid.Nil, "", err
|
||||
}
|
||||
if _, err := tx.Exec(ctx, `INSERT INTO credit_balances (company_id) VALUES ($1) ON CONFLICT DO NOTHING`, id); err != nil {
|
||||
return uuid.Nil, "", err
|
||||
}
|
||||
return id, name, nil
|
||||
}
|
||||
if err != pgx.ErrNoRows {
|
||||
return uuid.Nil, "", err
|
||||
}
|
||||
|
||||
err = tx.QueryRow(ctx, `INSERT INTO companies (name) VALUES ($1) RETURNING id`, name).Scan(&id)
|
||||
if err != nil {
|
||||
return uuid.Nil, "", err
|
||||
}
|
||||
if _, err := tx.Exec(ctx, `INSERT INTO company_settings (company_id) VALUES ($1) ON CONFLICT DO NOTHING`, id); err != nil {
|
||||
return uuid.Nil, "", err
|
||||
}
|
||||
if _, err := tx.Exec(ctx, `INSERT INTO credit_balances (company_id) VALUES ($1) ON CONFLICT DO NOTHING`, id); err != nil {
|
||||
return uuid.Nil, "", err
|
||||
}
|
||||
return id, name, nil
|
||||
}
|
||||
@@ -0,0 +1,109 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"encoding/json"
|
||||
"log"
|
||||
"strconv"
|
||||
|
||||
"github.com/google/uuid"
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
)
|
||||
|
||||
// migrateFiles copies file *metadata* only.
|
||||
//
|
||||
// Blobs strategy (cutover):
|
||||
// - Do NOT stream MySQL/local blob bytes through the migrator.
|
||||
// - Preserve legacy url/path in files.path and legacy id in metadata._legacy_file_id.
|
||||
// - Operators re-attach object storage / local volumes under the same relative paths,
|
||||
// or run a separate rsync/S3 sync keyed by legacy id after DNS freeze.
|
||||
// - raw_products.file_id is left unset until a follow-up remapper exists.
|
||||
func migrateFiles(
|
||||
ctx context.Context,
|
||||
mysqlDB *sql.DB,
|
||||
pg *pgxpool.Pool,
|
||||
companyMap, userMap, fileMap map[string]string,
|
||||
allow map[string]bool,
|
||||
report map[string]int,
|
||||
dryRun bool,
|
||||
) {
|
||||
if !mysqlTableExists(ctx, mysqlDB, "files") {
|
||||
log.Printf("files skipped: table missing (blobs strategy: metadata-only when present)")
|
||||
return
|
||||
}
|
||||
|
||||
clause, cargs := mysqlCompanyFilter("company_id", allow)
|
||||
rows, err := mysqlDB.QueryContext(ctx, `
|
||||
SELECT id, company_id, COALESCE(user_id, ''), file_name,
|
||||
COALESCE(file_type, ''), COALESCE(file_size, 0),
|
||||
COALESCE(status, 'uploaded'), url, metadata
|
||||
FROM files WHERE 1=1`+clause, cargs...)
|
||||
if err != nil {
|
||||
// Older dumps may lack metadata/url/status.
|
||||
rows, err = mysqlDB.QueryContext(ctx, `
|
||||
SELECT id, company_id, COALESCE(user_id, ''), file_name,
|
||||
COALESCE(file_type, ''), COALESCE(file_size, 0),
|
||||
'uploaded', NULL, NULL
|
||||
FROM files WHERE 1=1`+clause, cargs...)
|
||||
}
|
||||
if err != nil {
|
||||
log.Printf("files skipped: %v", err)
|
||||
return
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
for rows.Next() {
|
||||
var legacyID int64
|
||||
var companyLegacy, userLegacy, name, fileType, status string
|
||||
var size int64
|
||||
var url sql.NullString
|
||||
var metadata []byte
|
||||
if err := rows.Scan(&legacyID, &companyLegacy, &userLegacy, &name, &fileType, &size, &status, &url, &metadata); err != nil {
|
||||
report["files_skipped"]++
|
||||
continue
|
||||
}
|
||||
cid, ok := companyMap[companyLegacy]
|
||||
if !ok {
|
||||
report["files_skipped"]++
|
||||
continue
|
||||
}
|
||||
meta := map[string]any{}
|
||||
if len(metadata) > 0 {
|
||||
_ = json.Unmarshal(metadata, &meta)
|
||||
}
|
||||
meta["_legacy_file_id"] = legacyID
|
||||
meta["_blob_strategy"] = "metadata_only_resync_paths"
|
||||
if fileType != "" {
|
||||
meta["file_type"] = fileType
|
||||
}
|
||||
metaBytes, _ := json.Marshal(meta)
|
||||
|
||||
newID := uuid.New()
|
||||
legacyKey := strconv.FormatInt(legacyID, 10)
|
||||
fileMap[legacyKey] = newID.String()
|
||||
|
||||
var uid *string
|
||||
if userLegacy != "" {
|
||||
if mapped, ok := userMap[userLegacy]; ok {
|
||||
uid = &mapped
|
||||
}
|
||||
}
|
||||
|
||||
if dryRun {
|
||||
report["files"]++
|
||||
continue
|
||||
}
|
||||
_, err = pg.Exec(ctx, `
|
||||
INSERT INTO files (id, company_id, user_id, name, path, content_type, size_bytes, status, metadata)
|
||||
VALUES ($1, $2, $3::uuid, $4, $5, $6, $7, $8, $9::jsonb)`,
|
||||
newID, cid, uid, name, nullString(url), nullStr(fileType), size, status, string(metaBytes))
|
||||
if err != nil {
|
||||
log.Printf("files insert %d: %v", legacyID, err)
|
||||
delete(fileMap, legacyKey)
|
||||
report["files_skipped"]++
|
||||
continue
|
||||
}
|
||||
report["files"]++
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,165 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"log"
|
||||
"os"
|
||||
"path/filepath"
|
||||
|
||||
"github.com/google/uuid"
|
||||
)
|
||||
|
||||
// MigratorFixture is a minimal offline dump for dry-run without MySQL.
|
||||
// It is NOT a substitute for validating against production MySQL.
|
||||
type MigratorFixture struct {
|
||||
Companies []struct {
|
||||
ID string `json:"id"`
|
||||
Name string `json:"name"`
|
||||
Language string `json:"language"`
|
||||
} `json:"companies"`
|
||||
Users []struct {
|
||||
ID string `json:"id"`
|
||||
Email string `json:"email"`
|
||||
Name string `json:"name"`
|
||||
Active bool `json:"active"`
|
||||
} `json:"users"`
|
||||
AdminUsers []struct {
|
||||
UserID string `json:"user_id"`
|
||||
Email string `json:"email"`
|
||||
} `json:"admin_users"`
|
||||
Profiles []struct {
|
||||
CompanyID string `json:"company_id"`
|
||||
UserID string `json:"user_id"`
|
||||
Role string `json:"role"`
|
||||
Status string `json:"status"`
|
||||
} `json:"profiles"`
|
||||
XMLFeeds []struct {
|
||||
ID int64 `json:"id"`
|
||||
CompanyID string `json:"company_id"`
|
||||
Name string `json:"name"`
|
||||
FieldMappings json.RawMessage `json:"field_mappings"`
|
||||
} `json:"xml_feeds"`
|
||||
Files []struct {
|
||||
ID int64 `json:"id"`
|
||||
CompanyID string `json:"company_id"`
|
||||
FileName string `json:"file_name"`
|
||||
} `json:"files"`
|
||||
RawProducts int `json:"raw_products_count"`
|
||||
}
|
||||
|
||||
func loadFixture(path string) (*MigratorFixture, error) {
|
||||
b, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var f MigratorFixture
|
||||
if err := json.Unmarshal(b, &f); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &f, nil
|
||||
}
|
||||
|
||||
// runFixtureDryRun remaps fixture rows and writes id-map + validation-style counts
|
||||
// without connecting to MySQL or Postgres.
|
||||
func runFixtureDryRun(fixturePath, mapsDir, idMapPath string) {
|
||||
fx, err := loadFixture(fixturePath)
|
||||
if err != nil {
|
||||
log.Fatalf("fixture: %v", err)
|
||||
}
|
||||
if err := os.MkdirAll(mapsDir, 0o755); err != nil {
|
||||
log.Fatalf("maps dir: %v", err)
|
||||
}
|
||||
outIDMap := idMapPath
|
||||
if outIDMap == "" {
|
||||
outIDMap = filepath.Join(mapsDir, "id-map.json")
|
||||
}
|
||||
|
||||
report := map[string]int{}
|
||||
companyMap := map[string]string{}
|
||||
userMap := map[string]string{}
|
||||
feedMap := map[string]string{}
|
||||
fileMap := map[string]string{}
|
||||
|
||||
for _, c := range fx.Companies {
|
||||
companyMap[c.ID] = uuid.New().String()
|
||||
report["companies"]++
|
||||
}
|
||||
for _, u := range fx.Users {
|
||||
userMap[u.ID] = uuid.New().String()
|
||||
report["users"]++
|
||||
}
|
||||
for _, a := range fx.AdminUsers {
|
||||
if _, ok := userMap[a.UserID]; ok {
|
||||
report["admin_users"]++
|
||||
} else {
|
||||
report["admin_users_unmatched"]++
|
||||
}
|
||||
}
|
||||
for _, p := range fx.Profiles {
|
||||
if _, okC := companyMap[p.CompanyID]; !okC {
|
||||
report["memberships_skipped"]++
|
||||
continue
|
||||
}
|
||||
if _, okU := userMap[p.UserID]; !okU {
|
||||
report["memberships_skipped"]++
|
||||
continue
|
||||
}
|
||||
report["memberships"]++
|
||||
}
|
||||
for _, f := range fx.XMLFeeds {
|
||||
if _, ok := companyMap[f.CompanyID]; !ok {
|
||||
report["input_feeds_skipped"]++
|
||||
continue
|
||||
}
|
||||
feedMap[fmt.Sprintf("%d", f.ID)] = uuid.New().String()
|
||||
report["input_feeds"]++
|
||||
if len(f.FieldMappings) > 0 && string(f.FieldMappings) != "null" && string(f.FieldMappings) != "{}" {
|
||||
report["feed_mappings"]++
|
||||
}
|
||||
}
|
||||
for _, f := range fx.Files {
|
||||
if _, ok := companyMap[f.CompanyID]; !ok {
|
||||
report["files_skipped"]++
|
||||
continue
|
||||
}
|
||||
fileMap[fmt.Sprintf("%d", f.ID)] = uuid.New().String()
|
||||
report["files"]++
|
||||
}
|
||||
report["raw_products"] = fx.RawProducts
|
||||
|
||||
idDoc := NewIDMapDocument(userMap, companyMap, true)
|
||||
idDoc.Source = "fixture"
|
||||
idDoc.AttachEntityMaps(nil, nil, feedMap, nil, fileMap, report)
|
||||
if err := WriteIDMap(outIDMap, idDoc); err != nil {
|
||||
log.Fatalf("id-map: %v", err)
|
||||
}
|
||||
writeEntityMapFiles(mapsDir, companyMap, userMap, nil, nil, feedMap, nil, fileMap)
|
||||
|
||||
counts := []CountPair{
|
||||
{Entity: "companies", MySQL: int64(len(fx.Companies)), Note: "fixture"},
|
||||
{Entity: "users", MySQL: int64(len(fx.Users)), Note: "fixture; password_hash never imported"},
|
||||
{Entity: "admin_users", MySQL: int64(len(fx.AdminUsers)), Note: "→ is_platform_admin"},
|
||||
{Entity: "profiles", MySQL: int64(len(fx.Profiles)), Note: "→ memberships"},
|
||||
{Entity: "xml_feeds", MySQL: int64(len(fx.XMLFeeds)), Note: "→ input_feeds + feed_mappings"},
|
||||
{Entity: "files", MySQL: int64(len(fx.Files)), Note: "metadata only"},
|
||||
{Entity: "raw_products", MySQL: int64(fx.RawProducts), Note: "count-only in fixture"},
|
||||
}
|
||||
v := ValidationReport{
|
||||
Mode: "fixture-dry-run",
|
||||
Counts: counts,
|
||||
Orphans: []OrphanFinding{{
|
||||
Check: "skipped_fixture",
|
||||
Pass: true,
|
||||
Sample: "orphan checks need live Postgres after a real load",
|
||||
}},
|
||||
OK: true,
|
||||
}
|
||||
v.OrphanSummary = summarizeOrphans(v.Orphans)
|
||||
printValidation(v)
|
||||
_ = writeJSON(filepath.Join(mapsDir, "validation-report.json"), v)
|
||||
|
||||
printMigrationReport(report, true)
|
||||
fmt.Printf("wrote maps under %s (unified: %s)\n", mapsDir, outIDMap)
|
||||
fmt.Println("BLOCKER: fixture mode is not a substitute for dry-run against production MySQL — set MIGRATE_MYSQL_DSN and re-run before cutover.")
|
||||
}
|
||||
@@ -0,0 +1,619 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"encoding/json"
|
||||
"log"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/google/uuid"
|
||||
"github.com/jackc/pgx/v5"
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
)
|
||||
|
||||
// migrateGapDomains loads settings, formulas (standard fields), tags, woo, and usage snapshots.
|
||||
func migrateGapDomains(
|
||||
ctx context.Context,
|
||||
mysqlDB *sql.DB,
|
||||
pg *pgxpool.Pool,
|
||||
companyMap map[string]string,
|
||||
feedMap map[string]string,
|
||||
allow map[string]bool,
|
||||
domains domainSet,
|
||||
report map[string]int,
|
||||
dryRun bool,
|
||||
) {
|
||||
if domains.has("settings") {
|
||||
migrateCompanySettings(ctx, mysqlDB, pg, companyMap, allow, report, dryRun)
|
||||
}
|
||||
if domains.has("formulas") {
|
||||
migrateFieldGroupsAndStandards(ctx, mysqlDB, pg, companyMap, allow, report, dryRun)
|
||||
migrateStructuredDescriptionFields(ctx, mysqlDB, pg, companyMap, allow, report, dryRun)
|
||||
}
|
||||
if domains.has("tags") {
|
||||
migrateFeedTags(ctx, mysqlDB, pg, companyMap, feedMap, allow, report, dryRun)
|
||||
}
|
||||
if domains.has("woo") {
|
||||
migrateWooConfigs(ctx, mysqlDB, pg, companyMap, allow, report, dryRun)
|
||||
}
|
||||
if domains.has("usage") {
|
||||
migrateUsageIntoSettings(ctx, mysqlDB, pg, companyMap, allow, report, dryRun)
|
||||
}
|
||||
}
|
||||
|
||||
func migrateCompanySettings(
|
||||
ctx context.Context,
|
||||
mysqlDB *sql.DB,
|
||||
pg *pgxpool.Pool,
|
||||
companyMap map[string]string,
|
||||
allow map[string]bool,
|
||||
report map[string]int,
|
||||
dryRun bool,
|
||||
) {
|
||||
if !mysqlTableExists(ctx, mysqlDB, "company_settings") {
|
||||
log.Printf("company_settings skipped: table missing")
|
||||
return
|
||||
}
|
||||
lang := mysqlCoalesce(ctx, mysqlDB, "company_settings", "language", "'en'")
|
||||
merge := mysqlCoalesce(ctx, mysqlDB, "company_settings", "merge_products", "1")
|
||||
q := "SELECT company_id, " + lang + ", " + merge + " FROM company_settings WHERE company_id IS NOT NULL AND company_id <> ''"
|
||||
clause, args := mysqlCompanyFilter("company_id", allow)
|
||||
q += clause
|
||||
rows, err := mysqlDB.QueryContext(ctx, q, args...)
|
||||
if err != nil {
|
||||
log.Printf("company_settings skipped: %v", err)
|
||||
return
|
||||
}
|
||||
defer rows.Close()
|
||||
for rows.Next() {
|
||||
var companyLegacy, language string
|
||||
var mergeProducts int
|
||||
if err := rows.Scan(&companyLegacy, &language, &mergeProducts); err != nil {
|
||||
report["company_settings_skipped"]++
|
||||
continue
|
||||
}
|
||||
cid, ok := companyMap[companyLegacy]
|
||||
if !ok {
|
||||
report["company_settings_skipped"]++
|
||||
continue
|
||||
}
|
||||
if language == "" {
|
||||
language = "en"
|
||||
}
|
||||
settings := map[string]any{
|
||||
"language": language,
|
||||
"merge_products": mergeProducts == 1,
|
||||
"_legacy": true,
|
||||
}
|
||||
b, _ := json.Marshal(settings)
|
||||
if dryRun {
|
||||
report["company_settings"]++
|
||||
continue
|
||||
}
|
||||
_, err = pg.Exec(ctx, `
|
||||
INSERT INTO company_settings (company_id, settings, updated_at)
|
||||
VALUES ($1, $2::jsonb, now())
|
||||
ON CONFLICT (company_id) DO UPDATE SET
|
||||
settings = company_settings.settings || EXCLUDED.settings,
|
||||
updated_at = now()`, cid, string(b))
|
||||
if err != nil {
|
||||
log.Printf("company_settings %s: %v", companyLegacy, err)
|
||||
report["company_settings_skipped"]++
|
||||
continue
|
||||
}
|
||||
report["company_settings"]++
|
||||
}
|
||||
}
|
||||
|
||||
func migrateFieldGroupsAndStandards(
|
||||
ctx context.Context,
|
||||
mysqlDB *sql.DB,
|
||||
pg *pgxpool.Pool,
|
||||
companyMap map[string]string,
|
||||
allow map[string]bool,
|
||||
report map[string]int,
|
||||
dryRun bool,
|
||||
) {
|
||||
if !mysqlTableExists(ctx, mysqlDB, "field_groups") {
|
||||
log.Printf("field_groups skipped: table missing")
|
||||
return
|
||||
}
|
||||
groupMap := map[string]string{} // legacy group uuid → new uuid
|
||||
clause, args := mysqlCompanyFilter("company_id", allow)
|
||||
orderCol := mustQuoteMySQLIdent("order")
|
||||
q := "SELECT id, company_id, name, COALESCE(description, ''), COALESCE(" + orderCol + ", 0), COALESCE(is_system, 0) FROM field_groups WHERE company_id IS NOT NULL AND company_id <> ''" + clause
|
||||
// MySQL may use order without backticks in some dumps — try fallbacks.
|
||||
rows, err := mysqlDB.QueryContext(ctx, q, args...)
|
||||
if err != nil {
|
||||
q2 := `SELECT id, company_id, name, COALESCE(description, ''), 0, COALESCE(is_system, 0)
|
||||
FROM field_groups WHERE company_id IS NOT NULL AND company_id <> ''` + clause
|
||||
rows, err = mysqlDB.QueryContext(ctx, q2, args...)
|
||||
}
|
||||
if err != nil {
|
||||
log.Printf("field_groups skipped: %v", err)
|
||||
return
|
||||
}
|
||||
defer rows.Close()
|
||||
for rows.Next() {
|
||||
var legacyID, companyLegacy, name, desc string
|
||||
var order, isSystem int
|
||||
if err := rows.Scan(&legacyID, &companyLegacy, &name, &desc, &order, &isSystem); err != nil {
|
||||
report["field_groups_skipped"]++
|
||||
continue
|
||||
}
|
||||
cid, ok := companyMap[companyLegacy]
|
||||
if !ok {
|
||||
report["field_groups_skipped"]++
|
||||
continue
|
||||
}
|
||||
newID := uuid.New()
|
||||
if parsed, err := uuid.Parse(legacyID); err == nil {
|
||||
newID = parsed // preserve UUID when already uuid-shaped
|
||||
}
|
||||
groupMap[legacyID] = newID.String()
|
||||
if dryRun {
|
||||
report["field_groups"]++
|
||||
continue
|
||||
}
|
||||
_, err = pg.Exec(ctx, `
|
||||
INSERT INTO field_groups (id, company_id, name, description, "order", is_system)
|
||||
VALUES ($1, $2, $3, NULLIF($4, ''), $5, $6)
|
||||
ON CONFLICT (id) DO UPDATE SET
|
||||
name = EXCLUDED.name,
|
||||
description = EXCLUDED.description,
|
||||
"order" = EXCLUDED."order",
|
||||
updated_at = now()`,
|
||||
newID, cid, name, desc, order, isSystem == 1)
|
||||
if err != nil {
|
||||
log.Printf("field_group %s: %v", legacyID, err)
|
||||
report["field_groups_skipped"]++
|
||||
delete(groupMap, legacyID)
|
||||
continue
|
||||
}
|
||||
report["field_groups"]++
|
||||
}
|
||||
|
||||
if !mysqlTableExists(ctx, mysqlDB, "standard_fields") {
|
||||
return
|
||||
}
|
||||
keyCol := mustQuoteMySQLIdent("key")
|
||||
sq := "SELECT id, company_id, name, " + keyCol + ", type, group_id, COALESCE(is_required, 0), COALESCE(description, ''), COALESCE(default_value, ''), validation, COALESCE(is_system, 0) FROM standard_fields WHERE company_id IS NOT NULL AND company_id <> ''" + clause
|
||||
srows, err := mysqlDB.QueryContext(ctx, sq, args...)
|
||||
if err != nil {
|
||||
log.Printf("standard_fields skipped: %v", err)
|
||||
return
|
||||
}
|
||||
defer srows.Close()
|
||||
for srows.Next() {
|
||||
var legacyID, companyLegacy, name, key, typ, groupLegacy string
|
||||
var desc, defVal string
|
||||
var required, isSystem int
|
||||
var validation []byte
|
||||
if err := srows.Scan(&legacyID, &companyLegacy, &name, &key, &typ, &groupLegacy,
|
||||
&required, &desc, &defVal, &validation, &isSystem); err != nil {
|
||||
report["standard_fields_skipped"]++
|
||||
continue
|
||||
}
|
||||
cid, ok := companyMap[companyLegacy]
|
||||
if !ok {
|
||||
report["standard_fields_skipped"]++
|
||||
continue
|
||||
}
|
||||
gid, okG := groupMap[groupLegacy]
|
||||
if !okG {
|
||||
// group may already exist in PG with same UUID
|
||||
gid = groupLegacy
|
||||
}
|
||||
newID := uuid.New()
|
||||
if parsed, err := uuid.Parse(legacyID); err == nil {
|
||||
newID = parsed
|
||||
}
|
||||
if typ == "" {
|
||||
typ = "string"
|
||||
}
|
||||
if dryRun {
|
||||
report["standard_fields"]++
|
||||
continue
|
||||
}
|
||||
_, err = pg.Exec(ctx, `
|
||||
INSERT INTO standard_fields (
|
||||
id, company_id, name, key, type, group_id, is_required,
|
||||
description, default_value, validation, is_system
|
||||
) VALUES (
|
||||
$1, $2, $3, $4, $5, $6::uuid, $7, NULLIF($8, ''), NULLIF($9, ''),
|
||||
COALESCE($10::jsonb, '{}'::jsonb), $11
|
||||
)
|
||||
ON CONFLICT (company_id, key) DO UPDATE SET
|
||||
name = EXCLUDED.name,
|
||||
type = EXCLUDED.type,
|
||||
group_id = EXCLUDED.group_id,
|
||||
is_required = EXCLUDED.is_required,
|
||||
description = EXCLUDED.description,
|
||||
default_value = EXCLUDED.default_value,
|
||||
validation = EXCLUDED.validation,
|
||||
updated_at = now()`,
|
||||
newID, cid, name, key, typ, gid, required == 1, desc, defVal,
|
||||
jsonOrNull(validation), isSystem == 1)
|
||||
if err != nil {
|
||||
log.Printf("standard_field %s: %v", legacyID, err)
|
||||
report["standard_fields_skipped"]++
|
||||
continue
|
||||
}
|
||||
report["standard_fields"]++
|
||||
}
|
||||
}
|
||||
|
||||
func migrateStructuredDescriptionFields(
|
||||
ctx context.Context,
|
||||
mysqlDB *sql.DB,
|
||||
pg *pgxpool.Pool,
|
||||
companyMap map[string]string,
|
||||
allow map[string]bool,
|
||||
report map[string]int,
|
||||
dryRun bool,
|
||||
) {
|
||||
if !mysqlTableExists(ctx, mysqlDB, "structured_description_fields") {
|
||||
log.Printf("structured_description_fields skipped: table missing")
|
||||
return
|
||||
}
|
||||
clause, args := mysqlCompanyFilter("company_id", allow)
|
||||
q := `SELECT id, company_id, field_key, COALESCE(type, 'text')
|
||||
FROM structured_description_fields
|
||||
WHERE company_id IS NOT NULL AND company_id <> ''` + clause
|
||||
rows, err := mysqlDB.QueryContext(ctx, q, args...)
|
||||
if err != nil {
|
||||
log.Printf("structured_description_fields skipped: %v", err)
|
||||
return
|
||||
}
|
||||
defer rows.Close()
|
||||
for rows.Next() {
|
||||
var legacyID, companyLegacy, fieldKey, typ string
|
||||
if err := rows.Scan(&legacyID, &companyLegacy, &fieldKey, &typ); err != nil {
|
||||
report["structured_description_fields_skipped"]++
|
||||
continue
|
||||
}
|
||||
cid, ok := companyMap[companyLegacy]
|
||||
if !ok {
|
||||
report["structured_description_fields_skipped"]++
|
||||
continue
|
||||
}
|
||||
newID := uuid.New()
|
||||
if parsed, err := uuid.Parse(legacyID); err == nil {
|
||||
newID = parsed
|
||||
}
|
||||
if dryRun {
|
||||
report["structured_description_fields"]++
|
||||
continue
|
||||
}
|
||||
_, err = pg.Exec(ctx, `
|
||||
INSERT INTO structured_description_fields (id, company_id, field_key, type)
|
||||
VALUES ($1, $2, $3, $4)
|
||||
ON CONFLICT (company_id, field_key) DO UPDATE SET
|
||||
type = EXCLUDED.type, updated_at = now()`,
|
||||
newID, cid, fieldKey, typ)
|
||||
if err != nil {
|
||||
log.Printf("structured_description_field %s: %v", legacyID, err)
|
||||
report["structured_description_fields_skipped"]++
|
||||
continue
|
||||
}
|
||||
report["structured_description_fields"]++
|
||||
}
|
||||
}
|
||||
|
||||
func migrateFeedTags(
|
||||
ctx context.Context,
|
||||
mysqlDB *sql.DB,
|
||||
pg *pgxpool.Pool,
|
||||
companyMap, feedMap map[string]string,
|
||||
allow map[string]bool,
|
||||
report map[string]int,
|
||||
dryRun bool,
|
||||
) {
|
||||
if !mysqlTableExists(ctx, mysqlDB, "feed_tags") {
|
||||
log.Printf("feed_tags skipped: table missing")
|
||||
return
|
||||
}
|
||||
tagMap := map[string]string{}
|
||||
clause, args := mysqlCompanyFilter("company_id", allow)
|
||||
q := `SELECT id, company_id, name, COALESCE(color, '#888888')
|
||||
FROM feed_tags WHERE company_id IS NOT NULL AND company_id <> ''` + clause
|
||||
rows, err := mysqlDB.QueryContext(ctx, q, args...)
|
||||
if err != nil {
|
||||
log.Printf("feed_tags skipped: %v", err)
|
||||
return
|
||||
}
|
||||
defer rows.Close()
|
||||
for rows.Next() {
|
||||
var legacyID int64
|
||||
var companyLegacy, name, color string
|
||||
if err := rows.Scan(&legacyID, &companyLegacy, &name, &color); err != nil {
|
||||
report["feed_tags_skipped"]++
|
||||
continue
|
||||
}
|
||||
cid, ok := companyMap[companyLegacy]
|
||||
if !ok {
|
||||
report["feed_tags_skipped"]++
|
||||
continue
|
||||
}
|
||||
newID := uuid.New()
|
||||
tagMap[strconv.FormatInt(legacyID, 10)] = newID.String()
|
||||
if dryRun {
|
||||
report["feed_tags"]++
|
||||
continue
|
||||
}
|
||||
_, err = pg.Exec(ctx, `
|
||||
INSERT INTO feed_tags (id, company_id, name, color)
|
||||
VALUES ($1, $2, $3, $4)
|
||||
ON CONFLICT (company_id, name) DO UPDATE SET color = EXCLUDED.color`, newID, cid, name, color)
|
||||
if err != nil {
|
||||
var existing uuid.UUID
|
||||
if err2 := pg.QueryRow(ctx, `SELECT id FROM feed_tags WHERE company_id = $1 AND name = $2`, cid, name).Scan(&existing); err2 == nil {
|
||||
tagMap[strconv.FormatInt(legacyID, 10)] = existing.String()
|
||||
report["feed_tags"]++
|
||||
continue
|
||||
}
|
||||
log.Printf("feed_tag %d: %v", legacyID, err)
|
||||
report["feed_tags_skipped"]++
|
||||
continue
|
||||
}
|
||||
var existing uuid.UUID
|
||||
_ = pg.QueryRow(ctx, `SELECT id FROM feed_tags WHERE company_id = $1 AND name = $2`, cid, name).Scan(&existing)
|
||||
if existing != uuid.Nil {
|
||||
tagMap[strconv.FormatInt(legacyID, 10)] = existing.String()
|
||||
}
|
||||
report["feed_tags"]++
|
||||
}
|
||||
|
||||
if !mysqlTableExists(ctx, mysqlDB, "feed_tag_mappings") || len(tagMap) == 0 {
|
||||
return
|
||||
}
|
||||
mrows, err := mysqlDB.QueryContext(ctx, `SELECT feed_id, tag_id FROM feed_tag_mappings`)
|
||||
if err != nil {
|
||||
log.Printf("feed_tag_mappings skipped: %v", err)
|
||||
return
|
||||
}
|
||||
defer mrows.Close()
|
||||
for mrows.Next() {
|
||||
var feedLegacy, tagLegacy int64
|
||||
if err := mrows.Scan(&feedLegacy, &tagLegacy); err != nil {
|
||||
report["feed_tag_mappings_skipped"]++
|
||||
continue
|
||||
}
|
||||
fid, okF := feedMap[strconv.FormatInt(feedLegacy, 10)]
|
||||
tid, okT := tagMap[strconv.FormatInt(tagLegacy, 10)]
|
||||
if !okF || !okT {
|
||||
report["feed_tag_mappings_skipped"]++
|
||||
continue
|
||||
}
|
||||
if dryRun {
|
||||
report["feed_tag_mappings"]++
|
||||
continue
|
||||
}
|
||||
_, err = pg.Exec(ctx, `
|
||||
INSERT INTO feed_tag_mappings (feed_id, tag_id)
|
||||
VALUES ($1, $2) ON CONFLICT DO NOTHING`, fid, tid)
|
||||
if err != nil {
|
||||
report["feed_tag_mappings_skipped"]++
|
||||
continue
|
||||
}
|
||||
report["feed_tag_mappings"]++
|
||||
}
|
||||
}
|
||||
|
||||
func migrateWooConfigs(
|
||||
ctx context.Context,
|
||||
mysqlDB *sql.DB,
|
||||
pg *pgxpool.Pool,
|
||||
companyMap map[string]string,
|
||||
allow map[string]bool,
|
||||
report map[string]int,
|
||||
dryRun bool,
|
||||
) {
|
||||
// Legacy Woo settings live as per-company custom_fields named wc_*.
|
||||
if !mysqlTableExists(ctx, mysqlDB, "custom_fields") || !mysqlTableExists(ctx, mysqlDB, "feed_custom_field_values") {
|
||||
log.Printf("woocommerce_configs skipped: custom_fields tables missing")
|
||||
report["woocommerce_configs_note"] = 1
|
||||
return
|
||||
}
|
||||
clause, args := mysqlCompanyFilter("cf.company_id", allow)
|
||||
q := `
|
||||
SELECT cf.company_id, cf.name, COALESCE(fcfv.value, '')
|
||||
FROM custom_fields cf
|
||||
JOIN feed_custom_field_values fcfv ON fcfv.custom_field_id = cf.id
|
||||
WHERE cf.name LIKE 'wc_%'` + clause
|
||||
rows, err := mysqlDB.QueryContext(ctx, q, args...)
|
||||
if err != nil {
|
||||
log.Printf("woocommerce_configs skipped: %v", err)
|
||||
return
|
||||
}
|
||||
defer rows.Close()
|
||||
byCompany := map[string]map[string]string{}
|
||||
for rows.Next() {
|
||||
var companyLegacy, name, value string
|
||||
if err := rows.Scan(&companyLegacy, &name, &value); err != nil {
|
||||
continue
|
||||
}
|
||||
if _, ok := companyMap[companyLegacy]; !ok {
|
||||
continue
|
||||
}
|
||||
if byCompany[companyLegacy] == nil {
|
||||
byCompany[companyLegacy] = map[string]string{}
|
||||
}
|
||||
byCompany[companyLegacy][name] = value
|
||||
}
|
||||
if len(byCompany) == 0 {
|
||||
log.Printf("woocommerce_configs: no wc_* custom fields found (ok)")
|
||||
report["woocommerce_configs"] = 0
|
||||
return
|
||||
}
|
||||
for legacyCID, fields := range byCompany {
|
||||
cid := companyMap[legacyCID]
|
||||
enabled := strings.EqualFold(fields["wc_enabled"], "true") || fields["wc_enabled"] == "1"
|
||||
storeURL := firstNonEmpty(fields["wc_store_url"], fields["wc_url"], fields["wc_store"])
|
||||
consumerKey := firstNonEmpty(fields["wc_consumer_key"], fields["wc_key"])
|
||||
consumerSecret := firstNonEmpty(fields["wc_consumer_secret"], fields["wc_secret"])
|
||||
syncOpts, _ := json.Marshal(fields)
|
||||
if dryRun {
|
||||
report["woocommerce_configs"]++
|
||||
continue
|
||||
}
|
||||
_, err = pg.Exec(ctx, `
|
||||
INSERT INTO woocommerce_configs (
|
||||
company_id, store_url, consumer_key, consumer_secret, is_enabled, sync_options
|
||||
) VALUES ($1, $2, $3, $4, $5, $6::jsonb)
|
||||
ON CONFLICT (company_id) DO UPDATE SET
|
||||
store_url = EXCLUDED.store_url,
|
||||
consumer_key = EXCLUDED.consumer_key,
|
||||
consumer_secret = EXCLUDED.consumer_secret,
|
||||
is_enabled = EXCLUDED.is_enabled,
|
||||
sync_options = EXCLUDED.sync_options,
|
||||
updated_at = now()`,
|
||||
cid, storeURL, consumerKey, consumerSecret, enabled, string(syncOpts))
|
||||
if err != nil {
|
||||
log.Printf("woocommerce_configs %s: %v", legacyCID, err)
|
||||
report["woocommerce_configs_skipped"]++
|
||||
continue
|
||||
}
|
||||
report["woocommerce_configs"]++
|
||||
}
|
||||
}
|
||||
|
||||
// migrateUsageIntoSettings folds latest usage_metrics into company_settings.settings._legacy_usage.
|
||||
// v2 has no usage_metrics table; this preserves a portable snapshot without N+1.
|
||||
func migrateUsageIntoSettings(
|
||||
ctx context.Context,
|
||||
mysqlDB *sql.DB,
|
||||
pg *pgxpool.Pool,
|
||||
companyMap map[string]string,
|
||||
allow map[string]bool,
|
||||
report map[string]int,
|
||||
dryRun bool,
|
||||
) {
|
||||
if !mysqlTableExists(ctx, mysqlDB, "usage_metrics") {
|
||||
log.Printf("usage_metrics skipped: table missing")
|
||||
return
|
||||
}
|
||||
clause, args := mysqlCompanyFilter("company_id", allow)
|
||||
q := `
|
||||
SELECT company_id,
|
||||
SUM(COALESCE(credits_used, 0)),
|
||||
MAX(COALESCE(total_products, 0)),
|
||||
COUNT(*)
|
||||
FROM usage_metrics
|
||||
WHERE company_id IS NOT NULL AND company_id <> ''` + clause + `
|
||||
GROUP BY company_id`
|
||||
rows, err := mysqlDB.QueryContext(ctx, q, args...)
|
||||
if err != nil {
|
||||
log.Printf("usage_metrics skipped: %v", err)
|
||||
return
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
type snap struct {
|
||||
CreditsUsed float64 `json:"credits_used_sum"`
|
||||
MaxProducts int `json:"max_total_products"`
|
||||
MetricDays int `json:"metric_days"`
|
||||
}
|
||||
batch := make([][]any, 0)
|
||||
for rows.Next() {
|
||||
var companyLegacy string
|
||||
var credits float64
|
||||
var maxProducts, days int
|
||||
if err := rows.Scan(&companyLegacy, &credits, &maxProducts, &days); err != nil {
|
||||
report["usage_metrics_skipped"]++
|
||||
continue
|
||||
}
|
||||
cid, ok := companyMap[companyLegacy]
|
||||
if !ok {
|
||||
report["usage_metrics_skipped"]++
|
||||
continue
|
||||
}
|
||||
payload, _ := json.Marshal(map[string]any{
|
||||
"_legacy_usage": snap{CreditsUsed: credits, MaxProducts: maxProducts, MetricDays: days},
|
||||
})
|
||||
if dryRun {
|
||||
report["usage_metrics"]++
|
||||
continue
|
||||
}
|
||||
batch = append(batch, []any{cid, string(payload)})
|
||||
report["usage_metrics"]++
|
||||
}
|
||||
if dryRun || len(batch) == 0 {
|
||||
return
|
||||
}
|
||||
tx, err := pg.Begin(ctx)
|
||||
if err != nil {
|
||||
log.Printf("usage_metrics tx: %v", err)
|
||||
return
|
||||
}
|
||||
defer tx.Rollback(ctx)
|
||||
b := &pgx.Batch{}
|
||||
for _, row := range batch {
|
||||
b.Queue(`
|
||||
INSERT INTO company_settings (company_id, settings, updated_at)
|
||||
VALUES ($1, $2::jsonb, now())
|
||||
ON CONFLICT (company_id) DO UPDATE SET
|
||||
settings = company_settings.settings || EXCLUDED.settings,
|
||||
updated_at = now()`, row...)
|
||||
}
|
||||
br := tx.SendBatch(ctx, b)
|
||||
if err := br.Close(); err != nil {
|
||||
log.Printf("usage_metrics batch: %v", err)
|
||||
return
|
||||
}
|
||||
if err := tx.Commit(ctx); err != nil {
|
||||
log.Printf("usage_metrics commit: %v", err)
|
||||
}
|
||||
|
||||
// usage_limits → settings._legacy_usage_limits when present
|
||||
if mysqlTableExists(ctx, mysqlDB, "usage_limits") {
|
||||
lq := `SELECT company_id, tokens_per_minute, requests_per_minute, tokens_per_day, cost_limit, COALESCE(is_active, 1)
|
||||
FROM usage_limits WHERE company_id IS NOT NULL AND company_id <> ''`
|
||||
lc, la := mysqlCompanyFilter("company_id", allow)
|
||||
lrows, err := mysqlDB.QueryContext(ctx, lq+lc, la...)
|
||||
if err == nil {
|
||||
defer lrows.Close()
|
||||
for lrows.Next() {
|
||||
var companyLegacy string
|
||||
var tpm, rpm, tpd int
|
||||
var costLimit sql.NullInt64
|
||||
var active int
|
||||
if err := lrows.Scan(&companyLegacy, &tpm, &rpm, &tpd, &costLimit, &active); err != nil {
|
||||
continue
|
||||
}
|
||||
cid, ok := companyMap[companyLegacy]
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
lim := map[string]any{
|
||||
"tokens_per_minute": tpm,
|
||||
"requests_per_minute": rpm,
|
||||
"tokens_per_day": tpd,
|
||||
"is_active": active == 1,
|
||||
}
|
||||
if costLimit.Valid {
|
||||
lim["cost_limit"] = costLimit.Int64
|
||||
}
|
||||
payload, _ := json.Marshal(map[string]any{"_legacy_usage_limits": lim})
|
||||
_, _ = pg.Exec(ctx, `
|
||||
INSERT INTO company_settings (company_id, settings, updated_at)
|
||||
VALUES ($1, $2::jsonb, now())
|
||||
ON CONFLICT (company_id) DO UPDATE SET
|
||||
settings = company_settings.settings || EXCLUDED.settings,
|
||||
updated_at = now()`, cid, string(payload))
|
||||
report["usage_limits"]++
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func firstNonEmpty(vals ...string) string {
|
||||
for _, v := range vals {
|
||||
if strings.TrimSpace(v) != "" {
|
||||
return strings.TrimSpace(v)
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
@@ -0,0 +1,173 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"time"
|
||||
|
||||
"github.com/google/uuid"
|
||||
)
|
||||
|
||||
// IDMapDocument matches docs/schema-map.md (versioned unified ID map for cutover rehearsal).
|
||||
type IDMapDocument struct {
|
||||
Version int `json:"version"`
|
||||
GeneratedAt string `json:"generated_at"`
|
||||
Source string `json:"source"`
|
||||
Target string `json:"target"`
|
||||
Users map[string]string `json:"users"`
|
||||
Companies map[string]string `json:"companies"`
|
||||
Categories map[string]string `json:"categories,omitempty"`
|
||||
Attributes map[string]string `json:"attributes,omitempty"`
|
||||
Feeds map[string]string `json:"feeds,omitempty"`
|
||||
RawProducts map[string]string `json:"raw_products,omitempty"`
|
||||
Files map[string]string `json:"files,omitempty"`
|
||||
Meta IDMapMeta `json:"meta"`
|
||||
}
|
||||
|
||||
type IDMapMeta struct {
|
||||
UserCount int `json:"user_count"`
|
||||
CompanyCount int `json:"company_count"`
|
||||
DryRun bool `json:"dry_run"`
|
||||
Report map[string]int `json:"report,omitempty"`
|
||||
}
|
||||
|
||||
func copyMap(dst *map[string]string, src map[string]string) {
|
||||
if src == nil {
|
||||
return
|
||||
}
|
||||
if *dst == nil {
|
||||
*dst = map[string]string{}
|
||||
}
|
||||
for k, v := range src {
|
||||
(*dst)[k] = v
|
||||
}
|
||||
}
|
||||
|
||||
// AttachEntityMaps merges optional catalog/feed entity remaps into the document.
|
||||
func (d *IDMapDocument) AttachEntityMaps(
|
||||
categories, attributes, feeds, rawProducts, files map[string]string,
|
||||
report map[string]int,
|
||||
) {
|
||||
if d == nil {
|
||||
return
|
||||
}
|
||||
copyMap(&d.Categories, categories)
|
||||
copyMap(&d.Attributes, attributes)
|
||||
copyMap(&d.Feeds, feeds)
|
||||
copyMap(&d.RawProducts, rawProducts)
|
||||
copyMap(&d.Files, files)
|
||||
d.Meta.UserCount = len(d.Users)
|
||||
d.Meta.CompanyCount = len(d.Companies)
|
||||
if report != nil {
|
||||
d.Meta.Report = report
|
||||
}
|
||||
}
|
||||
|
||||
func NewIDMapDocument(users, companies map[string]string, dryRun bool) IDMapDocument {
|
||||
if users == nil {
|
||||
users = map[string]string{}
|
||||
}
|
||||
if companies == nil {
|
||||
companies = map[string]string{}
|
||||
}
|
||||
return IDMapDocument{
|
||||
Version: 1,
|
||||
GeneratedAt: time.Now().UTC().Format(time.RFC3339),
|
||||
Source: "mysql",
|
||||
Target: "postgres",
|
||||
Users: users,
|
||||
Companies: companies,
|
||||
Meta: IDMapMeta{
|
||||
UserCount: len(users),
|
||||
CompanyCount: len(companies),
|
||||
DryRun: dryRun,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func (d IDMapDocument) Validate() error {
|
||||
if d.Version != 1 {
|
||||
return fmt.Errorf("unsupported id map version %d", d.Version)
|
||||
}
|
||||
for legacy, id := range d.Users {
|
||||
if legacy == "" {
|
||||
return fmt.Errorf("empty user legacy id")
|
||||
}
|
||||
if _, err := uuid.Parse(id); err != nil {
|
||||
return fmt.Errorf("user %q maps to invalid uuid %q", legacy, id)
|
||||
}
|
||||
}
|
||||
for legacy, id := range d.Companies {
|
||||
if legacy == "" {
|
||||
return fmt.Errorf("empty company legacy id")
|
||||
}
|
||||
if _, err := uuid.Parse(id); err != nil {
|
||||
return fmt.Errorf("company %q maps to invalid uuid %q", legacy, id)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (d IDMapDocument) ResolveUser(legacy string) (uuid.UUID, bool) {
|
||||
raw, ok := d.Users[legacy]
|
||||
if !ok {
|
||||
return uuid.Nil, false
|
||||
}
|
||||
id, err := uuid.Parse(raw)
|
||||
if err != nil {
|
||||
return uuid.Nil, false
|
||||
}
|
||||
return id, true
|
||||
}
|
||||
|
||||
func (d IDMapDocument) ResolveCompany(legacy string) (uuid.UUID, bool) {
|
||||
raw, ok := d.Companies[legacy]
|
||||
if !ok {
|
||||
return uuid.Nil, false
|
||||
}
|
||||
id, err := uuid.Parse(raw)
|
||||
if err != nil {
|
||||
return uuid.Nil, false
|
||||
}
|
||||
return id, true
|
||||
}
|
||||
|
||||
func WriteIDMap(path string, d IDMapDocument) error {
|
||||
if err := d.Validate(); err != nil {
|
||||
return err
|
||||
}
|
||||
b, err := json.MarshalIndent(d, "", " ")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return os.WriteFile(path, b, 0o644)
|
||||
}
|
||||
|
||||
func ReadIDMap(path string) (IDMapDocument, error) {
|
||||
b, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return IDMapDocument{}, err
|
||||
}
|
||||
var d IDMapDocument
|
||||
if err := json.Unmarshal(b, &d); err != nil {
|
||||
return IDMapDocument{}, err
|
||||
}
|
||||
if err := d.Validate(); err != nil {
|
||||
return IDMapDocument{}, err
|
||||
}
|
||||
return d, nil
|
||||
}
|
||||
|
||||
func WriteIDMapDir(dir string, users, companies map[string]string, dryRun bool) (string, error) {
|
||||
if err := os.MkdirAll(dir, 0o755); err != nil {
|
||||
return "", err
|
||||
}
|
||||
path := filepath.Join(dir, "id-map.json")
|
||||
doc := NewIDMapDocument(users, companies, dryRun)
|
||||
if err := WriteIDMap(path, doc); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return path, nil
|
||||
}
|
||||
@@ -0,0 +1,104 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"github.com/google/uuid"
|
||||
)
|
||||
|
||||
func TestIDMapRoundTripFixture(t *testing.T) {
|
||||
t.Parallel()
|
||||
dir := t.TempDir()
|
||||
users := map[string]string{
|
||||
"user_2abcClerkId": "550e8400-e29b-41d4-a716-446655440000",
|
||||
}
|
||||
companies := map[string]string{
|
||||
"org_or_legacy_company_text_id": "6ba7b810-9dad-11d1-80b4-00c04fd430c8",
|
||||
}
|
||||
path, err := WriteIDMapDir(dir, users, companies, true)
|
||||
if err != nil {
|
||||
t.Fatalf("WriteIDMapDir: %v", err)
|
||||
}
|
||||
if filepath.Base(path) != "id-map.json" {
|
||||
t.Fatalf("unexpected path %s", path)
|
||||
}
|
||||
|
||||
doc, err := ReadIDMap(path)
|
||||
if err != nil {
|
||||
t.Fatalf("ReadIDMap: %v", err)
|
||||
}
|
||||
if doc.Version != 1 || !doc.Meta.DryRun {
|
||||
t.Fatalf("meta %#v", doc.Meta)
|
||||
}
|
||||
if doc.Meta.UserCount != 1 || doc.Meta.CompanyCount != 1 {
|
||||
t.Fatalf("counts user=%d company=%d", doc.Meta.UserCount, doc.Meta.CompanyCount)
|
||||
}
|
||||
|
||||
uid, ok := doc.ResolveUser("user_2abcClerkId")
|
||||
if !ok || uid.String() != "550e8400-e29b-41d4-a716-446655440000" {
|
||||
t.Fatalf("ResolveUser = %v ok=%v", uid, ok)
|
||||
}
|
||||
cid, ok := doc.ResolveCompany("org_or_legacy_company_text_id")
|
||||
if !ok || cid.String() != "6ba7b810-9dad-11d1-80b4-00c04fd430c8" {
|
||||
t.Fatalf("ResolveCompany = %v ok=%v", cid, ok)
|
||||
}
|
||||
if _, ok := doc.ResolveUser("missing"); ok {
|
||||
t.Fatal("expected missing user")
|
||||
}
|
||||
}
|
||||
|
||||
func TestIDMapValidateRejectsBadUUID(t *testing.T) {
|
||||
t.Parallel()
|
||||
doc := NewIDMapDocument(map[string]string{"u1": "not-a-uuid"}, nil, false)
|
||||
if err := doc.Validate(); err == nil {
|
||||
t.Fatal("expected validation error")
|
||||
}
|
||||
}
|
||||
|
||||
func TestIDMapValidateRejectsEmptyLegacy(t *testing.T) {
|
||||
t.Parallel()
|
||||
doc := NewIDMapDocument(map[string]string{"": uuid.New().String()}, nil, false)
|
||||
if err := doc.Validate(); err == nil {
|
||||
t.Fatal("expected empty legacy error")
|
||||
}
|
||||
}
|
||||
|
||||
func TestIDMapWriteRejectsInvalid(t *testing.T) {
|
||||
t.Parallel()
|
||||
path := filepath.Join(t.TempDir(), "bad.json")
|
||||
err := WriteIDMap(path, IDMapDocument{Version: 2})
|
||||
if err == nil {
|
||||
t.Fatal("expected write validation error")
|
||||
}
|
||||
if _, err := os.Stat(path); !os.IsNotExist(err) {
|
||||
t.Fatalf("file should not exist, stat err=%v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRemapOrphanDetectionFixture(t *testing.T) {
|
||||
t.Parallel()
|
||||
// Membership rows whose company/user legacy IDs are absent from the map are orphans.
|
||||
companies := map[string]string{"co_a": uuid.New().String()}
|
||||
users := map[string]string{"user_a": uuid.New().String()}
|
||||
doc := NewIDMapDocument(users, companies, false)
|
||||
|
||||
type membership struct{ CompanyLegacy, UserLegacy string }
|
||||
rows := []membership{
|
||||
{"co_a", "user_a"},
|
||||
{"co_missing", "user_a"},
|
||||
{"co_a", "user_missing"},
|
||||
}
|
||||
orphans := 0
|
||||
for _, m := range rows {
|
||||
_, okC := doc.ResolveCompany(m.CompanyLegacy)
|
||||
_, okU := doc.ResolveUser(m.UserLegacy)
|
||||
if !okC || !okU {
|
||||
orphans++
|
||||
}
|
||||
}
|
||||
if orphans != 2 {
|
||||
t.Fatalf("orphans = %d, want 2", orphans)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,496 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"log"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/google/uuid"
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
)
|
||||
|
||||
// migrateJobsDomain imports legacy processing_jobs (+ best-effort job products) and tasks.
|
||||
// Job rows are tagged ai_provider_mode='migrated' so retention cleanup preserves history.
|
||||
func migrateJobsDomain(
|
||||
ctx context.Context,
|
||||
mysqlDB *sql.DB,
|
||||
pg *pgxpool.Pool,
|
||||
companyMap, userMap, rawMap map[string]string,
|
||||
allow map[string]bool,
|
||||
domains domainSet,
|
||||
report map[string]int,
|
||||
dryRun bool,
|
||||
) {
|
||||
if !domains.has("jobs") {
|
||||
return
|
||||
}
|
||||
migrateProcessingJobs(ctx, mysqlDB, pg, companyMap, userMap, allow, report, dryRun)
|
||||
migrateProcessingJobProducts(ctx, mysqlDB, pg, rawMap, allow, report, dryRun)
|
||||
migrateLegacyTasks(ctx, mysqlDB, pg, companyMap, userMap, allow, report, dryRun)
|
||||
}
|
||||
|
||||
func migrateProcessingJobs(
|
||||
ctx context.Context,
|
||||
mysqlDB *sql.DB,
|
||||
pg *pgxpool.Pool,
|
||||
companyMap, userMap map[string]string,
|
||||
allow map[string]bool,
|
||||
report map[string]int,
|
||||
dryRun bool,
|
||||
) {
|
||||
if !mysqlTableExists(ctx, mysqlDB, "processing_jobs") {
|
||||
log.Printf("processing_jobs skipped: table missing")
|
||||
return
|
||||
}
|
||||
q := mysqlSelectList(
|
||||
"id",
|
||||
"company_id",
|
||||
mysqlCol(ctx, mysqlDB, "processing_jobs", "user_id", "NULL"),
|
||||
mysqlCoalesce(ctx, mysqlDB, "processing_jobs", "status", "'pending'"),
|
||||
mysqlCoalesce(ctx, mysqlDB, "processing_jobs", "total_products", "0"),
|
||||
mysqlCoalesce(ctx, mysqlDB, "processing_jobs", "processed_products", "0"),
|
||||
mysqlCol(ctx, mysqlDB, "processing_jobs", "error", "NULL"),
|
||||
mysqlCoalesce(ctx, mysqlDB, "processing_jobs", "processing_type", "'full'"),
|
||||
mysqlCoalesce(ctx, mysqlDB, "processing_jobs", "priority", "0"),
|
||||
mysqlCoalesce(ctx, mysqlDB, "processing_jobs", "estimated_tokens", "0"),
|
||||
mysqlCol(ctx, mysqlDB, "processing_jobs", "started_at", "NULL"),
|
||||
mysqlCol(ctx, mysqlDB, "processing_jobs", "completed_at", "NULL"),
|
||||
mysqlCoalesce(ctx, mysqlDB, "processing_jobs", "created_at", "NOW()"),
|
||||
mysqlCoalesce(ctx, mysqlDB, "processing_jobs", "updated_at", "NOW()"),
|
||||
) + " FROM processing_jobs WHERE 1=1"
|
||||
clause, cargs := mysqlCompanyFilter("company_id", allow)
|
||||
q += clause
|
||||
rows, err := mysqlDB.QueryContext(ctx, q, cargs...)
|
||||
if err != nil {
|
||||
log.Printf("processing_jobs skipped: %v", err)
|
||||
return
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
for rows.Next() {
|
||||
var (
|
||||
legacyID, companyLegacy string
|
||||
userLegacy sql.NullString
|
||||
status, processingType string
|
||||
errText sql.NullString
|
||||
totalProducts any
|
||||
processedProducts any
|
||||
priority any
|
||||
estimatedTokens any
|
||||
startedAt, completedAt sql.NullTime
|
||||
createdAt, updatedAt time.Time
|
||||
)
|
||||
if err := rows.Scan(
|
||||
&legacyID, &companyLegacy, &userLegacy, &status, &totalProducts, &processedProducts,
|
||||
&errText, &processingType, &priority, &estimatedTokens,
|
||||
&startedAt, &completedAt, &createdAt, &updatedAt,
|
||||
); err != nil {
|
||||
report["processing_jobs_skipped"]++
|
||||
continue
|
||||
}
|
||||
cid, ok := companyMap[companyLegacy]
|
||||
if !ok {
|
||||
report["processing_jobs_skipped"]++
|
||||
continue
|
||||
}
|
||||
jobID, err := uuid.Parse(strings.TrimSpace(legacyID))
|
||||
if err != nil {
|
||||
// Legacy dump mixes UUID and numeric string PKs — keep remaps stable.
|
||||
jobID = uuid.NewSHA1(uuid.NameSpaceOID, []byte("processing_job:"+strings.TrimSpace(legacyID)))
|
||||
}
|
||||
var userID *uuid.UUID
|
||||
if userLegacy.Valid && strings.TrimSpace(userLegacy.String) != "" {
|
||||
if mapped, ok := userMap[userLegacy.String]; ok {
|
||||
if parsed, err := uuid.Parse(mapped); err == nil {
|
||||
if !dryRun {
|
||||
var exists bool
|
||||
_ = pg.QueryRow(ctx, `SELECT EXISTS(SELECT 1 FROM users WHERE id = $1)`, parsed).Scan(&exists)
|
||||
if exists {
|
||||
userID = &parsed
|
||||
} else {
|
||||
report["processing_jobs_user_missing"]++
|
||||
}
|
||||
} else {
|
||||
userID = &parsed
|
||||
}
|
||||
}
|
||||
} else {
|
||||
report["processing_jobs_user_unmapped"]++
|
||||
}
|
||||
}
|
||||
normStatus := normalizeProcessingJobStatus(status)
|
||||
ptype := strings.TrimSpace(processingType)
|
||||
if ptype == "" {
|
||||
ptype = "full"
|
||||
}
|
||||
var errPtr *string
|
||||
if errText.Valid && strings.TrimSpace(errText.String) != "" {
|
||||
v := errText.String
|
||||
errPtr = &v
|
||||
}
|
||||
var startedPtr, completedPtr *time.Time
|
||||
if startedAt.Valid {
|
||||
t := startedAt.Time.UTC()
|
||||
startedPtr = &t
|
||||
}
|
||||
if completedAt.Valid {
|
||||
t := completedAt.Time.UTC()
|
||||
completedPtr = &t
|
||||
}
|
||||
if dryRun {
|
||||
report["processing_jobs"]++
|
||||
continue
|
||||
}
|
||||
_, err = pg.Exec(ctx, `
|
||||
INSERT INTO processing_jobs (
|
||||
id, company_id, user_id, status, total_products, processed_products,
|
||||
error, processing_type, priority, estimated_tokens,
|
||||
started_at, completed_at, created_at, updated_at,
|
||||
current_step, step_progress, ai_provider_mode
|
||||
) VALUES (
|
||||
$1, $2, $3, $4, $5, $6,
|
||||
$7, $8, $9, $10,
|
||||
$11, $12, $13, $14,
|
||||
'', '[]'::jsonb, 'migrated'
|
||||
)
|
||||
ON CONFLICT (id) DO UPDATE SET
|
||||
company_id = EXCLUDED.company_id,
|
||||
user_id = EXCLUDED.user_id,
|
||||
status = EXCLUDED.status,
|
||||
total_products = EXCLUDED.total_products,
|
||||
processed_products = EXCLUDED.processed_products,
|
||||
error = EXCLUDED.error,
|
||||
processing_type = EXCLUDED.processing_type,
|
||||
priority = EXCLUDED.priority,
|
||||
estimated_tokens = EXCLUDED.estimated_tokens,
|
||||
started_at = EXCLUDED.started_at,
|
||||
completed_at = EXCLUDED.completed_at,
|
||||
created_at = EXCLUDED.created_at,
|
||||
updated_at = EXCLUDED.updated_at,
|
||||
ai_provider_mode = 'migrated'`,
|
||||
jobID, cid, userID, normStatus,
|
||||
scanIntish(totalProducts), scanIntish(processedProducts),
|
||||
errPtr, ptype, scanIntish(priority), scanIntish(estimatedTokens),
|
||||
startedPtr, completedPtr, createdAt.UTC(), updatedAt.UTC(),
|
||||
)
|
||||
if err != nil {
|
||||
log.Printf("processing_job %s: %v", legacyID, err)
|
||||
report["processing_jobs_skipped"]++
|
||||
continue
|
||||
}
|
||||
report["processing_jobs"]++
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
log.Printf("processing_jobs rows: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func migrateProcessingJobProducts(
|
||||
ctx context.Context,
|
||||
mysqlDB *sql.DB,
|
||||
pg *pgxpool.Pool,
|
||||
rawMap map[string]string,
|
||||
allow map[string]bool,
|
||||
report map[string]int,
|
||||
dryRun bool,
|
||||
) {
|
||||
if !mysqlTableExists(ctx, mysqlDB, "processing_job_products") {
|
||||
return
|
||||
}
|
||||
if !mysqlTableExists(ctx, mysqlDB, "processing_jobs") {
|
||||
return
|
||||
}
|
||||
q := `
|
||||
SELECT pjp.id, pjp.job_id, pjp.raw_product_id, pjp.status, pjp.error,
|
||||
pjp.processed_product_id, pjp.created_at, pjp.updated_at
|
||||
FROM processing_job_products pjp
|
||||
JOIN processing_jobs pj ON pj.id = pjp.job_id
|
||||
WHERE 1=1`
|
||||
clause, cargs := mysqlCompanyFilter("pj.company_id", allow)
|
||||
q += clause
|
||||
rows, err := mysqlDB.QueryContext(ctx, q, cargs...)
|
||||
if err != nil {
|
||||
log.Printf("processing_job_products skipped: %v", err)
|
||||
return
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
for rows.Next() {
|
||||
var (
|
||||
legacyProdID int64
|
||||
jobLegacy string
|
||||
rawLegacy any
|
||||
status string
|
||||
errText sql.NullString
|
||||
processedLegacy sql.NullInt64
|
||||
createdAt, updatedAt time.Time
|
||||
)
|
||||
if err := rows.Scan(
|
||||
&legacyProdID, &jobLegacy, &rawLegacy, &status, &errText,
|
||||
&processedLegacy, &createdAt, &updatedAt,
|
||||
); err != nil {
|
||||
report["processing_job_products_skipped"]++
|
||||
continue
|
||||
}
|
||||
jobID, err := uuid.Parse(strings.TrimSpace(jobLegacy))
|
||||
if err != nil {
|
||||
jobID = uuid.NewSHA1(uuid.NameSpaceOID, []byte("processing_job:"+strings.TrimSpace(jobLegacy)))
|
||||
}
|
||||
rawKey := strings.TrimSpace(stringifyAnyID(rawLegacy))
|
||||
rawUUIDStr, ok := rawMap[rawKey]
|
||||
if !ok {
|
||||
report["processing_job_products_skipped"]++
|
||||
continue
|
||||
}
|
||||
rawUUID, err := uuid.Parse(rawUUIDStr)
|
||||
if err != nil {
|
||||
report["processing_job_products_skipped"]++
|
||||
continue
|
||||
}
|
||||
if dryRun {
|
||||
report["processing_job_products"]++
|
||||
continue
|
||||
}
|
||||
// Only attach when the remapped raw product still exists (GTIN dedupe may drop some).
|
||||
var exists bool
|
||||
if err := pg.QueryRow(ctx, `SELECT EXISTS(SELECT 1 FROM raw_products WHERE id = $1)`, rawUUID).Scan(&exists); err != nil || !exists {
|
||||
report["processing_job_products_skipped"]++
|
||||
continue
|
||||
}
|
||||
var jobExists bool
|
||||
if err := pg.QueryRow(ctx, `SELECT EXISTS(SELECT 1 FROM processing_jobs WHERE id = $1)`, jobID).Scan(&jobExists); err != nil || !jobExists {
|
||||
report["processing_job_products_skipped"]++
|
||||
continue
|
||||
}
|
||||
var errPtr *string
|
||||
if errText.Valid && strings.TrimSpace(errText.String) != "" {
|
||||
v := errText.String
|
||||
errPtr = &v
|
||||
}
|
||||
// Stable UUID from legacy int so resume is idempotent.
|
||||
prodID := uuid.NewSHA1(uuid.NameSpaceOID, []byte("pjp:"+strconv.FormatInt(legacyProdID, 10)))
|
||||
_, err = pg.Exec(ctx, `
|
||||
INSERT INTO processing_job_products (
|
||||
id, job_id, raw_product_id, status, error, processed_product_id, created_at, updated_at
|
||||
) VALUES ($1, $2, $3, $4, $5, NULL, $6, $7)
|
||||
ON CONFLICT (id) DO UPDATE SET
|
||||
status = EXCLUDED.status,
|
||||
error = EXCLUDED.error,
|
||||
updated_at = EXCLUDED.updated_at`,
|
||||
prodID, jobID, rawUUID, normalizeJobProductStatus(status), errPtr,
|
||||
createdAt.UTC(), updatedAt.UTC(),
|
||||
)
|
||||
if err != nil {
|
||||
report["processing_job_products_skipped"]++
|
||||
continue
|
||||
}
|
||||
report["processing_job_products"]++
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
log.Printf("processing_job_products rows: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func migrateLegacyTasks(
|
||||
ctx context.Context,
|
||||
mysqlDB *sql.DB,
|
||||
pg *pgxpool.Pool,
|
||||
companyMap, userMap map[string]string,
|
||||
allow map[string]bool,
|
||||
report map[string]int,
|
||||
dryRun bool,
|
||||
) {
|
||||
if !mysqlTableExists(ctx, mysqlDB, "tasks") {
|
||||
return
|
||||
}
|
||||
q := mysqlSelectList(
|
||||
"id",
|
||||
mysqlCol(ctx, mysqlDB, "tasks", "company_id", "NULL"),
|
||||
mysqlCol(ctx, mysqlDB, "tasks", "user_id", "NULL"),
|
||||
mysqlCoalesce(ctx, mysqlDB, "tasks", "task_name", "''"),
|
||||
mysqlCoalesce(ctx, mysqlDB, "tasks", "status", "'pending'"),
|
||||
mysqlCol(ctx, mysqlDB, "tasks", "start_time", "NULL"),
|
||||
mysqlCol(ctx, mysqlDB, "tasks", "end_time", "NULL"),
|
||||
mysqlCol(ctx, mysqlDB, "tasks", "log", "NULL"),
|
||||
mysqlCoalesce(ctx, mysqlDB, "tasks", "processing_products", "0"),
|
||||
mysqlCoalesce(ctx, mysqlDB, "tasks", "processed_products", "0"),
|
||||
mysqlCoalesce(ctx, mysqlDB, "tasks", "total_products", "0"),
|
||||
mysqlCol(ctx, mysqlDB, "tasks", "error_products", "NULL"),
|
||||
mysqlCol(ctx, mysqlDB, "tasks", "product_ids", "NULL"),
|
||||
mysqlCoalesce(ctx, mysqlDB, "tasks", "created_at", "NOW()"),
|
||||
mysqlCoalesce(ctx, mysqlDB, "tasks", "updated_at", "NOW()"),
|
||||
) + " FROM tasks WHERE 1=1"
|
||||
clause, cargs := mysqlCompanyFilter("company_id", allow)
|
||||
q += clause
|
||||
rows, err := mysqlDB.QueryContext(ctx, q, cargs...)
|
||||
if err != nil {
|
||||
log.Printf("tasks skipped: %v", err)
|
||||
return
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
for rows.Next() {
|
||||
var (
|
||||
legacyID int64
|
||||
companyLegacy, userLegacy sql.NullString
|
||||
taskName, status string
|
||||
startTime, endTime sql.NullTime
|
||||
logText sql.NullString
|
||||
processingProducts, processedProducts, totalP any
|
||||
errorProducts, productIDs sql.NullString
|
||||
createdAt, updatedAt time.Time
|
||||
)
|
||||
if err := rows.Scan(
|
||||
&legacyID, &companyLegacy, &userLegacy, &taskName, &status,
|
||||
&startTime, &endTime, &logText,
|
||||
&processingProducts, &processedProducts, &totalP,
|
||||
&errorProducts, &productIDs, &createdAt, &updatedAt,
|
||||
); err != nil {
|
||||
report["tasks_skipped"]++
|
||||
continue
|
||||
}
|
||||
if !companyLegacy.Valid || strings.TrimSpace(companyLegacy.String) == "" {
|
||||
report["tasks_skipped"]++
|
||||
continue
|
||||
}
|
||||
cid, ok := companyMap[companyLegacy.String]
|
||||
if !ok {
|
||||
report["tasks_skipped"]++
|
||||
continue
|
||||
}
|
||||
var userID *uuid.UUID
|
||||
if userLegacy.Valid && strings.TrimSpace(userLegacy.String) != "" {
|
||||
if mapped, ok := userMap[userLegacy.String]; ok {
|
||||
if parsed, err := uuid.Parse(mapped); err == nil {
|
||||
if !dryRun {
|
||||
var exists bool
|
||||
_ = pg.QueryRow(ctx, `SELECT EXISTS(SELECT 1 FROM users WHERE id = $1)`, parsed).Scan(&exists)
|
||||
if exists {
|
||||
userID = &parsed
|
||||
}
|
||||
} else {
|
||||
userID = &parsed
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
taskID := uuid.NewSHA1(uuid.NameSpaceOID, []byte("task:"+strconv.FormatInt(legacyID, 10)))
|
||||
var startPtr, endPtr *time.Time
|
||||
if startTime.Valid {
|
||||
t := startTime.Time.UTC()
|
||||
startPtr = &t
|
||||
}
|
||||
if endTime.Valid {
|
||||
t := endTime.Time.UTC()
|
||||
endPtr = &t
|
||||
}
|
||||
var logPtr *string
|
||||
if logText.Valid {
|
||||
v := logText.String
|
||||
logPtr = &v
|
||||
}
|
||||
errJSON := nullJSON(errorProducts)
|
||||
prodJSON := nullJSON(productIDs)
|
||||
if dryRun {
|
||||
report["tasks"]++
|
||||
continue
|
||||
}
|
||||
_, err = pg.Exec(ctx, `
|
||||
INSERT INTO tasks (
|
||||
id, company_id, user_id, task_name, status, start_time, end_time, log,
|
||||
processing_products, processed_products, total_products,
|
||||
error_products, product_ids, created_at, updated_at
|
||||
) VALUES (
|
||||
$1, $2, $3, $4, $5, $6, $7, $8,
|
||||
$9, $10, $11,
|
||||
$12::jsonb, $13::jsonb, $14, $15
|
||||
)
|
||||
ON CONFLICT (id) DO UPDATE SET
|
||||
status = EXCLUDED.status,
|
||||
end_time = EXCLUDED.end_time,
|
||||
log = EXCLUDED.log,
|
||||
processed_products = EXCLUDED.processed_products,
|
||||
updated_at = EXCLUDED.updated_at`,
|
||||
taskID, cid, userID, taskName, strings.TrimSpace(status),
|
||||
startPtr, endPtr, logPtr,
|
||||
scanIntish(processingProducts), scanIntish(processedProducts), scanIntish(totalP),
|
||||
errJSON, prodJSON, createdAt.UTC(), updatedAt.UTC(),
|
||||
)
|
||||
if err != nil {
|
||||
log.Printf("task %d: %v", legacyID, err)
|
||||
report["tasks_skipped"]++
|
||||
continue
|
||||
}
|
||||
report["tasks"]++
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
log.Printf("tasks rows: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func normalizeProcessingJobStatus(raw string) string {
|
||||
switch strings.ToLower(strings.TrimSpace(raw)) {
|
||||
case "completed", "success", "done":
|
||||
return "completed"
|
||||
case "failed", "error":
|
||||
return "failed"
|
||||
case "cancelled", "canceled", "skipped":
|
||||
return "cancelled"
|
||||
case "running", "processing":
|
||||
return "running"
|
||||
case "pending", "queued":
|
||||
return "pending"
|
||||
default:
|
||||
return "failed"
|
||||
}
|
||||
}
|
||||
|
||||
func normalizeJobProductStatus(raw string) string {
|
||||
switch strings.ToLower(strings.TrimSpace(raw)) {
|
||||
case "processed", "completed", "success", "done":
|
||||
return "processed"
|
||||
case "failed", "error":
|
||||
return "failed"
|
||||
case "cancelled", "canceled", "skipped":
|
||||
return "cancelled"
|
||||
case "processing", "running":
|
||||
return "processing"
|
||||
case "pending", "queued":
|
||||
return "pending"
|
||||
default:
|
||||
return "failed"
|
||||
}
|
||||
}
|
||||
|
||||
func stringifyAnyID(v any) string {
|
||||
switch x := v.(type) {
|
||||
case nil:
|
||||
return ""
|
||||
case int64:
|
||||
return strconv.FormatInt(x, 10)
|
||||
case int32:
|
||||
return strconv.FormatInt(int64(x), 10)
|
||||
case float64:
|
||||
return strconv.FormatInt(int64(x), 10)
|
||||
case []byte:
|
||||
return strings.TrimSpace(string(x))
|
||||
case string:
|
||||
return strings.TrimSpace(x)
|
||||
default:
|
||||
n := scanIntish(v)
|
||||
if n != 0 {
|
||||
return strconv.Itoa(n)
|
||||
}
|
||||
return ""
|
||||
}
|
||||
}
|
||||
|
||||
func nullJSON(ns sql.NullString) any {
|
||||
if !ns.Valid || strings.TrimSpace(ns.String) == "" {
|
||||
return nil
|
||||
}
|
||||
return strings.TrimSpace(ns.String)
|
||||
}
|
||||
@@ -0,0 +1,599 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/csv"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"log"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/descrybe/descrybe-v2/apps/api/internal/auth"
|
||||
"github.com/descrybe/descrybe-v2/apps/api/internal/billing"
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
)
|
||||
|
||||
const legacyEmailSuffix = "@legacy.local"
|
||||
|
||||
// legacyEmailRow is one Postgres user still on a synthetic Clerk-missing address.
|
||||
type legacyEmailRow struct {
|
||||
ID string `json:"id"`
|
||||
Email string `json:"email"`
|
||||
Name string `json:"name,omitempty"`
|
||||
LegacyUserID string `json:"legacy_user_id,omitempty"`
|
||||
Companies []string `json:"companies,omitempty"`
|
||||
A1Member bool `json:"a1_member"`
|
||||
LegacyCompanyIDs []string `json:"legacy_company_ids,omitempty"`
|
||||
}
|
||||
|
||||
type legacyEmailInventory struct {
|
||||
Version int `json:"version"`
|
||||
GeneratedAt string `json:"generated_at"`
|
||||
Count int `json:"count"`
|
||||
A1Members int `json:"a1_members"`
|
||||
Note string `json:"note"`
|
||||
Users []legacyEmailRow `json:"users"`
|
||||
// Emails is a Clerk-id → email stub map for operators to fill (or overwrite from a Clerk export).
|
||||
Emails map[string]string `json:"emails"`
|
||||
}
|
||||
|
||||
type emailPatchSkip struct {
|
||||
LegacyID string `json:"legacy_id,omitempty"`
|
||||
UserID string `json:"user_id,omitempty"`
|
||||
Email string `json:"email,omitempty"`
|
||||
Reason string `json:"reason"`
|
||||
}
|
||||
|
||||
type emailPatchAction struct {
|
||||
UserID string `json:"user_id"`
|
||||
LegacyID string `json:"legacy_id"`
|
||||
FromEmail string `json:"from_email"`
|
||||
ToEmail string `json:"to_email"`
|
||||
A1Member bool `json:"a1_member"`
|
||||
Name string `json:"name,omitempty"`
|
||||
}
|
||||
|
||||
type emailPatchPlan struct {
|
||||
Apply []emailPatchAction `json:"apply"`
|
||||
Skips []emailPatchSkip `json:"skips"`
|
||||
}
|
||||
|
||||
func runLegacyEmailTools(postgresURL, mapsDir, emailsFile, emailsOut string, listOnly, exportOnly, patch bool, dryRun, confirm bool) {
|
||||
if postgresURL == "" {
|
||||
log.Fatal("-postgres / DATABASE_URL is required for legacy-email tooling")
|
||||
}
|
||||
if !listOnly && !exportOnly && !patch {
|
||||
log.Fatal("pass -list-legacy-emails and/or -export-legacy-emails and/or -patch-emails")
|
||||
}
|
||||
if patch && strings.TrimSpace(emailsFile) == "" {
|
||||
log.Fatal("-emails-file is required with -patch-emails (Clerk export or emails map JSON/CSV)")
|
||||
}
|
||||
if err := guardLiveMutation(patch, dryRun, confirm, "-patch-emails"); err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
|
||||
ctx := context.Background()
|
||||
pg, err := pgxpool.New(ctx, postgresURL)
|
||||
if err != nil {
|
||||
log.Fatalf("postgres: %v", err)
|
||||
}
|
||||
defer pg.Close()
|
||||
|
||||
rows, err := listSyntheticLegacyEmails(ctx, pg)
|
||||
if err != nil {
|
||||
log.Fatalf("list @legacy.local users: %v", err)
|
||||
}
|
||||
fmt.Printf("legacy_local_users: %d\n", len(rows))
|
||||
a1 := 0
|
||||
for _, r := range rows {
|
||||
if r.A1Member {
|
||||
a1++
|
||||
}
|
||||
}
|
||||
fmt.Printf("a1_members_among_them: %d\n", a1)
|
||||
|
||||
if listOnly || (!exportOnly && !patch) {
|
||||
for _, r := range rows {
|
||||
a1Flag := ""
|
||||
if r.A1Member {
|
||||
a1Flag = "\ta1"
|
||||
}
|
||||
co := strings.Join(r.Companies, ",")
|
||||
fmt.Printf(" %s\t%s\t%s\t%s%s\n", r.ID, r.LegacyUserID, r.Email, co, a1Flag)
|
||||
}
|
||||
fmt.Printf("listed=%d\n", len(rows))
|
||||
}
|
||||
|
||||
if exportOnly {
|
||||
outPath := strings.TrimSpace(emailsOut)
|
||||
if outPath == "" {
|
||||
if strings.TrimSpace(mapsDir) == "" {
|
||||
mapsDir = "maps"
|
||||
}
|
||||
outPath = filepath.Join(mapsDir, "legacy-emails.json")
|
||||
}
|
||||
if err := writeLegacyEmailInventory(outPath, rows); err != nil {
|
||||
log.Fatalf("export legacy emails: %v", err)
|
||||
}
|
||||
fmt.Printf("exported=%d path=%s\n", len(rows), outPath)
|
||||
}
|
||||
|
||||
if !patch {
|
||||
return
|
||||
}
|
||||
|
||||
byLegacy, err := loadEmailPatchMap(emailsFile)
|
||||
if err != nil {
|
||||
log.Fatalf("load -emails-file: %v", err)
|
||||
}
|
||||
occupied, err := loadOccupiedEmails(ctx, pg)
|
||||
if err != nil {
|
||||
log.Fatalf("load occupied emails: %v", err)
|
||||
}
|
||||
plan := planEmailPatches(rows, byLegacy, occupied)
|
||||
fmt.Printf("patch_candidates: %d skips: %d dry_run=%v\n", len(plan.Apply), len(plan.Skips), dryRun)
|
||||
for _, s := range plan.Skips {
|
||||
fmt.Printf("skip\t%s\t%s\t%s\t%s\n", s.UserID, s.LegacyID, s.Email, s.Reason)
|
||||
}
|
||||
applied := 0
|
||||
for _, a := range plan.Apply {
|
||||
if a.A1Member {
|
||||
fmt.Printf("skip\t%s\t%s\t%s\ta1_member\n", a.UserID, a.LegacyID, a.FromEmail)
|
||||
continue
|
||||
}
|
||||
if dryRun {
|
||||
fmt.Printf("dry-run: would patch %s (%s) %s -> %s\n", a.UserID, a.LegacyID, a.FromEmail, a.ToEmail)
|
||||
applied++
|
||||
continue
|
||||
}
|
||||
ok, err := applyEmailPatch(ctx, pg, a)
|
||||
if err != nil {
|
||||
log.Printf("patch %s: %v", a.UserID, err)
|
||||
continue
|
||||
}
|
||||
if !ok {
|
||||
fmt.Printf("skip\t%s\t%s\t%s\tcurrent_email_no_longer_synthetic\n", a.UserID, a.LegacyID, a.FromEmail)
|
||||
continue
|
||||
}
|
||||
fmt.Printf("patched\t%s\t%s\t%s -> %s\n", a.UserID, a.LegacyID, a.FromEmail, a.ToEmail)
|
||||
applied++
|
||||
}
|
||||
fmt.Printf("applied=%d skipped=%d dry_run=%v\n", applied, len(plan.Skips), dryRun)
|
||||
}
|
||||
|
||||
func listSyntheticLegacyEmails(ctx context.Context, pg *pgxpool.Pool) ([]legacyEmailRow, error) {
|
||||
q := `
|
||||
SELECT u.id::text,
|
||||
u.email,
|
||||
COALESCE(u.name, ''),
|
||||
COALESCE(u.legacy_user_id, ''),
|
||||
COALESCE(string_agg(DISTINCT c.name, ', ' ORDER BY c.name), ''),
|
||||
COALESCE(string_agg(DISTINCT COALESCE(c.legacy_company_id, ''), ','), '')
|
||||
FROM users u
|
||||
LEFT JOIN memberships m ON m.user_id = u.id AND m.status = 'active'
|
||||
LEFT JOIN companies c ON c.id = m.company_id
|
||||
WHERE lower(u.email) LIKE '%@legacy.local'
|
||||
GROUP BY u.id
|
||||
ORDER BY u.email`
|
||||
rows, err := pg.Query(ctx, q)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
var out []legacyEmailRow
|
||||
for rows.Next() {
|
||||
var r legacyEmailRow
|
||||
var companiesCSV, legacyIDsCSV string
|
||||
if err := rows.Scan(&r.ID, &r.Email, &r.Name, &r.LegacyUserID, &companiesCSV, &legacyIDsCSV); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
r.Companies = splitCSVNonEmpty(companiesCSV)
|
||||
r.LegacyCompanyIDs = splitCSVNonEmpty(legacyIDsCSV)
|
||||
r.A1Member = rowIsA1Member(r)
|
||||
if r.LegacyUserID == "" {
|
||||
r.LegacyUserID = legacyIDFromSyntheticEmail(r.Email)
|
||||
}
|
||||
out = append(out, r)
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
func rowIsA1Member(r legacyEmailRow) bool {
|
||||
for _, id := range r.LegacyCompanyIDs {
|
||||
if billing.IsA1CohortCompany(id, "") {
|
||||
return true
|
||||
}
|
||||
}
|
||||
for _, name := range r.Companies {
|
||||
if strings.EqualFold(strings.TrimSpace(name), "A1 Slovenija") ||
|
||||
strings.EqualFold(strings.TrimSpace(name), "A1") {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func splitCSVNonEmpty(s string) []string {
|
||||
s = strings.TrimSpace(s)
|
||||
if s == "" {
|
||||
return nil
|
||||
}
|
||||
parts := strings.Split(s, ",")
|
||||
out := make([]string, 0, len(parts))
|
||||
for _, p := range parts {
|
||||
p = strings.TrimSpace(p)
|
||||
if p != "" {
|
||||
out = append(out, p)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func writeLegacyEmailInventory(path string, rows []legacyEmailRow) error {
|
||||
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
|
||||
return err
|
||||
}
|
||||
a1 := 0
|
||||
emails := map[string]string{}
|
||||
for _, r := range rows {
|
||||
if r.A1Member {
|
||||
a1++
|
||||
}
|
||||
key := strings.TrimSpace(r.LegacyUserID)
|
||||
if key == "" {
|
||||
key = legacyIDFromSyntheticEmail(r.Email)
|
||||
}
|
||||
if key != "" {
|
||||
emails[key] = ""
|
||||
}
|
||||
}
|
||||
inv := legacyEmailInventory{
|
||||
Version: 1,
|
||||
GeneratedAt: time.Now().UTC().Format(time.RFC3339),
|
||||
Count: len(rows),
|
||||
A1Members: a1,
|
||||
Note: "Fill emails{} from a Clerk user export (id → primary email), then: go run ./cmd/migrator -patch-emails -emails-file <path> -postgres $DATABASE_URL -dry-run (live apply needs -confirm). Never commit secrets. Patch only updates rows that still end with @legacy.local — A1 members and real live emails are never overwritten.",
|
||||
Users: rows,
|
||||
Emails: emails,
|
||||
}
|
||||
raw, err := json.MarshalIndent(inv, "", " ")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return os.WriteFile(path, append(raw, '\n'), 0o600)
|
||||
}
|
||||
|
||||
func loadOccupiedEmails(ctx context.Context, pg *pgxpool.Pool) (map[string]string, error) {
|
||||
rows, err := pg.Query(ctx, `SELECT id::text, lower(email) FROM users WHERE email IS NOT NULL AND trim(email) <> ''`)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
out := map[string]string{}
|
||||
for rows.Next() {
|
||||
var id, email string
|
||||
if err := rows.Scan(&id, &email); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out[strings.ToLower(strings.TrimSpace(email))] = id
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
func loadEmailPatchMap(path string) (map[string]string, error) {
|
||||
raw, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
ext := strings.ToLower(filepath.Ext(path))
|
||||
if ext == ".csv" {
|
||||
return parseEmailPatchCSV(raw)
|
||||
}
|
||||
return parseEmailPatchJSON(raw)
|
||||
}
|
||||
|
||||
func parseEmailPatchJSON(raw []byte) (map[string]string, error) {
|
||||
trimmed := strings.TrimSpace(string(raw))
|
||||
if trimmed == "" {
|
||||
return nil, fmt.Errorf("empty emails file")
|
||||
}
|
||||
|
||||
// Object map: {"user_xxx":"a@b.com"} or inventory {"emails":{...},"users":[...]}
|
||||
var obj map[string]json.RawMessage
|
||||
if err := json.Unmarshal(raw, &obj); err == nil {
|
||||
if emailsRaw, ok := obj["emails"]; ok {
|
||||
var emails map[string]string
|
||||
if err := json.Unmarshal(emailsRaw, &emails); err != nil {
|
||||
return nil, fmt.Errorf("emails object: %w", err)
|
||||
}
|
||||
return normalizeEmailPatchMap(emails), nil
|
||||
}
|
||||
if usersRaw, ok := obj["users"]; ok {
|
||||
m, err := parseEmailPatchArray(usersRaw)
|
||||
if err == nil && len(m) > 0 {
|
||||
return m, nil
|
||||
}
|
||||
}
|
||||
// Flat string map (all values JSON strings).
|
||||
var flat map[string]string
|
||||
if err := json.Unmarshal(raw, &flat); err == nil {
|
||||
// Reject inventory-shaped objects that decoded poorly (version etc.).
|
||||
if _, hasVersion := flat["version"]; !hasVersion && len(flat) > 0 {
|
||||
return normalizeEmailPatchMap(flat), nil
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
var arr []json.RawMessage
|
||||
if err := json.Unmarshal(raw, &arr); err == nil {
|
||||
return parseEmailPatchArray(raw)
|
||||
}
|
||||
return nil, fmt.Errorf("unsupported emails JSON (want map, {emails:{}}, {users:[]}, or array)")
|
||||
}
|
||||
|
||||
func parseEmailPatchArray(raw []byte) (map[string]string, error) {
|
||||
var rows []map[string]any
|
||||
if err := json.Unmarshal(raw, &rows); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out := map[string]string{}
|
||||
for _, row := range rows {
|
||||
id := firstString(row, "id", "legacy_user_id", "user_id", "clerk_id")
|
||||
email := firstString(row, "email", "primary_email_address", "primary_email", "real_email")
|
||||
if email == "" {
|
||||
if addrs, ok := row["email_addresses"].([]any); ok {
|
||||
email = primaryFromClerkEmailAddresses(addrs)
|
||||
}
|
||||
}
|
||||
id = strings.TrimSpace(id)
|
||||
email = strings.ToLower(strings.TrimSpace(email))
|
||||
if id == "" || email == "" {
|
||||
continue
|
||||
}
|
||||
out[id] = email
|
||||
}
|
||||
return normalizeEmailPatchMap(out), nil
|
||||
}
|
||||
|
||||
func primaryFromClerkEmailAddresses(addrs []any) string {
|
||||
for _, a := range addrs {
|
||||
m, ok := a.(map[string]any)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
email := firstString(m, "email_address", "email")
|
||||
if email == "" {
|
||||
continue
|
||||
}
|
||||
if primary, _ := m["primary"].(bool); primary {
|
||||
return email
|
||||
}
|
||||
}
|
||||
for _, a := range addrs {
|
||||
m, ok := a.(map[string]any)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
if email := firstString(m, "email_address", "email"); email != "" {
|
||||
return email
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func firstString(m map[string]any, keys ...string) string {
|
||||
for _, k := range keys {
|
||||
if v, ok := m[k]; ok {
|
||||
switch t := v.(type) {
|
||||
case string:
|
||||
if strings.TrimSpace(t) != "" {
|
||||
return t
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func parseEmailPatchCSV(raw []byte) (map[string]string, error) {
|
||||
r := csv.NewReader(strings.NewReader(string(raw)))
|
||||
r.TrimLeadingSpace = true
|
||||
records, err := r.ReadAll()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(records) == 0 {
|
||||
return nil, fmt.Errorf("empty CSV")
|
||||
}
|
||||
header := records[0]
|
||||
idIdx, emailIdx := -1, -1
|
||||
for i, h := range header {
|
||||
switch strings.ToLower(strings.TrimSpace(h)) {
|
||||
case "id", "legacy_user_id", "user_id", "clerk_id":
|
||||
if idIdx < 0 {
|
||||
idIdx = i
|
||||
}
|
||||
case "email", "primary_email_address", "primary_email", "real_email":
|
||||
if emailIdx < 0 {
|
||||
emailIdx = i
|
||||
}
|
||||
}
|
||||
}
|
||||
if idIdx < 0 || emailIdx < 0 {
|
||||
return nil, fmt.Errorf("CSV needs id/legacy_user_id and email/primary_email_address columns")
|
||||
}
|
||||
out := map[string]string{}
|
||||
for _, rec := range records[1:] {
|
||||
if idIdx >= len(rec) || emailIdx >= len(rec) {
|
||||
continue
|
||||
}
|
||||
id := strings.TrimSpace(rec[idIdx])
|
||||
email := strings.ToLower(strings.TrimSpace(rec[emailIdx]))
|
||||
if id == "" || email == "" {
|
||||
continue
|
||||
}
|
||||
out[id] = email
|
||||
}
|
||||
return normalizeEmailPatchMap(out), nil
|
||||
}
|
||||
|
||||
func normalizeEmailPatchMap(in map[string]string) map[string]string {
|
||||
out := map[string]string{}
|
||||
for k, v := range in {
|
||||
k = strings.TrimSpace(k)
|
||||
v = strings.ToLower(strings.TrimSpace(v))
|
||||
if k == "" || v == "" {
|
||||
continue
|
||||
}
|
||||
out[k] = v
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func legacyIDFromSyntheticEmail(email string) string {
|
||||
email = strings.ToLower(strings.TrimSpace(email))
|
||||
if !strings.HasSuffix(email, legacyEmailSuffix) {
|
||||
return ""
|
||||
}
|
||||
return strings.TrimSuffix(email, legacyEmailSuffix)
|
||||
}
|
||||
|
||||
// planEmailPatches builds apply/skip lists. Safety: only synthetic current emails;
|
||||
// never overwrite a real (non-@legacy.local) address; never mutate A1 members
|
||||
// (even with -confirm / dry-run apply lists).
|
||||
func planEmailPatches(rows []legacyEmailRow, byLegacy map[string]string, occupied map[string]string) emailPatchPlan {
|
||||
plan := emailPatchPlan{}
|
||||
if len(byLegacy) == 0 {
|
||||
plan.Skips = append(plan.Skips, emailPatchSkip{Reason: "empty_patch_map"})
|
||||
return plan
|
||||
}
|
||||
|
||||
matchedLegacy := map[string]bool{}
|
||||
for _, row := range rows {
|
||||
if row.A1Member {
|
||||
legacyID := strings.TrimSpace(row.LegacyUserID)
|
||||
if legacyID == "" {
|
||||
legacyID = legacyIDFromSyntheticEmail(row.Email)
|
||||
}
|
||||
if legacyID != "" {
|
||||
matchedLegacy[legacyID] = true
|
||||
}
|
||||
plan.Skips = append(plan.Skips, emailPatchSkip{
|
||||
UserID: row.ID,
|
||||
LegacyID: legacyID,
|
||||
Email: row.Email,
|
||||
Reason: "a1_member",
|
||||
})
|
||||
continue
|
||||
}
|
||||
if !auth.IsSyntheticLegacyEmail(row.Email) {
|
||||
plan.Skips = append(plan.Skips, emailPatchSkip{
|
||||
UserID: row.ID,
|
||||
LegacyID: row.LegacyUserID,
|
||||
Email: row.Email,
|
||||
Reason: "current_email_not_synthetic",
|
||||
})
|
||||
continue
|
||||
}
|
||||
legacyID := strings.TrimSpace(row.LegacyUserID)
|
||||
if legacyID == "" {
|
||||
legacyID = legacyIDFromSyntheticEmail(row.Email)
|
||||
}
|
||||
to, ok := byLegacy[legacyID]
|
||||
if !ok {
|
||||
plan.Skips = append(plan.Skips, emailPatchSkip{
|
||||
UserID: row.ID,
|
||||
LegacyID: legacyID,
|
||||
Email: row.Email,
|
||||
Reason: "no_mapping_in_emails_file",
|
||||
})
|
||||
continue
|
||||
}
|
||||
matchedLegacy[legacyID] = true
|
||||
to = strings.ToLower(strings.TrimSpace(to))
|
||||
if to == "" || strings.EqualFold(to, "replace_me@example.com") {
|
||||
plan.Skips = append(plan.Skips, emailPatchSkip{
|
||||
UserID: row.ID,
|
||||
LegacyID: legacyID,
|
||||
Email: row.Email,
|
||||
Reason: "empty_or_placeholder_target",
|
||||
})
|
||||
continue
|
||||
}
|
||||
if auth.IsSyntheticLegacyEmail(to) {
|
||||
plan.Skips = append(plan.Skips, emailPatchSkip{
|
||||
UserID: row.ID,
|
||||
LegacyID: legacyID,
|
||||
Email: row.Email,
|
||||
Reason: "target_still_synthetic",
|
||||
})
|
||||
continue
|
||||
}
|
||||
if !strings.Contains(to, "@") {
|
||||
plan.Skips = append(plan.Skips, emailPatchSkip{
|
||||
UserID: row.ID,
|
||||
LegacyID: legacyID,
|
||||
Email: row.Email,
|
||||
Reason: "target_invalid_email",
|
||||
})
|
||||
continue
|
||||
}
|
||||
if strings.EqualFold(to, row.Email) {
|
||||
plan.Skips = append(plan.Skips, emailPatchSkip{
|
||||
UserID: row.ID,
|
||||
LegacyID: legacyID,
|
||||
Email: row.Email,
|
||||
Reason: "unchanged",
|
||||
})
|
||||
continue
|
||||
}
|
||||
if owner, taken := occupied[to]; taken && owner != row.ID {
|
||||
plan.Skips = append(plan.Skips, emailPatchSkip{
|
||||
UserID: row.ID,
|
||||
LegacyID: legacyID,
|
||||
Email: row.Email,
|
||||
Reason: "target_email_owned_by_" + owner,
|
||||
})
|
||||
continue
|
||||
}
|
||||
plan.Apply = append(plan.Apply, emailPatchAction{
|
||||
UserID: row.ID,
|
||||
LegacyID: legacyID,
|
||||
FromEmail: row.Email,
|
||||
ToEmail: to,
|
||||
A1Member: row.A1Member,
|
||||
Name: row.Name,
|
||||
})
|
||||
}
|
||||
|
||||
for legacyID, email := range byLegacy {
|
||||
if matchedLegacy[legacyID] {
|
||||
continue
|
||||
}
|
||||
plan.Skips = append(plan.Skips, emailPatchSkip{
|
||||
LegacyID: legacyID,
|
||||
Email: email,
|
||||
Reason: "no_synthetic_user_for_legacy_id",
|
||||
})
|
||||
}
|
||||
return plan
|
||||
}
|
||||
|
||||
func applyEmailPatch(ctx context.Context, pg *pgxpool.Pool, a emailPatchAction) (bool, error) {
|
||||
// Defense in depth: SQL only updates rows that are still @legacy.local.
|
||||
tag, err := pg.Exec(ctx, `
|
||||
UPDATE users
|
||||
SET email = $2, updated_at = now()
|
||||
WHERE id = $1::uuid
|
||||
AND lower(email) LIKE '%@legacy.local'
|
||||
AND lower(email) = lower($3)`,
|
||||
a.UserID, a.ToEmail, a.FromEmail)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
return tag.RowsAffected() > 0, nil
|
||||
}
|
||||
@@ -0,0 +1,152 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/descrybe/descrybe-v2/apps/api/internal/billing"
|
||||
)
|
||||
|
||||
func TestLegacyIDFromSyntheticEmail(t *testing.T) {
|
||||
t.Parallel()
|
||||
if got := legacyIDFromSyntheticEmail("user_abc@legacy.local"); got != "user_abc" {
|
||||
t.Fatalf("got %q", got)
|
||||
}
|
||||
if got := legacyIDFromSyntheticEmail(" User_ABC@Legacy.Local "); got != "user_abc" {
|
||||
t.Fatalf("got %q", got)
|
||||
}
|
||||
if got := legacyIDFromSyntheticEmail("real@example.com"); got != "" {
|
||||
t.Fatalf("want empty, got %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseEmailPatchJSONMap(t *testing.T) {
|
||||
t.Parallel()
|
||||
m, err := parseEmailPatchJSON([]byte(`{"user_1":"A@Example.COM","user_2":""}`))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if m["user_1"] != "a@example.com" {
|
||||
t.Fatalf("got %#v", m)
|
||||
}
|
||||
if _, ok := m["user_2"]; ok {
|
||||
t.Fatal("empty emails must be dropped")
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseEmailPatchJSONInventoryEmails(t *testing.T) {
|
||||
t.Parallel()
|
||||
raw := []byte(`{
|
||||
"version": 1,
|
||||
"emails": {"user_x":"x@example.com"},
|
||||
"users": [{"legacy_user_id":"user_x","email":"user_x@legacy.local"}]
|
||||
}`)
|
||||
m, err := parseEmailPatchJSON(raw)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if m["user_x"] != "x@example.com" {
|
||||
t.Fatalf("got %#v", m)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseEmailPatchJSONArrayClerkish(t *testing.T) {
|
||||
t.Parallel()
|
||||
raw := []byte(`[
|
||||
{"id":"user_a","primary_email_address":"a@ex.com"},
|
||||
{"id":"user_b","email_addresses":[{"email_address":"b@ex.com","primary":true}]}
|
||||
]`)
|
||||
m, err := parseEmailPatchJSON(raw)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if m["user_a"] != "a@ex.com" || m["user_b"] != "b@ex.com" {
|
||||
t.Fatalf("got %#v", m)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseEmailPatchCSV(t *testing.T) {
|
||||
t.Parallel()
|
||||
m, err := parseEmailPatchCSV([]byte("id,primary_email_address\nuser_c,C@Ex.COM\n"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if m["user_c"] != "c@ex.com" {
|
||||
t.Fatalf("got %#v", m)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPlanEmailPatchesSafety(t *testing.T) {
|
||||
t.Parallel()
|
||||
rows := []legacyEmailRow{
|
||||
{ID: "u1", Email: "user_1@legacy.local", LegacyUserID: "user_1", A1Member: true},
|
||||
{ID: "u2", Email: "a1-primary@descrybe.local", LegacyUserID: "user_live", A1Member: true},
|
||||
{ID: "u3", Email: "user_3@legacy.local", LegacyUserID: "user_3"},
|
||||
{ID: "u4", Email: "user_4@legacy.local", LegacyUserID: "user_4"},
|
||||
{ID: "u5", Email: "user_5@legacy.local", LegacyUserID: "user_5"},
|
||||
}
|
||||
byLegacy := map[string]string{
|
||||
"user_1": "real1@example.com",
|
||||
"user_live": "should-not-apply@example.com",
|
||||
"user_3": "user_3@legacy.local",
|
||||
"user_4": "taken@example.com",
|
||||
"user_5": "real5@example.com",
|
||||
"user_missing": "ghost@example.com",
|
||||
}
|
||||
occupied := map[string]string{
|
||||
"user_1@legacy.local": "u1",
|
||||
"a1-primary@descrybe.local": "u2",
|
||||
"user_3@legacy.local": "u3",
|
||||
"user_4@legacy.local": "u4",
|
||||
"user_5@legacy.local": "u5",
|
||||
"taken@example.com": "other",
|
||||
}
|
||||
plan := planEmailPatches(rows, byLegacy, occupied)
|
||||
if len(plan.Apply) != 1 || plan.Apply[0].UserID != "u5" || plan.Apply[0].ToEmail != "real5@example.com" {
|
||||
t.Fatalf("apply=%#v", plan.Apply)
|
||||
}
|
||||
if plan.Apply[0].A1Member {
|
||||
t.Fatal("apply list must never include a1_member=true")
|
||||
}
|
||||
reasons := map[string]bool{}
|
||||
for _, s := range plan.Skips {
|
||||
reasons[s.Reason] = true
|
||||
if s.UserID == "u1" && s.Reason != "a1_member" {
|
||||
t.Fatalf("A1 synthetic email must skip as a1_member, skip=%#v", s)
|
||||
}
|
||||
if s.UserID == "u2" && s.Reason != "a1_member" {
|
||||
t.Fatalf("live A1 email must skip as a1_member, skip=%#v", s)
|
||||
}
|
||||
}
|
||||
for _, want := range []string{
|
||||
"a1_member",
|
||||
"target_still_synthetic",
|
||||
"target_email_owned_by_other",
|
||||
"no_synthetic_user_for_legacy_id",
|
||||
} {
|
||||
if !reasons[want] {
|
||||
t.Fatalf("missing skip reason %q in %#v", want, plan.Skips)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestPlanEmailPatchesEmptyMap(t *testing.T) {
|
||||
t.Parallel()
|
||||
plan := planEmailPatches(nil, nil, nil)
|
||||
if len(plan.Skips) != 1 || !strings.Contains(plan.Skips[0].Reason, "empty") {
|
||||
t.Fatalf("got %#v", plan.Skips)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRowIsA1Member(t *testing.T) {
|
||||
t.Parallel()
|
||||
if !rowIsA1Member(legacyEmailRow{LegacyCompanyIDs: []string{billing.A1LegacyCompanyID}}) {
|
||||
t.Fatal("expected A1 by legacy company id")
|
||||
}
|
||||
if !rowIsA1Member(legacyEmailRow{Companies: []string{"A1 Slovenija"}}) {
|
||||
t.Fatal("expected A1 by name")
|
||||
}
|
||||
if rowIsA1Member(legacyEmailRow{Companies: []string{"Acme"}}) {
|
||||
t.Fatal("Acme is not A1")
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,239 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"log"
|
||||
"strings"
|
||||
|
||||
"github.com/descrybe/descrybe-v2/apps/api/internal/billing"
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
)
|
||||
|
||||
// memberMembershipRow is one active membership with role=member (cutover promote candidate).
|
||||
type memberMembershipRow struct {
|
||||
UserID string
|
||||
Email string
|
||||
CompanyID string
|
||||
CompanyName string
|
||||
LegacyCompanyID string
|
||||
Role string
|
||||
Status string
|
||||
IsPlatformAdmin bool
|
||||
MustSetPassword bool
|
||||
}
|
||||
|
||||
// runMembershipRoleRepair lists and/or promotes active memberships from role=member
|
||||
// to company admin (role=admin). Postgres-only post-load operator tooling.
|
||||
// Live writes require confirm=true (no blind promotes). Prefer -dry-run first.
|
||||
// Unscoped promote (no email/user-id/company-id) is refused.
|
||||
// NEVER promotes A1 cohort memberships (a1=true) — dry-run and live both skip them.
|
||||
func runMembershipRoleRepair(
|
||||
postgresURL, email, userID, companyID string,
|
||||
listOnly, promote bool,
|
||||
dryRun, confirm bool,
|
||||
) {
|
||||
if postgresURL == "" {
|
||||
log.Fatal("-postgres / DATABASE_URL is required for membership role tooling")
|
||||
}
|
||||
if !listOnly && !promote {
|
||||
log.Fatal("pass -list-member-memberships and/or -promote-company-admins")
|
||||
}
|
||||
if err := validatePromoteTargets(promote, email, userID, companyID); err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
if err := guardLiveMutation(promote, dryRun, confirm, "-promote-company-admins"); err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
|
||||
ctx := context.Background()
|
||||
pg, err := pgxpool.New(ctx, postgresURL)
|
||||
if err != nil {
|
||||
log.Fatalf("postgres: %v", err)
|
||||
}
|
||||
defer pg.Close()
|
||||
|
||||
rows, err := listMemberMemberships(ctx, pg, email, userID, companyID)
|
||||
if err != nil {
|
||||
log.Fatalf("list member memberships: %v", err)
|
||||
}
|
||||
fmt.Printf("active_member_memberships: %d\n", len(rows))
|
||||
|
||||
listed := 0
|
||||
promoted := 0
|
||||
skipped := 0
|
||||
a1Candidates := 0
|
||||
a1Skipped := 0
|
||||
|
||||
for _, m := range rows {
|
||||
listed++
|
||||
a1 := billing.IsA1CohortCompany(m.LegacyCompanyID, m.CompanyName)
|
||||
if a1 {
|
||||
a1Candidates++
|
||||
}
|
||||
if listOnly || !promote {
|
||||
fmt.Printf(" %s\t%s\t%s\t%s\ta1=%v\tplatform_admin=%v\tmust_set_password=%v\n",
|
||||
m.UserID, m.Email, m.CompanyID, m.CompanyName, a1, m.IsPlatformAdmin, m.MustSetPassword)
|
||||
}
|
||||
mutate, skipReason := decideMembershipPromote(promote, a1)
|
||||
if !mutate {
|
||||
if promote && skipReason != "" {
|
||||
fmt.Printf("skip\t%s\t%s\t%s\t%s\t%s\n",
|
||||
m.UserID, m.Email, m.CompanyID, m.CompanyName, skipReason)
|
||||
skipped++
|
||||
if skipReason == "a1_cohort" {
|
||||
a1Skipped++
|
||||
}
|
||||
}
|
||||
continue
|
||||
}
|
||||
if dryRun {
|
||||
fmt.Printf("dry-run: would promote user %s (%s) on company %s (%s) member→admin a1=false\n",
|
||||
m.UserID, m.Email, m.CompanyID, m.CompanyName)
|
||||
promoted++
|
||||
continue
|
||||
}
|
||||
ok, err := promoteMembershipToAdmin(ctx, pg, m.CompanyID, m.UserID)
|
||||
if err != nil {
|
||||
log.Printf("promote user %s company %s: %v", m.UserID, m.CompanyID, err)
|
||||
skipped++
|
||||
continue
|
||||
}
|
||||
if !ok {
|
||||
skipped++
|
||||
continue
|
||||
}
|
||||
fmt.Printf("promoted user %s (%s) on company %s (%s) member→admin a1=false\n",
|
||||
m.UserID, m.Email, m.CompanyID, m.CompanyName)
|
||||
promoted++
|
||||
}
|
||||
|
||||
if promote {
|
||||
fmt.Printf("listed=%d promoted=%d skipped=%d a1_candidates=%d a1_skipped=%d dry_run=%v\n",
|
||||
listed, promoted, skipped, a1Candidates, a1Skipped, dryRun)
|
||||
} else {
|
||||
fmt.Printf("listed=%d a1_candidates=%d\n", listed, a1Candidates)
|
||||
}
|
||||
}
|
||||
|
||||
// decideMembershipPromote is the promote gate used by dry-run and -confirm.
|
||||
// A1 cohort rows always skip (never member→admin), even when confirm=true.
|
||||
func decideMembershipPromote(promote, a1 bool) (mutate bool, skipReason string) {
|
||||
if !promote {
|
||||
return false, ""
|
||||
}
|
||||
if a1 {
|
||||
return false, "a1_cohort"
|
||||
}
|
||||
return true, ""
|
||||
}
|
||||
|
||||
// validatePromoteTargets refuses unscoped live/dry promote of every member membership.
|
||||
// At least one of -email, -user-id, or -company-id is required. A1 rows are always skipped at promote time.
|
||||
func validatePromoteTargets(promote bool, email, userID, companyID string) error {
|
||||
if !promote {
|
||||
return nil
|
||||
}
|
||||
if strings.TrimSpace(email) == "" && strings.TrimSpace(userID) == "" && strings.TrimSpace(companyID) == "" {
|
||||
return fmt.Errorf("-promote-company-admins requires -email, -user-id, or -company-id (refusing unscoped promote; A1 rows are always skipped)")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// guardLiveMutation refuses mutating ops unless -confirm is set.
|
||||
// -dry-run always previews without writes (confirm is ignored).
|
||||
func guardLiveMutation(mutate, dryRun, confirm bool, flagHint string) error {
|
||||
if !mutate || dryRun {
|
||||
return nil
|
||||
}
|
||||
if !confirm {
|
||||
if strings.TrimSpace(flagHint) == "" {
|
||||
flagHint = "the mutating flag"
|
||||
}
|
||||
return fmt.Errorf("refusing live write: pass -dry-run to preview, or -confirm with %s (no blind live writes)", flagHint)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func listMemberMemberships(
|
||||
ctx context.Context,
|
||||
pg *pgxpool.Pool,
|
||||
email, userID, companyID string,
|
||||
) ([]memberMembershipRow, error) {
|
||||
q := `
|
||||
SELECT u.id::text,
|
||||
u.email,
|
||||
c.id::text,
|
||||
c.name,
|
||||
COALESCE(c.legacy_company_id, ''),
|
||||
m.role,
|
||||
m.status,
|
||||
u.is_platform_admin,
|
||||
u.must_set_password
|
||||
FROM memberships m
|
||||
JOIN users u ON u.id = m.user_id
|
||||
JOIN companies c ON c.id = m.company_id
|
||||
WHERE m.status = 'active'
|
||||
AND m.role = 'member'`
|
||||
args := make([]any, 0, 3)
|
||||
argN := 1
|
||||
if e := strings.TrimSpace(email); e != "" {
|
||||
q += fmt.Sprintf(" AND lower(u.email) = lower($%d)", argN)
|
||||
args = append(args, e)
|
||||
argN++
|
||||
}
|
||||
if uid := strings.TrimSpace(userID); uid != "" {
|
||||
q += fmt.Sprintf(" AND m.user_id = $%d::uuid", argN)
|
||||
args = append(args, uid)
|
||||
argN++
|
||||
}
|
||||
if cid := strings.TrimSpace(companyID); cid != "" {
|
||||
q += fmt.Sprintf(" AND m.company_id = $%d::uuid", argN)
|
||||
args = append(args, cid)
|
||||
argN++
|
||||
}
|
||||
q += `
|
||||
ORDER BY c.name, u.email`
|
||||
|
||||
rows, err := pg.Query(ctx, q, args...)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
var out []memberMembershipRow
|
||||
for rows.Next() {
|
||||
var m memberMembershipRow
|
||||
if err := rows.Scan(
|
||||
&m.UserID,
|
||||
&m.Email,
|
||||
&m.CompanyID,
|
||||
&m.CompanyName,
|
||||
&m.LegacyCompanyID,
|
||||
&m.Role,
|
||||
&m.Status,
|
||||
&m.IsPlatformAdmin,
|
||||
&m.MustSetPassword,
|
||||
); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out = append(out, m)
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
// promoteMembershipToAdmin sets an active member membership to admin.
|
||||
// Returns ok=false when no matching row was updated (already admin, inactive, or missing).
|
||||
func promoteMembershipToAdmin(ctx context.Context, pg *pgxpool.Pool, companyID, userID string) (bool, error) {
|
||||
tag, err := pg.Exec(ctx, `
|
||||
UPDATE memberships
|
||||
SET role = 'admin', updated_at = now()
|
||||
WHERE company_id = $1::uuid
|
||||
AND user_id = $2::uuid
|
||||
AND status = 'active'
|
||||
AND role = 'member'`, companyID, userID)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
return tag.RowsAffected() > 0, nil
|
||||
}
|
||||
@@ -0,0 +1,74 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestDecideMembershipPromoteSkipsA1(t *testing.T) {
|
||||
t.Parallel()
|
||||
mutate, reason := decideMembershipPromote(true, true)
|
||||
if mutate || reason != "a1_cohort" {
|
||||
t.Fatalf("a1=true must never promote (even with -confirm): mutate=%v reason=%q", mutate, reason)
|
||||
}
|
||||
mutate, reason = decideMembershipPromote(true, false)
|
||||
if !mutate || reason != "" {
|
||||
t.Fatalf("non-A1 should promote: mutate=%v reason=%q", mutate, reason)
|
||||
}
|
||||
mutate, reason = decideMembershipPromote(false, true)
|
||||
if mutate || reason != "" {
|
||||
t.Fatalf("list-only: mutate=%v reason=%q", mutate, reason)
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidatePromoteTargets(t *testing.T) {
|
||||
t.Parallel()
|
||||
if err := validatePromoteTargets(false, "", "", ""); err != nil {
|
||||
t.Fatalf("list-only: %v", err)
|
||||
}
|
||||
if err := validatePromoteTargets(true, "a@b.c", "", ""); err != nil {
|
||||
t.Fatalf("email: %v", err)
|
||||
}
|
||||
if err := validatePromoteTargets(true, "", "11111111-1111-1111-1111-111111111111", ""); err != nil {
|
||||
t.Fatalf("user-id: %v", err)
|
||||
}
|
||||
if err := validatePromoteTargets(true, "", "", "22222222-2222-2222-2222-222222222222"); err != nil {
|
||||
t.Fatalf("company-id alone: %v", err)
|
||||
}
|
||||
err := validatePromoteTargets(true, "", "", "")
|
||||
if err == nil || !strings.Contains(err.Error(), "requires -email, -user-id, or -company-id") {
|
||||
t.Fatalf("expected unscoped refuse, got %v", err)
|
||||
}
|
||||
err = validatePromoteTargets(true, " ", " ", " ")
|
||||
if err == nil || !strings.Contains(err.Error(), "requires -email, -user-id, or -company-id") {
|
||||
t.Fatalf("expected blank refuse, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGuardLiveMutation(t *testing.T) {
|
||||
t.Parallel()
|
||||
if err := guardLiveMutation(false, false, false, "-promote-company-admins"); err != nil {
|
||||
t.Fatalf("list-only: %v", err)
|
||||
}
|
||||
if err := guardLiveMutation(true, true, false, "-promote-company-admins"); err != nil {
|
||||
t.Fatalf("dry-run: %v", err)
|
||||
}
|
||||
if err := guardLiveMutation(true, false, true, "-promote-company-admins"); err != nil {
|
||||
t.Fatalf("confirm: %v", err)
|
||||
}
|
||||
err := guardLiveMutation(true, false, false, "-promote-company-admins")
|
||||
if err == nil || !strings.Contains(err.Error(), "no blind live writes") {
|
||||
t.Fatalf("expected blind-write refusal, got %v", err)
|
||||
}
|
||||
if !strings.Contains(err.Error(), "-promote-company-admins") {
|
||||
t.Fatalf("expected flag hint in error, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGuardLiveAssignDelegates(t *testing.T) {
|
||||
t.Parallel()
|
||||
err := guardLiveAssign(true, false, false)
|
||||
if err == nil || !strings.Contains(err.Error(), "-assign-missing-plans") {
|
||||
t.Fatalf("expected assign hint, got %v", err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,42 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestLoadFixture(t *testing.T) {
|
||||
fx, err := loadFixture("testdata/fixture.json")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(fx.Companies) != 1 || len(fx.Users) != 2 {
|
||||
t.Fatalf("unexpected fixture sizes: companies=%d users=%d", len(fx.Companies), len(fx.Users))
|
||||
}
|
||||
if len(fx.AdminUsers) != 1 {
|
||||
t.Fatalf("expected admin_users")
|
||||
}
|
||||
if len(fx.XMLFeeds) != 1 || len(fx.XMLFeeds[0].FieldMappings) == 0 {
|
||||
t.Fatalf("expected feed mappings in fixture")
|
||||
}
|
||||
}
|
||||
|
||||
func TestEnsureJSON(t *testing.T) {
|
||||
if string(ensureJSON(nil)) != "{}" {
|
||||
t.Fatalf("nil -> {}")
|
||||
}
|
||||
if string(ensureJSON([]byte("not-json"))) != "{}" {
|
||||
t.Fatalf("invalid -> {}")
|
||||
}
|
||||
in := []byte(`{"a":1}`)
|
||||
if string(ensureJSON(in)) != `{"a":1}` {
|
||||
t.Fatalf("valid passthrough")
|
||||
}
|
||||
}
|
||||
|
||||
func TestAttachEntityMaps(t *testing.T) {
|
||||
doc := NewIDMapDocument(map[string]string{"u1": "550e8400-e29b-41d4-a716-446655440000"}, map[string]string{"c1": "6ba7b810-9dad-11d1-80b4-00c04fd430c8"}, true)
|
||||
doc.AttachEntityMaps(nil, nil, map[string]string{"10": "6ba7b810-9dad-11d1-80b4-00c04fd430c8"}, nil, nil, map[string]int{"feeds": 1})
|
||||
if len(doc.Feeds) != 1 || doc.Meta.Report["feeds"] != 1 {
|
||||
t.Fatalf("attach failed: %#v", doc)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,76 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"fmt"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// mysqlCol returns a quoted column name if present, otherwise a SQL literal/expression fallback.
|
||||
func mysqlCol(ctx context.Context, db *sql.DB, table, column, fallbackExpr string) string {
|
||||
if mysqlColumnExists(ctx, db, table, column) {
|
||||
q, err := quoteMySQLIdent(column)
|
||||
if err != nil {
|
||||
return fallbackExpr
|
||||
}
|
||||
return q
|
||||
}
|
||||
return fallbackExpr
|
||||
}
|
||||
|
||||
// mysqlCoalesce returns COALESCE(column, fallback) when column exists, else fallback alone.
|
||||
func mysqlCoalesce(ctx context.Context, db *sql.DB, table, column, fallbackExpr string) string {
|
||||
if mysqlColumnExists(ctx, db, table, column) {
|
||||
q, err := quoteMySQLIdent(column)
|
||||
if err != nil {
|
||||
return fallbackExpr
|
||||
}
|
||||
return fmt.Sprintf("COALESCE(%s, %s)", q, fallbackExpr)
|
||||
}
|
||||
return fallbackExpr
|
||||
}
|
||||
|
||||
// mysqlSelectList builds "SELECT a, b, ..." from expressions (already resolved).
|
||||
func mysqlSelectList(exprs ...string) string {
|
||||
return "SELECT " + strings.Join(exprs, ", ")
|
||||
}
|
||||
|
||||
// scanIntish scans MySQL INT/DECIMAL/string numeric values into an int.
|
||||
func scanIntish(v any) int {
|
||||
switch x := v.(type) {
|
||||
case int64:
|
||||
return int(x)
|
||||
case int32:
|
||||
return int(x)
|
||||
case float64:
|
||||
return int(x)
|
||||
case []byte:
|
||||
var n float64
|
||||
if _, err := fmt.Sscanf(string(x), "%f", &n); err == nil {
|
||||
return int(n)
|
||||
}
|
||||
case string:
|
||||
var n float64
|
||||
if _, err := fmt.Sscanf(x, "%f", &n); err == nil {
|
||||
return int(n)
|
||||
}
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
// queryFirstOK tries queries in order until one succeeds (for column-shape fallbacks).
|
||||
func queryFirstOK(ctx context.Context, db *sql.DB, queries ...string) (*sql.Rows, error) {
|
||||
var last error
|
||||
for _, q := range queries {
|
||||
rows, err := db.QueryContext(ctx, q)
|
||||
if err == nil {
|
||||
return rows, nil
|
||||
}
|
||||
last = err
|
||||
}
|
||||
if last == nil {
|
||||
return nil, fmt.Errorf("no queries provided")
|
||||
}
|
||||
return nil, last
|
||||
}
|
||||
@@ -0,0 +1,41 @@
|
||||
package main
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestScanIntish(t *testing.T) {
|
||||
cases := []struct {
|
||||
in any
|
||||
want int
|
||||
}{
|
||||
{int64(42), 42},
|
||||
{float64(3.9), 3},
|
||||
{[]byte("12.50"), 12},
|
||||
{"7", 7},
|
||||
{nil, 0},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
if got := scanIntish(tc.in); got != tc.want {
|
||||
t.Fatalf("scanIntish(%v)=%d want %d", tc.in, got, tc.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestSummarizeOrphans(t *testing.T) {
|
||||
s := summarizeOrphans([]OrphanFinding{
|
||||
{Check: "a", Pass: true},
|
||||
{Check: "b", Pass: false},
|
||||
{Check: "platform_admins", Pass: true, Count: 2},
|
||||
{Check: "skipped_dry_run", Pass: true},
|
||||
})
|
||||
if s.Passed != 3 || s.Failed != 1 || s.Total != 4 {
|
||||
t.Fatalf("summary %#v", s)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMysqlSelectList(t *testing.T) {
|
||||
got := mysqlSelectList("id", "COALESCE(name, id)", "'en'")
|
||||
want := "SELECT id, COALESCE(name, id), 'en'"
|
||||
if got != want {
|
||||
t.Fatalf("got %q", got)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,176 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"log"
|
||||
"net/url"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/descrybe/descrybe-v2/apps/api/internal/auth"
|
||||
"github.com/google/uuid"
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
)
|
||||
|
||||
// SetPasswordHook is a one-time accept-invite style token for a migrated user.
|
||||
// SMTP delivery is owned by mailhooks / WS7 — this only prepares durable invite rows + a local artifact.
|
||||
type SetPasswordHook struct {
|
||||
UserID uuid.UUID `json:"user_id"`
|
||||
Email string `json:"email"`
|
||||
CompanyID uuid.UUID `json:"company_id"`
|
||||
Role string `json:"role"`
|
||||
Token string `json:"token"`
|
||||
URL string `json:"url"`
|
||||
ExpiresAt time.Time `json:"expires_at"`
|
||||
InviteID uuid.UUID `json:"invite_id,omitempty"`
|
||||
}
|
||||
|
||||
func webOrigin() string {
|
||||
o := strings.TrimSpace(os.Getenv("WEB_ORIGIN"))
|
||||
if o == "" {
|
||||
o = "http://localhost:5174"
|
||||
}
|
||||
return strings.TrimRight(o, "/")
|
||||
}
|
||||
|
||||
func setPasswordInviteURL(token string) string {
|
||||
return webOrigin() + "/accept-invite?token=" + url.QueryEscape(token)
|
||||
}
|
||||
|
||||
// prepareSetPasswordHooks creates invites for active users with must_set_password=true.
|
||||
// Tokens are returned once for the artifact (do not commit). AcceptInvite sets password
|
||||
// only when must_set_password is still true (existing accounts with a password must verify it).
|
||||
func prepareSetPasswordHooks(
|
||||
ctx context.Context,
|
||||
pg *pgxpool.Pool,
|
||||
ttl time.Duration,
|
||||
dryRun bool,
|
||||
report map[string]int,
|
||||
) ([]SetPasswordHook, error) {
|
||||
if dryRun {
|
||||
report["set_password_hooks_skipped_dry_run"]++
|
||||
return nil, nil
|
||||
}
|
||||
if ttl <= 0 {
|
||||
ttl = 7 * 24 * time.Hour
|
||||
}
|
||||
|
||||
rows, err := pg.Query(ctx, `
|
||||
SELECT u.id, u.email, m.company_id, m.role
|
||||
FROM users u
|
||||
JOIN memberships m ON m.user_id = u.id AND m.status = 'active'
|
||||
WHERE u.must_set_password = true AND u.is_active = true
|
||||
ORDER BY u.email, m.created_at
|
||||
`)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("list must_set_password users: %w", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
seen := map[uuid.UUID]bool{}
|
||||
var hooks []SetPasswordHook
|
||||
expires := time.Now().UTC().Add(ttl)
|
||||
|
||||
for rows.Next() {
|
||||
var h SetPasswordHook
|
||||
if err := rows.Scan(&h.UserID, &h.Email, &h.CompanyID, &h.Role); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if seen[h.UserID] {
|
||||
continue
|
||||
}
|
||||
seen[h.UserID] = true
|
||||
if auth.IsSyntheticLegacyEmail(h.Email) {
|
||||
report["set_password_hooks_skipped_synthetic"]++
|
||||
continue
|
||||
}
|
||||
if h.Role == "" {
|
||||
h.Role = "member"
|
||||
}
|
||||
token, err := auth.RandomToken(24)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
h.Token = token
|
||||
h.ExpiresAt = expires
|
||||
h.URL = setPasswordInviteURL(h.Token)
|
||||
|
||||
// Expire prior unaccepted invites for this email+company so re-issue is safe.
|
||||
_, _ = pg.Exec(ctx, `
|
||||
UPDATE invites
|
||||
SET expires_at = least(expires_at, now())
|
||||
WHERE company_id = $1 AND lower(email) = lower($2) AND accepted_at IS NULL`,
|
||||
h.CompanyID, h.Email)
|
||||
|
||||
err = pg.QueryRow(ctx, `
|
||||
INSERT INTO invites (company_id, email, role, token, expires_at)
|
||||
VALUES ($1, lower($2), $3, $4, $5)
|
||||
RETURNING id`,
|
||||
h.CompanyID, h.Email, h.Role, auth.HashInviteToken(h.Token), h.ExpiresAt,
|
||||
).Scan(&h.InviteID)
|
||||
if err != nil {
|
||||
log.Printf("set-password invite for user_id=%s: %v", h.UserID, err)
|
||||
report["set_password_hooks_skipped"]++
|
||||
continue
|
||||
}
|
||||
hooks = append(hooks, h)
|
||||
report["set_password_hooks"]++
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return hooks, nil
|
||||
}
|
||||
|
||||
func writeSetPasswordArtifacts(mapsDir string, hooks []SetPasswordHook) error {
|
||||
if len(hooks) == 0 {
|
||||
return nil
|
||||
}
|
||||
if err := os.MkdirAll(mapsDir, 0o755); err != nil {
|
||||
return err
|
||||
}
|
||||
// Canonical mailhooks path + operator-friendly alias with URLs.
|
||||
hookPath := filepath.Join(mapsDir, "set-password-hooks.json")
|
||||
invitePath := filepath.Join(mapsDir, "password_invites.json")
|
||||
if err := writeJSON(hookPath, hooks); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := writeJSON(invitePath, hooks); err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Printf("wrote %d set-password invites to %s and %s (do not commit)\n", len(hooks), invitePath, hookPath)
|
||||
fmt.Println("=== Set-password invite URLs ===")
|
||||
for _, h := range hooks {
|
||||
fmt.Printf("%s\t%s\n", h.Email, h.URL)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// setPasswordByEmail is a local/dev bootstrap: force a password for one migrated user.
|
||||
func setPasswordByEmail(ctx context.Context, pg *pgxpool.Pool, emailPass string) error {
|
||||
parts := strings.SplitN(emailPass, ":", 2)
|
||||
if len(parts) != 2 || strings.TrimSpace(parts[0]) == "" || parts[1] == "" {
|
||||
return fmt.Errorf("-set-password expects email:password")
|
||||
}
|
||||
email := strings.ToLower(strings.TrimSpace(parts[0]))
|
||||
password := parts[1]
|
||||
hash, err := auth.HashPassword(password)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
ct, err := pg.Exec(ctx, `
|
||||
UPDATE users
|
||||
SET password_hash = $2, must_set_password = false, updated_at = now()
|
||||
WHERE lower(email) = $1`, email, hash)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if ct.RowsAffected() == 0 {
|
||||
return fmt.Errorf("no user with email %s", email)
|
||||
}
|
||||
fmt.Printf("set password for %s (must_set_password=false)\n", email)
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,34 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestSetPasswordInviteURL(t *testing.T) {
|
||||
t.Setenv("WEB_ORIGIN", "https://app.example.com/")
|
||||
got := setPasswordInviteURL("tok+1")
|
||||
wantPrefix := "https://app.example.com/accept-invite?token="
|
||||
if !strings.HasPrefix(got, wantPrefix) {
|
||||
t.Fatalf("got %q", got)
|
||||
}
|
||||
if !strings.Contains(got, "tok%2B1") && !strings.Contains(got, "tok+1") {
|
||||
t.Fatalf("token not in URL: %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSetPasswordByEmailParse(t *testing.T) {
|
||||
err := setPasswordByEmail(context.TODO(), nil, "bad")
|
||||
if err == nil || !strings.Contains(err.Error(), "email:password") {
|
||||
t.Fatalf("expected parse error, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestWebOriginDefault(t *testing.T) {
|
||||
os.Unsetenv("WEB_ORIGIN")
|
||||
if webOrigin() != "http://localhost:5174" {
|
||||
t.Fatalf("default origin")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,296 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"fmt"
|
||||
"log"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
)
|
||||
|
||||
// CountPair is one MySQL vs Postgres table count comparison.
|
||||
type CountPair struct {
|
||||
Entity string `json:"entity"`
|
||||
MySQL int64 `json:"mysql"`
|
||||
Postgres int64 `json:"postgres"`
|
||||
Delta int64 `json:"delta"`
|
||||
Note string `json:"note,omitempty"`
|
||||
}
|
||||
|
||||
// OrphanFinding is a remapped FK that does not resolve in Postgres.
|
||||
type OrphanFinding struct {
|
||||
Check string `json:"check"`
|
||||
Count int64 `json:"count"`
|
||||
Sample string `json:"sample,omitempty"`
|
||||
Pass bool `json:"pass"`
|
||||
}
|
||||
|
||||
// ValidationReport is written next to the ID map for cutover verification.
|
||||
type ValidationReport struct {
|
||||
Mode string `json:"mode"`
|
||||
Counts []CountPair `json:"counts"`
|
||||
PostgresCounts map[string]int64 `json:"postgres_counts,omitempty"`
|
||||
Orphans []OrphanFinding `json:"orphans"`
|
||||
OrphanSummary OrphanSummary `json:"orphan_summary"`
|
||||
OK bool `json:"ok"`
|
||||
}
|
||||
|
||||
// OrphanSummary is a compact end-of-run pass/fail tally.
|
||||
type OrphanSummary struct {
|
||||
Passed int `json:"passed"`
|
||||
Failed int `json:"failed"`
|
||||
Total int `json:"total"`
|
||||
}
|
||||
|
||||
func mysqlCount(ctx context.Context, db *sql.DB, table string) (int64, error) {
|
||||
quoted, err := quoteMySQLIdent(table)
|
||||
if err != nil {
|
||||
return -1, err
|
||||
}
|
||||
if !mysqlTableExists(ctx, db, table) {
|
||||
return -1, fmt.Errorf("missing")
|
||||
}
|
||||
var n int64
|
||||
err = db.QueryRowContext(ctx, "SELECT COUNT(*) FROM "+quoted).Scan(&n)
|
||||
return n, err
|
||||
}
|
||||
|
||||
func pgCount(ctx context.Context, pg *pgxpool.Pool, table string) (int64, error) {
|
||||
quoted, err := quotePGIdent(table)
|
||||
if err != nil {
|
||||
return -1, err
|
||||
}
|
||||
var n int64
|
||||
err = pg.QueryRow(ctx, "SELECT COUNT(*) FROM "+quoted).Scan(&n)
|
||||
return n, err
|
||||
}
|
||||
|
||||
// pgVerificationTables are Postgres targets printed at end-of-run.
|
||||
var pgVerificationTables = []string{
|
||||
"companies",
|
||||
"users",
|
||||
"memberships",
|
||||
"plans",
|
||||
"company_plans",
|
||||
"credit_balances",
|
||||
"categories",
|
||||
"attributes",
|
||||
"category_attributes",
|
||||
"custom_variables",
|
||||
"input_feeds",
|
||||
"feed_mappings",
|
||||
"export_feeds",
|
||||
"raw_products",
|
||||
"processed_products",
|
||||
"files",
|
||||
}
|
||||
|
||||
func collectPostgresCounts(ctx context.Context, pg *pgxpool.Pool, dryRun bool) map[string]int64 {
|
||||
out := map[string]int64{}
|
||||
if dryRun || pg == nil {
|
||||
return out
|
||||
}
|
||||
for _, t := range pgVerificationTables {
|
||||
if n, err := pgCount(ctx, pg, t); err == nil {
|
||||
out[t] = n
|
||||
} else {
|
||||
out[t] = -1
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// buildCountReport compares allowlisted MySQL source tables to Postgres targets.
|
||||
func buildCountReport(ctx context.Context, mysqlDB *sql.DB, pg *pgxpool.Pool, dryRun bool) []CountPair {
|
||||
pairs := []struct{ mysql, postgres, note string }{
|
||||
{"companies", "companies", ""},
|
||||
{"profiles", "memberships", "profiles → memberships"},
|
||||
{"users", "users", "no password_hash imported"},
|
||||
{"admin_users", "", "folded into users.is_platform_admin"},
|
||||
{"plans", "plans", ""},
|
||||
{"company_plans", "company_plans", ""},
|
||||
{"credit_balances", "credit_balances", ""},
|
||||
{"categories", "categories", ""},
|
||||
{"attributes", "attributes", ""},
|
||||
{"category_attributes", "category_attributes", ""},
|
||||
{"custom_variables", "custom_variables", "label/example → value"},
|
||||
{"xml_feeds", "input_feeds", "xml_feeds → input_feeds"},
|
||||
{"raw_products", "raw_products", ""},
|
||||
{"processed_products", "processed_products", ""},
|
||||
{"export_feeds", "export_feeds", ""},
|
||||
{"files", "files", "metadata only; blobs not copied"},
|
||||
{"company_settings", "company_settings", "partial: language + merge_products only"},
|
||||
{"api_keys", "", "not migrated; clients must create new keys"},
|
||||
{"processing_jobs", "processing_jobs", "migrated when domain jobs enabled (ai_provider_mode=migrated)"},
|
||||
}
|
||||
|
||||
out := make([]CountPair, 0, len(pairs))
|
||||
for _, p := range pairs {
|
||||
cp := CountPair{Entity: p.mysql, Note: p.note, MySQL: -1, Postgres: -1}
|
||||
if n, err := mysqlCount(ctx, mysqlDB, p.mysql); err == nil {
|
||||
cp.MySQL = n
|
||||
} else {
|
||||
cp.Note = strings.TrimSpace(cp.Note + " mysql_missing")
|
||||
}
|
||||
if p.postgres != "" && !dryRun {
|
||||
if n, err := pgCount(ctx, pg, p.postgres); err == nil {
|
||||
cp.Postgres = n
|
||||
if cp.MySQL >= 0 {
|
||||
cp.Delta = cp.Postgres - cp.MySQL
|
||||
}
|
||||
} else {
|
||||
cp.Note = strings.TrimSpace(cp.Note + " pg_error")
|
||||
}
|
||||
}
|
||||
out = append(out, cp)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// checkOrphanFKs runs Postgres-side orphan queries after a live load.
|
||||
// Dry-run skips (no writes to validate).
|
||||
func checkOrphanFKs(ctx context.Context, pg *pgxpool.Pool, dryRun bool) []OrphanFinding {
|
||||
if dryRun {
|
||||
return []OrphanFinding{{
|
||||
Check: "skipped_dry_run",
|
||||
Pass: true,
|
||||
Sample: "orphan FK checks require a live Postgres load",
|
||||
}}
|
||||
}
|
||||
|
||||
checks := []struct {
|
||||
name string
|
||||
sql string
|
||||
}{
|
||||
{"memberships_missing_user", `SELECT COUNT(*) FROM memberships m LEFT JOIN users u ON u.id = m.user_id WHERE u.id IS NULL`},
|
||||
{"memberships_missing_company", `SELECT COUNT(*) FROM memberships m LEFT JOIN companies c ON c.id = m.company_id WHERE c.id IS NULL`},
|
||||
{"categories_missing_company", `SELECT COUNT(*) FROM categories x LEFT JOIN companies c ON c.id = x.company_id WHERE c.id IS NULL`},
|
||||
{"attributes_missing_company", `SELECT COUNT(*) FROM attributes x LEFT JOIN companies c ON c.id = x.company_id WHERE c.id IS NULL`},
|
||||
{"custom_variables_missing_company", `SELECT COUNT(*) FROM custom_variables x LEFT JOIN companies c ON c.id = x.company_id WHERE c.id IS NULL`},
|
||||
{"raw_products_missing_company", `SELECT COUNT(*) FROM raw_products r LEFT JOIN companies c ON c.id = r.company_id WHERE c.id IS NULL`},
|
||||
{"raw_products_missing_feed", `SELECT COUNT(*) FROM raw_products r LEFT JOIN input_feeds f ON f.id = r.feed_id WHERE r.feed_id IS NOT NULL AND f.id IS NULL`},
|
||||
{"processed_missing_company", `SELECT COUNT(*) FROM processed_products p LEFT JOIN companies c ON c.id = p.company_id WHERE c.id IS NULL`},
|
||||
{"processed_missing_raw", `SELECT COUNT(*) FROM processed_products p LEFT JOIN raw_products r ON r.id = p.raw_product_id WHERE p.raw_product_id IS NOT NULL AND r.id IS NULL`},
|
||||
{"processed_missing_feed", `SELECT COUNT(*) FROM processed_products p LEFT JOIN input_feeds f ON f.id = p.feed_id WHERE p.feed_id IS NOT NULL AND f.id IS NULL`},
|
||||
{"export_feeds_missing_company", `SELECT COUNT(*) FROM export_feeds e LEFT JOIN companies c ON c.id = e.company_id WHERE c.id IS NULL`},
|
||||
{"export_feeds_missing_source", `SELECT COUNT(*) FROM export_feeds e LEFT JOIN input_feeds f ON f.id = e.source_feed_id WHERE e.source_feed_id IS NOT NULL AND f.id IS NULL`},
|
||||
{"feed_mappings_missing_feed", `SELECT COUNT(*) FROM feed_mappings m LEFT JOIN input_feeds f ON f.id = m.feed_id WHERE f.id IS NULL`},
|
||||
{"files_missing_company", `SELECT COUNT(*) FROM files f LEFT JOIN companies c ON c.id = f.company_id WHERE c.id IS NULL`},
|
||||
{"company_plans_missing_plan", `SELECT COUNT(*) FROM company_plans cp LEFT JOIN plans p ON p.id = cp.plan_id WHERE p.id IS NULL`},
|
||||
{"companies_without_active_plan", `SELECT COUNT(*) FROM companies c WHERE NOT EXISTS (SELECT 1 FROM company_plans cp WHERE cp.company_id = c.id AND cp.is_active = true)`},
|
||||
{"platform_admins", `SELECT COUNT(*) FROM users WHERE is_platform_admin = true`},
|
||||
}
|
||||
|
||||
out := make([]OrphanFinding, 0, len(checks))
|
||||
for _, c := range checks {
|
||||
var n int64
|
||||
err := pg.QueryRow(ctx, c.sql).Scan(&n)
|
||||
f := OrphanFinding{Check: c.name, Count: n, Pass: err == nil}
|
||||
if err != nil {
|
||||
f.Pass = false
|
||||
f.Sample = err.Error()
|
||||
} else if c.name == "platform_admins" {
|
||||
// Informational — not an orphan.
|
||||
f.Pass = true
|
||||
} else if c.name == "companies_without_active_plan" {
|
||||
// Informational cutover signal (use -list-companies-without-plans / -assign-missing-plans).
|
||||
f.Pass = true
|
||||
if n > 0 {
|
||||
f.Sample = fmt.Sprintf("%d companies lack an active plan", n)
|
||||
}
|
||||
} else {
|
||||
f.Pass = n == 0
|
||||
}
|
||||
out = append(out, f)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func summarizeOrphans(orphans []OrphanFinding) OrphanSummary {
|
||||
s := OrphanSummary{Total: len(orphans)}
|
||||
for _, o := range orphans {
|
||||
if o.Check == "skipped_dry_run" || o.Check == "skipped_fixture" || o.Check == "platform_admins" || o.Check == "companies_without_active_plan" {
|
||||
s.Passed++
|
||||
continue
|
||||
}
|
||||
if o.Pass {
|
||||
s.Passed++
|
||||
} else {
|
||||
s.Failed++
|
||||
}
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
func printValidation(v ValidationReport) {
|
||||
fmt.Println("=== Validation report ===")
|
||||
fmt.Printf("mode: %s ok=%v\n", v.Mode, v.OK)
|
||||
|
||||
fmt.Println("-- mysql vs postgres counts --")
|
||||
for _, c := range v.Counts {
|
||||
fmt.Printf("%s mysql=%d postgres=%d delta=%d %s\n", c.Entity, c.MySQL, c.Postgres, c.Delta, c.Note)
|
||||
}
|
||||
|
||||
if len(v.PostgresCounts) > 0 {
|
||||
fmt.Println("-- postgres table counts --")
|
||||
keys := make([]string, 0, len(v.PostgresCounts))
|
||||
for k := range v.PostgresCounts {
|
||||
keys = append(keys, k)
|
||||
}
|
||||
sort.Strings(keys)
|
||||
for _, k := range keys {
|
||||
fmt.Printf("%s: %d\n", k, v.PostgresCounts[k])
|
||||
}
|
||||
}
|
||||
|
||||
fmt.Println("-- orphans --")
|
||||
names := make([]string, 0, len(v.Orphans))
|
||||
byName := map[string]OrphanFinding{}
|
||||
for _, o := range v.Orphans {
|
||||
names = append(names, o.Check)
|
||||
byName[o.Check] = o
|
||||
}
|
||||
sort.Strings(names)
|
||||
for _, name := range names {
|
||||
o := byName[name]
|
||||
status := "PASS"
|
||||
if !o.Pass {
|
||||
status = "FAIL"
|
||||
}
|
||||
fmt.Printf("%s %s count=%d %s\n", status, o.Check, o.Count, o.Sample)
|
||||
}
|
||||
fmt.Printf("-- orphan summary -- passed=%d failed=%d total=%d\n",
|
||||
v.OrphanSummary.Passed, v.OrphanSummary.Failed, v.OrphanSummary.Total)
|
||||
}
|
||||
|
||||
func runValidation(
|
||||
ctx context.Context,
|
||||
mysqlDB *sql.DB,
|
||||
pg *pgxpool.Pool,
|
||||
dryRun bool,
|
||||
) ValidationReport {
|
||||
mode := "live"
|
||||
if dryRun {
|
||||
mode = "dry-run"
|
||||
}
|
||||
counts := buildCountReport(ctx, mysqlDB, pg, dryRun)
|
||||
pgCounts := collectPostgresCounts(ctx, pg, dryRun)
|
||||
orphans := checkOrphanFKs(ctx, pg, dryRun)
|
||||
summary := summarizeOrphans(orphans)
|
||||
ok := summary.Failed == 0
|
||||
v := ValidationReport{
|
||||
Mode: mode,
|
||||
Counts: counts,
|
||||
PostgresCounts: pgCounts,
|
||||
Orphans: orphans,
|
||||
OrphanSummary: summary,
|
||||
OK: ok,
|
||||
}
|
||||
if !ok {
|
||||
log.Printf("validation: orphan FK failures present — inspect report before DNS cutover")
|
||||
}
|
||||
return v
|
||||
}
|
||||
@@ -0,0 +1,53 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"regexp"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// SQL identifiers in this migrator are always static allowlisted names or
|
||||
// programmer-supplied column paths — never end-user free text. Still quote
|
||||
// and validate before interpolating into DDL/DML to fail closed on mistakes.
|
||||
|
||||
var sqlIdentSegment = regexp.MustCompile(`^[A-Za-z_][A-Za-z0-9_]*$`)
|
||||
|
||||
func quoteMySQLIdent(ident string) (string, error) {
|
||||
if !sqlIdentSegment.MatchString(ident) {
|
||||
return "", fmt.Errorf("invalid MySQL identifier %q", ident)
|
||||
}
|
||||
return "`" + strings.ReplaceAll(ident, "`", "``") + "`", nil
|
||||
}
|
||||
|
||||
// quoteMySQLIdentPath quotes dotted paths such as company_id or cf.company_id.
|
||||
func quoteMySQLIdentPath(path string) (string, error) {
|
||||
path = strings.TrimSpace(path)
|
||||
if path == "" {
|
||||
return "", fmt.Errorf("empty MySQL identifier path")
|
||||
}
|
||||
parts := strings.Split(path, ".")
|
||||
out := make([]string, len(parts))
|
||||
for i, part := range parts {
|
||||
q, err := quoteMySQLIdent(part)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
out[i] = q
|
||||
}
|
||||
return strings.Join(out, "."), nil
|
||||
}
|
||||
|
||||
func mustQuoteMySQLIdent(ident string) string {
|
||||
q, err := quoteMySQLIdent(ident)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
return q
|
||||
}
|
||||
|
||||
func quotePGIdent(ident string) (string, error) {
|
||||
if !sqlIdentSegment.MatchString(ident) {
|
||||
return "", fmt.Errorf("invalid Postgres identifier %q", ident)
|
||||
}
|
||||
return `"` + strings.ReplaceAll(ident, `"`, `""`) + `"`, nil
|
||||
}
|
||||
@@ -0,0 +1,42 @@
|
||||
package main
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestQuoteMySQLIdent(t *testing.T) {
|
||||
got, err := quoteMySQLIdent("order")
|
||||
if err != nil || got != "`order`" {
|
||||
t.Fatalf("order: got %q err=%v", got, err)
|
||||
}
|
||||
if _, err := quoteMySQLIdent("users; DROP TABLE x"); err == nil {
|
||||
t.Fatal("expected reject for injection payload")
|
||||
}
|
||||
if _, err := quoteMySQLIdent("a-b"); err == nil {
|
||||
t.Fatal("expected reject for hyphen")
|
||||
}
|
||||
}
|
||||
|
||||
func TestQuoteMySQLIdentPath(t *testing.T) {
|
||||
got, err := quoteMySQLIdentPath("cf.company_id")
|
||||
if err != nil || got != "`cf`.`company_id`" {
|
||||
t.Fatalf("path: got %q err=%v", got, err)
|
||||
}
|
||||
if _, err := quoteMySQLIdentPath("cf.company_id;--"); err == nil {
|
||||
t.Fatal("expected reject")
|
||||
}
|
||||
}
|
||||
|
||||
func TestQuotePGIdent(t *testing.T) {
|
||||
got, err := quotePGIdent("companies")
|
||||
if err != nil || got != `"companies"` {
|
||||
t.Fatalf("got %q err=%v", got, err)
|
||||
}
|
||||
if _, err := quotePGIdent(`companies" OR 1=1`); err == nil {
|
||||
t.Fatal("expected reject")
|
||||
}
|
||||
}
|
||||
|
||||
func TestMustQuoteMySQLIdent(t *testing.T) {
|
||||
if got := mustQuoteMySQLIdent("key"); got != "`key`" {
|
||||
t.Fatalf("got %q", got)
|
||||
}
|
||||
}
|
||||
+30
@@ -0,0 +1,30 @@
|
||||
{
|
||||
"companies": [
|
||||
{"id": "co_legacy_1", "name": "Acme Feeds", "language": "en"}
|
||||
],
|
||||
"users": [
|
||||
{"id": "user_clerk_admin", "email": "admin@example.com", "name": "Admin", "active": true},
|
||||
{"id": "user_clerk_member", "email": "member@example.com", "name": "Member", "active": true}
|
||||
],
|
||||
"admin_users": [
|
||||
{"user_id": "user_clerk_admin", "email": "admin@example.com"}
|
||||
],
|
||||
"profiles": [
|
||||
{"company_id": "co_legacy_1", "user_id": "user_clerk_admin", "role": "admin", "status": "active"},
|
||||
{"company_id": "co_legacy_1", "user_id": "user_clerk_member", "role": "member", "status": "active"}
|
||||
],
|
||||
"xml_feeds": [
|
||||
{
|
||||
"id": 101,
|
||||
"company_id": "co_legacy_1",
|
||||
"name": "Demo XML",
|
||||
"field_mappings": {
|
||||
"title": {"xpath": "/item/title", "fieldName": "title", "originalName": "title", "isRequired": true}
|
||||
}
|
||||
}
|
||||
],
|
||||
"files": [
|
||||
{"id": 1, "company_id": "co_legacy_1", "file_name": "upload.csv"}
|
||||
],
|
||||
"raw_products_count": 3
|
||||
}
|
||||
Reference in New Issue
Block a user