package main import ( "context" "database/sql" "encoding/json" "log" "strconv" "strings" "github.com/google/uuid" "github.com/jackc/pgx/v5" "github.com/jackc/pgx/v5/pgxpool" ) // migrateGapDomains loads settings, formulas (standard fields), tags, woo, and usage snapshots. func migrateGapDomains( ctx context.Context, mysqlDB *sql.DB, pg *pgxpool.Pool, companyMap map[string]string, feedMap map[string]string, allow map[string]bool, domains domainSet, report map[string]int, dryRun bool, ) { if domains.has("settings") { migrateCompanySettings(ctx, mysqlDB, pg, companyMap, allow, report, dryRun) } if domains.has("formulas") { migrateFieldGroupsAndStandards(ctx, mysqlDB, pg, companyMap, allow, report, dryRun) migrateStructuredDescriptionFields(ctx, mysqlDB, pg, companyMap, allow, report, dryRun) } if domains.has("tags") { migrateFeedTags(ctx, mysqlDB, pg, companyMap, feedMap, allow, report, dryRun) } if domains.has("woo") { migrateWooConfigs(ctx, mysqlDB, pg, companyMap, allow, report, dryRun) } if domains.has("usage") { migrateUsageIntoSettings(ctx, mysqlDB, pg, companyMap, allow, report, dryRun) } } func migrateCompanySettings( ctx context.Context, mysqlDB *sql.DB, pg *pgxpool.Pool, companyMap map[string]string, allow map[string]bool, report map[string]int, dryRun bool, ) { if !mysqlTableExists(ctx, mysqlDB, "company_settings") { log.Printf("company_settings skipped: table missing") return } lang := mysqlCoalesce(ctx, mysqlDB, "company_settings", "language", "'en'") merge := mysqlCoalesce(ctx, mysqlDB, "company_settings", "merge_products", "1") q := "SELECT company_id, " + lang + ", " + merge + " FROM company_settings WHERE company_id IS NOT NULL AND company_id <> ''" clause, args := mysqlCompanyFilter("company_id", allow) q += clause rows, err := mysqlDB.QueryContext(ctx, q, args...) if err != nil { log.Printf("company_settings skipped: %v", err) return } defer rows.Close() for rows.Next() { var companyLegacy, language string var mergeProducts int if err := rows.Scan(&companyLegacy, &language, &mergeProducts); err != nil { report["company_settings_skipped"]++ continue } cid, ok := companyMap[companyLegacy] if !ok { report["company_settings_skipped"]++ continue } if language == "" { language = "en" } settings := map[string]any{ "language": language, "merge_products": mergeProducts == 1, "_legacy": true, } b, _ := json.Marshal(settings) if dryRun { report["company_settings"]++ continue } _, err = pg.Exec(ctx, ` INSERT INTO company_settings (company_id, settings, updated_at) VALUES ($1, $2::jsonb, now()) ON CONFLICT (company_id) DO UPDATE SET settings = company_settings.settings || EXCLUDED.settings, updated_at = now()`, cid, string(b)) if err != nil { log.Printf("company_settings %s: %v", companyLegacy, err) report["company_settings_skipped"]++ continue } report["company_settings"]++ } } func migrateFieldGroupsAndStandards( ctx context.Context, mysqlDB *sql.DB, pg *pgxpool.Pool, companyMap map[string]string, allow map[string]bool, report map[string]int, dryRun bool, ) { if !mysqlTableExists(ctx, mysqlDB, "field_groups") { log.Printf("field_groups skipped: table missing") return } groupMap := map[string]string{} // legacy group uuid → new uuid clause, args := mysqlCompanyFilter("company_id", allow) orderCol := mustQuoteMySQLIdent("order") q := "SELECT id, company_id, name, COALESCE(description, ''), COALESCE(" + orderCol + ", 0), COALESCE(is_system, 0) FROM field_groups WHERE company_id IS NOT NULL AND company_id <> ''" + clause // MySQL may use order without backticks in some dumps — try fallbacks. rows, err := mysqlDB.QueryContext(ctx, q, args...) if err != nil { q2 := `SELECT id, company_id, name, COALESCE(description, ''), 0, COALESCE(is_system, 0) FROM field_groups WHERE company_id IS NOT NULL AND company_id <> ''` + clause rows, err = mysqlDB.QueryContext(ctx, q2, args...) } if err != nil { log.Printf("field_groups skipped: %v", err) return } defer rows.Close() for rows.Next() { var legacyID, companyLegacy, name, desc string var order, isSystem int if err := rows.Scan(&legacyID, &companyLegacy, &name, &desc, &order, &isSystem); err != nil { report["field_groups_skipped"]++ continue } cid, ok := companyMap[companyLegacy] if !ok { report["field_groups_skipped"]++ continue } newID := uuid.New() if parsed, err := uuid.Parse(legacyID); err == nil { newID = parsed // preserve UUID when already uuid-shaped } groupMap[legacyID] = newID.String() if dryRun { report["field_groups"]++ continue } _, err = pg.Exec(ctx, ` INSERT INTO field_groups (id, company_id, name, description, "order", is_system) VALUES ($1, $2, $3, NULLIF($4, ''), $5, $6) ON CONFLICT (id) DO UPDATE SET name = EXCLUDED.name, description = EXCLUDED.description, "order" = EXCLUDED."order", updated_at = now()`, newID, cid, name, desc, order, isSystem == 1) if err != nil { log.Printf("field_group %s: %v", legacyID, err) report["field_groups_skipped"]++ delete(groupMap, legacyID) continue } report["field_groups"]++ } if !mysqlTableExists(ctx, mysqlDB, "standard_fields") { return } keyCol := mustQuoteMySQLIdent("key") sq := "SELECT id, company_id, name, " + keyCol + ", type, group_id, COALESCE(is_required, 0), COALESCE(description, ''), COALESCE(default_value, ''), validation, COALESCE(is_system, 0) FROM standard_fields WHERE company_id IS NOT NULL AND company_id <> ''" + clause srows, err := mysqlDB.QueryContext(ctx, sq, args...) if err != nil { log.Printf("standard_fields skipped: %v", err) return } defer srows.Close() for srows.Next() { var legacyID, companyLegacy, name, key, typ, groupLegacy string var desc, defVal string var required, isSystem int var validation []byte if err := srows.Scan(&legacyID, &companyLegacy, &name, &key, &typ, &groupLegacy, &required, &desc, &defVal, &validation, &isSystem); err != nil { report["standard_fields_skipped"]++ continue } cid, ok := companyMap[companyLegacy] if !ok { report["standard_fields_skipped"]++ continue } gid, okG := groupMap[groupLegacy] if !okG { // group may already exist in PG with same UUID gid = groupLegacy } newID := uuid.New() if parsed, err := uuid.Parse(legacyID); err == nil { newID = parsed } if typ == "" { typ = "string" } if dryRun { report["standard_fields"]++ continue } _, err = pg.Exec(ctx, ` INSERT INTO standard_fields ( id, company_id, name, key, type, group_id, is_required, description, default_value, validation, is_system ) VALUES ( $1, $2, $3, $4, $5, $6::uuid, $7, NULLIF($8, ''), NULLIF($9, ''), COALESCE($10::jsonb, '{}'::jsonb), $11 ) ON CONFLICT (company_id, key) DO UPDATE SET name = EXCLUDED.name, type = EXCLUDED.type, group_id = EXCLUDED.group_id, is_required = EXCLUDED.is_required, description = EXCLUDED.description, default_value = EXCLUDED.default_value, validation = EXCLUDED.validation, updated_at = now()`, newID, cid, name, key, typ, gid, required == 1, desc, defVal, jsonOrNull(validation), isSystem == 1) if err != nil { log.Printf("standard_field %s: %v", legacyID, err) report["standard_fields_skipped"]++ continue } report["standard_fields"]++ } } func migrateStructuredDescriptionFields( ctx context.Context, mysqlDB *sql.DB, pg *pgxpool.Pool, companyMap map[string]string, allow map[string]bool, report map[string]int, dryRun bool, ) { if !mysqlTableExists(ctx, mysqlDB, "structured_description_fields") { log.Printf("structured_description_fields skipped: table missing") return } clause, args := mysqlCompanyFilter("company_id", allow) q := `SELECT id, company_id, field_key, COALESCE(type, 'text') FROM structured_description_fields WHERE company_id IS NOT NULL AND company_id <> ''` + clause rows, err := mysqlDB.QueryContext(ctx, q, args...) if err != nil { log.Printf("structured_description_fields skipped: %v", err) return } defer rows.Close() for rows.Next() { var legacyID, companyLegacy, fieldKey, typ string if err := rows.Scan(&legacyID, &companyLegacy, &fieldKey, &typ); err != nil { report["structured_description_fields_skipped"]++ continue } cid, ok := companyMap[companyLegacy] if !ok { report["structured_description_fields_skipped"]++ continue } newID := uuid.New() if parsed, err := uuid.Parse(legacyID); err == nil { newID = parsed } if dryRun { report["structured_description_fields"]++ continue } _, err = pg.Exec(ctx, ` INSERT INTO structured_description_fields (id, company_id, field_key, type) VALUES ($1, $2, $3, $4) ON CONFLICT (company_id, field_key) DO UPDATE SET type = EXCLUDED.type, updated_at = now()`, newID, cid, fieldKey, typ) if err != nil { log.Printf("structured_description_field %s: %v", legacyID, err) report["structured_description_fields_skipped"]++ continue } report["structured_description_fields"]++ } } func migrateFeedTags( ctx context.Context, mysqlDB *sql.DB, pg *pgxpool.Pool, companyMap, feedMap map[string]string, allow map[string]bool, report map[string]int, dryRun bool, ) { if !mysqlTableExists(ctx, mysqlDB, "feed_tags") { log.Printf("feed_tags skipped: table missing") return } tagMap := map[string]string{} clause, args := mysqlCompanyFilter("company_id", allow) q := `SELECT id, company_id, name, COALESCE(color, '#888888') FROM feed_tags WHERE company_id IS NOT NULL AND company_id <> ''` + clause rows, err := mysqlDB.QueryContext(ctx, q, args...) if err != nil { log.Printf("feed_tags skipped: %v", err) return } defer rows.Close() for rows.Next() { var legacyID int64 var companyLegacy, name, color string if err := rows.Scan(&legacyID, &companyLegacy, &name, &color); err != nil { report["feed_tags_skipped"]++ continue } cid, ok := companyMap[companyLegacy] if !ok { report["feed_tags_skipped"]++ continue } newID := uuid.New() tagMap[strconv.FormatInt(legacyID, 10)] = newID.String() if dryRun { report["feed_tags"]++ continue } _, err = pg.Exec(ctx, ` INSERT INTO feed_tags (id, company_id, name, color) VALUES ($1, $2, $3, $4) ON CONFLICT (company_id, name) DO UPDATE SET color = EXCLUDED.color`, newID, cid, name, color) if err != nil { var existing uuid.UUID if err2 := pg.QueryRow(ctx, `SELECT id FROM feed_tags WHERE company_id = $1 AND name = $2`, cid, name).Scan(&existing); err2 == nil { tagMap[strconv.FormatInt(legacyID, 10)] = existing.String() report["feed_tags"]++ continue } log.Printf("feed_tag %d: %v", legacyID, err) report["feed_tags_skipped"]++ continue } var existing uuid.UUID _ = pg.QueryRow(ctx, `SELECT id FROM feed_tags WHERE company_id = $1 AND name = $2`, cid, name).Scan(&existing) if existing != uuid.Nil { tagMap[strconv.FormatInt(legacyID, 10)] = existing.String() } report["feed_tags"]++ } if !mysqlTableExists(ctx, mysqlDB, "feed_tag_mappings") || len(tagMap) == 0 { return } mrows, err := mysqlDB.QueryContext(ctx, `SELECT feed_id, tag_id FROM feed_tag_mappings`) if err != nil { log.Printf("feed_tag_mappings skipped: %v", err) return } defer mrows.Close() for mrows.Next() { var feedLegacy, tagLegacy int64 if err := mrows.Scan(&feedLegacy, &tagLegacy); err != nil { report["feed_tag_mappings_skipped"]++ continue } fid, okF := feedMap[strconv.FormatInt(feedLegacy, 10)] tid, okT := tagMap[strconv.FormatInt(tagLegacy, 10)] if !okF || !okT { report["feed_tag_mappings_skipped"]++ continue } if dryRun { report["feed_tag_mappings"]++ continue } _, err = pg.Exec(ctx, ` INSERT INTO feed_tag_mappings (feed_id, tag_id) VALUES ($1, $2) ON CONFLICT DO NOTHING`, fid, tid) if err != nil { report["feed_tag_mappings_skipped"]++ continue } report["feed_tag_mappings"]++ } } func migrateWooConfigs( ctx context.Context, mysqlDB *sql.DB, pg *pgxpool.Pool, companyMap map[string]string, allow map[string]bool, report map[string]int, dryRun bool, ) { // Legacy Woo settings live as per-company custom_fields named wc_*. if !mysqlTableExists(ctx, mysqlDB, "custom_fields") || !mysqlTableExists(ctx, mysqlDB, "feed_custom_field_values") { log.Printf("woocommerce_configs skipped: custom_fields tables missing") report["woocommerce_configs_note"] = 1 return } clause, args := mysqlCompanyFilter("cf.company_id", allow) q := ` SELECT cf.company_id, cf.name, COALESCE(fcfv.value, '') FROM custom_fields cf JOIN feed_custom_field_values fcfv ON fcfv.custom_field_id = cf.id WHERE cf.name LIKE 'wc_%'` + clause rows, err := mysqlDB.QueryContext(ctx, q, args...) if err != nil { log.Printf("woocommerce_configs skipped: %v", err) return } defer rows.Close() byCompany := map[string]map[string]string{} for rows.Next() { var companyLegacy, name, value string if err := rows.Scan(&companyLegacy, &name, &value); err != nil { continue } if _, ok := companyMap[companyLegacy]; !ok { continue } if byCompany[companyLegacy] == nil { byCompany[companyLegacy] = map[string]string{} } byCompany[companyLegacy][name] = value } if len(byCompany) == 0 { log.Printf("woocommerce_configs: no wc_* custom fields found (ok)") report["woocommerce_configs"] = 0 return } for legacyCID, fields := range byCompany { cid := companyMap[legacyCID] enabled := strings.EqualFold(fields["wc_enabled"], "true") || fields["wc_enabled"] == "1" storeURL := firstNonEmpty(fields["wc_store_url"], fields["wc_url"], fields["wc_store"]) consumerKey := firstNonEmpty(fields["wc_consumer_key"], fields["wc_key"]) consumerSecret := firstNonEmpty(fields["wc_consumer_secret"], fields["wc_secret"]) syncOpts, _ := json.Marshal(fields) if dryRun { report["woocommerce_configs"]++ continue } _, err = pg.Exec(ctx, ` INSERT INTO woocommerce_configs ( company_id, store_url, consumer_key, consumer_secret, is_enabled, sync_options ) VALUES ($1, $2, $3, $4, $5, $6::jsonb) ON CONFLICT (company_id) DO UPDATE SET store_url = EXCLUDED.store_url, consumer_key = EXCLUDED.consumer_key, consumer_secret = EXCLUDED.consumer_secret, is_enabled = EXCLUDED.is_enabled, sync_options = EXCLUDED.sync_options, updated_at = now()`, cid, storeURL, consumerKey, consumerSecret, enabled, string(syncOpts)) if err != nil { log.Printf("woocommerce_configs %s: %v", legacyCID, err) report["woocommerce_configs_skipped"]++ continue } report["woocommerce_configs"]++ } } // migrateUsageIntoSettings folds latest usage_metrics into company_settings.settings._legacy_usage. // v2 has no usage_metrics table; this preserves a portable snapshot without N+1. func migrateUsageIntoSettings( ctx context.Context, mysqlDB *sql.DB, pg *pgxpool.Pool, companyMap map[string]string, allow map[string]bool, report map[string]int, dryRun bool, ) { if !mysqlTableExists(ctx, mysqlDB, "usage_metrics") { log.Printf("usage_metrics skipped: table missing") return } clause, args := mysqlCompanyFilter("company_id", allow) q := ` SELECT company_id, SUM(COALESCE(credits_used, 0)), MAX(COALESCE(total_products, 0)), COUNT(*) FROM usage_metrics WHERE company_id IS NOT NULL AND company_id <> ''` + clause + ` GROUP BY company_id` rows, err := mysqlDB.QueryContext(ctx, q, args...) if err != nil { log.Printf("usage_metrics skipped: %v", err) return } defer rows.Close() type snap struct { CreditsUsed float64 `json:"credits_used_sum"` MaxProducts int `json:"max_total_products"` MetricDays int `json:"metric_days"` } batch := make([][]any, 0) for rows.Next() { var companyLegacy string var credits float64 var maxProducts, days int if err := rows.Scan(&companyLegacy, &credits, &maxProducts, &days); err != nil { report["usage_metrics_skipped"]++ continue } cid, ok := companyMap[companyLegacy] if !ok { report["usage_metrics_skipped"]++ continue } payload, _ := json.Marshal(map[string]any{ "_legacy_usage": snap{CreditsUsed: credits, MaxProducts: maxProducts, MetricDays: days}, }) if dryRun { report["usage_metrics"]++ continue } batch = append(batch, []any{cid, string(payload)}) report["usage_metrics"]++ } if dryRun || len(batch) == 0 { return } tx, err := pg.Begin(ctx) if err != nil { log.Printf("usage_metrics tx: %v", err) return } defer tx.Rollback(ctx) b := &pgx.Batch{} for _, row := range batch { b.Queue(` INSERT INTO company_settings (company_id, settings, updated_at) VALUES ($1, $2::jsonb, now()) ON CONFLICT (company_id) DO UPDATE SET settings = company_settings.settings || EXCLUDED.settings, updated_at = now()`, row...) } br := tx.SendBatch(ctx, b) if err := br.Close(); err != nil { log.Printf("usage_metrics batch: %v", err) return } if err := tx.Commit(ctx); err != nil { log.Printf("usage_metrics commit: %v", err) } // usage_limits → settings._legacy_usage_limits when present if mysqlTableExists(ctx, mysqlDB, "usage_limits") { lq := `SELECT company_id, tokens_per_minute, requests_per_minute, tokens_per_day, cost_limit, COALESCE(is_active, 1) FROM usage_limits WHERE company_id IS NOT NULL AND company_id <> ''` lc, la := mysqlCompanyFilter("company_id", allow) lrows, err := mysqlDB.QueryContext(ctx, lq+lc, la...) if err == nil { defer lrows.Close() for lrows.Next() { var companyLegacy string var tpm, rpm, tpd int var costLimit sql.NullInt64 var active int if err := lrows.Scan(&companyLegacy, &tpm, &rpm, &tpd, &costLimit, &active); err != nil { continue } cid, ok := companyMap[companyLegacy] if !ok { continue } lim := map[string]any{ "tokens_per_minute": tpm, "requests_per_minute": rpm, "tokens_per_day": tpd, "is_active": active == 1, } if costLimit.Valid { lim["cost_limit"] = costLimit.Int64 } payload, _ := json.Marshal(map[string]any{"_legacy_usage_limits": lim}) _, _ = pg.Exec(ctx, ` INSERT INTO company_settings (company_id, settings, updated_at) VALUES ($1, $2::jsonb, now()) ON CONFLICT (company_id) DO UPDATE SET settings = company_settings.settings || EXCLUDED.settings, updated_at = now()`, cid, string(payload)) report["usage_limits"]++ } } } } func firstNonEmpty(vals ...string) string { for _, v := range vals { if strings.TrimSpace(v) != "" { return strings.TrimSpace(v) } } return "" }