package main import ( "context" "database/sql" "log" "strconv" "strings" "time" "github.com/google/uuid" "github.com/jackc/pgx/v5/pgxpool" ) // migrateJobsDomain imports legacy processing_jobs (+ best-effort job products) and tasks. // Job rows are tagged ai_provider_mode='migrated' so retention cleanup preserves history. func migrateJobsDomain( ctx context.Context, mysqlDB *sql.DB, pg *pgxpool.Pool, companyMap, userMap, rawMap map[string]string, allow map[string]bool, domains domainSet, report map[string]int, dryRun bool, ) { if !domains.has("jobs") { return } migrateProcessingJobs(ctx, mysqlDB, pg, companyMap, userMap, allow, report, dryRun) migrateProcessingJobProducts(ctx, mysqlDB, pg, rawMap, allow, report, dryRun) migrateLegacyTasks(ctx, mysqlDB, pg, companyMap, userMap, allow, report, dryRun) } func migrateProcessingJobs( ctx context.Context, mysqlDB *sql.DB, pg *pgxpool.Pool, companyMap, userMap map[string]string, allow map[string]bool, report map[string]int, dryRun bool, ) { if !mysqlTableExists(ctx, mysqlDB, "processing_jobs") { log.Printf("processing_jobs skipped: table missing") return } q := mysqlSelectList( "id", "company_id", mysqlCol(ctx, mysqlDB, "processing_jobs", "user_id", "NULL"), mysqlCoalesce(ctx, mysqlDB, "processing_jobs", "status", "'pending'"), mysqlCoalesce(ctx, mysqlDB, "processing_jobs", "total_products", "0"), mysqlCoalesce(ctx, mysqlDB, "processing_jobs", "processed_products", "0"), mysqlCol(ctx, mysqlDB, "processing_jobs", "error", "NULL"), mysqlCoalesce(ctx, mysqlDB, "processing_jobs", "processing_type", "'full'"), mysqlCoalesce(ctx, mysqlDB, "processing_jobs", "priority", "0"), mysqlCoalesce(ctx, mysqlDB, "processing_jobs", "estimated_tokens", "0"), mysqlCol(ctx, mysqlDB, "processing_jobs", "started_at", "NULL"), mysqlCol(ctx, mysqlDB, "processing_jobs", "completed_at", "NULL"), mysqlCoalesce(ctx, mysqlDB, "processing_jobs", "created_at", "NOW()"), mysqlCoalesce(ctx, mysqlDB, "processing_jobs", "updated_at", "NOW()"), ) + " FROM processing_jobs WHERE 1=1" clause, cargs := mysqlCompanyFilter("company_id", allow) q += clause rows, err := mysqlDB.QueryContext(ctx, q, cargs...) if err != nil { log.Printf("processing_jobs skipped: %v", err) return } defer rows.Close() for rows.Next() { var ( legacyID, companyLegacy string userLegacy sql.NullString status, processingType string errText sql.NullString totalProducts any processedProducts any priority any estimatedTokens any startedAt, completedAt sql.NullTime createdAt, updatedAt time.Time ) if err := rows.Scan( &legacyID, &companyLegacy, &userLegacy, &status, &totalProducts, &processedProducts, &errText, &processingType, &priority, &estimatedTokens, &startedAt, &completedAt, &createdAt, &updatedAt, ); err != nil { report["processing_jobs_skipped"]++ continue } cid, ok := companyMap[companyLegacy] if !ok { report["processing_jobs_skipped"]++ continue } jobID, err := uuid.Parse(strings.TrimSpace(legacyID)) if err != nil { // Legacy dump mixes UUID and numeric string PKs — keep remaps stable. jobID = uuid.NewSHA1(uuid.NameSpaceOID, []byte("processing_job:"+strings.TrimSpace(legacyID))) } var userID *uuid.UUID if userLegacy.Valid && strings.TrimSpace(userLegacy.String) != "" { if mapped, ok := userMap[userLegacy.String]; ok { if parsed, err := uuid.Parse(mapped); err == nil { if !dryRun { var exists bool _ = pg.QueryRow(ctx, `SELECT EXISTS(SELECT 1 FROM users WHERE id = $1)`, parsed).Scan(&exists) if exists { userID = &parsed } else { report["processing_jobs_user_missing"]++ } } else { userID = &parsed } } } else { report["processing_jobs_user_unmapped"]++ } } normStatus := normalizeProcessingJobStatus(status) ptype := strings.TrimSpace(processingType) if ptype == "" { ptype = "full" } var errPtr *string if errText.Valid && strings.TrimSpace(errText.String) != "" { v := errText.String errPtr = &v } var startedPtr, completedPtr *time.Time if startedAt.Valid { t := startedAt.Time.UTC() startedPtr = &t } if completedAt.Valid { t := completedAt.Time.UTC() completedPtr = &t } if dryRun { report["processing_jobs"]++ continue } _, err = pg.Exec(ctx, ` INSERT INTO processing_jobs ( id, company_id, user_id, status, total_products, processed_products, error, processing_type, priority, estimated_tokens, started_at, completed_at, created_at, updated_at, current_step, step_progress, ai_provider_mode ) VALUES ( $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, '', '[]'::jsonb, 'migrated' ) ON CONFLICT (id) DO UPDATE SET company_id = EXCLUDED.company_id, user_id = EXCLUDED.user_id, status = EXCLUDED.status, total_products = EXCLUDED.total_products, processed_products = EXCLUDED.processed_products, error = EXCLUDED.error, processing_type = EXCLUDED.processing_type, priority = EXCLUDED.priority, estimated_tokens = EXCLUDED.estimated_tokens, started_at = EXCLUDED.started_at, completed_at = EXCLUDED.completed_at, created_at = EXCLUDED.created_at, updated_at = EXCLUDED.updated_at, ai_provider_mode = 'migrated'`, jobID, cid, userID, normStatus, scanIntish(totalProducts), scanIntish(processedProducts), errPtr, ptype, scanIntish(priority), scanIntish(estimatedTokens), startedPtr, completedPtr, createdAt.UTC(), updatedAt.UTC(), ) if err != nil { log.Printf("processing_job %s: %v", legacyID, err) report["processing_jobs_skipped"]++ continue } report["processing_jobs"]++ } if err := rows.Err(); err != nil { log.Printf("processing_jobs rows: %v", err) } } func migrateProcessingJobProducts( ctx context.Context, mysqlDB *sql.DB, pg *pgxpool.Pool, rawMap map[string]string, allow map[string]bool, report map[string]int, dryRun bool, ) { if !mysqlTableExists(ctx, mysqlDB, "processing_job_products") { return } if !mysqlTableExists(ctx, mysqlDB, "processing_jobs") { return } q := ` SELECT pjp.id, pjp.job_id, pjp.raw_product_id, pjp.status, pjp.error, pjp.processed_product_id, pjp.created_at, pjp.updated_at FROM processing_job_products pjp JOIN processing_jobs pj ON pj.id = pjp.job_id WHERE 1=1` clause, cargs := mysqlCompanyFilter("pj.company_id", allow) q += clause rows, err := mysqlDB.QueryContext(ctx, q, cargs...) if err != nil { log.Printf("processing_job_products skipped: %v", err) return } defer rows.Close() for rows.Next() { var ( legacyProdID int64 jobLegacy string rawLegacy any status string errText sql.NullString processedLegacy sql.NullInt64 createdAt, updatedAt time.Time ) if err := rows.Scan( &legacyProdID, &jobLegacy, &rawLegacy, &status, &errText, &processedLegacy, &createdAt, &updatedAt, ); err != nil { report["processing_job_products_skipped"]++ continue } jobID, err := uuid.Parse(strings.TrimSpace(jobLegacy)) if err != nil { jobID = uuid.NewSHA1(uuid.NameSpaceOID, []byte("processing_job:"+strings.TrimSpace(jobLegacy))) } rawKey := strings.TrimSpace(stringifyAnyID(rawLegacy)) rawUUIDStr, ok := rawMap[rawKey] if !ok { report["processing_job_products_skipped"]++ continue } rawUUID, err := uuid.Parse(rawUUIDStr) if err != nil { report["processing_job_products_skipped"]++ continue } if dryRun { report["processing_job_products"]++ continue } // Only attach when the remapped raw product still exists (GTIN dedupe may drop some). var exists bool if err := pg.QueryRow(ctx, `SELECT EXISTS(SELECT 1 FROM raw_products WHERE id = $1)`, rawUUID).Scan(&exists); err != nil || !exists { report["processing_job_products_skipped"]++ continue } var jobExists bool if err := pg.QueryRow(ctx, `SELECT EXISTS(SELECT 1 FROM processing_jobs WHERE id = $1)`, jobID).Scan(&jobExists); err != nil || !jobExists { report["processing_job_products_skipped"]++ continue } var errPtr *string if errText.Valid && strings.TrimSpace(errText.String) != "" { v := errText.String errPtr = &v } // Stable UUID from legacy int so resume is idempotent. prodID := uuid.NewSHA1(uuid.NameSpaceOID, []byte("pjp:"+strconv.FormatInt(legacyProdID, 10))) _, err = pg.Exec(ctx, ` INSERT INTO processing_job_products ( id, job_id, raw_product_id, status, error, processed_product_id, created_at, updated_at ) VALUES ($1, $2, $3, $4, $5, NULL, $6, $7) ON CONFLICT (id) DO UPDATE SET status = EXCLUDED.status, error = EXCLUDED.error, updated_at = EXCLUDED.updated_at`, prodID, jobID, rawUUID, normalizeJobProductStatus(status), errPtr, createdAt.UTC(), updatedAt.UTC(), ) if err != nil { report["processing_job_products_skipped"]++ continue } report["processing_job_products"]++ } if err := rows.Err(); err != nil { log.Printf("processing_job_products rows: %v", err) } } func migrateLegacyTasks( ctx context.Context, mysqlDB *sql.DB, pg *pgxpool.Pool, companyMap, userMap map[string]string, allow map[string]bool, report map[string]int, dryRun bool, ) { if !mysqlTableExists(ctx, mysqlDB, "tasks") { return } q := mysqlSelectList( "id", mysqlCol(ctx, mysqlDB, "tasks", "company_id", "NULL"), mysqlCol(ctx, mysqlDB, "tasks", "user_id", "NULL"), mysqlCoalesce(ctx, mysqlDB, "tasks", "task_name", "''"), mysqlCoalesce(ctx, mysqlDB, "tasks", "status", "'pending'"), mysqlCol(ctx, mysqlDB, "tasks", "start_time", "NULL"), mysqlCol(ctx, mysqlDB, "tasks", "end_time", "NULL"), mysqlCol(ctx, mysqlDB, "tasks", "log", "NULL"), mysqlCoalesce(ctx, mysqlDB, "tasks", "processing_products", "0"), mysqlCoalesce(ctx, mysqlDB, "tasks", "processed_products", "0"), mysqlCoalesce(ctx, mysqlDB, "tasks", "total_products", "0"), mysqlCol(ctx, mysqlDB, "tasks", "error_products", "NULL"), mysqlCol(ctx, mysqlDB, "tasks", "product_ids", "NULL"), mysqlCoalesce(ctx, mysqlDB, "tasks", "created_at", "NOW()"), mysqlCoalesce(ctx, mysqlDB, "tasks", "updated_at", "NOW()"), ) + " FROM tasks WHERE 1=1" clause, cargs := mysqlCompanyFilter("company_id", allow) q += clause rows, err := mysqlDB.QueryContext(ctx, q, cargs...) if err != nil { log.Printf("tasks skipped: %v", err) return } defer rows.Close() for rows.Next() { var ( legacyID int64 companyLegacy, userLegacy sql.NullString taskName, status string startTime, endTime sql.NullTime logText sql.NullString processingProducts, processedProducts, totalP any errorProducts, productIDs sql.NullString createdAt, updatedAt time.Time ) if err := rows.Scan( &legacyID, &companyLegacy, &userLegacy, &taskName, &status, &startTime, &endTime, &logText, &processingProducts, &processedProducts, &totalP, &errorProducts, &productIDs, &createdAt, &updatedAt, ); err != nil { report["tasks_skipped"]++ continue } if !companyLegacy.Valid || strings.TrimSpace(companyLegacy.String) == "" { report["tasks_skipped"]++ continue } cid, ok := companyMap[companyLegacy.String] if !ok { report["tasks_skipped"]++ continue } var userID *uuid.UUID if userLegacy.Valid && strings.TrimSpace(userLegacy.String) != "" { if mapped, ok := userMap[userLegacy.String]; ok { if parsed, err := uuid.Parse(mapped); err == nil { if !dryRun { var exists bool _ = pg.QueryRow(ctx, `SELECT EXISTS(SELECT 1 FROM users WHERE id = $1)`, parsed).Scan(&exists) if exists { userID = &parsed } } else { userID = &parsed } } } } taskID := uuid.NewSHA1(uuid.NameSpaceOID, []byte("task:"+strconv.FormatInt(legacyID, 10))) var startPtr, endPtr *time.Time if startTime.Valid { t := startTime.Time.UTC() startPtr = &t } if endTime.Valid { t := endTime.Time.UTC() endPtr = &t } var logPtr *string if logText.Valid { v := logText.String logPtr = &v } errJSON := nullJSON(errorProducts) prodJSON := nullJSON(productIDs) if dryRun { report["tasks"]++ continue } _, err = pg.Exec(ctx, ` INSERT INTO tasks ( id, company_id, user_id, task_name, status, start_time, end_time, log, processing_products, processed_products, total_products, error_products, product_ids, created_at, updated_at ) VALUES ( $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12::jsonb, $13::jsonb, $14, $15 ) ON CONFLICT (id) DO UPDATE SET status = EXCLUDED.status, end_time = EXCLUDED.end_time, log = EXCLUDED.log, processed_products = EXCLUDED.processed_products, updated_at = EXCLUDED.updated_at`, taskID, cid, userID, taskName, strings.TrimSpace(status), startPtr, endPtr, logPtr, scanIntish(processingProducts), scanIntish(processedProducts), scanIntish(totalP), errJSON, prodJSON, createdAt.UTC(), updatedAt.UTC(), ) if err != nil { log.Printf("task %d: %v", legacyID, err) report["tasks_skipped"]++ continue } report["tasks"]++ } if err := rows.Err(); err != nil { log.Printf("tasks rows: %v", err) } } func normalizeProcessingJobStatus(raw string) string { switch strings.ToLower(strings.TrimSpace(raw)) { case "completed", "success", "done": return "completed" case "failed", "error": return "failed" case "cancelled", "canceled", "skipped": return "cancelled" case "running", "processing": return "running" case "pending", "queued": return "pending" default: return "failed" } } func normalizeJobProductStatus(raw string) string { switch strings.ToLower(strings.TrimSpace(raw)) { case "processed", "completed", "success", "done": return "processed" case "failed", "error": return "failed" case "cancelled", "canceled", "skipped": return "cancelled" case "processing", "running": return "processing" case "pending", "queued": return "pending" default: return "failed" } } func stringifyAnyID(v any) string { switch x := v.(type) { case nil: return "" case int64: return strconv.FormatInt(x, 10) case int32: return strconv.FormatInt(int64(x), 10) case float64: return strconv.FormatInt(int64(x), 10) case []byte: return strings.TrimSpace(string(x)) case string: return strings.TrimSpace(x) default: n := scanIntish(v) if n != 0 { return strconv.Itoa(n) } return "" } } func nullJSON(ns sql.NullString) any { if !ns.Valid || strings.TrimSpace(ns.String) == "" { return nil } return strings.TrimSpace(ns.String) }