Files
descrybe/apps/api/cmd/migrator/jobs.go
T

497 lines
15 KiB
Go
Raw Normal View History

package main
import (
"context"
"database/sql"
"log"
"strconv"
"strings"
"time"
"github.com/google/uuid"
"github.com/jackc/pgx/v5/pgxpool"
)
// migrateJobsDomain imports legacy processing_jobs (+ best-effort job products) and tasks.
// Job rows are tagged ai_provider_mode='migrated' so retention cleanup preserves history.
func migrateJobsDomain(
ctx context.Context,
mysqlDB *sql.DB,
pg *pgxpool.Pool,
companyMap, userMap, rawMap map[string]string,
allow map[string]bool,
domains domainSet,
report map[string]int,
dryRun bool,
) {
if !domains.has("jobs") {
return
}
migrateProcessingJobs(ctx, mysqlDB, pg, companyMap, userMap, allow, report, dryRun)
migrateProcessingJobProducts(ctx, mysqlDB, pg, rawMap, allow, report, dryRun)
migrateLegacyTasks(ctx, mysqlDB, pg, companyMap, userMap, allow, report, dryRun)
}
func migrateProcessingJobs(
ctx context.Context,
mysqlDB *sql.DB,
pg *pgxpool.Pool,
companyMap, userMap map[string]string,
allow map[string]bool,
report map[string]int,
dryRun bool,
) {
if !mysqlTableExists(ctx, mysqlDB, "processing_jobs") {
log.Printf("processing_jobs skipped: table missing")
return
}
q := mysqlSelectList(
"id",
"company_id",
mysqlCol(ctx, mysqlDB, "processing_jobs", "user_id", "NULL"),
mysqlCoalesce(ctx, mysqlDB, "processing_jobs", "status", "'pending'"),
mysqlCoalesce(ctx, mysqlDB, "processing_jobs", "total_products", "0"),
mysqlCoalesce(ctx, mysqlDB, "processing_jobs", "processed_products", "0"),
mysqlCol(ctx, mysqlDB, "processing_jobs", "error", "NULL"),
mysqlCoalesce(ctx, mysqlDB, "processing_jobs", "processing_type", "'full'"),
mysqlCoalesce(ctx, mysqlDB, "processing_jobs", "priority", "0"),
mysqlCoalesce(ctx, mysqlDB, "processing_jobs", "estimated_tokens", "0"),
mysqlCol(ctx, mysqlDB, "processing_jobs", "started_at", "NULL"),
mysqlCol(ctx, mysqlDB, "processing_jobs", "completed_at", "NULL"),
mysqlCoalesce(ctx, mysqlDB, "processing_jobs", "created_at", "NOW()"),
mysqlCoalesce(ctx, mysqlDB, "processing_jobs", "updated_at", "NOW()"),
) + " FROM processing_jobs WHERE 1=1"
clause, cargs := mysqlCompanyFilter("company_id", allow)
q += clause
rows, err := mysqlDB.QueryContext(ctx, q, cargs...)
if err != nil {
log.Printf("processing_jobs skipped: %v", err)
return
}
defer rows.Close()
for rows.Next() {
var (
legacyID, companyLegacy string
userLegacy sql.NullString
status, processingType string
errText sql.NullString
totalProducts any
processedProducts any
priority any
estimatedTokens any
startedAt, completedAt sql.NullTime
createdAt, updatedAt time.Time
)
if err := rows.Scan(
&legacyID, &companyLegacy, &userLegacy, &status, &totalProducts, &processedProducts,
&errText, &processingType, &priority, &estimatedTokens,
&startedAt, &completedAt, &createdAt, &updatedAt,
); err != nil {
report["processing_jobs_skipped"]++
continue
}
cid, ok := companyMap[companyLegacy]
if !ok {
report["processing_jobs_skipped"]++
continue
}
jobID, err := uuid.Parse(strings.TrimSpace(legacyID))
if err != nil {
// Legacy dump mixes UUID and numeric string PKs — keep remaps stable.
jobID = uuid.NewSHA1(uuid.NameSpaceOID, []byte("processing_job:"+strings.TrimSpace(legacyID)))
}
var userID *uuid.UUID
if userLegacy.Valid && strings.TrimSpace(userLegacy.String) != "" {
if mapped, ok := userMap[userLegacy.String]; ok {
if parsed, err := uuid.Parse(mapped); err == nil {
if !dryRun {
var exists bool
_ = pg.QueryRow(ctx, `SELECT EXISTS(SELECT 1 FROM users WHERE id = $1)`, parsed).Scan(&exists)
if exists {
userID = &parsed
} else {
report["processing_jobs_user_missing"]++
}
} else {
userID = &parsed
}
}
} else {
report["processing_jobs_user_unmapped"]++
}
}
normStatus := normalizeProcessingJobStatus(status)
ptype := strings.TrimSpace(processingType)
if ptype == "" {
ptype = "full"
}
var errPtr *string
if errText.Valid && strings.TrimSpace(errText.String) != "" {
v := errText.String
errPtr = &v
}
var startedPtr, completedPtr *time.Time
if startedAt.Valid {
t := startedAt.Time.UTC()
startedPtr = &t
}
if completedAt.Valid {
t := completedAt.Time.UTC()
completedPtr = &t
}
if dryRun {
report["processing_jobs"]++
continue
}
_, err = pg.Exec(ctx, `
INSERT INTO processing_jobs (
id, company_id, user_id, status, total_products, processed_products,
error, processing_type, priority, estimated_tokens,
started_at, completed_at, created_at, updated_at,
current_step, step_progress, ai_provider_mode
) VALUES (
$1, $2, $3, $4, $5, $6,
$7, $8, $9, $10,
$11, $12, $13, $14,
'', '[]'::jsonb, 'migrated'
)
ON CONFLICT (id) DO UPDATE SET
company_id = EXCLUDED.company_id,
user_id = EXCLUDED.user_id,
status = EXCLUDED.status,
total_products = EXCLUDED.total_products,
processed_products = EXCLUDED.processed_products,
error = EXCLUDED.error,
processing_type = EXCLUDED.processing_type,
priority = EXCLUDED.priority,
estimated_tokens = EXCLUDED.estimated_tokens,
started_at = EXCLUDED.started_at,
completed_at = EXCLUDED.completed_at,
created_at = EXCLUDED.created_at,
updated_at = EXCLUDED.updated_at,
ai_provider_mode = 'migrated'`,
jobID, cid, userID, normStatus,
scanIntish(totalProducts), scanIntish(processedProducts),
errPtr, ptype, scanIntish(priority), scanIntish(estimatedTokens),
startedPtr, completedPtr, createdAt.UTC(), updatedAt.UTC(),
)
if err != nil {
log.Printf("processing_job %s: %v", legacyID, err)
report["processing_jobs_skipped"]++
continue
}
report["processing_jobs"]++
}
if err := rows.Err(); err != nil {
log.Printf("processing_jobs rows: %v", err)
}
}
func migrateProcessingJobProducts(
ctx context.Context,
mysqlDB *sql.DB,
pg *pgxpool.Pool,
rawMap map[string]string,
allow map[string]bool,
report map[string]int,
dryRun bool,
) {
if !mysqlTableExists(ctx, mysqlDB, "processing_job_products") {
return
}
if !mysqlTableExists(ctx, mysqlDB, "processing_jobs") {
return
}
q := `
SELECT pjp.id, pjp.job_id, pjp.raw_product_id, pjp.status, pjp.error,
pjp.processed_product_id, pjp.created_at, pjp.updated_at
FROM processing_job_products pjp
JOIN processing_jobs pj ON pj.id = pjp.job_id
WHERE 1=1`
clause, cargs := mysqlCompanyFilter("pj.company_id", allow)
q += clause
rows, err := mysqlDB.QueryContext(ctx, q, cargs...)
if err != nil {
log.Printf("processing_job_products skipped: %v", err)
return
}
defer rows.Close()
for rows.Next() {
var (
legacyProdID int64
jobLegacy string
rawLegacy any
status string
errText sql.NullString
processedLegacy sql.NullInt64
createdAt, updatedAt time.Time
)
if err := rows.Scan(
&legacyProdID, &jobLegacy, &rawLegacy, &status, &errText,
&processedLegacy, &createdAt, &updatedAt,
); err != nil {
report["processing_job_products_skipped"]++
continue
}
jobID, err := uuid.Parse(strings.TrimSpace(jobLegacy))
if err != nil {
jobID = uuid.NewSHA1(uuid.NameSpaceOID, []byte("processing_job:"+strings.TrimSpace(jobLegacy)))
}
rawKey := strings.TrimSpace(stringifyAnyID(rawLegacy))
rawUUIDStr, ok := rawMap[rawKey]
if !ok {
report["processing_job_products_skipped"]++
continue
}
rawUUID, err := uuid.Parse(rawUUIDStr)
if err != nil {
report["processing_job_products_skipped"]++
continue
}
if dryRun {
report["processing_job_products"]++
continue
}
// Only attach when the remapped raw product still exists (GTIN dedupe may drop some).
var exists bool
if err := pg.QueryRow(ctx, `SELECT EXISTS(SELECT 1 FROM raw_products WHERE id = $1)`, rawUUID).Scan(&exists); err != nil || !exists {
report["processing_job_products_skipped"]++
continue
}
var jobExists bool
if err := pg.QueryRow(ctx, `SELECT EXISTS(SELECT 1 FROM processing_jobs WHERE id = $1)`, jobID).Scan(&jobExists); err != nil || !jobExists {
report["processing_job_products_skipped"]++
continue
}
var errPtr *string
if errText.Valid && strings.TrimSpace(errText.String) != "" {
v := errText.String
errPtr = &v
}
// Stable UUID from legacy int so resume is idempotent.
prodID := uuid.NewSHA1(uuid.NameSpaceOID, []byte("pjp:"+strconv.FormatInt(legacyProdID, 10)))
_, err = pg.Exec(ctx, `
INSERT INTO processing_job_products (
id, job_id, raw_product_id, status, error, processed_product_id, created_at, updated_at
) VALUES ($1, $2, $3, $4, $5, NULL, $6, $7)
ON CONFLICT (id) DO UPDATE SET
status = EXCLUDED.status,
error = EXCLUDED.error,
updated_at = EXCLUDED.updated_at`,
prodID, jobID, rawUUID, normalizeJobProductStatus(status), errPtr,
createdAt.UTC(), updatedAt.UTC(),
)
if err != nil {
report["processing_job_products_skipped"]++
continue
}
report["processing_job_products"]++
}
if err := rows.Err(); err != nil {
log.Printf("processing_job_products rows: %v", err)
}
}
func migrateLegacyTasks(
ctx context.Context,
mysqlDB *sql.DB,
pg *pgxpool.Pool,
companyMap, userMap map[string]string,
allow map[string]bool,
report map[string]int,
dryRun bool,
) {
if !mysqlTableExists(ctx, mysqlDB, "tasks") {
return
}
q := mysqlSelectList(
"id",
mysqlCol(ctx, mysqlDB, "tasks", "company_id", "NULL"),
mysqlCol(ctx, mysqlDB, "tasks", "user_id", "NULL"),
mysqlCoalesce(ctx, mysqlDB, "tasks", "task_name", "''"),
mysqlCoalesce(ctx, mysqlDB, "tasks", "status", "'pending'"),
mysqlCol(ctx, mysqlDB, "tasks", "start_time", "NULL"),
mysqlCol(ctx, mysqlDB, "tasks", "end_time", "NULL"),
mysqlCol(ctx, mysqlDB, "tasks", "log", "NULL"),
mysqlCoalesce(ctx, mysqlDB, "tasks", "processing_products", "0"),
mysqlCoalesce(ctx, mysqlDB, "tasks", "processed_products", "0"),
mysqlCoalesce(ctx, mysqlDB, "tasks", "total_products", "0"),
mysqlCol(ctx, mysqlDB, "tasks", "error_products", "NULL"),
mysqlCol(ctx, mysqlDB, "tasks", "product_ids", "NULL"),
mysqlCoalesce(ctx, mysqlDB, "tasks", "created_at", "NOW()"),
mysqlCoalesce(ctx, mysqlDB, "tasks", "updated_at", "NOW()"),
) + " FROM tasks WHERE 1=1"
clause, cargs := mysqlCompanyFilter("company_id", allow)
q += clause
rows, err := mysqlDB.QueryContext(ctx, q, cargs...)
if err != nil {
log.Printf("tasks skipped: %v", err)
return
}
defer rows.Close()
for rows.Next() {
var (
legacyID int64
companyLegacy, userLegacy sql.NullString
taskName, status string
startTime, endTime sql.NullTime
logText sql.NullString
processingProducts, processedProducts, totalP any
errorProducts, productIDs sql.NullString
createdAt, updatedAt time.Time
)
if err := rows.Scan(
&legacyID, &companyLegacy, &userLegacy, &taskName, &status,
&startTime, &endTime, &logText,
&processingProducts, &processedProducts, &totalP,
&errorProducts, &productIDs, &createdAt, &updatedAt,
); err != nil {
report["tasks_skipped"]++
continue
}
if !companyLegacy.Valid || strings.TrimSpace(companyLegacy.String) == "" {
report["tasks_skipped"]++
continue
}
cid, ok := companyMap[companyLegacy.String]
if !ok {
report["tasks_skipped"]++
continue
}
var userID *uuid.UUID
if userLegacy.Valid && strings.TrimSpace(userLegacy.String) != "" {
if mapped, ok := userMap[userLegacy.String]; ok {
if parsed, err := uuid.Parse(mapped); err == nil {
if !dryRun {
var exists bool
_ = pg.QueryRow(ctx, `SELECT EXISTS(SELECT 1 FROM users WHERE id = $1)`, parsed).Scan(&exists)
if exists {
userID = &parsed
}
} else {
userID = &parsed
}
}
}
}
taskID := uuid.NewSHA1(uuid.NameSpaceOID, []byte("task:"+strconv.FormatInt(legacyID, 10)))
var startPtr, endPtr *time.Time
if startTime.Valid {
t := startTime.Time.UTC()
startPtr = &t
}
if endTime.Valid {
t := endTime.Time.UTC()
endPtr = &t
}
var logPtr *string
if logText.Valid {
v := logText.String
logPtr = &v
}
errJSON := nullJSON(errorProducts)
prodJSON := nullJSON(productIDs)
if dryRun {
report["tasks"]++
continue
}
_, err = pg.Exec(ctx, `
INSERT INTO tasks (
id, company_id, user_id, task_name, status, start_time, end_time, log,
processing_products, processed_products, total_products,
error_products, product_ids, created_at, updated_at
) VALUES (
$1, $2, $3, $4, $5, $6, $7, $8,
$9, $10, $11,
$12::jsonb, $13::jsonb, $14, $15
)
ON CONFLICT (id) DO UPDATE SET
status = EXCLUDED.status,
end_time = EXCLUDED.end_time,
log = EXCLUDED.log,
processed_products = EXCLUDED.processed_products,
updated_at = EXCLUDED.updated_at`,
taskID, cid, userID, taskName, strings.TrimSpace(status),
startPtr, endPtr, logPtr,
scanIntish(processingProducts), scanIntish(processedProducts), scanIntish(totalP),
errJSON, prodJSON, createdAt.UTC(), updatedAt.UTC(),
)
if err != nil {
log.Printf("task %d: %v", legacyID, err)
report["tasks_skipped"]++
continue
}
report["tasks"]++
}
if err := rows.Err(); err != nil {
log.Printf("tasks rows: %v", err)
}
}
func normalizeProcessingJobStatus(raw string) string {
switch strings.ToLower(strings.TrimSpace(raw)) {
case "completed", "success", "done":
return "completed"
case "failed", "error":
return "failed"
case "cancelled", "canceled", "skipped":
return "cancelled"
case "running", "processing":
return "running"
case "pending", "queued":
return "pending"
default:
return "failed"
}
}
func normalizeJobProductStatus(raw string) string {
switch strings.ToLower(strings.TrimSpace(raw)) {
case "processed", "completed", "success", "done":
return "processed"
case "failed", "error":
return "failed"
case "cancelled", "canceled", "skipped":
return "cancelled"
case "processing", "running":
return "processing"
case "pending", "queued":
return "pending"
default:
return "failed"
}
}
func stringifyAnyID(v any) string {
switch x := v.(type) {
case nil:
return ""
case int64:
return strconv.FormatInt(x, 10)
case int32:
return strconv.FormatInt(int64(x), 10)
case float64:
return strconv.FormatInt(int64(x), 10)
case []byte:
return strings.TrimSpace(string(x))
case string:
return strings.TrimSpace(x)
default:
n := scanIntish(v)
if n != 0 {
return strconv.Itoa(n)
}
return ""
}
}
func nullJSON(ns sql.NullString) any {
if !ns.Valid || strings.TrimSpace(ns.String) == "" {
return nil
}
return strings.TrimSpace(ns.String)
}