Initial commit of Descrybe v2 without local scratch artifacts.
Drop one-shot tmp/axe scripts and agent i18n scratch so the Gitea tree is deployable.
This commit is contained in:
@@ -0,0 +1,496 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"log"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/google/uuid"
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
)
|
||||
|
||||
// migrateJobsDomain imports legacy processing_jobs (+ best-effort job products) and tasks.
|
||||
// Job rows are tagged ai_provider_mode='migrated' so retention cleanup preserves history.
|
||||
func migrateJobsDomain(
|
||||
ctx context.Context,
|
||||
mysqlDB *sql.DB,
|
||||
pg *pgxpool.Pool,
|
||||
companyMap, userMap, rawMap map[string]string,
|
||||
allow map[string]bool,
|
||||
domains domainSet,
|
||||
report map[string]int,
|
||||
dryRun bool,
|
||||
) {
|
||||
if !domains.has("jobs") {
|
||||
return
|
||||
}
|
||||
migrateProcessingJobs(ctx, mysqlDB, pg, companyMap, userMap, allow, report, dryRun)
|
||||
migrateProcessingJobProducts(ctx, mysqlDB, pg, rawMap, allow, report, dryRun)
|
||||
migrateLegacyTasks(ctx, mysqlDB, pg, companyMap, userMap, allow, report, dryRun)
|
||||
}
|
||||
|
||||
func migrateProcessingJobs(
|
||||
ctx context.Context,
|
||||
mysqlDB *sql.DB,
|
||||
pg *pgxpool.Pool,
|
||||
companyMap, userMap map[string]string,
|
||||
allow map[string]bool,
|
||||
report map[string]int,
|
||||
dryRun bool,
|
||||
) {
|
||||
if !mysqlTableExists(ctx, mysqlDB, "processing_jobs") {
|
||||
log.Printf("processing_jobs skipped: table missing")
|
||||
return
|
||||
}
|
||||
q := mysqlSelectList(
|
||||
"id",
|
||||
"company_id",
|
||||
mysqlCol(ctx, mysqlDB, "processing_jobs", "user_id", "NULL"),
|
||||
mysqlCoalesce(ctx, mysqlDB, "processing_jobs", "status", "'pending'"),
|
||||
mysqlCoalesce(ctx, mysqlDB, "processing_jobs", "total_products", "0"),
|
||||
mysqlCoalesce(ctx, mysqlDB, "processing_jobs", "processed_products", "0"),
|
||||
mysqlCol(ctx, mysqlDB, "processing_jobs", "error", "NULL"),
|
||||
mysqlCoalesce(ctx, mysqlDB, "processing_jobs", "processing_type", "'full'"),
|
||||
mysqlCoalesce(ctx, mysqlDB, "processing_jobs", "priority", "0"),
|
||||
mysqlCoalesce(ctx, mysqlDB, "processing_jobs", "estimated_tokens", "0"),
|
||||
mysqlCol(ctx, mysqlDB, "processing_jobs", "started_at", "NULL"),
|
||||
mysqlCol(ctx, mysqlDB, "processing_jobs", "completed_at", "NULL"),
|
||||
mysqlCoalesce(ctx, mysqlDB, "processing_jobs", "created_at", "NOW()"),
|
||||
mysqlCoalesce(ctx, mysqlDB, "processing_jobs", "updated_at", "NOW()"),
|
||||
) + " FROM processing_jobs WHERE 1=1"
|
||||
clause, cargs := mysqlCompanyFilter("company_id", allow)
|
||||
q += clause
|
||||
rows, err := mysqlDB.QueryContext(ctx, q, cargs...)
|
||||
if err != nil {
|
||||
log.Printf("processing_jobs skipped: %v", err)
|
||||
return
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
for rows.Next() {
|
||||
var (
|
||||
legacyID, companyLegacy string
|
||||
userLegacy sql.NullString
|
||||
status, processingType string
|
||||
errText sql.NullString
|
||||
totalProducts any
|
||||
processedProducts any
|
||||
priority any
|
||||
estimatedTokens any
|
||||
startedAt, completedAt sql.NullTime
|
||||
createdAt, updatedAt time.Time
|
||||
)
|
||||
if err := rows.Scan(
|
||||
&legacyID, &companyLegacy, &userLegacy, &status, &totalProducts, &processedProducts,
|
||||
&errText, &processingType, &priority, &estimatedTokens,
|
||||
&startedAt, &completedAt, &createdAt, &updatedAt,
|
||||
); err != nil {
|
||||
report["processing_jobs_skipped"]++
|
||||
continue
|
||||
}
|
||||
cid, ok := companyMap[companyLegacy]
|
||||
if !ok {
|
||||
report["processing_jobs_skipped"]++
|
||||
continue
|
||||
}
|
||||
jobID, err := uuid.Parse(strings.TrimSpace(legacyID))
|
||||
if err != nil {
|
||||
// Legacy dump mixes UUID and numeric string PKs — keep remaps stable.
|
||||
jobID = uuid.NewSHA1(uuid.NameSpaceOID, []byte("processing_job:"+strings.TrimSpace(legacyID)))
|
||||
}
|
||||
var userID *uuid.UUID
|
||||
if userLegacy.Valid && strings.TrimSpace(userLegacy.String) != "" {
|
||||
if mapped, ok := userMap[userLegacy.String]; ok {
|
||||
if parsed, err := uuid.Parse(mapped); err == nil {
|
||||
if !dryRun {
|
||||
var exists bool
|
||||
_ = pg.QueryRow(ctx, `SELECT EXISTS(SELECT 1 FROM users WHERE id = $1)`, parsed).Scan(&exists)
|
||||
if exists {
|
||||
userID = &parsed
|
||||
} else {
|
||||
report["processing_jobs_user_missing"]++
|
||||
}
|
||||
} else {
|
||||
userID = &parsed
|
||||
}
|
||||
}
|
||||
} else {
|
||||
report["processing_jobs_user_unmapped"]++
|
||||
}
|
||||
}
|
||||
normStatus := normalizeProcessingJobStatus(status)
|
||||
ptype := strings.TrimSpace(processingType)
|
||||
if ptype == "" {
|
||||
ptype = "full"
|
||||
}
|
||||
var errPtr *string
|
||||
if errText.Valid && strings.TrimSpace(errText.String) != "" {
|
||||
v := errText.String
|
||||
errPtr = &v
|
||||
}
|
||||
var startedPtr, completedPtr *time.Time
|
||||
if startedAt.Valid {
|
||||
t := startedAt.Time.UTC()
|
||||
startedPtr = &t
|
||||
}
|
||||
if completedAt.Valid {
|
||||
t := completedAt.Time.UTC()
|
||||
completedPtr = &t
|
||||
}
|
||||
if dryRun {
|
||||
report["processing_jobs"]++
|
||||
continue
|
||||
}
|
||||
_, err = pg.Exec(ctx, `
|
||||
INSERT INTO processing_jobs (
|
||||
id, company_id, user_id, status, total_products, processed_products,
|
||||
error, processing_type, priority, estimated_tokens,
|
||||
started_at, completed_at, created_at, updated_at,
|
||||
current_step, step_progress, ai_provider_mode
|
||||
) VALUES (
|
||||
$1, $2, $3, $4, $5, $6,
|
||||
$7, $8, $9, $10,
|
||||
$11, $12, $13, $14,
|
||||
'', '[]'::jsonb, 'migrated'
|
||||
)
|
||||
ON CONFLICT (id) DO UPDATE SET
|
||||
company_id = EXCLUDED.company_id,
|
||||
user_id = EXCLUDED.user_id,
|
||||
status = EXCLUDED.status,
|
||||
total_products = EXCLUDED.total_products,
|
||||
processed_products = EXCLUDED.processed_products,
|
||||
error = EXCLUDED.error,
|
||||
processing_type = EXCLUDED.processing_type,
|
||||
priority = EXCLUDED.priority,
|
||||
estimated_tokens = EXCLUDED.estimated_tokens,
|
||||
started_at = EXCLUDED.started_at,
|
||||
completed_at = EXCLUDED.completed_at,
|
||||
created_at = EXCLUDED.created_at,
|
||||
updated_at = EXCLUDED.updated_at,
|
||||
ai_provider_mode = 'migrated'`,
|
||||
jobID, cid, userID, normStatus,
|
||||
scanIntish(totalProducts), scanIntish(processedProducts),
|
||||
errPtr, ptype, scanIntish(priority), scanIntish(estimatedTokens),
|
||||
startedPtr, completedPtr, createdAt.UTC(), updatedAt.UTC(),
|
||||
)
|
||||
if err != nil {
|
||||
log.Printf("processing_job %s: %v", legacyID, err)
|
||||
report["processing_jobs_skipped"]++
|
||||
continue
|
||||
}
|
||||
report["processing_jobs"]++
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
log.Printf("processing_jobs rows: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func migrateProcessingJobProducts(
|
||||
ctx context.Context,
|
||||
mysqlDB *sql.DB,
|
||||
pg *pgxpool.Pool,
|
||||
rawMap map[string]string,
|
||||
allow map[string]bool,
|
||||
report map[string]int,
|
||||
dryRun bool,
|
||||
) {
|
||||
if !mysqlTableExists(ctx, mysqlDB, "processing_job_products") {
|
||||
return
|
||||
}
|
||||
if !mysqlTableExists(ctx, mysqlDB, "processing_jobs") {
|
||||
return
|
||||
}
|
||||
q := `
|
||||
SELECT pjp.id, pjp.job_id, pjp.raw_product_id, pjp.status, pjp.error,
|
||||
pjp.processed_product_id, pjp.created_at, pjp.updated_at
|
||||
FROM processing_job_products pjp
|
||||
JOIN processing_jobs pj ON pj.id = pjp.job_id
|
||||
WHERE 1=1`
|
||||
clause, cargs := mysqlCompanyFilter("pj.company_id", allow)
|
||||
q += clause
|
||||
rows, err := mysqlDB.QueryContext(ctx, q, cargs...)
|
||||
if err != nil {
|
||||
log.Printf("processing_job_products skipped: %v", err)
|
||||
return
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
for rows.Next() {
|
||||
var (
|
||||
legacyProdID int64
|
||||
jobLegacy string
|
||||
rawLegacy any
|
||||
status string
|
||||
errText sql.NullString
|
||||
processedLegacy sql.NullInt64
|
||||
createdAt, updatedAt time.Time
|
||||
)
|
||||
if err := rows.Scan(
|
||||
&legacyProdID, &jobLegacy, &rawLegacy, &status, &errText,
|
||||
&processedLegacy, &createdAt, &updatedAt,
|
||||
); err != nil {
|
||||
report["processing_job_products_skipped"]++
|
||||
continue
|
||||
}
|
||||
jobID, err := uuid.Parse(strings.TrimSpace(jobLegacy))
|
||||
if err != nil {
|
||||
jobID = uuid.NewSHA1(uuid.NameSpaceOID, []byte("processing_job:"+strings.TrimSpace(jobLegacy)))
|
||||
}
|
||||
rawKey := strings.TrimSpace(stringifyAnyID(rawLegacy))
|
||||
rawUUIDStr, ok := rawMap[rawKey]
|
||||
if !ok {
|
||||
report["processing_job_products_skipped"]++
|
||||
continue
|
||||
}
|
||||
rawUUID, err := uuid.Parse(rawUUIDStr)
|
||||
if err != nil {
|
||||
report["processing_job_products_skipped"]++
|
||||
continue
|
||||
}
|
||||
if dryRun {
|
||||
report["processing_job_products"]++
|
||||
continue
|
||||
}
|
||||
// Only attach when the remapped raw product still exists (GTIN dedupe may drop some).
|
||||
var exists bool
|
||||
if err := pg.QueryRow(ctx, `SELECT EXISTS(SELECT 1 FROM raw_products WHERE id = $1)`, rawUUID).Scan(&exists); err != nil || !exists {
|
||||
report["processing_job_products_skipped"]++
|
||||
continue
|
||||
}
|
||||
var jobExists bool
|
||||
if err := pg.QueryRow(ctx, `SELECT EXISTS(SELECT 1 FROM processing_jobs WHERE id = $1)`, jobID).Scan(&jobExists); err != nil || !jobExists {
|
||||
report["processing_job_products_skipped"]++
|
||||
continue
|
||||
}
|
||||
var errPtr *string
|
||||
if errText.Valid && strings.TrimSpace(errText.String) != "" {
|
||||
v := errText.String
|
||||
errPtr = &v
|
||||
}
|
||||
// Stable UUID from legacy int so resume is idempotent.
|
||||
prodID := uuid.NewSHA1(uuid.NameSpaceOID, []byte("pjp:"+strconv.FormatInt(legacyProdID, 10)))
|
||||
_, err = pg.Exec(ctx, `
|
||||
INSERT INTO processing_job_products (
|
||||
id, job_id, raw_product_id, status, error, processed_product_id, created_at, updated_at
|
||||
) VALUES ($1, $2, $3, $4, $5, NULL, $6, $7)
|
||||
ON CONFLICT (id) DO UPDATE SET
|
||||
status = EXCLUDED.status,
|
||||
error = EXCLUDED.error,
|
||||
updated_at = EXCLUDED.updated_at`,
|
||||
prodID, jobID, rawUUID, normalizeJobProductStatus(status), errPtr,
|
||||
createdAt.UTC(), updatedAt.UTC(),
|
||||
)
|
||||
if err != nil {
|
||||
report["processing_job_products_skipped"]++
|
||||
continue
|
||||
}
|
||||
report["processing_job_products"]++
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
log.Printf("processing_job_products rows: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func migrateLegacyTasks(
|
||||
ctx context.Context,
|
||||
mysqlDB *sql.DB,
|
||||
pg *pgxpool.Pool,
|
||||
companyMap, userMap map[string]string,
|
||||
allow map[string]bool,
|
||||
report map[string]int,
|
||||
dryRun bool,
|
||||
) {
|
||||
if !mysqlTableExists(ctx, mysqlDB, "tasks") {
|
||||
return
|
||||
}
|
||||
q := mysqlSelectList(
|
||||
"id",
|
||||
mysqlCol(ctx, mysqlDB, "tasks", "company_id", "NULL"),
|
||||
mysqlCol(ctx, mysqlDB, "tasks", "user_id", "NULL"),
|
||||
mysqlCoalesce(ctx, mysqlDB, "tasks", "task_name", "''"),
|
||||
mysqlCoalesce(ctx, mysqlDB, "tasks", "status", "'pending'"),
|
||||
mysqlCol(ctx, mysqlDB, "tasks", "start_time", "NULL"),
|
||||
mysqlCol(ctx, mysqlDB, "tasks", "end_time", "NULL"),
|
||||
mysqlCol(ctx, mysqlDB, "tasks", "log", "NULL"),
|
||||
mysqlCoalesce(ctx, mysqlDB, "tasks", "processing_products", "0"),
|
||||
mysqlCoalesce(ctx, mysqlDB, "tasks", "processed_products", "0"),
|
||||
mysqlCoalesce(ctx, mysqlDB, "tasks", "total_products", "0"),
|
||||
mysqlCol(ctx, mysqlDB, "tasks", "error_products", "NULL"),
|
||||
mysqlCol(ctx, mysqlDB, "tasks", "product_ids", "NULL"),
|
||||
mysqlCoalesce(ctx, mysqlDB, "tasks", "created_at", "NOW()"),
|
||||
mysqlCoalesce(ctx, mysqlDB, "tasks", "updated_at", "NOW()"),
|
||||
) + " FROM tasks WHERE 1=1"
|
||||
clause, cargs := mysqlCompanyFilter("company_id", allow)
|
||||
q += clause
|
||||
rows, err := mysqlDB.QueryContext(ctx, q, cargs...)
|
||||
if err != nil {
|
||||
log.Printf("tasks skipped: %v", err)
|
||||
return
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
for rows.Next() {
|
||||
var (
|
||||
legacyID int64
|
||||
companyLegacy, userLegacy sql.NullString
|
||||
taskName, status string
|
||||
startTime, endTime sql.NullTime
|
||||
logText sql.NullString
|
||||
processingProducts, processedProducts, totalP any
|
||||
errorProducts, productIDs sql.NullString
|
||||
createdAt, updatedAt time.Time
|
||||
)
|
||||
if err := rows.Scan(
|
||||
&legacyID, &companyLegacy, &userLegacy, &taskName, &status,
|
||||
&startTime, &endTime, &logText,
|
||||
&processingProducts, &processedProducts, &totalP,
|
||||
&errorProducts, &productIDs, &createdAt, &updatedAt,
|
||||
); err != nil {
|
||||
report["tasks_skipped"]++
|
||||
continue
|
||||
}
|
||||
if !companyLegacy.Valid || strings.TrimSpace(companyLegacy.String) == "" {
|
||||
report["tasks_skipped"]++
|
||||
continue
|
||||
}
|
||||
cid, ok := companyMap[companyLegacy.String]
|
||||
if !ok {
|
||||
report["tasks_skipped"]++
|
||||
continue
|
||||
}
|
||||
var userID *uuid.UUID
|
||||
if userLegacy.Valid && strings.TrimSpace(userLegacy.String) != "" {
|
||||
if mapped, ok := userMap[userLegacy.String]; ok {
|
||||
if parsed, err := uuid.Parse(mapped); err == nil {
|
||||
if !dryRun {
|
||||
var exists bool
|
||||
_ = pg.QueryRow(ctx, `SELECT EXISTS(SELECT 1 FROM users WHERE id = $1)`, parsed).Scan(&exists)
|
||||
if exists {
|
||||
userID = &parsed
|
||||
}
|
||||
} else {
|
||||
userID = &parsed
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
taskID := uuid.NewSHA1(uuid.NameSpaceOID, []byte("task:"+strconv.FormatInt(legacyID, 10)))
|
||||
var startPtr, endPtr *time.Time
|
||||
if startTime.Valid {
|
||||
t := startTime.Time.UTC()
|
||||
startPtr = &t
|
||||
}
|
||||
if endTime.Valid {
|
||||
t := endTime.Time.UTC()
|
||||
endPtr = &t
|
||||
}
|
||||
var logPtr *string
|
||||
if logText.Valid {
|
||||
v := logText.String
|
||||
logPtr = &v
|
||||
}
|
||||
errJSON := nullJSON(errorProducts)
|
||||
prodJSON := nullJSON(productIDs)
|
||||
if dryRun {
|
||||
report["tasks"]++
|
||||
continue
|
||||
}
|
||||
_, err = pg.Exec(ctx, `
|
||||
INSERT INTO tasks (
|
||||
id, company_id, user_id, task_name, status, start_time, end_time, log,
|
||||
processing_products, processed_products, total_products,
|
||||
error_products, product_ids, created_at, updated_at
|
||||
) VALUES (
|
||||
$1, $2, $3, $4, $5, $6, $7, $8,
|
||||
$9, $10, $11,
|
||||
$12::jsonb, $13::jsonb, $14, $15
|
||||
)
|
||||
ON CONFLICT (id) DO UPDATE SET
|
||||
status = EXCLUDED.status,
|
||||
end_time = EXCLUDED.end_time,
|
||||
log = EXCLUDED.log,
|
||||
processed_products = EXCLUDED.processed_products,
|
||||
updated_at = EXCLUDED.updated_at`,
|
||||
taskID, cid, userID, taskName, strings.TrimSpace(status),
|
||||
startPtr, endPtr, logPtr,
|
||||
scanIntish(processingProducts), scanIntish(processedProducts), scanIntish(totalP),
|
||||
errJSON, prodJSON, createdAt.UTC(), updatedAt.UTC(),
|
||||
)
|
||||
if err != nil {
|
||||
log.Printf("task %d: %v", legacyID, err)
|
||||
report["tasks_skipped"]++
|
||||
continue
|
||||
}
|
||||
report["tasks"]++
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
log.Printf("tasks rows: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func normalizeProcessingJobStatus(raw string) string {
|
||||
switch strings.ToLower(strings.TrimSpace(raw)) {
|
||||
case "completed", "success", "done":
|
||||
return "completed"
|
||||
case "failed", "error":
|
||||
return "failed"
|
||||
case "cancelled", "canceled", "skipped":
|
||||
return "cancelled"
|
||||
case "running", "processing":
|
||||
return "running"
|
||||
case "pending", "queued":
|
||||
return "pending"
|
||||
default:
|
||||
return "failed"
|
||||
}
|
||||
}
|
||||
|
||||
func normalizeJobProductStatus(raw string) string {
|
||||
switch strings.ToLower(strings.TrimSpace(raw)) {
|
||||
case "processed", "completed", "success", "done":
|
||||
return "processed"
|
||||
case "failed", "error":
|
||||
return "failed"
|
||||
case "cancelled", "canceled", "skipped":
|
||||
return "cancelled"
|
||||
case "processing", "running":
|
||||
return "processing"
|
||||
case "pending", "queued":
|
||||
return "pending"
|
||||
default:
|
||||
return "failed"
|
||||
}
|
||||
}
|
||||
|
||||
func stringifyAnyID(v any) string {
|
||||
switch x := v.(type) {
|
||||
case nil:
|
||||
return ""
|
||||
case int64:
|
||||
return strconv.FormatInt(x, 10)
|
||||
case int32:
|
||||
return strconv.FormatInt(int64(x), 10)
|
||||
case float64:
|
||||
return strconv.FormatInt(int64(x), 10)
|
||||
case []byte:
|
||||
return strings.TrimSpace(string(x))
|
||||
case string:
|
||||
return strings.TrimSpace(x)
|
||||
default:
|
||||
n := scanIntish(v)
|
||||
if n != 0 {
|
||||
return strconv.Itoa(n)
|
||||
}
|
||||
return ""
|
||||
}
|
||||
}
|
||||
|
||||
func nullJSON(ns sql.NullString) any {
|
||||
if !ns.Valid || strings.TrimSpace(ns.String) == "" {
|
||||
return nil
|
||||
}
|
||||
return strings.TrimSpace(ns.String)
|
||||
}
|
||||
Reference in New Issue
Block a user