60 lines
1.9 KiB
Go
60 lines
1.9 KiB
Go
|
package processing
|
||
|
|
|
||
|
|
import (
|
||
|
|
"context"
|
||
|
|
"fmt"
|
||
|
|
|
||
|
|
"github.com/jackc/pgx/v5/pgxpool"
|
||
|
|
)
|
||
|
|
|
||
|
|
// StuckAgeInterval is the shared SQL age used by CleanupStuck and claim-time
|
||
|
|
// reclaim in loadPendingItems for stranded processing_job_products.
|
||
|
|
const StuckAgeInterval = "2 hours"
|
||
|
|
|
||
|
|
// StuckCleanupResult counts rows touched by CleanupStuck.
|
||
|
|
type StuckCleanupResult struct {
|
||
|
|
JobsMarkedFailed int64
|
||
|
|
ProductsReset int64
|
||
|
|
SyncJobsMarkedFailed int64
|
||
|
|
}
|
||
|
|
|
||
|
|
// CleanupStuck aligns worker and admin stuck-cleanup semantics:
|
||
|
|
// mark long-running jobs failed, reset stranded processing_job_products
|
||
|
|
// from 'processing' back to 'pending', and fail aged running feed_sync_jobs
|
||
|
|
// so they are not left unclaimable forever.
|
||
|
|
func CleanupStuck(ctx context.Context, pool *pgxpool.Pool) (StuckCleanupResult, error) {
|
||
|
|
var out StuckCleanupResult
|
||
|
|
if pool == nil {
|
||
|
|
return out, fmt.Errorf("cleanup stuck: nil pool")
|
||
|
|
}
|
||
|
|
|
||
|
|
ctJobs, err := pool.Exec(ctx, `
|
||
|
|
UPDATE processing_jobs
|
||
|
|
SET status = 'failed', error = 'stuck cleanup', completed_at = now(), updated_at = now()
|
||
|
|
WHERE status = 'running' AND updated_at < now() - interval '`+StuckAgeInterval+`'`)
|
||
|
|
if err != nil {
|
||
|
|
return out, fmt.Errorf("cleanup stuck jobs: %w", err)
|
||
|
|
}
|
||
|
|
out.JobsMarkedFailed = ctJobs.RowsAffected()
|
||
|
|
|
||
|
|
ctProd, err := pool.Exec(ctx, `
|
||
|
|
UPDATE processing_job_products
|
||
|
|
SET status = 'pending', updated_at = now()
|
||
|
|
WHERE status = 'processing' AND updated_at < now() - interval '`+StuckAgeInterval+`'`)
|
||
|
|
if err != nil {
|
||
|
|
return out, fmt.Errorf("cleanup stuck products: %w", err)
|
||
|
|
}
|
||
|
|
out.ProductsReset = ctProd.RowsAffected()
|
||
|
|
|
||
|
|
ctSync, err := pool.Exec(ctx, `
|
||
|
|
UPDATE feed_sync_jobs
|
||
|
|
SET status = 'failed', error = 'stuck cleanup', completed_at = now(), updated_at = now()
|
||
|
|
WHERE status = 'running' AND updated_at < now() - interval '`+StuckAgeInterval+`'`)
|
||
|
|
if err != nil {
|
||
|
|
return out, fmt.Errorf("cleanup stuck sync jobs: %w", err)
|
||
|
|
}
|
||
|
|
out.SyncJobsMarkedFailed = ctSync.RowsAffected()
|
||
|
|
|
||
|
|
return out, nil
|
||
|
|
}
|