Initial commit of Descrybe v2 without local scratch artifacts.

Drop one-shot tmp/axe scripts and agent i18n scratch so the Gitea tree is deployable.
2026-08-09 22:47:43 +02:00
commit 8580c996c3
1285 changed files with 325780 additions and 0 deletions
@@ -0,0 +1,114 @@
+package jobs
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"time"
+
+	"github.com/jackc/pgx/v5"
+	"github.com/jackc/pgx/v5/pgxpool"
+)
+
+// ProcessingWorkerID is the durable heartbeat row for cmd/worker.
+const ProcessingWorkerID = "processing"
+
+// DefaultHeartbeatStaleAfter is how long /readyz tolerates a missing touch.
+// Worker poll defaults to 250ms; 60s absorbs brief deploys without masking death.
+const DefaultHeartbeatStaleAfter = 60 * time.Second
+
+// TouchHeartbeat upserts last_seen_at for workerID (call from the worker poll loop).
+func TouchHeartbeat(ctx context.Context, pool *pgxpool.Pool, workerID string) error {
+	if pool == nil {
+		return fmt.Errorf("jobs heartbeat: pool unavailable")
+	}
+	if workerID == "" {
+		workerID = ProcessingWorkerID
+	}
+	_, err := pool.Exec(ctx, `
+		INSERT INTO worker_heartbeats (worker_id, last_seen_at, updated_at)
+		VALUES ($1, now(), now())
+		ON CONFLICT (worker_id) DO UPDATE
+		SET last_seen_at = now(), updated_at = now()`, workerID)
+	return err
+}
+
+// WorkerProbe is the /readyz worker + queue snapshot (no driver detail in ErrMsg).
+type WorkerProbe struct {
+	OK           bool
+	WorkerCheck  string // ok | missing | stale | fail | unavailable
+	QueueCheck   string // ok | fail | unavailable
+	ErrMsg       string // short stable code for clients
+	Reason       string // optional operator remediation (safe, no secrets)
+	PendingJobs  int64
+	LastSeenAgeS int64 // seconds since last heartbeat; -1 when missing
+}
+
+// HeartbeatQuerier is satisfied by *pgxpool.Pool (and test stubs).
+type HeartbeatQuerier interface {
+	QueryRow(ctx context.Context, sql string, args ...any) pgx.Row
+}
+
+// ProbeWorkerReadiness checks the processing worker heartbeat and pending queue depth.
+func ProbeWorkerReadiness(ctx context.Context, q HeartbeatQuerier, staleAfter time.Duration) WorkerProbe {
+	out := WorkerProbe{
+		WorkerCheck:  "unavailable",
+		QueueCheck:   "unavailable",
+		LastSeenAgeS: -1,
+	}
+	if q == nil {
+		out.ErrMsg = "worker probe unavailable"
+		out.Reason = "Database pool unavailable; cannot probe worker heartbeat."
+		return out
+	}
+	if staleAfter <= 0 {
+		staleAfter = DefaultHeartbeatStaleAfter
+	}
+
+	var pending int64
+	if err := q.QueryRow(ctx, `
+		SELECT COUNT(*)::bigint FROM processing_jobs WHERE status = 'pending'`).Scan(&pending); err != nil {
+		out.WorkerCheck = "fail"
+		out.QueueCheck = "fail"
+		out.ErrMsg = "queue depth query failed"
+		out.Reason = "Could not read processing job queue depth; check DATABASE_URL and Postgres."
+		return out
+	}
+	out.PendingJobs = pending
+	out.QueueCheck = "ok"
+
+	var lastSeen time.Time
+	err := q.QueryRow(ctx, `
+		SELECT last_seen_at FROM worker_heartbeats WHERE worker_id = $1`, ProcessingWorkerID).Scan(&lastSeen)
+	if errors.Is(err, pgx.ErrNoRows) {
+		out.WorkerCheck = "missing"
+		out.ErrMsg = "worker heartbeat missing"
+		out.Reason = "No processing worker heartbeat. API-only readiness 503 is expected — start the worker (npm run dev includes it, or npm run dev:worker)."
+		return out
+	}
+	if err != nil {
+		out.WorkerCheck = "fail"
+		out.ErrMsg = "worker heartbeat query failed"
+		out.Reason = "Could not read worker_heartbeats; ensure goose migration 039_worker_heartbeats is applied."
+		return out
+	}
+
+	age := time.Since(lastSeen)
+	if age < 0 {
+		age = 0
+	}
+	out.LastSeenAgeS = int64(age / time.Second)
+	if age > staleAfter {
+		out.WorkerCheck = "stale"
+		out.ErrMsg = "worker heartbeat stale"
+		out.Reason = fmt.Sprintf(
+			"Processing worker heartbeat older than %s. API-only readiness 503 is expected — start or restart the worker (npm run dev includes it, or npm run dev:worker).",
+			staleAfter,
+		)
+		return out
+	}
+
+	out.OK = true
+	out.WorkerCheck = "ok"
+	return out
+}
@@ -0,0 +1,94 @@
+package jobs_test
+
+import (
+	"context"
+	"errors"
+	"testing"
+	"time"
+
+	"github.com/descrybe/descrybe-v2/apps/api/internal/jobs"
+	"github.com/jackc/pgx/v5"
+)
+
+type stubRow struct {
+	scan func(dest ...any) error
+}
+
+func (r stubRow) Scan(dest ...any) error {
+	if r.scan == nil {
+		return pgx.ErrNoRows
+	}
+	return r.scan(dest...)
+}
+
+type stubQuerier struct {
+	pending    int64
+	pendingErr error
+	lastSeen   time.Time
+	seenErr    error
+	calls      int
+}
+
+func (q *stubQuerier) QueryRow(ctx context.Context, sql string, args ...any) pgx.Row {
+	q.calls++
+	if q.calls == 1 {
+		return stubRow{scan: func(dest ...any) error {
+			if q.pendingErr != nil {
+				return q.pendingErr
+			}
+			*(dest[0].(*int64)) = q.pending
+			return nil
+		}}
+	}
+	return stubRow{scan: func(dest ...any) error {
+		if q.seenErr != nil {
+			return q.seenErr
+		}
+		*(dest[0].(*time.Time)) = q.lastSeen
+		return nil
+	}}
+}
+
+func TestProbeWorkerReadinessOK(t *testing.T) {
+	t.Parallel()
+	q := &stubQuerier{pending: 3, lastSeen: time.Now()}
+	probe := jobs.ProbeWorkerReadiness(context.Background(), q, time.Minute)
+	if !probe.OK || probe.WorkerCheck != "ok" || probe.QueueCheck != "ok" || probe.PendingJobs != 3 {
+		t.Fatalf("probe = %#v", probe)
+	}
+}
+
+func TestProbeWorkerReadinessMissing(t *testing.T) {
+	t.Parallel()
+	q := &stubQuerier{seenErr: pgx.ErrNoRows}
+	probe := jobs.ProbeWorkerReadiness(context.Background(), q, time.Minute)
+	if probe.OK || probe.WorkerCheck != "missing" || probe.ErrMsg == "" || probe.Reason == "" {
+		t.Fatalf("probe = %#v", probe)
+	}
+}
+
+func TestProbeWorkerReadinessStale(t *testing.T) {
+	t.Parallel()
+	q := &stubQuerier{lastSeen: time.Now().Add(-2 * time.Minute)}
+	probe := jobs.ProbeWorkerReadiness(context.Background(), q, time.Minute)
+	if probe.OK || probe.WorkerCheck != "stale" || probe.Reason == "" {
+		t.Fatalf("probe = %#v", probe)
+	}
+}
+
+func TestProbeWorkerReadinessNil(t *testing.T) {
+	t.Parallel()
+	probe := jobs.ProbeWorkerReadiness(context.Background(), nil, 0)
+	if probe.OK || probe.WorkerCheck != "unavailable" || probe.Reason == "" {
+		t.Fatalf("probe = %#v", probe)
+	}
+}
+
+func TestProbeWorkerReadinessQueueFail(t *testing.T) {
+	t.Parallel()
+	q := &stubQuerier{pendingErr: errors.New("closed")}
+	probe := jobs.ProbeWorkerReadiness(context.Background(), q, time.Minute)
+	if probe.OK || probe.QueueCheck != "fail" || probe.WorkerCheck != "fail" {
+		t.Fatalf("probe = %#v", probe)
+	}
+}
@@ -0,0 +1,103 @@
+package jobs
+
+import (
+	"context"
+	"fmt"
+	"log"
+	"time"
+
+	"github.com/jackc/pgx/v5"
+	"github.com/jackc/pgx/v5/pgxpool"
+)
+
+// Notify channel names used by EnqueueProcessingJob / EnqueueFeedSyncJob.
+const (
+	ChannelProcessingJobs = "processing_jobs"
+	ChannelFeedSyncJobs   = "feed_sync_jobs"
+)
+
+// ListenWake LISTENs on the given Postgres channels until ctx is done.
+// Each notification (and a successful LISTEN) non-blocking-signals wake so
+// the worker can claim work without waiting for the poll ticker.
+// On connection errors it reconnects after a short backoff.
+func ListenWake(ctx context.Context, pool *pgxpool.Pool, wake chan<- struct{}, channels ...string) error {
+	if wake == nil {
+		return fmt.Errorf("listen wake: nil wake channel")
+	}
+	if len(channels) == 0 {
+		return fmt.Errorf("listen wake: no channels")
+	}
+	for _, ch := range channels {
+		if err := validateNotifyChannel(ch); err != nil {
+			return err
+		}
+	}
+	if pool == nil {
+		return fmt.Errorf("listen wake: nil pool")
+	}
+
+	backoff := time.Second
+	for {
+		if err := ctx.Err(); err != nil {
+			return err
+		}
+		err := listenOnce(ctx, pool, wake, channels)
+		if ctx.Err() != nil {
+			return ctx.Err()
+		}
+		log.Printf("jobs: listen wake reconnect after: %v", err)
+		select {
+		case <-ctx.Done():
+			return ctx.Err()
+		case <-time.After(backoff):
+		}
+		if backoff < 30*time.Second {
+			backoff *= 2
+		} else {
+			backoff = 30 * time.Second
+		}
+	}
+}
+
+func listenOnce(ctx context.Context, pool *pgxpool.Pool, wake chan<- struct{}, channels []string) error {
+	conn, err := pool.Acquire(ctx)
+	if err != nil {
+		return fmt.Errorf("acquire: %w", err)
+	}
+	defer conn.Release()
+
+	for _, ch := range channels {
+		if _, err := conn.Exec(ctx, "LISTEN "+pgx.Identifier{ch}.Sanitize()); err != nil {
+			return fmt.Errorf("LISTEN %s: %w", ch, err)
+		}
+	}
+	// Catch work enqueued before LISTEN connected.
+	signalWake(wake)
+
+	for {
+		if _, err := conn.Conn().WaitForNotification(ctx); err != nil {
+			return err
+		}
+		signalWake(wake)
+	}
+}
+
+func signalWake(wake chan<- struct{}) {
+	select {
+	case wake <- struct{}{}:
+	default:
+	}
+}
+
+func validateNotifyChannel(name string) error {
+	if name == "" {
+		return fmt.Errorf("listen wake: empty channel")
+	}
+	for _, r := range name {
+		if (r >= 'a' && r <= 'z') || (r >= '0' && r <= '9') || r == '_' {
+			continue
+		}
+		return fmt.Errorf("listen wake: invalid channel %q", name)
+	}
+	return nil
+}
@@ -0,0 +1,45 @@
+package jobs
+
+import (
+	"context"
+	"testing"
+)
+
+func TestListenWakeNilArgs(t *testing.T) {
+	wake := make(chan struct{}, 1)
+	if err := ListenWake(context.Background(), nil, wake, ChannelFeedSyncJobs); err == nil {
+		t.Fatal("expected error for nil pool")
+	}
+	if err := ListenWake(context.Background(), nil, nil, ChannelFeedSyncJobs); err == nil {
+		t.Fatal("expected error for nil wake")
+	}
+}
+
+func TestListenWakeRejectsInvalidChannel(t *testing.T) {
+	wake := make(chan struct{}, 1)
+	if err := ListenWake(context.Background(), nil, wake, "feed-sync"); err == nil {
+		t.Fatal("expected invalid channel error")
+	}
+	if err := validateNotifyChannel(ChannelProcessingJobs); err != nil {
+		t.Fatalf("processing channel: %v", err)
+	}
+	if err := validateNotifyChannel(ChannelFeedSyncJobs); err != nil {
+		t.Fatalf("feed sync channel: %v", err)
+	}
+}
+
+func TestSignalWakeNonBlocking(t *testing.T) {
+	wake := make(chan struct{}, 1)
+	signalWake(wake)
+	signalWake(wake) // must not block when buffer full
+	select {
+	case <-wake:
+	default:
+		t.Fatal("expected one wake signal")
+	}
+	select {
+	case <-wake:
+		t.Fatal("expected coalesced wake (no second signal)")
+	default:
+	}
+}
@@ -0,0 +1,62 @@
+package jobs
+
+import (
+	"context"
+	"fmt"
+	"log"
+
+	"github.com/google/uuid"
+	"github.com/jackc/pgx/v5/pgxpool"
+)
+
+// Queue enqueues processing and feed-sync work for the worker poller (SKIP LOCKED claim).
+// ASSUMPTION: full River client + river migrations are deferred; Postgres pending
+// jobs + FOR UPDATE SKIP LOCKED is the production queue for P0-4 MVP.
+type Queue struct {
+	Pool *pgxpool.Pool
+}
+
+func NewQueue(pool *pgxpool.Pool) *Queue {
+	return &Queue{Pool: pool}
+}
+
+// EnqueueProcessingJob ensures the job is pending and wakes listeners via NOTIFY.
+func (q *Queue) EnqueueProcessingJob(ctx context.Context, jobID uuid.UUID) error {
+	if q == nil || q.Pool == nil {
+		return fmt.Errorf("jobs queue not configured")
+	}
+	ct, err := q.Pool.Exec(ctx, `
+		UPDATE processing_jobs
+		SET status = 'pending', updated_at = now(),
+			error = CASE WHEN status = 'failed' THEN NULL ELSE error END
+		WHERE id = $1 AND status IN ('pending', 'failed')`, jobID)
+	if err != nil {
+		return err
+	}
+	if ct.RowsAffected() == 0 {
+		// Already running/completed/cancelled — still notify in case worker is idle.
+		var status string
+		_ = q.Pool.QueryRow(ctx, `SELECT status FROM processing_jobs WHERE id = $1`, jobID).Scan(&status)
+		log.Printf("jobs: enqueue %s status=%s (no status change)", jobID, status)
+	}
+	_, _ = q.Pool.Exec(ctx, `SELECT pg_notify('processing_jobs', $1)`, jobID.String())
+	return nil
+}
+
+// EnqueueFeedSyncJob wakes listeners for an already-pending feed_sync_jobs row.
+// Row creation + mapping gates live in feeds.EnqueueSync; this mirrors processing NOTIFY.
+func (q *Queue) EnqueueFeedSyncJob(ctx context.Context, jobID uuid.UUID) error {
+	if q == nil || q.Pool == nil {
+		return fmt.Errorf("jobs queue not configured")
+	}
+	var status string
+	err := q.Pool.QueryRow(ctx, `SELECT status FROM feed_sync_jobs WHERE id = $1`, jobID).Scan(&status)
+	if err != nil {
+		return err
+	}
+	if status != "pending" && status != "running" {
+		log.Printf("jobs: feed sync enqueue %s status=%s (notify only)", jobID, status)
+	}
+	_, _ = q.Pool.Exec(ctx, `SELECT pg_notify('feed_sync_jobs', $1)`, jobID.String())
+	return nil
+}
@@ -0,0 +1,29 @@
+package jobs
+
+import (
+	"context"
+	"testing"
+
+	"github.com/google/uuid"
+)
+
+func TestEnqueueFeedSyncJobNilQueue(t *testing.T) {
+	var q *Queue
+	err := q.EnqueueFeedSyncJob(context.Background(), uuid.New())
+	if err == nil {
+		t.Fatal("expected error for nil queue")
+	}
+	q = &Queue{}
+	err = q.EnqueueFeedSyncJob(context.Background(), uuid.New())
+	if err == nil {
+		t.Fatal("expected error for nil pool")
+	}
+}
+
+func TestEnqueueProcessingJobNilQueue(t *testing.T) {
+	var q *Queue
+	err := q.EnqueueProcessingJob(context.Background(), uuid.New())
+	if err == nil {
+		t.Fatal("expected error for nil queue")
+	}
+}
@@ -0,0 +1,65 @@
+package jobs
+
+import "sync"
+
+// DefaultSyncWorkers is the in-process bound for concurrent feed/Woo/Shopify syncs.
+// Claim paths use FOR UPDATE SKIP LOCKED so each slot gets a distinct job.
+const DefaultSyncWorkers = 1
+
+// MaxSyncWorkers caps in-process sync parallelism (DB pool + upstream API RPM).
+const MaxSyncWorkers = 2
+
+// ClampSyncWorkers bounds n to [1, MaxSyncWorkers].
+func ClampSyncWorkers(n int) int {
+	if n < 1 {
+		return 1
+	}
+	if n > MaxSyncWorkers {
+		return MaxSyncWorkers
+	}
+	return n
+}
+
+// SyncSlots limits concurrent sync Process* goroutines across feed/Woo/Shopify claims.
+type SyncSlots struct {
+	Workers int
+	sem     chan struct{}
+	wg      sync.WaitGroup
+}
+
+// NewSyncSlots creates a bounded slot set for concurrent sync jobs.
+func NewSyncSlots(workers int) *SyncSlots {
+	w := ClampSyncWorkers(workers)
+	return &SyncSlots{
+		Workers: w,
+		sem:     make(chan struct{}, w),
+	}
+}
+
+// Wait blocks until all in-flight sync goroutines finish.
+func (s *SyncSlots) Wait() {
+	s.wg.Wait()
+}
+
+// TryStart claims one free slot (non-blocking). claim must be SKIP LOCKED–safe.
+// If claim fails, the slot is released. On success, run executes in a new goroutine.
+func (s *SyncSlots) TryStart(claim func() error, run func()) (started bool, claimErr error) {
+	select {
+	case s.sem <- struct{}{}:
+	default:
+		return false, nil
+	}
+
+	if err := claim(); err != nil {
+		<-s.sem
+		return false, err
+	}
+
+	s.wg.Add(1)
+	go func() {
+		defer s.wg.Done()
+		defer func() { <-s.sem }()
+		run()
+	}()
+	return true, nil
+}
@@ -0,0 +1,84 @@
+package jobs
+
+import (
+	"errors"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	"github.com/jackc/pgx/v5"
+)
+
+func TestClampSyncWorkers(t *testing.T) {
+	t.Parallel()
+	cases := []struct {
+		in, want int
+	}{
+		{0, 1},
+		{-2, 1},
+		{1, 1},
+		{MaxSyncWorkers, MaxSyncWorkers},
+		{MaxSyncWorkers + 3, MaxSyncWorkers},
+	}
+	for _, tc := range cases {
+		if got := ClampSyncWorkers(tc.in); got != tc.want {
+			t.Fatalf("ClampSyncWorkers(%d)=%d want %d", tc.in, got, tc.want)
+		}
+	}
+}
+
+func TestSyncSlotsBoundsConcurrent(t *testing.T) {
+	t.Parallel()
+	slots := NewSyncSlots(2)
+
+	var inflight atomic.Int32
+	var maxInflight atomic.Int32
+	var claimed atomic.Int32
+
+	claim := func() error {
+		if claimed.Add(1) > 4 {
+			return pgx.ErrNoRows
+		}
+		return nil
+	}
+
+	for i := 0; i < 8; i++ {
+		_, err := slots.TryStart(claim, func() {
+			n := inflight.Add(1)
+			for {
+				cur := maxInflight.Load()
+				if n <= cur || maxInflight.CompareAndSwap(cur, n) {
+					break
+				}
+			}
+			time.Sleep(30 * time.Millisecond)
+			inflight.Add(-1)
+		})
+		if err != nil && !errors.Is(err, pgx.ErrNoRows) {
+			t.Fatalf("claim: %v", err)
+		}
+	}
+	slots.Wait()
+	if maxInflight.Load() > 2 {
+		t.Fatalf("max inflight=%d want <=2", maxInflight.Load())
+	}
+}
+
+func TestSyncSlotsRejectsWhenFull(t *testing.T) {
+	t.Parallel()
+	slots := NewSyncSlots(1)
+	block := make(chan struct{})
+	started, err := slots.TryStart(func() error { return nil }, func() { <-block })
+	if err != nil || !started {
+		t.Fatalf("first start: started=%v err=%v", started, err)
+	}
+	started, err = slots.TryStart(func() error {
+		t.Fatal("should not claim when full")
+		return nil
+	}, func() {})
+	if err != nil || started {
+		t.Fatalf("second start: started=%v err=%v", started, err)
+	}
+	close(block)
+	slots.Wait()
+}