fix

apps/api/cmd/seed-a1/category_backfill.go

@@ -10,6 +10,7 @@ import (
 	"strings"
 
 	"github.com/descrybe/descrybe-v2/apps/api/internal/billing"
+	"github.com/descrybe/descrybe-v2/apps/api/internal/processing"
 	"github.com/google/uuid"
 	"github.com/jackc/pgx/v5/pgxpool"
 )
@@ -34,30 +35,9 @@ type categoryBackfillResult struct {
 }
 
 // backfillMappedCategoriesFromProcessed copies processed_products.category into
-// raw_products.mapped_data.category for A1 only. Legacy dumps store category on
-// processed rows (unique_id codes); feed mappings never mapped a category field,
-// so re-processing without this backfill yields Uncategorized / grey C coverage.
+// raw_products.mapped_data.category for A1 only. Delegates to processing.
 func backfillMappedCategoriesFromProcessed(ctx context.Context, pg *pgxpool.Pool, companyID uuid.UUID) (int64, error) {
-	ct, err := pg.Exec(ctx, `
-		UPDATE raw_products r
-		SET mapped_data = jsonb_set(
-			COALESCE(r.mapped_data, '{}'::jsonb),
-			'{category}',
-			to_jsonb(p.category),
-			true
-		),
-		updated_at = now()
-		FROM processed_products p
-		WHERE p.raw_product_id = r.id
-		  AND p.company_id = $1
-		  AND r.company_id = $1
-		  AND COALESCE(NULLIF(trim(p.category), ''), '') <> ''
-		  AND lower(trim(p.category)) <> 'none'
-		  AND COALESCE(NULLIF(trim(r.mapped_data->>'category'), ''), '') = ''`, companyID)
-	if err != nil {
-		return 0, fmt.Errorf("backfill mapped category: %w", err)
-	}
-	return ct.RowsAffected(), nil
+	return processing.BackfillMappedCategoriesFromProcessed(ctx, pg, companyID)
 }
 
 // backfillCategoriesFromMySQLDump streams dump processed_products for the A1

apps/api/internal/catalog/clone.go

@@ -358,6 +358,15 @@ func cloneCompanyCatalog(ctx context.Context, pool *pgxpool.Pool, src, dest uuid
 	}
 	add("raw_products", ct.RowsAffected())
 
+	var mappedWithCat int64
+	if err := tx.QueryRow(ctx, `
+		SELECT COUNT(*) FROM raw_products
+		WHERE company_id = $1
+			AND COALESCE(NULLIF(trim(mapped_data->>'category'), ''), '') <> ''`, dest).Scan(&mappedWithCat); err != nil {
+		return nil, fmt.Errorf("count mapped categories: %w", err)
+	}
+	add("mapped_with_category", mappedWithCat)
+
 	ct, err = tx.Exec(ctx, `
 		INSERT INTO processed_products (
 			id, company_id, user_id, product_id, name, category, description, processed_description,

apps/api/internal/processing/category_backfill.go

@@ -92,6 +92,67 @@ func BackfillProcessedCategoriesFromMapped(ctx context.Context, pool *pgxpool.Po
 	return out, nil
 }
 
+// BackfillMappedCategoriesFromProcessed copies processed_products.category into
+// raw_products.mapped_data.category when mapped category is empty. Legacy A1 dumps
+// often store category only on processed rows; feed mappings do not map category,
+// so Fix Catalog / re-seed must push processed → mapped for Products UI coverage.
+// Does not invent categories.
+func BackfillMappedCategoriesFromProcessed(ctx context.Context, pool *pgxpool.Pool, companyID uuid.UUID) (int64, error) {
+	if pool == nil {
+		return 0, fmt.Errorf("mapped category backfill: nil pool")
+	}
+	if companyID == uuid.Nil {
+		return 0, fmt.Errorf("mapped category backfill: empty company id")
+	}
+	ct, err := pool.Exec(ctx, `
+		UPDATE raw_products r
+		SET mapped_data = jsonb_set(
+			COALESCE(r.mapped_data, '{}'::jsonb),
+			'{category}',
+			to_jsonb(p.category),
+			true
+		),
+		updated_at = now()
+		FROM processed_products p
+		WHERE p.raw_product_id = r.id
+		  AND p.company_id = $1
+		  AND r.company_id = $1
+		  AND COALESCE(NULLIF(trim(p.category), ''), '') <> ''
+		  AND lower(trim(p.category)) <> 'none'
+		  AND COALESCE(NULLIF(trim(r.mapped_data->>'category'), ''), '') = ''`, companyID)
+	if err != nil {
+		return 0, fmt.Errorf("mapped category backfill: %w", err)
+	}
+	return ct.RowsAffected(), nil
+}
+
+// CountMappedCategoryCoverage returns how many raw_products rows have a non-empty
+// mapped_data.category vs empty, plus taxonomy row count for the company.
+func CountMappedCategoryCoverage(ctx context.Context, pool *pgxpool.Pool, companyID uuid.UUID) (withCat, withoutCat, taxonomy int64, err error) {
+	if pool == nil {
+		return 0, 0, 0, fmt.Errorf("mapped category coverage: nil pool")
+	}
+	err = pool.QueryRow(ctx, `
+		SELECT
+			COUNT(*) FILTER (
+				WHERE COALESCE(NULLIF(trim(mapped_data->>'category'), ''), '') <> ''
+			),
+			COUNT(*) FILTER (
+				WHERE COALESCE(NULLIF(trim(mapped_data->>'category'), ''), '') = ''
+			)
+		FROM raw_products
+		WHERE company_id = $1`, companyID).Scan(&withCat, &withoutCat)
+	if err != nil {
+		return 0, 0, 0, fmt.Errorf("mapped category coverage: %w", err)
+	}
+	err = pool.QueryRow(ctx, `
+		SELECT COUNT(*) FROM categories WHERE company_id = $1`, companyID).Scan(&taxonomy)
+	if err != nil {
+		return withCat, withoutCat, 0, fmt.Errorf("taxonomy count: %w", err)
+	}
+	return withCat, withoutCat, taxonomy, nil
+}
+
 // CountProcessedCategoryCoverage returns how many processed rows for companyID have
 // a non-empty category vs empty/"none".
 func CountProcessedCategoryCoverage(ctx context.Context, pool *pgxpool.Pool, companyID uuid.UUID) (withCat, withoutCat int64, err error) {

apps/api/internal/processing/category_test.go

@@ -470,3 +470,82 @@ func TestBackfillProcessedCategoriesFromMapped(t *testing.T) {
 		t.Fatalf("coverage with=%d without=%d", withCat, withoutCat)
 	}
 }
+
+func TestBackfillMappedCategoriesFromProcessed(t *testing.T) {
+	dsn := os.Getenv("DATABASE_URL")
+	if dsn == "" {
+		t.Skip("DATABASE_URL not set")
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+	defer cancel()
+
+	pg, err := pgxpool.New(ctx, dsn)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer pg.Close()
+
+	companyID := uuid.New()
+	rawID := uuid.New()
+	ppID := uuid.New()
+	gtin := "map-bf-" + companyID.String()[:8]
+
+	if _, err := pg.Exec(ctx, `INSERT INTO companies (id, name) VALUES ($1, $2)`, companyID, "mapped-backfill-test"); err != nil {
+		t.Fatal(err)
+	}
+	defer func() {
+		_, _ = pg.Exec(context.Background(), `DELETE FROM processed_products WHERE company_id = $1`, companyID)
+		_, _ = pg.Exec(context.Background(), `DELETE FROM raw_products WHERE company_id = $1`, companyID)
+		_, _ = pg.Exec(context.Background(), `DELETE FROM companies WHERE id = $1`, companyID)
+	}()
+
+	if _, err := pg.Exec(ctx, `
+		INSERT INTO raw_products (id, company_id, gtin, raw_data, mapped_data, is_processed, processing_status)
+		VALUES ($1, $2, $3, '{}'::jsonb, '{"name":"Widget"}'::jsonb, true, 'processed')`,
+		rawID, companyID, gtin); err != nil {
+		t.Fatal(err)
+	}
+	if _, err := pg.Exec(ctx, `
+		INSERT INTO processed_products (
+			id, company_id, product_id, name, category, description, processed_name, processed_description,
+			raw_product_id, status, attributes, processed_attributes, field_sources
+		) VALUES (
+			$1, $2, $3, 'Widget', '42', 'x', 'Widget', 'x',
+			$4, 'needs_review', '{}'::jsonb, '{}'::jsonb, '{}'::jsonb
+		)`, ppID, companyID, gtin, rawID); err != nil {
+		t.Fatal(err)
+	}
+
+	n, err := BackfillMappedCategoriesFromProcessed(ctx, pg, companyID)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if n != 1 {
+		t.Fatalf("updated=%d want 1", n)
+	}
+	var mappedCat string
+	if err := pg.QueryRow(ctx, `
+		SELECT COALESCE(mapped_data->>'category', '') FROM raw_products WHERE id = $1`, rawID).Scan(&mappedCat); err != nil {
+		t.Fatal(err)
+	}
+	if mappedCat != "42" {
+		t.Fatalf("mapped category=%q want 42", mappedCat)
+	}
+
+	n2, err := BackfillMappedCategoriesFromProcessed(ctx, pg, companyID)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if n2 != 0 {
+		t.Fatalf("second backfill updated=%d want 0", n2)
+	}
+
+	withCat, withoutCat, taxonomy, err := CountMappedCategoryCoverage(ctx, pg, companyID)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if withCat != 1 || withoutCat != 0 || taxonomy != 0 {
+		t.Fatalf("mapped coverage with=%d without=%d taxonomy=%d", withCat, withoutCat, taxonomy)
+	}
+}

apps/api/internal/processing/fix_company_catalog.go

@@ -41,9 +41,19 @@ type FixCompanyCatalogResult struct {
 
 	WeakHashesCleared    int `json:"weak_hashes_cleared"`
 	CategoriesBackfilled int `json:"categories_backfilled"`
+	// MappedCategoriesBackfilled is processed → mapped_data.category copies.
+	MappedCategoriesBackfilled int `json:"mapped_categories_backfilled"`
+	// Coverage after Fix (Products UI reads mapped_data.category for raw rows).
+	TaxonomyCategories    int64 `json:"taxonomy_categories"`
+	MappedWithCategory    int64 `json:"mapped_with_category"`
+	MappedWithoutCategory int64 `json:"mapped_without_category"`
+	ProcessedWithCategory    int64 `json:"processed_with_category"`
+	ProcessedWithoutCategory int64 `json:"processed_without_category"`
 	// Hashes / Categories alias the long names for flash.admin.fixA1Success.
 	Hashes     int `json:"hashes"`
 	Categories int `json:"categories"`
+	// MappedBackfilled aliases mapped_categories_backfilled for flash {mapped_backfilled}.
+	MappedBackfilled int `json:"mapped_backfilled"`
 
 	DescriptionsNormalized int   `json:"descriptions_normalized"`
 	DescriptionsBackfilled int   `json:"descriptions_backfilled"`
@@ -106,6 +116,14 @@ func FixCompanyCatalog(ctx context.Context, pool *pgxpool.Pool, companyID uuid.U
 	out.MetaBackfilled = hygiene.MetaBackfilled
 	out.ProductsScanned = hygiene.ProductsScanned
 
+	if opts.BackfillCategories {
+		mappedN, err := BackfillMappedCategoriesFromProcessed(ctx, pool, companyID)
+		if err != nil {
+			return out, fmt.Errorf("mapped category backfill: %w", err)
+		}
+		out.MappedCategoriesBackfilled = int(mappedN)
+	}
+
 	sanitized, err := BackfillCompanyProcessedAttributes(ctx, pool, companyID)
 	if err != nil {
 		return out, fmt.Errorf("sanitize attributes: %w", err)
@@ -119,9 +137,24 @@ func FixCompanyCatalog(ctx context.Context, pool *pgxpool.Pool, companyID uuid.U
 	out.ReprocessNeededCount = needed
 	out.ReprocessSampleRawProductIDs = sample
 
-	// Short keys for flash.admin.fixA1Success {prompts,hashes,categories}.
+	mWith, mWithout, taxonomy, err := CountMappedCategoryCoverage(ctx, pool, companyID)
+	if err != nil {
+		return out, err
+	}
+	out.MappedWithCategory = mWith
+	out.MappedWithoutCategory = mWithout
+	out.TaxonomyCategories = taxonomy
+	pWith, pWithout, err := CountProcessedCategoryCoverage(ctx, pool, companyID)
+	if err != nil {
+		return out, err
+	}
+	out.ProcessedWithCategory = pWith
+	out.ProcessedWithoutCategory = pWithout
+
+	// Short keys for flash.admin.fixA1Success {prompts,hashes,categories,mapped_backfilled}.
 	out.Prompts = out.CategoryPromptsUpdated
 	out.Hashes = out.WeakHashesCleared
 	out.Categories = out.CategoriesBackfilled
+	out.MappedBackfilled = out.MappedCategoriesBackfilled
 	return out, nil
 }

apps/api/internal/processing/pipeline.go

@@ -598,6 +598,7 @@ func (p *Pipeline) ProcessJob(ctx context.Context, jobID uuid.UUID) error {
 		return err
 	}
 	if !IsProcessableJobStatus(status) {
+		log.Printf("processing: skip job=%s status=%s reason=not_processable", jobID, status)
 		return nil
 	}
 
@@ -625,6 +626,9 @@ func (p *Pipeline) ProcessJob(ctx context.Context, jobID uuid.UUID) error {
 		modeLabel = label
 	}
 
+	log.Printf("processing: start job=%s company=%s type=%s mode=%s prior_processed=%d",
+		jobID, companyID, processingType, modeLabel, alreadyProcessed)
+
 	progress := InitialStepProgress(processingType)
 	if len(progress) > 0 {
 		progress[0].Status = "running"
@@ -648,6 +652,11 @@ func (p *Pipeline) ProcessJob(ctx context.Context, jobID uuid.UUID) error {
 	batch := resolveBatchSize(p.BatchSize)
 	progressEvery := resolveProgressEvery(p.ProgressEvery)
 	jobCache := p.loadJobScopedCache(ctx, companyID, jobID)
+	if !jobCache.stepPolicy().AllowAI {
+		log.Printf("processing: ai_skip job=%s reason=entitlement_can_use_ai", jobID)
+	} else if !jobEngine.CompleterEnabled() {
+		log.Printf("processing: ai_skip job=%s reason=openai_not_configured", jobID)
+	}
 	processed := alreadyProcessed
 	failed := 0
 	tokenTotal := 0
@@ -672,6 +681,7 @@ func (p *Pipeline) ProcessJob(ctx context.Context, jobID uuid.UUID) error {
 			flushProgress(true)
 			if errors.Is(err, errJobCancelled) {
 				_ = p.reclaimOrphanedProcessingItems(ctx, jobID)
+				log.Printf("processing: cancelled job=%s", jobID)
 				return nil
 			}
 			return fmt.Errorf("processing: check cancel job=%s: %w", jobID, err)
@@ -728,6 +738,7 @@ func (p *Pipeline) ProcessJob(ctx context.Context, jobID uuid.UUID) error {
 					flushProgress(true)
 					if errors.Is(err, errJobCancelled) {
 						_ = p.reclaimOrphanedProcessingItems(ctx, jobID)
+						log.Printf("processing: cancelled job=%s", jobID)
 						return nil
 					}
 					return fmt.Errorf("processing: check cancel job=%s: %w", jobID, err)
@@ -807,7 +818,8 @@ func (p *Pipeline) ProcessJob(ctx context.Context, jobID uuid.UUID) error {
 			current_step = $6, step_progress = $7::jsonb,
 			ai_provider_mode = $8
 		WHERE id = $1 AND status = 'running'`, jobID, finalStatus, processed, errMsg, tokenTotal, finalStep, finalJSON, finalMode)
-	log.Printf("processing: finished job=%s status=%s processed=%d failed=%d mode=%s", jobID, finalStatus, processed, failed, finalMode)
+	log.Printf("processing: finished job=%s status=%s processed=%d failed=%d mode=%s event=%s",
+		jobID, finalStatus, processed, failed, finalMode, finalStatus)
 	return err
 }
 
@@ -1556,7 +1568,12 @@ func (p *Pipeline) processOne(ctx context.Context, companyID, jobID uuid.UUID, i
 	// name tokens are not wiped. Empty taxonomy set skips filtering (unit tests).
 	if cache != nil {
 		coerceCategoryToCompanyUniqueID(&result, cache.categoryNamesByUID, cache.categoryUniqueIDs)
+		catPreFilter := strings.TrimSpace(result.Category)
 		filterCategoryIfInvalid(&result, cache.categoryUniqueIDs)
+		if catPreFilter != "" && strings.TrimSpace(result.Category) == "" {
+			log.Printf("processing: category_empty job=%s raw=%s rejected=%s reason=unknown_unique_id",
+				jobID, it.RawID, catPreFilter)
+		}
 		syncCategoryName(&result, cache.categoryNamesByUID)
 	}
 	// Re-apply after category validation so allowlist matches the persisted category.