greeneclipse/descrybe
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fix

Browse Source
This commit is contained in:
greeneclipse
2026-08-17 21:20:45 +02:00
parent 321f11e817
commit 6fcdc74843
157 changed files with 2895 additions and 7544 deletions
Download Patch File Download Diff File Expand all files Collapse all files
apps/api/cmd/migrator/postimport.go
+2 -2
View File
@@ -4,13 +4,13 @@ import (
"context"
"fmt"
"log"
"net/url"
"os"
"path/filepath"
"strings"
"time"
"github.com/descrybe/descrybe-v2/apps/api/internal/auth"
"github.com/descrybe/descrybe-v2/apps/api/internal/mail"
"github.com/google/uuid"
"github.com/jackc/pgx/v5/pgxpool"
)
@@ -37,7 +37,7 @@ func webOrigin() string {
}
func setPasswordInviteURL(token string) string {
return webOrigin() + "/accept-invite?token=" + url.QueryEscape(token)
return mail.AcceptInviteURL(webOrigin(), token)
}
// prepareSetPasswordHooks creates invites for active users with must_set_password=true.
apps/api/cmd/migrator/postimport_test.go
+2 -2
View File
@@ -10,7 +10,7 @@ import (
func TestSetPasswordInviteURL(t *testing.T) {
t.Setenv("WEB_ORIGIN", "https://app.example.com/")
got := setPasswordInviteURL("tok+1")
wantPrefix := "https://app.example.com/accept-invite?token="
wantPrefix := "https://app.example.com/accept-invite#token="
if !strings.HasPrefix(got, wantPrefix) {
t.Fatalf("got %q", got)
}
@@ -31,4 +31,4 @@ func TestWebOriginDefault(t *testing.T) {
if webOrigin() != "http://localhost:5174" {
t.Fatalf("default origin")
}
}
}
apps/api/internal/auth/errors.go
+2 -1
View File
@@ -20,6 +20,8 @@ func ClientError(err error) (msg string, ok bool) {
switch {
case err == nil:
return "", false
case errors.Is(err, ErrSyntheticEmail):
return "this email cannot receive invites", true
case errors.Is(err, ErrRegisterFieldsRequired),
errors.Is(err, ErrPasswordTooShort),
errors.Is(err, ErrPasswordAlreadySet),
@@ -32,7 +34,6 @@ func ClientError(err error) (msg string, ok bool) {
errors.Is(err, ErrInviteNotFound),
errors.Is(err, ErrTokenInvalid),
errors.Is(err, ErrEmailRequired),
errors.Is(err, ErrSyntheticEmail),
errors.Is(err, ErrNotEligibleSetPassword),
errors.Is(err, ErrEmailMismatch),
errors.Is(err, ErrNotCompanyOwner),
apps/api/internal/auth/ownership_test.go
+15
View File
@@ -1,6 +1,7 @@
package auth
import (
"strings"
"testing"
)
@@ -18,3 +19,17 @@ func TestOwnershipErrorClientFacing(t *testing.T) {
}
}
}
func TestClientError_syntheticEmailPublicString(t *testing.T) {
t.Parallel()
msg, ok := ClientError(ErrSyntheticEmail)
if !ok {
t.Fatal("ErrSyntheticEmail should be a client error")
}
if msg != "this email cannot receive invites" {
t.Fatalf("public msg=%q", msg)
}
if strings.Contains(msg, "synthetic") || strings.Contains(msg, "migration") {
t.Fatalf("internal wording leaked: %q", msg)
}
}
apps/api/internal/catalog/service.go
+2 -2
View File
@@ -1265,7 +1265,7 @@ func (s *Service) GetProcessedProduct(ctx context.Context, companyID, id uuid.UU
item["content_language"] = primary
item["content_languages"] = company.LoadContentLanguages(ctx, s.Pool, companyID)
if loc, err := company.DecodeLocalizedContent(item["localized_content"]); err == nil {
item["localized_content"] = loc
item["localized_content"] = loc.Public()
}
return item, nil
}
@@ -1328,7 +1328,7 @@ func (s *Service) GetRawProduct(ctx context.Context, companyID, id uuid.UUID) (m
item["content_language"] = primary
item["content_languages"] = company.LoadContentLanguages(ctx, s.Pool, companyID)
if loc, err := company.DecodeLocalizedContent(item["localized_content"]); err == nil {
item["localized_content"] = loc
item["localized_content"] = loc.Public()
}
return item, nil
}
apps/api/internal/company/lang_content.go
+15
View File
@@ -31,6 +31,21 @@ type LocalizedFields struct {
// LocalizedContent is language-code → per-language product output fields.
type LocalizedContent map[string]LocalizedFields
// Public returns a copy safe for API/UI clients. Drops enhance_input_hash
// (pipeline cache key) so GET product payloads cannot leak job internals.
// UpdateProcessedProduct merges from DB, so omitting the hash on GET is safe.
func (c LocalizedContent) Public() LocalizedContent {
if len(c) == 0 {
return c
}
out := make(LocalizedContent, len(c))
for lang, fields := range c {
fields.EnhanceInputHash = ""
out[lang] = fields
}
return out
}
// SanitizeLangPromptMap validates language codes, sanitizes prompts, and drops empties.
// Accepts LangPromptAny ("*") as a shared any-language prompt key.
func SanitizeLangPromptMap(in map[string]string, maxRunes int) (LangPromptMap, error) {
apps/api/internal/company/lang_content_test.go
+30
View File
@@ -2,6 +2,7 @@ package company
import (
"encoding/json"
"strings"
"testing"
)
@@ -101,3 +102,32 @@ func TestLocalizedContentRoundTrip(t *testing.T) {
t.Fatalf("got %#v", f)
}
}
func TestLocalizedContentPublicOmitsEnhanceHash(t *testing.T) {
t.Parallel()
c := LocalizedContent{
"sl": {
ProcessedName: "Naslov",
ProcessedDescription: "Opis",
MetaTitle: "Meta",
EnhanceInputHash: "deadbeef",
},
}
pub := c.Public()
if pub["sl"].EnhanceInputHash != "" {
t.Fatalf("public hash leaked: %#v", pub["sl"])
}
if pub["sl"].ProcessedName != "Naslov" || pub["sl"].MetaTitle != "Meta" {
t.Fatalf("public stripped too much: %#v", pub["sl"])
}
if c["sl"].EnhanceInputHash != "deadbeef" {
t.Fatalf("Public must not mutate storage copy: %#v", c["sl"])
}
b, err := json.Marshal(pub)
if err != nil {
t.Fatal(err)
}
if strings.Contains(string(b), "enhance_input_hash") || strings.Contains(string(b), "deadbeef") {
t.Fatalf("hash in JSON: %s", b)
}
}
apps/api/internal/httpapi/accept_invite_url_test.go
+1 -1
View File
@@ -9,7 +9,7 @@ import (
func TestAcceptInviteURL(t *testing.T) {
t.Parallel()
got := mail.AcceptInviteURL("http://localhost:28472/", "abc123")
want := "http://localhost:28472/accept-invite?token=abc123"
want := "http://localhost:28472/accept-invite#token=abc123"
if got != want {
t.Fatalf("AcceptInviteURL = %q want %q", got, want)
}
apps/api/internal/httpapi/catalog_handlers.go
+26
View File
@@ -7,6 +7,7 @@ import (
"github.com/descrybe/descrybe-v2/apps/api/internal/catalog"
"github.com/descrybe/descrybe-v2/apps/api/internal/company"
"github.com/descrybe/descrybe-v2/apps/api/internal/processing"
"github.com/go-chi/chi/v5"
"github.com/google/uuid"
"github.com/jackc/pgx/v5"
@@ -403,6 +404,7 @@ func (s *Server) handleListProducts(w http.ResponseWriter, r *http.Request) {
if detailed {
attachProductQuality(items)
}
stripCatalogSEOMetaIfOmitted(processing.CompanyOmitsSEOMeta(r.Context(), s.Pool, cid), items...)
resp := map[string]any{"products": items, "total": total, "kind": "processed", "limit": limit, "offset": offset, "detailed": detailed}
if nextCursor, nextAfter := catalog.NextProductCursor(f, items, limit); nextCursor != "" || nextAfter != "" {
if nextCursor != "" {
@@ -438,6 +440,7 @@ func (s *Server) handleGetProduct(w http.ResponseWriter, r *http.Request) {
return
}
}
stripCatalogSEOMetaIfOmitted(processing.CompanyOmitsSEOMeta(r.Context(), s.Pool, cid), item)
JSON(w, http.StatusOK, item)
}
@@ -458,5 +461,28 @@ func (s *Server) handleUpdateProduct(w http.ResponseWriter, r *http.Request) {
ClientOrLog(w, http.StatusBadRequest, "could not update product", err, catalog.ClientError)
return
}
stripCatalogSEOMetaIfOmitted(processing.CompanyOmitsSEOMeta(r.Context(), s.Pool, cid), item)
JSON(w, http.StatusOK, item)
}
// stripCatalogSEOMetaIfOmitted drops meta_title / meta_description from dashboard
// product payloads when omit is true (same detection as V1 process / CompanyOmitsSEOMeta).
func stripCatalogSEOMetaIfOmitted(omit bool, items ...map[string]any) {
if !omit {
return
}
for _, item := range items {
if item == nil {
continue
}
delete(item, "meta_title")
delete(item, "meta_description")
if loc, ok := item["localized_content"].(company.LocalizedContent); ok {
for lang, fields := range loc {
fields.MetaTitle = ""
fields.MetaDescription = ""
loc[lang] = fields
}
}
}
}
apps/api/internal/httpapi/catalog_handlers_seo_test.go
+76
View File
@@ -0,0 +1,76 @@
package httpapi
import (
"testing"
"github.com/descrybe/descrybe-v2/apps/api/internal/company"
"github.com/descrybe/descrybe-v2/apps/api/internal/processing"
"github.com/google/uuid"
)
func catalogSEOItem() map[string]any {
return map[string]any{
"name": "Widget",
"meta_title": "T",
"meta_description": "D",
"localized_content": company.LocalizedContent{
"sl": {ProcessedName: "N", MetaTitle: "LT", MetaDescription: "LD", EnhanceInputHash: "abc"},
},
}
}
func assertCatalogSEOOmitted(t *testing.T, item map[string]any) {
t.Helper()
if _, ok := item["meta_title"]; ok {
t.Fatalf("must omit meta_title: %v", item)
}
if _, ok := item["meta_description"]; ok {
t.Fatalf("must omit meta_description: %v", item)
}
loc := item["localized_content"].(company.LocalizedContent)
if loc["sl"].MetaTitle != "" || loc["sl"].MetaDescription != "" {
t.Fatalf("localized meta leaked: %#v", loc["sl"])
}
if loc["sl"].ProcessedName != "N" {
t.Fatalf("stripped too much: %#v", loc["sl"])
}
}
func TestStripCatalogSEOMetaIfOmitted(t *testing.T) {
t.Parallel()
a1 := uuid.MustParse("604f23a8-b66e-4b21-8b45-0d72b68f4790")
other := uuid.MustParse("11111111-1111-1111-1111-111111111111")
item := catalogSEOItem()
stripCatalogSEOMetaIfOmitted(processing.CompanyOmitsSEOMetaLookup(other, "", "Acme", false), item)
if item["meta_title"] != "T" {
t.Fatalf("ordinary company must keep meta: %v", item["meta_title"])
}
stripCatalogSEOMetaIfOmitted(processing.CompanyOmitsSEOMetaLookup(a1, "", "", false), item)
assertCatalogSEOOmitted(t, item)
}
func TestStripCatalogSEOMetaIfOmitted_demoByName(t *testing.T) {
t.Parallel()
other := uuid.MustParse("11111111-1111-1111-1111-111111111111")
omit := processing.CompanyOmitsSEOMetaLookup(other, "", " platform demo ", false)
if !omit {
t.Fatal("Platform Demo by name must omit SEO meta")
}
item := catalogSEOItem()
stripCatalogSEOMetaIfOmitted(omit, item)
assertCatalogSEOOmitted(t, item)
}
func TestStripCatalogSEOMetaIfOmitted_a1PromptMarkers(t *testing.T) {
t.Parallel()
other := uuid.MustParse("11111111-1111-1111-1111-111111111111")
omit := processing.CompanyOmitsSEOMetaLookup(other, "", "Acme", true)
if !omit {
t.Fatal("A1-style prompt markers must omit SEO meta")
}
item := catalogSEOItem()
stripCatalogSEOMetaIfOmitted(omit, item)
assertCatalogSEOOmitted(t, item)
}
apps/api/internal/httpapi/catalog_v1_handlers.go
+17
View File
@@ -24,6 +24,23 @@ func presentV1Category(item map[string]any) map[string]any {
}
}
// handleV1GetCategory serves GET /api/v1/categories/{id} as the public list DTO
// (no prompts, formulas, or has_prompt). Dashboard GET stays on handleGetCategory.
func (s *Server) handleV1GetCategory(w http.ResponseWriter, r *http.Request) {
cid, _ := CompanyIDFromContext(r.Context())
id, err := uuid.Parse(chi.URLParam(r, "id"))
if err != nil {
Error(w, http.StatusBadRequest, "invalid id")
return
}
item, err := s.Catalog.GetCategory(r.Context(), cid, id)
if err != nil {
Error(w, http.StatusNotFound, "not found")
return
}
v1OK(w, http.StatusOK, presentV1Category(item), nil)
}
func presentV1Attribute(item map[string]any) map[string]any {
out := map[string]any{
"id": item["id"],
apps/api/internal/httpapi/catalog_v1_handlers_test.go
+13 -5
View File
@@ -17,6 +17,9 @@ func TestPresentV1CategoryAndAttribute(t *testing.T) {
cat := presentV1Category(map[string]any{
"id": id, "unique_id": "electronics", "name": "Electronics",
"created_at": ts, "updated_at": ts,
"prompt": "--- Title --- secret", "prompts": map[string]string{"en": "x"},
"has_prompt": true, "title_template": []any{"--- Title ---"},
"description_template": []any{"x"},
})
if cat["unique_id"] != "electronics" || cat["name"] != "Electronics" {
t.Fatalf("category=%v", cat)
@@ -24,6 +27,11 @@ func TestPresentV1CategoryAndAttribute(t *testing.T) {
if cat["created_at"] != "2026-07-01T08:00:00Z" {
t.Fatalf("created_at=%v", cat["created_at"])
}
for _, leak := range []string{"prompt", "prompts", "has_prompt", "title_template", "description_template"} {
if _, ok := cat[leak]; ok {
t.Fatalf("public category leaked %q: %v", leak, cat)
}
}
attr := presentV1Attribute(map[string]any{
"id": id, "attribute_key": "color", "name": "Color", "value_type": "string",
@@ -41,11 +49,11 @@ func TestPresentV1CategoryAndAttribute(t *testing.T) {
func TestV1OpenAPICategoriesAttributesLegacyContract(t *testing.T) {
body := string(v1OpenAPIYAML)
needles := []string{
"LegacyCategoriesResponse",
"LegacyAttributesResponse",
"LegacyCategoryCreateResponse",
"LegacyAttributeCreateResponse",
"LegacySuccessMessage",
"CategoriesEnvelope",
"AttributesEnvelope",
"CategoryCreateEnvelope",
"AttributeCreateEnvelope",
"SuccessMessage",
"category_unique_id",
"attribute_key",
"parent_id",
apps/api/internal/httpapi/company_invite_test.go
+1 -1
View File
@@ -10,7 +10,7 @@ import (
func TestInviteAcceptURLAlwaysShareable(t *testing.T) {
t.Parallel()
url := mail.AcceptInviteURL("https://app.example.com/", "tok_abc")
want := "https://app.example.com/accept-invite?token=tok_abc"
want := "https://app.example.com/accept-invite#token=tok_abc"
if url != want {
t.Fatalf("AcceptInviteURL = %q want %q", url, want)
}
apps/api/internal/httpapi/health_test.go
+29
View File
@@ -413,6 +413,35 @@ func TestRouterMetricsHiddenInProductionForRemote(t *testing.T) {
}
}
func TestRouterMetricsHiddenInProductionForSpoofedLoopbackXFF(t *testing.T) {
t.Parallel()
s := testAPIServer()
s.Config.AppEnv = "production"
s.Config.MetricsPublic = false
s.Config.TrustedProxies = []string{"10.0.0.0/8"}
h := s.Router()
req := httptest.NewRequest(http.MethodGet, "/metrics", nil)
req.RemoteAddr = "10.0.0.5:443"
req.Header.Set("X-Forwarded-For", "127.0.0.1")
rec := httptest.NewRecorder()
h.ServeHTTP(rec, req)
if rec.Code != http.StatusNotFound {
t.Fatalf("trusted proxy spoofed loopback XFF status=%d want 404", rec.Code)
}
s.Config.MetricsPublic = true
hPub := s.Router()
reqPub := httptest.NewRequest(http.MethodGet, "/metrics", nil)
reqPub.RemoteAddr = "10.0.0.5:443"
reqPub.Header.Set("X-Forwarded-For", "127.0.0.1")
recPub := httptest.NewRecorder()
hPub.ServeHTTP(recPub, reqPub)
if recPub.Code != http.StatusOK {
t.Fatalf("METRICS_PUBLIC trusted proxy XFF status=%d", recPub.Code)
}
}
func assertGateBody(t *testing.T, rec *httptest.ResponseRecorder, errorCode string, maintenance, readOnly bool) {
t.Helper()
var body map[string]any
apps/api/internal/httpapi/processing_handlers.go
+1
View File
@@ -119,6 +119,7 @@ func (s *Server) handleGetProcessingJob(w http.ResponseWriter, r *http.Request)
Error(w, http.StatusInternalServerError, "load failed")
return
}
items = processing.ApplyV1SEOMetaPolicy(cid, items)
JSON(w, http.StatusOK, processing.FormatJobStatusResponse(job, items, true))
return
}
apps/api/internal/httpapi/products_v1_handlers.go
+78
View File
@@ -1,6 +1,7 @@
package httpapi
import (
"errors"
"net/http"
"strconv"
"strings"
@@ -8,6 +9,10 @@ import (
"github.com/descrybe/descrybe-v2/apps/api/internal/catalog"
"github.com/descrybe/descrybe-v2/apps/api/internal/marketing"
"github.com/descrybe/descrybe-v2/apps/api/internal/processing"
"github.com/go-chi/chi/v5"
"github.com/google/uuid"
"github.com/jackc/pgx/v5"
)
const (
@@ -88,6 +93,49 @@ func presentV1Product(item map[string]any) map[string]any {
}
}
func presentV1ProductDetail(item map[string]any) map[string]any {
out := presentV1Product(item)
desc := firstNonEmpty(asMapString(item["processed_description"]), asMapString(item["description"]))
if desc == "" {
out["description"] = nil
} else {
out["description"] = desc
}
out["attributes"] = firstPublicAttributes(item)
mapped, _ := item["mapped_data"].(map[string]any)
main, more := catalog.ExtractProductImages(mapped, nil)
if main == "" {
out["main_image"] = nil
} else {
out["main_image"] = main
}
if len(more) > 0 {
out["more_images"] = more
} else {
out["more_images"] = nil
}
if eprel, ok := item["eprel"]; ok {
out["eprel"] = eprel
} else {
out["eprel"] = nil
}
return out
}
func firstPublicAttributes(item map[string]any) any {
for _, key := range []string{"processed_attributes", "attributes"} {
v := item[key]
if v == nil {
continue
}
if m, ok := v.(map[string]any); ok && len(m) == 0 {
continue
}
return v
}
return map[string]any{}
}
func nullIfEmptyAny(v any) any {
if v == nil {
return nil
@@ -150,6 +198,36 @@ func (s *Server) handleV1ListProducts(w http.ResponseWriter, r *http.Request) {
v1OK(w, http.StatusOK, data, v1ProductListMeta(page, limit, total))
}
// handleV1GetProduct serves GET /api/v1/products/{id} as a public DTO aligned with
// process items (name, description, attributes, images, eprel). Dashboard GET stays
// on handleGetProduct.
func (s *Server) handleV1GetProduct(w http.ResponseWriter, r *http.Request) {
cid, _ := CompanyIDFromContext(r.Context())
id, err := uuid.Parse(chi.URLParam(r, "id"))
if err != nil {
Error(w, http.StatusBadRequest, "invalid id")
return
}
item, err := s.Catalog.GetProcessedProduct(r.Context(), cid, id)
if err != nil {
if !errors.Is(err, pgx.ErrNoRows) {
Error(w, http.StatusInternalServerError, "lookup failed")
return
}
item, err = s.Catalog.GetRawProduct(r.Context(), cid, id)
if err != nil {
if errors.Is(err, pgx.ErrNoRows) {
Error(w, http.StatusNotFound, "not found")
return
}
Error(w, http.StatusInternalServerError, "lookup failed")
return
}
}
stripCatalogSEOMetaIfOmitted(processing.CompanyOmitsSEOMeta(r.Context(), s.Pool, cid), item)
v1OK(w, http.StatusOK, presentV1ProductDetail(item), nil)
}
// handleV1ListProductQuality serves GET /api/v1/products/quality with the legacy
// { data, meta } envelope (quality rows + page/limit/total).
func (s *Server) handleV1ListProductQuality(w http.ResponseWriter, r *http.Request) {
apps/api/internal/httpapi/products_v1_handlers_test.go
+46 -1
View File
@@ -56,6 +56,51 @@ func TestPresentV1ProductFields(t *testing.T) {
}
}
func TestPresentV1ProductDetailOmitsMappedData(t *testing.T) {
ts := time.Date(2026, 8, 1, 10, 15, 0, 0, time.UTC)
row := map[string]any{
"id": "a1b2c3d4-e5f6-7890-abcd-ef1234567890",
"product_id": "SKU-1001",
"name": "Wireless earbuds",
"processed_name": "Acme Wireless Earbuds",
"category": "electronics/audio",
"status": "completed",
"feed_id": "22222222-2222-2222-2222-222222222222",
"description": "raw desc",
"processed_description": "A detailed product description that is long enough for scoring.",
"attributes": map[string]any{"color": "Black"},
"processed_attributes": map[string]any{"color": "Midnight"},
"mapped_data": map[string]any{"image": "https://example.com/earbuds.jpg", "main_image": "https://example.com/main.jpg"},
"has_processed_name": true,
"has_eprel": false,
"eprel": nil,
"created_at": ts,
"updated_at": ts,
}
out := presentV1ProductDetail(row)
for _, key := range []string{"name", "description", "attributes", "main_image", "more_images", "eprel"} {
if _, ok := out[key]; !ok {
t.Fatalf("missing field %q", key)
}
}
for _, leak := range []string{
"mapped_data", "has_processed_name", "has_processed_description",
"has_attributes", "has_processed_attributes", "has_eprel",
"processed_name", "processed_description", "processed_attributes",
} {
if _, ok := out[leak]; ok {
t.Fatalf("unexpected internal field %q", leak)
}
}
if out["description"] != "raw desc" && out["description"] != "A detailed product description that is long enough for scoring." {
t.Fatalf("description=%v", out["description"])
}
attrs, _ := out["attributes"].(map[string]any)
if attrs["color"] != "Midnight" {
t.Fatalf("attributes=%v", out["attributes"])
}
}
func TestV1ProductStatusAll(t *testing.T) {
if got := v1ProductStatus("all"); got != "" {
t.Fatalf("all -> %q want empty", got)
@@ -99,7 +144,7 @@ func TestV1OpenAPIIncludesLegacyProductsEnvelope(t *testing.T) {
"name: sortBy",
"name: feedId",
"totalPages",
"LegacyLimit",
"PublicLimit",
"required: [data, meta]",
} {
if !strings.Contains(body, needle) {
apps/api/internal/httpapi/v1.go
+4 -5
View File
@@ -8,7 +8,6 @@ import (
"net/http"
"strings"
"github.com/descrybe/descrybe-v2/apps/api/internal/processing"
"github.com/go-chi/chi/v5"
"github.com/google/uuid"
)
@@ -66,7 +65,7 @@ func (s *Server) mountV1(r chi.Router) {
// Not an alias of POST/GET /process (flat ProcessingJob).
r.Post("/products/process", s.handleV1StartProcess)
r.Get("/products/process/{id}", s.handleV1GetProcess)
r.Get("/products/{id}", s.handleGetProduct)
r.Get("/products/{id}", s.handleV1GetProduct)
r.Patch("/products/{id}", s.handleUpdateProduct)
// Content calendar — separate from email /api/campaigns (session UI).
@@ -79,7 +78,7 @@ func (s *Server) mountV1(r chi.Router) {
r.Get("/categories", s.handleV1ListCategories)
r.Post("/categories", s.handleV1CreateCategory)
r.Post("/categories/create", s.handleV1CreateCategory) // legacy alias
r.Get("/categories/{id}", s.handleGetCategory)
r.Get("/categories/{id}", s.handleV1GetCategory)
r.Patch("/categories/{id}", s.handleUpdateCategory)
r.Delete("/categories/{id}", s.handleV1DeleteCategory)
@@ -113,7 +112,7 @@ func (s *Server) mountV1(r chi.Router) {
// Dashboard-style jobs (flat JSON / 202). Prefer /products/process for legacy integrations.
r.Post("/process", s.handleStartProcessingJob)
r.Get("/process", s.handleV1ListProcessJobs)
r.Get("/process/{id}", s.handleGetProcessingJob)
r.Get("/process/{id}", s.handleV1GetProcessJob)
r.Post("/process/{id}/cancel", s.handleCancelProcessingJob)
r.Post("/process/{id}/terminate", s.handleCancelProcessingJob)
r.Post("/process/{id}/retry", s.handleRetryProcessingJob)
@@ -133,7 +132,7 @@ func (s *Server) handleV1ListProcessJobs(w http.ResponseWriter, r *http.Request)
Error(w, http.StatusInternalServerError, "list failed")
return
}
JSON(w, http.StatusOK, map[string]any{"jobs": processing.FormatListJobsResponse(items), "limit": limit})
JSON(w, http.StatusOK, map[string]any{"jobs": presentV1JobList(items), "limit": limit})
}
func (s *Server) handleV1OpenAPI(w http.ResponseWriter, r *http.Request) {
apps/api/internal/httpapi/v1_auth_test.go
+1 -1
View File
@@ -118,7 +118,7 @@ func TestV1OpenAPIDocumentsPublicAPIAuth(t *testing.T) {
"Retry-After",
"rate limit exceeded",
"code: unauthorized",
"LegacyAPIError",
"CodedAPIError",
"security: []",
} {
if !strings.Contains(body, want) {
apps/api/internal/httpapi/v1_csrf_tenant_test.go
+3 -3
View File
@@ -154,9 +154,9 @@ func TestV1OpenAPIIncludesProcessAndFeeds(t *testing.T) {
"raw_product_ids",
"items[].ean",
"process_id",
"LegacyStartProcessByEAN",
"StartProcessByEAN",
"StartProcessByRawIDs",
"LegacyProcessCompleted",
"ProcessCompletedExample",
"X-API-Key",
"https://descrybe.io/api/v1",
"BearerAuth",
@@ -185,7 +185,7 @@ func TestV1OpenAPIIncludesProcessAndFeeds(t *testing.T) {
"CSRFHeader",
"code: unauthorized",
"message: Unauthorized",
"legacy envelope",
"coded envelope",
"/process/{id}/retry:",
} {
if !strings.Contains(body, needle) {
apps/api/internal/httpapi/v1_domain_crud_integration_test.go
+21 -12
View File
@@ -123,12 +123,18 @@ func TestV1DomainResourceCRUD(t *testing.T) {
t.Fatalf("get category status=%d body=%s", rec.Code, rec.Body.String())
}
gotCat := decode(t, rec)
if _, hasData := gotCat["data"]; hasData {
t.Fatalf("GET /categories/{uuid} should be flat dashboard JSON, got envelope: %v", gotCat)
}
if gotCat["unique_id"] != catUnique {
gotCatData, _ := gotCat["data"].(map[string]any)
if gotCatData["unique_id"] != catUnique {
t.Fatalf("get category=%v", gotCat)
}
for _, leak := range []string{"prompt", "prompts", "has_prompt", "title_template", "description_template"} {
if _, ok := gotCatData[leak]; ok {
t.Fatalf("v1 GET category leaked %q: %v", leak, gotCatData)
}
if _, ok := gotCat[leak]; ok {
t.Fatalf("v1 GET category envelope leaked %q: %v", leak, gotCat)
}
}
rec = do(http.MethodPatch, "/api/v1/categories/"+catUUID.String(),
`{"name":"Electronics & Audio"}`)
@@ -323,12 +329,15 @@ func TestV1DomainResourceCRUD(t *testing.T) {
t.Fatalf("get product status=%d body=%s", rec.Code, rec.Body.String())
}
gotProd := decode(t, rec)
if _, hasData := gotProd["data"]; hasData {
t.Fatalf("GET /products/{id} should be flat JSON, got envelope: %v", gotProd)
}
if fmt.Sprint(gotProd["product_id"]) != "SKU-1001" {
gotProdData, _ := gotProd["data"].(map[string]any)
if fmt.Sprint(gotProdData["product_id"]) != "SKU-1001" {
t.Fatalf("get product=%v", gotProd)
}
for _, leak := range []string{"mapped_data", "has_processed_name", "has_eprel", "processed_name"} {
if _, ok := gotProdData[leak]; ok {
t.Fatalf("v1 GET product leaked %q: %v", leak, gotProdData)
}
}
rec = do(http.MethodPatch, "/api/v1/products/"+productID.String(),
`{"processed_name":"Acme Wireless Earbuds ANC Midnight","status":"completed"}`)
@@ -427,7 +436,7 @@ func mountV1DomainTestRouter(s *Server) http.Handler {
r.Route("/api/v1", func(r chi.Router) {
r.Get("/products", s.handleV1ListProducts)
r.Get("/products/quality", s.handleV1ListProductQuality)
r.Get("/products/{id}", s.handleGetProduct)
r.Get("/products/{id}", s.handleV1GetProduct)
r.Patch("/products/{id}", s.handleUpdateProduct)
r.Get("/marketing/calendar", s.handleGetMarketingCalendar)
@@ -438,7 +447,7 @@ func mountV1DomainTestRouter(s *Server) http.Handler {
r.Get("/categories", s.handleV1ListCategories)
r.Post("/categories", s.handleV1CreateCategory)
r.Post("/categories/create", s.handleV1CreateCategory)
r.Get("/categories/{id}", s.handleGetCategory)
r.Get("/categories/{id}", s.handleV1GetCategory)
r.Patch("/categories/{id}", s.handleUpdateCategory)
r.Delete("/categories/{id}", s.handleV1DeleteCategory)
@@ -480,9 +489,9 @@ func TestV1OpenAPIDocumentsDomainCRUDSurface(t *testing.T) {
"/marketing/calendar:",
"/products:",
"/feeds/{id}/sync-process-sample:",
"Flat category JSON",
"no prompts or formulas",
"flat PresentFeed",
"flat ProcessedProduct",
"without mapped_data",
"CategoryDetail",
"FeedDeleted",
"FeedMappings",
apps/api/internal/httpapi/v1_feeds_test.go
+1 -1
View File
@@ -9,7 +9,7 @@ func TestV1OpenAPIFeedsLegacyContract(t *testing.T) {
t.Parallel()
body := string(v1OpenAPIYAML)
for _, needle := range []string{
"Legacy public contract — { data: Feed[], meta: { page, limit, total } }",
"Paged list — { data: Feed[], meta: { page, limit, total } }",
"required: [name, item_path]",
"jobId:",
"FeedSyncResponse",
apps/api/internal/httpapi/v1_openapi.go
+450 -447
View File
File diff suppressed because it is too large Load Diff
apps/api/internal/httpapi/v1_openapi_test.go
+160 -6
View File
@@ -415,25 +415,25 @@ func TestV1OpenAPIYAMLProcessDualIDsAndGatesDocumentsContracts(t *testing.T) {
}
}
// TestV1OpenAPIYAMLLegacyProcessItemEprelShape locks LegacyProcessItem.eprel
// TestV1OpenAPIYAMLProcessItemEprelShape locks ProcessItem.eprel
// to the live nested object from eprel.MergeInto / extractEPRELFromAttrs.
func TestV1OpenAPIYAMLLegacyProcessItemEprelShape(t *testing.T) {
func TestV1OpenAPIYAMLProcessItemEprelShape(t *testing.T) {
t.Parallel()
root := mustParseOpenAPIRoot(t, v1OpenAPIYAML)
comps, _ := root["components"].(map[string]any)
schemas, _ := comps["schemas"].(map[string]any)
item, _ := schemas["LegacyProcessItem"].(map[string]any)
item, _ := schemas["ProcessItem"].(map[string]any)
itemProps, _ := item["properties"].(map[string]any)
eprel, _ := itemProps["eprel"].(map[string]any)
eprelProps, _ := eprel["properties"].(map[string]any)
if eprelProps == nil {
t.Fatal("LegacyProcessItem.eprel.properties missing")
t.Fatal("ProcessItem.eprel.properties missing")
}
want := []string{"id", "label", "pdf", "energy_class", "energy_scale"}
for _, key := range want {
if _, ok := eprelProps[key]; !ok {
t.Fatalf("LegacyProcessItem.eprel missing property %q (live shape)", key)
t.Fatalf("ProcessItem.eprel missing property %q (live shape)", key)
}
}
if len(eprelProps) != len(want) {
@@ -441,7 +441,161 @@ func TestV1OpenAPIYAMLLegacyProcessItemEprelShape(t *testing.T) {
for k := range eprelProps {
keys = append(keys, k)
}
t.Fatalf("LegacyProcessItem.eprel properties = %v, want exactly %v", keys, want)
t.Fatalf("ProcessItem.eprel properties = %v, want exactly %v", keys, want)
}
}
func TestV1OpenAPIYAMLNoLegacyWording(t *testing.T) {
t.Parallel()
assertOpenAPIYAMLOmitsToken(t, "legacy")
}
func TestV1OpenAPIYAMLNoA1Wording(t *testing.T) {
t.Parallel()
// Case-sensitive: lowercase "a1" appears in UUIDs and must stay allowed.
assertOpenAPIYAMLOmitsToken(t, "A1")
}
func TestV1OpenAPIYAMLOmitsInternalPromptWording(t *testing.T) {
t.Parallel()
for _, token := range []string{
"role-section prompts",
"plain description, meta",
"after AI/manual edit",
} {
assertOpenAPIYAMLOmitsToken(t, token)
}
}
func TestV1OpenAPIYAMLProcessItemCompletedExamplesOmitLeakyShape(t *testing.T) {
t.Parallel()
root := mustParseOpenAPIRoot(t, v1OpenAPIYAML)
leaky := []string{"id", "title", "meta_title", "meta_description"}
assertProcessItems := func(where string, items []any) {
t.Helper()
if len(items) == 0 {
t.Fatalf("%s: expected completed ProcessItem example", where)
}
for i, raw := range items {
item, _ := raw.(map[string]any)
if item == nil {
t.Fatalf("%s[%d]: not an object", where, i)
}
if _, ok := item["ean"]; !ok {
t.Fatalf("%s[%d]: missing ean", where, i)
}
for _, k := range leaky {
if _, ok := item[k]; ok {
t.Fatalf("%s[%d] must omit %q (do not teach leaky ProcessItem shape)", where, i, k)
}
}
}
}
comps, _ := root["components"].(map[string]any)
examples, _ := comps["examples"].(map[string]any)
completed, _ := examples["ProcessCompletedExample"].(map[string]any)
val, _ := completed["value"].(map[string]any)
data, _ := val["data"].(map[string]any)
items, _ := data["items"].([]any)
assertProcessItems("ProcessCompletedExample", items)
paths, _ := root["paths"].(map[string]any)
productsProcess := openAPIJSONExamples(t, paths, "/products/process/{id}", "get", "200")
prodCompleted, _ := productsProcess["completed"].(map[string]any)
prodVal, _ := prodCompleted["value"].(map[string]any)
prodData, _ := prodVal["data"].(map[string]any)
prodItems, _ := prodData["items"].([]any)
assertProcessItems("GET /products/process/{id} completed", prodItems)
legacyProcess := openAPIJSONExamples(t, paths, "/process/{id}", "get", "200")
legacyCompleted, _ := legacyProcess["completed"].(map[string]any)
legacyVal, _ := legacyCompleted["value"].(map[string]any)
legacyItems, _ := legacyVal["items"].([]any)
assertProcessItems("GET /process/{id} completed", legacyItems)
}
func openAPIJSONExamples(t *testing.T, paths map[string]any, path, method, status string) map[string]any {
t.Helper()
p, _ := paths[path].(map[string]any)
op, _ := p[method].(map[string]any)
resps, _ := op["responses"].(map[string]any)
resp, _ := resps[status].(map[string]any)
content, _ := resp["content"].(map[string]any)
appJSON, _ := content["application/json"].(map[string]any)
examples, _ := appJSON["examples"].(map[string]any)
if examples == nil {
t.Fatalf("%s %s %s: missing application/json examples", method, path, status)
}
return examples
}
func assertOpenAPIYAMLOmitsToken(t *testing.T, token string) {
t.Helper()
doc := string(v1OpenAPIYAML)
if token == "legacy" {
doc = strings.ToLower(doc)
}
if i := strings.Index(doc, token); i >= 0 {
start := i - 40
if start < 0 {
start = 0
}
end := i + 40
if end > len(doc) {
end = len(doc)
}
t.Fatalf("public OpenAPI YAML must not mention %q (near %q)", token, doc[start:end])
}
}
func TestV1OpenAPIYAMLEmptySecurityOnlyOnPublicProbes(t *testing.T) {
t.Parallel()
root := mustParseOpenAPIRoot(t, v1OpenAPIYAML)
sec, _ := root["security"].([]any)
if len(sec) == 0 {
t.Fatal("document-level security must require API key schemes")
}
var hasBearer, hasAPIKey bool
for _, item := range sec {
m, _ := item.(map[string]any)
if _, ok := m["BearerAuth"]; ok {
hasBearer = true
}
if _, ok := m["ApiKeyAuth"]; ok {
hasAPIKey = true
}
}
if !hasBearer || !hasAPIKey {
t.Fatal("document-level security must include BearerAuth and ApiKeyAuth")
}
paths, _ := root["paths"].(map[string]any)
methods := []string{"get", "post", "put", "patch", "delete"}
var bad []string
for p, raw := range paths {
item, _ := raw.(map[string]any)
for _, m := range methods {
op, _ := item[m].(map[string]any)
if op == nil {
continue
}
rawSec, ok := op["security"]
if !ok {
continue
}
arr, _ := rawSec.([]any)
if len(arr) != 0 {
continue
}
if p != "/health" && p != "/openapi.yaml" {
bad = append(bad, m+" "+p)
}
}
}
if len(bad) > 0 {
t.Fatalf("empty security (no API key) on non-probe operations: %v", bad)
}
}
apps/api/internal/httpapi/v1_process_handlers.go
+97 -2
View File
@@ -227,6 +227,7 @@ func (s *Server) handleV1GetProcess(w http.ResponseWriter, r *http.Request) {
v1Err(w, http.StatusInternalServerError, "internal_server_error", "Internal server error")
return
}
items = processing.ApplyV1SEOMetaPolicy(cid, items)
data := map[string]any{
"status": status,
"process_id": job.ID.String(),
@@ -245,8 +246,9 @@ func (s *Server) handleV1GetProcess(w http.ResponseWriter, r *http.Request) {
v1OK(w, http.StatusOK, data, nil)
case "FAILED":
errMsg := "Processing failed"
if job.Error != nil && *job.Error != "" {
errMsg = *job.Error
if job.Error != nil && strings.TrimSpace(*job.Error) != "" {
log.Printf("httpapi: v1 process job %s failed: %s", job.ID, redactForLog(*job.Error))
errMsg = publicV1JobError(*job.Error)
}
v1OK(w, http.StatusOK, map[string]any{
"status": status,
@@ -269,6 +271,99 @@ func (s *Server) handleV1GetProcess(w http.ResponseWriter, r *http.Request) {
}
}
func publicV1JobError(raw string) string {
raw = strings.TrimSpace(raw)
if raw == "" {
return "Processing failed"
}
out := processing.PublicV1Error(raw)
if strings.TrimSpace(out) == "" {
return "Processing failed"
}
return out
}
func presentV1Job(job processing.Job) map[string]any {
out := map[string]any{
"id": job.ID,
"status": job.Status,
"total_products": job.TotalProducts,
"processed_products": job.ProcessedProducts,
"processing_type": processing.ProcessingTypeForAPIResponse(job.ProcessingType),
"created_at": formatV1Timestamp(job.CreatedAt),
}
if job.StartedAt != nil {
out["started_at"] = formatV1Timestamp(*job.StartedAt)
}
if job.CompletedAt != nil {
out["completed_at"] = formatV1Timestamp(*job.CompletedAt)
}
if job.Error != nil && strings.TrimSpace(*job.Error) != "" {
out["error"] = publicV1JobError(*job.Error)
}
return out
}
func presentV1JobList(jobs []processing.Job) []any {
if len(jobs) == 0 {
return []any{}
}
formatted := processing.FormatListJobsResponse(jobs)
out := make([]any, len(formatted))
for i, row := range formatted {
base := presentV1Job(jobs[i])
raw, err := json.Marshal(row)
if err != nil {
out[i] = base
continue
}
var extra map[string]any
if err := json.Unmarshal(raw, &extra); err != nil {
out[i] = base
continue
}
for _, k := range []string{"sibling_job_ids", "job_count", "total_products_queued"} {
if v, ok := extra[k]; ok {
base[k] = v
}
}
out[i] = base
}
return out
}
// handleV1GetProcessJob serves GET /api/v1/process/{id} with a public job DTO
// (no current_step, step notes, or company_id). Dashboard GET stays on handleGetProcessingJob.
func (s *Server) handleV1GetProcessJob(w http.ResponseWriter, r *http.Request) {
cid, ok := CompanyIDFromContext(r.Context())
if !ok || cid == uuid.Nil {
Error(w, http.StatusUnauthorized, "unauthorized")
return
}
id, err := uuid.Parse(chi.URLParam(r, "id"))
if err != nil {
Error(w, http.StatusBadRequest, "invalid id")
return
}
job, err := s.getV1ProcessJob(r.Context(), cid, id)
if err != nil {
Error(w, http.StatusNotFound, "not found")
return
}
out := presentV1Job(job)
if processing.JobStatusIncludesProducts(job.Status) {
items, loadErr := s.loadV1ProcessJobItems(r.Context(), cid, id, job.ProcessingType)
if loadErr != nil {
Error(w, http.StatusInternalServerError, "load failed")
return
}
items = processing.ApplyV1SEOMetaPolicy(cid, items)
out["items"] = items
out["total_items"] = len(items)
}
JSON(w, http.StatusOK, out)
}
func (s *Server) ensureRawV1Items(ctx context.Context, companyID uuid.UUID, items []catalog.V1ProcessItem) ([]uuid.UUID, []catalog.EnsureRawResult, []string, error) {
if s != nil && s.testEnsureRawV1Items != nil {
return s.testEnsureRawV1Items(ctx, companyID, items)
apps/api/internal/httpapi/v1_process_handlers_test.go
+212 -3
View File
@@ -337,7 +337,6 @@ func TestHandleV1GetProcessCompletedIncludesProcessedItems(t *testing.T) {
t.Parallel()
cid := uuid.MustParse("11111111-1111-1111-1111-111111111111")
jobID := uuid.MustParse("aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa")
productID := "bbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb"
s := &Server{
testGetJob: func(_ context.Context, companyID, id uuid.UUID) (processing.Job, error) {
if companyID != cid || id != jobID {
@@ -357,7 +356,7 @@ func TestHandleV1GetProcessCompletedIncludesProcessedItems(t *testing.T) {
}
return []processing.V1ProcessJobItem{{
"ean": "0123456789012",
"id": productID,
"id": "bbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb",
"status": "processed",
"title": "Acme Widget",
}}, nil
@@ -393,9 +392,12 @@ func TestHandleV1GetProcessCompletedIncludesProcessedItems(t *testing.T) {
if !ok {
t.Fatalf("item=%T", items[0])
}
if item["status"] != "processed" || item["id"] != productID || item["ean"] != "0123456789012" {
if item["status"] != "processed" || item["ean"] != "0123456789012" {
t.Fatalf("item=%v", item)
}
if _, ok := item["id"]; ok {
t.Fatalf("id must be omitted from public process items: %v", item)
}
if _, ok := body.Data["processed_at"]; !ok {
t.Fatalf("missing processed_at: %v", body.Data)
}
@@ -447,6 +449,90 @@ func TestHandleGetProcessingJobCompletedIncludesItems(t *testing.T) {
}
}
func TestHandleV1GetProcessJobOmitsAIEnhance(t *testing.T) {
t.Parallel()
cid := uuid.MustParse("11111111-1111-1111-1111-111111111111")
jobID := uuid.MustParse("aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa")
note := "GPT: rewrite title with --- Title --- markers"
errMsg := "openai: invalid api-key sk-live-secret"
s := &Server{
testGetJob: func(_ context.Context, companyID, id uuid.UUID) (processing.Job, error) {
return processing.Job{
ID: jobID,
CompanyID: companyID,
Status: "processing",
ProcessingType: "full",
CurrentStep: "ai_enhance",
StepProgress: []processing.StepProgress{{
Step: "ai_enhance", Status: "running", Note: note,
}},
Error: &errMsg,
TotalProducts: 2,
ProcessedProducts: 1,
}, nil
},
}
ctx := context.WithValue(context.Background(), ctxCompanyID, cid)
rec := httptest.NewRecorder()
req := httptest.NewRequest(http.MethodGet, "/api/v1/process/"+jobID.String(), nil)
rctx := chi.NewRouteContext()
rctx.URLParams.Add("id", jobID.String())
req = req.WithContext(context.WithValue(ctx, chi.RouteCtxKey, rctx))
s.handleV1GetProcessJob(rec, req)
if rec.Code != http.StatusOK {
t.Fatalf("status=%d body=%s", rec.Code, rec.Body.String())
}
raw := rec.Body.String()
if strings.Contains(raw, "ai_enhance") {
t.Fatalf("public process poll leaked ai_enhance: %s", raw)
}
if strings.Contains(raw, "GPT:") || strings.Contains(raw, "--- Title ---") {
t.Fatalf("public process poll leaked step notes: %s", raw)
}
if strings.Contains(raw, "current_step") || strings.Contains(raw, "step_progress") || strings.Contains(raw, "company_id") {
t.Fatalf("public process poll leaked internals: %s", raw)
}
var body map[string]any
if err := json.Unmarshal(rec.Body.Bytes(), &body); err != nil {
t.Fatal(err)
}
if body["status"] != "processing" {
t.Fatalf("status=%v", body["status"])
}
if body["total_products"].(float64) != 2 || body["processed_products"].(float64) != 1 {
t.Fatalf("counts=%v", body)
}
}
func TestHandleGetProcessingJobKeepsCurrentStep(t *testing.T) {
t.Parallel()
cid := uuid.MustParse("11111111-1111-1111-1111-111111111111")
jobID := uuid.MustParse("aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa")
s := &Server{
testGetJob: func(_ context.Context, companyID, id uuid.UUID) (processing.Job, error) {
return processing.Job{
ID: jobID,
CompanyID: companyID,
Status: "processing",
CurrentStep: "ai_enhance",
}, nil
},
}
ctx := context.WithValue(context.Background(), ctxCompanyID, cid)
rec := httptest.NewRecorder()
req := httptest.NewRequest(http.MethodGet, "/api/processing/jobs/"+jobID.String(), nil)
rctx := chi.NewRouteContext()
rctx.URLParams.Add("id", jobID.String())
req = req.WithContext(context.WithValue(ctx, chi.RouteCtxKey, rctx))
s.handleGetProcessingJob(rec, req)
if rec.Code != http.StatusOK {
t.Fatalf("status=%d body=%s", rec.Code, rec.Body.String())
}
if !strings.Contains(rec.Body.String(), "ai_enhance") {
t.Fatalf("dashboard GET should keep current_step: %s", rec.Body.String())
}
}
func TestHandleV1ListProcessJobsUnauthorizedWithoutCompany(t *testing.T) {
t.Parallel()
s := &Server{}
@@ -500,3 +586,126 @@ func TestAssertV1ProcessGatesNilBilling(t *testing.T) {
t.Fatalf("nil billing must no-op: %v", err)
}
}
func processJobWithSEOMetaItems(cid, jobID uuid.UUID) *Server {
return &Server{
testGetJob: func(_ context.Context, companyID, id uuid.UUID) (processing.Job, error) {
return processing.Job{
ID: jobID,
CompanyID: companyID,
Status: "completed",
ProcessingType: "full",
TotalProducts: 1,
}, nil
},
testLoadV1ProcessJobItems: func(context.Context, uuid.UUID, uuid.UUID, string) ([]processing.V1ProcessJobItem, error) {
return []processing.V1ProcessJobItem{{
"ean": "0123456789012",
"status": "processed",
"name": "Acme Widget",
"meta_title": "Acme | Widget",
"meta_description": "A widget for tests",
}}, nil
},
}
}
func getProcessJSONItem(t *testing.T, rec *httptest.ResponseRecorder, wrapped bool) map[string]any {
t.Helper()
if rec.Code != http.StatusOK {
t.Fatalf("status=%d body=%s", rec.Code, rec.Body.String())
}
var raw map[string]any
if err := json.Unmarshal(rec.Body.Bytes(), &raw); err != nil {
t.Fatal(err)
}
data := raw
if wrapped {
var ok bool
data, ok = raw["data"].(map[string]any)
if !ok {
t.Fatalf("data=%T body=%s", raw["data"], rec.Body.String())
}
}
items, ok := data["items"].([]any)
if !ok || len(items) != 1 {
t.Fatalf("items=%v", data["items"])
}
item, ok := items[0].(map[string]any)
if !ok {
t.Fatalf("item=%T", items[0])
}
return item
}
func TestA1ProcessEndpointsOmitSEOMeta(t *testing.T) {
t.Parallel()
cid := uuid.MustParse("604f23a8-b66e-4b21-8b45-0d72b68f4790")
jobID := uuid.MustParse("aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa")
s := processJobWithSEOMetaItems(cid, jobID)
ctx := context.WithValue(context.Background(), ctxCompanyID, cid)
legacyRec := httptest.NewRecorder()
legacyReq := httptest.NewRequest(http.MethodGet, "/api/v1/products/process/"+jobID.String(), nil)
rctx := chi.NewRouteContext()
rctx.URLParams.Add("id", jobID.String())
legacyReq = legacyReq.WithContext(context.WithValue(ctx, chi.RouteCtxKey, rctx))
s.handleV1GetProcess(legacyRec, legacyReq)
legacyItem := getProcessJSONItem(t, legacyRec, true)
if _, ok := legacyItem["meta_title"]; ok {
t.Fatalf("GET /products/process/{id} A1 meta_title=%v", legacyItem["meta_title"])
}
if _, ok := legacyItem["meta_description"]; ok {
t.Fatalf("GET /products/process/{id} A1 meta_description=%v", legacyItem["meta_description"])
}
jobRec := httptest.NewRecorder()
jobReq := httptest.NewRequest(http.MethodGet, "/api/v1/process/"+jobID.String(), nil)
jrctx := chi.NewRouteContext()
jrctx.URLParams.Add("id", jobID.String())
jobReq = jobReq.WithContext(context.WithValue(ctx, chi.RouteCtxKey, jrctx))
s.handleGetProcessingJob(jobRec, jobReq)
jobItem := getProcessJSONItem(t, jobRec, false)
if _, ok := jobItem["meta_title"]; ok {
t.Fatalf("GET /process/{id} A1 meta_title=%v", jobItem["meta_title"])
}
if _, ok := jobItem["meta_description"]; ok {
t.Fatalf("GET /process/{id} A1 meta_description=%v", jobItem["meta_description"])
}
}
func TestNonA1ProcessEndpointsKeepSEOMeta(t *testing.T) {
t.Parallel()
cid := uuid.MustParse("11111111-1111-1111-1111-111111111111")
jobID := uuid.MustParse("aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa")
s := processJobWithSEOMetaItems(cid, jobID)
ctx := context.WithValue(context.Background(), ctxCompanyID, cid)
legacyRec := httptest.NewRecorder()
legacyReq := httptest.NewRequest(http.MethodGet, "/api/v1/products/process/"+jobID.String(), nil)
rctx := chi.NewRouteContext()
rctx.URLParams.Add("id", jobID.String())
legacyReq = legacyReq.WithContext(context.WithValue(ctx, chi.RouteCtxKey, rctx))
s.handleV1GetProcess(legacyRec, legacyReq)
legacyItem := getProcessJSONItem(t, legacyRec, true)
if legacyItem["meta_title"] != "Acme | Widget" {
t.Fatalf("non-A1 GET /products/process/{id} meta_title=%v", legacyItem["meta_title"])
}
if legacyItem["meta_description"] != "A widget for tests" {
t.Fatalf("non-A1 GET /products/process/{id} meta_description=%v", legacyItem["meta_description"])
}
jobRec := httptest.NewRecorder()
jobReq := httptest.NewRequest(http.MethodGet, "/api/v1/process/"+jobID.String(), nil)
jrctx := chi.NewRouteContext()
jrctx.URLParams.Add("id", jobID.String())
jobReq = jobReq.WithContext(context.WithValue(ctx, chi.RouteCtxKey, jrctx))
s.handleGetProcessingJob(jobRec, jobReq)
jobItem := getProcessJSONItem(t, jobRec, false)
if jobItem["meta_title"] != "Acme | Widget" {
t.Fatalf("non-A1 GET /process/{id} meta_title=%v", jobItem["meta_title"])
}
if jobItem["meta_description"] != "A widget for tests" {
t.Fatalf("non-A1 GET /process/{id} meta_description=%v", jobItem["meta_description"])
}
}
apps/api/internal/mail/mailer.go
+14 -11
View File
@@ -2,6 +2,7 @@ package mail
import (
"fmt"
"html"
"log"
"net"
"net/smtp"
@@ -109,31 +110,33 @@ func hasHeaderBreak(v string) bool {
func InviteMessage(webOrigin, email, token, companyName string) Message {
link := AcceptInviteURL(webOrigin, token)
text := fmt.Sprintf("You have been invited to %s on Descrybe.\n\nAccept: %s\n", companyName, link)
html := fmt.Sprintf(
htmlBody := fmt.Sprintf(
`<p>You have been invited to <strong>%s</strong> on Descrybe.</p><p><a href="%s">Accept invite</a></p>`,
companyName, link,
html.EscapeString(companyName), html.EscapeString(link),
)
return Message{To: email, Subject: "You are invited to Descrybe", Text: text, HTML: html}
return Message{To: email, Subject: "You are invited to Descrybe", Text: text, HTML: htmlBody}
}
// AcceptInviteURL builds the durable invite / set-password accept link (hashed invite tokens).
// Token is placed in the URL fragment so it is not sent on the page GET (Referer/access logs).
func AcceptInviteURL(webOrigin, token string) string {
return strings.TrimRight(webOrigin, "/") + "/accept-invite?token=" + token
return strings.TrimRight(webOrigin, "/") + "/accept-invite#token=" + token
}
// SetPasswordURL builds the HMAC set-password accept-invite link.
// Token is placed in the URL fragment so it is not sent on the page GET (Referer/access logs).
func SetPasswordURL(webOrigin, token string) string {
return strings.TrimRight(webOrigin, "/") + "/accept-invite?token=" + token + "&mode=set-password"
return strings.TrimRight(webOrigin, "/") + "/accept-invite#token=" + token + "&mode=set-password"
}
func SetPasswordMessage(webOrigin, email, token string) Message {
link := SetPasswordURL(webOrigin, token)
text := fmt.Sprintf("Set your Descrybe password:\n\n%s\n\nThis link expires in 72 hours.\n", link)
html := fmt.Sprintf(
htmlBody := fmt.Sprintf(
`<p>Set your Descrybe password:</p><p><a href="%s">Set password</a></p><p>This link expires in 72 hours.</p>`,
link,
html.EscapeString(link),
)
return Message{To: email, Subject: "Set your Descrybe password", Text: text, HTML: html}
return Message{To: email, Subject: "Set your Descrybe password", Text: text, HTML: htmlBody}
}
// MigratedSetPasswordMessage uses migrator invite tokens (accept-invite flow).
@@ -143,11 +146,11 @@ func MigratedSetPasswordMessage(webOrigin, email, token string) Message {
"Your Descrybe account was migrated. Set your password here:\n\n%s\n\nIf you did not expect this email, ignore it.\n",
link,
)
html := fmt.Sprintf(
htmlBody := fmt.Sprintf(
`<p>Your Descrybe account was migrated.</p><p><a href="%s">Set your password</a></p><p>If you did not expect this email, ignore it.</p>`,
link,
html.EscapeString(link),
)
return Message{To: email, Subject: "Set your Descrybe password", Text: text, HTML: html}
return Message{To: email, Subject: "Set your Descrybe password", Text: text, HTML: htmlBody}
}
// ResetPasswordURL builds the self-serve forgot-password reset link.
apps/api/internal/mail/mailer_test.go
+32 -1
View File
@@ -118,9 +118,32 @@ func TestNewDynamicResolvesPerCall(t *testing.T) {
}
}
func TestInviteAndSetPasswordMessagesEscapeHTML(t *testing.T) {
name := `Acme <script>alert("x")</script> & Co`
msg := InviteMessage("https://app.example", "a@b.c", "tok", name)
if strings.Contains(msg.HTML, "<script>") {
t.Fatalf("unescaped script in HTML: %s", msg.HTML)
}
if !strings.Contains(msg.HTML, "Acme &lt;script&gt;") || !strings.Contains(msg.HTML, "&amp; Co") {
t.Fatalf("expected escaped company name, got %s", msg.HTML)
}
if !strings.Contains(msg.Text, name) {
t.Fatalf("text should keep company name: %s", msg.Text)
}
set := SetPasswordMessage("https://app.example", "a@b.c", `tok"onclick="alert(1)`)
if strings.Contains(set.HTML, `"onclick=`) {
t.Fatalf("unescaped token in set-password HTML: %s", set.HTML)
}
mig := MigratedSetPasswordMessage("https://app.example", "a@b.c", `tok"><img src=x>`)
if strings.Contains(mig.HTML, "<img") {
t.Fatalf("unescaped token in migrated HTML: %s", mig.HTML)
}
}
func TestSetPasswordURL(t *testing.T) {
got := SetPasswordURL("http://localhost:5174/", "tok123")
want := "http://localhost:5174/accept-invite?token=tok123&mode=set-password"
want := "http://localhost:5174/accept-invite#token=tok123&mode=set-password"
if got != want {
t.Fatalf("SetPasswordURL=%q want %q", got, want)
}
@@ -129,3 +152,11 @@ func TestSetPasswordURL(t *testing.T) {
t.Fatalf("SetPasswordMessage text missing link: %q", msg.Text)
}
}
func TestAcceptInviteURLUsesFragment(t *testing.T) {
got := AcceptInviteURL("http://localhost:5174/", "tok123")
want := "http://localhost:5174/accept-invite#token=tok123"
if got != want {
t.Fatalf("AcceptInviteURL=%q want %q", got, want)
}
}
apps/api/internal/metrics/metrics.go
+17 -2
View File
@@ -185,11 +185,13 @@ func Handler() http.Handler {
}
// Gate restricts Prometheus scrapes in production: allow when metricsPublic is true
// (METRICS_PUBLIC=1) or the peer is loopback. Non-production always allows (local scrapes).
// (METRICS_PUBLIC=1) or the immediate TCP peer is loopback. Non-production always
// allows (local scrapes). Loopback is ignored when client-IP proxy headers are
// present so TrustedRealIP cannot mint 127.0.0.1 from X-Forwarded-For.
func Gate(isProduction, metricsPublic bool) func(http.Handler) http.Handler {
return func(next http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if !isProduction || metricsPublic || isLoopbackRemoteAddr(r.RemoteAddr) {
if !isProduction || metricsPublic || allowLoopbackMetricsPeer(r) {
next.ServeHTTP(w, r)
return
}
@@ -198,6 +200,19 @@ func Gate(isProduction, metricsPublic bool) func(http.Handler) http.Handler {
}
}
func allowLoopbackMetricsPeer(r *http.Request) bool {
if forwardedClientIPPresent(r) {
return false
}
return isLoopbackRemoteAddr(r.RemoteAddr)
}
func forwardedClientIPPresent(r *http.Request) bool {
return strings.TrimSpace(r.Header.Get("X-Forwarded-For")) != "" ||
strings.TrimSpace(r.Header.Get("X-Real-IP")) != "" ||
strings.TrimSpace(r.Header.Get("True-Client-IP")) != ""
}
func isLoopbackRemoteAddr(remoteAddr string) bool {
host := strings.TrimSpace(remoteAddr)
if host == "" {
apps/api/internal/metrics/metrics_test.go
+14
View File
@@ -134,3 +134,17 @@ func TestGateAllowsPublicFlagInProduction(t *testing.T) {
t.Fatalf("METRICS_PUBLIC status=%d", rec.Code)
}
}
func TestGateBlocksSpoofedLoopbackXFFInProduction(t *testing.T) {
t.Cleanup(Reset)
Reset()
h := Gate(true, false)(Handler())
req := httptest.NewRequest(http.MethodGet, "/metrics", nil)
req.RemoteAddr = "127.0.0.1:54321"
req.Header.Set("X-Forwarded-For", "127.0.0.1")
rec := httptest.NewRecorder()
h.ServeHTTP(rec, req)
if rec.Code != http.StatusNotFound {
t.Fatalf("spoofed XFF loopback status=%d want 404", rec.Code)
}
}
apps/api/internal/processing/sanitize.go
+94 -7
View File
@@ -67,11 +67,17 @@ func truncateRunes(s string, max int) string {
return s
}
const (
publicErrProcessingFailed = "processing_failed"
publicErrProviderUnavailable = "provider_unavailable"
)
// TruncateError returns a safe, short error string for DB storage / API clients.
// Secret-like substrings and logredact matches become an opaque message so
// job step_progress notes and v1 item errors cannot leak keys/JWTs/DSNs/emails.
// Common provider transport failures are rewritten to short user-facing text
// (no dial URLs / Go net strings) while preserving unrelated provider messages.
// (no dial URLs / Go net strings). Remaining provider internals map to stable
// public codes (processing_failed / provider_unavailable).
func TruncateError(err error) string {
if err == nil {
return ""
@@ -94,20 +100,101 @@ func TruncateError(err error) string {
return friendly
}
// Drop retry-exhaustion wrapper when the inner message is already clear.
cleaned := redacted
cleaned := stripOpenAIRetryPrefix(redacted)
if friendly := classifyProviderError(cleaned); friendly != "" {
return friendly
}
if code := mapPublicErrorCode(cleaned); code != "" {
return code
}
return truncateRunes(cleaned, 500)
}
// PublicV1Error maps a stored job/item error to a stable public v1 string.
// Dashboard notes / step_progress may still carry operator detail; public JSON must not.
func PublicV1Error(msg string) string {
msg = strings.TrimSpace(msg)
if msg == "" {
return ""
}
switch msg {
case publicErrProcessingFailed, publicErrProviderUnavailable,
"Product data not available", "Processing failed",
"provider error (details redacted)":
return msg
}
if strings.HasPrefix(msg, "processing.job.error.") {
return msg
}
if strings.Contains(strings.ToLower(msg), "insufficient credits") {
return msg
}
if friendly := classifyProviderError(msg); friendly != "" {
return friendly
}
cleaned := stripOpenAIRetryPrefix(msg)
if friendly := classifyProviderError(cleaned); friendly != "" {
return friendly
}
if code := mapPublicErrorCode(cleaned); code != "" {
return code
}
if looksLikeInternalProviderError(cleaned) {
return publicErrProcessingFailed
}
return truncateRunes(cleaned, 500)
}
func stripOpenAIRetryPrefix(msg string) string {
cleaned := msg
for _, prefix := range []string{
"openai retries exhausted: ",
"openai embedding retries exhausted: ",
} {
if strings.HasPrefix(strings.ToLower(cleaned), prefix) {
cleaned = strings.TrimSpace(cleaned[len(prefix):])
break
return strings.TrimSpace(cleaned[len(prefix):])
}
}
if friendly := classifyProviderError(cleaned); friendly != "" {
return friendly
return cleaned
}
func mapPublicErrorCode(msg string) string {
lower := strings.ToLower(strings.TrimSpace(msg))
if lower == "" {
return ""
}
return truncateRunes(cleaned, 500)
switch {
case strings.Contains(lower, "max_tokens"),
strings.Contains(lower, "length-capped"):
return publicErrProcessingFailed
case strings.Contains(lower, "openai unset"),
strings.Contains(lower, "platform openai"):
return publicErrProviderUnavailable
case strings.Contains(lower, "ai_enhance: skipped"):
return publicErrProcessingFailed
case strings.Contains(lower, "unavailable"),
strings.Contains(lower, "http 503"),
strings.Contains(lower, "overloaded"):
return publicErrProviderUnavailable
case strings.Contains(lower, "openai"),
strings.Contains(lower, "gpt-"),
strings.Contains(lower, " for model"):
return publicErrProcessingFailed
}
return ""
}
func looksLikeInternalProviderError(msg string) bool {
lower := strings.ToLower(msg)
for _, n := range []string{
"openai", "max_tokens", "length-capped", "gpt-", "ai_enhance",
"finish_reason", "chat/completions",
} {
if strings.Contains(lower, n) {
return true
}
}
return false
}
// classifyProviderError maps common OpenAI-compatible transport/auth failures
apps/api/internal/processing/sanitize_test.go
+61 -4
View File
@@ -49,6 +49,65 @@ func TestTruncateError_redactsSecrets(t *testing.T) {
}
}
func TestTruncateError_mapsInternalProviderLeaks(t *testing.T) {
cases := []struct {
in string
want string
}{
{
in: `openai length-capped at max_tokens=16000 for model "gpt-5.4" (prefer a faster non-reasoning product-enhance model): unexpected EOF`,
want: "processing_failed",
},
{
in: "ai_enhance: skipped (platform OpenAI unset; configure admin settings or company BYOK)",
want: "provider_unavailable",
},
{
in: "openai retries exhausted: green-chat unavailable",
want: "provider_unavailable",
},
{
in: "upstream 503: model overloaded",
want: "provider_unavailable",
},
}
for _, tc := range cases {
got := TruncateError(errString(tc.in))
if got != tc.want {
t.Fatalf("TruncateError(%q)=%q want %q", tc.in, got, tc.want)
}
if strings.Contains(strings.ToLower(got), "openai") ||
strings.Contains(strings.ToLower(got), "max_tokens") ||
strings.Contains(got, "gpt-") {
t.Fatalf("internal leak in TruncateError: %q → %q", tc.in, got)
}
}
}
func TestPublicV1Error_stripsProviderInternals(t *testing.T) {
cases := []struct {
in string
want string
}{
{in: `openai length-capped at max_tokens=8000 for model "gpt-4o"`, want: "processing_failed"},
{in: "ai_enhance: skipped (platform OpenAI unset; configure admin settings or company BYOK)", want: "provider_unavailable"},
{in: "processing_failed", want: "processing_failed"},
{in: "Product data not available", want: "Product data not available"},
{in: "processing.job.error.all_failed|count=3", want: "processing.job.error.all_failed|count=3"},
{in: "insufficient credits", want: "insufficient credits"},
}
for _, tc := range cases {
got := PublicV1Error(tc.in)
if got != tc.want {
t.Fatalf("PublicV1Error(%q)=%q want %q", tc.in, got, tc.want)
}
lower := strings.ToLower(got)
if strings.Contains(lower, "openai") || strings.Contains(lower, "max_tokens") {
t.Fatalf("public v1 leak: %q → %q", tc.in, got)
}
}
}
func TestTruncateError_classifiesProviderFailures(t *testing.T) {
cases := []struct {
in string
@@ -68,10 +127,8 @@ in: "unauthorized", want: "AI provider rejected the API key"},
in: "openai http 401", want: "AI provider rejected the API key"},
{in: "rate limited or server error", want: "AI provider temporarily unavailable"},
{in: "too many requests", want: "AI provider rate limited"},
{
in: "upstream 503: model overloaded", want: "upstream 503: model overloaded"},
{
in: "openai retries exhausted: green-chat unavailable", want: "green-chat unavailable"},
{in: "upstream 503: model overloaded", want: "provider_unavailable"},
{in: "openai retries exhausted: green-chat unavailable", want: "provider_unavailable"},
}
for _, tc := range cases {
got := TruncateError(errString(tc.in))
apps/api/internal/processing/title_spacing_test.go
+8 -3
View File
@@ -46,6 +46,7 @@ func TestV1ProcessJobItemMarshalJSON_order(t *testing.T) {
"category": "Hladilniki",
"category_id": "11",
"processed_product_id": "bbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb",
"gpt_response": "should-not-ship",
"status": "processed",
}
b, err := item.MarshalJSON()
@@ -56,11 +57,15 @@ func TestV1ProcessJobItemMarshalJSON_order(t *testing.T) {
eanAt := strings.Index(s, `"ean"`)
titleAt := strings.Index(s, `"title"`)
attrsAt := strings.Index(s, `"attributes"`)
ppAt := strings.Index(s, `"processed_product_id"`)
if eanAt < 0 || titleAt < 0 || attrsAt < 0 || ppAt < 0 {
if eanAt < 0 || titleAt < 0 || attrsAt < 0 {
t.Fatalf("missing keys in %s", s)
}
if !(eanAt < titleAt && titleAt < attrsAt && attrsAt < ppAt) {
if !(eanAt < titleAt && titleAt < attrsAt) {
t.Fatalf("bad key order in %s", s)
}
for _, leak := range []string{`"processed_product_id"`, `"gpt_response"`} {
if strings.Contains(s, leak) {
t.Fatalf("allowlist MarshalJSON must omit %s: %s", leak, s)
}
}
}
apps/api/internal/processing/v1_item_json.go
+32 -34
View File
@@ -5,7 +5,7 @@ import (
"encoding/json"
)
// v1ProcessItemKeyOrder is the human-readable LegacyProcessItem property order.
// v1ProcessItemKeyOrder is the human-readable ProcessItem property order.
// Important catalog fields first; internal ids / SEO last.
var v1ProcessItemKeyOrder = []string{
"ean",
@@ -23,13 +23,39 @@ var v1ProcessItemKeyOrder = []string{
"error",
"meta_title",
"meta_description",
"id",
"processed_product_id",
"raw_product_id",
}
// MarshalJSON emits LegacyProcessItem keys in a stable human-readable order.
// ASSUMPTION: JSON object key order is part of the V1 readability contract for A1.
// v1ProcessItemInternalKeys must never appear on public ProcessItem JSON.
var v1ProcessItemInternalKeys = map[string]struct{}{
"id": {},
"processed_product_id": {},
"raw_product_id": {},
"field_sources": {},
"enhance_input_hash": {},
"ai_enhance": {},
"finish_reason": {},
"gpt_response": {},
"total_tokens": {},
"prompt_tokens": {},
"completion_tokens": {},
"ai_provider_mode": {},
"notes": {},
"worker": {},
"model": {},
"provider": {},
}
func stripV1ProcessItemInternalKeys(item V1ProcessJobItem) {
if item == nil {
return
}
for k := range v1ProcessItemInternalKeys {
delete(item, k)
}
}
// MarshalJSON emits only the public ProcessItem allowlist in a stable order.
// Unexpected keys (including pipeline internals) cannot ship.
func (item V1ProcessJobItem) MarshalJSON() ([]byte, error) {
if item == nil {
return []byte("null"), nil
@@ -55,43 +81,15 @@ func (item V1ProcessJobItem) MarshalJSON() ([]byte, error) {
buf.Write(vb)
return nil
}
seen := map[string]struct{}{}
for _, k := range v1ProcessItemKeyOrder {
v, ok := item[k]
if !ok {
continue
}
seen[k] = struct{}{}
if err := writePair(k, v); err != nil {
return nil, err
}
}
// Preserve any unexpected keys deterministically (sorted via encoding/json map).
extras := map[string]any{}
for k, v := range item {
if _, ok := seen[k]; ok {
continue
}
extras[k] = v
}
if len(extras) > 0 {
eb, err := json.Marshal(extras)
if err != nil {
return nil, err
}
// eb is `{...}`; splice inner pairs.
inner := bytes.TrimSpace(eb)
if len(inner) >= 2 && inner[0] == '{' && inner[len(inner)-1] == '}' {
inner = inner[1 : len(inner)-1]
if len(bytes.TrimSpace(inner)) > 0 {
if !first {
buf.WriteByte(',')
}
first = false
buf.Write(inner)
}
}
}
buf.WriteByte('}')
return buf.Bytes(), nil
}
apps/api/internal/processing/v1_legacy.go
+85 -16
View File
@@ -13,6 +13,7 @@ import (
"github.com/descrybe/descrybe-v2/apps/api/internal/company"
"github.com/descrybe/descrybe-v2/apps/api/internal/eprel"
"github.com/google/uuid"
"github.com/jackc/pgx/v5/pgxpool"
)
var v1PartialSteps = []string{"category", "title", "description", "attributes"}
@@ -274,7 +275,7 @@ func (p *Pipeline) LoadV1ProcessJobItems(ctx context.Context, companyID, jobID u
"error": "Product data not available",
}
if itemError != nil && *itemError != "" {
item["error"] = *itemError
item["error"] = PublicV1Error(*itemError)
}
full = append(full, item)
continue
@@ -309,6 +310,9 @@ func (p *Pipeline) LoadV1ProcessJobItems(ctx context.Context, companyID, jobID u
if descTxt != nil {
descOut = v1PreserveDescription(*descTxt)
}
if isPromptLeakageTitle(descOut) || isPromptLabelTitle(descOut) {
descOut = ""
}
eprelVal := extractEPRELFromAttrs(attrs)
attrs = SanitizeV1ProcessAttributesAllowed(attrs, allowedAttrs)
@@ -453,7 +457,7 @@ func (p *Pipeline) LoadV1ProcessJobItems(ctx context.Context, companyID, jobID u
// Internal ids (id / processed_product_id / raw_product_id) are omitted from
// the public V1 process item shape — use catalog APIs when a UUID is needed.
if itemError != nil && *itemError != "" {
item["error"] = *itemError
item["error"] = PublicV1Error(*itemError)
}
if len(attrs) > 0 {
item["attributes"] = attrs
@@ -484,35 +488,72 @@ func nullIfEmptyPtr(s *string) any {
return *s
}
// companyOmitsSEOMeta is true when V1/process must omit meta_title / meta_description:
// A1 cohort (legacy id), Platform Demo (A1-cloned prompts, no legacy id), or any
// company whose categories store A1-style role-section enhance prompts.
func companyOmitsSEOMeta(ctx context.Context, p *Pipeline, companyID uuid.UUID) bool {
if p == nil || p.Pool == nil {
// Known v2 companies.id values that omit SEO meta on V1 process payloads.
// A1 Slovenija is also matched via billing.A1LegacyCompanyID (legacy_company_id / PK).
const (
a1V2CompanyID = "604f23a8-b66e-4b21-8b45-0d72b68f4790"
platformDemoV2CompanyID = "2b3159b0-fc08-415b-b248-35ed02a6baab"
)
// CompanyOmitsSEOMetaID is true for A1 Slovenija and Platform Demo by companies.id
// (or the migrated MySQL company id). Does not grant Legacy plan privileges.
func CompanyOmitsSEOMetaID(companyID uuid.UUID) bool {
if companyID == uuid.Nil {
return false
}
id := strings.ToLower(companyID.String())
switch id {
case a1V2CompanyID, platformDemoV2CompanyID, strings.ToLower(billing.A1LegacyCompanyID):
return true
}
return false
}
// CompanyOmitsSEOMetaLookup is the shared A1/Demo SEO-omit decision used by V1 process
// and catalog list/get/update. Matches hardcoded companies.id, A1 cohort (legacy id),
// Platform Demo by name, or A1-style --- Title --- / --- Description --- / --- Meta ---
// category prompts.
func CompanyOmitsSEOMetaLookup(companyID uuid.UUID, legacyID, name string, hasA1SectionPrompts bool) bool {
if CompanyOmitsSEOMetaID(companyID) {
return true
}
if billing.IsA1CohortCompany(legacyID, "") {
return true
}
if strings.EqualFold(strings.TrimSpace(name), "Platform Demo") {
return true
}
return hasA1SectionPrompts
}
// CompanyOmitsSEOMeta is true when catalog/V1 process must omit meta_title /
// meta_description for this company (same rules as CompanyOmitsSEOMetaLookup).
func CompanyOmitsSEOMeta(ctx context.Context, pool *pgxpool.Pool, companyID uuid.UUID) bool {
if CompanyOmitsSEOMetaID(companyID) {
return true
}
if pool == nil {
return false
}
var legacy, name string
err := p.Pool.QueryRow(ctx, `
err := pool.QueryRow(ctx, `
SELECT COALESCE(legacy_company_id, ''), COALESCE(name, '')
FROM companies
WHERE id = $1`, companyID).Scan(&legacy, &name)
if err != nil {
return false
}
if billing.IsA1CohortCompany(legacy, "") {
return true
}
if strings.EqualFold(strings.TrimSpace(name), "Platform Demo") {
if CompanyOmitsSEOMetaLookup(companyID, legacy, name, false) {
return true
}
var hasA1Prompts bool
err = p.Pool.QueryRow(ctx, `
err = pool.QueryRow(ctx, `
SELECT EXISTS (
SELECT 1 FROM categories
WHERE company_id = $1
AND prompt ILIKE '%--- Title ---%'
AND prompt ILIKE '%--- Description ---%'
AND prompt ILIKE '%--- Meta ---%'
AND prompt ILIKE '%--- Title ---%'
AND prompt ILIKE '%--- Description ---%'
AND prompt ILIKE '%--- Meta ---%'
LIMIT 1
)`, companyID).Scan(&hasA1Prompts)
if err != nil {
@@ -521,6 +562,34 @@ func companyOmitsSEOMeta(ctx context.Context, p *Pipeline, companyID uuid.UUID)
return hasA1Prompts
}
// StripV1SEOMeta removes meta_title / meta_description keys (omit, not null).
func StripV1SEOMeta(items []V1ProcessJobItem) []V1ProcessJobItem {
for _, item := range items {
if item == nil {
continue
}
delete(item, "meta_title")
delete(item, "meta_description")
}
return items
}
// ApplyV1SEOMetaPolicy omits SEO meta on A1/Demo process items by companies.id.
func ApplyV1SEOMetaPolicy(companyID uuid.UUID, items []V1ProcessJobItem) []V1ProcessJobItem {
if !CompanyOmitsSEOMetaID(companyID) {
return items
}
return StripV1SEOMeta(items)
}
// companyOmitsSEOMeta is the pipeline wrapper around CompanyOmitsSEOMeta.
func companyOmitsSEOMeta(ctx context.Context, p *Pipeline, companyID uuid.UUID) bool {
if p == nil {
return CompanyOmitsSEOMeta(ctx, nil, companyID)
}
return CompanyOmitsSEOMeta(ctx, p.Pool, companyID)
}
func derefStringPtr(s *string) string {
if s == nil {
return ""
apps/api/internal/processing/v1_legacy_test.go
+61
View File
@@ -190,6 +190,37 @@ func TestApplyV1ProcessItemIDsDualMode(t *testing.T) {
}
}
func TestV1ProcessJobItemMarshalJSONOmitsInternals(t *testing.T) {
item := V1ProcessJobItem{
"ean": "1",
"status": "processed",
"name": "T",
"id": "bbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb",
"ai_enhance": true,
"finish_reason": "stop",
"enhance_input_hash": "deadbeef",
"field_sources": map[string]any{"enhance_input_hash": "deadbeef"},
"prompt_tokens": 9,
"worker": "river",
}
raw, err := json.Marshal(item)
if err != nil {
t.Fatal(err)
}
s := string(raw)
for _, junk := range []string{
`"id"`, `"ai_enhance"`, `"finish_reason"`, `"enhance_input_hash"`,
`"field_sources"`, `"prompt_tokens"`, `"worker"`, "deadbeef",
} {
if strings.Contains(s, junk) {
t.Fatalf("internal %q leaked in JSON: %s", junk, s)
}
}
if !strings.Contains(s, `"ean":"1"`) || !strings.Contains(s, `"name":"T"`) {
t.Fatalf("public fields missing: %s", s)
}
}
func TestFormatJobStatusResponseAddsItems(t *testing.T) {
jobID := mustParseTestUUID(t, "aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa")
job := Job{ID: jobID, Status: "completed", TotalProducts: 1}
@@ -218,3 +249,33 @@ func mustParseTestUUID(t *testing.T, s string) uuid.UUID {
}
return id
}
func TestCompanyOmitsSEOMetaLookup(t *testing.T) {
t.Parallel()
a1 := mustParseTestUUID(t, "604f23a8-b66e-4b21-8b45-0d72b68f4790")
other := mustParseTestUUID(t, "11111111-1111-1111-1111-111111111111")
cases := []struct {
name string
id uuid.UUID
legacy string
coName string
prompts bool
wantOmit bool
}{
{name: "ordinary", id: other, coName: "Acme", wantOmit: false},
{name: "a1_uuid", id: a1, wantOmit: true},
{name: "demo_by_name", id: other, coName: "Platform Demo", wantOmit: true},
{name: "demo_by_name_case", id: other, coName: " platform DEMO ", wantOmit: true},
{name: "a1_prompt_markers", id: other, coName: "Acme", prompts: true, wantOmit: true},
{name: "a1_legacy_id", id: other, legacy: "97e1a309-3d23-4aa2-b518-8e8d7afdfec7", wantOmit: true},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
t.Parallel()
got := CompanyOmitsSEOMetaLookup(tc.id, tc.legacy, tc.coName, tc.prompts)
if got != tc.wantOmit {
t.Fatalf("got %v want %v", got, tc.wantOmit)
}
})
}
}
apps/api/internal/processing/v1_process_item.go
+9 -5
View File
@@ -9,7 +9,7 @@ import (
)
// V1ProcessItemScorecard scores one COMPLETED legacy process item against
// OpenAPI LegacyProcessItem + A1 poll expectations.
// OpenAPI ProcessItem + A1 poll expectations.
type V1ProcessItemScorecard struct {
EAN string `json:"ean"`
Status string `json:"status"`
@@ -184,7 +184,7 @@ func ScoreV1ProcessCompletedItem(item V1ProcessJobItem, opts ScoreV1ProcessItemO
return sc
}
// EnforceV1ProcessCompletedItem fills LegacyProcessItem projection gaps for a successful item:
// EnforceV1ProcessCompletedItem fills ProcessItem projection gaps for a successful item:
// nonempty description when title exists (formula HTML preserved), readable title spacing,
// category display name as primary, optional SEO meta (unless omitSEOMeta), clean attrs.
// allowed is the company attribute_key set (canonicalized). When nil, only
@@ -209,6 +209,7 @@ func EnforceV1ProcessCompletedItemOpts(item V1ProcessJobItem, opts EnforceV1Opts
if item == nil {
return item
}
stripV1ProcessItemInternalKeys(item)
status, _ := item["status"].(string)
if status != "processed" {
return item
@@ -230,6 +231,9 @@ func EnforceV1ProcessCompletedItemOpts(item V1ProcessJobItem, opts EnforceV1Opts
if title == "" {
title = ensureReadableTitleSpacing(stringFromItem(item, "title"))
}
if isPromptLabelTitle(title) {
title = preferredProductTitle(stringFromItem(item, "ean"), title)
}
if title != "" {
item["name"] = title
delete(item, "title")
@@ -237,9 +241,6 @@ func EnforceV1ProcessCompletedItemOpts(item V1ProcessJobItem, opts EnforceV1Opts
item["name"] = nil
delete(item, "title")
}
delete(item, "id")
delete(item, "processed_product_id")
delete(item, "raw_product_id")
catID, catName := projectV1CategoryFields(item)
catLabel := catName
@@ -248,6 +249,9 @@ func EnforceV1ProcessCompletedItemOpts(item V1ProcessJobItem, opts EnforceV1Opts
}
desc, _ := descriptionFromItem(item)
if isPromptLeakageTitle(desc) || isPromptLabelTitle(desc) {
desc = ""
}
needsDesc := title != "" && (desc == "" ||
isWeakPriorEnhanceDescription(desc, title) ||
descriptionEchoesTitle(desc, title) ||
apps/api/internal/processing/v1_process_item_test.go
+131 -1
View File
@@ -4,6 +4,9 @@ import (
"fmt"
"strings"
"testing"
"github.com/descrybe/descrybe-v2/apps/api/internal/billing"
"github.com/google/uuid"
)
func TestScoreV1ProcessCompletedItem_processedStructure(t *testing.T) {
@@ -149,11 +152,25 @@ func TestEnforceV1ProcessCompletedItem_fillsDescriptionMetaSanitizes(t *testing.
func TestEnforceV1ProcessCompletedItem_skipsTerminalStatuses(t *testing.T) {
t.Parallel()
item := V1ProcessJobItem{"ean": "1", "status": "not_found", "error": "missing"}
item := V1ProcessJobItem{
"ean": "1",
"status": "not_found",
"error": "missing",
"ai_enhance": true,
"finish_reason": "stop",
"field_sources": map[string]any{"enhance_input_hash": "abc"},
"prompt_tokens": 12,
"id": "bbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb",
}
out := EnforceV1ProcessCompletedItem(item, "", nil)
if _, ok := out["title"]; ok {
t.Fatalf("should not invent fields for not_found: %v", out)
}
for _, junk := range []string{"id", "ai_enhance", "finish_reason", "field_sources", "prompt_tokens"} {
if _, ok := out[junk]; ok {
t.Fatalf("%s must be stripped from public items: %v", junk, out)
}
}
}
func TestEnforceV1ProcessCompletedItem_categoryFromCallerPreserved(t *testing.T) {
@@ -328,6 +345,119 @@ func TestEnforceV1ProcessCompletedItem_refreshesPromptLeakageMeta(t *testing.T)
}
}
func TestCompanyOmitsSEOMetaID(t *testing.T) {
t.Parallel()
a1 := uuid.MustParse("604f23a8-b66e-4b21-8b45-0d72b68f4790")
demo := uuid.MustParse("2b3159b0-fc08-415b-b248-35ed02a6baab")
legacy := uuid.MustParse(billing.A1LegacyCompanyID)
other := uuid.MustParse("11111111-1111-1111-1111-111111111111")
if !CompanyOmitsSEOMetaID(a1) || !CompanyOmitsSEOMetaID(demo) || !CompanyOmitsSEOMetaID(legacy) {
t.Fatal("A1, Demo, and legacy A1 ids must omit SEO meta")
}
if CompanyOmitsSEOMetaID(other) || CompanyOmitsSEOMetaID(uuid.Nil) {
t.Fatal("unrelated company ids must keep SEO meta")
}
}
func TestApplyV1SEOMetaPolicy_omitsOnlyA1(t *testing.T) {
t.Parallel()
item := V1ProcessJobItem{
"ean": "1",
"status": "processed",
"meta_title": "T",
"meta_description": "D",
}
a1 := uuid.MustParse("604f23a8-b66e-4b21-8b45-0d72b68f4790")
out := ApplyV1SEOMetaPolicy(a1, []V1ProcessJobItem{item})
if _, ok := out[0]["meta_title"]; ok {
t.Fatal("A1 must omit meta_title")
}
if _, ok := out[0]["meta_description"]; ok {
t.Fatal("A1 must omit meta_description")
}
keep := V1ProcessJobItem{
"ean": "1",
"status": "processed",
"meta_title": "T",
"meta_description": "D",
}
other := uuid.MustParse("11111111-1111-1111-1111-111111111111")
kept := ApplyV1SEOMetaPolicy(other, []V1ProcessJobItem{keep})
if kept[0]["meta_title"] != "T" || kept[0]["meta_description"] != "D" {
t.Fatalf("non-A1 must keep SEO meta: %v", kept[0])
}
}
func TestEnforceV1ProcessCompletedItemOpts_omitSEOMeta(t *testing.T) {
t.Parallel()
item := V1ProcessJobItem{
"ean": "1",
"status": "processed",
"title": "Sony Headphones",
"description": "Sony Headphones deliver clear audio with noise cancelling for daily commuting.",
"meta_title": "Sony | Headphones",
"meta_description": "Sony Headphones deliver clear audio.",
"category": "48",
"category_name": "Slušalke",
"attributes": map[string]any{"brand": "Sony"},
"eprel": nil,
}
out := EnforceV1ProcessCompletedItemOpts(item, EnforceV1Opts{Language: "sl", OmitSEOMeta: true})
if _, ok := out["meta_title"]; ok {
t.Fatalf("meta_title should be omitted: %v", out["meta_title"])
}
if _, ok := out["meta_description"]; ok {
t.Fatalf("meta_description should be omitted: %v", out["meta_description"])
}
sc := ScoreV1ProcessCompletedItem(out, ScoreV1ProcessItemOptions{MappedCategory: "48", OmitSEOMeta: true})
if !sc.OK {
t.Fatalf("omit meta should still score OK, flags=%v", sc.FailFlags)
}
}
func TestEnforceV1ProcessCompletedItemOpts_A1PollScrubsPromptDump(t *testing.T) {
t.Parallel()
item := V1ProcessJobItem{
"ean": "8606019604493",
"status": "processed",
"name": "--- Title ---\nReply with ONLY JSON\nSchema: {\"name\":\"string\"}",
"description": "--- Description ---\nReply with ONLY JSON (no markdown)\n" +
"Title formula: Brand + Model. Follow any constraints that follow.\n" +
"Schema: {\"description\":\"string\"}",
"meta_title": "Sony | short retail title; follow any Title formula constr…",
"meta_description": "prefer 1-3 factual paragraphs as ONE string",
"category": "50",
"category_name": "Cookers",
"attributes": map[string]any{"brand": "Vox", "product_model": "EHT6020WG"},
}
out := EnforceV1ProcessCompletedItemOpts(item, EnforceV1Opts{Language: "en", OmitSEOMeta: true})
name := fmt.Sprint(out["name"])
desc := fmt.Sprint(out["description"])
for _, phrase := range []string{
"--- Title ---",
"--- Description ---",
"Reply with ONLY JSON",
"Title formula",
"Schema:",
} {
if strings.Contains(name, phrase) || strings.Contains(desc, phrase) {
t.Fatalf("A1 poll must not return prompt dump %q: name=%q desc=%q", phrase, name, desc)
}
}
if isPromptLeakageTitle(name) || isPromptLeakageTitle(desc) {
t.Fatalf("A1 poll still looks like prompt leakage: name=%q desc=%q", name, desc)
}
if desc == "" || desc == "<nil>" || out["description"] == nil {
t.Fatalf("A1 poll should replace leaked description, got %v", out["description"])
}
if _, ok := out["meta_title"]; ok {
t.Fatal("A1 poll must omit meta_title")
}
if _, ok := out["meta_description"]; ok {
t.Fatal("A1 poll must omit meta_description")
}
}
func TestEnforceV1ProcessCompletedItem_preservesFormulaHTML(t *testing.T) {
t.Parallel()
htmlDesc := `<h1>Anker Soundcore Space One Pro</h1><p>Zložljive ANC slušalke z bogatim zvokom.</p><ul><li>Bluetooth 5.3</li></ul>`