Initial commit of Descrybe v2 without local scratch artifacts.

Drop one-shot tmp/axe scripts and agent i18n scratch so the Gitea tree is deployable.
This commit is contained in:
2026-08-09 22:47:43 +02:00
commit 8580c996c3
1285 changed files with 325780 additions and 0 deletions
+299
View File
@@ -0,0 +1,299 @@
package woocommerce
import (
"bytes"
"context"
"encoding/base64"
"encoding/json"
"fmt"
"io"
"net/http"
"net/url"
"strconv"
"strings"
"time"
"github.com/descrybe/descrybe-v2/apps/api/internal/security"
)
const (
apiPrefix = "/wp-json/wc/v3"
defaultTimeout = 30 * time.Second
maxBodyBytes = 8 << 20
maxRateLimitRetries = 5
maxSKULookupChunk = 50
)
type Client struct {
BaseURL string
ConsumerKey string
ConsumerSecret string
HTTP *http.Client
}
type ProductPayload struct {
ID int `json:"id,omitempty"`
Name string `json:"name,omitempty"`
Type string `json:"type,omitempty"`
Status string `json:"status,omitempty"`
Description string `json:"description,omitempty"`
ShortDescription string `json:"short_description,omitempty"`
SKU string `json:"sku,omitempty"`
RegularPrice string `json:"regular_price,omitempty"`
Categories []map[string]any `json:"categories,omitempty"`
Attributes []map[string]any `json:"attributes,omitempty"`
Images []map[string]string `json:"images,omitempty"`
MetaData []MetaDatum `json:"meta_data,omitempty"`
ManageStock *bool `json:"manage_stock,omitempty"`
StockStatus string `json:"stock_status,omitempty"`
}
type MetaDatum struct {
Key string `json:"key"`
Value string `json:"value"`
}
type Product struct {
ID int `json:"id"`
SKU string `json:"sku"`
Name string `json:"name"`
}
type BatchRequest struct {
Create []ProductPayload `json:"create,omitempty"`
Update []ProductPayload `json:"update,omitempty"`
}
type BatchResponse struct {
Create []Product `json:"create"`
Update []Product `json:"update"`
}
type Category struct {
ID int `json:"id"`
Name string `json:"name"`
Slug string `json:"slug"`
}
type Attribute struct {
ID int `json:"id"`
Name string `json:"name"`
Slug string `json:"slug"`
}
func NewClient(baseURL, key, secret string, httpClient *http.Client) *Client {
if httpClient == nil {
httpClient = security.SafeHTTPClient(defaultTimeout, true)
}
return &Client{
BaseURL: strings.TrimRight(baseURL, "/"),
ConsumerKey: key,
ConsumerSecret: secret,
HTTP: httpClient,
}
}
func (c *Client) TestConnection(ctx context.Context) error {
_, err := c.do(ctx, http.MethodGet, "/products", map[string]string{"per_page": "1"}, nil)
return err
}
func (c *Client) ListProductsBySKU(ctx context.Context, sku string) ([]Product, error) {
found, err := c.ListProductsBySKUs(ctx, []string{sku})
if err != nil {
return nil, err
}
sku = strings.TrimSpace(sku)
if p, ok := found[sku]; ok {
return []Product{p}, nil
}
return nil, nil
}
// ListProductsBySKUs resolves many SKUs via comma-separated WC filters (chunked).
// Keys in the returned map are the exact requested SKUs that matched.
func (c *Client) ListProductsBySKUs(ctx context.Context, skus []string) (map[string]Product, error) {
seen := make(map[string]struct{}, len(skus))
clean := make([]string, 0, len(skus))
for _, sku := range skus {
sku = strings.TrimSpace(sku)
if sku == "" {
continue
}
if _, ok := seen[sku]; ok {
continue
}
seen[sku] = struct{}{}
clean = append(clean, sku)
}
out := make(map[string]Product, len(clean))
if len(clean) == 0 {
return out, nil
}
chunkSize := maxSKULookupChunk
for i := 0; i < len(clean); i += chunkSize {
end := i + chunkSize
if end > len(clean) {
end = len(clean)
}
chunk := clean[i:end]
want := make(map[string]struct{}, len(chunk))
for _, sku := range chunk {
want[sku] = struct{}{}
}
perPage := len(chunk)
if perPage < 10 {
perPage = 10
}
if perPage > 100 {
perPage = 100
}
raw, err := c.do(ctx, http.MethodGet, "/products", map[string]string{
"sku": strings.Join(chunk, ","),
"per_page": strconv.Itoa(perPage),
}, nil)
if err != nil {
return nil, err
}
var products []Product
if err := json.Unmarshal(raw, &products); err != nil {
return nil, err
}
for _, p := range products {
sku := strings.TrimSpace(p.SKU)
if _, ok := want[sku]; !ok {
continue
}
if _, exists := out[sku]; exists {
continue
}
out[sku] = p
}
}
return out, nil
}
func (c *Client) BatchProducts(ctx context.Context, req BatchRequest) (BatchResponse, error) {
raw, err := c.do(ctx, http.MethodPost, "/products/batch", nil, req)
if err != nil {
return BatchResponse{}, err
}
var out BatchResponse
if err := json.Unmarshal(raw, &out); err != nil {
return BatchResponse{}, err
}
return out, nil
}
func (c *Client) ListCategories(ctx context.Context) ([]Category, error) {
raw, err := c.do(ctx, http.MethodGet, "/products/categories", map[string]string{"per_page": "100"}, nil)
if err != nil {
return nil, err
}
var out []Category
if err := json.Unmarshal(raw, &out); err != nil {
return nil, err
}
return out, nil
}
func (c *Client) ListAttributes(ctx context.Context) ([]Attribute, error) {
raw, err := c.do(ctx, http.MethodGet, "/products/attributes", map[string]string{"per_page": "100"}, nil)
if err != nil {
return nil, err
}
var out []Attribute
if err := json.Unmarshal(raw, &out); err != nil {
return nil, err
}
return out, nil
}
func retryAfterWait(h http.Header, attempt int) time.Duration {
if ra := strings.TrimSpace(h.Get("Retry-After")); ra != "" {
if secs, err := strconv.Atoi(ra); err == nil && secs >= 0 {
return time.Duration(secs) * time.Second
}
}
shift := attempt
if shift > 4 {
shift = 4
}
return time.Duration(1<<shift) * time.Second
}
func (c *Client) do(ctx context.Context, method, path string, query map[string]string, body any) ([]byte, error) {
u, err := url.Parse(c.BaseURL + apiPrefix + path)
if err != nil {
return nil, err
}
q := u.Query()
for k, v := range query {
q.Set(k, v)
}
u.RawQuery = q.Encode()
var bodyBytes []byte
if body != nil {
bodyBytes, err = json.Marshal(body)
if err != nil {
return nil, err
}
}
var lastErr error
for attempt := 0; attempt <= maxRateLimitRetries; attempt++ {
var rdr io.Reader
if bodyBytes != nil {
rdr = bytes.NewReader(bodyBytes)
}
req, err := http.NewRequestWithContext(ctx, method, u.String(), rdr)
if err != nil {
return nil, err
}
req.Header.Set("Accept", "application/json")
req.Header.Set("User-Agent", "Descrybe-WooCommerce/2.0")
if bodyBytes != nil {
req.Header.Set("Content-Type", "application/json")
}
token := base64.StdEncoding.EncodeToString([]byte(c.ConsumerKey + ":" + c.ConsumerSecret))
req.Header.Set("Authorization", "Basic "+token)
res, err := c.HTTP.Do(req)
if err != nil {
return nil, err
}
limited := io.LimitReader(res.Body, maxBodyBytes+1)
raw, err := io.ReadAll(limited)
res.Body.Close()
if err != nil {
return nil, err
}
if len(raw) > maxBodyBytes {
return nil, fmt.Errorf("woocommerce response too large")
}
if res.StatusCode == http.StatusTooManyRequests || res.StatusCode == http.StatusServiceUnavailable {
lastErr = fmt.Errorf("woocommerce api %s: rate limited", strconv.Itoa(res.StatusCode))
if attempt == maxRateLimitRetries {
break
}
wait := retryAfterWait(res.Header, attempt)
select {
case <-ctx.Done():
return nil, ctx.Err()
case <-time.After(wait):
}
continue
}
if res.StatusCode < 200 || res.StatusCode >= 300 {
msg := strings.TrimSpace(string(raw))
if len(msg) > 400 {
msg = msg[:400] + "…"
}
return nil, fmt.Errorf("woocommerce api %s: %s", strconv.Itoa(res.StatusCode), msg)
}
return raw, nil
}
return nil, lastErr
}
+110
View File
@@ -0,0 +1,110 @@
package woocommerce
import (
"crypto/aes"
"crypto/cipher"
"crypto/rand"
"crypto/sha256"
"encoding/base64"
"encoding/hex"
"errors"
"io"
"strings"
"github.com/descrybe/descrybe-v2/apps/api/internal/config"
)
const encPrefix = "enc:v1:"
// DeriveKey builds a 32-byte AES key. Prefer explicitKey (hex/base64 of 32 bytes,
// or any passphrase hashed). When empty, derives from fallbackMaterial so local
// setups work without a dedicated env var (still at-rest encryption).
// In production (APP_ENV=production|prod), explicitKey is required; otherwise
// returns nil so encrypt/decrypt fail closed instead of hashing DATABASE_URL.
func DeriveKey(explicitKey, fallbackMaterial string) []byte {
explicitKey = strings.TrimSpace(explicitKey)
if explicitKey != "" {
if b, err := decodeKeyMaterial(explicitKey); err == nil {
return b
}
sum := sha256.Sum256([]byte(explicitKey))
return sum[:]
}
if config.IsProductionEnv() {
return nil
}
sum := sha256.Sum256([]byte("descrybe-woo-v1|" + fallbackMaterial))
return sum[:]
}
func decodeKeyMaterial(s string) ([]byte, error) {
if b, err := base64.StdEncoding.DecodeString(s); err == nil && len(b) == 32 {
return b, nil
}
if b, err := base64.RawStdEncoding.DecodeString(s); err == nil && len(b) == 32 {
return b, nil
}
if b, err := hex.DecodeString(s); err == nil && len(b) == 32 {
return b, nil
}
return nil, errors.New("invalid key material")
}
func EncryptSecret(key []byte, plaintext string) (string, error) {
if plaintext == "" {
return "", nil
}
if len(key) != 32 {
return "", errors.New("encryption key must be 32 bytes")
}
block, err := aes.NewCipher(key)
if err != nil {
return "", err
}
gcm, err := cipher.NewGCM(block)
if err != nil {
return "", err
}
nonce := make([]byte, gcm.NonceSize())
if _, err := io.ReadFull(rand.Reader, nonce); err != nil {
return "", err
}
sealed := gcm.Seal(nonce, nonce, []byte(plaintext), nil)
return encPrefix + base64.RawStdEncoding.EncodeToString(sealed), nil
}
func DecryptSecret(key []byte, stored string) (string, error) {
if stored == "" {
return "", nil
}
if !strings.HasPrefix(stored, encPrefix) {
if config.IsProductionEnv() {
return "", errors.New("plaintext secrets are not allowed when APP_ENV=production")
}
return stored, nil
}
if len(key) != 32 {
return "", errors.New("encryption key must be 32 bytes")
}
raw, err := base64.RawStdEncoding.DecodeString(strings.TrimPrefix(stored, encPrefix))
if err != nil {
return "", err
}
block, err := aes.NewCipher(key)
if err != nil {
return "", err
}
gcm, err := cipher.NewGCM(block)
if err != nil {
return "", err
}
if len(raw) < gcm.NonceSize() {
return "", errors.New("ciphertext too short")
}
nonce, ciphertext := raw[:gcm.NonceSize()], raw[gcm.NonceSize():]
plain, err := gcm.Open(nil, nonce, ciphertext, nil)
if err != nil {
return "", err
}
return string(plain), nil
}
@@ -0,0 +1,202 @@
package woocommerce
import (
"encoding/json"
"errors"
"fmt"
"testing"
"time"
"github.com/google/uuid"
)
func TestEncryptDecryptRoundTrip(t *testing.T) {
t.Setenv("APP_ENV", "development")
key := DeriveKey("test-passphrase", "fallback")
enc, err := EncryptSecret(key, "ck_secret_value")
if err != nil {
t.Fatal(err)
}
if enc == "" || enc == "ck_secret_value" {
t.Fatalf("expected ciphertext, got %q", enc)
}
plain, err := DecryptSecret(key, enc)
if err != nil {
t.Fatal(err)
}
if plain != "ck_secret_value" {
t.Fatalf("got %q", plain)
}
}
func TestDecryptLegacyPlaintext(t *testing.T) {
t.Setenv("APP_ENV", "development")
key := DeriveKey("x", "y")
plain, err := DecryptSecret(key, "legacy-plain")
if err != nil {
t.Fatal(err)
}
if plain != "legacy-plain" {
t.Fatalf("got %q", plain)
}
}
func TestDecryptLegacyPlaintextRejectedInProduction(t *testing.T) {
t.Setenv("APP_ENV", "production")
key := DeriveKey("x", "y")
if _, err := DecryptSecret(key, "legacy-plain"); err == nil {
t.Fatal("expected plaintext decrypt rejected in production")
}
}
func TestDeriveKeyRejectsFallbackInProduction(t *testing.T) {
t.Setenv("APP_ENV", "production")
if key := DeriveKey("", "postgres://local"); key != nil {
t.Fatalf("expected nil key without explicit material in production, got len=%d", len(key))
}
t.Setenv("APP_ENV", "development")
if key := DeriveKey("", "postgres://local"); len(key) != 32 {
t.Fatalf("expected fallback key in development, got len=%d", len(key))
}
}
func TestParseSyncOptionsClampsLimits(t *testing.T) {
raw := []byte(`{"sync_limit":99999,"batch_size":500,"orders_sync_limit":99999,"reviews_sync_limit":-1,"schedule_interval_hours":999}`)
opt := parseSyncOptions(raw)
if opt.SyncLimit != defaultSyncLimit {
t.Fatalf("sync_limit=%d", opt.SyncLimit)
}
if opt.BatchSize != defaultBatchSize {
t.Fatalf("batch_size=%d", opt.BatchSize)
}
if opt.OrdersSyncLimit != 0 {
t.Fatalf("orders_sync_limit=%d", opt.OrdersSyncLimit)
}
if opt.ReviewsSyncLimit != 0 {
t.Fatalf("reviews_sync_limit=%d", opt.ReviewsSyncLimit)
}
if opt.ScheduleIntervalHours != 0 {
t.Fatalf("schedule_interval_hours=%d", opt.ScheduleIntervalHours)
}
}
func TestParseSyncOptionsPrunesProductIDs(t *testing.T) {
ids := make(map[string]int, maxProductIDMap+50)
for i := 0; i < maxProductIDMap+50; i++ {
ids[fmt.Sprintf("sku-%d", i)] = i + 1
}
raw, err := json.Marshal(map[string]any{"product_ids": ids})
if err != nil {
t.Fatal(err)
}
opt := parseSyncOptions(raw)
if len(opt.ProductIDs) != maxProductIDMap {
t.Fatalf("product_ids len=%d want %d", len(opt.ProductIDs), maxProductIDMap)
}
}
func TestParseSyncOptionsScheduleAndFilterParams(t *testing.T) {
raw := []byte(`{
"schedule_interval_hours":48,
"match_strategy":"barcode",
"orders_modified_after":"2026-01-01T00:00:00Z",
"product_ids":{"SKU-1":7}
}`)
opt := parseSyncOptions(raw)
if opt.ScheduleIntervalHours != 48 {
t.Fatalf("schedule_interval_hours=%d want 48", opt.ScheduleIntervalHours)
}
if opt.MatchStrategy != "barcode" {
t.Fatalf("match_strategy=%q", opt.MatchStrategy)
}
if opt.OrdersModifiedAfter != "2026-01-01T00:00:00Z" {
t.Fatalf("orders_modified_after=%q", opt.OrdersModifiedAfter)
}
if opt.ProductIDs["SKU-1"] != 7 {
t.Fatalf("product_ids=%v", opt.ProductIDs)
}
maxRaw := []byte(fmt.Sprintf(`{"schedule_interval_hours":%d}`, maxScheduleIntervalH))
if got := parseSyncOptions(maxRaw).ScheduleIntervalHours; got != maxScheduleIntervalH {
t.Fatalf("max schedule kept=%d want %d", got, maxScheduleIntervalH)
}
neg := parseSyncOptions([]byte(`{"schedule_interval_hours":-1,"match_strategy":""}`))
if neg.ScheduleIntervalHours != 0 {
t.Fatalf("negative schedule=%d", neg.ScheduleIntervalHours)
}
if neg.MatchStrategy != "sku" {
t.Fatalf("empty match_strategy default=%q", neg.MatchStrategy)
}
}
func TestResolveScheduleIntervalAndDue(t *testing.T) {
if got := resolveScheduleInterval(0, 0); got != 6*time.Hour {
t.Fatalf("default interval=%s", got)
}
if got := resolveScheduleInterval(8, 2*time.Hour); got != 8*time.Hour {
t.Fatalf("custom interval=%s", got)
}
now := time.Date(2026, 8, 9, 12, 0, 0, 0, time.UTC)
if !isDueForSchedule(nil, now, time.Hour) {
t.Fatal("nil last should be due")
}
recent := now.Add(-15 * time.Minute)
if isDueForSchedule(&recent, now, time.Hour) {
t.Fatal("recent sync should not be due")
}
stale := now.Add(-90 * time.Minute)
if !isDueForSchedule(&stale, now, time.Hour) {
t.Fatal("stale sync should be due")
}
}
func TestUpdateScheduleRejectsInvalidInterval(t *testing.T) {
t.Parallel()
s := &Service{} // Pool nil — validation must fail before any DB I/O
cid := uuid.MustParse("11111111-1111-1111-1111-111111111111")
_, err := s.UpdateSchedule(t.Context(), cid, -1, false)
if !errors.Is(err, ErrInvalidScheduleInterval) {
t.Fatalf("negative hours: err=%v", err)
}
_, err = s.UpdateSchedule(t.Context(), cid, maxScheduleIntervalH+1, false)
if !errors.Is(err, ErrInvalidScheduleInterval) {
t.Fatalf("over-max hours: err=%v", err)
}
msg, ok := ClientError(ErrInvalidScheduleInterval)
if !ok || msg == "" {
t.Fatal("ClientError mapping missing for ErrInvalidScheduleInterval")
}
}
func TestShouldEnqueueScheduledPaused(t *testing.T) {
now := time.Date(2026, 8, 9, 12, 0, 0, 0, time.UTC)
if shouldEnqueueScheduled(true, 6, nil, now, 6*time.Hour) {
t.Fatal("paused schedule must not enqueue")
}
if !shouldEnqueueScheduled(false, 6, nil, now, 6*time.Hour) {
t.Fatal("unpaused with nil last should enqueue")
}
recent := now.Add(-30 * time.Minute)
if shouldEnqueueScheduled(false, 6, &recent, now, 6*time.Hour) {
t.Fatal("recent sync within interval must not enqueue")
}
opt := parseSyncOptions([]byte(`{"schedule_interval_hours":12,"schedule_paused":true}`))
if !opt.SchedulePaused || opt.ScheduleIntervalHours != 12 {
t.Fatalf("parse paused=%v hours=%d", opt.SchedulePaused, opt.ScheduleIntervalHours)
}
}
func TestNormalizeOrderListFilter(t *testing.T) {
got := normalizeOrderListFilter(OrderListFilter{Limit: 0, Offset: -2})
if got.Limit != 50 || got.Offset != 0 {
t.Fatalf("defaults got limit=%d offset=%d", got.Limit, got.Offset)
}
got = normalizeOrderListFilter(OrderListFilter{Limit: 500, Offset: 4})
if got.Limit != 200 || got.Offset != 4 {
t.Fatalf("over-max clamp got limit=%d offset=%d", got.Limit, got.Offset)
}
got = normalizeOrderListFilter(OrderListFilter{Limit: 40, Offset: 1, Status: "processing", Email: "x@y.z"})
if got.Limit != 40 || got.Offset != 1 || got.Status != "processing" || got.Email != "x@y.z" {
t.Fatalf("preserve got %+v", got)
}
}
+21
View File
@@ -0,0 +1,21 @@
package woocommerce
import "errors"
// ClientError reports whether err is a known client-facing WooCommerce config/sync error.
func ClientError(err error) (msg string, ok bool) {
switch {
case err == nil:
return "", false
case errors.Is(err, ErrInvalidStoreURL),
errors.Is(err, ErrBlockedStoreURL),
errors.Is(err, ErrNotConfigured),
errors.Is(err, ErrNotEnabled),
errors.Is(err, ErrMissingCreds),
errors.Is(err, ErrInvalidSyncScope),
errors.Is(err, ErrInvalidScheduleInterval):
return err.Error(), true
default:
return "", false
}
}
@@ -0,0 +1,384 @@
package woocommerce
import (
"context"
"encoding/json"
"strings"
"time"
"github.com/google/uuid"
)
type OrderListFilter struct {
Status string
Email string
Since *time.Time
Limit int
Offset int
}
type ReviewListFilter struct {
Status string
ProductID int64
MinRating int
Limit int
Offset int
}
type OrderRow struct {
ID uuid.UUID `json:"id"`
ExternalID int64 `json:"external_id"`
Status string `json:"status"`
Currency string `json:"currency"`
Total *string `json:"total,omitempty"`
CustomerID *int64 `json:"customer_id,omitempty"`
CustomerEmail *string `json:"customer_email,omitempty"`
CustomerName *string `json:"customer_name,omitempty"`
OrderedAt *time.Time `json:"ordered_at,omitempty"`
SyncedAt time.Time `json:"synced_at"`
Payload json.RawMessage `json:"payload"`
}
type ReviewRow struct {
ID uuid.UUID `json:"id"`
ExternalID int64 `json:"external_id"`
ProductID *int64 `json:"product_id,omitempty"`
ProductName string `json:"product_name"`
Status string `json:"status"`
Reviewer string `json:"reviewer"`
ReviewerEmail string `json:"reviewer_email"`
Rating *int `json:"rating,omitempty"`
Review string `json:"review"`
ReviewedAt *time.Time `json:"reviewed_at,omitempty"`
SyncedAt time.Time `json:"synced_at"`
Payload json.RawMessage `json:"payload"`
}
type AudienceCustomer struct {
Email string `json:"email"`
Name string `json:"name,omitempty"`
}
type AudienceResult struct {
Customers []AudienceCustomer `json:"customers"`
Total int `json:"total"`
Note string `json:"note"`
}
func normalizeOrderListFilter(f OrderListFilter) OrderListFilter {
if f.Limit <= 0 {
f.Limit = 50
}
if f.Limit > 200 {
f.Limit = 200
}
if f.Offset < 0 {
f.Offset = 0
}
return f
}
func (s *Service) ListOrders(ctx context.Context, companyID uuid.UUID, f OrderListFilter) ([]OrderRow, int, error) {
f = normalizeOrderListFilter(f)
args := []any{companyID}
where := []string{"company_id = $1"}
n := 2
if status := strings.TrimSpace(f.Status); status != "" {
where = append(where, "status = $"+itoa(n))
args = append(args, status)
n++
}
if email := strings.TrimSpace(strings.ToLower(f.Email)); email != "" {
where = append(where, "lower(customer_email) = $"+itoa(n))
args = append(args, email)
n++
}
if f.Since != nil {
where = append(where, "ordered_at >= $"+itoa(n))
args = append(args, *f.Since)
n++
}
whereSQL := strings.Join(where, " AND ")
var total int
if err := s.Pool.QueryRow(ctx, `SELECT count(*) FROM woo_orders WHERE `+whereSQL, args...).Scan(&total); err != nil {
return nil, 0, err
}
args = append(args, f.Limit, f.Offset)
rows, err := s.Pool.Query(ctx, `
SELECT id, external_id, status, currency, total::text, customer_id, customer_email, customer_name,
ordered_at, synced_at, payload
FROM woo_orders
WHERE `+whereSQL+`
ORDER BY ordered_at DESC NULLS LAST, external_id DESC
LIMIT $`+itoa(n)+` OFFSET $`+itoa(n+1), args...)
if err != nil {
return nil, 0, err
}
defer rows.Close()
out := make([]OrderRow, 0)
for rows.Next() {
var row OrderRow
if err := rows.Scan(
&row.ID, &row.ExternalID, &row.Status, &row.Currency, &row.Total, &row.CustomerID,
&row.CustomerEmail, &row.CustomerName, &row.OrderedAt, &row.SyncedAt, &row.Payload,
); err != nil {
return nil, 0, err
}
out = append(out, row)
}
return out, total, rows.Err()
}
func (s *Service) ListReviews(ctx context.Context, companyID uuid.UUID, f ReviewListFilter) ([]ReviewRow, int, error) {
if f.Limit <= 0 {
f.Limit = 50
}
if f.Limit > 200 {
f.Limit = 200
}
if f.Offset < 0 {
f.Offset = 0
}
args := []any{companyID}
where := []string{"company_id = $1"}
n := 2
if status := strings.TrimSpace(f.Status); status != "" {
where = append(where, "status = $"+itoa(n))
args = append(args, status)
n++
}
if f.ProductID > 0 {
where = append(where, "product_id = $"+itoa(n))
args = append(args, f.ProductID)
n++
}
if f.MinRating > 0 {
where = append(where, "rating >= $"+itoa(n))
args = append(args, f.MinRating)
n++
}
whereSQL := strings.Join(where, " AND ")
var total int
if err := s.Pool.QueryRow(ctx, `SELECT count(*) FROM product_reviews WHERE `+whereSQL, args...).Scan(&total); err != nil {
return nil, 0, err
}
args = append(args, f.Limit, f.Offset)
rows, err := s.Pool.Query(ctx, `
SELECT id, external_id, product_id, product_name, status, reviewer, reviewer_email,
rating, review, reviewed_at, synced_at, payload
FROM product_reviews
WHERE `+whereSQL+`
ORDER BY reviewed_at DESC NULLS LAST, external_id DESC
LIMIT $`+itoa(n)+` OFFSET $`+itoa(n+1), args...)
if err != nil {
return nil, 0, err
}
defer rows.Close()
out := make([]ReviewRow, 0)
for rows.Next() {
var row ReviewRow
if err := rows.Scan(
&row.ID, &row.ExternalID, &row.ProductID, &row.ProductName, &row.Status, &row.Reviewer,
&row.ReviewerEmail, &row.Rating, &row.Review, &row.ReviewedAt, &row.SyncedAt, &row.Payload,
); err != nil {
return nil, 0, err
}
out = append(out, row)
}
return out, total, rows.Err()
}
// AudienceBoughtCategories returns distinct customers who bought category X
// and (optionally) did not buy category Y. Best-effort from synced orders:
// matches line-item categories JSON and/or local processed_products.category by SKU/product_id.
func (s *Service) AudienceBoughtCategories(ctx context.Context, companyID uuid.UUID, boughtCategory, notBoughtCategory string, limit int) (AudienceResult, error) {
boughtCategory = strings.TrimSpace(boughtCategory)
notBoughtCategory = strings.TrimSpace(notBoughtCategory)
if boughtCategory == "" {
return AudienceResult{Customers: []AudienceCustomer{}, Note: "bought_category is required"}, nil
}
if limit <= 0 {
limit = 500
}
if limit > 5000 {
limit = 5000
}
const note = "best-effort from synced Woo orders (line-item categories + processed_products by sku/product_id)"
rows, err := s.Pool.Query(ctx, `
WITH buyers AS (
SELECT DISTINCT lower(o.customer_email) AS email, COALESCE(o.customer_name, '') AS name
FROM woo_orders o
JOIN woo_order_items i ON i.order_id = o.id AND i.company_id = o.company_id
LEFT JOIN processed_products p
ON p.company_id = o.company_id
AND (
(i.sku <> '' AND p.product_id = i.sku)
OR (i.product_id IS NOT NULL AND p.product_id = i.product_id::text)
)
WHERE o.company_id = $1
AND o.customer_email IS NOT NULL AND o.customer_email <> ''
AND o.status IN ('completed', 'processing', 'on-hold')
AND (
EXISTS (
SELECT 1
FROM jsonb_array_elements(
CASE
WHEN jsonb_typeof(i.categories) = 'array' THEN i.categories
ELSE '[]'::jsonb
END
) cat(val)
WHERE lower(cat.val #>> '{}') = lower($2)
)
OR lower(COALESCE(p.category, '')) = lower($2)
)
),
excluded AS (
SELECT DISTINCT lower(o.customer_email) AS email
FROM woo_orders o
JOIN woo_order_items i ON i.order_id = o.id AND i.company_id = o.company_id
LEFT JOIN processed_products p
ON p.company_id = o.company_id
AND (
(i.sku <> '' AND p.product_id = i.sku)
OR (i.product_id IS NOT NULL AND p.product_id = i.product_id::text)
)
WHERE o.company_id = $1
AND $3 <> ''
AND o.customer_email IS NOT NULL AND o.customer_email <> ''
AND o.status IN ('completed', 'processing', 'on-hold')
AND (
EXISTS (
SELECT 1
FROM jsonb_array_elements(
CASE
WHEN jsonb_typeof(i.categories) = 'array' THEN i.categories
ELSE '[]'::jsonb
END
) cat(val)
WHERE lower(cat.val #>> '{}') = lower($3)
)
OR lower(COALESCE(p.category, '')) = lower($3)
)
)
SELECT b.email, b.name
FROM buyers b
WHERE NOT EXISTS (SELECT 1 FROM excluded e WHERE e.email = b.email)
ORDER BY b.email
LIMIT $4`, companyID, boughtCategory, notBoughtCategory, limit)
if err != nil {
return AudienceResult{}, err
}
defer rows.Close()
out := make([]AudienceCustomer, 0)
for rows.Next() {
var c AudienceCustomer
if err := rows.Scan(&c.Email, &c.Name); err != nil {
return AudienceResult{}, err
}
out = append(out, c)
}
if err := rows.Err(); err != nil {
return AudienceResult{}, err
}
return AudienceResult{Customers: out, Total: len(out), Note: note}, nil
}
// AudienceAnyOrdersExcept returns distinct customers with any qualifying Woo order,
// optionally excluding those who bought notBoughtCategory (best-effort category match).
func (s *Service) AudienceAnyOrdersExcept(ctx context.Context, companyID uuid.UUID, notBoughtCategory string, limit int) (AudienceResult, error) {
notBoughtCategory = strings.TrimSpace(notBoughtCategory)
if limit <= 0 {
limit = 500
}
if limit > 5000 {
limit = 5000
}
const note = "best-effort from synced Woo orders (any order, optional not_bought_category exclude)"
rows, err := s.Pool.Query(ctx, `
WITH buyers AS (
SELECT DISTINCT lower(o.customer_email) AS email, COALESCE(o.customer_name, '') AS name
FROM woo_orders o
WHERE o.company_id = $1
AND o.customer_email IS NOT NULL AND o.customer_email <> ''
AND o.status IN ('completed', 'processing', 'on-hold')
),
excluded AS (
SELECT DISTINCT lower(o.customer_email) AS email
FROM woo_orders o
JOIN woo_order_items i ON i.order_id = o.id AND i.company_id = o.company_id
LEFT JOIN processed_products p
ON p.company_id = o.company_id
AND (
(i.sku <> '' AND p.product_id = i.sku)
OR (i.product_id IS NOT NULL AND p.product_id = i.product_id::text)
)
WHERE o.company_id = $1
AND $2 <> ''
AND o.customer_email IS NOT NULL AND o.customer_email <> ''
AND o.status IN ('completed', 'processing', 'on-hold')
AND (
EXISTS (
SELECT 1
FROM jsonb_array_elements(
CASE
WHEN jsonb_typeof(i.categories) = 'array' THEN i.categories
ELSE '[]'::jsonb
END
) cat(val)
WHERE lower(cat.val #>> '{}') = lower($2)
)
OR lower(COALESCE(p.category, '')) = lower($2)
)
)
SELECT b.email, b.name
FROM buyers b
WHERE NOT EXISTS (SELECT 1 FROM excluded e WHERE e.email = b.email)
ORDER BY b.email
LIMIT $3`, companyID, notBoughtCategory, limit)
if err != nil {
return AudienceResult{}, err
}
defer rows.Close()
out := make([]AudienceCustomer, 0)
for rows.Next() {
var c AudienceCustomer
if err := rows.Scan(&c.Email, &c.Name); err != nil {
return AudienceResult{}, err
}
out = append(out, c)
}
if err := rows.Err(); err != nil {
return AudienceResult{}, err
}
return AudienceResult{Customers: out, Total: len(out), Note: note}, nil
}
func itoa(n int) string {
if n == 0 {
return "0"
}
const digits = "0123456789"
if n < 10 {
return digits[n : n+1]
}
var b [12]byte
i := len(b)
for n > 0 {
i--
b[i] = byte('0' + n%10)
n /= 10
}
return string(b[i:])
}
@@ -0,0 +1,151 @@
package woocommerce
import (
"context"
"encoding/json"
"strconv"
"time"
)
const (
defaultListPerPage = 50
maxListPerPage = 100
)
type OrderBilling struct {
Email string `json:"email"`
FirstName string `json:"first_name"`
LastName string `json:"last_name"`
}
type OrderLineItem struct {
ID int `json:"id"`
Name string `json:"name"`
ProductID int `json:"product_id"`
VariationID int `json:"variation_id"`
Quantity int `json:"quantity"`
Total string `json:"total"`
SKU string `json:"sku"`
MetaData json.RawMessage `json:"meta_data"`
}
type Order struct {
ID int `json:"id"`
Status string `json:"status"`
Currency string `json:"currency"`
Total string `json:"total"`
CustomerID int `json:"customer_id"`
DateCreated string `json:"date_created"`
DateCreatedGMT string `json:"date_created_gmt"`
Billing OrderBilling `json:"billing"`
LineItems []OrderLineItem `json:"line_items"`
Raw json.RawMessage `json:"-"`
}
type ProductReview struct {
ID int `json:"id"`
ProductID int `json:"product_id"`
Status string `json:"status"`
Reviewer string `json:"reviewer"`
ReviewerEmail string `json:"reviewer_email"`
Review string `json:"review"`
Rating int `json:"rating"`
DateCreated string `json:"date_created"`
DateCreatedGMT string `json:"date_created_gmt"`
ProductName string `json:"product_name"`
Raw json.RawMessage `json:"-"`
}
func clampPerPage(perPage int) int {
if perPage <= 0 {
return defaultListPerPage
}
if perPage > maxListPerPage {
return maxListPerPage
}
return perPage
}
// ListOrdersPage fetches one page of WooCommerce orders.
func (c *Client) ListOrdersPage(ctx context.Context, page, perPage int, modifiedAfter string) ([]Order, []byte, error) {
perPage = clampPerPage(perPage)
if page <= 0 {
page = 1
}
q := map[string]string{
"page": strconv.Itoa(page),
"per_page": strconv.Itoa(perPage),
"orderby": "date",
"order": "desc",
}
if modifiedAfter != "" {
q["modified_after"] = modifiedAfter
}
raw, err := c.do(ctx, "GET", "/orders", q, nil)
if err != nil {
return nil, nil, err
}
var out []Order
if err := json.Unmarshal(raw, &out); err != nil {
return nil, nil, err
}
// Keep full payload slices aligned with unmarshaled rows.
var rawItems []json.RawMessage
_ = json.Unmarshal(raw, &rawItems)
for i := range out {
if i < len(rawItems) {
out[i].Raw = rawItems[i]
}
}
return out, raw, nil
}
// ListProductReviewsPage fetches one page of WooCommerce product reviews.
func (c *Client) ListProductReviewsPage(ctx context.Context, page, perPage int) ([]ProductReview, []byte, error) {
perPage = clampPerPage(perPage)
if page <= 0 {
page = 1
}
q := map[string]string{
"page": strconv.Itoa(page),
"per_page": strconv.Itoa(perPage),
"orderby": "date",
"order": "desc",
}
raw, err := c.do(ctx, "GET", "/products/reviews", q, nil)
if err != nil {
return nil, nil, err
}
var out []ProductReview
if err := json.Unmarshal(raw, &out); err != nil {
return nil, nil, err
}
var rawItems []json.RawMessage
_ = json.Unmarshal(raw, &rawItems)
for i := range out {
if i < len(rawItems) {
out[i].Raw = rawItems[i]
}
}
return out, raw, nil
}
func parseWooTime(vals ...string) *time.Time {
layouts := []string{
time.RFC3339,
"2006-01-02T15:04:05",
"2006-01-02 15:04:05",
}
for _, v := range vals {
if v == "" {
continue
}
for _, layout := range layouts {
if t, err := time.Parse(layout, v); err == nil {
u := t.UTC()
return &u
}
}
}
return nil
}
@@ -0,0 +1,36 @@
package woocommerce
import (
"encoding/json"
"testing"
)
func TestExtractItemCategories(t *testing.T) {
meta, _ := json.Marshal([]map[string]any{
{"key": "categories", "value": []any{"Shoes", "Men"}},
{"key": "foo", "value": "bar"},
})
item := OrderLineItem{MetaData: meta}
got := extractItemCategories(item)
if len(got) != 2 {
t.Fatalf("expected 2 categories, got %#v", got)
}
}
func TestParseWooTime(t *testing.T) {
if parseWooTime("2024-01-02T03:04:05") == nil {
t.Fatal("expected parsed time")
}
if parseWooTime("") != nil {
t.Fatal("expected nil")
}
}
func TestClampPerPage(t *testing.T) {
if clampPerPage(0) != defaultListPerPage {
t.Fatal("default")
}
if clampPerPage(500) != maxListPerPage {
t.Fatal("max")
}
}
@@ -0,0 +1,354 @@
package woocommerce
import (
"context"
"errors"
"encoding/json"
"fmt"
"math/big"
"strings"
"time"
"github.com/google/uuid"
"github.com/jackc/pgx/v5"
)
const (
defaultOrdersSyncLimit = 500
defaultReviewsSyncLimit = 500
defaultPullPageSize = 50
)
type OrdersSyncSummary struct {
Pages int `json:"pages"`
Fetched int `json:"fetched"`
Upserted int `json:"upserted"`
ItemsSaved int `json:"items_saved"`
Failed int `json:"failed"`
}
type ReviewsSyncSummary struct {
Pages int `json:"pages"`
Fetched int `json:"fetched"`
Upserted int `json:"upserted"`
Failed int `json:"failed"`
}
func (s *Service) EnqueueOrdersSync(ctx context.Context, companyID uuid.UUID) (map[string]any, error) {
if err := s.requireEnabledCreds(ctx, companyID); err != nil {
return nil, err
}
sc, err := s.loadStored(ctx, companyID)
if err != nil {
return nil, err
}
opt := parseSyncOptions(sc.syncOptions)
opt.PendingOrdersSync = true
if err := s.saveSyncOptions(ctx, companyID, opt); err != nil {
return nil, err
}
return map[string]any{"status": "accepted", "message": "woocommerce orders sync queued"}, nil
}
func (s *Service) EnqueueReviewsSync(ctx context.Context, companyID uuid.UUID) (map[string]any, error) {
if err := s.requireEnabledCreds(ctx, companyID); err != nil {
return nil, err
}
sc, err := s.loadStored(ctx, companyID)
if err != nil {
return nil, err
}
opt := parseSyncOptions(sc.syncOptions)
opt.PendingReviewsSync = true
if err := s.saveSyncOptions(ctx, companyID, opt); err != nil {
return nil, err
}
return map[string]any{"status": "accepted", "message": "woocommerce reviews sync queued"}, nil
}
func (s *Service) requireEnabledCreds(ctx context.Context, companyID uuid.UUID) error {
sc, err := s.loadStored(ctx, companyID)
if errors.Is(err, pgx.ErrNoRows) {
return ErrNotConfigured
}
if err != nil {
return err
}
if !sc.enabled {
return ErrNotEnabled
}
key, err := DecryptSecret(s.Key, sc.keyEnc)
if err != nil {
return err
}
secret, err := DecryptSecret(s.Key, sc.secretEnc)
if err != nil {
return err
}
if sc.storeURL == "" || key == "" || secret == "" {
return ErrMissingCreds
}
return nil
}
func (s *Service) saveSyncOptions(ctx context.Context, companyID uuid.UUID, opt SyncOptions) error {
pruneStringIntMap(opt.ProductIDs, maxProductIDMap)
raw, err := json.Marshal(opt)
if err != nil {
return err
}
_, err = s.Pool.Exec(ctx, `
UPDATE woocommerce_configs SET sync_options = $2, updated_at = now()
WHERE company_id = $1`, companyID, raw)
return err
}
// ClaimNextPendingJob claims the next pending Woo sync (products, orders, or reviews).
func (s *Service) ClaimNextPendingJob(ctx context.Context) (uuid.UUID, string, error) {
var companyID uuid.UUID
var kind string
err := s.Pool.QueryRow(ctx, `
WITH candidate AS (
SELECT company_id,
CASE
WHEN COALESCE(sync_options->>'pending_sync', 'false') = 'true' THEN 'products'
WHEN COALESCE(sync_options->>'pending_orders_sync', 'false') = 'true' THEN 'orders'
WHEN COALESCE(sync_options->>'pending_reviews_sync', 'false') = 'true' THEN 'reviews'
ELSE ''
END AS kind
FROM woocommerce_configs
WHERE is_enabled = true
AND (
COALESCE(sync_options->>'pending_sync', 'false') = 'true'
OR COALESCE(sync_options->>'pending_orders_sync', 'false') = 'true'
OR COALESCE(sync_options->>'pending_reviews_sync', 'false') = 'true'
)
ORDER BY updated_at ASC
LIMIT 1
FOR UPDATE SKIP LOCKED
)
UPDATE woocommerce_configs c
SET sync_options = CASE candidate.kind
WHEN 'products' THEN jsonb_set(COALESCE(c.sync_options, '{}'::jsonb), '{pending_sync}', 'false'::jsonb, true)
WHEN 'orders' THEN jsonb_set(COALESCE(c.sync_options, '{}'::jsonb), '{pending_orders_sync}', 'false'::jsonb, true)
WHEN 'reviews' THEN jsonb_set(COALESCE(c.sync_options, '{}'::jsonb), '{pending_reviews_sync}', 'false'::jsonb, true)
ELSE c.sync_options
END,
updated_at = now()
FROM candidate
WHERE c.company_id = candidate.company_id AND candidate.kind <> ''
RETURNING c.company_id, candidate.kind`).Scan(&companyID, &kind)
return companyID, kind, err
}
// SyncOrders pulls Woo orders in pages and upserts company-scoped rows.
func (s *Service) SyncOrders(ctx context.Context, companyID uuid.UUID) (OrdersSyncSummary, error) {
client, _, opt, err := s.clientFor(ctx, companyID)
if err != nil {
return OrdersSyncSummary{}, err
}
limit := opt.OrdersSyncLimit
if limit <= 0 || limit > maxOrdersSyncLimit {
limit = defaultOrdersSyncLimit
}
pageSize := defaultPullPageSize
summary := OrdersSyncSummary{}
after := strings.TrimSpace(opt.OrdersModifiedAfter)
for page := 1; summary.Fetched < limit; page++ {
remaining := limit - summary.Fetched
perPage := pageSize
if remaining < perPage {
perPage = remaining
}
orders, _, err := client.ListOrdersPage(ctx, page, perPage, after)
if err != nil {
opt.PendingOrdersSync = false
opt.LastOrdersSyncStatus = "failed"
opt.LastOrdersSyncError = truncateErr(err)
_ = s.saveSyncOptions(ctx, companyID, opt)
return summary, err
}
if len(orders) == 0 {
break
}
summary.Pages++
summary.Fetched += len(orders)
for _, order := range orders {
nItems, err := s.upsertOrder(ctx, companyID, order)
if err != nil {
summary.Failed++
continue
}
summary.Upserted++
summary.ItemsSaved += nItems
}
if len(orders) < perPage {
break
}
}
opt.PendingOrdersSync = false
if summary.Failed > 0 && summary.Upserted == 0 {
opt.LastOrdersSyncStatus = "failed"
opt.LastOrdersSyncError = "all order upserts failed"
} else if summary.Failed > 0 {
opt.LastOrdersSyncStatus = "partial"
opt.LastOrdersSyncError = "some order upserts failed"
} else {
opt.LastOrdersSyncStatus = "success"
opt.LastOrdersSyncError = ""
}
now := timeNowUTC()
opt.LastOrdersSyncedAt = &now
if err := s.saveSyncOptions(ctx, companyID, opt); err != nil {
return summary, err
}
return summary, nil
}
func (s *Service) upsertOrder(ctx context.Context, companyID uuid.UUID, order Order) (int, error) {
payload := order.Raw
if len(payload) == 0 {
b, err := json.Marshal(order)
if err != nil {
return 0, err
}
payload = b
}
email := strings.TrimSpace(strings.ToLower(order.Billing.Email))
name := strings.TrimSpace(strings.TrimSpace(order.Billing.FirstName + " " + order.Billing.LastName))
orderedAt := parseWooTime(order.DateCreatedGMT, order.DateCreated)
total := parseDecimal(order.Total)
tx, err := s.Pool.Begin(ctx)
if err != nil {
return 0, err
}
defer tx.Rollback(ctx)
var orderID uuid.UUID
err = tx.QueryRow(ctx, `
INSERT INTO woo_orders (
company_id, external_id, status, currency, total, customer_id, customer_email, customer_name,
ordered_at, payload, synced_at, updated_at
) VALUES (
$1, $2, $3, $4, $5, NULLIF($6, 0), NULLIF($7, ''), NULLIF($8, ''),
$9, $10::jsonb, now(), now()
)
ON CONFLICT (company_id, external_id) DO UPDATE SET
status = EXCLUDED.status,
currency = EXCLUDED.currency,
total = EXCLUDED.total,
customer_id = EXCLUDED.customer_id,
customer_email = EXCLUDED.customer_email,
customer_name = EXCLUDED.customer_name,
ordered_at = EXCLUDED.ordered_at,
payload = EXCLUDED.payload,
synced_at = now(),
updated_at = now()
RETURNING id`,
companyID, order.ID, order.Status, order.Currency, total, order.CustomerID, email, name, orderedAt, payload,
).Scan(&orderID)
if err != nil {
return 0, err
}
_, err = tx.Exec(ctx, `DELETE FROM woo_order_items WHERE company_id = $1 AND order_id = $2`, companyID, orderID)
if err != nil {
return 0, err
}
saved := 0
for _, item := range order.LineItems {
itemPayload, _ := json.Marshal(item)
cats := extractItemCategories(item)
catsRaw, _ := json.Marshal(cats)
itemTotal := parseDecimal(item.Total)
_, err := tx.Exec(ctx, `
INSERT INTO woo_order_items (
company_id, order_id, external_id, product_id, variation_id, sku, name, quantity, total, categories, payload, updated_at
) VALUES (
$1, $2, $3, NULLIF($4, 0), NULLIF($5, 0), $6, $7, $8, $9, $10::jsonb, $11::jsonb, now()
)
ON CONFLICT (company_id, order_id, external_id) DO UPDATE SET
product_id = EXCLUDED.product_id,
variation_id = EXCLUDED.variation_id,
sku = EXCLUDED.sku,
name = EXCLUDED.name,
quantity = EXCLUDED.quantity,
total = EXCLUDED.total,
categories = EXCLUDED.categories,
payload = EXCLUDED.payload,
updated_at = now()`,
companyID, orderID, item.ID, item.ProductID, item.VariationID, strings.TrimSpace(item.SKU),
strings.TrimSpace(item.Name), item.Quantity, itemTotal, catsRaw, itemPayload,
)
if err != nil {
return 0, err
}
saved++
}
if err := tx.Commit(ctx); err != nil {
return 0, err
}
return saved, nil
}
func extractItemCategories(item OrderLineItem) []any {
out := make([]any, 0)
if len(item.MetaData) == 0 {
return out
}
var meta []map[string]any
if json.Unmarshal(item.MetaData, &meta) != nil {
return out
}
for _, m := range meta {
key := strings.ToLower(fmt.Sprint(m["key"]))
if key != "categories" && key != "_categories" && key != "category" && !strings.Contains(key, "categor") {
continue
}
switch v := m["value"].(type) {
case string:
if strings.TrimSpace(v) != "" {
out = append(out, strings.TrimSpace(v))
}
case []any:
out = append(out, v...)
case map[string]any:
if name, ok := v["name"].(string); ok && name != "" {
out = append(out, name)
}
if id, ok := v["id"]; ok {
out = append(out, id)
}
}
}
return out
}
func parseDecimal(s string) *string {
s = strings.TrimSpace(s)
if s == "" {
return nil
}
if _, _, err := big.ParseFloat(s, 10, 64, big.ToNearestEven); err != nil {
return nil
}
return &s
}
func truncateErr(err error) string {
if err == nil {
return ""
}
msg := err.Error()
if len(msg) > 400 {
return msg[:400] + "…"
}
return msg
}
func timeNowUTC() time.Time { return time.Now().UTC() }
@@ -0,0 +1,118 @@
package woocommerce
import (
"context"
"encoding/json"
"strings"
"time"
"github.com/google/uuid"
)
// SyncReviews pulls Woo product reviews in pages and upserts company-scoped rows.
func (s *Service) SyncReviews(ctx context.Context, companyID uuid.UUID) (ReviewsSyncSummary, error) {
client, _, opt, err := s.clientFor(ctx, companyID)
if err != nil {
return ReviewsSyncSummary{}, err
}
limit := opt.ReviewsSyncLimit
if limit <= 0 || limit > maxReviewsSyncLimit {
limit = defaultReviewsSyncLimit
}
pageSize := defaultPullPageSize
summary := ReviewsSyncSummary{}
for page := 1; summary.Fetched < limit; page++ {
remaining := limit - summary.Fetched
perPage := pageSize
if remaining < perPage {
perPage = remaining
}
reviews, _, err := client.ListProductReviewsPage(ctx, page, perPage)
if err != nil {
opt.PendingReviewsSync = false
opt.LastReviewsSyncStatus = "failed"
opt.LastReviewsSyncError = truncateErr(err)
_ = s.saveSyncOptions(ctx, companyID, opt)
return summary, err
}
if len(reviews) == 0 {
break
}
summary.Pages++
summary.Fetched += len(reviews)
for _, review := range reviews {
if err := s.upsertReview(ctx, companyID, review); err != nil {
summary.Failed++
continue
}
summary.Upserted++
}
if len(reviews) < perPage {
break
}
}
opt.PendingReviewsSync = false
if summary.Failed > 0 && summary.Upserted == 0 {
opt.LastReviewsSyncStatus = "failed"
opt.LastReviewsSyncError = "all review upserts failed"
} else if summary.Failed > 0 {
opt.LastReviewsSyncStatus = "partial"
opt.LastReviewsSyncError = "some review upserts failed"
} else {
opt.LastReviewsSyncStatus = "success"
opt.LastReviewsSyncError = ""
}
now := time.Now().UTC()
opt.LastReviewsSyncedAt = &now
if err := s.saveSyncOptions(ctx, companyID, opt); err != nil {
return summary, err
}
return summary, nil
}
func (s *Service) upsertReview(ctx context.Context, companyID uuid.UUID, review ProductReview) error {
payload := review.Raw
if len(payload) == 0 {
b, err := json.Marshal(review)
if err != nil {
return err
}
payload = b
}
reviewedAt := parseWooTime(review.DateCreatedGMT, review.DateCreated)
email := strings.TrimSpace(strings.ToLower(review.ReviewerEmail))
_, err := s.Pool.Exec(ctx, `
INSERT INTO product_reviews (
company_id, external_id, product_id, product_name, status, reviewer, reviewer_email,
rating, review, reviewed_at, payload, synced_at, updated_at
) VALUES (
$1, $2, NULLIF($3, 0), $4, $5, $6, $7,
$8, $9, $10, $11::jsonb, now(), now()
)
ON CONFLICT (company_id, external_id) DO UPDATE SET
product_id = EXCLUDED.product_id,
product_name = EXCLUDED.product_name,
status = EXCLUDED.status,
reviewer = EXCLUDED.reviewer,
reviewer_email = EXCLUDED.reviewer_email,
rating = EXCLUDED.rating,
review = EXCLUDED.review,
reviewed_at = EXCLUDED.reviewed_at,
payload = EXCLUDED.payload,
synced_at = now(),
updated_at = now()`,
companyID, review.ID, review.ProductID, strings.TrimSpace(review.ProductName),
strings.TrimSpace(review.Status), strings.TrimSpace(review.Reviewer), email,
nullableInt(review.Rating), strings.TrimSpace(review.Review), reviewedAt, payload,
)
return err
}
func nullableInt(v int) *int {
if v == 0 {
return nil
}
return &v
}
+449
View File
@@ -0,0 +1,449 @@
package woocommerce
import (
"context"
"encoding/json"
"errors"
"net/http"
"strings"
"time"
"github.com/descrybe/descrybe-v2/apps/api/internal/security"
"github.com/google/uuid"
"github.com/jackc/pgx/v5"
"github.com/jackc/pgx/v5/pgxpool"
)
var (
ErrNotConfigured = errors.New("woocommerce not configured")
ErrNotEnabled = errors.New("woocommerce sync is disabled")
ErrMissingCreds = errors.New("woocommerce credentials missing")
ErrInvalidSyncScope = errors.New("invalid product sync scope")
ErrInvalidScheduleInterval = errors.New("schedule_interval_hours must be between 0 and 168")
)
type Service struct {
Pool *pgxpool.Pool
Key []byte
HTTPClient *http.Client
}
type Config struct {
StoreURL string `json:"store_url"`
IsEnabled bool `json:"is_enabled"`
Configured bool `json:"configured"`
LastSyncedAt *time.Time `json:"last_synced_at,omitempty"`
LastTestAt *time.Time `json:"last_test_at,omitempty"`
LastTestStatus *string `json:"last_test_status,omitempty"`
HasCredentials bool `json:"has_credentials"`
PendingSync bool `json:"pending_sync"`
PendingOrdersSync bool `json:"pending_orders_sync"`
PendingReviewsSync bool `json:"pending_reviews_sync"`
MatchStrategy string `json:"match_strategy"`
LastSyncStatus string `json:"last_sync_status,omitempty"`
LastSyncError string `json:"last_sync_error,omitempty"`
LastSyncSummary *SyncSummary `json:"last_sync_summary,omitempty"`
ProductMapCount int `json:"product_map_count"`
SyncLimit int `json:"sync_limit,omitempty"`
LastOrdersSyncedAt *time.Time `json:"last_orders_synced_at,omitempty"`
LastOrdersSyncStatus string `json:"last_orders_sync_status,omitempty"`
LastOrdersSyncError string `json:"last_orders_sync_error,omitempty"`
LastReviewsSyncedAt *time.Time `json:"last_reviews_synced_at,omitempty"`
LastReviewsSyncStatus string `json:"last_reviews_sync_status,omitempty"`
LastReviewsSyncError string `json:"last_reviews_sync_error,omitempty"`
CategoryMaps int `json:"category_map_count"`
AttributeMaps int `json:"attribute_map_count"`
ScheduleIntervalHours int `json:"schedule_interval_hours"`
SchedulePaused bool `json:"schedule_paused"`
}
func NewService(pool *pgxpool.Pool, key []byte) *Service {
return &Service{
Pool: pool,
Key: key,
// Dial-time SSRF; loopback allowed for local mock Woo only (NormalizeStoreURL
// already requires https except localhost).
HTTPClient: security.SafeHTTPClient(defaultTimeout, true),
}
}
type storedConfig struct {
storeURL, keyEnc, secretEnc string
enabled bool
syncOptions []byte
lastSync, lastTest *time.Time
status *string
}
func (s *Service) loadStored(ctx context.Context, companyID uuid.UUID) (storedConfig, error) {
var sc storedConfig
err := s.Pool.QueryRow(ctx, `
SELECT store_url, consumer_key, consumer_secret, is_enabled, sync_options, last_synced_at, last_test_at, last_test_status
FROM woocommerce_configs WHERE company_id = $1`, companyID).Scan(
&sc.storeURL, &sc.keyEnc, &sc.secretEnc, &sc.enabled, &sc.syncOptions, &sc.lastSync, &sc.lastTest, &sc.status,
)
return sc, err
}
func (s *Service) GetConfig(ctx context.Context, companyID uuid.UUID) (Config, error) {
sc, err := s.loadStored(ctx, companyID)
if errors.Is(err, pgx.ErrNoRows) {
return Config{}, err
}
if err != nil {
return Config{}, err
}
opt := parseSyncOptions(sc.syncOptions)
key, _ := DecryptSecret(s.Key, sc.keyEnc)
secret, _ := DecryptSecret(s.Key, sc.secretEnc)
return Config{
StoreURL: sc.storeURL,
IsEnabled: sc.enabled,
Configured: true,
LastSyncedAt: sc.lastSync,
LastTestAt: sc.lastTest,
LastTestStatus: sc.status,
HasCredentials: key != "" && secret != "",
PendingSync: opt.PendingSync,
PendingOrdersSync: opt.PendingOrdersSync,
PendingReviewsSync: opt.PendingReviewsSync,
MatchStrategy: opt.MatchStrategy,
LastSyncStatus: opt.LastSyncStatus,
LastSyncError: opt.LastSyncError,
LastSyncSummary: opt.LastSyncSummary,
ProductMapCount: len(opt.ProductIDs),
SyncLimit: opt.SyncLimit,
LastOrdersSyncedAt: opt.LastOrdersSyncedAt,
LastOrdersSyncStatus: opt.LastOrdersSyncStatus,
LastOrdersSyncError: opt.LastOrdersSyncError,
LastReviewsSyncedAt: opt.LastReviewsSyncedAt,
LastReviewsSyncStatus: opt.LastReviewsSyncStatus,
LastReviewsSyncError: opt.LastReviewsSyncError,
CategoryMaps: len(opt.CategoryMappings),
AttributeMaps: len(opt.AttributeMappings),
ScheduleIntervalHours: opt.ScheduleIntervalHours,
SchedulePaused: opt.SchedulePaused,
}, nil
}
func (s *Service) UpdateConfig(ctx context.Context, companyID uuid.UUID, storeURL, key, secret string, enabled bool) (Config, error) {
normalized, err := NormalizeStoreURL(storeURL)
if err != nil {
return Config{}, err
}
keyEnc := ""
secretEnc := ""
if strings.TrimSpace(key) != "" {
keyEnc, err = EncryptSecret(s.Key, strings.TrimSpace(key))
if err != nil {
return Config{}, err
}
}
if strings.TrimSpace(secret) != "" {
secretEnc, err = EncryptSecret(s.Key, strings.TrimSpace(secret))
if err != nil {
return Config{}, err
}
}
_, err = s.Pool.Exec(ctx, `
INSERT INTO woocommerce_configs (company_id, store_url, consumer_key, consumer_secret, is_enabled, updated_at)
VALUES ($1, $2, $3, $4, $5, now())
ON CONFLICT (company_id) DO UPDATE SET
store_url = EXCLUDED.store_url,
consumer_key = CASE WHEN EXCLUDED.consumer_key <> '' THEN EXCLUDED.consumer_key ELSE woocommerce_configs.consumer_key END,
consumer_secret = CASE WHEN EXCLUDED.consumer_secret <> '' THEN EXCLUDED.consumer_secret ELSE woocommerce_configs.consumer_secret END,
is_enabled = EXCLUDED.is_enabled,
updated_at = now()`,
companyID, normalized, keyEnc, secretEnc, enabled)
if err != nil {
return Config{}, err
}
return s.GetConfig(ctx, companyID)
}
func (s *Service) UpdateMaps(ctx context.Context, companyID uuid.UUID, categoryMaps map[string]CategoryMap, attributeMaps map[string]AttributeMap, matchStrategy string) (Config, error) {
sc, err := s.loadStored(ctx, companyID)
if errors.Is(err, pgx.ErrNoRows) {
return Config{}, ErrNotConfigured
}
if err != nil {
return Config{}, err
}
opt := parseSyncOptions(sc.syncOptions)
if categoryMaps != nil {
opt.CategoryMappings = categoryMaps
}
if attributeMaps != nil {
opt.AttributeMappings = attributeMaps
}
if matchStrategy != "" {
opt.MatchStrategy = matchStrategy
}
raw, err := json.Marshal(opt)
if err != nil {
return Config{}, err
}
_, err = s.Pool.Exec(ctx, `
UPDATE woocommerce_configs SET sync_options = $2, updated_at = now() WHERE company_id = $1`,
companyID, raw)
if err != nil {
return Config{}, err
}
return s.GetConfig(ctx, companyID)
}
// UpdateSchedule sets auto product-sync interval hours and pause flag on sync_options.
// hours 0 means the worker default (6h). Manual sync remains available when paused.
func (s *Service) UpdateSchedule(ctx context.Context, companyID uuid.UUID, hours int, paused bool) (Config, error) {
if hours < 0 || hours > maxScheduleIntervalH {
return Config{}, ErrInvalidScheduleInterval
}
sc, err := s.loadStored(ctx, companyID)
if errors.Is(err, pgx.ErrNoRows) {
return Config{}, ErrNotConfigured
}
if err != nil {
return Config{}, err
}
opt := parseSyncOptions(sc.syncOptions)
opt.ScheduleIntervalHours = hours
opt.SchedulePaused = paused
raw, err := json.Marshal(opt)
if err != nil {
return Config{}, err
}
_, err = s.Pool.Exec(ctx, `
UPDATE woocommerce_configs SET sync_options = $2, updated_at = now() WHERE company_id = $1`,
companyID, raw)
if err != nil {
return Config{}, err
}
return s.GetConfig(ctx, companyID)
}
func (s *Service) clientFor(ctx context.Context, companyID uuid.UUID) (*Client, storedConfig, SyncOptions, error) {
sc, err := s.loadStored(ctx, companyID)
if errors.Is(err, pgx.ErrNoRows) {
return nil, sc, SyncOptions{}, ErrNotConfigured
}
if err != nil {
return nil, sc, SyncOptions{}, err
}
key, err := DecryptSecret(s.Key, sc.keyEnc)
if err != nil {
return nil, sc, SyncOptions{}, err
}
secret, err := DecryptSecret(s.Key, sc.secretEnc)
if err != nil {
return nil, sc, SyncOptions{}, err
}
if sc.storeURL == "" || key == "" || secret == "" {
return nil, sc, SyncOptions{}, ErrMissingCreds
}
opt := parseSyncOptions(sc.syncOptions)
return NewClient(sc.storeURL, key, secret, s.HTTPClient), sc, opt, nil
}
func (s *Service) TestConnection(ctx context.Context, companyID uuid.UUID) (map[string]any, error) {
client, _, _, err := s.clientFor(ctx, companyID)
status := "ok"
message := "connection successful"
if err != nil {
status = "failed"
message = err.Error()
_, _ = s.Pool.Exec(ctx, `
UPDATE woocommerce_configs SET last_test_at = now(), last_test_status = $2, updated_at = now()
WHERE company_id = $1`, companyID, status)
return map[string]any{"status": status, "message": message}, err
}
if err := client.TestConnection(ctx); err != nil {
status = "failed"
message = "connection failed"
_, execErr := s.Pool.Exec(ctx, `
UPDATE woocommerce_configs SET last_test_at = now(), last_test_status = $2, updated_at = now()
WHERE company_id = $1`, companyID, status)
if execErr != nil {
return map[string]any{"status": status, "message": message}, execErr
}
return map[string]any{"status": status, "message": message}, err
}
if _, err := s.Pool.Exec(ctx, `
UPDATE woocommerce_configs SET last_test_at = now(), last_test_status = $2, updated_at = now()
WHERE company_id = $1`, companyID, status); err != nil {
return map[string]any{"status": status, "message": message}, err
}
return map[string]any{"status": status, "message": message}, nil
}
// EnqueueSync marks the company for worker pickup (idempotent).
// Optional scopes[0] selects which products to push (status/category/limit/ids).
func (s *Service) EnqueueSync(ctx context.Context, companyID uuid.UUID, scopes ...ProductSyncScope) (map[string]any, error) {
sc, err := s.loadStored(ctx, companyID)
if errors.Is(err, pgx.ErrNoRows) {
return nil, ErrNotConfigured
}
if err != nil {
return nil, err
}
if !sc.enabled {
return nil, ErrNotEnabled
}
key, err := DecryptSecret(s.Key, sc.keyEnc)
if err != nil {
return nil, err
}
secret, err := DecryptSecret(s.Key, sc.secretEnc)
if err != nil {
return nil, err
}
if sc.storeURL == "" || key == "" || secret == "" {
return nil, ErrMissingCreds
}
opt := parseSyncOptions(sc.syncOptions)
if len(scopes) > 0 {
if err := applyProductSyncScope(&opt, scopes[0]); err != nil {
return nil, err
}
} else {
clearOneShotSyncFilters(&opt)
}
opt.PendingSync = true
raw, err := json.Marshal(opt)
if err != nil {
return nil, err
}
ct, err := s.Pool.Exec(ctx, `
UPDATE woocommerce_configs SET sync_options = $2, updated_at = now()
WHERE company_id = $1 AND is_enabled = true`, companyID, raw)
if err != nil {
return nil, err
}
if ct.RowsAffected() == 0 {
return nil, ErrNotEnabled
}
return map[string]any{
"status": "accepted",
"message": "woocommerce sync queued",
}, nil
}
// ClaimNextPending claims one enabled company with pending product sync for the worker.
func (s *Service) ClaimNextPending(ctx context.Context) (uuid.UUID, error) {
var companyID uuid.UUID
err := s.Pool.QueryRow(ctx, `
UPDATE woocommerce_configs c
SET sync_options = jsonb_set(COALESCE(c.sync_options, '{}'::jsonb), '{pending_sync}', 'false'::jsonb, true),
updated_at = now()
WHERE c.company_id = (
SELECT company_id FROM woocommerce_configs
WHERE is_enabled = true
AND COALESCE(sync_options->>'pending_sync', 'false') = 'true'
ORDER BY updated_at ASC
LIMIT 1
FOR UPDATE SKIP LOCKED
)
RETURNING c.company_id`).Scan(&companyID)
return companyID, err
}
// EnqueueDueScheduled marks enabled Woo configs as pending when last_synced_at is older
// than schedule_interval_hours (or defaultInterval). Safe/idempotent; worker ClaimNextPending drains.
func (s *Service) EnqueueDueScheduled(ctx context.Context, defaultInterval time.Duration) (int, error) {
rows, err := s.Pool.Query(ctx, `
SELECT company_id, sync_options, last_synced_at
FROM woocommerce_configs
WHERE is_enabled = true
AND COALESCE(sync_options->>'pending_sync', 'false') <> 'true'`)
if err != nil {
return 0, err
}
defer rows.Close()
n := 0
now := time.Now().UTC()
for rows.Next() {
var companyID uuid.UUID
var raw []byte
var last *time.Time
if err := rows.Scan(&companyID, &raw, &last); err != nil {
return n, err
}
opt := parseSyncOptions(raw)
if !shouldEnqueueScheduled(opt.SchedulePaused, opt.ScheduleIntervalHours, last, now, defaultInterval) {
continue
}
if _, err := s.EnqueueSync(ctx, companyID); err != nil {
continue
}
n++
}
return n, rows.Err()
}
// SyncCompany pushes processed products to WooCommerce in batches.
func (s *Service) SyncCompany(ctx context.Context, companyID uuid.UUID) (SyncSummary, error) {
client, _, opt, err := s.clientFor(ctx, companyID)
if err != nil {
return SyncSummary{}, err
}
rows, err := s.loadProductsForSync(ctx, companyID, opt)
if err != nil {
return SyncSummary{}, err
}
summary, err := s.pushProducts(ctx, client, companyID, &opt, rows)
if err != nil {
return summary, err
}
if summary.Failed == 0 {
opt.LastSyncStatus = "success"
opt.LastSyncError = ""
} else if summary.Created+summary.Updated == 0 {
opt.LastSyncStatus = "failed"
opt.LastSyncError = "all product pushes failed"
} else {
opt.LastSyncStatus = "partial"
opt.LastSyncError = "some product pushes failed"
}
sumCopy := summary
opt.LastSyncSummary = &sumCopy
opt.PendingSync = false
clearOneShotSyncFilters(&opt)
pruneStringIntMap(opt.ProductIDs, maxProductIDMap)
raw, err := json.Marshal(opt)
if err != nil {
return summary, err
}
_, err = s.Pool.Exec(ctx, `
UPDATE woocommerce_configs
SET sync_options = $2, last_synced_at = now(), updated_at = now()
WHERE company_id = $1`, companyID, raw)
if err != nil {
return summary, err
}
return summary, nil
}
// FetchRemoteMaps loads categories/attributes from Woo for mapping UI helpers.
func (s *Service) FetchRemoteMaps(ctx context.Context, companyID uuid.UUID) (map[string]any, error) {
client, _, _, err := s.clientFor(ctx, companyID)
if err != nil {
return nil, err
}
cats, err := client.ListCategories(ctx)
if err != nil {
return nil, err
}
attrs, err := client.ListAttributes(ctx)
if err != nil {
return nil, err
}
return map[string]any{
"categories": cats,
"attributes": attrs,
}, nil
}
// SyncStub kept as deprecated alias for EnqueueSync during transition.
func (s *Service) SyncStub(ctx context.Context, companyID uuid.UUID) (map[string]any, error) {
return s.EnqueueSync(ctx, companyID)
}
+513
View File
@@ -0,0 +1,513 @@
package woocommerce
import (
"context"
"encoding/json"
"fmt"
"strconv"
"strings"
"time"
"github.com/google/uuid"
)
const (
defaultBatchSize = 25
defaultSyncLimit = 200
maxBatchSize = 100
maxSyncLimit = 1000
maxOrdersSyncLimit = 5000
maxReviewsSyncLimit = 5000
maxProductIDMap = 5000
maxScheduleIntervalH = 168 // 7 days
metaDescrybeID = "_descrybe_product_id"
)
type SyncOptions struct {
CategoryMappings map[string]CategoryMap `json:"category_mappings"`
AttributeMappings map[string]AttributeMap `json:"attribute_mappings"`
ProductIDs map[string]int `json:"product_ids"`
MatchStrategy string `json:"match_strategy"`
BatchSize int `json:"batch_size"`
SyncLimit int `json:"sync_limit"`
PendingSync bool `json:"pending_sync"`
PendingOrdersSync bool `json:"pending_orders_sync"`
PendingReviewsSync bool `json:"pending_reviews_sync"`
LastSyncStatus string `json:"last_sync_status"`
LastSyncError string `json:"last_sync_error"`
LastSyncSummary *SyncSummary `json:"last_sync_summary,omitempty"`
LastOrdersSyncStatus string `json:"last_orders_sync_status"`
LastOrdersSyncError string `json:"last_orders_sync_error"`
LastOrdersSyncedAt *time.Time `json:"last_orders_synced_at,omitempty"`
LastReviewsSyncStatus string `json:"last_reviews_sync_status"`
LastReviewsSyncError string `json:"last_reviews_sync_error"`
LastReviewsSyncedAt *time.Time `json:"last_reviews_synced_at,omitempty"`
OrdersSyncLimit int `json:"orders_sync_limit"`
ReviewsSyncLimit int `json:"reviews_sync_limit"`
OrdersModifiedAfter string `json:"orders_modified_after"`
ScheduleIntervalHours int `json:"schedule_interval_hours"`
SchedulePaused bool `json:"schedule_paused"`
// One-shot selection for the next product sync (cleared when sync finishes).
SyncFilterStatus string `json:"sync_filter_status,omitempty"`
SyncFilterCategory string `json:"sync_filter_category,omitempty"`
SyncOnlyIDs []string `json:"sync_only_ids,omitempty"`
}
type CategoryMap struct {
Name string `json:"name"`
Slug string `json:"slug"`
WCID int `json:"wc_id"`
}
type AttributeMap struct {
Name string `json:"name"`
WCID int `json:"wc_id"`
}
type SyncSummary struct {
Total int `json:"total"`
Created int `json:"created"`
Updated int `json:"updated"`
Failed int `json:"failed"`
Skipped int `json:"skipped"`
}
type syncProductRow struct {
ID uuid.UUID
ProductID *string
Name *string
Category *string
Description *string
ProcessedName *string
ProcessedDescription *string
MetaDescription *string
Attributes []byte
ProcessedAttributes []byte
GTIN *string
MappedData []byte
}
func parseSyncOptions(raw []byte) SyncOptions {
opt := SyncOptions{
CategoryMappings: map[string]CategoryMap{},
AttributeMappings: map[string]AttributeMap{},
ProductIDs: map[string]int{},
MatchStrategy: "sku",
BatchSize: defaultBatchSize,
SyncLimit: defaultSyncLimit,
}
if len(raw) == 0 {
return opt
}
_ = json.Unmarshal(raw, &opt)
if opt.CategoryMappings == nil {
opt.CategoryMappings = map[string]CategoryMap{}
}
if opt.AttributeMappings == nil {
opt.AttributeMappings = map[string]AttributeMap{}
}
if opt.ProductIDs == nil {
opt.ProductIDs = map[string]int{}
}
pruneStringIntMap(opt.ProductIDs, maxProductIDMap)
if opt.MatchStrategy == "" {
opt.MatchStrategy = "sku"
}
if opt.BatchSize <= 0 || opt.BatchSize > maxBatchSize {
opt.BatchSize = defaultBatchSize
}
if opt.SyncLimit <= 0 || opt.SyncLimit > maxSyncLimit {
opt.SyncLimit = defaultSyncLimit
}
if opt.OrdersSyncLimit < 0 || opt.OrdersSyncLimit > maxOrdersSyncLimit {
opt.OrdersSyncLimit = 0
}
if opt.ReviewsSyncLimit < 0 || opt.ReviewsSyncLimit > maxReviewsSyncLimit {
opt.ReviewsSyncLimit = 0
}
if opt.ScheduleIntervalHours < 0 || opt.ScheduleIntervalHours > maxScheduleIntervalH {
opt.ScheduleIntervalHours = 0
}
opt.SyncOnlyIDs = pruneSyncOnlyIDs(opt.SyncOnlyIDs, maxSyncOnlyIDs)
return opt
}
// resolveScheduleInterval maps schedule_interval_hours to a duration (default when hours<=0).
func resolveScheduleInterval(hours int, defaultInterval time.Duration) time.Duration {
if defaultInterval <= 0 {
defaultInterval = 6 * time.Hour
}
if hours > 0 {
return time.Duration(hours) * time.Hour
}
return defaultInterval
}
// isDueForSchedule reports whether last sync is missing or older than interval.
func isDueForSchedule(last *time.Time, now time.Time, interval time.Duration) bool {
if last == nil {
return true
}
return now.Sub(last.UTC()) >= interval
}
// shouldEnqueueScheduled reports whether auto product sync should run now.
// Paused schedules never enqueue; interval 0 uses defaultInterval (worker default 6h).
func shouldEnqueueScheduled(paused bool, hours int, last *time.Time, now time.Time, defaultInterval time.Duration) bool {
if paused {
return false
}
return isDueForSchedule(last, now, resolveScheduleInterval(hours, defaultInterval))
}
func (s *Service) loadProductsForSync(ctx context.Context, companyID uuid.UUID, opt SyncOptions) ([]syncProductRow, error) {
limit := opt.SyncLimit
if limit <= 0 || limit > maxSyncLimit {
limit = defaultSyncLimit
}
args := []any{companyID}
where := []string{"p.company_id = $1"}
if status := strings.TrimSpace(opt.SyncFilterStatus); status != "" {
if status == "needs_review" {
where = append(where, "p.status IN ('needs_review', 'processed')")
} else {
args = append(args, status)
where = append(where, fmt.Sprintf("p.status = $%d", len(args)))
}
}
if cat := strings.TrimSpace(opt.SyncFilterCategory); cat != "" {
args = append(args, cat)
n := len(args)
where = append(where, fmt.Sprintf("(p.category = $%d OR lower(p.category) = lower($%d))", n, n))
}
if len(opt.SyncOnlyIDs) > 0 {
ids := make([]uuid.UUID, 0, len(opt.SyncOnlyIDs))
for _, raw := range opt.SyncOnlyIDs {
id, err := uuid.Parse(raw)
if err != nil {
continue
}
ids = append(ids, id)
}
if len(ids) == 0 {
return nil, nil
}
args = append(args, ids)
where = append(where, fmt.Sprintf("p.id = ANY($%d::uuid[])", len(args)))
}
args = append(args, limit)
limN := len(args)
q := fmt.Sprintf(`
SELECT p.id, p.product_id, p.name, p.category, p.description, p.processed_name, p.processed_description,
p.meta_description, p.attributes, p.processed_attributes, r.gtin, r.mapped_data
FROM processed_products p
LEFT JOIN raw_products r ON r.id = p.raw_product_id AND r.company_id = p.company_id
WHERE %s
ORDER BY p.updated_at DESC
LIMIT $%d`, strings.Join(where, " AND "), limN)
rows, err := s.Pool.Query(ctx, q, args...)
if err != nil {
return nil, err
}
defer rows.Close()
out := make([]syncProductRow, 0)
for rows.Next() {
var row syncProductRow
if err := rows.Scan(
&row.ID, &row.ProductID, &row.Name, &row.Category, &row.Description,
&row.ProcessedName, &row.ProcessedDescription, &row.MetaDescription,
&row.Attributes, &row.ProcessedAttributes, &row.GTIN, &row.MappedData,
); err != nil {
return nil, err
}
out = append(out, row)
}
return out, rows.Err()
}
func deref(s *string) string {
if s == nil {
return ""
}
return strings.TrimSpace(*s)
}
func firstNonEmpty(vals ...string) string {
for _, v := range vals {
if strings.TrimSpace(v) != "" {
return strings.TrimSpace(v)
}
}
return ""
}
func mappedString(mapped []byte, keys ...string) string {
if len(mapped) == 0 {
return ""
}
var m map[string]any
if json.Unmarshal(mapped, &m) != nil {
return ""
}
for _, k := range keys {
if v, ok := m[k]; ok {
switch t := v.(type) {
case string:
if strings.TrimSpace(t) != "" {
return strings.TrimSpace(t)
}
case float64:
return strconv.FormatFloat(t, 'f', -1, 64)
case json.Number:
return t.String()
default:
s := strings.TrimSpace(fmt.Sprint(t))
if s != "" && s != "<nil>" {
return s
}
}
}
}
return ""
}
func mappedImages(mapped []byte) []map[string]string {
if len(mapped) == 0 {
return nil
}
var m map[string]any
if json.Unmarshal(mapped, &m) != nil {
return nil
}
raw, ok := m["images"]
if !ok {
return nil
}
out := make([]map[string]string, 0)
switch t := raw.(type) {
case []any:
for _, item := range t {
switch u := item.(type) {
case string:
if u != "" {
out = append(out, map[string]string{"src": u})
}
case map[string]any:
if src, ok := u["src"].(string); ok && src != "" {
out = append(out, map[string]string{"src": src})
}
}
}
case string:
if t != "" {
out = append(out, map[string]string{"src": t})
}
}
return out
}
func (row syncProductRow) toPayload(opt SyncOptions) ProductPayload {
name := firstNonEmpty(deref(row.ProcessedName), deref(row.Name), "Product")
desc := firstNonEmpty(deref(row.ProcessedDescription), deref(row.Description))
shortDesc := deref(row.MetaDescription)
sku := firstNonEmpty(mappedString(row.MappedData, "sku", "SKU"), deref(row.ProductID), row.ID.String())
price := mappedString(row.MappedData, "price", "regular_price")
ean := firstNonEmpty(mappedString(row.MappedData, "ean", "gtin", "EAN"), deref(row.GTIN))
manage := false
payload := ProductPayload{
Name: name,
Type: "simple",
Status: "publish",
Description: desc,
ShortDescription: shortDesc,
SKU: sku,
RegularPrice: price,
Images: mappedImages(row.MappedData),
ManageStock: &manage,
StockStatus: "instock",
MetaData: []MetaDatum{
{Key: metaDescrybeID, Value: row.ID.String()},
{Key: "_descrybe_category", Value: deref(row.Category)},
{Key: "ean", Value: ean},
},
}
cat := deref(row.Category)
if cat != "" {
if m, ok := opt.CategoryMappings[cat]; ok {
entry := map[string]any{"name": firstNonEmpty(m.Name, cat)}
if m.WCID > 0 {
entry["id"] = m.WCID
}
if m.Slug != "" {
entry["slug"] = m.Slug
}
payload.Categories = []map[string]any{entry}
} else {
payload.Categories = []map[string]any{{"name": cat}}
}
}
attrsRaw := row.ProcessedAttributes
if len(attrsRaw) == 0 {
attrsRaw = row.Attributes
}
var attrs map[string]any
if len(attrsRaw) > 0 && json.Unmarshal(attrsRaw, &attrs) == nil {
i := 0
for key, val := range attrs {
options := []string{}
switch t := val.(type) {
case []any:
for _, x := range t {
options = append(options, fmt.Sprint(x))
}
default:
options = []string{fmt.Sprint(val)}
}
name := key
entry := map[string]any{
"name": name,
"visible": true,
"variation": false,
"options": options,
"position": i,
}
if m, ok := opt.AttributeMappings[key]; ok {
if m.Name != "" {
entry["name"] = m.Name
}
if m.WCID > 0 {
entry["id"] = m.WCID
}
}
payload.Attributes = append(payload.Attributes, entry)
i++
}
}
return payload
}
func (s *Service) pushProducts(ctx context.Context, client *Client, companyID uuid.UUID, opt *SyncOptions, rows []syncProductRow) (SyncSummary, error) {
summary := SyncSummary{Total: len(rows)}
creates := make([]ProductPayload, 0)
updates := make([]ProductPayload, 0)
createKeys := make([]string, 0)
updateKeys := make([]string, 0)
type pendingSKU struct {
key string
payload ProductPayload
}
needSKU := make([]pendingSKU, 0)
skus := make([]string, 0)
for _, row := range rows {
key := row.ID.String()
payload := row.toPayload(*opt)
if wcID, ok := opt.ProductIDs[key]; ok && wcID > 0 {
payload.ID = wcID
updates = append(updates, payload)
updateKeys = append(updateKeys, key)
continue
}
if opt.MatchStrategy == "sku" && payload.SKU != "" {
needSKU = append(needSKU, pendingSKU{key: key, payload: payload})
skus = append(skus, payload.SKU)
continue
}
creates = append(creates, payload)
createKeys = append(createKeys, key)
}
if len(needSKU) > 0 {
found, err := client.ListProductsBySKUs(ctx, skus)
if err != nil {
summary.Failed += len(needSKU)
} else {
for _, item := range needSKU {
if p, ok := found[item.payload.SKU]; ok && p.ID > 0 {
item.payload.ID = p.ID
opt.ProductIDs[item.key] = p.ID
updates = append(updates, item.payload)
updateKeys = append(updateKeys, item.key)
continue
}
creates = append(creates, item.payload)
createKeys = append(createKeys, item.key)
}
}
}
batchSize := opt.BatchSize
if batchSize <= 0 {
batchSize = defaultBatchSize
}
for i := 0; i < len(creates); i += batchSize {
end := i + batchSize
if end > len(creates) {
end = len(creates)
}
chunk := creates[i:end]
keys := createKeys[i:end]
res, err := client.BatchProducts(ctx, BatchRequest{Create: chunk})
if err != nil {
summary.Failed += len(chunk)
continue
}
for idx, p := range res.Create {
if idx < len(keys) && p.ID > 0 {
opt.ProductIDs[keys[idx]] = p.ID
summary.Created++
} else {
summary.Failed++
}
}
if len(res.Create) < len(chunk) {
summary.Failed += len(chunk) - len(res.Create)
}
}
for i := 0; i < len(updates); i += batchSize {
end := i + batchSize
if end > len(updates) {
end = len(updates)
}
chunk := updates[i:end]
keys := updateKeys[i:end]
res, err := client.BatchProducts(ctx, BatchRequest{Update: chunk})
if err != nil {
summary.Failed += len(chunk)
continue
}
for idx, p := range res.Update {
if idx < len(keys) && p.ID > 0 {
opt.ProductIDs[keys[idx]] = p.ID
summary.Updated++
} else {
summary.Failed++
}
}
if len(res.Update) < len(chunk) {
summary.Failed += len(chunk) - len(res.Update)
}
}
_ = companyID
return summary, nil
}
func pruneStringIntMap(m map[string]int, max int) {
if max <= 0 || len(m) <= max {
return
}
n := len(m) - max
for k := range m {
delete(m, k)
n--
if n <= 0 {
return
}
}
}
@@ -0,0 +1,134 @@
package woocommerce
import (
"encoding/json"
"io"
"net/http"
"net/http/httptest"
"strings"
"sync/atomic"
"testing"
"time"
"github.com/google/uuid"
)
func TestListProductsBySKUsOneRequest(t *testing.T) {
var calls atomic.Int32
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.URL.Path != "/wp-json/wc/v3/products" {
t.Fatalf("path %s", r.URL.Path)
}
calls.Add(1)
sku := r.URL.Query().Get("sku")
if !strings.Contains(sku, "A1") || !strings.Contains(sku, "B2") {
t.Fatalf("sku query %q", sku)
}
_, _ = io.WriteString(w, `[{"id":11,"sku":"A1","name":"A"},{"id":22,"sku":"B2","name":"B"}]`)
}))
defer srv.Close()
c := NewClient(srv.URL, "ck", "cs", srv.Client())
found, err := c.ListProductsBySKUs(t.Context(), []string{"A1", "B2", "MISSING"})
if err != nil {
t.Fatal(err)
}
if calls.Load() != 1 {
t.Fatalf("calls=%d", calls.Load())
}
if found["A1"].ID != 11 || found["B2"].ID != 22 {
t.Fatalf("%+v", found)
}
if _, ok := found["MISSING"]; ok {
t.Fatal("unexpected missing hit")
}
}
func TestPushProductsBatchesSKULookup(t *testing.T) {
var skuCalls, batchCalls atomic.Int32
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
switch {
case r.Method == http.MethodGet && r.URL.Path == "/wp-json/wc/v3/products":
skuCalls.Add(1)
_, _ = io.WriteString(w, `[{"id":99,"sku":"EXISTING","name":"Exist"}]`)
case r.Method == http.MethodPost && r.URL.Path == "/wp-json/wc/v3/products/batch":
batchCalls.Add(1)
var req BatchRequest
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
t.Fatal(err)
}
if len(req.Create) == 1 && req.Create[0].SKU == "NEW" {
_, _ = io.WriteString(w, `{"create":[{"id":100,"sku":"NEW"}],"update":[]}`)
return
}
if len(req.Update) == 1 && req.Update[0].SKU == "EXISTING" {
_, _ = io.WriteString(w, `{"create":[],"update":[{"id":99,"sku":"EXISTING"}]}`)
return
}
t.Fatalf("unexpected batch %+v", req)
default:
t.Fatalf("unexpected %s %s", r.Method, r.URL.Path)
}
}))
defer srv.Close()
c := NewClient(srv.URL, "ck", "cs", srv.Client())
svc := &Service{}
idExisting := uuid.MustParse("11111111-1111-1111-1111-111111111111")
idNew := uuid.MustParse("22222222-2222-2222-2222-222222222222")
pidExisting := "EXISTING"
pidNew := "NEW"
name := "Product"
opt := SyncOptions{
ProductIDs: map[string]int{},
MatchStrategy: "sku",
BatchSize: 25,
}
rows := []syncProductRow{
{ID: idExisting, ProductID: &pidExisting, Name: &name, MappedData: []byte(`{"sku":"EXISTING","price":"1"}`)},
{ID: idNew, ProductID: &pidNew, Name: &name, MappedData: []byte(`{"sku":"NEW","price":"2"}`)},
}
summary, err := svc.pushProducts(t.Context(), c, uuid.Nil, &opt, rows)
if err != nil {
t.Fatal(err)
}
if skuCalls.Load() != 1 {
t.Fatalf("expected 1 SKU lookup, got %d", skuCalls.Load())
}
if batchCalls.Load() != 2 {
t.Fatalf("expected create+update batches, got %d", batchCalls.Load())
}
if summary.Created != 1 || summary.Updated != 1 || summary.Failed != 0 {
t.Fatalf("summary=%+v", summary)
}
if opt.ProductIDs[idExisting.String()] != 99 || opt.ProductIDs[idNew.String()] != 100 {
t.Fatalf("product ids=%v", opt.ProductIDs)
}
}
func TestWooCommerceRetries429(t *testing.T) {
var calls atomic.Int32
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
n := calls.Add(1)
if n == 1 {
w.Header().Set("Retry-After", "0")
w.WriteHeader(http.StatusTooManyRequests)
_, _ = io.WriteString(w, `{"code":"woocommerce_rest_cannot_view"}`)
return
}
_, _ = io.WriteString(w, `[]`)
}))
defer srv.Close()
c := NewClient(srv.URL, "ck", "cs", srv.Client())
start := time.Now()
if err := c.TestConnection(t.Context()); err != nil {
t.Fatal(err)
}
if calls.Load() != 2 {
t.Fatalf("calls=%d", calls.Load())
}
if time.Since(start) > 3*time.Second {
t.Fatal("retry waited too long")
}
}
@@ -0,0 +1,97 @@
package woocommerce
import (
"fmt"
"strings"
"github.com/google/uuid"
)
const (
maxSyncOnlyIDs = 500
maxCategoryFilterLen = 200
)
// ProductSyncScope is an optional one-shot filter for EnqueueSync / POST /sync.
// Empty fields mean "no filter" (sync recent processed products up to SyncLimit).
type ProductSyncScope struct {
Limit int `json:"sync_limit,omitempty"`
Status string `json:"status,omitempty"`
Category string `json:"category,omitempty"`
ProductIDs []string `json:"product_ids,omitempty"`
}
func clearOneShotSyncFilters(opt *SyncOptions) {
if opt == nil {
return
}
opt.SyncFilterStatus = ""
opt.SyncFilterCategory = ""
opt.SyncOnlyIDs = nil
}
func normalizeSyncFilterStatus(status string) (string, error) {
s := strings.TrimSpace(strings.ToLower(status))
if s == "" {
return "", nil
}
switch s {
case "completed", "needs_review", "error", "processing", "processed":
return s, nil
default:
return "", fmt.Errorf("%w: invalid status", ErrInvalidSyncScope)
}
}
func applyProductSyncScope(opt *SyncOptions, scope ProductSyncScope) error {
if opt == nil {
return fmt.Errorf("%w: missing options", ErrInvalidSyncScope)
}
clearOneShotSyncFilters(opt)
if scope.Limit > 0 {
if scope.Limit > maxSyncLimit {
return fmt.Errorf("%w: sync_limit max %d", ErrInvalidSyncScope, maxSyncLimit)
}
opt.SyncLimit = scope.Limit
}
status, err := normalizeSyncFilterStatus(scope.Status)
if err != nil {
return err
}
opt.SyncFilterStatus = status
cat := strings.TrimSpace(scope.Category)
if len(cat) > maxCategoryFilterLen {
return fmt.Errorf("%w: category too long", ErrInvalidSyncScope)
}
opt.SyncFilterCategory = cat
if len(scope.ProductIDs) > maxSyncOnlyIDs {
return fmt.Errorf("%w: product_ids max %d", ErrInvalidSyncScope, maxSyncOnlyIDs)
}
cleaned := make([]string, 0, len(scope.ProductIDs))
seen := make(map[string]struct{}, len(scope.ProductIDs))
for _, raw := range scope.ProductIDs {
id := strings.TrimSpace(raw)
if id == "" {
continue
}
parsed, err := uuid.Parse(id)
if err != nil {
return fmt.Errorf("%w: invalid product_ids", ErrInvalidSyncScope)
}
key := parsed.String()
if _, ok := seen[key]; ok {
continue
}
seen[key] = struct{}{}
cleaned = append(cleaned, key)
}
opt.SyncOnlyIDs = cleaned
return nil
}
func pruneSyncOnlyIDs(ids []string, max int) []string {
if max <= 0 || len(ids) <= max {
return ids
}
return ids[:max]
}
@@ -0,0 +1,42 @@
package woocommerce
import (
"errors"
"strings"
"testing"
"github.com/google/uuid"
)
func TestApplyProductSyncScope(t *testing.T) {
opt := SyncOptions{SyncLimit: defaultSyncLimit}
id := uuid.New().String()
err := applyProductSyncScope(&opt, ProductSyncScope{
Limit: 25,
Status: "needs_review",
Category: "Sofas",
ProductIDs: []string{id},
})
if err != nil {
t.Fatal(err)
}
if opt.SyncLimit != 25 || opt.SyncFilterStatus != "needs_review" || opt.SyncFilterCategory != "Sofas" {
t.Fatalf("opt=%#v", opt)
}
if len(opt.SyncOnlyIDs) != 1 || opt.SyncOnlyIDs[0] != id {
t.Fatalf("ids=%v", opt.SyncOnlyIDs)
}
if err := applyProductSyncScope(&opt, ProductSyncScope{Status: "bad"}); !errors.Is(err, ErrInvalidSyncScope) {
t.Fatalf("err=%v", err)
}
if err := applyProductSyncScope(&opt, ProductSyncScope{Category: strings.Repeat("x", maxCategoryFilterLen+1)}); !errors.Is(err, ErrInvalidSyncScope) {
t.Fatalf("category length err=%v", err)
}
tooMany := make([]string, maxSyncOnlyIDs+1)
for i := range tooMany {
tooMany[i] = uuid.New().String()
}
if err := applyProductSyncScope(&opt, ProductSyncScope{ProductIDs: tooMany}); !errors.Is(err, ErrInvalidSyncScope) {
t.Fatalf("product_ids max err=%v", err)
}
}
+119
View File
@@ -0,0 +1,119 @@
package woocommerce
import (
"errors"
"fmt"
"net"
"net/url"
"strings"
)
var (
ErrInvalidStoreURL = errors.New("invalid store url")
ErrBlockedStoreURL = errors.New("store url host is not allowed")
)
// NormalizeStoreURL validates and normalizes a WooCommerce store base URL.
// Requires https (http only for localhost). Blocks metadata and private ranges (SSRF).
func NormalizeStoreURL(raw string) (string, error) {
raw = strings.TrimSpace(raw)
if raw == "" {
return "", ErrInvalidStoreURL
}
if !strings.Contains(raw, "://") {
raw = "https://" + raw
}
u, err := url.Parse(raw)
if err != nil || u.Host == "" {
return "", ErrInvalidStoreURL
}
scheme := strings.ToLower(u.Scheme)
if scheme != "https" && scheme != "http" {
return "", ErrInvalidStoreURL
}
host := strings.ToLower(u.Hostname())
if host == "" {
return "", ErrInvalidStoreURL
}
if host == "metadata.google.internal" || host == "metadata" {
return "", ErrBlockedStoreURL
}
ips, err := resolveHostIPs(host)
if err != nil {
if isLiteralIP(host) {
return "", err
}
} else {
for _, ip := range ips {
if !allowedIP(ip, host) {
return "", ErrBlockedStoreURL
}
}
}
if scheme == "http" && !isLoopbackHost(host) {
return "", fmt.Errorf("%w: https required (http only allowed for localhost)", ErrInvalidStoreURL)
}
u.Scheme = scheme
if u.Port() != "" {
u.Host = net.JoinHostPort(u.Hostname(), u.Port())
} else {
u.Host = u.Hostname()
}
u.Path = strings.TrimRight(u.Path, "/")
u.RawQuery = ""
u.Fragment = ""
u.User = nil
return u.String(), nil
}
func resolveHostIPs(host string) ([]net.IP, error) {
if ip := net.ParseIP(host); ip != nil {
return []net.IP{ip}, nil
}
addrs, err := net.LookupIP(host)
if err != nil {
return nil, ErrInvalidStoreURL
}
return addrs, nil
}
func isLiteralIP(host string) bool {
return net.ParseIP(host) != nil
}
func isLoopbackHost(host string) bool {
if host == "localhost" {
return true
}
ip := net.ParseIP(host)
return ip != nil && ip.IsLoopback()
}
func allowedIP(ip net.IP, host string) bool {
if ip.IsLoopback() {
return isLoopbackHost(host)
}
if ip.IsUnspecified() || ip.IsMulticast() || ip.IsLinkLocalUnicast() || ip.IsLinkLocalMulticast() {
return false
}
if ip4 := ip.To4(); ip4 != nil {
if ip4[0] == 10 {
return false
}
if ip4[0] == 172 && ip4[1] >= 16 && ip4[1] <= 31 {
return false
}
if ip4[0] == 192 && ip4[1] == 168 {
return false
}
if ip4[0] == 169 && ip4[1] == 254 {
return false
}
if ip4[0] == 100 && ip4[1] >= 64 && ip4[1] <= 127 {
return false
}
} else if ip.IsPrivate() {
return false
}
return true
}
+40
View File
@@ -0,0 +1,40 @@
package woocommerce
import (
"errors"
"testing"
)
func TestNormalizeStoreURLHTTPS(t *testing.T) {
got, err := NormalizeStoreURL("https://example.com/shop/")
if err != nil {
t.Fatal(err)
}
if got != "https://example.com/shop" {
t.Fatalf("got %q", got)
}
}
func TestNormalizeStoreURLBlocksPrivateIP(t *testing.T) {
_, err := NormalizeStoreURL("https://192.168.1.10")
if !errors.Is(err, ErrBlockedStoreURL) {
t.Fatalf("expected blocked, got %v", err)
}
}
func TestNormalizeStoreURLRejectsPlainHTTPRemote(t *testing.T) {
_, err := NormalizeStoreURL("http://example.com")
if err == nil {
t.Fatal("expected error")
}
}
func TestNormalizeStoreURLAllowsLocalhostHTTP(t *testing.T) {
got, err := NormalizeStoreURL("http://127.0.0.1:8080")
if err != nil {
t.Fatal(err)
}
if got != "http://127.0.0.1:8080" {
t.Fatalf("got %q", got)
}
}