Initial commit of Descrybe v2 without local scratch artifacts.
Drop one-shot tmp/axe scripts and agent i18n scratch so the Gitea tree is deployable.
This commit is contained in:
@@ -0,0 +1,513 @@
|
||||
package woocommerce
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/google/uuid"
|
||||
)
|
||||
|
||||
const (
|
||||
defaultBatchSize = 25
|
||||
defaultSyncLimit = 200
|
||||
maxBatchSize = 100
|
||||
maxSyncLimit = 1000
|
||||
maxOrdersSyncLimit = 5000
|
||||
maxReviewsSyncLimit = 5000
|
||||
maxProductIDMap = 5000
|
||||
maxScheduleIntervalH = 168 // 7 days
|
||||
metaDescrybeID = "_descrybe_product_id"
|
||||
)
|
||||
|
||||
type SyncOptions struct {
|
||||
CategoryMappings map[string]CategoryMap `json:"category_mappings"`
|
||||
AttributeMappings map[string]AttributeMap `json:"attribute_mappings"`
|
||||
ProductIDs map[string]int `json:"product_ids"`
|
||||
MatchStrategy string `json:"match_strategy"`
|
||||
BatchSize int `json:"batch_size"`
|
||||
SyncLimit int `json:"sync_limit"`
|
||||
PendingSync bool `json:"pending_sync"`
|
||||
PendingOrdersSync bool `json:"pending_orders_sync"`
|
||||
PendingReviewsSync bool `json:"pending_reviews_sync"`
|
||||
LastSyncStatus string `json:"last_sync_status"`
|
||||
LastSyncError string `json:"last_sync_error"`
|
||||
LastSyncSummary *SyncSummary `json:"last_sync_summary,omitempty"`
|
||||
LastOrdersSyncStatus string `json:"last_orders_sync_status"`
|
||||
LastOrdersSyncError string `json:"last_orders_sync_error"`
|
||||
LastOrdersSyncedAt *time.Time `json:"last_orders_synced_at,omitempty"`
|
||||
LastReviewsSyncStatus string `json:"last_reviews_sync_status"`
|
||||
LastReviewsSyncError string `json:"last_reviews_sync_error"`
|
||||
LastReviewsSyncedAt *time.Time `json:"last_reviews_synced_at,omitempty"`
|
||||
OrdersSyncLimit int `json:"orders_sync_limit"`
|
||||
ReviewsSyncLimit int `json:"reviews_sync_limit"`
|
||||
OrdersModifiedAfter string `json:"orders_modified_after"`
|
||||
ScheduleIntervalHours int `json:"schedule_interval_hours"`
|
||||
SchedulePaused bool `json:"schedule_paused"`
|
||||
// One-shot selection for the next product sync (cleared when sync finishes).
|
||||
SyncFilterStatus string `json:"sync_filter_status,omitempty"`
|
||||
SyncFilterCategory string `json:"sync_filter_category,omitempty"`
|
||||
SyncOnlyIDs []string `json:"sync_only_ids,omitempty"`
|
||||
}
|
||||
|
||||
type CategoryMap struct {
|
||||
Name string `json:"name"`
|
||||
Slug string `json:"slug"`
|
||||
WCID int `json:"wc_id"`
|
||||
}
|
||||
|
||||
type AttributeMap struct {
|
||||
Name string `json:"name"`
|
||||
WCID int `json:"wc_id"`
|
||||
}
|
||||
|
||||
type SyncSummary struct {
|
||||
Total int `json:"total"`
|
||||
Created int `json:"created"`
|
||||
Updated int `json:"updated"`
|
||||
Failed int `json:"failed"`
|
||||
Skipped int `json:"skipped"`
|
||||
}
|
||||
|
||||
type syncProductRow struct {
|
||||
ID uuid.UUID
|
||||
ProductID *string
|
||||
Name *string
|
||||
Category *string
|
||||
Description *string
|
||||
ProcessedName *string
|
||||
ProcessedDescription *string
|
||||
MetaDescription *string
|
||||
Attributes []byte
|
||||
ProcessedAttributes []byte
|
||||
GTIN *string
|
||||
MappedData []byte
|
||||
}
|
||||
|
||||
func parseSyncOptions(raw []byte) SyncOptions {
|
||||
opt := SyncOptions{
|
||||
CategoryMappings: map[string]CategoryMap{},
|
||||
AttributeMappings: map[string]AttributeMap{},
|
||||
ProductIDs: map[string]int{},
|
||||
MatchStrategy: "sku",
|
||||
BatchSize: defaultBatchSize,
|
||||
SyncLimit: defaultSyncLimit,
|
||||
}
|
||||
if len(raw) == 0 {
|
||||
return opt
|
||||
}
|
||||
_ = json.Unmarshal(raw, &opt)
|
||||
if opt.CategoryMappings == nil {
|
||||
opt.CategoryMappings = map[string]CategoryMap{}
|
||||
}
|
||||
if opt.AttributeMappings == nil {
|
||||
opt.AttributeMappings = map[string]AttributeMap{}
|
||||
}
|
||||
if opt.ProductIDs == nil {
|
||||
opt.ProductIDs = map[string]int{}
|
||||
}
|
||||
pruneStringIntMap(opt.ProductIDs, maxProductIDMap)
|
||||
if opt.MatchStrategy == "" {
|
||||
opt.MatchStrategy = "sku"
|
||||
}
|
||||
if opt.BatchSize <= 0 || opt.BatchSize > maxBatchSize {
|
||||
opt.BatchSize = defaultBatchSize
|
||||
}
|
||||
if opt.SyncLimit <= 0 || opt.SyncLimit > maxSyncLimit {
|
||||
opt.SyncLimit = defaultSyncLimit
|
||||
}
|
||||
if opt.OrdersSyncLimit < 0 || opt.OrdersSyncLimit > maxOrdersSyncLimit {
|
||||
opt.OrdersSyncLimit = 0
|
||||
}
|
||||
if opt.ReviewsSyncLimit < 0 || opt.ReviewsSyncLimit > maxReviewsSyncLimit {
|
||||
opt.ReviewsSyncLimit = 0
|
||||
}
|
||||
if opt.ScheduleIntervalHours < 0 || opt.ScheduleIntervalHours > maxScheduleIntervalH {
|
||||
opt.ScheduleIntervalHours = 0
|
||||
}
|
||||
opt.SyncOnlyIDs = pruneSyncOnlyIDs(opt.SyncOnlyIDs, maxSyncOnlyIDs)
|
||||
return opt
|
||||
}
|
||||
|
||||
// resolveScheduleInterval maps schedule_interval_hours to a duration (default when hours<=0).
|
||||
func resolveScheduleInterval(hours int, defaultInterval time.Duration) time.Duration {
|
||||
if defaultInterval <= 0 {
|
||||
defaultInterval = 6 * time.Hour
|
||||
}
|
||||
if hours > 0 {
|
||||
return time.Duration(hours) * time.Hour
|
||||
}
|
||||
return defaultInterval
|
||||
}
|
||||
|
||||
// isDueForSchedule reports whether last sync is missing or older than interval.
|
||||
func isDueForSchedule(last *time.Time, now time.Time, interval time.Duration) bool {
|
||||
if last == nil {
|
||||
return true
|
||||
}
|
||||
return now.Sub(last.UTC()) >= interval
|
||||
}
|
||||
|
||||
// shouldEnqueueScheduled reports whether auto product sync should run now.
|
||||
// Paused schedules never enqueue; interval 0 uses defaultInterval (worker default 6h).
|
||||
func shouldEnqueueScheduled(paused bool, hours int, last *time.Time, now time.Time, defaultInterval time.Duration) bool {
|
||||
if paused {
|
||||
return false
|
||||
}
|
||||
return isDueForSchedule(last, now, resolveScheduleInterval(hours, defaultInterval))
|
||||
}
|
||||
|
||||
func (s *Service) loadProductsForSync(ctx context.Context, companyID uuid.UUID, opt SyncOptions) ([]syncProductRow, error) {
|
||||
limit := opt.SyncLimit
|
||||
if limit <= 0 || limit > maxSyncLimit {
|
||||
limit = defaultSyncLimit
|
||||
}
|
||||
args := []any{companyID}
|
||||
where := []string{"p.company_id = $1"}
|
||||
if status := strings.TrimSpace(opt.SyncFilterStatus); status != "" {
|
||||
if status == "needs_review" {
|
||||
where = append(where, "p.status IN ('needs_review', 'processed')")
|
||||
} else {
|
||||
args = append(args, status)
|
||||
where = append(where, fmt.Sprintf("p.status = $%d", len(args)))
|
||||
}
|
||||
}
|
||||
if cat := strings.TrimSpace(opt.SyncFilterCategory); cat != "" {
|
||||
args = append(args, cat)
|
||||
n := len(args)
|
||||
where = append(where, fmt.Sprintf("(p.category = $%d OR lower(p.category) = lower($%d))", n, n))
|
||||
}
|
||||
if len(opt.SyncOnlyIDs) > 0 {
|
||||
ids := make([]uuid.UUID, 0, len(opt.SyncOnlyIDs))
|
||||
for _, raw := range opt.SyncOnlyIDs {
|
||||
id, err := uuid.Parse(raw)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
ids = append(ids, id)
|
||||
}
|
||||
if len(ids) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
args = append(args, ids)
|
||||
where = append(where, fmt.Sprintf("p.id = ANY($%d::uuid[])", len(args)))
|
||||
}
|
||||
args = append(args, limit)
|
||||
limN := len(args)
|
||||
q := fmt.Sprintf(`
|
||||
SELECT p.id, p.product_id, p.name, p.category, p.description, p.processed_name, p.processed_description,
|
||||
p.meta_description, p.attributes, p.processed_attributes, r.gtin, r.mapped_data
|
||||
FROM processed_products p
|
||||
LEFT JOIN raw_products r ON r.id = p.raw_product_id AND r.company_id = p.company_id
|
||||
WHERE %s
|
||||
ORDER BY p.updated_at DESC
|
||||
LIMIT $%d`, strings.Join(where, " AND "), limN)
|
||||
rows, err := s.Pool.Query(ctx, q, args...)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
out := make([]syncProductRow, 0)
|
||||
for rows.Next() {
|
||||
var row syncProductRow
|
||||
if err := rows.Scan(
|
||||
&row.ID, &row.ProductID, &row.Name, &row.Category, &row.Description,
|
||||
&row.ProcessedName, &row.ProcessedDescription, &row.MetaDescription,
|
||||
&row.Attributes, &row.ProcessedAttributes, &row.GTIN, &row.MappedData,
|
||||
); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out = append(out, row)
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
func deref(s *string) string {
|
||||
if s == nil {
|
||||
return ""
|
||||
}
|
||||
return strings.TrimSpace(*s)
|
||||
}
|
||||
|
||||
func firstNonEmpty(vals ...string) string {
|
||||
for _, v := range vals {
|
||||
if strings.TrimSpace(v) != "" {
|
||||
return strings.TrimSpace(v)
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func mappedString(mapped []byte, keys ...string) string {
|
||||
if len(mapped) == 0 {
|
||||
return ""
|
||||
}
|
||||
var m map[string]any
|
||||
if json.Unmarshal(mapped, &m) != nil {
|
||||
return ""
|
||||
}
|
||||
for _, k := range keys {
|
||||
if v, ok := m[k]; ok {
|
||||
switch t := v.(type) {
|
||||
case string:
|
||||
if strings.TrimSpace(t) != "" {
|
||||
return strings.TrimSpace(t)
|
||||
}
|
||||
case float64:
|
||||
return strconv.FormatFloat(t, 'f', -1, 64)
|
||||
case json.Number:
|
||||
return t.String()
|
||||
default:
|
||||
s := strings.TrimSpace(fmt.Sprint(t))
|
||||
if s != "" && s != "<nil>" {
|
||||
return s
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func mappedImages(mapped []byte) []map[string]string {
|
||||
if len(mapped) == 0 {
|
||||
return nil
|
||||
}
|
||||
var m map[string]any
|
||||
if json.Unmarshal(mapped, &m) != nil {
|
||||
return nil
|
||||
}
|
||||
raw, ok := m["images"]
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
out := make([]map[string]string, 0)
|
||||
switch t := raw.(type) {
|
||||
case []any:
|
||||
for _, item := range t {
|
||||
switch u := item.(type) {
|
||||
case string:
|
||||
if u != "" {
|
||||
out = append(out, map[string]string{"src": u})
|
||||
}
|
||||
case map[string]any:
|
||||
if src, ok := u["src"].(string); ok && src != "" {
|
||||
out = append(out, map[string]string{"src": src})
|
||||
}
|
||||
}
|
||||
}
|
||||
case string:
|
||||
if t != "" {
|
||||
out = append(out, map[string]string{"src": t})
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func (row syncProductRow) toPayload(opt SyncOptions) ProductPayload {
|
||||
name := firstNonEmpty(deref(row.ProcessedName), deref(row.Name), "Product")
|
||||
desc := firstNonEmpty(deref(row.ProcessedDescription), deref(row.Description))
|
||||
shortDesc := deref(row.MetaDescription)
|
||||
sku := firstNonEmpty(mappedString(row.MappedData, "sku", "SKU"), deref(row.ProductID), row.ID.String())
|
||||
price := mappedString(row.MappedData, "price", "regular_price")
|
||||
ean := firstNonEmpty(mappedString(row.MappedData, "ean", "gtin", "EAN"), deref(row.GTIN))
|
||||
|
||||
manage := false
|
||||
payload := ProductPayload{
|
||||
Name: name,
|
||||
Type: "simple",
|
||||
Status: "publish",
|
||||
Description: desc,
|
||||
ShortDescription: shortDesc,
|
||||
SKU: sku,
|
||||
RegularPrice: price,
|
||||
Images: mappedImages(row.MappedData),
|
||||
ManageStock: &manage,
|
||||
StockStatus: "instock",
|
||||
MetaData: []MetaDatum{
|
||||
{Key: metaDescrybeID, Value: row.ID.String()},
|
||||
{Key: "_descrybe_category", Value: deref(row.Category)},
|
||||
{Key: "ean", Value: ean},
|
||||
},
|
||||
}
|
||||
|
||||
cat := deref(row.Category)
|
||||
if cat != "" {
|
||||
if m, ok := opt.CategoryMappings[cat]; ok {
|
||||
entry := map[string]any{"name": firstNonEmpty(m.Name, cat)}
|
||||
if m.WCID > 0 {
|
||||
entry["id"] = m.WCID
|
||||
}
|
||||
if m.Slug != "" {
|
||||
entry["slug"] = m.Slug
|
||||
}
|
||||
payload.Categories = []map[string]any{entry}
|
||||
} else {
|
||||
payload.Categories = []map[string]any{{"name": cat}}
|
||||
}
|
||||
}
|
||||
|
||||
attrsRaw := row.ProcessedAttributes
|
||||
if len(attrsRaw) == 0 {
|
||||
attrsRaw = row.Attributes
|
||||
}
|
||||
var attrs map[string]any
|
||||
if len(attrsRaw) > 0 && json.Unmarshal(attrsRaw, &attrs) == nil {
|
||||
i := 0
|
||||
for key, val := range attrs {
|
||||
options := []string{}
|
||||
switch t := val.(type) {
|
||||
case []any:
|
||||
for _, x := range t {
|
||||
options = append(options, fmt.Sprint(x))
|
||||
}
|
||||
default:
|
||||
options = []string{fmt.Sprint(val)}
|
||||
}
|
||||
name := key
|
||||
entry := map[string]any{
|
||||
"name": name,
|
||||
"visible": true,
|
||||
"variation": false,
|
||||
"options": options,
|
||||
"position": i,
|
||||
}
|
||||
if m, ok := opt.AttributeMappings[key]; ok {
|
||||
if m.Name != "" {
|
||||
entry["name"] = m.Name
|
||||
}
|
||||
if m.WCID > 0 {
|
||||
entry["id"] = m.WCID
|
||||
}
|
||||
}
|
||||
payload.Attributes = append(payload.Attributes, entry)
|
||||
i++
|
||||
}
|
||||
}
|
||||
return payload
|
||||
}
|
||||
|
||||
func (s *Service) pushProducts(ctx context.Context, client *Client, companyID uuid.UUID, opt *SyncOptions, rows []syncProductRow) (SyncSummary, error) {
|
||||
summary := SyncSummary{Total: len(rows)}
|
||||
creates := make([]ProductPayload, 0)
|
||||
updates := make([]ProductPayload, 0)
|
||||
createKeys := make([]string, 0)
|
||||
updateKeys := make([]string, 0)
|
||||
|
||||
type pendingSKU struct {
|
||||
key string
|
||||
payload ProductPayload
|
||||
}
|
||||
needSKU := make([]pendingSKU, 0)
|
||||
skus := make([]string, 0)
|
||||
|
||||
for _, row := range rows {
|
||||
key := row.ID.String()
|
||||
payload := row.toPayload(*opt)
|
||||
if wcID, ok := opt.ProductIDs[key]; ok && wcID > 0 {
|
||||
payload.ID = wcID
|
||||
updates = append(updates, payload)
|
||||
updateKeys = append(updateKeys, key)
|
||||
continue
|
||||
}
|
||||
if opt.MatchStrategy == "sku" && payload.SKU != "" {
|
||||
needSKU = append(needSKU, pendingSKU{key: key, payload: payload})
|
||||
skus = append(skus, payload.SKU)
|
||||
continue
|
||||
}
|
||||
creates = append(creates, payload)
|
||||
createKeys = append(createKeys, key)
|
||||
}
|
||||
|
||||
if len(needSKU) > 0 {
|
||||
found, err := client.ListProductsBySKUs(ctx, skus)
|
||||
if err != nil {
|
||||
summary.Failed += len(needSKU)
|
||||
} else {
|
||||
for _, item := range needSKU {
|
||||
if p, ok := found[item.payload.SKU]; ok && p.ID > 0 {
|
||||
item.payload.ID = p.ID
|
||||
opt.ProductIDs[item.key] = p.ID
|
||||
updates = append(updates, item.payload)
|
||||
updateKeys = append(updateKeys, item.key)
|
||||
continue
|
||||
}
|
||||
creates = append(creates, item.payload)
|
||||
createKeys = append(createKeys, item.key)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
batchSize := opt.BatchSize
|
||||
if batchSize <= 0 {
|
||||
batchSize = defaultBatchSize
|
||||
}
|
||||
for i := 0; i < len(creates); i += batchSize {
|
||||
end := i + batchSize
|
||||
if end > len(creates) {
|
||||
end = len(creates)
|
||||
}
|
||||
chunk := creates[i:end]
|
||||
keys := createKeys[i:end]
|
||||
res, err := client.BatchProducts(ctx, BatchRequest{Create: chunk})
|
||||
if err != nil {
|
||||
summary.Failed += len(chunk)
|
||||
continue
|
||||
}
|
||||
for idx, p := range res.Create {
|
||||
if idx < len(keys) && p.ID > 0 {
|
||||
opt.ProductIDs[keys[idx]] = p.ID
|
||||
summary.Created++
|
||||
} else {
|
||||
summary.Failed++
|
||||
}
|
||||
}
|
||||
if len(res.Create) < len(chunk) {
|
||||
summary.Failed += len(chunk) - len(res.Create)
|
||||
}
|
||||
}
|
||||
|
||||
for i := 0; i < len(updates); i += batchSize {
|
||||
end := i + batchSize
|
||||
if end > len(updates) {
|
||||
end = len(updates)
|
||||
}
|
||||
chunk := updates[i:end]
|
||||
keys := updateKeys[i:end]
|
||||
res, err := client.BatchProducts(ctx, BatchRequest{Update: chunk})
|
||||
if err != nil {
|
||||
summary.Failed += len(chunk)
|
||||
continue
|
||||
}
|
||||
for idx, p := range res.Update {
|
||||
if idx < len(keys) && p.ID > 0 {
|
||||
opt.ProductIDs[keys[idx]] = p.ID
|
||||
summary.Updated++
|
||||
} else {
|
||||
summary.Failed++
|
||||
}
|
||||
}
|
||||
if len(res.Update) < len(chunk) {
|
||||
summary.Failed += len(chunk) - len(res.Update)
|
||||
}
|
||||
}
|
||||
|
||||
_ = companyID
|
||||
return summary, nil
|
||||
}
|
||||
|
||||
func pruneStringIntMap(m map[string]int, max int) {
|
||||
if max <= 0 || len(m) <= max {
|
||||
return
|
||||
}
|
||||
n := len(m) - max
|
||||
for k := range m {
|
||||
delete(m, k)
|
||||
n--
|
||||
if n <= 0 {
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user