Initial commit of Descrybe v2 without local scratch artifacts.
Drop one-shot tmp/axe scripts and agent i18n scratch so the Gitea tree is deployable.
This commit is contained in:
@@ -0,0 +1,404 @@
|
||||
package feeds
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"os"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
const (
|
||||
// defaultMaxDownloadBytes caps HTTP downloads and local upload sources.
|
||||
// Bodies stream to a temp file (not RAM), so hundreds of MiB are safe for
|
||||
// large merchant catalogs. Keep within ~200–500 MiB; raise carefully if disk
|
||||
// and downloadTimeout remain adequate. Exceeding returns downloadTooLarge().
|
||||
defaultMaxDownloadBytes int64 = 256 << 20 // 256 MiB
|
||||
downloadTimeout = 60 * time.Second
|
||||
dialTimeout = 10 * time.Second
|
||||
maxRedirects = 5
|
||||
defaultUserAgent = "DescrybeFeedSync/2.0"
|
||||
)
|
||||
|
||||
// maxDownloadBytes bounds feed downloads (temp file / local source size). Mutable for tests.
|
||||
var maxDownloadBytes = defaultMaxDownloadBytes
|
||||
|
||||
var (
|
||||
errURLRequired = errors.New("feed url required")
|
||||
errURLScheme = errors.New("url scheme must be http or https")
|
||||
errURLPrivate = errors.New("url resolves to a private or blocked address")
|
||||
errURLFTP = errors.New("ftp/ftps feed sync is not supported yet")
|
||||
errDownloadTooLarge = errors.New("feed download exceeds size limit")
|
||||
)
|
||||
|
||||
// downloadTooLarge returns errDownloadTooLarge with the active size limit for clients.
|
||||
func downloadTooLarge() error {
|
||||
if maxDownloadBytes >= 1<<20 {
|
||||
return fmt.Errorf("%w (max %d MiB)", errDownloadTooLarge, maxDownloadBytes>>20)
|
||||
}
|
||||
return fmt.Errorf("%w (max %d bytes)", errDownloadTooLarge, maxDownloadBytes)
|
||||
}
|
||||
|
||||
var (
|
||||
allowMu sync.RWMutex
|
||||
allowConfigured bool
|
||||
allowHosts map[string]struct{}
|
||||
allowCIDRs []*net.IPNet
|
||||
)
|
||||
|
||||
// ConfigurePrivateAllowlist sets hostnames and CIDRs that may bypass the private-IP SSRF block.
|
||||
// Intended for tests and optional startup wiring; production normally uses FEED_URL_PRIVATE_ALLOWLIST
|
||||
// and/or admin platform setting feeds.private_url_allowlist.
|
||||
func ConfigurePrivateAllowlist(hosts []string, cidrs []string) error {
|
||||
h := make(map[string]struct{}, len(hosts))
|
||||
for _, raw := range hosts {
|
||||
raw = strings.ToLower(strings.TrimSpace(raw))
|
||||
if raw != "" {
|
||||
h[raw] = struct{}{}
|
||||
}
|
||||
}
|
||||
nets := make([]*net.IPNet, 0, len(cidrs))
|
||||
for _, raw := range cidrs {
|
||||
raw = strings.TrimSpace(raw)
|
||||
if raw == "" {
|
||||
continue
|
||||
}
|
||||
_, n, err := net.ParseCIDR(raw)
|
||||
if err != nil {
|
||||
return fmt.Errorf("invalid allowlist cidr %q: %w", raw, err)
|
||||
}
|
||||
nets = append(nets, n)
|
||||
}
|
||||
allowMu.Lock()
|
||||
allowHosts = h
|
||||
allowCIDRs = nets
|
||||
allowConfigured = true
|
||||
allowMu.Unlock()
|
||||
return nil
|
||||
}
|
||||
|
||||
// ApplyPrivateAllowlistCSV merges env FEED_URL_PRIVATE_ALLOWLIST with an optional
|
||||
// admin/settings CSV (settings override by appending unique entries).
|
||||
func ApplyPrivateAllowlistCSV(settingsCSV string) {
|
||||
parts := make([]string, 0, 8)
|
||||
for _, src := range []string{os.Getenv("FEED_URL_PRIVATE_ALLOWLIST"), settingsCSV} {
|
||||
for _, part := range strings.Split(src, ",") {
|
||||
part = strings.TrimSpace(part)
|
||||
if part != "" {
|
||||
parts = append(parts, part)
|
||||
}
|
||||
}
|
||||
}
|
||||
hosts := make([]string, 0)
|
||||
cidrs := make([]string, 0)
|
||||
seen := map[string]struct{}{}
|
||||
for _, part := range parts {
|
||||
key := strings.ToLower(part)
|
||||
if _, ok := seen[key]; ok {
|
||||
continue
|
||||
}
|
||||
seen[key] = struct{}{}
|
||||
if strings.Contains(part, "/") {
|
||||
cidrs = append(cidrs, part)
|
||||
} else {
|
||||
hosts = append(hosts, part)
|
||||
}
|
||||
}
|
||||
_ = ConfigurePrivateAllowlist(hosts, cidrs)
|
||||
}
|
||||
|
||||
func ensureAllowlist() {
|
||||
allowMu.RLock()
|
||||
done := allowConfigured
|
||||
allowMu.RUnlock()
|
||||
if done {
|
||||
return
|
||||
}
|
||||
allowMu.Lock()
|
||||
defer allowMu.Unlock()
|
||||
if allowConfigured {
|
||||
return
|
||||
}
|
||||
allowHosts = map[string]struct{}{}
|
||||
allowCIDRs = nil
|
||||
raw := strings.TrimSpace(os.Getenv("FEED_URL_PRIVATE_ALLOWLIST"))
|
||||
for _, part := range strings.Split(raw, ",") {
|
||||
part = strings.TrimSpace(part)
|
||||
if part == "" {
|
||||
continue
|
||||
}
|
||||
if strings.Contains(part, "/") {
|
||||
if _, n, err := net.ParseCIDR(part); err == nil {
|
||||
allowCIDRs = append(allowCIDRs, n)
|
||||
}
|
||||
continue
|
||||
}
|
||||
allowHosts[strings.ToLower(part)] = struct{}{}
|
||||
}
|
||||
allowConfigured = true
|
||||
}
|
||||
|
||||
// ValidateFeedURL checks a feed source URL for SSRF before persist.
|
||||
// Empty URL is allowed (local upload sources). Does not fetch the URL.
|
||||
func ValidateFeedURL(ctx context.Context, rawURL string) error {
|
||||
ensureAllowlist()
|
||||
rawURL = strings.TrimSpace(rawURL)
|
||||
if rawURL == "" {
|
||||
return nil
|
||||
}
|
||||
if len(rawURL) > 2048 {
|
||||
return errURLScheme
|
||||
}
|
||||
lower := strings.ToLower(rawURL)
|
||||
if strings.HasPrefix(lower, "ftp://") || strings.HasPrefix(lower, "ftps://") {
|
||||
return errURLFTP
|
||||
}
|
||||
u, err := url.Parse(rawURL)
|
||||
if err != nil || u.Host == "" {
|
||||
return ClientMsg("invalid url")
|
||||
}
|
||||
if u.Scheme != "http" && u.Scheme != "https" {
|
||||
return errURLScheme
|
||||
}
|
||||
return assertPublicHost(ctx, u.Hostname())
|
||||
}
|
||||
|
||||
// downloadFeed streams an http(s) feed to a temp file with SSRF controls and a hard size cap.
|
||||
// Callers must Close the returned blob to remove the temp file.
|
||||
func downloadFeed(ctx context.Context, rawURL string) (*feedBlob, error) {
|
||||
ensureAllowlist()
|
||||
rawURL = strings.TrimSpace(rawURL)
|
||||
if rawURL == "" {
|
||||
return nil, errURLRequired
|
||||
}
|
||||
lower := strings.ToLower(rawURL)
|
||||
if strings.HasPrefix(lower, "ftp://") || strings.HasPrefix(lower, "ftps://") {
|
||||
return nil, errURLFTP
|
||||
}
|
||||
|
||||
u, err := url.Parse(rawURL)
|
||||
if err != nil || u.Host == "" {
|
||||
return nil, ClientMsg("invalid url")
|
||||
}
|
||||
if u.Scheme != "http" && u.Scheme != "https" {
|
||||
return nil, errURLScheme
|
||||
}
|
||||
if err := assertPublicHost(ctx, u.Hostname()); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
client := &http.Client{
|
||||
Timeout: downloadTimeout,
|
||||
Transport: ssrfTransport(),
|
||||
CheckRedirect: func(req *http.Request, via []*http.Request) error {
|
||||
if len(via) >= maxRedirects {
|
||||
return ClientMsg("too many redirects")
|
||||
}
|
||||
if req.URL.Scheme != "http" && req.URL.Scheme != "https" {
|
||||
return errURLScheme
|
||||
}
|
||||
return assertPublicHost(req.Context(), req.URL.Hostname())
|
||||
},
|
||||
}
|
||||
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, u.String(), nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
req.Header.Set("User-Agent", defaultUserAgent)
|
||||
req.Header.Set("Accept", "text/csv,application/csv,application/xml,text/xml,application/atom+xml,*/*")
|
||||
|
||||
res, err := client.Do(req)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("download failed: %w", err)
|
||||
}
|
||||
defer res.Body.Close()
|
||||
|
||||
if res.StatusCode < 200 || res.StatusCode >= 300 {
|
||||
return nil, fmt.Errorf("download status %d", res.StatusCode)
|
||||
}
|
||||
if res.ContentLength > maxDownloadBytes {
|
||||
return nil, downloadTooLarge()
|
||||
}
|
||||
|
||||
tmp, err := os.CreateTemp("", "descrybe-feed-*")
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("temp file: %w", err)
|
||||
}
|
||||
tmpPath := tmp.Name()
|
||||
cleanup := true
|
||||
defer func() {
|
||||
_ = tmp.Close()
|
||||
if cleanup {
|
||||
_ = os.Remove(tmpPath)
|
||||
}
|
||||
}()
|
||||
|
||||
limited := io.LimitReader(res.Body, maxDownloadBytes+1)
|
||||
written, err := io.Copy(tmp, limited)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("read body: %w", err)
|
||||
}
|
||||
if written > maxDownloadBytes {
|
||||
return nil, downloadTooLarge()
|
||||
}
|
||||
if err := tmp.Close(); err != nil {
|
||||
return nil, fmt.Errorf("close temp: %w", err)
|
||||
}
|
||||
cleanup = false
|
||||
return &feedBlob{
|
||||
path: tmpPath,
|
||||
contentType: res.Header.Get("Content-Type"),
|
||||
size: written,
|
||||
owned: true,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func ssrfTransport() *http.Transport {
|
||||
dialer := &net.Dialer{Timeout: dialTimeout, KeepAlive: 30 * time.Second}
|
||||
return &http.Transport{
|
||||
// Never honor HTTP(S)_PROXY: dialing a proxy skips destination SSRF checks.
|
||||
Proxy: nil,
|
||||
DialContext: func(ctx context.Context, network, addr string) (net.Conn, error) {
|
||||
host, port, err := net.SplitHostPort(addr)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := assertPublicHost(ctx, host); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
ips, err := resolveHostIPs(ctx, host)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var lastErr error
|
||||
for _, ip := range ips {
|
||||
if isBlockedIP(ip) && !isAllowlistedHostOrIP(host, ip) {
|
||||
lastErr = errURLPrivate
|
||||
continue
|
||||
}
|
||||
target := net.JoinHostPort(ip.String(), port)
|
||||
conn, err := dialer.DialContext(ctx, network, target)
|
||||
if err == nil {
|
||||
return conn, nil
|
||||
}
|
||||
lastErr = err
|
||||
}
|
||||
if lastErr == nil {
|
||||
lastErr = errURLPrivate
|
||||
}
|
||||
return nil, lastErr
|
||||
},
|
||||
ForceAttemptHTTP2: true,
|
||||
MaxIdleConns: 10,
|
||||
IdleConnTimeout: 90 * time.Second,
|
||||
TLSHandshakeTimeout: 10 * time.Second,
|
||||
ExpectContinueTimeout: 1 * time.Second,
|
||||
ResponseHeaderTimeout: 30 * time.Second,
|
||||
}
|
||||
}
|
||||
|
||||
func resolveHostIPs(ctx context.Context, host string) ([]net.IP, error) {
|
||||
if ip := net.ParseIP(host); ip != nil {
|
||||
return []net.IP{ip}, nil
|
||||
}
|
||||
addrs, err := net.DefaultResolver.LookupIPAddr(ctx, host)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("dns lookup: %w", err)
|
||||
}
|
||||
out := make([]net.IP, 0, len(addrs))
|
||||
for _, a := range addrs {
|
||||
out = append(out, a.IP)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func assertPublicHost(ctx context.Context, host string) error {
|
||||
ensureAllowlist()
|
||||
host = strings.TrimSpace(host)
|
||||
if host == "" {
|
||||
return errURLPrivate
|
||||
}
|
||||
lower := strings.ToLower(host)
|
||||
allowMu.RLock()
|
||||
_, hostAllowed := allowHosts[lower]
|
||||
allowMu.RUnlock()
|
||||
if hostAllowed {
|
||||
return nil
|
||||
}
|
||||
if lower == "localhost" || strings.HasSuffix(lower, ".localhost") || strings.HasSuffix(lower, ".local") {
|
||||
return errURLPrivate
|
||||
}
|
||||
if ip := net.ParseIP(host); ip != nil {
|
||||
if isBlockedIP(ip) && !isAllowlistedIP(ip) {
|
||||
return errURLPrivate
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
addrs, err := resolveHostIPs(ctx, host)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if len(addrs) == 0 {
|
||||
return errURLPrivate
|
||||
}
|
||||
for _, ip := range addrs {
|
||||
if isBlockedIP(ip) && !isAllowlistedIP(ip) {
|
||||
return errURLPrivate
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func isAllowlistedHostOrIP(host string, ip net.IP) bool {
|
||||
ensureAllowlist()
|
||||
allowMu.RLock()
|
||||
_, ok := allowHosts[strings.ToLower(strings.TrimSpace(host))]
|
||||
allowMu.RUnlock()
|
||||
if ok {
|
||||
return true
|
||||
}
|
||||
return isAllowlistedIP(ip)
|
||||
}
|
||||
|
||||
func isAllowlistedIP(ip net.IP) bool {
|
||||
ensureAllowlist()
|
||||
if ip == nil {
|
||||
return false
|
||||
}
|
||||
allowMu.RLock()
|
||||
defer allowMu.RUnlock()
|
||||
for _, n := range allowCIDRs {
|
||||
if n.Contains(ip) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func isBlockedIP(ip net.IP) bool {
|
||||
if ip == nil {
|
||||
return true
|
||||
}
|
||||
if ip.IsLoopback() || ip.IsPrivate() || ip.IsLinkLocalUnicast() || ip.IsLinkLocalMulticast() ||
|
||||
ip.IsMulticast() || ip.IsUnspecified() {
|
||||
return true
|
||||
}
|
||||
// AWS/GCP/Azure metadata and CGNAT.
|
||||
if ip4 := ip.To4(); ip4 != nil {
|
||||
if ip4[0] == 169 && ip4[1] == 254 {
|
||||
return true
|
||||
}
|
||||
if ip4[0] == 100 && ip4[1] >= 64 && ip4[1] <= 127 {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
@@ -0,0 +1,59 @@
|
||||
package feeds
|
||||
|
||||
import (
|
||||
"errors"
|
||||
|
||||
"github.com/jackc/pgx/v5"
|
||||
)
|
||||
|
||||
// ErrFormatMismatch is returned when the public export URL extension does not
|
||||
// match the feed's configured format. Public HTTP handlers must map this to the
|
||||
// same opaque 404 as an unknown token (no existence oracle).
|
||||
var ErrFormatMismatch = errors.New("format mismatch")
|
||||
|
||||
// ErrNotFound is returned when a company-scoped feed (or related row) is missing.
|
||||
var ErrNotFound = errors.New("not found")
|
||||
|
||||
// clientError is a validation/business message safe to return to API clients.
|
||||
type clientError struct {
|
||||
msg string
|
||||
}
|
||||
|
||||
func (e *clientError) Error() string { return e.msg }
|
||||
|
||||
// ClientMsg marks a message as safe to expose in HTTP 4xx responses.
|
||||
func ClientMsg(msg string) error {
|
||||
return &clientError{msg: msg}
|
||||
}
|
||||
|
||||
// ClientError reports whether err is a known client-facing feeds error.
|
||||
func ClientError(err error) (msg string, ok bool) {
|
||||
if err == nil {
|
||||
return "", false
|
||||
}
|
||||
var ce *clientError
|
||||
if errors.As(err, &ce) {
|
||||
return ce.msg, true
|
||||
}
|
||||
switch {
|
||||
case errors.Is(err, ErrNotFound), errors.Is(err, pgx.ErrNoRows):
|
||||
return "not found", true
|
||||
case errors.Is(err, ErrFormatMismatch),
|
||||
errors.Is(err, errURLRequired),
|
||||
errors.Is(err, errURLScheme),
|
||||
errors.Is(err, errURLPrivate),
|
||||
errors.Is(err, errURLFTP),
|
||||
errors.Is(err, errDownloadTooLarge),
|
||||
errors.Is(err, errParseTooManyRows),
|
||||
errors.Is(err, errSourceRequired),
|
||||
errors.Is(err, errLocalSource):
|
||||
return err.Error(), true
|
||||
default:
|
||||
return "", false
|
||||
}
|
||||
}
|
||||
|
||||
// IsNotFound reports whether err means a missing feed/resource.
|
||||
func IsNotFound(err error) bool {
|
||||
return errors.Is(err, ErrNotFound) || errors.Is(err, pgx.ErrNoRows)
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,98 @@
|
||||
package feeds
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/google/uuid"
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
)
|
||||
|
||||
// TestRotateExportFeedPublicToken creates an ephemeral sandbox company (never A1 /
|
||||
// Platform Demo), rotates the export public token, and asserts revoke semantics.
|
||||
func TestRotateExportFeedPublicToken(t *testing.T) {
|
||||
dsn := strings.TrimSpace(os.Getenv("DATABASE_URL"))
|
||||
if dsn == "" {
|
||||
t.Skip("DATABASE_URL not set")
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer cancel()
|
||||
pg, err := pgxpool.New(ctx, dsn)
|
||||
if err != nil {
|
||||
t.Fatalf("pool: %v", err)
|
||||
}
|
||||
defer pg.Close()
|
||||
|
||||
companyID := uuid.New()
|
||||
prefix := companyID.String()[:8]
|
||||
_, err = pg.Exec(ctx, `INSERT INTO companies (id, name) VALUES ($1, $2)`,
|
||||
companyID, "export-rotate-"+prefix)
|
||||
if err != nil {
|
||||
t.Fatalf("seed company: %v", err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
_, _ = pg.Exec(context.Background(), `DELETE FROM companies WHERE id = $1`, companyID)
|
||||
})
|
||||
|
||||
svc := &Service{Pool: pg}
|
||||
created, err := svc.CreateExportFeed(ctx, companyID, CreateExportInput{
|
||||
Name: "rotate-smoke-" + prefix,
|
||||
Format: "xml",
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("CreateExportFeed: %v", err)
|
||||
}
|
||||
oldTok, _ := created["public_token"].(string)
|
||||
if !validPublicToken(oldTok) || len(oldTok) != 64 {
|
||||
t.Fatalf("create public_token=%q want 64-hex", oldTok)
|
||||
}
|
||||
feedID, ok := created["id"].(uuid.UUID)
|
||||
if !ok {
|
||||
idStr := fmt.Sprint(created["id"])
|
||||
feedID, err = uuid.Parse(idStr)
|
||||
if err != nil {
|
||||
t.Fatalf("export id: %v (%v)", err, created["id"])
|
||||
}
|
||||
}
|
||||
|
||||
rotated, err := svc.RotateExportFeedPublicToken(ctx, companyID, feedID)
|
||||
if err != nil {
|
||||
t.Fatalf("RotateExportFeedPublicToken: %v", err)
|
||||
}
|
||||
newTok, _ := rotated["public_token"].(string)
|
||||
if !validPublicToken(newTok) || len(newTok) != 64 {
|
||||
t.Fatalf("rotated public_token=%q want 64-hex", newTok)
|
||||
}
|
||||
if newTok == oldTok {
|
||||
t.Fatal("rotate must replace public_token")
|
||||
}
|
||||
|
||||
got, err := svc.GetExportFeed(ctx, companyID, feedID)
|
||||
if err != nil {
|
||||
t.Fatalf("GetExportFeed: %v", err)
|
||||
}
|
||||
if fmt.Sprint(got["public_token"]) != newTok {
|
||||
t.Fatalf("persisted token=%v want %s", got["public_token"], newTok)
|
||||
}
|
||||
|
||||
var byOld int
|
||||
err = pg.QueryRow(ctx, `
|
||||
SELECT COUNT(*) FROM export_feeds
|
||||
WHERE company_id = $1 AND public_token = $2`, companyID, oldTok).Scan(&byOld)
|
||||
if err != nil {
|
||||
t.Fatalf("count old token: %v", err)
|
||||
}
|
||||
if byOld != 0 {
|
||||
t.Fatal("old public_token still present after rotate (not revoked)")
|
||||
}
|
||||
|
||||
otherCompany := uuid.New()
|
||||
_, err = svc.RotateExportFeedPublicToken(ctx, otherCompany, feedID)
|
||||
if err == nil || !strings.Contains(err.Error(), "not found") {
|
||||
t.Fatalf("cross-tenant rotate err=%v want not found", err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,382 @@
|
||||
package feeds
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestValidPublicToken(t *testing.T) {
|
||||
if validPublicToken("../etc/passwd") {
|
||||
t.Fatal("path traversal token must be rejected")
|
||||
}
|
||||
if validPublicToken("short") {
|
||||
t.Fatal("short token must be rejected")
|
||||
}
|
||||
if validPublicToken("0123456789abcdef") { // 16 hex / 64-bit — below floor
|
||||
t.Fatal("undersized token must be rejected")
|
||||
}
|
||||
if validPublicToken("0123456789abcdef0123456789abcde") { // odd length
|
||||
t.Fatal("odd-length hex must be rejected")
|
||||
}
|
||||
if !validPublicToken("0123456789abcdef0123456789abcdef") {
|
||||
t.Fatal("32-hex token should be accepted")
|
||||
}
|
||||
tok, err := newPublicExportToken()
|
||||
if err != nil {
|
||||
t.Fatalf("newPublicExportToken: %v", err)
|
||||
}
|
||||
if len(tok) != 64 {
|
||||
t.Fatalf("expected 64 hex chars, got %d", len(tok))
|
||||
}
|
||||
if !validPublicToken(tok) {
|
||||
t.Fatal("fresh public export token should be accepted")
|
||||
}
|
||||
tok2, err := newPublicExportToken()
|
||||
if err != nil {
|
||||
t.Fatalf("newPublicExportToken second: %v", err)
|
||||
}
|
||||
if tok == tok2 {
|
||||
t.Fatal("rotated/fresh tokens must differ (CSPRNG collision)")
|
||||
}
|
||||
}
|
||||
|
||||
func TestNewPublicExportTokenIs256BitHex(t *testing.T) {
|
||||
t.Parallel()
|
||||
for i := 0; i < 8; i++ {
|
||||
tok, err := newPublicExportToken()
|
||||
if err != nil {
|
||||
t.Fatalf("newPublicExportToken: %v", err)
|
||||
}
|
||||
if len(tok) != publicExportTokenBytes*2 {
|
||||
t.Fatalf("len=%d want %d", len(tok), publicExportTokenBytes*2)
|
||||
}
|
||||
if !validPublicToken(tok) {
|
||||
t.Fatalf("token %q rejected by validPublicToken", tok)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestSanitizeXMLName(t *testing.T) {
|
||||
if got := sanitizeXMLName("prod uct!", "product"); got != "prod_uct_" {
|
||||
t.Fatalf("got %q", got)
|
||||
}
|
||||
if got := sanitizeXMLName("", "product"); got != "product" {
|
||||
t.Fatalf("empty fallback got %q", got)
|
||||
}
|
||||
if got := sanitizeXMLName("g:id", "field"); got != "g:id" {
|
||||
t.Fatalf("namespace colon got %q", got)
|
||||
}
|
||||
if got := sanitizeXMLName("g:title", "field"); got != "g:title" {
|
||||
t.Fatalf("g:title got %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseExportTemplateDefaults(t *testing.T) {
|
||||
tpl := parseExportTemplate([]byte("{}"))
|
||||
if tpl.Root != defaultExportRoot || tpl.Item != defaultExportItem {
|
||||
t.Fatalf("unexpected defaults %#v", tpl)
|
||||
}
|
||||
if len(tpl.Fields) < 3 {
|
||||
t.Fatal("expected default fields")
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseExportTemplateMappingsSorted(t *testing.T) {
|
||||
tpl := parseExportTemplate([]byte(`{"mappings":{"z":"status","a":"name"}}`))
|
||||
if len(tpl.Fields) != 2 {
|
||||
t.Fatalf("fields=%d", len(tpl.Fields))
|
||||
}
|
||||
if tpl.Fields[0].Key != "a" || tpl.Fields[1].Key != "z" {
|
||||
t.Fatalf("unsorted keys: %#v", tpl.Fields)
|
||||
}
|
||||
}
|
||||
|
||||
func TestProductFieldValuePrefersProcessed(t *testing.T) {
|
||||
name := "Widget"
|
||||
processed := "Widget Pro"
|
||||
p := exportProduct{Name: &name, ProcessedName: &processed, Attributes: []byte(`{"color":"red"}`)}
|
||||
if got := productFieldValue(p, "name"); got != "Widget Pro" {
|
||||
t.Fatalf("name=%q", got)
|
||||
}
|
||||
if got := productFieldValue(p, "attr.color"); got != "red" {
|
||||
t.Fatalf("attr=%q", got)
|
||||
}
|
||||
if !strings.Contains(xmlEscape(`a&b<c>`), "&") {
|
||||
t.Fatal("xmlEscape broken")
|
||||
}
|
||||
}
|
||||
|
||||
func TestSanitizeExportFileName(t *testing.T) {
|
||||
if got := sanitizeExportFileName("My Feed!"); got != "my_feed" {
|
||||
t.Fatalf("got %q", got)
|
||||
}
|
||||
if got := sanitizeExportFileName(""); got != "export" {
|
||||
t.Fatalf("empty got %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestScalarAttrStringStructuredAndNull(t *testing.T) {
|
||||
if got := scalarAttrString(nil); got != "" {
|
||||
t.Fatalf("nil=%q", got)
|
||||
}
|
||||
if got := scalarAttrString(map[string]any{"key": "oblika-zaslona-2", "name": "ukrivljen"}); got != "ukrivljen" {
|
||||
t.Fatalf("name object=%q", got)
|
||||
}
|
||||
if got := scalarAttrString(map[string]any{"key": "brand", "value": "CoolCo"}); got != "CoolCo" {
|
||||
t.Fatalf("value object=%q", got)
|
||||
}
|
||||
out := map[string]string{}
|
||||
putAttrValue(out, "barva", nil)
|
||||
putAttrValue(out, "oblika-zaslona", map[string]any{"key": "oblika-zaslona-2", "name": "ukrivljen"})
|
||||
if _, ok := out["barva"]; ok {
|
||||
t.Fatal("nil attr should be skipped")
|
||||
}
|
||||
if out["oblika-zaslona"] != "ukrivljen" {
|
||||
t.Fatalf("oblika=%q", out["oblika-zaslona"])
|
||||
}
|
||||
}
|
||||
|
||||
func sampleProcessedProduct() exportProduct {
|
||||
pid := "4897098683545"
|
||||
name := "Fridge X"
|
||||
procName := "Fridge X Energy"
|
||||
status := "processed"
|
||||
return exportProduct{
|
||||
ProductID: &pid,
|
||||
Name: &name,
|
||||
ProcessedName: &procName,
|
||||
Status: &status,
|
||||
Attributes: []byte(`{"color":"silver"}`),
|
||||
ProcessedAttributes: []byte(`{
|
||||
"eprel_id": "246834",
|
||||
"brand": {"key":"brand","name":"CoolCo","value":"CoolCo"},
|
||||
"specifications": [
|
||||
{"key":"battery_life","value":"30 hours"},
|
||||
{"key":"weight","value":"250g"}
|
||||
],
|
||||
"eprel": {
|
||||
"energy_class": "E",
|
||||
"energy_scale": "A-G",
|
||||
"label": "https://eprel.ec.europa.eu/api/product/246834/labels?format=png",
|
||||
"pdf": "https://eprel.ec.europa.eu/fiches/example.pdf"
|
||||
}
|
||||
}`),
|
||||
MappedData: []byte(`{"eprel_id":"should-not-win"}`),
|
||||
}
|
||||
}
|
||||
|
||||
func TestFlattenSpecificationsAndEprel(t *testing.T) {
|
||||
p := sampleProcessedProduct()
|
||||
if got := productFieldValue(p, "eprel_id"); got != "246834" {
|
||||
t.Fatalf("eprel_id=%q", got)
|
||||
}
|
||||
if got := productFieldValue(p, "energy_class"); got != "E" {
|
||||
t.Fatalf("energy_class=%q", got)
|
||||
}
|
||||
if got := productFieldValue(p, "eprel_energy_class"); got != "E" {
|
||||
t.Fatalf("eprel_energy_class=%q", got)
|
||||
}
|
||||
if got := productFieldValue(p, "eprel_label"); !strings.Contains(got, "eprel.ec.europa.eu") {
|
||||
t.Fatalf("eprel_label=%q", got)
|
||||
}
|
||||
if got := productFieldValue(p, "spec.battery_life"); got != "30 hours" {
|
||||
t.Fatalf("spec.battery_life=%q", got)
|
||||
}
|
||||
if got := productFieldValue(p, "attr.weight"); got != "250g" {
|
||||
t.Fatalf("attr.weight=%q", got)
|
||||
}
|
||||
if got := productFieldValue(p, "attr.brand"); got != "CoolCo" {
|
||||
t.Fatalf("brand=%q", got)
|
||||
}
|
||||
specs := productFieldValue(p, "specifications")
|
||||
if !strings.Contains(specs, "battery_life: 30 hours") || !strings.Contains(specs, "weight: 250g") {
|
||||
t.Fatalf("specifications=%q", specs)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFlattenSpecificationsObject(t *testing.T) {
|
||||
p := exportProduct{
|
||||
ProcessedAttributes: []byte(`{"specifications":{"color":"red","size":"L"}}`),
|
||||
}
|
||||
if got := productFieldValue(p, "spec.color"); got != "red" {
|
||||
t.Fatalf("got %q", got)
|
||||
}
|
||||
if got := productFieldValue(p, "specifications.size"); got != "L" {
|
||||
t.Fatalf("got %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestExportXMLSnippetWithSpecsAndEprel(t *testing.T) {
|
||||
tpl := exportTemplate{
|
||||
Root: "products",
|
||||
Item: "product",
|
||||
Fields: []exportField{
|
||||
{Key: "product_id", Source: "product_id"},
|
||||
{Key: "name", Source: "name"},
|
||||
{Key: "eprel_id", Source: "eprel_id"},
|
||||
{Key: "energy_class", Source: "energy_class"},
|
||||
{Key: "eprel_label", Source: "eprel_label"},
|
||||
{Key: "specs", Source: "specifications.*"},
|
||||
},
|
||||
}
|
||||
out, n, err := renderExportSnippet("xml", tpl, []exportProduct{sampleProcessedProduct()}, "Demo Feed")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if n != 1 {
|
||||
t.Fatalf("count=%d", n)
|
||||
}
|
||||
for _, want := range []string{
|
||||
`<?xml version="1.0" encoding="UTF-8"?>`,
|
||||
`<products feed="Demo Feed">`,
|
||||
`<product_id>4897098683545</product_id>`,
|
||||
`<name>Fridge X Energy</name>`,
|
||||
`<eprel_id>246834</eprel_id>`,
|
||||
`<energy_class>E</energy_class>`,
|
||||
`<battery_life>30 hours</battery_life>`,
|
||||
`<weight>250g</weight>`,
|
||||
`</product>`,
|
||||
`</products>`,
|
||||
} {
|
||||
if !strings.Contains(out, want) {
|
||||
t.Fatalf("missing %q in:\n%s", want, out)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestExportCSVSnippetWithEprel(t *testing.T) {
|
||||
tpl := exportTemplate{
|
||||
Fields: []exportField{
|
||||
{Key: "product_id", Source: "product_id"},
|
||||
{Key: "eprel_id", Source: "eprel_id"},
|
||||
{Key: "energy_class", Source: "energy_class"},
|
||||
{Key: "battery_life", Source: "spec.battery_life"},
|
||||
},
|
||||
}
|
||||
out, n, err := renderExportSnippet("csv", tpl, []exportProduct{sampleProcessedProduct()}, "")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if n != 1 {
|
||||
t.Fatalf("count=%d", n)
|
||||
}
|
||||
lines := strings.Split(strings.TrimSpace(out), "\n")
|
||||
if len(lines) != 2 {
|
||||
t.Fatalf("lines=%v", lines)
|
||||
}
|
||||
if lines[0] != "product_id,eprel_id,energy_class,battery_life" {
|
||||
t.Fatalf("header=%q", lines[0])
|
||||
}
|
||||
if lines[1] != "4897098683545,246834,E,30 hours" {
|
||||
t.Fatalf("row=%q", lines[1])
|
||||
}
|
||||
}
|
||||
|
||||
func TestProcessedAttributesWinOverMappedData(t *testing.T) {
|
||||
p := sampleProcessedProduct()
|
||||
if got := productFieldValue(p, "eprel_id"); got != "246834" {
|
||||
t.Fatalf("expected processed eprel_id, got %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestStreamXMLUsesProductSeqWithoutCollecting(t *testing.T) {
|
||||
tpl := exportTemplate{
|
||||
Root: "products",
|
||||
Item: "product",
|
||||
Fields: []exportField{
|
||||
{Key: "product_id", Source: "product_id"},
|
||||
{Key: "name", Source: "name"},
|
||||
},
|
||||
}
|
||||
yielded := 0
|
||||
seq := exportProductSeq(func(yield func(exportProduct) error) error {
|
||||
for i := 0; i < 3; i++ {
|
||||
yielded++
|
||||
if err := yield(sampleProcessedProduct()); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
})
|
||||
var buf bytes.Buffer
|
||||
n, err := streamXML(&buf, seq, tpl, "Batch Feed")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if n != 3 || yielded != 3 {
|
||||
t.Fatalf("count=%d yielded=%d", n, yielded)
|
||||
}
|
||||
out := buf.String()
|
||||
if !strings.Contains(out, `<products feed="Batch Feed">`) || !strings.Contains(out, "</products>") {
|
||||
t.Fatalf("bad xml:\n%s", out)
|
||||
}
|
||||
if strings.Count(out, "<product>") != 3 {
|
||||
t.Fatalf("expected 3 products, got:\n%s", out)
|
||||
}
|
||||
}
|
||||
|
||||
func TestStreamCSVUsesProductSeqWithoutCollecting(t *testing.T) {
|
||||
tpl := exportTemplate{
|
||||
Fields: []exportField{
|
||||
{Key: "product_id", Source: "product_id"},
|
||||
{Key: "name", Source: "name"},
|
||||
},
|
||||
}
|
||||
seq := exportProductSeq(func(yield func(exportProduct) error) error {
|
||||
return yield(sampleProcessedProduct())
|
||||
})
|
||||
var buf bytes.Buffer
|
||||
n, err := streamCSV(&buf, seq, tpl)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if n != 1 {
|
||||
t.Fatalf("count=%d", n)
|
||||
}
|
||||
lines := strings.Split(strings.TrimSpace(buf.String()), "\n")
|
||||
if len(lines) != 2 {
|
||||
t.Fatalf("lines=%v", lines)
|
||||
}
|
||||
if lines[0] != "product_id,name" {
|
||||
t.Fatalf("header=%q", lines[0])
|
||||
}
|
||||
if lines[1] != "4897098683545,Fridge X Energy" {
|
||||
t.Fatalf("row=%q", lines[1])
|
||||
}
|
||||
}
|
||||
|
||||
func TestExportBatchSizeBoundsMemoryPages(t *testing.T) {
|
||||
if exportBatchSize <= 0 || exportBatchSize > exportMaxProducts {
|
||||
t.Fatalf("exportBatchSize=%d exportMaxProducts=%d", exportBatchSize, exportMaxProducts)
|
||||
}
|
||||
if exportChunkHint <= 0 || exportChunkHint > exportBatchSize {
|
||||
t.Fatalf("exportChunkHint=%d should be positive and <= batch", exportChunkHint)
|
||||
}
|
||||
if exportSelectedMaxProducts <= 0 || exportSelectedMaxProducts > exportMaxProducts {
|
||||
t.Fatalf("exportSelectedMaxProducts=%d must be in (0, %d]", exportSelectedMaxProducts, exportMaxProducts)
|
||||
}
|
||||
}
|
||||
|
||||
func TestStreamXMLPropagatesSeqError(t *testing.T) {
|
||||
tpl := exportTemplate{
|
||||
Root: "products",
|
||||
Item: "product",
|
||||
Fields: []exportField{{Key: "name", Source: "name"}},
|
||||
}
|
||||
seq := exportProductSeq(func(yield func(exportProduct) error) error {
|
||||
if err := yield(sampleProcessedProduct()); err != nil {
|
||||
return err
|
||||
}
|
||||
return errors.New("boom")
|
||||
})
|
||||
var buf bytes.Buffer
|
||||
n, err := streamXML(&buf, seq, tpl, "x")
|
||||
if err == nil || err.Error() != "boom" {
|
||||
t.Fatalf("err=%v count=%d", err, n)
|
||||
}
|
||||
if n != 1 {
|
||||
t.Fatalf("expected 1 product written before error, got %d", n)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,529 @@
|
||||
package feeds
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/csv"
|
||||
"encoding/xml"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"sort"
|
||||
"strings"
|
||||
"unicode"
|
||||
|
||||
"github.com/google/uuid"
|
||||
)
|
||||
|
||||
const (
|
||||
schemaSampleBytes = 512 << 10 // 512 KiB preview window
|
||||
schemaMaxRows = 25
|
||||
schemaMaxSamples = 5
|
||||
schemaMaxFields = 200
|
||||
previewMaxLines = 80
|
||||
)
|
||||
|
||||
// SchemaField is one discovered source column/xpath with sample values.
|
||||
type SchemaField struct {
|
||||
Path string `json:"path"`
|
||||
FieldName string `json:"field_name"`
|
||||
DataType string `json:"data_type"`
|
||||
SampleValues []string `json:"sample_values"`
|
||||
UniqueValuesCount int `json:"unique_values_count"`
|
||||
SuggestedTarget string `json:"suggested_target,omitempty"`
|
||||
}
|
||||
|
||||
// SchemaExtractResult is returned by POST /feeds/{id}/extract-schema.
|
||||
type SchemaExtractResult struct {
|
||||
FeedID string `json:"feed_id"`
|
||||
Format string `json:"format"`
|
||||
SuggestedPath string `json:"suggested_item_path,omitempty"`
|
||||
ItemPath string `json:"item_path,omitempty"`
|
||||
Fields []SchemaField `json:"fields"`
|
||||
SampleRows int `json:"sample_rows"`
|
||||
Preview string `json:"preview,omitempty"`
|
||||
PreviewTruncated bool `json:"preview_truncated,omitempty"`
|
||||
}
|
||||
|
||||
// ExtractSchema downloads a bounded sample of the feed and returns field paths + samples.
|
||||
func (s *Service) ExtractSchema(ctx context.Context, companyID, feedID uuid.UUID, itemPathHint string) (*SchemaExtractResult, error) {
|
||||
feed, err := s.Get(ctx, companyID, feedID)
|
||||
if err != nil {
|
||||
if IsNotFound(err) {
|
||||
return nil, ErrNotFound
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
urlStr, _ := feed["url"].(string)
|
||||
feedType, _ := feed["feed_type"].(string)
|
||||
|
||||
itemPathHint = strings.TrimSpace(itemPathHint)
|
||||
if itemPathHint == "" {
|
||||
if m, err := s.GetMappings(ctx, companyID, feedID); err == nil {
|
||||
itemPathHint = itemPathFromMappings(m["mappings"])
|
||||
}
|
||||
if itemPathHint == "" {
|
||||
if opts, ok := feed["options"].(map[string]any); ok {
|
||||
if v, ok := opts["item_path"].(string); ok {
|
||||
itemPathHint = strings.TrimSpace(v)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
src, err := s.loadFeedSource(ctx, companyID, feed)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer src.Close()
|
||||
|
||||
f, err := src.Open()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer f.Close()
|
||||
|
||||
truncated := src.size > schemaSampleBytes
|
||||
data, err := io.ReadAll(io.LimitReader(f, schemaSampleBytes))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
format := detectFeedFormat(feedType, src.contentType, urlStr, data)
|
||||
out := &SchemaExtractResult{
|
||||
FeedID: feedID.String(),
|
||||
Format: format,
|
||||
ItemPath: itemPathHint,
|
||||
PreviewTruncated: truncated,
|
||||
}
|
||||
|
||||
switch format {
|
||||
case "xml":
|
||||
suggested := guessXMLItemPath(data)
|
||||
out.SuggestedPath = suggested
|
||||
local := itemLocalFromPath(itemPathHint)
|
||||
if local == "" {
|
||||
local = itemLocalFromPath(suggested)
|
||||
out.ItemPath = suggested
|
||||
} else {
|
||||
out.ItemPath = itemPathHint
|
||||
}
|
||||
fields, rows, err := extractXMLSchema(data, local)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out.Fields = fields
|
||||
out.SampleRows = rows
|
||||
out.Preview = buildXMLPreview(data, local)
|
||||
default:
|
||||
fields, rows, preview, err := extractCSVSchema(data)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out.Fields = fields
|
||||
out.SampleRows = rows
|
||||
out.Preview = preview
|
||||
out.SuggestedPath = ""
|
||||
out.ItemPath = ""
|
||||
}
|
||||
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func itemPathFromMappings(raw any) string {
|
||||
switch t := raw.(type) {
|
||||
case map[string]any:
|
||||
if v, ok := t["item_path"].(string); ok {
|
||||
if p := strings.TrimSpace(v); p != "" {
|
||||
return p
|
||||
}
|
||||
}
|
||||
if fields, ok := t["fields"]; ok {
|
||||
if p := itemPathFromMappings(fields); p != "" {
|
||||
return p
|
||||
}
|
||||
}
|
||||
if nested, ok := t["mappings"]; ok {
|
||||
return itemPathFromMappings(nested)
|
||||
}
|
||||
return ""
|
||||
case []any, []FieldMapping:
|
||||
return deriveItemPathFromMappings(parseMappings(t))
|
||||
default:
|
||||
if parsed := parseMappings(raw); len(parsed) > 0 {
|
||||
return deriveItemPathFromMappings(parsed)
|
||||
}
|
||||
return ""
|
||||
}
|
||||
}
|
||||
|
||||
// deriveItemPathFromMappings picks the common parent path of mapping xpaths
|
||||
// (e.g. Export/Item/ID + Export/Item/name -> Export/Item).
|
||||
func deriveItemPathFromMappings(mappings []FieldMapping) string {
|
||||
var partsLists [][]string
|
||||
for _, m := range mappings {
|
||||
src := m.sourceKey()
|
||||
if src == "" {
|
||||
continue
|
||||
}
|
||||
src = strings.Trim(strings.ReplaceAll(src, "\\", "/"), "/")
|
||||
if !strings.Contains(src, "/") {
|
||||
continue
|
||||
}
|
||||
parts := strings.Split(src, "/")
|
||||
if len(parts) < 2 {
|
||||
continue
|
||||
}
|
||||
// Drop the leaf field segment.
|
||||
partsLists = append(partsLists, parts[:len(parts)-1])
|
||||
}
|
||||
if len(partsLists) == 0 {
|
||||
return ""
|
||||
}
|
||||
common := partsLists[0]
|
||||
for _, parts := range partsLists[1:] {
|
||||
n := len(common)
|
||||
if len(parts) < n {
|
||||
n = len(parts)
|
||||
}
|
||||
i := 0
|
||||
for i < n && strings.EqualFold(common[i], parts[i]) {
|
||||
i++
|
||||
}
|
||||
common = common[:i]
|
||||
if len(common) == 0 {
|
||||
return ""
|
||||
}
|
||||
}
|
||||
return strings.Join(common, "/")
|
||||
}
|
||||
|
||||
func itemLocalFromPath(path string) string {
|
||||
path = strings.Trim(strings.TrimSpace(path), "/")
|
||||
if path == "" {
|
||||
return ""
|
||||
}
|
||||
if i := strings.LastIndex(path, "/"); i >= 0 {
|
||||
return path[i+1:]
|
||||
}
|
||||
return path
|
||||
}
|
||||
|
||||
func guessXMLItemPath(data []byte) string {
|
||||
sample := string(data)
|
||||
if len(sample) > 64<<10 {
|
||||
sample = sample[:64<<10]
|
||||
}
|
||||
lower := strings.ToLower(sample)
|
||||
|
||||
type cand struct {
|
||||
local string
|
||||
full string
|
||||
}
|
||||
cands := []cand{
|
||||
{"item", "rss/channel/item"},
|
||||
{"product", "products/product"},
|
||||
{"entry", "feed/entry"},
|
||||
{"offer", "offers/offer"},
|
||||
{"row", "rows/row"},
|
||||
}
|
||||
for _, c := range cands {
|
||||
if strings.Contains(lower, "<"+c.local) || strings.Contains(lower, ":"+c.local) {
|
||||
if path := findFirstTagPath(data, c.local); path != "" {
|
||||
return path
|
||||
}
|
||||
return c.full
|
||||
}
|
||||
}
|
||||
return "rss/channel/item"
|
||||
}
|
||||
|
||||
func findFirstTagPath(data []byte, local string) string {
|
||||
dec := xml.NewDecoder(bytes.NewReader(data))
|
||||
dec.Strict = false
|
||||
var stack []string
|
||||
for {
|
||||
tok, err := dec.Token()
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
switch t := tok.(type) {
|
||||
case xml.StartElement:
|
||||
stack = append(stack, t.Name.Local)
|
||||
if localNameEquals(t.Name, local) {
|
||||
return strings.Join(stack, "/")
|
||||
}
|
||||
case xml.EndElement:
|
||||
if len(stack) > 0 {
|
||||
stack = stack[:len(stack)-1]
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
type fieldAcc struct {
|
||||
path string
|
||||
name string
|
||||
samples []string
|
||||
seen map[string]struct{}
|
||||
dataType string
|
||||
}
|
||||
|
||||
func extractXMLSchema(data []byte, itemLocal string) ([]SchemaField, int, error) {
|
||||
if itemLocal == "" {
|
||||
itemLocal = guessXMLItemLocal(data)
|
||||
}
|
||||
acc := map[string]*fieldAcc{}
|
||||
rows := 0
|
||||
_, err := parseXMLItems(bytes.NewReader(data), itemLocal, func(row feedRow) error {
|
||||
rows++
|
||||
if rows > schemaMaxRows {
|
||||
return errStopSchema
|
||||
}
|
||||
accumulateRowFields(acc, row)
|
||||
return nil
|
||||
})
|
||||
if err != nil && !errors.Is(err, errStopSchema) {
|
||||
return nil, rows, err
|
||||
}
|
||||
return finalizeSchema(acc), rows, nil
|
||||
}
|
||||
|
||||
func accumulateRowFields(acc map[string]*fieldAcc, row feedRow) {
|
||||
for k, v := range row {
|
||||
v = strings.TrimSpace(v)
|
||||
if v == "" {
|
||||
continue
|
||||
}
|
||||
if strings.HasPrefix(k, "@") && !strings.Contains(k, "/") {
|
||||
// Bare attribute dupes are noise; path-qualified @ kept below.
|
||||
continue
|
||||
}
|
||||
fa := acc[k]
|
||||
if fa == nil {
|
||||
fa = &fieldAcc{
|
||||
path: k,
|
||||
name: leafName(k),
|
||||
seen: map[string]struct{}{},
|
||||
}
|
||||
acc[k] = fa
|
||||
}
|
||||
if _, ok := fa.seen[v]; !ok {
|
||||
fa.seen[v] = struct{}{}
|
||||
if len(fa.samples) < schemaMaxSamples {
|
||||
fa.samples = append(fa.samples, truncateSample(v))
|
||||
}
|
||||
}
|
||||
if fa.dataType == "" {
|
||||
fa.dataType = inferDataType(v)
|
||||
} else if fa.dataType != "string" {
|
||||
t := inferDataType(v)
|
||||
if t != fa.dataType {
|
||||
fa.dataType = "string"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
var errStopSchema = errors.New("schema sample limit")
|
||||
|
||||
func extractCSVSchema(data []byte) ([]SchemaField, int, string, error) {
|
||||
r := csv.NewReader(bytes.NewReader(data))
|
||||
r.ReuseRecord = true
|
||||
r.LazyQuotes = true
|
||||
r.TrimLeadingSpace = true
|
||||
r.FieldsPerRecord = -1
|
||||
|
||||
header, err := r.Read()
|
||||
if err != nil {
|
||||
return nil, 0, "", fmt.Errorf("csv header: %w", err)
|
||||
}
|
||||
cols := make([]string, len(header))
|
||||
for i, h := range header {
|
||||
cols[i] = strings.TrimSpace(h)
|
||||
}
|
||||
|
||||
acc := map[string]*fieldAcc{}
|
||||
for _, c := range cols {
|
||||
if c == "" {
|
||||
continue
|
||||
}
|
||||
acc[c] = &fieldAcc{path: c, name: c, seen: map[string]struct{}{}, dataType: ""}
|
||||
}
|
||||
|
||||
rows := 0
|
||||
var previewLines []string
|
||||
previewLines = append(previewLines, strings.Join(cols, ","))
|
||||
|
||||
for {
|
||||
rec, err := r.Read()
|
||||
if err == io.EOF {
|
||||
break
|
||||
}
|
||||
if err != nil {
|
||||
return nil, rows, "", fmt.Errorf("csv row %d: %w", rows+1, err)
|
||||
}
|
||||
rows++
|
||||
if rows <= 5 {
|
||||
previewLines = append(previewLines, strings.Join(rec, ","))
|
||||
}
|
||||
if rows > schemaMaxRows {
|
||||
break
|
||||
}
|
||||
row := make(feedRow, len(cols))
|
||||
for i, col := range cols {
|
||||
if col == "" || i >= len(rec) {
|
||||
continue
|
||||
}
|
||||
row[col] = strings.TrimSpace(rec[i])
|
||||
}
|
||||
expandSpecificationFields(row)
|
||||
accumulateRowFields(acc, row)
|
||||
}
|
||||
|
||||
preview := strings.Join(previewLines, "\n")
|
||||
return finalizeSchema(acc), rows, preview, nil
|
||||
}
|
||||
|
||||
func finalizeSchema(acc map[string]*fieldAcc) []SchemaField {
|
||||
preferNestedFieldPaths(acc)
|
||||
keys := make([]string, 0, len(acc))
|
||||
for k := range acc {
|
||||
keys = append(keys, k)
|
||||
}
|
||||
sort.SliceStable(keys, func(i, j int) bool {
|
||||
di, dj := strings.Count(keys[i], "/"), strings.Count(keys[j], "/")
|
||||
if di != dj {
|
||||
return di < dj
|
||||
}
|
||||
return keys[i] < keys[j]
|
||||
})
|
||||
out := make([]SchemaField, 0, len(keys))
|
||||
for _, k := range keys {
|
||||
fa := acc[k]
|
||||
dt := fa.dataType
|
||||
if dt == "" {
|
||||
dt = "string"
|
||||
}
|
||||
// Nested CDATA/HTML parent blobs stay as string; children are preferred for mapping.
|
||||
if isSpecFieldKey(k) && hasPrefixedChildrenAcc(acc, k) {
|
||||
dt = "object"
|
||||
}
|
||||
out = append(out, SchemaField{
|
||||
Path: fa.path,
|
||||
FieldName: fa.name,
|
||||
DataType: dt,
|
||||
SampleValues: fa.samples,
|
||||
UniqueValuesCount: len(fa.seen),
|
||||
SuggestedTarget: SuggestTarget(fa.name),
|
||||
})
|
||||
if out[len(out)-1].SuggestedTarget == "" {
|
||||
out[len(out)-1].SuggestedTarget = SuggestTarget(fa.path)
|
||||
}
|
||||
if len(out) >= schemaMaxFields {
|
||||
break
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// preferNestedFieldPaths drops bare leaf keys when a nested path ending with
|
||||
// the same leaf exists (e.g. keep specifications/Color, drop Color).
|
||||
func preferNestedFieldPaths(acc map[string]*fieldAcc) {
|
||||
nestedLeaves := map[string]struct{}{}
|
||||
for k := range acc {
|
||||
if strings.Contains(k, "/") {
|
||||
nestedLeaves[strings.ToLower(leafName(k))] = struct{}{}
|
||||
}
|
||||
}
|
||||
for k := range acc {
|
||||
if strings.Contains(k, "/") {
|
||||
continue
|
||||
}
|
||||
if _, ok := nestedLeaves[strings.ToLower(k)]; ok {
|
||||
delete(acc, k)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func hasPrefixedChildrenAcc(acc map[string]*fieldAcc, prefix string) bool {
|
||||
p := strings.TrimSuffix(prefix, "/") + "/"
|
||||
for k := range acc {
|
||||
if strings.HasPrefix(k, p) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func leafName(path string) string {
|
||||
path = strings.TrimSpace(path)
|
||||
if i := strings.LastIndex(path, "/"); i >= 0 {
|
||||
return path[i+1:]
|
||||
}
|
||||
return path
|
||||
}
|
||||
|
||||
func truncateSample(s string) string {
|
||||
if len(s) > 120 {
|
||||
return s[:117] + "..."
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
func inferDataType(v string) string {
|
||||
v = strings.TrimSpace(v)
|
||||
if v == "" {
|
||||
return "string"
|
||||
}
|
||||
lower := strings.ToLower(v)
|
||||
if lower == "true" || lower == "false" {
|
||||
return "boolean"
|
||||
}
|
||||
dot := 0
|
||||
digits := 0
|
||||
for i, r := range v {
|
||||
if r == '-' && i == 0 {
|
||||
continue
|
||||
}
|
||||
if r == '.' {
|
||||
dot++
|
||||
if dot > 1 {
|
||||
return "string"
|
||||
}
|
||||
continue
|
||||
}
|
||||
if !unicode.IsDigit(r) {
|
||||
return "string"
|
||||
}
|
||||
digits++
|
||||
}
|
||||
if digits == 0 {
|
||||
return "string"
|
||||
}
|
||||
if dot == 1 {
|
||||
return "number"
|
||||
}
|
||||
return "integer"
|
||||
}
|
||||
|
||||
func buildXMLPreview(data []byte, itemLocal string) string {
|
||||
sample := string(data)
|
||||
if len(sample) > schemaSampleBytes {
|
||||
sample = sample[:schemaSampleBytes]
|
||||
}
|
||||
lines := strings.Split(sample, "\n")
|
||||
out := make([]string, 0, previewMaxLines)
|
||||
for _, line := range lines {
|
||||
trimmed := strings.TrimSpace(line)
|
||||
if trimmed == "" {
|
||||
continue
|
||||
}
|
||||
out = append(out, line)
|
||||
if len(out) >= previewMaxLines {
|
||||
break
|
||||
}
|
||||
}
|
||||
_ = itemLocal
|
||||
return strings.Join(out, "\n")
|
||||
}
|
||||
@@ -0,0 +1,70 @@
|
||||
package feeds
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestExtractCSVSchema(t *testing.T) {
|
||||
data := []byte("ean,title,price\n123,Widget,9.99\n456,Gadget,12.50\n")
|
||||
fields, rows, preview, err := extractCSVSchema(data)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if rows != 2 {
|
||||
t.Fatalf("rows=%d", rows)
|
||||
}
|
||||
if len(fields) != 3 {
|
||||
t.Fatalf("fields=%d", len(fields))
|
||||
}
|
||||
if !strings.Contains(preview, "ean,title,price") {
|
||||
t.Fatalf("preview missing header: %q", preview)
|
||||
}
|
||||
if fields[0].Path != "ean" {
|
||||
t.Fatalf("first field %q", fields[0].Path)
|
||||
}
|
||||
}
|
||||
|
||||
func TestExtractXMLSchema(t *testing.T) {
|
||||
data := []byte(`<?xml version="1.0"?>
|
||||
<rss><channel>
|
||||
<item><title>A</title><g:gtin>111</g:gtin></item>
|
||||
<item><title>B</title><g:gtin>222</g:gtin></item>
|
||||
</channel></rss>`)
|
||||
fields, rows, err := extractXMLSchema(data, "item")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if rows != 2 {
|
||||
t.Fatalf("rows=%d", rows)
|
||||
}
|
||||
if len(fields) == 0 {
|
||||
t.Fatal("expected fields")
|
||||
}
|
||||
foundTitle := false
|
||||
for _, f := range fields {
|
||||
if f.FieldName == "title" || f.Path == "title" {
|
||||
foundTitle = true
|
||||
}
|
||||
}
|
||||
if !foundTitle {
|
||||
t.Fatalf("title not found in %#v", fields)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseMappingsWrapped(t *testing.T) {
|
||||
raw := map[string]any{
|
||||
"item_path": "rss/channel/item",
|
||||
"fields": []any{
|
||||
map[string]any{"source": "gtin", "target": "gtin"},
|
||||
map[string]any{"source": "title", "target": "title"},
|
||||
},
|
||||
}
|
||||
got := parseMappings(raw)
|
||||
if len(got) != 2 {
|
||||
t.Fatalf("got %d mappings: %#v", len(got), got)
|
||||
}
|
||||
if itemPathFromMappings(raw) != "rss/channel/item" {
|
||||
t.Fatalf("item path: %q", itemPathFromMappings(raw))
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,96 @@
|
||||
package feeds
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/google/uuid"
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
)
|
||||
|
||||
func TestListAndExportFeedsSQLPagination(t *testing.T) {
|
||||
dsn := strings.TrimSpace(os.Getenv("DATABASE_URL"))
|
||||
if dsn == "" {
|
||||
t.Skip("DATABASE_URL not set")
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer cancel()
|
||||
pg, err := pgxpool.New(ctx, dsn)
|
||||
if err != nil {
|
||||
t.Fatalf("pool: %v", err)
|
||||
}
|
||||
defer pg.Close()
|
||||
|
||||
companyID := uuid.New()
|
||||
_, err = pg.Exec(ctx, `INSERT INTO companies (id, name) VALUES ($1, $2)`,
|
||||
companyID, "list-page-"+companyID.String()[:8])
|
||||
if err != nil {
|
||||
t.Fatalf("seed company: %v", err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
_, _ = pg.Exec(context.Background(), `DELETE FROM companies WHERE id = $1`, companyID)
|
||||
})
|
||||
|
||||
svc := &Service{Pool: pg}
|
||||
for i := 0; i < 3; i++ {
|
||||
_, err := pg.Exec(ctx, `
|
||||
INSERT INTO input_feeds (company_id, name, url, feed_type, status, sync_interval_minutes, options)
|
||||
VALUES ($1, $2, '', 'csv', 'unmapped', 60, '{}'::jsonb)`,
|
||||
companyID, fmt.Sprintf("feed-%d", i))
|
||||
if err != nil {
|
||||
t.Fatalf("insert feed: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
page, total, _, _, err := svc.List(ctx, companyID, 2, 0, "")
|
||||
if err != nil {
|
||||
t.Fatalf("List: %v", err)
|
||||
}
|
||||
if total != 3 {
|
||||
t.Fatalf("total=%d want 3", total)
|
||||
}
|
||||
if len(page) != 2 {
|
||||
t.Fatalf("page len=%d want 2", len(page))
|
||||
}
|
||||
for i, item := range page {
|
||||
v, ok := item["mapping_incomplete"].(bool)
|
||||
if !ok {
|
||||
t.Fatalf("page[%d] missing mapping_incomplete bool: %#v", i, item["mapping_incomplete"])
|
||||
}
|
||||
if !v {
|
||||
t.Fatalf("page[%d] mapping_incomplete=false want true for unmapped feed without mappings", i)
|
||||
}
|
||||
presented := PresentFeed(item)
|
||||
if presented["mapping_incomplete"] != true {
|
||||
t.Fatalf("PresentFeed mapping_incomplete=%v", presented["mapping_incomplete"])
|
||||
}
|
||||
}
|
||||
|
||||
page2, total2, _, _, err := svc.List(ctx, companyID, 2, 2, "")
|
||||
if err != nil {
|
||||
t.Fatalf("List page2: %v", err)
|
||||
}
|
||||
if total2 != 3 || len(page2) != 1 {
|
||||
t.Fatalf("page2 len=%d total=%d", len(page2), total2)
|
||||
}
|
||||
|
||||
matched, matchedTotal, _, _, err := svc.List(ctx, companyID, 10, 0, "feed-1")
|
||||
if err != nil {
|
||||
t.Fatalf("List search: %v", err)
|
||||
}
|
||||
if matchedTotal != 1 || len(matched) != 1 {
|
||||
t.Fatalf("search len=%d total=%d want 1", len(matched), matchedTotal)
|
||||
}
|
||||
|
||||
exp, expTotal, err := svc.ListExportFeeds(ctx, companyID, 10, 0)
|
||||
if err != nil {
|
||||
t.Fatalf("ListExportFeeds: %v", err)
|
||||
}
|
||||
if expTotal != 0 || len(exp) != 0 {
|
||||
t.Fatalf("export feeds: len=%d total=%d", len(exp), expTotal)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,323 @@
|
||||
package feeds
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// FieldMapping maps a feed source column/xpath onto a canonical product field.
|
||||
type FieldMapping struct {
|
||||
Source string `json:"source,omitempty"`
|
||||
Column string `json:"column,omitempty"`
|
||||
XPath string `json:"xpath,omitempty"`
|
||||
Target string `json:"target,omitempty"`
|
||||
FieldName string `json:"fieldName,omitempty"`
|
||||
Field string `json:"field,omitempty"`
|
||||
}
|
||||
|
||||
func (m FieldMapping) sourceKey() string {
|
||||
for _, v := range []string{m.Source, m.Column, m.XPath} {
|
||||
if strings.TrimSpace(v) != "" {
|
||||
return strings.TrimSpace(v)
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func (m FieldMapping) targetKey() string {
|
||||
for _, v := range []string{m.Target, m.FieldName, m.Field} {
|
||||
if strings.TrimSpace(v) != "" {
|
||||
return strings.TrimSpace(v)
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// parseMappings accepts legacy object maps or array forms.
|
||||
func parseMappings(raw any) []FieldMapping {
|
||||
if raw == nil {
|
||||
return nil
|
||||
}
|
||||
switch t := raw.(type) {
|
||||
case []FieldMapping:
|
||||
return t
|
||||
case []any:
|
||||
out := make([]FieldMapping, 0, len(t))
|
||||
for _, item := range t {
|
||||
if m, ok := fieldMappingFromUIEntry(item); ok {
|
||||
out = append(out, m)
|
||||
continue
|
||||
}
|
||||
b, err := json.Marshal(item)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
var m FieldMapping
|
||||
if json.Unmarshal(b, &m) != nil {
|
||||
continue
|
||||
}
|
||||
if m.sourceKey() != "" && m.targetKey() != "" {
|
||||
out = append(out, m)
|
||||
}
|
||||
}
|
||||
return out
|
||||
case map[string]any:
|
||||
// Prefer wrapped { item_path, fields|mappings: [...] } from the mapping UI.
|
||||
if fields, ok := t["fields"]; ok {
|
||||
return parseMappings(fields)
|
||||
}
|
||||
if nested, ok := t["mappings"]; ok {
|
||||
return parseMappings(nested)
|
||||
}
|
||||
out := make([]FieldMapping, 0, len(t))
|
||||
for source, val := range t {
|
||||
if source == "item_path" {
|
||||
continue
|
||||
}
|
||||
switch v := val.(type) {
|
||||
case string:
|
||||
out = append(out, FieldMapping{Source: source, Target: v})
|
||||
case map[string]any:
|
||||
b, _ := json.Marshal(v)
|
||||
var m FieldMapping
|
||||
_ = json.Unmarshal(b, &m)
|
||||
if m.sourceKey() == "" {
|
||||
m.Source = source
|
||||
}
|
||||
if m.targetKey() == "" {
|
||||
continue
|
||||
}
|
||||
out = append(out, m)
|
||||
default:
|
||||
b, err := json.Marshal(v)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
var m FieldMapping
|
||||
if json.Unmarshal(b, &m) != nil {
|
||||
continue
|
||||
}
|
||||
if m.sourceKey() == "" {
|
||||
m.Source = source
|
||||
}
|
||||
if m.targetKey() != "" {
|
||||
out = append(out, m)
|
||||
}
|
||||
}
|
||||
}
|
||||
return out
|
||||
default:
|
||||
b, err := json.Marshal(raw)
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
var arr []FieldMapping
|
||||
if json.Unmarshal(b, &arr) == nil && len(arr) > 0 {
|
||||
return arr
|
||||
}
|
||||
var obj map[string]any
|
||||
if json.Unmarshal(b, &obj) == nil {
|
||||
return parseMappings(obj)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
// fieldMappingFromUIEntry unwraps dashboard rows shaped like
|
||||
// {"key":"Export/Item/ID","mapping":{"fieldName":"id","xpath":"Export/Item/ID"}}.
|
||||
func fieldMappingFromUIEntry(item any) (FieldMapping, bool) {
|
||||
m, ok := item.(map[string]any)
|
||||
if !ok {
|
||||
return FieldMapping{}, false
|
||||
}
|
||||
nested, ok := m["mapping"]
|
||||
if !ok {
|
||||
return FieldMapping{}, false
|
||||
}
|
||||
nestedMap, ok := nested.(map[string]any)
|
||||
if !ok {
|
||||
b, err := json.Marshal(nested)
|
||||
if err != nil {
|
||||
return FieldMapping{}, false
|
||||
}
|
||||
nestedMap = map[string]any{}
|
||||
if json.Unmarshal(b, &nestedMap) != nil {
|
||||
return FieldMapping{}, false
|
||||
}
|
||||
}
|
||||
b, err := json.Marshal(nestedMap)
|
||||
if err != nil {
|
||||
return FieldMapping{}, false
|
||||
}
|
||||
var fm FieldMapping
|
||||
if json.Unmarshal(b, &fm) != nil {
|
||||
return FieldMapping{}, false
|
||||
}
|
||||
if key, _ := m["key"].(string); strings.TrimSpace(key) != "" {
|
||||
if fm.XPath == "" {
|
||||
fm.XPath = strings.TrimSpace(key)
|
||||
}
|
||||
if fm.Source == "" && fm.Column == "" {
|
||||
fm.Source = strings.TrimSpace(key)
|
||||
}
|
||||
}
|
||||
if fm.sourceKey() == "" || fm.targetKey() == "" {
|
||||
return FieldMapping{}, false
|
||||
}
|
||||
return fm, true
|
||||
}
|
||||
|
||||
// applyMappings copies source values into mapped_data as-is (including "0").
|
||||
// Derivation and zero-dimension cleanup happen later in processing.EnrichMapped.
|
||||
func applyMappings(row map[string]string, mappings []FieldMapping) (mapped map[string]any, gtin string) {
|
||||
mapped = make(map[string]any, len(mappings))
|
||||
for _, m := range mappings {
|
||||
src := m.sourceKey()
|
||||
tgt := m.targetKey()
|
||||
if src == "" || tgt == "" || strings.EqualFold(tgt, "none") {
|
||||
continue
|
||||
}
|
||||
if isSpecificationsTarget(tgt) {
|
||||
if obj := resolveSpecifications(row, src); obj != nil {
|
||||
if raw, ok := obj["_raw"]; ok && len(obj) == 1 {
|
||||
mapped["specifications"] = raw
|
||||
} else {
|
||||
delete(obj, "_raw")
|
||||
mapped["specifications"] = obj
|
||||
}
|
||||
if !strings.EqualFold(tgt, "specifications") {
|
||||
mapped[tgt] = mapped["specifications"]
|
||||
}
|
||||
}
|
||||
continue
|
||||
}
|
||||
val, ok := lookupRow(row, src)
|
||||
if !ok || strings.TrimSpace(val) == "" {
|
||||
continue
|
||||
}
|
||||
val = strings.TrimSpace(val)
|
||||
mapped[tgt] = val
|
||||
if strings.EqualFold(tgt, "gtin") || strings.EqualFold(tgt, "ean") || strings.EqualFold(tgt, "upc") {
|
||||
gtin = val
|
||||
mapped["gtin"] = val
|
||||
}
|
||||
}
|
||||
if gtin == "" {
|
||||
for _, k := range []string{"gtin", "ean", "upc", "EAN", "GTIN", "barcode"} {
|
||||
if v, ok := lookupRow(row, k); ok && strings.TrimSpace(v) != "" {
|
||||
gtin = strings.TrimSpace(v)
|
||||
mapped["gtin"] = gtin
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
return mapped, gtin
|
||||
}
|
||||
|
||||
func isSpecificationsTarget(tgt string) bool {
|
||||
switch strings.ToLower(strings.TrimSpace(tgt)) {
|
||||
case "specifications", "specs", "specification":
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
// resolveSpecifications builds a label→value map from nested XML children,
|
||||
// CDATA HTML, or flat strings. Empty/missing yields nil.
|
||||
func resolveSpecifications(row map[string]string, src string) map[string]string {
|
||||
src = strings.Trim(strings.TrimSpace(src), "/")
|
||||
if children := collectPrefixed(row, src); len(children) > 0 {
|
||||
return normalizeSpecKeys(children)
|
||||
}
|
||||
if val, ok := lookupRow(row, src); ok {
|
||||
if pairs := ParseSpecifications(val); len(pairs) > 0 {
|
||||
return specsToMap(pairs)
|
||||
}
|
||||
// Keep non-empty raw blob so UI/backfill can still parse later.
|
||||
if strings.TrimSpace(val) != "" && !isEmptySpecBlob(val) {
|
||||
return map[string]string{"_raw": strings.TrimSpace(val)}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func normalizeSpecKeys(in map[string]string) map[string]string {
|
||||
if len(in) == 0 {
|
||||
return nil
|
||||
}
|
||||
out := make(map[string]string, len(in))
|
||||
for k, v := range in {
|
||||
key := CanonicalAttributeKey(k)
|
||||
if key == "" || strings.TrimSpace(v) == "" {
|
||||
continue
|
||||
}
|
||||
out[key] = v
|
||||
}
|
||||
if len(out) == 0 {
|
||||
return nil
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func lookupRow(row map[string]string, key string) (string, bool) {
|
||||
key = strings.Trim(strings.TrimSpace(key), "/")
|
||||
if key == "" {
|
||||
return "", false
|
||||
}
|
||||
if v, ok := row[key]; ok {
|
||||
return v, true
|
||||
}
|
||||
// Case-insensitive exact path match (prefer longest key).
|
||||
var bestKey string
|
||||
for k := range row {
|
||||
if strings.EqualFold(k, key) {
|
||||
if len(k) >= len(bestKey) {
|
||||
bestKey = k
|
||||
}
|
||||
}
|
||||
}
|
||||
if bestKey != "" {
|
||||
return row[bestKey], true
|
||||
}
|
||||
if !strings.Contains(key, "/") {
|
||||
return "", false
|
||||
}
|
||||
// Nested path: prefer a unique row key that ends with the same path suffix.
|
||||
leaf := leafName(key)
|
||||
suffix := "/" + strings.ToLower(key)
|
||||
var suffixHits []string
|
||||
var leafHits []string
|
||||
for k := range row {
|
||||
lk := strings.ToLower(k)
|
||||
if strings.HasSuffix(lk, suffix) || lk == strings.ToLower(key) {
|
||||
suffixHits = append(suffixHits, k)
|
||||
continue
|
||||
}
|
||||
if strings.EqualFold(leafName(k), leaf) {
|
||||
leafHits = append(leafHits, k)
|
||||
}
|
||||
}
|
||||
if len(suffixHits) == 1 {
|
||||
return row[suffixHits[0]], true
|
||||
}
|
||||
if len(suffixHits) > 1 {
|
||||
// Prefer the shortest (most specific relative) match.
|
||||
best := suffixHits[0]
|
||||
for _, h := range suffixHits[1:] {
|
||||
if len(h) < len(best) {
|
||||
best = h
|
||||
}
|
||||
}
|
||||
return row[best], true
|
||||
}
|
||||
// Fall back to bare leaf only when unambiguous.
|
||||
if len(leafHits) == 1 {
|
||||
return row[leafHits[0]], true
|
||||
}
|
||||
if v, ok := row[leaf]; ok && len(leafHits) <= 1 {
|
||||
return v, true
|
||||
}
|
||||
return "", false
|
||||
}
|
||||
@@ -0,0 +1,233 @@
|
||||
package feeds
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/csv"
|
||||
"encoding/xml"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// defaultMaxParseRows caps CSV/XML product rows per sync. Parse is streaming
|
||||
// (one row at a time); the bound limits sync duration and DB write volume for
|
||||
// oversized catalogs. Exceeding returns parseTooManyRows() with the numeric limit.
|
||||
const defaultMaxParseRows = 1_000_000
|
||||
|
||||
// maxParseRows is the active row cap. Mutable for tests.
|
||||
var maxParseRows = defaultMaxParseRows
|
||||
|
||||
// errParseTooManyRows is the sentinel for ClientError / errors.Is checks.
|
||||
var errParseTooManyRows = errors.New("feed exceeds max row limit")
|
||||
|
||||
// parseTooManyRows returns errParseTooManyRows with the active row limit for clients.
|
||||
func parseTooManyRows() error {
|
||||
return fmt.Errorf("%w (%d)", errParseTooManyRows, maxParseRows)
|
||||
}
|
||||
|
||||
// feedRow is a normalized flat record from CSV or XML.
|
||||
type feedRow map[string]string
|
||||
|
||||
func detectFeedFormat(feedType, contentType, urlHint string, sample []byte) string {
|
||||
ft := strings.ToLower(strings.TrimSpace(feedType))
|
||||
if ft == "csv" || ft == "xml" {
|
||||
return ft
|
||||
}
|
||||
ct := strings.ToLower(contentType)
|
||||
u := strings.ToLower(urlHint)
|
||||
switch {
|
||||
case strings.Contains(ct, "csv") || strings.HasSuffix(u, ".csv"):
|
||||
return "csv"
|
||||
case strings.Contains(ct, "xml") || strings.HasSuffix(u, ".xml"):
|
||||
return "xml"
|
||||
}
|
||||
trimmed := bytes.TrimSpace(sample)
|
||||
if len(trimmed) > 0 && trimmed[0] == '<' {
|
||||
return "xml"
|
||||
}
|
||||
return "csv"
|
||||
}
|
||||
|
||||
// parseCSV streams rows via callback to avoid holding the full matrix when possible.
|
||||
// The CSV reader still tokenizes; we only keep one row at a time in the callback path.
|
||||
func parseCSV(r io.Reader, onRow func(feedRow) error) (int, error) {
|
||||
cr := csv.NewReader(r)
|
||||
cr.ReuseRecord = true
|
||||
cr.LazyQuotes = true
|
||||
cr.TrimLeadingSpace = true
|
||||
cr.FieldsPerRecord = -1
|
||||
|
||||
header, err := cr.Read()
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("csv header: %w", err)
|
||||
}
|
||||
cols := make([]string, len(header))
|
||||
for i, h := range header {
|
||||
cols[i] = strings.TrimSpace(h)
|
||||
}
|
||||
count := 0
|
||||
for {
|
||||
rec, err := cr.Read()
|
||||
if err == io.EOF {
|
||||
break
|
||||
}
|
||||
if err != nil {
|
||||
return count, fmt.Errorf("csv row %d: %w", count+1, err)
|
||||
}
|
||||
count++
|
||||
if count > maxParseRows {
|
||||
return count, parseTooManyRows()
|
||||
}
|
||||
row := make(feedRow, len(cols))
|
||||
for i, col := range cols {
|
||||
if col == "" {
|
||||
continue
|
||||
}
|
||||
if i < len(rec) {
|
||||
row[col] = rec[i]
|
||||
} else {
|
||||
row[col] = ""
|
||||
}
|
||||
}
|
||||
expandSpecificationFields(row)
|
||||
if err := onRow(row); err != nil {
|
||||
return count, err
|
||||
}
|
||||
}
|
||||
return count, nil
|
||||
}
|
||||
|
||||
// parseXMLItems streams element-local text maps for repeating item tags.
|
||||
// When itemLocal is empty and r is seekable, a small prefix is sniffed then rewound.
|
||||
func parseXMLItems(r io.Reader, itemLocal string, onRow func(feedRow) error) (int, error) {
|
||||
itemLocal = strings.TrimSpace(itemLocal)
|
||||
if itemLocal == "" {
|
||||
var err error
|
||||
itemLocal, r, err = resolveXMLItemLocal(r)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
}
|
||||
dec := xml.NewDecoder(r)
|
||||
dec.Strict = false
|
||||
count := 0
|
||||
for {
|
||||
tok, err := dec.Token()
|
||||
if err == io.EOF {
|
||||
break
|
||||
}
|
||||
if err != nil {
|
||||
return count, fmt.Errorf("xml: %w", err)
|
||||
}
|
||||
se, ok := tok.(xml.StartElement)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
if !localNameEquals(se.Name, itemLocal) {
|
||||
continue
|
||||
}
|
||||
row, err := readXMLElementMap(dec, se)
|
||||
if err != nil {
|
||||
return count, err
|
||||
}
|
||||
expandSpecificationFields(row)
|
||||
count++
|
||||
if count > maxParseRows {
|
||||
return count, parseTooManyRows()
|
||||
}
|
||||
if err := onRow(row); err != nil {
|
||||
return count, err
|
||||
}
|
||||
}
|
||||
return count, nil
|
||||
}
|
||||
|
||||
const xmlItemGuessBytes = 64 << 10
|
||||
|
||||
func resolveXMLItemLocal(r io.Reader) (string, io.Reader, error) {
|
||||
if rs, ok := r.(io.ReadSeeker); ok {
|
||||
sample := make([]byte, xmlItemGuessBytes)
|
||||
n, err := io.ReadFull(rs, sample)
|
||||
if err != nil && err != io.EOF && err != io.ErrUnexpectedEOF {
|
||||
return "", nil, err
|
||||
}
|
||||
sample = sample[:n]
|
||||
local := guessXMLItemLocal(sample)
|
||||
if _, err := rs.Seek(0, io.SeekStart); err != nil {
|
||||
return "", nil, err
|
||||
}
|
||||
return local, rs, nil
|
||||
}
|
||||
sample, err := io.ReadAll(io.LimitReader(r, xmlItemGuessBytes))
|
||||
if err != nil {
|
||||
return "", nil, err
|
||||
}
|
||||
local := guessXMLItemLocal(sample)
|
||||
return local, io.MultiReader(bytes.NewReader(sample), r), nil
|
||||
}
|
||||
|
||||
func guessXMLItemLocal(data []byte) string {
|
||||
sample := string(data)
|
||||
if len(sample) > xmlItemGuessBytes {
|
||||
sample = sample[:xmlItemGuessBytes]
|
||||
}
|
||||
lower := strings.ToLower(sample)
|
||||
for _, cand := range []string{"item", "product", "entry", "offer", "row"} {
|
||||
if strings.Contains(lower, "<"+cand) || strings.Contains(lower, ":"+cand) {
|
||||
return cand
|
||||
}
|
||||
}
|
||||
return "item"
|
||||
}
|
||||
|
||||
func localNameEquals(n xml.Name, local string) bool {
|
||||
return strings.EqualFold(n.Local, local)
|
||||
}
|
||||
|
||||
func readXMLElementMap(dec *xml.Decoder, start xml.StartElement) (feedRow, error) {
|
||||
row := make(feedRow)
|
||||
for _, a := range start.Attr {
|
||||
key := "@" + a.Name.Local
|
||||
row[key] = a.Value
|
||||
row[start.Name.Local+"/"+key] = a.Value
|
||||
}
|
||||
var path []string
|
||||
for {
|
||||
tok, err := dec.Token()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
switch t := tok.(type) {
|
||||
case xml.StartElement:
|
||||
path = append(path, t.Name.Local)
|
||||
for _, a := range t.Attr {
|
||||
key := strings.Join(path, "/") + "/@" + a.Name.Local
|
||||
row["@"+a.Name.Local] = a.Value
|
||||
row[key] = a.Value
|
||||
}
|
||||
case xml.EndElement:
|
||||
if len(path) == 0 {
|
||||
return row, nil
|
||||
}
|
||||
path = path[:len(path)-1]
|
||||
case xml.CharData:
|
||||
text := strings.TrimSpace(string(t))
|
||||
if text == "" || len(path) == 0 {
|
||||
continue
|
||||
}
|
||||
leaf := path[len(path)-1]
|
||||
full := strings.Join(path, "/")
|
||||
// Prefer nested path as source of truth; only set bare leaf when
|
||||
// unique (no other nested field already owns this leaf name).
|
||||
if prev, ok := row[full]; ok && prev != "" && prev != text {
|
||||
row[full] = prev + " " + text
|
||||
} else {
|
||||
row[full] = text
|
||||
}
|
||||
if prev, ok := row[leaf]; !ok || prev == "" || prev == text || prev == row[full] {
|
||||
row[leaf] = row[full]
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,53 @@
|
||||
package feeds
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestParseUIKeyMappingArray(t *testing.T) {
|
||||
raw := []any{
|
||||
map[string]any{
|
||||
"key": "Export/Item/ID",
|
||||
"mapping": map[string]any{
|
||||
"fieldName": "id",
|
||||
"xpath": "Export/Item/ID",
|
||||
"originalName": "ID",
|
||||
},
|
||||
},
|
||||
map[string]any{
|
||||
"key": "Export/Item/name",
|
||||
"mapping": map[string]any{
|
||||
"fieldName": "name",
|
||||
"xpath": "Export/Item/name",
|
||||
},
|
||||
},
|
||||
map[string]any{
|
||||
"key": "Export/Item/EAN",
|
||||
"mapping": map[string]any{
|
||||
"fieldName": "gtin",
|
||||
"xpath": "Export/Item/EAN",
|
||||
},
|
||||
},
|
||||
}
|
||||
got := parseMappings(raw)
|
||||
if len(got) != 3 {
|
||||
t.Fatalf("got %d mappings: %#v", len(got), got)
|
||||
}
|
||||
if got[0].targetKey() != "id" || got[0].sourceKey() != "Export/Item/ID" {
|
||||
t.Fatalf("first=%#v", got[0])
|
||||
}
|
||||
path := itemPathFromMappings(raw)
|
||||
if path != "Export/Item" {
|
||||
t.Fatalf("item path=%q", path)
|
||||
}
|
||||
}
|
||||
|
||||
func TestItemPathFromWrappedMappings(t *testing.T) {
|
||||
raw := map[string]any{
|
||||
"item_path": "rss/channel/item",
|
||||
"mappings": []any{
|
||||
map[string]any{"source": "title", "target": "title"},
|
||||
},
|
||||
}
|
||||
if itemPathFromMappings(raw) != "rss/channel/item" {
|
||||
t.Fatalf("path=%q", itemPathFromMappings(raw))
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,140 @@
|
||||
package feeds
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/google/uuid"
|
||||
)
|
||||
|
||||
// PresentFeed maps an input_feeds row to the public/legacy feed DTO.
|
||||
// Legacy fields (item_path, is_active, last_synced, product_count) are always set;
|
||||
// v2 fields (feed_type, sync_interval_minutes, last_synced_at, options) are included for dual-support.
|
||||
func PresentFeed(feed map[string]any) map[string]any {
|
||||
if feed == nil {
|
||||
return nil
|
||||
}
|
||||
opts, _ := feed["options"].(map[string]any)
|
||||
if opts == nil {
|
||||
opts = map[string]any{}
|
||||
}
|
||||
itemPath, _ := opts["item_path"].(string)
|
||||
if itemPath == "" {
|
||||
itemPath, _ = feed["item_path"].(string)
|
||||
}
|
||||
status, _ := feed["status"].(string)
|
||||
lastSynced := formatAPITime(feed["last_synced_at"])
|
||||
productsUpdated := formatAPITime(feed["products_updated_at"])
|
||||
productCount := intFromAny(feed["product_count"])
|
||||
mappingFieldCount := intFromAny(feed["mapping_field_count"])
|
||||
hasMappings := mappingFieldCount > 0
|
||||
if v, ok := feed["has_mappings"].(bool); ok {
|
||||
hasMappings = v || mappingFieldCount > 0
|
||||
}
|
||||
mappingIncomplete := !hasMappings
|
||||
if v, ok := feed["mapping_incomplete"].(bool); ok {
|
||||
mappingIncomplete = v
|
||||
}
|
||||
// last_data_at: live sync timestamp when present, else latest raw product update
|
||||
// (covers MySQL→Postgres imports where last_synced_at was never set).
|
||||
lastDataAt := lastSynced
|
||||
if lastDataAt == nil {
|
||||
lastDataAt = productsUpdated
|
||||
}
|
||||
|
||||
out := map[string]any{
|
||||
"id": stringifyID(feed["id"]),
|
||||
"name": feed["name"],
|
||||
"url": nullish(feed["url"]),
|
||||
"item_path": itemPath,
|
||||
"is_active": strings.EqualFold(status, "active"),
|
||||
"product_count": productCount,
|
||||
"mapping_field_count": mappingFieldCount,
|
||||
"has_mappings": hasMappings,
|
||||
"mapping_incomplete": mappingIncomplete,
|
||||
"status": status,
|
||||
"last_synced": lastSynced,
|
||||
"created_at": formatAPITime(feed["created_at"]),
|
||||
"updated_at": formatAPITime(feed["updated_at"]),
|
||||
"feed_type": feed["feed_type"],
|
||||
"sync_interval_minutes": feed["sync_interval_minutes"],
|
||||
"last_synced_at": lastSynced,
|
||||
"products_updated_at": productsUpdated,
|
||||
"last_data_at": lastDataAt,
|
||||
"options": opts,
|
||||
}
|
||||
if deltas, ok := opts["last_sync_deltas"].(map[string]any); ok && deltas != nil {
|
||||
out["last_sync_deltas"] = deltas
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func intFromAny(v any) int {
|
||||
switch t := v.(type) {
|
||||
case int:
|
||||
return t
|
||||
case int32:
|
||||
return int(t)
|
||||
case int64:
|
||||
return int(t)
|
||||
case float64:
|
||||
return int(t)
|
||||
default:
|
||||
return 0
|
||||
}
|
||||
}
|
||||
|
||||
// PresentFeeds maps a page of feed rows through PresentFeed.
|
||||
func PresentFeeds(items []map[string]any) []map[string]any {
|
||||
out := make([]map[string]any, 0, len(items))
|
||||
for _, item := range items {
|
||||
out = append(out, PresentFeed(item))
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func stringifyID(v any) any {
|
||||
switch t := v.(type) {
|
||||
case uuid.UUID:
|
||||
return t.String()
|
||||
case [16]byte:
|
||||
return uuid.UUID(t).String()
|
||||
default:
|
||||
return v
|
||||
}
|
||||
}
|
||||
|
||||
func nullish(v any) any {
|
||||
if v == nil {
|
||||
return nil
|
||||
}
|
||||
if s, ok := v.(string); ok && s == "" {
|
||||
return nil
|
||||
}
|
||||
return v
|
||||
}
|
||||
|
||||
func formatAPITime(v any) any {
|
||||
if v == nil {
|
||||
return nil
|
||||
}
|
||||
switch t := v.(type) {
|
||||
case time.Time:
|
||||
if t.IsZero() {
|
||||
return nil
|
||||
}
|
||||
return t.UTC().Format(time.RFC3339)
|
||||
case *time.Time:
|
||||
if t == nil || t.IsZero() {
|
||||
return nil
|
||||
}
|
||||
return t.UTC().Format(time.RFC3339)
|
||||
case string:
|
||||
if t == "" {
|
||||
return nil
|
||||
}
|
||||
return t
|
||||
default:
|
||||
return v
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,143 @@
|
||||
package feeds
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestPresentFeedLegacyFields(t *testing.T) {
|
||||
t.Parallel()
|
||||
ts := time.Date(2026, 8, 4, 11, 0, 0, 0, time.UTC)
|
||||
got := PresentFeed(map[string]any{
|
||||
"id": "22222222-2222-2222-2222-222222222222",
|
||||
"name": "Main catalog",
|
||||
"url": "https://example.com/feed.xml",
|
||||
"feed_type": "xml",
|
||||
"status": "active",
|
||||
"sync_interval_minutes": 60,
|
||||
"last_synced_at": ts,
|
||||
"options": map[string]any{"item_path": "channel/item"},
|
||||
"product_count": int64(12),
|
||||
"created_at": ts,
|
||||
"updated_at": ts,
|
||||
})
|
||||
if got["item_path"] != "channel/item" {
|
||||
t.Fatalf("item_path=%v", got["item_path"])
|
||||
}
|
||||
if got["is_active"] != true {
|
||||
t.Fatalf("is_active=%v", got["is_active"])
|
||||
}
|
||||
if got["product_count"] != 12 {
|
||||
t.Fatalf("product_count=%v", got["product_count"])
|
||||
}
|
||||
if got["last_synced"] != "2026-08-04T11:00:00Z" {
|
||||
t.Fatalf("last_synced=%v", got["last_synced"])
|
||||
}
|
||||
if got["last_synced_at"] != got["last_synced"] {
|
||||
t.Fatalf("dual last_synced_at mismatch")
|
||||
}
|
||||
if got["feed_type"] != "xml" {
|
||||
t.Fatalf("feed_type=%v", got["feed_type"])
|
||||
}
|
||||
if got["last_data_at"] != got["last_synced_at"] {
|
||||
t.Fatalf("last_data_at should prefer live sync: %v", got["last_data_at"])
|
||||
}
|
||||
}
|
||||
|
||||
func TestPresentFeedLastDataFromProducts(t *testing.T) {
|
||||
t.Parallel()
|
||||
ts := time.Date(2026, 8, 4, 11, 0, 0, 0, time.UTC)
|
||||
got := PresentFeed(map[string]any{
|
||||
"id": "1",
|
||||
"name": "Imported",
|
||||
"status": "active",
|
||||
"product_count": int64(50),
|
||||
"mapping_field_count": 19,
|
||||
"has_mappings": true,
|
||||
"products_updated_at": ts,
|
||||
"last_synced_at": nil,
|
||||
})
|
||||
if got["last_synced_at"] != nil {
|
||||
t.Fatalf("last_synced_at=%v", got["last_synced_at"])
|
||||
}
|
||||
if got["last_data_at"] != "2026-08-04T11:00:00Z" {
|
||||
t.Fatalf("last_data_at=%v", got["last_data_at"])
|
||||
}
|
||||
if got["mapping_field_count"] != 19 {
|
||||
t.Fatalf("mapping_field_count=%v", got["mapping_field_count"])
|
||||
}
|
||||
if got["has_mappings"] != true {
|
||||
t.Fatalf("has_mappings=%v", got["has_mappings"])
|
||||
}
|
||||
if got["mapping_incomplete"] != false {
|
||||
t.Fatalf("mapping_incomplete=%v want false when has_mappings and unset", got["mapping_incomplete"])
|
||||
}
|
||||
if got["product_count"] != 50 {
|
||||
t.Fatalf("product_count=%v", got["product_count"])
|
||||
}
|
||||
}
|
||||
|
||||
func TestPresentFeedMappingIncomplete(t *testing.T) {
|
||||
t.Parallel()
|
||||
got := PresentFeed(map[string]any{
|
||||
"id": "1",
|
||||
"name": "Mapped incomplete",
|
||||
"status": "mapped",
|
||||
"mapping_field_count": 2,
|
||||
"has_mappings": true,
|
||||
"mapping_incomplete": true,
|
||||
})
|
||||
if got["mapping_incomplete"] != true {
|
||||
t.Fatalf("mapping_incomplete=%v", got["mapping_incomplete"])
|
||||
}
|
||||
}
|
||||
|
||||
func TestPresentFeedsPreservesMappingIncomplete(t *testing.T) {
|
||||
t.Parallel()
|
||||
out := PresentFeeds([]map[string]any{
|
||||
{
|
||||
"id": "a",
|
||||
"name": "Incomplete",
|
||||
"status": "mapped",
|
||||
"mapping_field_count": 1,
|
||||
"has_mappings": true,
|
||||
"mapping_incomplete": true,
|
||||
},
|
||||
{
|
||||
"id": "b",
|
||||
"name": "Complete",
|
||||
"status": "active",
|
||||
"mapping_field_count": 3,
|
||||
"has_mappings": true,
|
||||
"mapping_incomplete": false,
|
||||
},
|
||||
})
|
||||
if len(out) != 2 {
|
||||
t.Fatalf("len=%d", len(out))
|
||||
}
|
||||
if out[0]["mapping_incomplete"] != true || out[1]["mapping_incomplete"] != false {
|
||||
t.Fatalf("got %#v %#v", out[0]["mapping_incomplete"], out[1]["mapping_incomplete"])
|
||||
}
|
||||
}
|
||||
|
||||
func TestPresentFeedEmptyURLAndInactive(t *testing.T) {
|
||||
t.Parallel()
|
||||
got := PresentFeed(map[string]any{
|
||||
"id": "1",
|
||||
"name": "Draft",
|
||||
"url": "",
|
||||
"status": "unmapped",
|
||||
"options": map[string]any{},
|
||||
"created_at": time.Unix(0, 0).UTC(),
|
||||
"updated_at": time.Unix(0, 0).UTC(),
|
||||
})
|
||||
if got["url"] != nil {
|
||||
t.Fatalf("url=%v want nil", got["url"])
|
||||
}
|
||||
if got["is_active"] != false {
|
||||
t.Fatalf("is_active=%v", got["is_active"])
|
||||
}
|
||||
if got["item_path"] != "" {
|
||||
t.Fatalf("item_path=%v", got["item_path"])
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,635 @@
|
||||
package feeds
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"github.com/google/uuid"
|
||||
"github.com/jackc/pgx/v5"
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
)
|
||||
|
||||
type Service struct {
|
||||
Pool *pgxpool.Pool
|
||||
UploadDir string
|
||||
}
|
||||
|
||||
// CreateInput is the payload for creating an input feed (URL and/or uploaded CSV).
|
||||
// Legacy clients send name + item_path (url optional). V2 clients send name + url/file
|
||||
// plus optional feed_type / sync_interval_minutes (or legacy sync_frequency in hours).
|
||||
type CreateInput struct {
|
||||
Name string
|
||||
URL string
|
||||
ItemPath string
|
||||
FeedType string
|
||||
SyncIntervalMinutes int
|
||||
SyncFrequencyHours int // legacy alias; converted to minutes when SyncIntervalMinutes unset
|
||||
Options map[string]any
|
||||
}
|
||||
|
||||
func (s *Service) List(ctx context.Context, companyID uuid.UUID, limit, offset int, q string) ([]map[string]any, int64, int64, int64, error) {
|
||||
q = strings.TrimSpace(q)
|
||||
where := `company_id = $1`
|
||||
args := []any{companyID}
|
||||
if q != "" {
|
||||
where += ` AND (
|
||||
COALESCE(name, '') ILIKE '%' || $2 || '%' OR
|
||||
COALESCE(url, '') ILIKE '%' || $2 || '%' OR
|
||||
COALESCE(feed_type, '') ILIKE '%' || $2 || '%' OR
|
||||
COALESCE(status, '') ILIKE '%' || $2 || '%' OR
|
||||
COALESCE(options->>'source_filename', '') ILIKE '%' || $2 || '%'
|
||||
)`
|
||||
args = append(args, q)
|
||||
}
|
||||
var total int64
|
||||
if err := s.Pool.QueryRow(ctx, `SELECT count(*) FROM input_feeds WHERE `+where, args...).Scan(&total); err != nil {
|
||||
return nil, 0, 0, 0, err
|
||||
}
|
||||
// active_total = truly syncing; mapped_total = fields saved but not activated.
|
||||
var activeTotal, mappedTotal int64
|
||||
if err := s.Pool.QueryRow(ctx,
|
||||
`SELECT
|
||||
count(*) FILTER (WHERE lower(COALESCE(status, '')) = 'active'),
|
||||
count(*) FILTER (WHERE lower(COALESCE(status, '')) = 'mapped')
|
||||
FROM input_feeds WHERE `+where,
|
||||
args...,
|
||||
).Scan(&activeTotal, &mappedTotal); err != nil {
|
||||
return nil, 0, 0, 0, err
|
||||
}
|
||||
limitArg := len(args) + 1
|
||||
offsetArg := len(args) + 2
|
||||
query := fmt.Sprintf(`
|
||||
SELECT id, name, url, feed_type, status, sync_interval_minutes, last_synced_at, options, created_at, updated_at,
|
||||
(SELECT count(*)::int FROM raw_products rp WHERE rp.feed_id = input_feeds.id AND rp.company_id = input_feeds.company_id) AS product_count,
|
||||
(SELECT max(rp.updated_at) FROM raw_products rp WHERE rp.feed_id = input_feeds.id AND rp.company_id = input_feeds.company_id) AS products_updated_at
|
||||
FROM input_feeds WHERE %s
|
||||
ORDER BY created_at DESC LIMIT $%d OFFSET $%d`, where, limitArg, offsetArg)
|
||||
queryArgs := append(append([]any{}, args...), limit, offset)
|
||||
rows, err := s.Pool.Query(ctx, query, queryArgs...)
|
||||
if err != nil {
|
||||
return nil, 0, 0, 0, err
|
||||
}
|
||||
defer rows.Close()
|
||||
items, err := scanMaps(rows, []string{"id", "name", "url", "feed_type", "status", "sync_interval_minutes", "last_synced_at", "options", "created_at", "updated_at", "product_count", "products_updated_at"})
|
||||
if err != nil {
|
||||
return nil, 0, 0, 0, err
|
||||
}
|
||||
if err := s.attachMappingFieldCounts(ctx, companyID, items); err != nil {
|
||||
return nil, 0, 0, 0, err
|
||||
}
|
||||
return items, total, activeTotal, mappedTotal, nil
|
||||
}
|
||||
|
||||
// ProductTotals is company-scoped catalog counts across all feeds.
|
||||
type ProductTotals struct {
|
||||
Total int64
|
||||
Processed int64
|
||||
Unprocessed int64
|
||||
}
|
||||
|
||||
// CompanyProductTotals returns company-scoped catalog counts for feeds/dashboard cards.
|
||||
// ASSUMPTION: Total = count(raw_products); Processed = count(processed_products);
|
||||
// Unprocessed = count(raw where processing_status='unprocessed'). These are not a
|
||||
// partition of Total (P+U≠Total by design). Do not redefine without documenting a new ASSUMPTION.
|
||||
func (s *Service) CompanyProductTotals(ctx context.Context, companyID uuid.UUID) (ProductTotals, error) {
|
||||
var t ProductTotals
|
||||
err := s.Pool.QueryRow(ctx, `
|
||||
SELECT
|
||||
(SELECT count(*)::bigint FROM raw_products WHERE company_id = $1),
|
||||
(SELECT count(*)::bigint FROM processed_products WHERE company_id = $1),
|
||||
(SELECT count(*)::bigint FROM raw_products
|
||||
WHERE company_id = $1 AND lower(COALESCE(processing_status, '')) = 'unprocessed')`,
|
||||
companyID,
|
||||
).Scan(&t.Total, &t.Processed, &t.Unprocessed)
|
||||
return t, err
|
||||
}
|
||||
|
||||
func (s *Service) Create(ctx context.Context, companyID uuid.UUID, in CreateInput) (map[string]any, error) {
|
||||
name := strings.TrimSpace(in.Name)
|
||||
if name == "" {
|
||||
return nil, ClientMsg("name required")
|
||||
}
|
||||
url := strings.TrimSpace(in.URL)
|
||||
if err := ValidateFeedURL(ctx, url); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
opts := in.Options
|
||||
if opts == nil {
|
||||
opts = map[string]any{}
|
||||
}
|
||||
itemPath := strings.TrimSpace(in.ItemPath)
|
||||
if itemPath == "" {
|
||||
if p, ok := opts["item_path"].(string); ok {
|
||||
itemPath = strings.TrimSpace(p)
|
||||
}
|
||||
}
|
||||
if itemPath != "" {
|
||||
opts["item_path"] = itemPath
|
||||
}
|
||||
hasLocal := sourcePathFromOptions(opts) != ""
|
||||
// Dual-support: legacy create allows name + item_path without url/file.
|
||||
if url == "" && !hasLocal && itemPath == "" {
|
||||
return nil, errSourceRequired
|
||||
}
|
||||
feedType := strings.ToLower(strings.TrimSpace(in.FeedType))
|
||||
if feedType == "" {
|
||||
if hasLocal {
|
||||
feedType = "csv"
|
||||
} else {
|
||||
feedType = "xml"
|
||||
}
|
||||
}
|
||||
if feedType != "xml" && feedType != "csv" {
|
||||
return nil, ClientMsg("feed_type must be xml or csv")
|
||||
}
|
||||
interval := in.SyncIntervalMinutes
|
||||
if interval <= 0 && in.SyncFrequencyHours > 0 {
|
||||
interval = in.SyncFrequencyHours * 60
|
||||
}
|
||||
if interval <= 0 {
|
||||
interval = 60
|
||||
}
|
||||
optsBytes, err := json.Marshal(opts)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var id uuid.UUID
|
||||
err = s.Pool.QueryRow(ctx, `
|
||||
INSERT INTO input_feeds (company_id, name, url, feed_type, status, sync_interval_minutes, options)
|
||||
VALUES ($1, $2, $3, $4, 'unmapped', $5, $6::jsonb) RETURNING id`,
|
||||
companyID, name, nullStr(url), feedType, interval, optsBytes).Scan(&id)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return s.Get(ctx, companyID, id)
|
||||
}
|
||||
|
||||
func (s *Service) Get(ctx context.Context, companyID, id uuid.UUID) (map[string]any, error) {
|
||||
row := s.Pool.QueryRow(ctx, `
|
||||
SELECT id, name, url, feed_type, status, sync_interval_minutes, last_synced_at, options, created_at, updated_at,
|
||||
(SELECT count(*)::int FROM raw_products rp WHERE rp.feed_id = input_feeds.id AND rp.company_id = input_feeds.company_id) AS product_count,
|
||||
(SELECT max(rp.updated_at) FROM raw_products rp WHERE rp.feed_id = input_feeds.id AND rp.company_id = input_feeds.company_id) AS products_updated_at
|
||||
FROM input_feeds WHERE id = $1 AND company_id = $2`, id, companyID)
|
||||
item, err := scanMap(row, []string{"id", "name", "url", "feed_type", "status", "sync_interval_minutes", "last_synced_at", "options", "created_at", "updated_at", "product_count", "products_updated_at"})
|
||||
if err != nil {
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return nil, ErrNotFound
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
if err := s.attachMappingFieldCounts(ctx, companyID, []map[string]any{item}); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return item, nil
|
||||
}
|
||||
|
||||
// attachMappingFieldCounts sets mapping_field_count / has_mappings / mapping_incomplete
|
||||
// on each feed row from the active feed_mappings document (one query for the page).
|
||||
// mapping_incomplete mirrors list-chip blocking preflight (empty/required/item_path).
|
||||
func (s *Service) attachMappingFieldCounts(ctx context.Context, companyID uuid.UUID, items []map[string]any) error {
|
||||
if len(items) == 0 {
|
||||
return nil
|
||||
}
|
||||
ids := make([]uuid.UUID, 0, len(items))
|
||||
index := make(map[uuid.UUID]map[string]any, len(items))
|
||||
for _, item := range items {
|
||||
id, ok := asUUID(item["id"])
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
ids = append(ids, id)
|
||||
index[id] = item
|
||||
item["mapping_field_count"] = 0
|
||||
item["has_mappings"] = false
|
||||
item["mapping_incomplete"] = true
|
||||
}
|
||||
if len(ids) == 0 {
|
||||
return nil
|
||||
}
|
||||
required, err := s.loadRequiredStandardFields(ctx, companyID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
rows, err := s.Pool.Query(ctx, `
|
||||
SELECT DISTINCT ON (feed_id) feed_id, mappings
|
||||
FROM feed_mappings
|
||||
WHERE company_id = $1 AND is_active = true AND feed_id = ANY($2::uuid[])
|
||||
ORDER BY feed_id, version DESC`, companyID, ids)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer rows.Close()
|
||||
for rows.Next() {
|
||||
var feedID uuid.UUID
|
||||
var raw []byte
|
||||
if err := rows.Scan(&feedID, &raw); err != nil {
|
||||
return err
|
||||
}
|
||||
item := index[feedID]
|
||||
if item == nil {
|
||||
continue
|
||||
}
|
||||
var parsed any
|
||||
if err := json.Unmarshal(raw, &parsed); err != nil {
|
||||
continue
|
||||
}
|
||||
mappings := parseMappings(parsed)
|
||||
n := len(mappings)
|
||||
item["mapping_field_count"] = n
|
||||
item["has_mappings"] = n > 0
|
||||
item["mapping_incomplete"] = mappingDocIncomplete(item, parsed, mappings, required)
|
||||
}
|
||||
return rows.Err()
|
||||
}
|
||||
|
||||
func isCSVFeedType(feedType string) bool {
|
||||
t := strings.ToLower(strings.TrimSpace(feedType))
|
||||
return t == "csv" || t == "excel"
|
||||
}
|
||||
|
||||
func feedItemPathHint(item map[string]any, mappingsRaw any) string {
|
||||
if p := itemPathFromMappings(mappingsRaw); p != "" {
|
||||
return p
|
||||
}
|
||||
if opts, ok := item["options"].(map[string]any); ok {
|
||||
if v, ok := opts["item_path"].(string); ok {
|
||||
if p := strings.TrimSpace(v); p != "" {
|
||||
return p
|
||||
}
|
||||
}
|
||||
}
|
||||
if v, ok := item["item_path"].(string); ok {
|
||||
return strings.TrimSpace(v)
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// mappingDocIncomplete reports list-chip blocking gaps (empty mappings, required targets, XML item_path).
|
||||
func mappingDocIncomplete(item map[string]any, mappingsRaw any, mappings []FieldMapping, required []requiredStandardField) bool {
|
||||
if err := validateMappingsForSync(mappings, required); err != nil {
|
||||
return true
|
||||
}
|
||||
feedType, _ := item["feed_type"].(string)
|
||||
if !isCSVFeedType(feedType) && feedItemPathHint(item, mappingsRaw) == "" {
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func asUUID(v any) (uuid.UUID, bool) {
|
||||
switch t := v.(type) {
|
||||
case uuid.UUID:
|
||||
return t, true
|
||||
case [16]byte:
|
||||
return uuid.UUID(t), true
|
||||
case string:
|
||||
id, err := uuid.Parse(t)
|
||||
return id, err == nil
|
||||
default:
|
||||
return uuid.Nil, false
|
||||
}
|
||||
}
|
||||
|
||||
func (s *Service) Update(ctx context.Context, companyID, id uuid.UUID, body map[string]any) (map[string]any, error) {
|
||||
name, _ := body["name"].(string)
|
||||
url, _ := body["url"].(string)
|
||||
status, _ := body["status"].(string)
|
||||
feedType, _ := body["feed_type"].(string)
|
||||
itemPath, _ := body["item_path"].(string)
|
||||
if err := ValidateFeedURL(ctx, url); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
feedType = strings.ToLower(strings.TrimSpace(feedType))
|
||||
if feedType != "" && feedType != "xml" && feedType != "csv" {
|
||||
return nil, ClientMsg("feed_type must be xml or csv")
|
||||
}
|
||||
interval := 0
|
||||
switch v := body["sync_interval_minutes"].(type) {
|
||||
case float64:
|
||||
interval = int(v)
|
||||
case int:
|
||||
interval = v
|
||||
case json.Number:
|
||||
n, _ := v.Int64()
|
||||
interval = int(n)
|
||||
}
|
||||
if interval <= 0 {
|
||||
switch v := body["sync_frequency"].(type) {
|
||||
case float64:
|
||||
interval = int(v) * 60
|
||||
case int:
|
||||
interval = v * 60
|
||||
}
|
||||
}
|
||||
|
||||
ct, err := s.Pool.Exec(ctx, `
|
||||
UPDATE input_feeds SET
|
||||
name = CASE WHEN $3 <> '' THEN $3 ELSE name END,
|
||||
url = CASE WHEN $4 <> '' THEN $4 ELSE url END,
|
||||
status = CASE WHEN $5 <> '' THEN $5 ELSE status END,
|
||||
feed_type = CASE WHEN $6 <> '' THEN $6 ELSE feed_type END,
|
||||
sync_interval_minutes = CASE WHEN $7 > 0 THEN $7 ELSE sync_interval_minutes END,
|
||||
options = CASE
|
||||
WHEN $8 <> '' THEN COALESCE(options, '{}'::jsonb) || jsonb_build_object('item_path', to_jsonb($8::text))
|
||||
ELSE options
|
||||
END,
|
||||
updated_at = now()
|
||||
WHERE id = $1 AND company_id = $2`,
|
||||
id, companyID, name, url, status, feedType, interval, strings.TrimSpace(itemPath))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if ct.RowsAffected() == 0 {
|
||||
return nil, ErrNotFound
|
||||
}
|
||||
return s.Get(ctx, companyID, id)
|
||||
}
|
||||
|
||||
func (s *Service) Delete(ctx context.Context, companyID, id uuid.UUID) error {
|
||||
ct, err := s.Pool.Exec(ctx, `DELETE FROM input_feeds WHERE id = $1 AND company_id = $2`, id, companyID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if ct.RowsAffected() == 0 {
|
||||
return ErrNotFound
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Service) GetMappings(ctx context.Context, companyID, feedID uuid.UUID) (map[string]any, error) {
|
||||
var id uuid.UUID
|
||||
var version int
|
||||
var mappings []byte
|
||||
err := s.Pool.QueryRow(ctx, `
|
||||
SELECT id, version, mappings FROM feed_mappings
|
||||
WHERE feed_id = $1 AND company_id = $2 AND is_active = true
|
||||
ORDER BY version DESC LIMIT 1`, feedID, companyID).Scan(&id, &version, &mappings)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var m any
|
||||
_ = json.Unmarshal(mappings, &m)
|
||||
return map[string]any{"id": id, "version": version, "mappings": m}, nil
|
||||
}
|
||||
|
||||
func (s *Service) PutMappings(ctx context.Context, companyID, feedID uuid.UUID, mappings any) (map[string]any, error) {
|
||||
b, err := json.Marshal(mappings)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var version int
|
||||
_ = s.Pool.QueryRow(ctx, `
|
||||
SELECT COALESCE(MAX(version), 0) FROM feed_mappings WHERE feed_id = $1`, feedID).Scan(&version)
|
||||
version++
|
||||
_, _ = s.Pool.Exec(ctx, `UPDATE feed_mappings SET is_active = false WHERE feed_id = $1`, feedID)
|
||||
var id uuid.UUID
|
||||
err = s.Pool.QueryRow(ctx, `
|
||||
INSERT INTO feed_mappings (feed_id, company_id, version, mappings, is_active)
|
||||
VALUES ($1, $2, $3, $4, true) RETURNING id`, feedID, companyID, version, b).Scan(&id)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// Keep feed.options.item_path in sync for Sync() XML item selection, and
|
||||
// flip unmapped -> mapped whenever at least one field mapping is saved.
|
||||
path := itemPathFromMappings(mappings)
|
||||
hasFields := len(parseMappings(mappings)) > 0
|
||||
switch {
|
||||
case path != "":
|
||||
_, _ = s.Pool.Exec(ctx, `
|
||||
UPDATE input_feeds SET
|
||||
options = COALESCE(options, '{}'::jsonb) || jsonb_build_object('item_path', to_jsonb($3::text)),
|
||||
status = CASE WHEN status = 'unmapped' THEN 'mapped' ELSE status END,
|
||||
updated_at = now()
|
||||
WHERE id = $1 AND company_id = $2`, feedID, companyID, path)
|
||||
case hasFields:
|
||||
_, _ = s.Pool.Exec(ctx, `
|
||||
UPDATE input_feeds SET
|
||||
status = CASE WHEN status = 'unmapped' THEN 'mapped' ELSE status END,
|
||||
updated_at = now()
|
||||
WHERE id = $1 AND company_id = $2`, feedID, companyID)
|
||||
}
|
||||
return map[string]any{"id": id, "version": version, "mappings": mappings}, nil
|
||||
}
|
||||
|
||||
func (s *Service) ListExportFeeds(ctx context.Context, companyID uuid.UUID, limit, offset int) ([]map[string]any, int64, error) {
|
||||
var total int64
|
||||
if err := s.Pool.QueryRow(ctx, `SELECT count(*) FROM export_feeds WHERE company_id = $1`, companyID).Scan(&total); err != nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
rows, err := s.Pool.Query(ctx, `
|
||||
SELECT id, name, source_feed_id, format, public_token, is_active, last_generated_at, created_at, updated_at
|
||||
FROM export_feeds WHERE company_id = $1
|
||||
ORDER BY created_at DESC LIMIT $2 OFFSET $3`, companyID, limit, offset)
|
||||
if err != nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
defer rows.Close()
|
||||
items, err := scanMaps(rows, []string{"id", "name", "source_feed_id", "format", "public_token", "is_active", "last_generated_at", "created_at", "updated_at"})
|
||||
if err != nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
return items, total, nil
|
||||
}
|
||||
|
||||
func (s *Service) GetExportFeed(ctx context.Context, companyID, id uuid.UUID) (map[string]any, error) {
|
||||
rows, err := s.Pool.Query(ctx, `
|
||||
SELECT id, name, source_feed_id, format, public_token, template, filters, is_active, last_generated_at, created_at, updated_at
|
||||
FROM export_feeds WHERE id = $1 AND company_id = $2`, id, companyID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
items, err := scanMaps(rows, []string{"id", "name", "source_feed_id", "format", "public_token", "template", "filters", "is_active", "last_generated_at", "created_at", "updated_at"})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(items) == 0 {
|
||||
return nil, errors.New("not found")
|
||||
}
|
||||
return items[0], nil
|
||||
}
|
||||
|
||||
func (s *Service) DeleteExportFeed(ctx context.Context, companyID, id uuid.UUID) error {
|
||||
ct, err := s.Pool.Exec(ctx, `DELETE FROM export_feeds WHERE id = $1 AND company_id = $2`, id, companyID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if ct.RowsAffected() == 0 {
|
||||
return errors.New("not found")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Service) UpdateExportFeed(ctx context.Context, companyID, id uuid.UUID, name *string, isActive *bool, template, filters any) (map[string]any, error) {
|
||||
current, err := s.GetExportFeed(ctx, companyID, id)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if name != nil {
|
||||
n := strings.TrimSpace(*name)
|
||||
if n == "" {
|
||||
return nil, ClientMsg("name required")
|
||||
}
|
||||
if _, err := s.Pool.Exec(ctx, `
|
||||
UPDATE export_feeds SET name = $3, updated_at = now() WHERE id = $1 AND company_id = $2`, id, companyID, n); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
if isActive != nil {
|
||||
if _, err := s.Pool.Exec(ctx, `
|
||||
UPDATE export_feeds SET is_active = $3, updated_at = now() WHERE id = $1 AND company_id = $2`, id, companyID, *isActive); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
if template != nil || filters != nil {
|
||||
tpl := template
|
||||
flt := filters
|
||||
if tpl == nil {
|
||||
tpl = current["template"]
|
||||
}
|
||||
if flt == nil {
|
||||
flt = current["filters"]
|
||||
}
|
||||
if _, err := s.UpdateExportFeedTemplate(ctx, companyID, id, tpl, flt); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
return s.GetExportFeed(ctx, companyID, id)
|
||||
}
|
||||
|
||||
func (s *Service) CreateExportFeed(ctx context.Context, companyID uuid.UUID, in CreateExportInput) (map[string]any, error) {
|
||||
if strings.TrimSpace(in.Name) == "" {
|
||||
return nil, ClientMsg("name required")
|
||||
}
|
||||
format := strings.ToLower(strings.TrimSpace(in.Format))
|
||||
if format == "" {
|
||||
format = "xml"
|
||||
}
|
||||
if format != "xml" && format != "csv" {
|
||||
return nil, ClientMsg("format must be xml or csv")
|
||||
}
|
||||
var src *uuid.UUID
|
||||
if in.SourceFeedID != nil && *in.SourceFeedID != "" {
|
||||
id, err := uuid.Parse(*in.SourceFeedID)
|
||||
if err != nil {
|
||||
return nil, ClientMsg("invalid source_feed_id")
|
||||
}
|
||||
src = &id
|
||||
}
|
||||
tplBytes := []byte("{}")
|
||||
if in.Template != nil {
|
||||
b, err := json.Marshal(in.Template)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
tplBytes = b
|
||||
}
|
||||
filterBytes := []byte("{}")
|
||||
if in.Filters != nil {
|
||||
b, err := json.Marshal(in.Filters)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
filterBytes = b
|
||||
}
|
||||
token, err := newPublicExportToken()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var id uuid.UUID
|
||||
err = s.Pool.QueryRow(ctx, `
|
||||
INSERT INTO export_feeds (company_id, name, source_feed_id, format, template, filters, public_token)
|
||||
VALUES ($1, $2, $3, $4, $5::jsonb, $6::jsonb, $7) RETURNING id, public_token`,
|
||||
companyID, in.Name, src, format, tplBytes, filterBytes, token).Scan(&id, &token)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return map[string]any{
|
||||
"id": id, "name": in.Name, "format": format, "public_token": token,
|
||||
"template": in.Template, "filters": in.Filters,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// RotateExportFeedPublicToken replaces the public URL token (revokes the previous URL).
|
||||
func (s *Service) RotateExportFeedPublicToken(ctx context.Context, companyID, id uuid.UUID) (map[string]any, error) {
|
||||
token, err := newPublicExportToken()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
ct, err := s.Pool.Exec(ctx, `
|
||||
UPDATE export_feeds SET public_token = $3, updated_at = now()
|
||||
WHERE id = $1 AND company_id = $2`, id, companyID, token)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if ct.RowsAffected() == 0 {
|
||||
return nil, errors.New("not found")
|
||||
}
|
||||
return s.GetExportFeed(ctx, companyID, id)
|
||||
}
|
||||
|
||||
func nullStr(s string) *string {
|
||||
if s == "" {
|
||||
return nil
|
||||
}
|
||||
return &s
|
||||
}
|
||||
|
||||
func xmlEscape(s string) string {
|
||||
r := strings.NewReplacer("&", "&", "<", "<", ">", ">", `"`, """)
|
||||
return r.Replace(s)
|
||||
}
|
||||
|
||||
func scanMaps(rows pgx.Rows, cols []string) ([]map[string]any, error) {
|
||||
out := make([]map[string]any, 0)
|
||||
for rows.Next() {
|
||||
vals := make([]any, len(cols))
|
||||
ptrs := make([]any, len(cols))
|
||||
for i := range vals {
|
||||
ptrs[i] = &vals[i]
|
||||
}
|
||||
if err := rows.Scan(ptrs...); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
m := make(map[string]any, len(cols))
|
||||
for i, c := range cols {
|
||||
m[c] = normalize(vals[i])
|
||||
}
|
||||
out = append(out, m)
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
func scanMap(row pgx.Row, cols []string) (map[string]any, error) {
|
||||
vals := make([]any, len(cols))
|
||||
ptrs := make([]any, len(cols))
|
||||
for i := range vals {
|
||||
ptrs[i] = &vals[i]
|
||||
}
|
||||
if err := row.Scan(ptrs...); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
m := make(map[string]any, len(cols))
|
||||
for i, c := range cols {
|
||||
m[c] = normalize(vals[i])
|
||||
}
|
||||
return m, nil
|
||||
}
|
||||
|
||||
func normalize(v any) any {
|
||||
switch t := v.(type) {
|
||||
case []byte:
|
||||
var j any
|
||||
if json.Unmarshal(t, &j) == nil {
|
||||
return j
|
||||
}
|
||||
return string(t)
|
||||
case [16]byte:
|
||||
return uuid.UUID(t).String()
|
||||
default:
|
||||
return v
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,155 @@
|
||||
package feeds
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
"github.com/google/uuid"
|
||||
)
|
||||
|
||||
var (
|
||||
errSourceRequired = errors.New("feed url or uploaded CSV source required")
|
||||
errLocalSource = errors.New("local feed source unavailable")
|
||||
)
|
||||
|
||||
// feedBlob is feed content on disk. Close removes owned temp files (HTTP downloads).
|
||||
// Local uploads reference the existing path and Close is a no-op.
|
||||
type feedBlob struct {
|
||||
path string
|
||||
contentType string
|
||||
size int64
|
||||
owned bool
|
||||
}
|
||||
|
||||
// Close removes the temp file when this blob owns it.
|
||||
func (b *feedBlob) Close() error {
|
||||
if b == nil || !b.owned || b.path == "" {
|
||||
return nil
|
||||
}
|
||||
err := os.Remove(b.path)
|
||||
b.path = ""
|
||||
b.owned = false
|
||||
return err
|
||||
}
|
||||
|
||||
// Open returns a new read handle at the start of the blob.
|
||||
func (b *feedBlob) Open() (*os.File, error) {
|
||||
if b == nil || b.path == "" {
|
||||
return nil, errors.New("feed blob closed or empty")
|
||||
}
|
||||
return os.Open(b.path)
|
||||
}
|
||||
|
||||
// Sniff reads up to n bytes from the start of the blob (for format detection).
|
||||
func (b *feedBlob) Sniff(n int) ([]byte, error) {
|
||||
if n <= 0 {
|
||||
return nil, nil
|
||||
}
|
||||
f, err := b.Open()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer f.Close()
|
||||
buf := make([]byte, n)
|
||||
nr, err := io.ReadFull(f, buf)
|
||||
if err == io.EOF || err == io.ErrUnexpectedEOF {
|
||||
err = nil
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return buf[:nr], nil
|
||||
}
|
||||
|
||||
// loadFeedSource returns on-disk feed content from a local upload or HTTP(S) URL.
|
||||
// Callers must Close the blob when finished.
|
||||
func (s *Service) loadFeedSource(ctx context.Context, companyID uuid.UUID, feed map[string]any) (*feedBlob, error) {
|
||||
if path := sourcePathFromOptions(feed["options"]); path != "" {
|
||||
return s.readLocalFeed(companyID, path)
|
||||
}
|
||||
urlStr, _ := feed["url"].(string)
|
||||
urlStr = strings.TrimSpace(urlStr)
|
||||
if urlStr == "" {
|
||||
return nil, errSourceRequired
|
||||
}
|
||||
return downloadFeed(ctx, urlStr)
|
||||
}
|
||||
|
||||
func sourcePathFromOptions(raw any) string {
|
||||
opts, ok := raw.(map[string]any)
|
||||
if !ok || opts == nil {
|
||||
return ""
|
||||
}
|
||||
for _, key := range []string{"source_path", "local_path", "file_path"} {
|
||||
if v, ok := opts[key].(string); ok {
|
||||
if p := strings.TrimSpace(v); p != "" {
|
||||
return p
|
||||
}
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func (s *Service) readLocalFeed(companyID uuid.UUID, rel string) (*feedBlob, error) {
|
||||
uploadDir := strings.TrimSpace(s.UploadDir)
|
||||
if uploadDir == "" {
|
||||
return nil, ClientMsg("upload directory not configured")
|
||||
}
|
||||
abs, err := resolveCompanyPath(uploadDir, companyID, rel)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
info, err := os.Stat(abs)
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
return nil, fmt.Errorf("%w: file missing", errLocalSource)
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
if !info.Mode().IsRegular() {
|
||||
return nil, ClientMsg("invalid source path")
|
||||
}
|
||||
if info.Size() > maxDownloadBytes {
|
||||
return nil, downloadTooLarge()
|
||||
}
|
||||
ct := "text/csv"
|
||||
lower := strings.ToLower(abs)
|
||||
if strings.HasSuffix(lower, ".xml") {
|
||||
ct = "application/xml"
|
||||
}
|
||||
return &feedBlob{
|
||||
path: abs,
|
||||
contentType: ct,
|
||||
size: info.Size(),
|
||||
owned: false,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func resolveCompanyPath(uploadDir string, companyID uuid.UUID, rel string) (string, error) {
|
||||
rel = filepath.ToSlash(strings.TrimSpace(rel))
|
||||
if rel == "" || strings.Contains(rel, "..") {
|
||||
return "", ClientMsg("invalid source path")
|
||||
}
|
||||
prefix := companyID.String() + "/"
|
||||
if !strings.HasPrefix(rel, prefix) {
|
||||
return "", ClientMsg("forbidden source path")
|
||||
}
|
||||
base, err := filepath.Abs(uploadDir)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
abs, err := filepath.Abs(filepath.Join(uploadDir, filepath.FromSlash(rel)))
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
sep := string(os.PathSeparator)
|
||||
if abs != base && !strings.HasPrefix(abs, base+sep) {
|
||||
return "", ClientMsg("forbidden source path")
|
||||
}
|
||||
return abs, nil
|
||||
}
|
||||
@@ -0,0 +1,109 @@
|
||||
package feeds
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/google/uuid"
|
||||
)
|
||||
|
||||
func TestResolveCompanyPath(t *testing.T) {
|
||||
t.Parallel()
|
||||
base := t.TempDir()
|
||||
cid := uuid.New()
|
||||
rel := cid.String() + "/sample.csv"
|
||||
absWant := filepath.Join(base, filepath.FromSlash(rel))
|
||||
if err := os.MkdirAll(filepath.Dir(absWant), 0o750); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(absWant, []byte("a,b\n1,2\n"), 0o640); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
got, err := resolveCompanyPath(base, cid, rel)
|
||||
if err != nil {
|
||||
t.Fatalf("resolve: %v", err)
|
||||
}
|
||||
if filepath.Clean(got) != filepath.Clean(absWant) {
|
||||
t.Fatalf("got %q want %q", got, absWant)
|
||||
}
|
||||
|
||||
if _, err := resolveCompanyPath(base, cid, "../etc/passwd"); err == nil {
|
||||
t.Fatal("expected traversal reject")
|
||||
}
|
||||
other := uuid.New()
|
||||
if _, err := resolveCompanyPath(base, cid, other.String()+"/x.csv"); err == nil {
|
||||
t.Fatal("expected company mismatch reject")
|
||||
}
|
||||
}
|
||||
|
||||
func TestReadLocalFeed(t *testing.T) {
|
||||
t.Parallel()
|
||||
base := t.TempDir()
|
||||
cid := uuid.New()
|
||||
rel := cid.String() + "/products.csv"
|
||||
abs := filepath.Join(base, filepath.FromSlash(rel))
|
||||
if err := os.MkdirAll(filepath.Dir(abs), 0o750); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
payload := []byte("ean,title\n123,Widget\n")
|
||||
if err := os.WriteFile(abs, payload, 0o640); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
svc := &Service{UploadDir: base}
|
||||
blob, err := svc.readLocalFeed(cid, rel)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { _ = blob.Close() })
|
||||
if blob.contentType != "text/csv" {
|
||||
t.Fatalf("content-type %q", blob.contentType)
|
||||
}
|
||||
data, err := os.ReadFile(blob.path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if string(data) != string(payload) {
|
||||
t.Fatalf("payload mismatch")
|
||||
}
|
||||
if blob.owned {
|
||||
t.Fatal("local feed must not own path")
|
||||
}
|
||||
}
|
||||
|
||||
func TestReadLocalFeedRejectsOversized(t *testing.T) {
|
||||
t.Parallel()
|
||||
old := maxDownloadBytes
|
||||
maxDownloadBytes = 32
|
||||
t.Cleanup(func() { maxDownloadBytes = old })
|
||||
|
||||
base := t.TempDir()
|
||||
cid := uuid.New()
|
||||
rel := cid.String() + "/big.csv"
|
||||
abs := filepath.Join(base, filepath.FromSlash(rel))
|
||||
if err := os.MkdirAll(filepath.Dir(abs), 0o750); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(abs, []byte(strings.Repeat("x", 40)), 0o640); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
svc := &Service{UploadDir: base}
|
||||
_, err := svc.readLocalFeed(cid, rel)
|
||||
if !errors.Is(err, errDownloadTooLarge) {
|
||||
t.Fatalf("err=%v want errDownloadTooLarge", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSourcePathFromOptions(t *testing.T) {
|
||||
t.Parallel()
|
||||
if p := sourcePathFromOptions(map[string]any{"source_path": " a/b.csv "}); p != "a/b.csv" {
|
||||
t.Fatalf("got %q", p)
|
||||
}
|
||||
if p := sourcePathFromOptions(nil); p != "" {
|
||||
t.Fatalf("got %q", p)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,539 @@
|
||||
package feeds
|
||||
|
||||
import (
|
||||
"html"
|
||||
"regexp"
|
||||
"strings"
|
||||
"unicode"
|
||||
)
|
||||
|
||||
const (
|
||||
maxSpecPairs = 200
|
||||
maxSpecRawBytes = 64 << 10 // 64 KiB per specifications blob
|
||||
maxSpecLabelRunes = 120
|
||||
maxSpecValueRunes = 2000
|
||||
)
|
||||
|
||||
var (
|
||||
// Accept </li> and broken </> closers seen in A1 feed CDATA.
|
||||
reHTMLLi = regexp.MustCompile(`(?is)<li\b[^>]*>(.*?)(?:</li\s*>|</\s*>)`)
|
||||
reHTMLTag = regexp.MustCompile(`(?is)<[^>]+>`)
|
||||
reMultiSpace = regexp.MustCompile(`\s+`)
|
||||
)
|
||||
|
||||
// SpecPair is one label/value extracted from a specifications blob.
|
||||
type SpecPair struct {
|
||||
Label string
|
||||
Value string
|
||||
}
|
||||
|
||||
// ParseSpecifications accepts nested-expanded text, CDATA HTML lists, or flat
|
||||
// CSV-like strings. Empty / missing / blank HTML returns nil (not an error).
|
||||
func ParseSpecifications(raw string) []SpecPair {
|
||||
raw = strings.TrimSpace(raw)
|
||||
if raw == "" || len(raw) > maxSpecRawBytes {
|
||||
return nil
|
||||
}
|
||||
if isEmptySpecBlob(raw) {
|
||||
return nil
|
||||
}
|
||||
|
||||
var pairs []SpecPair
|
||||
switch {
|
||||
case looksLikeHTMLList(raw):
|
||||
pairs = parseHTMLSpecList(raw)
|
||||
case looksLikeFlatSpecs(raw):
|
||||
pairs = parseFlatSpecs(raw)
|
||||
default:
|
||||
// Single "Label: value" line still counts as flat.
|
||||
if p, ok := splitLabelValue(raw); ok {
|
||||
pairs = []SpecPair{p}
|
||||
}
|
||||
}
|
||||
return clampSpecPairs(pairs)
|
||||
}
|
||||
|
||||
// expandSpecificationFields mutates row: for specification-like keys whose
|
||||
// value is HTML/flat text, add nested paths key/Label → value. Nested XML
|
||||
// children are already present as key/child from the XML walker.
|
||||
func expandSpecificationFields(row feedRow) {
|
||||
if len(row) == 0 {
|
||||
return
|
||||
}
|
||||
keys := make([]string, 0, 8)
|
||||
for k, v := range row {
|
||||
if !isSpecFieldKey(k) {
|
||||
continue
|
||||
}
|
||||
if strings.TrimSpace(v) == "" {
|
||||
continue
|
||||
}
|
||||
// Already has nested children — leave tree as-is; still parse text if useful.
|
||||
if hasPrefixedChildren(row, k) && !looksLikeHTMLList(v) && !looksLikeFlatSpecs(v) {
|
||||
continue
|
||||
}
|
||||
keys = append(keys, k)
|
||||
}
|
||||
for _, k := range keys {
|
||||
pairs := ParseSpecifications(row[k])
|
||||
for _, p := range pairs {
|
||||
seg := sanitizePathSegment(p.Label)
|
||||
if seg == "" {
|
||||
continue
|
||||
}
|
||||
path := k + "/" + seg
|
||||
if prev, ok := row[path]; ok && strings.TrimSpace(prev) != "" {
|
||||
continue
|
||||
}
|
||||
row[path] = p.Value
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func isSpecFieldKey(key string) bool {
|
||||
leaf := strings.ToLower(leafName(key))
|
||||
leaf = strings.TrimPrefix(leaf, "@")
|
||||
switch leaf {
|
||||
case "specifications", "specification", "specs", "spec", "features", "feature", "attributes_raw":
|
||||
return true
|
||||
}
|
||||
return strings.Contains(leaf, "specification")
|
||||
}
|
||||
|
||||
func hasPrefixedChildren(row feedRow, prefix string) bool {
|
||||
prefix = strings.TrimSuffix(prefix, "/") + "/"
|
||||
for k := range row {
|
||||
if strings.HasPrefix(k, prefix) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func collectPrefixed(row feedRow, prefix string) map[string]string {
|
||||
prefix = strings.TrimSuffix(strings.TrimSpace(prefix), "/")
|
||||
if prefix == "" {
|
||||
return nil
|
||||
}
|
||||
p := prefix + "/"
|
||||
out := map[string]string{}
|
||||
for k, v := range row {
|
||||
if !strings.HasPrefix(k, p) {
|
||||
continue
|
||||
}
|
||||
rest := k[len(p):]
|
||||
if rest == "" || strings.Contains(rest, "/") {
|
||||
continue
|
||||
}
|
||||
v = strings.TrimSpace(v)
|
||||
if v == "" {
|
||||
continue
|
||||
}
|
||||
out[rest] = v
|
||||
}
|
||||
if len(out) == 0 {
|
||||
return nil
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func isEmptySpecBlob(raw string) bool {
|
||||
stripped := strings.TrimSpace(reHTMLTag.ReplaceAllString(raw, " "))
|
||||
stripped = html.UnescapeString(stripped)
|
||||
stripped = strings.TrimSpace(reMultiSpace.ReplaceAllString(stripped, " "))
|
||||
return stripped == ""
|
||||
}
|
||||
|
||||
func looksLikeHTMLList(raw string) bool {
|
||||
lower := strings.ToLower(raw)
|
||||
return strings.Contains(lower, "<li") || (strings.Contains(lower, "<ul") && strings.Contains(lower, "</ul>"))
|
||||
}
|
||||
|
||||
func looksLikeFlatSpecs(raw string) bool {
|
||||
if looksLikeHTMLList(raw) {
|
||||
return false
|
||||
}
|
||||
// Multiple label:value pairs separated by ; | newline or comma between pairs.
|
||||
if strings.Count(raw, ":") >= 2 {
|
||||
return true
|
||||
}
|
||||
if strings.Count(raw, "=") >= 2 && (strings.Contains(raw, ";") || strings.Contains(raw, "|") || strings.Contains(raw, "\n")) {
|
||||
return true
|
||||
}
|
||||
if strings.Contains(raw, ";") && strings.Contains(raw, ":") {
|
||||
return true
|
||||
}
|
||||
if strings.Contains(raw, "|") && strings.Contains(raw, ":") {
|
||||
return true
|
||||
}
|
||||
if strings.Contains(raw, "\n") && strings.Contains(raw, ":") {
|
||||
return true
|
||||
}
|
||||
// Quoted CSV-ish pairs: "Brand","Acme"; "Model","X1"
|
||||
if strings.Count(raw, `"`) >= 4 && (strings.Contains(raw, ";") || strings.Contains(raw, ",")) {
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func parseHTMLSpecList(raw string) []SpecPair {
|
||||
matches := reHTMLLi.FindAllStringSubmatch(raw, maxSpecPairs+1)
|
||||
if len(matches) == 0 {
|
||||
// Fallback: strip tags and try flat parse.
|
||||
plain := strings.TrimSpace(reHTMLTag.ReplaceAllString(raw, "\n"))
|
||||
plain = html.UnescapeString(plain)
|
||||
return parseFlatSpecs(plain)
|
||||
}
|
||||
out := make([]SpecPair, 0, len(matches))
|
||||
for _, m := range matches {
|
||||
inner := strings.TrimSpace(reHTMLTag.ReplaceAllString(m[1], " "))
|
||||
inner = html.UnescapeString(inner)
|
||||
inner = strings.TrimSpace(reMultiSpace.ReplaceAllString(inner, " "))
|
||||
if inner == "" {
|
||||
continue
|
||||
}
|
||||
if p, ok := splitLabelValue(inner); ok {
|
||||
out = append(out, p)
|
||||
}
|
||||
// Bare list items without "Label: value" are skipped — suppliers should
|
||||
// send explicit pairs; inventing key→"true" produces junk attributes.
|
||||
if len(out) >= maxSpecPairs {
|
||||
break
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func parseFlatSpecs(raw string) []SpecPair {
|
||||
raw = strings.ReplaceAll(raw, "\r\n", "\n")
|
||||
raw = strings.ReplaceAll(raw, "\r", "\n")
|
||||
|
||||
chunks := splitSpecChunks(raw)
|
||||
out := make([]SpecPair, 0, len(chunks))
|
||||
for _, chunk := range chunks {
|
||||
chunk = strings.TrimSpace(chunk)
|
||||
if chunk == "" {
|
||||
continue
|
||||
}
|
||||
// CSV-ish "Label","value" or Label,value
|
||||
if strings.Contains(chunk, ",") {
|
||||
if p, ok := parseCSVSpecChunk(chunk); ok {
|
||||
out = append(out, p)
|
||||
if len(out) >= maxSpecPairs {
|
||||
break
|
||||
}
|
||||
continue
|
||||
}
|
||||
}
|
||||
if p, ok := splitLabelValue(chunk); ok {
|
||||
out = append(out, p)
|
||||
}
|
||||
if len(out) >= maxSpecPairs {
|
||||
break
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func splitSpecChunks(raw string) []string {
|
||||
// Prefer strong separators first.
|
||||
for _, sep := range []string{"\n", ";", "|"} {
|
||||
if strings.Contains(raw, sep) {
|
||||
return strings.Split(raw, sep)
|
||||
}
|
||||
}
|
||||
// Comma only when it looks like paired entries (has colon/equals).
|
||||
if strings.Contains(raw, ",") && (strings.Contains(raw, ":") || strings.Contains(raw, "=")) {
|
||||
return strings.Split(raw, ",")
|
||||
}
|
||||
return []string{raw}
|
||||
}
|
||||
|
||||
func parseCSVSpecChunk(chunk string) (SpecPair, bool) {
|
||||
parts := strings.SplitN(chunk, ",", 2)
|
||||
if len(parts) != 2 {
|
||||
return SpecPair{}, false
|
||||
}
|
||||
label := strings.Trim(strings.TrimSpace(parts[0]), `"'`)
|
||||
value := strings.Trim(strings.TrimSpace(parts[1]), `"'`)
|
||||
if label == "" || value == "" {
|
||||
return SpecPair{}, false
|
||||
}
|
||||
return SpecPair{
|
||||
Label: truncateRunes(label, maxSpecLabelRunes),
|
||||
Value: truncateRunes(value, maxSpecValueRunes),
|
||||
}, true
|
||||
}
|
||||
|
||||
func splitLabelValue(s string) (SpecPair, bool) {
|
||||
s = strings.TrimSpace(s)
|
||||
if s == "" {
|
||||
return SpecPair{}, false
|
||||
}
|
||||
// Prefer "Label: value" / "Label:value" / "Label - value" / "Label = value"
|
||||
for _, sep := range []string{":", ":", "=", "–", "—"} {
|
||||
if i := strings.Index(s, sep); i > 0 {
|
||||
label := strings.TrimSpace(s[:i])
|
||||
value := strings.TrimSpace(s[i+len(sep):])
|
||||
if label != "" && value != "" && !looksLikeURLScheme(label) {
|
||||
return SpecPair{
|
||||
Label: truncateRunes(label, maxSpecLabelRunes),
|
||||
Value: truncateRunes(value, maxSpecValueRunes),
|
||||
}, true
|
||||
}
|
||||
}
|
||||
}
|
||||
// "Label - value" with spaces (avoid splitting hyphenated words alone)
|
||||
if i := strings.Index(s, " - "); i > 0 {
|
||||
label := strings.TrimSpace(s[:i])
|
||||
value := strings.TrimSpace(s[i+3:])
|
||||
if label != "" && value != "" {
|
||||
return SpecPair{
|
||||
Label: truncateRunes(label, maxSpecLabelRunes),
|
||||
Value: truncateRunes(value, maxSpecValueRunes),
|
||||
}, true
|
||||
}
|
||||
}
|
||||
return SpecPair{}, false
|
||||
}
|
||||
|
||||
func looksLikeURLScheme(label string) bool {
|
||||
lower := strings.ToLower(strings.TrimSpace(label))
|
||||
return lower == "http" || lower == "https" || lower == "ftp"
|
||||
}
|
||||
|
||||
func sanitizePathSegment(label string) string {
|
||||
label = strings.TrimSpace(label)
|
||||
if label == "" {
|
||||
return ""
|
||||
}
|
||||
var b strings.Builder
|
||||
b.Grow(len(label))
|
||||
prevUS := false
|
||||
for _, r := range label {
|
||||
switch {
|
||||
case unicode.IsLetter(r) || unicode.IsDigit(r):
|
||||
b.WriteRune(r)
|
||||
prevUS = false
|
||||
case r == '_' || r == '-' || r == '.':
|
||||
b.WriteRune(r)
|
||||
prevUS = false
|
||||
case unicode.IsSpace(r) || r == '/' || r == '\\':
|
||||
if !prevUS && b.Len() > 0 {
|
||||
b.WriteByte('_')
|
||||
prevUS = true
|
||||
}
|
||||
default:
|
||||
// drop punctuation
|
||||
}
|
||||
}
|
||||
out := strings.Trim(b.String(), "._-")
|
||||
return truncateRunes(out, maxSpecLabelRunes)
|
||||
}
|
||||
|
||||
func clampSpecPairs(pairs []SpecPair) []SpecPair {
|
||||
if len(pairs) == 0 {
|
||||
return nil
|
||||
}
|
||||
if len(pairs) > maxSpecPairs {
|
||||
pairs = pairs[:maxSpecPairs]
|
||||
}
|
||||
seen := make(map[string]struct{}, len(pairs))
|
||||
out := make([]SpecPair, 0, len(pairs))
|
||||
for _, p := range pairs {
|
||||
p.Label = strings.TrimSpace(p.Label)
|
||||
p.Value = strings.TrimSpace(p.Value)
|
||||
if p.Label == "" || p.Value == "" {
|
||||
continue
|
||||
}
|
||||
key := strings.ToLower(p.Label)
|
||||
if _, ok := seen[key]; ok {
|
||||
continue
|
||||
}
|
||||
seen[key] = struct{}{}
|
||||
out = append(out, p)
|
||||
}
|
||||
if len(out) == 0 {
|
||||
return nil
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func truncateRunes(s string, max int) string {
|
||||
if max <= 0 || s == "" {
|
||||
return s
|
||||
}
|
||||
n := 0
|
||||
for i := range s {
|
||||
if n == max {
|
||||
return s[:i]
|
||||
}
|
||||
n++
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
func specsToMap(pairs []SpecPair) map[string]string {
|
||||
if len(pairs) == 0 {
|
||||
return nil
|
||||
}
|
||||
out := make(map[string]string, len(pairs))
|
||||
for _, p := range pairs {
|
||||
key := CanonicalAttributeKey(p.Label)
|
||||
if key == "" || strings.TrimSpace(p.Value) == "" {
|
||||
continue
|
||||
}
|
||||
out[key] = p.Value
|
||||
}
|
||||
if len(out) == 0 {
|
||||
return nil
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// compactAttributeKey strips separators for alias lookup (net_height / net-height / netheight → netheight).
|
||||
func compactAttributeKey(key string) string {
|
||||
var b strings.Builder
|
||||
b.Grow(len(key))
|
||||
for _, r := range strings.ToLower(key) {
|
||||
r = foldLatinRune(r)
|
||||
if (r >= 'a' && r <= 'z') || (r >= '0' && r <= '9') {
|
||||
b.WriteRune(r)
|
||||
}
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
|
||||
// Known supplier / locale labels → Descrybe standard field keys (snake_case).
|
||||
// Freeform specs keep kebab-case from AttributeKeyFromLabel.
|
||||
var attributeKeyAliases = map[string]string{
|
||||
"visina": "net_height",
|
||||
"height": "net_height",
|
||||
"netheight": "net_height",
|
||||
"sirina": "net_width",
|
||||
"width": "net_width",
|
||||
"netwidth": "net_width",
|
||||
"globina": "net_depth",
|
||||
"depth": "net_depth",
|
||||
"netdepth": "net_depth",
|
||||
"netmass": "net_mass",
|
||||
"mass": "net_mass",
|
||||
"weight": "net_mass",
|
||||
"teza": "net_mass",
|
||||
"productmodel": "product_model",
|
||||
"model": "product_model",
|
||||
"eprelid": "eprel_id",
|
||||
"eprel": "eprel_id",
|
||||
"energyclass": "energy_class",
|
||||
"energijskirazred": "energy_class",
|
||||
}
|
||||
|
||||
// IsValidAttributeKey rejects empty / punctuation-only / boolean junk keys.
|
||||
func IsValidAttributeKey(key string) bool {
|
||||
key = strings.TrimSpace(key)
|
||||
if key == "" || len(key) < 2 {
|
||||
return false
|
||||
}
|
||||
compact := compactAttributeKey(key)
|
||||
if len(compact) < 2 {
|
||||
return false
|
||||
}
|
||||
hasLetter := false
|
||||
for _, r := range compact {
|
||||
if r >= 'a' && r <= 'z' {
|
||||
hasLetter = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if !hasLetter {
|
||||
return false
|
||||
}
|
||||
switch compact {
|
||||
case "true", "false", "yes", "no", "null", "undefined", "none", "n", "y":
|
||||
return false
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// CanonicalAttributeKey normalizes a human or feed label to a stable attribute key.
|
||||
// Known dimension/identity aliases map to STANDARD_FIELDS snake_case; other labels
|
||||
// become kebab-case. Invalid / junk labels return "".
|
||||
func CanonicalAttributeKey(label string) string {
|
||||
slug := AttributeKeyFromLabel(label)
|
||||
compact := compactAttributeKey(slug)
|
||||
if compact == "" {
|
||||
compact = compactAttributeKey(label)
|
||||
}
|
||||
if alias, ok := attributeKeyAliases[compact]; ok {
|
||||
return alias
|
||||
}
|
||||
if slug == "" || !IsValidAttributeKey(slug) {
|
||||
return ""
|
||||
}
|
||||
return slug
|
||||
}
|
||||
|
||||
// AttributeKeyFromLabel turns a human spec label into a kebab-case attribute_key
|
||||
// (e.g. "Energijski razred" → "energijski-razred") matching company attributes.
|
||||
func AttributeKeyFromLabel(label string) string {
|
||||
label = strings.TrimSpace(label)
|
||||
if label == "" {
|
||||
return ""
|
||||
}
|
||||
var b strings.Builder
|
||||
b.Grow(len(label))
|
||||
prevHyphen := false
|
||||
for _, r := range strings.ToLower(label) {
|
||||
r = foldLatinRune(r)
|
||||
switch {
|
||||
case r >= 'a' && r <= 'z', r >= '0' && r <= '9':
|
||||
b.WriteRune(r)
|
||||
prevHyphen = false
|
||||
case unicode.IsSpace(r) || r == '_' || r == '/' || r == '\\' || r == '.' || r == ':' || r == '-':
|
||||
if !prevHyphen && b.Len() > 0 {
|
||||
b.WriteByte('-')
|
||||
prevHyphen = true
|
||||
}
|
||||
default:
|
||||
// drop other punctuation
|
||||
}
|
||||
}
|
||||
return strings.Trim(b.String(), "-")
|
||||
}
|
||||
|
||||
func foldLatinRune(r rune) rune {
|
||||
switch r {
|
||||
case 'š', 'ś', 'ş':
|
||||
return 's'
|
||||
case 'č', 'ć', 'ç':
|
||||
return 'c'
|
||||
case 'ž', 'ź', 'ż':
|
||||
return 'z'
|
||||
case 'đ':
|
||||
return 'd'
|
||||
case 'ň', 'ń':
|
||||
return 'n'
|
||||
case 'ř':
|
||||
return 'r'
|
||||
case 'ť':
|
||||
return 't'
|
||||
case 'ď':
|
||||
return 'd'
|
||||
case 'ľ', 'ĺ':
|
||||
return 'l'
|
||||
case 'ä', 'á', 'à', 'â', 'ã', 'å':
|
||||
return 'a'
|
||||
case 'ë', 'é', 'è', 'ê':
|
||||
return 'e'
|
||||
case 'ï', 'í', 'ì', 'î':
|
||||
return 'i'
|
||||
case 'ö', 'ó', 'ò', 'ô', 'õ':
|
||||
return 'o'
|
||||
case 'ü', 'ú', 'ù', 'û':
|
||||
return 'u'
|
||||
case 'ý', 'ÿ':
|
||||
return 'y'
|
||||
default:
|
||||
return r
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,280 @@
|
||||
package feeds
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestParseSpecificationsA1BrokenClosers(t *testing.T) {
|
||||
raw := "<ul><li>Energijski razred: E</><li>Dimenzije: 605x95x395 </></ul>"
|
||||
pairs := ParseSpecifications(raw)
|
||||
if len(pairs) != 2 {
|
||||
t.Fatalf("got %d pairs: %#v", len(pairs), pairs)
|
||||
}
|
||||
if pairs[0].Label != "Energijski razred" || pairs[0].Value != "E" {
|
||||
t.Fatalf("first=%#v", pairs[0])
|
||||
}
|
||||
if pairs[1].Label != "Dimenzije" || pairs[1].Value != "605x95x395" {
|
||||
t.Fatalf("second=%#v", pairs[1])
|
||||
}
|
||||
m := specsToMap(pairs)
|
||||
if m["energy_class"] != "E" || m["dimenzije"] != "605x95x395" {
|
||||
t.Fatalf("map=%#v", m)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAttributeKeyFromLabel(t *testing.T) {
|
||||
if got := AttributeKeyFromLabel("Energijski razred"); got != "energijski-razred" {
|
||||
t.Fatalf("got %q", got)
|
||||
}
|
||||
if got := AttributeKeyFromLabel("Širina"); got != "sirina" {
|
||||
t.Fatalf("got %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCanonicalAttributeKey(t *testing.T) {
|
||||
cases := map[string]string{
|
||||
"Visina": "net_height",
|
||||
"netheight": "net_height",
|
||||
"net_height": "net_height",
|
||||
"Širina": "net_width",
|
||||
"Globina": "net_depth",
|
||||
"netMass": "net_mass",
|
||||
"Teža": "net_mass",
|
||||
"Energijski razred": "energy_class",
|
||||
":": "",
|
||||
"true": "",
|
||||
"": "",
|
||||
}
|
||||
for in, want := range cases {
|
||||
if got := CanonicalAttributeKey(in); got != want {
|
||||
t.Fatalf("%q → %q want %q", in, got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseSpecificationsSkipsBareListItems(t *testing.T) {
|
||||
raw := `<ul><li>:</li><li>Color: Red</li><li>Waterproof</li></ul>`
|
||||
pairs := ParseSpecifications(raw)
|
||||
if len(pairs) != 1 || pairs[0].Label != "Color" || pairs[0].Value != "Red" {
|
||||
t.Fatalf("got %#v", pairs)
|
||||
}
|
||||
m := specsToMap(pairs)
|
||||
if _, ok := m[":"]; ok {
|
||||
t.Fatalf("junk key present: %#v", m)
|
||||
}
|
||||
if m["color"] != "Red" {
|
||||
t.Fatalf("map=%#v", m)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseSpecificationsHTML(t *testing.T) {
|
||||
raw := `<ul><li>Color: Red</li><li>Size: Large</li><li>Material: Cotton</li></ul>`
|
||||
pairs := ParseSpecifications(raw)
|
||||
if len(pairs) != 3 {
|
||||
t.Fatalf("got %d pairs: %#v", len(pairs), pairs)
|
||||
}
|
||||
if pairs[0].Label != "Color" || pairs[0].Value != "Red" {
|
||||
t.Fatalf("first=%#v", pairs[0])
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseSpecificationsHTMLEmpty(t *testing.T) {
|
||||
for _, raw := range []string{"", " ", "<ul></ul>", "<ul><li></li></ul>", "<![CDATA[]]>"} {
|
||||
if got := ParseSpecifications(raw); got != nil {
|
||||
t.Fatalf("raw=%q got %#v", raw, got)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseSpecificationsFlat(t *testing.T) {
|
||||
raw := "Color: Red; Size: L; Weight: 1.2 kg"
|
||||
pairs := ParseSpecifications(raw)
|
||||
if len(pairs) != 3 {
|
||||
t.Fatalf("got %d: %#v", len(pairs), pairs)
|
||||
}
|
||||
pipe := ParseSpecifications("Voltage: 230V | Frequency: 50Hz")
|
||||
if len(pipe) != 2 {
|
||||
t.Fatalf("pipe=%#v", pipe)
|
||||
}
|
||||
nl := ParseSpecifications("Width: 10\nHeight: 20")
|
||||
if len(nl) != 2 {
|
||||
t.Fatalf("nl=%#v", nl)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseSpecificationsCSVLike(t *testing.T) {
|
||||
raw := `"Brand","Acme","Model","X1"`
|
||||
// Single chunk without strong separators — treat as one line; may not split.
|
||||
// Use semicolon CSV-ish pairs:
|
||||
raw = `"Brand","Acme"; "Model","X1"`
|
||||
pairs := ParseSpecifications(raw)
|
||||
if len(pairs) < 2 {
|
||||
t.Fatalf("got %#v", pairs)
|
||||
}
|
||||
}
|
||||
|
||||
func TestExpandSpecificationFieldsNestedXML(t *testing.T) {
|
||||
xmlBody := `<?xml version="1.0"?>
|
||||
<Export>
|
||||
<Item>
|
||||
<EAN>5901234123457</EAN>
|
||||
<name>Washer</name>
|
||||
<brand>Janus</brand>
|
||||
<specifications>
|
||||
<EnergyClass>A</EnergyClass>
|
||||
<Capacity>8 kg</Capacity>
|
||||
</specifications>
|
||||
<EPRELID></EPRELID>
|
||||
<netMass>72</netMass>
|
||||
</Item>
|
||||
</Export>`
|
||||
var row feedRow
|
||||
n, err := parseXMLItems(strings.NewReader(xmlBody), "Item", func(r feedRow) error {
|
||||
row = r
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if n != 1 {
|
||||
t.Fatalf("n=%d", n)
|
||||
}
|
||||
if row["specifications/EnergyClass"] != "A" {
|
||||
t.Fatalf("nested energy=%q row=%v", row["specifications/EnergyClass"], row)
|
||||
}
|
||||
if row["specifications/Capacity"] != "8 kg" {
|
||||
t.Fatalf("capacity=%q", row["specifications/Capacity"])
|
||||
}
|
||||
v, ok := lookupRow(row, "specifications/EnergyClass")
|
||||
if !ok || v != "A" {
|
||||
t.Fatalf("lookup nested: ok=%v v=%q", ok, v)
|
||||
}
|
||||
}
|
||||
|
||||
func TestExpandSpecificationFieldsCDATA(t *testing.T) {
|
||||
xmlBody := `<?xml version="1.0"?>
|
||||
<products>
|
||||
<item>
|
||||
<gtin>111</gtin>
|
||||
<specifications><![CDATA[<ul><li>Color: Blue</li><li>Finish: Matte</li></ul>]]></specifications>
|
||||
</item>
|
||||
</products>`
|
||||
var row feedRow
|
||||
_, err := parseXMLItems(strings.NewReader(xmlBody), "item", func(r feedRow) error {
|
||||
row = r
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if row["specifications/Color"] != "Blue" {
|
||||
t.Fatalf("color=%q row=%v", row["specifications/Color"], row)
|
||||
}
|
||||
if row["specifications/Finish"] != "Matte" {
|
||||
t.Fatalf("finish=%q", row["specifications/Finish"])
|
||||
}
|
||||
}
|
||||
|
||||
func TestExpandSpecificationFieldsFlatAndEmpty(t *testing.T) {
|
||||
xmlBody := `<?xml version="1.0"?>
|
||||
<products>
|
||||
<item>
|
||||
<ean>222</ean>
|
||||
<specifications>Width: 10; Height: 20</specifications>
|
||||
</item>
|
||||
<item>
|
||||
<ean>333</ean>
|
||||
<specifications><![CDATA[<ul></ul>]]></specifications>
|
||||
</item>
|
||||
</products>`
|
||||
var rows []feedRow
|
||||
_, err := parseXMLItems(strings.NewReader(xmlBody), "item", func(r feedRow) error {
|
||||
cp := make(feedRow, len(r))
|
||||
for k, v := range r {
|
||||
cp[k] = v
|
||||
}
|
||||
rows = append(rows, cp)
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(rows) != 2 {
|
||||
t.Fatalf("rows=%d", len(rows))
|
||||
}
|
||||
if rows[0]["specifications/Width"] != "10" {
|
||||
t.Fatalf("width=%q", rows[0]["specifications/Width"])
|
||||
}
|
||||
// Empty HTML must not invent children.
|
||||
for k := range rows[1] {
|
||||
if strings.HasPrefix(k, "specifications/") {
|
||||
t.Fatalf("unexpected child %q", k)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestApplyMappingsSpecificationsObject(t *testing.T) {
|
||||
row := feedRow{
|
||||
"EAN": "5901234123457",
|
||||
"specifications": `<ul><li>Color: Red</li></ul>`,
|
||||
"specifications/Color": "Red",
|
||||
"specifications/EnergyClass": "B",
|
||||
}
|
||||
expandSpecificationFields(row)
|
||||
mappings := parseMappings([]any{
|
||||
map[string]any{"source": "EAN", "target": "gtin"},
|
||||
map[string]any{"source": "specifications", "target": "specifications"},
|
||||
map[string]any{"source": "specifications/EnergyClass", "target": "energy_class"},
|
||||
})
|
||||
mapped, gtin := applyMappings(row, mappings)
|
||||
if gtin != "5901234123457" {
|
||||
t.Fatalf("gtin=%q", gtin)
|
||||
}
|
||||
specs, ok := mapped["specifications"].(map[string]string)
|
||||
if !ok || specs["color"] != "Red" {
|
||||
t.Fatalf("specs=%#v", mapped["specifications"])
|
||||
}
|
||||
if mapped["energy_class"] != "B" {
|
||||
t.Fatalf("energy=%v", mapped["energy_class"])
|
||||
}
|
||||
}
|
||||
|
||||
func TestExtractXMLSchemaNestedSpecs(t *testing.T) {
|
||||
data := []byte(`<?xml version="1.0"?>
|
||||
<Export><Item>
|
||||
<EAN>1</EAN>
|
||||
<specifications><Color>Red</Color><Size>M</Size></specifications>
|
||||
</Item></Export>`)
|
||||
fields, rows, err := extractXMLSchema(data, "Item")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if rows != 1 {
|
||||
t.Fatalf("rows=%d", rows)
|
||||
}
|
||||
paths := map[string]bool{}
|
||||
for _, f := range fields {
|
||||
paths[f.Path] = true
|
||||
}
|
||||
if !paths["specifications/Color"] || !paths["specifications/Size"] {
|
||||
t.Fatalf("missing nested paths: %#v", paths)
|
||||
}
|
||||
// Bare Color leaf should be suppressed when nested exists.
|
||||
if paths["Color"] {
|
||||
t.Fatalf("bare Color should be dropped: %#v", paths)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLookupRowNestedPrefersPath(t *testing.T) {
|
||||
row := feedRow{
|
||||
"name": "Product",
|
||||
"specifications": "ignored",
|
||||
"specifications/foo": "nested",
|
||||
"other/foo": "other",
|
||||
}
|
||||
v, ok := lookupRow(row, "specifications/foo")
|
||||
if !ok || v != "nested" {
|
||||
t.Fatalf("got ok=%v v=%q", ok, v)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,188 @@
|
||||
package feeds
|
||||
|
||||
import "strings"
|
||||
|
||||
// MappingSuggestion is a suggested source→target pair from schema extraction.
|
||||
type MappingSuggestion struct {
|
||||
Source string `json:"source"`
|
||||
Target string `json:"target"`
|
||||
Confidence string `json:"confidence"`
|
||||
Score float64 `json:"score"`
|
||||
}
|
||||
|
||||
var sourceAliases = map[string][]string{
|
||||
"gtin": {"gtin"},
|
||||
"ean": {"gtin"},
|
||||
"upc": {"gtin"},
|
||||
"barcode": {"gtin"},
|
||||
"title": {"title"},
|
||||
"name": {"title"},
|
||||
"productname": {"title"},
|
||||
"brand": {"brand"},
|
||||
"manufacturer": {"brand"},
|
||||
"description": {"description"},
|
||||
"desc": {"description"},
|
||||
"price": {"price"},
|
||||
"regularprice": {"price"},
|
||||
"saleprice": {"sale_price", "price"},
|
||||
"currency": {"currency"},
|
||||
"image": {"image_url", "main_image", "image"},
|
||||
"imageurl": {"image_url", "main_image", "image"},
|
||||
"imagelink": {"image_url", "main_image", "image"},
|
||||
"mainimage": {"image_url", "main_image", "image"},
|
||||
"mainimageurl": {"image_url", "main_image", "image"},
|
||||
"link": {"product_url"},
|
||||
"url": {"product_url"},
|
||||
"producturl": {"product_url"},
|
||||
"sku": {"sku"},
|
||||
"mpn": {"mpn"},
|
||||
"category": {"category"},
|
||||
"availability": {"availability"},
|
||||
"stockstatus": {"availability", "stock_status"},
|
||||
"stock": {"stock"},
|
||||
"quantity": {"stock"},
|
||||
"qty": {"stock"},
|
||||
"color": {"color"},
|
||||
"size": {"size"},
|
||||
"material": {"material"},
|
||||
"weight": {"weight"},
|
||||
"netmass": {"weight"},
|
||||
"purchaseprice": {"purchase_price", "price"},
|
||||
"buyprice": {"purchase_price"},
|
||||
"cost": {"purchase_price"},
|
||||
"costprice": {"purchase_price"},
|
||||
"officiallink": {"official_link"},
|
||||
"warranty": {"warranty"},
|
||||
"garancija": {"warranty"},
|
||||
"service": {"service"},
|
||||
"servis": {"service"},
|
||||
"productmodel": {"product_model"},
|
||||
"model": {"product_model"},
|
||||
"modelnumber": {"product_model"},
|
||||
"eprelid": {"eprel_id"},
|
||||
"eprel": {"eprel_id"},
|
||||
"specifications": {"specs", "specifications"},
|
||||
"specs": {"specs", "specifications"},
|
||||
"specification": {"specs", "specifications"},
|
||||
"techspecs": {"specs", "specifications"},
|
||||
}
|
||||
|
||||
func normalizeSuggestKey(raw string) string {
|
||||
s := strings.ToLower(strings.TrimSpace(raw))
|
||||
var b strings.Builder
|
||||
for _, r := range s {
|
||||
if (r >= 'a' && r <= 'z') || (r >= '0' && r <= '9') {
|
||||
b.WriteRune(r)
|
||||
}
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
|
||||
func leafSuggestKey(path string) string {
|
||||
leaf := path
|
||||
if i := strings.LastIndex(path, "/"); i >= 0 {
|
||||
leaf = path[i+1:]
|
||||
}
|
||||
if i := strings.LastIndex(leaf, ":"); i >= 0 {
|
||||
leaf = leaf[i+1:]
|
||||
}
|
||||
return normalizeSuggestKey(leaf)
|
||||
}
|
||||
|
||||
// SuggestTarget returns the best canonical target key for a single source name/path,
|
||||
// or "" when no alias matches. Used during schema extract to annotate fields.
|
||||
func SuggestTarget(source string) string {
|
||||
keys := []string{normalizeSuggestKey(source), leafSuggestKey(source)}
|
||||
for _, key := range keys {
|
||||
if key == "" {
|
||||
continue
|
||||
}
|
||||
if aliases, ok := sourceAliases[key]; ok && len(aliases) > 0 {
|
||||
return aliases[0]
|
||||
}
|
||||
// Exact identity for known-looking keys already in alias map as targets.
|
||||
for _, aliases := range sourceAliases {
|
||||
for _, a := range aliases {
|
||||
if a == key {
|
||||
return a
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// SuggestMappings fuzzy-matches schema fields onto enabled target keys (1:1).
|
||||
func SuggestMappings(schema []SchemaField, enabledTargets []string) []MappingSuggestion {
|
||||
enabled := make(map[string]struct{}, len(enabledTargets))
|
||||
for _, t := range enabledTargets {
|
||||
t = strings.TrimSpace(t)
|
||||
if t != "" {
|
||||
enabled[t] = struct{}{}
|
||||
}
|
||||
}
|
||||
if len(schema) == 0 || len(enabled) == 0 {
|
||||
return nil
|
||||
}
|
||||
used := map[string]struct{}{}
|
||||
type scored struct {
|
||||
MappingSuggestion
|
||||
order int
|
||||
}
|
||||
var candidates []scored
|
||||
for i, f := range schema {
|
||||
keys := []string{
|
||||
normalizeSuggestKey(f.FieldName),
|
||||
leafSuggestKey(f.Path),
|
||||
normalizeSuggestKey(f.Path),
|
||||
}
|
||||
var hit *MappingSuggestion
|
||||
for _, key := range keys {
|
||||
if key == "" {
|
||||
continue
|
||||
}
|
||||
if _, ok := enabled[key]; ok {
|
||||
if _, taken := used[key]; !taken {
|
||||
hit = &MappingSuggestion{Source: f.Path, Target: key, Confidence: "exact", Score: 1}
|
||||
break
|
||||
}
|
||||
}
|
||||
if aliases, ok := sourceAliases[key]; ok {
|
||||
for _, a := range aliases {
|
||||
if _, ok := enabled[a]; !ok {
|
||||
continue
|
||||
}
|
||||
if _, taken := used[a]; taken {
|
||||
continue
|
||||
}
|
||||
hit = &MappingSuggestion{Source: f.Path, Target: a, Confidence: "alias", Score: 0.95}
|
||||
break
|
||||
}
|
||||
if hit != nil {
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
if hit != nil {
|
||||
candidates = append(candidates, scored{MappingSuggestion: *hit, order: i})
|
||||
}
|
||||
}
|
||||
// Prefer higher score, then earlier schema order.
|
||||
for i := 0; i < len(candidates); i++ {
|
||||
for j := i + 1; j < len(candidates); j++ {
|
||||
if candidates[j].Score > candidates[i].Score ||
|
||||
(candidates[j].Score == candidates[i].Score && candidates[j].order < candidates[i].order) {
|
||||
candidates[i], candidates[j] = candidates[j], candidates[i]
|
||||
}
|
||||
}
|
||||
}
|
||||
out := make([]MappingSuggestion, 0, len(candidates))
|
||||
for _, c := range candidates {
|
||||
if _, taken := used[c.Target]; taken {
|
||||
continue
|
||||
}
|
||||
used[c.Target] = struct{}{}
|
||||
out = append(out, c.MappingSuggestion)
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -0,0 +1,99 @@
|
||||
package feeds
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestSuggestTarget_b2bFields(t *testing.T) {
|
||||
cases := map[string]string{
|
||||
"purchasePrice": "purchase_price",
|
||||
"stockStatus": "availability",
|
||||
"officialLink": "official_link",
|
||||
"warranty": "warranty",
|
||||
"service": "service",
|
||||
"productModel": "product_model",
|
||||
"EPRELID": "eprel_id",
|
||||
"EAN": "gtin",
|
||||
"name": "title",
|
||||
"netMass": "weight",
|
||||
"mainImage": "image_url",
|
||||
}
|
||||
for in, want := range cases {
|
||||
if got := SuggestTarget(in); got != want {
|
||||
t.Fatalf("%s: got %q want %q", in, got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestNormalizeSuggestKey(t *testing.T) {
|
||||
cases := map[string]string{
|
||||
"EAN": "ean",
|
||||
"purchasePrice": "purchaseprice",
|
||||
"mainImage": "mainimage",
|
||||
"EPRELID": "eprelid",
|
||||
"specifications": "specifications",
|
||||
}
|
||||
for in, want := range cases {
|
||||
got := normalizeSuggestKey(in)
|
||||
if got != want {
|
||||
t.Fatalf("%q: got %q want %q", in, got, want)
|
||||
}
|
||||
}
|
||||
if leafSuggestKey("rss/channel/item/g:gtin") != "gtin" {
|
||||
t.Fatalf("leaf g:gtin → %q", leafSuggestKey("rss/channel/item/g:gtin"))
|
||||
}
|
||||
}
|
||||
|
||||
func TestSuggestMappingsAliases(t *testing.T) {
|
||||
schema := []SchemaField{
|
||||
{Path: "EAN", FieldName: "EAN"},
|
||||
{Path: "name", FieldName: "name"},
|
||||
{Path: "brand", FieldName: "brand"},
|
||||
{Path: "purchasePrice", FieldName: "purchasePrice"},
|
||||
{Path: "stock", FieldName: "stock"},
|
||||
{Path: "mainImage", FieldName: "mainImage"},
|
||||
{Path: "EPRELID", FieldName: "EPRELID"},
|
||||
{Path: "specifications", FieldName: "specifications"},
|
||||
}
|
||||
targets := []string{
|
||||
"gtin", "title", "brand", "price", "purchase_price", "stock", "image_url", "eprel_id", "specs",
|
||||
}
|
||||
got := SuggestMappings(schema, targets)
|
||||
bySrc := map[string]string{}
|
||||
for _, s := range got {
|
||||
bySrc[s.Source] = s.Target
|
||||
}
|
||||
want := map[string]string{
|
||||
"EAN": "gtin",
|
||||
"name": "title",
|
||||
"brand": "brand",
|
||||
"purchasePrice": "purchase_price",
|
||||
"stock": "stock",
|
||||
"mainImage": "image_url",
|
||||
"EPRELID": "eprel_id",
|
||||
"specifications": "specs",
|
||||
}
|
||||
for src, tgt := range want {
|
||||
if bySrc[src] != tgt {
|
||||
t.Fatalf("%s → %q, want %q (all=%v)", src, bySrc[src], tgt, bySrc)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestSuggestMappingsRespectsEnabled(t *testing.T) {
|
||||
schema := []SchemaField{{Path: "EAN", FieldName: "EAN"}, {Path: "name", FieldName: "name"}}
|
||||
got := SuggestMappings(schema, []string{"title"})
|
||||
if len(got) != 1 || got[0].Target != "title" {
|
||||
t.Fatalf("expected only title, got %#v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLeafSuggestKey(t *testing.T) {
|
||||
if leafSuggestKey("rss/channel/item/g:gtin") != "gtin" {
|
||||
t.Fatalf("got %q", leafSuggestKey("rss/channel/item/g:gtin"))
|
||||
}
|
||||
if !strings.Contains(normalizeSuggestKey("Foo-Bar"), "foobar") {
|
||||
t.Fatal("normalize")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,865 @@
|
||||
package feeds
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"io"
|
||||
"strings"
|
||||
|
||||
"github.com/google/uuid"
|
||||
"github.com/jackc/pgx/v5"
|
||||
)
|
||||
|
||||
const upsertChunkSize = 100
|
||||
|
||||
type syncStats struct {
|
||||
Total int
|
||||
Synced int
|
||||
Skipped int
|
||||
Unchanged int
|
||||
Progress int
|
||||
ContentHash string
|
||||
UnchangedFeed bool
|
||||
Deltas syncDeltaCounts
|
||||
}
|
||||
|
||||
type pendingProduct struct {
|
||||
GTIN string
|
||||
RawData map[string]string
|
||||
MappedData map[string]any
|
||||
ContentHash string
|
||||
}
|
||||
|
||||
// Sync downloads the feed URL, parses CSV/XML, applies mappings, and upserts raw_products
|
||||
// in chunks with progress updates on feed_sync_jobs. Replaces SyncStub.
|
||||
func (s *Service) Sync(ctx context.Context, companyID, feedID uuid.UUID) (map[string]any, error) {
|
||||
feed, err := s.Get(ctx, companyID, feedID)
|
||||
if err != nil {
|
||||
if IsNotFound(err) {
|
||||
return nil, ErrNotFound
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
if err := s.ensureMappingsReadyForSync(ctx, companyID, feedID); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
jobID, err := s.createSyncJob(ctx, companyID, feedID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := s.markJobRunning(ctx, jobID); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
stats, runErr := s.runSync(ctx, companyID, feedID, jobID, feed)
|
||||
if runErr != nil {
|
||||
_ = s.failJob(ctx, jobID, runErr.Error(), stats)
|
||||
return nil, runErr
|
||||
}
|
||||
if err := s.completeJob(ctx, jobID, stats); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
_ = s.persistFeedSyncMeta(ctx, companyID, feedID, jobID, stats)
|
||||
|
||||
job, err := s.GetSyncJob(ctx, companyID, feedID, jobID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return enrichJobWithDeltas(job, stats.Deltas), nil
|
||||
}
|
||||
|
||||
// EnqueueSync creates a pending feed_sync_jobs row and wakes the worker via NOTIFY.
|
||||
// Same-feed pending jobs are reused (no duplicate pending stack). The API process
|
||||
// does not run sync work (no unbound goroutines). Returns the job id immediately
|
||||
// for 202/poll (dashboard) or legacy 200 + jobId (v1).
|
||||
func (s *Service) EnqueueSync(ctx context.Context, companyID, feedID uuid.UUID) (uuid.UUID, error) {
|
||||
if _, err := s.Get(ctx, companyID, feedID); err != nil {
|
||||
if IsNotFound(err) {
|
||||
return uuid.Nil, ErrNotFound
|
||||
}
|
||||
return uuid.Nil, err
|
||||
}
|
||||
if err := s.ensureMappingsReadyForSync(ctx, companyID, feedID); err != nil {
|
||||
return uuid.Nil, err
|
||||
}
|
||||
jobID, err := s.findOrCreatePendingSyncJob(ctx, companyID, feedID)
|
||||
if err != nil {
|
||||
return uuid.Nil, err
|
||||
}
|
||||
_, _ = s.Pool.Exec(ctx, `SELECT pg_notify('feed_sync_jobs', $1)`, jobID.String())
|
||||
return jobID, nil
|
||||
}
|
||||
|
||||
// ClaimNextPendingSyncJob claims one pending feed_sync_jobs row (FOR UPDATE SKIP LOCKED)
|
||||
// and marks it running for the worker.
|
||||
func (s *Service) ClaimNextPendingSyncJob(ctx context.Context) (jobID, companyID, feedID uuid.UUID, err error) {
|
||||
err = s.Pool.QueryRow(ctx, `
|
||||
WITH candidate AS (
|
||||
SELECT id FROM feed_sync_jobs
|
||||
WHERE status = 'pending'
|
||||
ORDER BY created_at ASC, id ASC
|
||||
LIMIT 1
|
||||
FOR UPDATE SKIP LOCKED
|
||||
)
|
||||
UPDATE feed_sync_jobs j
|
||||
SET status = 'running', started_at = now(), updated_at = now()
|
||||
FROM candidate
|
||||
WHERE j.id = candidate.id
|
||||
RETURNING j.id, j.company_id, j.feed_id`).Scan(&jobID, &companyID, &feedID)
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return uuid.Nil, uuid.Nil, uuid.Nil, pgx.ErrNoRows
|
||||
}
|
||||
return jobID, companyID, feedID, err
|
||||
}
|
||||
|
||||
// ProcessSyncJob runs sync for a job already claimed (status=running) by ClaimNextPendingSyncJob.
|
||||
func (s *Service) ProcessSyncJob(ctx context.Context, companyID, feedID, jobID uuid.UUID) error {
|
||||
feed, err := s.Get(ctx, companyID, feedID)
|
||||
if err != nil {
|
||||
msg := err.Error()
|
||||
if IsNotFound(err) {
|
||||
msg = "feed not found"
|
||||
}
|
||||
_ = s.failJob(ctx, jobID, msg, syncStats{})
|
||||
return err
|
||||
}
|
||||
stats, runErr := s.runSync(ctx, companyID, feedID, jobID, feed)
|
||||
if runErr != nil {
|
||||
_ = s.failJob(ctx, jobID, runErr.Error(), stats)
|
||||
return runErr
|
||||
}
|
||||
if err := s.completeJob(ctx, jobID, stats); err != nil {
|
||||
_ = s.failJob(ctx, jobID, err.Error(), stats)
|
||||
return err
|
||||
}
|
||||
_ = s.persistFeedSyncMeta(ctx, companyID, feedID, jobID, stats)
|
||||
return nil
|
||||
}
|
||||
|
||||
// SyncStub is retained as a compatibility alias for Sync.
|
||||
func (s *Service) SyncStub(ctx context.Context, companyID, feedID uuid.UUID) (map[string]any, error) {
|
||||
return s.Sync(ctx, companyID, feedID)
|
||||
}
|
||||
|
||||
func (s *Service) GetSyncJob(ctx context.Context, companyID, feedID, jobID uuid.UUID) (map[string]any, error) {
|
||||
row := s.Pool.QueryRow(ctx, `
|
||||
SELECT id, feed_id, company_id, status, started_at, completed_at,
|
||||
products_synced, products_total, products_skipped, products_unchanged,
|
||||
progress, content_hash, error, created_at, updated_at
|
||||
FROM feed_sync_jobs
|
||||
WHERE id = $1 AND company_id = $2 AND feed_id = $3`, jobID, companyID, feedID)
|
||||
job, err := scanMap(row, []string{
|
||||
"id", "feed_id", "company_id", "status", "started_at", "completed_at",
|
||||
"products_synced", "products_total", "products_skipped", "products_unchanged",
|
||||
"progress", "content_hash", "error", "created_at", "updated_at",
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if deltas, ok := s.loadFeedLastSyncDeltas(ctx, companyID, feedID); ok {
|
||||
if matchJobID(deltas["job_id"], jobID) {
|
||||
return enrichJobWithDeltasMap(job, deltas), nil
|
||||
}
|
||||
}
|
||||
return job, nil
|
||||
}
|
||||
|
||||
func (s *Service) persistFeedSyncMeta(ctx context.Context, companyID, feedID, jobID uuid.UUID, stats syncStats) error {
|
||||
stats.Deltas.JobID = jobID.String()
|
||||
stats.Deltas.Unchanged = stats.Unchanged
|
||||
stats.Deltas.Skipped = stats.Skipped
|
||||
deltaJSON, err := json.Marshal(stats.Deltas.asMap())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_, err = s.Pool.Exec(ctx, `
|
||||
UPDATE input_feeds SET last_synced_at = now(), updated_at = now(),
|
||||
options = COALESCE(options, '{}'::jsonb) || jsonb_build_object(
|
||||
'last_content_hash', to_jsonb($2::text),
|
||||
'last_sync_deltas', $3::jsonb
|
||||
)
|
||||
WHERE id = $1 AND company_id = $4`, feedID, stats.ContentHash, deltaJSON, companyID)
|
||||
return err
|
||||
}
|
||||
|
||||
func (s *Service) loadFeedLastSyncDeltas(ctx context.Context, companyID, feedID uuid.UUID) (map[string]any, bool) {
|
||||
var raw []byte
|
||||
err := s.Pool.QueryRow(ctx, `
|
||||
SELECT options->'last_sync_deltas' FROM input_feeds
|
||||
WHERE id = $1 AND company_id = $2`, feedID, companyID).Scan(&raw)
|
||||
if err != nil || len(raw) == 0 || string(raw) == "null" {
|
||||
return nil, false
|
||||
}
|
||||
var m map[string]any
|
||||
if err := json.Unmarshal(raw, &m); err != nil || m == nil {
|
||||
return nil, false
|
||||
}
|
||||
return m, true
|
||||
}
|
||||
|
||||
func enrichJobWithDeltas(job map[string]any, d syncDeltaCounts) map[string]any {
|
||||
return enrichJobWithDeltasMap(job, d.asMap())
|
||||
}
|
||||
|
||||
func enrichJobWithDeltasMap(job map[string]any, deltas map[string]any) map[string]any {
|
||||
if job == nil {
|
||||
return nil
|
||||
}
|
||||
out := make(map[string]any, len(job)+1)
|
||||
for k, v := range job {
|
||||
out[k] = v
|
||||
}
|
||||
out["deltas"] = deltas
|
||||
out["price_changed"] = deltas["price_changed"]
|
||||
out["stock_changed"] = deltas["stock_changed"]
|
||||
out["availability_changed"] = deltas["availability_changed"]
|
||||
out["title_changed"] = deltas["title_changed"]
|
||||
out["other_changed"] = deltas["other_changed"]
|
||||
out["products_new"] = deltas["new"]
|
||||
return out
|
||||
}
|
||||
|
||||
func matchJobID(v any, id uuid.UUID) bool {
|
||||
switch t := v.(type) {
|
||||
case string:
|
||||
return strings.EqualFold(strings.TrimSpace(t), id.String())
|
||||
case uuid.UUID:
|
||||
return t == id
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
func (s *Service) ListSyncJobs(ctx context.Context, companyID, feedID uuid.UUID, limit int) ([]map[string]any, error) {
|
||||
if limit <= 0 {
|
||||
limit = 50
|
||||
}
|
||||
if limit > 200 {
|
||||
limit = 200
|
||||
}
|
||||
rows, err := s.Pool.Query(ctx, `
|
||||
SELECT id, feed_id, status, started_at, completed_at,
|
||||
products_synced, products_total, products_skipped, products_unchanged,
|
||||
progress, content_hash, error, created_at
|
||||
FROM feed_sync_jobs
|
||||
WHERE company_id = $1 AND feed_id = $2
|
||||
ORDER BY created_at DESC LIMIT $3`, companyID, feedID, limit)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
return scanMaps(rows, []string{
|
||||
"id", "feed_id", "status", "started_at", "completed_at",
|
||||
"products_synced", "products_total", "products_skipped", "products_unchanged",
|
||||
"progress", "content_hash", "error", "created_at",
|
||||
})
|
||||
}
|
||||
|
||||
func (s *Service) runSync(ctx context.Context, companyID, feedID, jobID uuid.UUID, feed map[string]any) (syncStats, error) {
|
||||
var stats syncStats
|
||||
urlStr, _ := feed["url"].(string)
|
||||
feedType, _ := feed["feed_type"].(string)
|
||||
|
||||
src, err := s.loadFeedSource(ctx, companyID, feed)
|
||||
if err != nil {
|
||||
return stats, err
|
||||
}
|
||||
defer src.Close()
|
||||
|
||||
hash, err := sha256HexFile(src)
|
||||
if err != nil {
|
||||
return stats, err
|
||||
}
|
||||
stats.ContentHash = hash
|
||||
|
||||
prev, _ := s.lastContentHash(ctx, feedID)
|
||||
if prev != "" && prev == hash {
|
||||
stats.UnchangedFeed = true
|
||||
stats.Progress = 100
|
||||
_ = s.updateJobProgress(ctx, jobID, stats)
|
||||
return stats, nil
|
||||
}
|
||||
|
||||
mappingsRaw, mapErr := s.loadMappingsRaw(ctx, companyID, feedID)
|
||||
if mapErr != nil && !errors.Is(mapErr, pgx.ErrNoRows) {
|
||||
return stats, mapErr
|
||||
}
|
||||
mappings := activeMappings(parseMappings(mappingsRaw))
|
||||
if len(mappings) == 0 {
|
||||
return stats, ClientMsg("no field mappings defined for feed")
|
||||
}
|
||||
|
||||
itemPath := "item"
|
||||
if path := itemPathFromMappings(mappingsRaw); path != "" {
|
||||
itemPath = itemLocalFromPath(path)
|
||||
} else if opts, ok := feed["options"].(map[string]any); ok {
|
||||
if v, ok := opts["item_path"].(string); ok && strings.TrimSpace(v) != "" {
|
||||
itemPath = itemLocalFromPath(v)
|
||||
}
|
||||
}
|
||||
if itemPath == "" {
|
||||
itemPath = "item"
|
||||
}
|
||||
|
||||
sample, sniffErr := src.Sniff(4096)
|
||||
if sniffErr != nil {
|
||||
return stats, sniffErr
|
||||
}
|
||||
format := detectFeedFormat(feedType, contentTypeFromBlob(src), urlStr, sample)
|
||||
chunk := make([]pendingProduct, 0, upsertChunkSize)
|
||||
|
||||
flush := func(force bool) error {
|
||||
if len(chunk) == 0 {
|
||||
return nil
|
||||
}
|
||||
if !force && len(chunk) < upsertChunkSize {
|
||||
return nil
|
||||
}
|
||||
synced, unchanged, skipped, deltas, err := s.upsertChunk(ctx, companyID, feedID, jobID, chunk)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
stats.Synced += synced
|
||||
stats.Unchanged += unchanged
|
||||
stats.Skipped += skipped
|
||||
stats.Deltas.New += deltas.New
|
||||
stats.Deltas.PriceChanged += deltas.PriceChanged
|
||||
stats.Deltas.StockChanged += deltas.StockChanged
|
||||
stats.Deltas.AvailabilityChanged += deltas.AvailabilityChanged
|
||||
stats.Deltas.TitleChanged += deltas.TitleChanged
|
||||
stats.Deltas.OtherChanged += deltas.OtherChanged
|
||||
chunk = chunk[:0]
|
||||
done := stats.Synced + stats.Unchanged + stats.Skipped
|
||||
if stats.Total > 0 {
|
||||
stats.Progress = done * 100 / stats.Total
|
||||
if stats.Progress > 99 {
|
||||
stats.Progress = 99
|
||||
}
|
||||
}
|
||||
_ = s.updateJobProgress(ctx, jobID, stats)
|
||||
return nil
|
||||
}
|
||||
|
||||
onRow := func(row feedRow) error {
|
||||
stats.Total++
|
||||
mapped, gtin := applyMappings(row, mappings)
|
||||
if gtin == "" {
|
||||
stats.Skipped++
|
||||
return nil
|
||||
}
|
||||
rawCopy := make(map[string]string, len(row))
|
||||
for k, v := range row {
|
||||
rawCopy[k] = v
|
||||
}
|
||||
mh := sha256Hex([]byte(mustJSON(mapped)))
|
||||
chunk = append(chunk, pendingProduct{
|
||||
GTIN: gtin, RawData: rawCopy, MappedData: mapped, ContentHash: mh,
|
||||
})
|
||||
return flush(false)
|
||||
}
|
||||
|
||||
body, err := src.Open()
|
||||
if err != nil {
|
||||
return stats, err
|
||||
}
|
||||
defer body.Close()
|
||||
|
||||
var parseCount int
|
||||
switch format {
|
||||
case "xml":
|
||||
parseCount, err = parseXMLItems(body, itemPath, onRow)
|
||||
default:
|
||||
parseCount, err = parseCSV(body, onRow)
|
||||
}
|
||||
if err != nil {
|
||||
return stats, err
|
||||
}
|
||||
if parseCount == 0 {
|
||||
return stats, ClientMsg("feed contained no rows")
|
||||
}
|
||||
if err := flush(true); err != nil {
|
||||
return stats, err
|
||||
}
|
||||
stats.Progress = 100
|
||||
_ = s.updateJobProgress(ctx, jobID, stats)
|
||||
return stats, nil
|
||||
}
|
||||
|
||||
type existingProduct struct {
|
||||
ID uuid.UUID
|
||||
MappedData []byte
|
||||
}
|
||||
|
||||
const (
|
||||
// Set-based upserts (UNNEST / ANY) — one statement per op kind, queued in a single
|
||||
// pgx.Batch round-trip. Mirrors catalog/import_csv.go patterns.
|
||||
upsertSQLInsertSet = `
|
||||
INSERT INTO raw_products (
|
||||
company_id, gtin, feed_id, raw_data, mapped_data, sync_job_id,
|
||||
is_processed, processing_status, updated_at
|
||||
)
|
||||
SELECT $1, v.gtin, $2, v.raw_data::jsonb, v.mapped_data::jsonb, $3, false, 'unprocessed', now()
|
||||
FROM unnest($4::text[], $5::text[], $6::text[]) AS v(gtin, raw_data, mapped_data)
|
||||
ON CONFLICT (company_id, gtin) DO UPDATE SET
|
||||
feed_id = EXCLUDED.feed_id,
|
||||
raw_data = EXCLUDED.raw_data,
|
||||
mapped_data = EXCLUDED.mapped_data,
|
||||
sync_job_id = EXCLUDED.sync_job_id,
|
||||
is_processed = false,
|
||||
processing_status = 'unprocessed',
|
||||
updated_at = now()`
|
||||
upsertSQLTouchSet = `
|
||||
UPDATE raw_products SET sync_job_id = $3, feed_id = $4, updated_at = now()
|
||||
WHERE company_id = $1 AND id = ANY($2::uuid[])`
|
||||
upsertSQLUpdateSet = `
|
||||
UPDATE raw_products AS r SET
|
||||
feed_id = $2,
|
||||
raw_data = v.raw_data::jsonb,
|
||||
mapped_data = v.mapped_data::jsonb,
|
||||
sync_job_id = $3,
|
||||
is_processed = false,
|
||||
processing_status = 'unprocessed',
|
||||
updated_at = now()
|
||||
FROM unnest($4::uuid[], $5::text[], $6::text[]) AS v(id, raw_data, mapped_data)
|
||||
WHERE r.id = v.id AND r.company_id = $1`
|
||||
)
|
||||
|
||||
type upsertOpKind int
|
||||
|
||||
const (
|
||||
upsertOpInsert upsertOpKind = iota
|
||||
upsertOpTouch
|
||||
upsertOpUpdate
|
||||
)
|
||||
|
||||
type upsertOp struct {
|
||||
kind upsertOpKind
|
||||
gtin string
|
||||
id uuid.UUID
|
||||
rawJSON []byte
|
||||
mappedJSON []byte
|
||||
changes []string
|
||||
}
|
||||
|
||||
// classifyUpsertOps decides insert / touch / update per row without DB I/O so a chunk
|
||||
// can be applied with set-based UNNEST statements (≤3) in one pgx.Batch round-trip.
|
||||
// Touch keeps is_processed/processing_status when mapped_data is equal (canonical JSON).
|
||||
// Updates/inserts stamp mapped_data._sync_changes for seller filters (price/stock/…).
|
||||
func classifyUpsertOps(chunk []pendingProduct, existing map[string]existingProduct) (ops []upsertOp, skipped int) {
|
||||
ops = make([]upsertOp, 0, len(chunk))
|
||||
for _, p := range chunk {
|
||||
rawJSON, err := json.Marshal(p.RawData)
|
||||
if err != nil {
|
||||
skipped++
|
||||
continue
|
||||
}
|
||||
ex, found := existing[p.GTIN]
|
||||
cleanMapped := stripSyncChanges(p.MappedData)
|
||||
if !found {
|
||||
withFlags := withSyncChanges(cleanMapped, []string{"new"})
|
||||
mappedJSON, err := json.Marshal(withFlags)
|
||||
if err != nil {
|
||||
skipped++
|
||||
continue
|
||||
}
|
||||
ops = append(ops, upsertOp{
|
||||
kind: upsertOpInsert, gtin: p.GTIN, rawJSON: rawJSON, mappedJSON: mappedJSON,
|
||||
changes: []string{"new"},
|
||||
})
|
||||
continue
|
||||
}
|
||||
// Category (and similar extras) live outside feed mappings — keep them.
|
||||
cleanMapped = preserveSyncedExtras(ex.MappedData, cleanMapped)
|
||||
plainJSON, err := json.Marshal(cleanMapped)
|
||||
if err != nil {
|
||||
skipped++
|
||||
continue
|
||||
}
|
||||
if bytesEqualJSON(stripSyncChangesBytes(ex.MappedData), plainJSON) {
|
||||
ops = append(ops, upsertOp{kind: upsertOpTouch, id: ex.ID})
|
||||
continue
|
||||
}
|
||||
changes := detectMappedChanges(ex.MappedData, cleanMapped)
|
||||
withFlags := withSyncChanges(cleanMapped, changes)
|
||||
mappedJSON, err := json.Marshal(withFlags)
|
||||
if err != nil {
|
||||
skipped++
|
||||
continue
|
||||
}
|
||||
ops = append(ops, upsertOp{
|
||||
kind: upsertOpUpdate, id: ex.ID, rawJSON: rawJSON, mappedJSON: mappedJSON,
|
||||
changes: changes,
|
||||
})
|
||||
}
|
||||
return ops, skipped
|
||||
}
|
||||
|
||||
func stripSyncChangesBytes(raw []byte) []byte {
|
||||
if len(raw) == 0 {
|
||||
return raw
|
||||
}
|
||||
var m map[string]any
|
||||
if err := json.Unmarshal(raw, &m); err != nil {
|
||||
return raw
|
||||
}
|
||||
out, err := json.Marshal(stripSyncChanges(m))
|
||||
if err != nil {
|
||||
return raw
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// dedupePendingByGTIN keeps first-seen order but last-seen payload per GTIN so a single
|
||||
// UNNEST INSERT cannot hit "cannot affect row a second time".
|
||||
func dedupePendingByGTIN(chunk []pendingProduct) []pendingProduct {
|
||||
if len(chunk) < 2 {
|
||||
return chunk
|
||||
}
|
||||
by := make(map[string]pendingProduct, len(chunk))
|
||||
order := make([]string, 0, len(chunk))
|
||||
for _, p := range chunk {
|
||||
if _, ok := by[p.GTIN]; !ok {
|
||||
order = append(order, p.GTIN)
|
||||
}
|
||||
by[p.GTIN] = p
|
||||
}
|
||||
if len(order) == len(chunk) {
|
||||
return chunk
|
||||
}
|
||||
out := make([]pendingProduct, 0, len(order))
|
||||
for _, gtin := range order {
|
||||
out = append(out, by[gtin])
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func partitionUpsertOps(ops []upsertOp) (inserts, touches, updates []upsertOp) {
|
||||
for _, op := range ops {
|
||||
switch op.kind {
|
||||
case upsertOpInsert:
|
||||
inserts = append(inserts, op)
|
||||
case upsertOpTouch:
|
||||
touches = append(touches, op)
|
||||
default:
|
||||
updates = append(updates, op)
|
||||
}
|
||||
}
|
||||
return inserts, touches, updates
|
||||
}
|
||||
|
||||
func (s *Service) upsertChunk(ctx context.Context, companyID, feedID, jobID uuid.UUID, chunk []pendingProduct) (synced, unchanged, skipped int, deltas syncDeltaCounts, err error) {
|
||||
chunk = dedupePendingByGTIN(chunk)
|
||||
if len(chunk) == 0 {
|
||||
return 0, 0, 0, deltas, nil
|
||||
}
|
||||
gtins := make([]string, 0, len(chunk))
|
||||
for _, p := range chunk {
|
||||
gtins = append(gtins, p.GTIN)
|
||||
}
|
||||
existing, err := s.loadExistingByGTIN(ctx, companyID, gtins)
|
||||
if err != nil {
|
||||
return 0, 0, 0, deltas, err
|
||||
}
|
||||
|
||||
ops, skipped := classifyUpsertOps(chunk, existing)
|
||||
if len(ops) == 0 {
|
||||
deltas.Skipped = skipped
|
||||
return 0, 0, skipped, deltas, nil
|
||||
}
|
||||
|
||||
inserts, touches, updates := partitionUpsertOps(ops)
|
||||
batch := &pgx.Batch{}
|
||||
queued := 0
|
||||
if len(inserts) > 0 {
|
||||
gtinCol := make([]string, len(inserts))
|
||||
rawCol := make([]string, len(inserts))
|
||||
mappedCol := make([]string, len(inserts))
|
||||
for i, op := range inserts {
|
||||
gtinCol[i] = op.gtin
|
||||
rawCol[i] = string(op.rawJSON)
|
||||
mappedCol[i] = string(op.mappedJSON)
|
||||
deltas.addChanges(op.changes)
|
||||
}
|
||||
batch.Queue(upsertSQLInsertSet, companyID, feedID, jobID, gtinCol, rawCol, mappedCol)
|
||||
queued++
|
||||
}
|
||||
if len(touches) > 0 {
|
||||
ids := make([]uuid.UUID, len(touches))
|
||||
for i, op := range touches {
|
||||
ids[i] = op.id
|
||||
}
|
||||
batch.Queue(upsertSQLTouchSet, companyID, ids, jobID, feedID)
|
||||
queued++
|
||||
}
|
||||
if len(updates) > 0 {
|
||||
ids := make([]uuid.UUID, len(updates))
|
||||
rawCol := make([]string, len(updates))
|
||||
mappedCol := make([]string, len(updates))
|
||||
for i, op := range updates {
|
||||
ids[i] = op.id
|
||||
rawCol[i] = string(op.rawJSON)
|
||||
mappedCol[i] = string(op.mappedJSON)
|
||||
deltas.addChanges(op.changes)
|
||||
}
|
||||
batch.Queue(upsertSQLUpdateSet, companyID, feedID, jobID, ids, rawCol, mappedCol)
|
||||
queued++
|
||||
}
|
||||
|
||||
br := s.Pool.SendBatch(ctx, batch)
|
||||
defer br.Close()
|
||||
for i := 0; i < queued; i++ {
|
||||
if _, err := br.Exec(); err != nil {
|
||||
return synced, unchanged, skipped, deltas, err
|
||||
}
|
||||
}
|
||||
// Content inserts/updates stamp raw as unprocessed — drop catalog rows so
|
||||
// processed_products cannot outlive that reset (matches resetRawProducts).
|
||||
// Touches keep processing_status and must not invalidate catalog.
|
||||
if len(inserts) > 0 || len(updates) > 0 {
|
||||
invalidateIDs := make([]uuid.UUID, 0, len(updates))
|
||||
for _, op := range updates {
|
||||
invalidateIDs = append(invalidateIDs, op.id)
|
||||
}
|
||||
invalidateGTINs := make([]string, 0, len(inserts))
|
||||
for _, op := range inserts {
|
||||
invalidateGTINs = append(invalidateGTINs, op.gtin)
|
||||
}
|
||||
if _, err := s.Pool.Exec(ctx, `
|
||||
DELETE FROM processed_products
|
||||
WHERE company_id = $1
|
||||
AND (
|
||||
raw_product_id = ANY($2::uuid[])
|
||||
OR raw_product_id IN (
|
||||
SELECT id FROM raw_products
|
||||
WHERE company_id = $1 AND gtin = ANY($3::text[])
|
||||
)
|
||||
)`, companyID, invalidateIDs, invalidateGTINs); err != nil {
|
||||
return synced, unchanged, skipped, deltas, err
|
||||
}
|
||||
}
|
||||
synced = len(inserts) + len(updates)
|
||||
unchanged = len(touches)
|
||||
deltas.Unchanged = unchanged
|
||||
deltas.Skipped = skipped
|
||||
return synced, unchanged, skipped, deltas, nil
|
||||
}
|
||||
|
||||
func (s *Service) loadExistingByGTIN(ctx context.Context, companyID uuid.UUID, gtins []string) (map[string]existingProduct, error) {
|
||||
out := make(map[string]existingProduct, len(gtins))
|
||||
if len(gtins) == 0 {
|
||||
return out, nil
|
||||
}
|
||||
rows, err := s.Pool.Query(ctx, `
|
||||
SELECT id, gtin, mapped_data FROM raw_products
|
||||
WHERE company_id = $1 AND gtin = ANY($2::text[])`, companyID, gtins)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
for rows.Next() {
|
||||
var ex existingProduct
|
||||
var gtin string
|
||||
if err := rows.Scan(&ex.ID, >in, &ex.MappedData); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out[gtin] = ex
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
func (s *Service) createSyncJob(ctx context.Context, companyID, feedID uuid.UUID) (uuid.UUID, error) {
|
||||
var id uuid.UUID
|
||||
err := s.Pool.QueryRow(ctx, `
|
||||
INSERT INTO feed_sync_jobs (feed_id, company_id, status)
|
||||
VALUES ($1, $2, 'pending') RETURNING id`, feedID, companyID).Scan(&id)
|
||||
return id, err
|
||||
}
|
||||
|
||||
// findOrCreatePendingSyncJob returns an existing pending job for the feed, or inserts one.
|
||||
// Uses a transaction advisory lock so concurrent enqueues do not stack duplicates.
|
||||
func (s *Service) findOrCreatePendingSyncJob(ctx context.Context, companyID, feedID uuid.UUID) (uuid.UUID, error) {
|
||||
tx, err := s.Pool.Begin(ctx)
|
||||
if err != nil {
|
||||
return uuid.Nil, err
|
||||
}
|
||||
defer tx.Rollback(ctx)
|
||||
|
||||
if _, err := tx.Exec(ctx, `SELECT pg_advisory_xact_lock(hashtextextended($1::text, 0))`, feedID.String()); err != nil {
|
||||
return uuid.Nil, err
|
||||
}
|
||||
|
||||
var id uuid.UUID
|
||||
err = tx.QueryRow(ctx, `
|
||||
SELECT id FROM feed_sync_jobs
|
||||
WHERE feed_id = $1 AND company_id = $2 AND status = 'pending'
|
||||
ORDER BY created_at
|
||||
LIMIT 1`, feedID, companyID).Scan(&id)
|
||||
if err == nil {
|
||||
if err := tx.Commit(ctx); err != nil {
|
||||
return uuid.Nil, err
|
||||
}
|
||||
return id, nil
|
||||
}
|
||||
if !errors.Is(err, pgx.ErrNoRows) {
|
||||
return uuid.Nil, err
|
||||
}
|
||||
|
||||
err = tx.QueryRow(ctx, `
|
||||
INSERT INTO feed_sync_jobs (feed_id, company_id, status)
|
||||
VALUES ($1, $2, 'pending') RETURNING id`, feedID, companyID).Scan(&id)
|
||||
if err != nil {
|
||||
return uuid.Nil, err
|
||||
}
|
||||
if err := tx.Commit(ctx); err != nil {
|
||||
return uuid.Nil, err
|
||||
}
|
||||
return id, nil
|
||||
}
|
||||
|
||||
func (s *Service) markJobRunning(ctx context.Context, jobID uuid.UUID) error {
|
||||
_, err := s.Pool.Exec(ctx, `
|
||||
UPDATE feed_sync_jobs SET status = 'running', started_at = now(), updated_at = now()
|
||||
WHERE id = $1`, jobID)
|
||||
return err
|
||||
}
|
||||
|
||||
func (s *Service) updateJobProgress(ctx context.Context, jobID uuid.UUID, stats syncStats) error {
|
||||
_, err := s.Pool.Exec(ctx, `
|
||||
UPDATE feed_sync_jobs SET
|
||||
products_synced = $2,
|
||||
products_total = $3,
|
||||
products_skipped = $4,
|
||||
products_unchanged = $5,
|
||||
progress = $6,
|
||||
content_hash = NULLIF($7, ''),
|
||||
updated_at = now()
|
||||
WHERE id = $1`,
|
||||
jobID, stats.Synced, stats.Total, stats.Skipped, stats.Unchanged, stats.Progress, stats.ContentHash)
|
||||
return err
|
||||
}
|
||||
|
||||
func (s *Service) completeJob(ctx context.Context, jobID uuid.UUID, stats syncStats) error {
|
||||
_, err := s.Pool.Exec(ctx, `
|
||||
UPDATE feed_sync_jobs SET
|
||||
status = 'completed',
|
||||
completed_at = now(),
|
||||
products_synced = $2,
|
||||
products_total = $3,
|
||||
products_skipped = $4,
|
||||
products_unchanged = $5,
|
||||
progress = 100,
|
||||
content_hash = NULLIF($6, ''),
|
||||
updated_at = now()
|
||||
WHERE id = $1`,
|
||||
jobID, stats.Synced, stats.Total, stats.Skipped, stats.Unchanged, stats.ContentHash)
|
||||
return err
|
||||
}
|
||||
|
||||
func (s *Service) failJob(ctx context.Context, jobID uuid.UUID, msg string, stats syncStats) error {
|
||||
_, err := s.Pool.Exec(ctx, `
|
||||
UPDATE feed_sync_jobs SET
|
||||
status = 'failed',
|
||||
error = $2,
|
||||
completed_at = now(),
|
||||
products_synced = $3,
|
||||
products_total = $4,
|
||||
products_skipped = $5,
|
||||
products_unchanged = $6,
|
||||
progress = $7,
|
||||
content_hash = NULLIF($8, ''),
|
||||
updated_at = now()
|
||||
WHERE id = $1`,
|
||||
jobID, truncateErr(msg), stats.Synced, stats.Total, stats.Skipped, stats.Unchanged, stats.Progress, stats.ContentHash)
|
||||
return err
|
||||
}
|
||||
|
||||
func (s *Service) lastContentHash(ctx context.Context, feedID uuid.UUID) (string, error) {
|
||||
var hash *string
|
||||
err := s.Pool.QueryRow(ctx, `
|
||||
SELECT content_hash FROM feed_sync_jobs
|
||||
WHERE feed_id = $1 AND status = 'completed' AND content_hash IS NOT NULL AND content_hash <> ''
|
||||
ORDER BY completed_at DESC NULLS LAST LIMIT 1`, feedID).Scan(&hash)
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return "", nil
|
||||
}
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if hash == nil {
|
||||
return "", nil
|
||||
}
|
||||
return *hash, nil
|
||||
}
|
||||
|
||||
func (s *Service) loadMappingsRaw(ctx context.Context, companyID, feedID uuid.UUID) (any, error) {
|
||||
var raw []byte
|
||||
err := s.Pool.QueryRow(ctx, `
|
||||
SELECT mappings FROM feed_mappings
|
||||
WHERE feed_id = $1 AND company_id = $2 AND is_active = true
|
||||
ORDER BY version DESC LIMIT 1`, feedID, companyID).Scan(&raw)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var m any
|
||||
if err := json.Unmarshal(raw, &m); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return m, nil
|
||||
}
|
||||
|
||||
func sha256Hex(b []byte) string {
|
||||
sum := sha256.Sum256(b)
|
||||
return hex.EncodeToString(sum[:])
|
||||
}
|
||||
|
||||
func sha256HexFile(src *feedBlob) (string, error) {
|
||||
f, err := src.Open()
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
defer f.Close()
|
||||
h := sha256.New()
|
||||
if _, err := io.Copy(h, f); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return hex.EncodeToString(h.Sum(nil)), nil
|
||||
}
|
||||
|
||||
func contentTypeFromBlob(src *feedBlob) string {
|
||||
if src == nil {
|
||||
return ""
|
||||
}
|
||||
return src.contentType
|
||||
}
|
||||
|
||||
func mustJSON(v any) string {
|
||||
b, err := json.Marshal(v)
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
return string(b)
|
||||
}
|
||||
|
||||
func bytesEqualJSON(a, b []byte) bool {
|
||||
if len(a) == 0 && len(b) == 0 {
|
||||
return true
|
||||
}
|
||||
var xa, xb any
|
||||
if json.Unmarshal(a, &xa) != nil || json.Unmarshal(b, &xb) != nil {
|
||||
return string(a) == string(b)
|
||||
}
|
||||
ba, _ := json.Marshal(xa)
|
||||
bb, _ := json.Marshal(xb)
|
||||
return string(ba) == string(bb)
|
||||
}
|
||||
|
||||
func truncateErr(msg string) string {
|
||||
if len(msg) > 2000 {
|
||||
return msg[:2000]
|
||||
}
|
||||
return msg
|
||||
}
|
||||
@@ -0,0 +1,112 @@
|
||||
package feeds
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"os"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/descrybe/descrybe-v2/apps/api/internal/jobs"
|
||||
"github.com/google/uuid"
|
||||
"github.com/jackc/pgx/v5"
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
)
|
||||
|
||||
func TestClaimNextPendingSyncJobConcurrentDistinct(t *testing.T) {
|
||||
dsn := strings.TrimSpace(os.Getenv("DATABASE_URL"))
|
||||
if dsn == "" {
|
||||
t.Skip("DATABASE_URL not set")
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer cancel()
|
||||
|
||||
pg, err := pgxpool.New(ctx, dsn)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// Clean up while the pool is still open (t.Cleanup runs after deferred Close).
|
||||
defer pg.Close()
|
||||
|
||||
companyID := uuid.New()
|
||||
_, err = pg.Exec(ctx, `INSERT INTO companies (id, name) VALUES ($1, $2)`,
|
||||
companyID, "sync-claim-"+companyID.String()[:8])
|
||||
if err != nil {
|
||||
t.Fatalf("seed company: %v", err)
|
||||
}
|
||||
defer func() {
|
||||
_, _ = pg.Exec(context.Background(), `DELETE FROM companies WHERE id = $1`, companyID)
|
||||
}()
|
||||
|
||||
var feedID uuid.UUID
|
||||
err = pg.QueryRow(ctx, `
|
||||
INSERT INTO input_feeds (company_id, name, url, feed_type, status, sync_interval_minutes, options)
|
||||
VALUES ($1, 'claim-feed', 'https://example.com/feed.csv', 'csv', 'active', 60, '{}'::jsonb)
|
||||
RETURNING id`, companyID).Scan(&feedID)
|
||||
if err != nil {
|
||||
t.Fatalf("seed feed: %v", err)
|
||||
}
|
||||
|
||||
// Live dev workers also ClaimNextPendingSyncJob; seed a buffer so N concurrent
|
||||
// claimants still succeed when the worker steals a few pending rows.
|
||||
const n = 4
|
||||
seedN := n + jobs.MaxSyncWorkers + 4
|
||||
jobIDs := make([]uuid.UUID, 0, seedN)
|
||||
for i := 0; i < seedN; i++ {
|
||||
var id uuid.UUID
|
||||
err = pg.QueryRow(ctx, `
|
||||
INSERT INTO feed_sync_jobs (feed_id, company_id, status)
|
||||
VALUES ($1, $2, 'pending') RETURNING id`, feedID, companyID).Scan(&id)
|
||||
if err != nil {
|
||||
t.Fatalf("seed sync job: %v", err)
|
||||
}
|
||||
jobIDs = append(jobIDs, id)
|
||||
}
|
||||
defer func() {
|
||||
for _, id := range jobIDs {
|
||||
_, _ = pg.Exec(context.Background(), `DELETE FROM feed_sync_jobs WHERE id = $1`, id)
|
||||
}
|
||||
}()
|
||||
|
||||
svc := &Service{Pool: pg}
|
||||
claimed := make([]uuid.UUID, n)
|
||||
var wg sync.WaitGroup
|
||||
wg.Add(n)
|
||||
for i := 0; i < n; i++ {
|
||||
go func(i int) {
|
||||
defer wg.Done()
|
||||
deadline := time.Now().Add(3 * time.Second)
|
||||
for {
|
||||
id, _, _, claimErr := svc.ClaimNextPendingSyncJob(ctx)
|
||||
if claimErr == nil {
|
||||
claimed[i] = id
|
||||
return
|
||||
}
|
||||
if !errors.Is(claimErr, pgx.ErrNoRows) {
|
||||
t.Errorf("ClaimNextPendingSyncJob: %v", claimErr)
|
||||
return
|
||||
}
|
||||
if time.Now().After(deadline) {
|
||||
t.Errorf("ClaimNextPendingSyncJob: no rows after retries")
|
||||
return
|
||||
}
|
||||
time.Sleep(5 * time.Millisecond)
|
||||
}
|
||||
}(i)
|
||||
}
|
||||
wg.Wait()
|
||||
|
||||
seen := make(map[uuid.UUID]struct{}, n)
|
||||
for _, id := range claimed {
|
||||
if id == uuid.Nil {
|
||||
t.Fatal("nil claim")
|
||||
}
|
||||
if _, ok := seen[id]; ok {
|
||||
t.Fatalf("duplicate claim %s (SKIP LOCKED failed)", id)
|
||||
}
|
||||
seen[id] = struct{}{}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,222 @@
|
||||
package feeds
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// Reserved mapped_data key holding seller-facing change flags from the latest
|
||||
// sync that modified the row. Cleared/replaced on the next content update.
|
||||
const syncChangesMappedKey = "_sync_changes"
|
||||
|
||||
// Seller-high-value mapped field groups compared during feed upserts.
|
||||
var (
|
||||
syncPriceFields = []string{"price", "sale_price", "purchase_price"}
|
||||
syncStockFields = []string{"stock", "quantity", "qty"}
|
||||
syncAvailFields = []string{"availability", "stock_status", "in_stock"}
|
||||
syncTitleFields = []string{"title", "name", "product_name"}
|
||||
)
|
||||
|
||||
// syncDeltaCounts is the MVP seller summary written to input_feeds.options.last_sync_deltas.
|
||||
type syncDeltaCounts struct {
|
||||
JobID string `json:"job_id,omitempty"`
|
||||
New int `json:"new"`
|
||||
PriceChanged int `json:"price_changed"`
|
||||
StockChanged int `json:"stock_changed"`
|
||||
AvailabilityChanged int `json:"availability_changed"`
|
||||
TitleChanged int `json:"title_changed"`
|
||||
OtherChanged int `json:"other_changed"`
|
||||
Unchanged int `json:"unchanged"`
|
||||
Skipped int `json:"skipped"`
|
||||
}
|
||||
|
||||
func (d *syncDeltaCounts) addChanges(changes []string) {
|
||||
if len(changes) == 0 {
|
||||
d.OtherChanged++
|
||||
return
|
||||
}
|
||||
seen := map[string]struct{}{}
|
||||
for _, c := range changes {
|
||||
if _, ok := seen[c]; ok {
|
||||
continue
|
||||
}
|
||||
seen[c] = struct{}{}
|
||||
switch c {
|
||||
case "new":
|
||||
d.New++
|
||||
case "price":
|
||||
d.PriceChanged++
|
||||
case "stock":
|
||||
d.StockChanged++
|
||||
case "availability":
|
||||
d.AvailabilityChanged++
|
||||
case "title":
|
||||
d.TitleChanged++
|
||||
default:
|
||||
d.OtherChanged++
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (d syncDeltaCounts) asMap() map[string]any {
|
||||
return map[string]any{
|
||||
"job_id": d.JobID,
|
||||
"new": d.New,
|
||||
"price_changed": d.PriceChanged,
|
||||
"stock_changed": d.StockChanged,
|
||||
"availability_changed": d.AvailabilityChanged,
|
||||
"title_changed": d.TitleChanged,
|
||||
"other_changed": d.OtherChanged,
|
||||
"unchanged": d.Unchanged,
|
||||
"skipped": d.Skipped,
|
||||
}
|
||||
}
|
||||
|
||||
func mappedScalar(m map[string]any, key string) string {
|
||||
if m == nil {
|
||||
return ""
|
||||
}
|
||||
v, ok := m[key]
|
||||
if !ok || v == nil {
|
||||
return ""
|
||||
}
|
||||
switch t := v.(type) {
|
||||
case string:
|
||||
return strings.TrimSpace(t)
|
||||
case float64:
|
||||
return strings.TrimSpace(strings.TrimRight(strings.TrimRight(
|
||||
strings.ReplaceAll(jsonNumber(t), "e+0", "e+"), "0"), "."))
|
||||
case json.Number:
|
||||
return strings.TrimSpace(t.String())
|
||||
case bool:
|
||||
if t {
|
||||
return "true"
|
||||
}
|
||||
return "false"
|
||||
default:
|
||||
b, err := json.Marshal(t)
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
return strings.TrimSpace(string(b))
|
||||
}
|
||||
}
|
||||
|
||||
func jsonNumber(f float64) string {
|
||||
b, err := json.Marshal(f)
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
return string(b)
|
||||
}
|
||||
|
||||
func fieldGroupChanged(oldM, newM map[string]any, keys []string) bool {
|
||||
for _, k := range keys {
|
||||
if mappedScalar(oldM, k) != mappedScalar(newM, k) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// detectMappedChanges compares prior mapped_data JSON to the new mapped map.
|
||||
// Returns change kind tags: price, stock, availability, title, other.
|
||||
func detectMappedChanges(oldJSON []byte, newMapped map[string]any) []string {
|
||||
var oldM map[string]any
|
||||
if len(oldJSON) > 0 {
|
||||
_ = json.Unmarshal(oldJSON, &oldM)
|
||||
}
|
||||
if oldM == nil {
|
||||
oldM = map[string]any{}
|
||||
}
|
||||
cleanNew := stripSyncChanges(newMapped)
|
||||
var out []string
|
||||
if fieldGroupChanged(oldM, cleanNew, syncPriceFields) {
|
||||
out = append(out, "price")
|
||||
}
|
||||
if fieldGroupChanged(oldM, cleanNew, syncStockFields) {
|
||||
out = append(out, "stock")
|
||||
}
|
||||
if fieldGroupChanged(oldM, cleanNew, syncAvailFields) {
|
||||
out = append(out, "availability")
|
||||
}
|
||||
if fieldGroupChanged(oldM, cleanNew, syncTitleFields) {
|
||||
out = append(out, "title")
|
||||
}
|
||||
// Any other mapped key change (excluding reserved meta).
|
||||
if len(out) == 0 && !mappedEqualIgnoringSyncMeta(oldM, cleanNew) {
|
||||
out = append(out, "other")
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func stripSyncChanges(m map[string]any) map[string]any {
|
||||
if m == nil {
|
||||
return map[string]any{}
|
||||
}
|
||||
out := make(map[string]any, len(m))
|
||||
for k, v := range m {
|
||||
if k == syncChangesMappedKey {
|
||||
continue
|
||||
}
|
||||
out[k] = v
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// preserveSyncedExtras keeps fields that feed mappings never set (notably
|
||||
// category unique_id from legacy assignment / seed backfill) when a sync
|
||||
// remaps the row. Without this, upserts replace mapped_data wholesale and
|
||||
// wipe category even though the feed has no category column.
|
||||
func preserveSyncedExtras(existingJSON []byte, mapped map[string]any) map[string]any {
|
||||
out := stripSyncChanges(mapped)
|
||||
if len(existingJSON) == 0 {
|
||||
return out
|
||||
}
|
||||
var old map[string]any
|
||||
if err := json.Unmarshal(existingJSON, &old); err != nil || old == nil {
|
||||
return out
|
||||
}
|
||||
old = stripSyncChanges(old)
|
||||
if mappedScalar(out, "category") == "" {
|
||||
if cat := mappedScalar(old, "category"); cat != "" && !strings.EqualFold(cat, "none") {
|
||||
out["category"] = cat
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func mappedEqualIgnoringSyncMeta(a, b map[string]any) bool {
|
||||
aa := stripSyncChanges(a)
|
||||
bb := stripSyncChanges(b)
|
||||
ab, err1 := json.Marshal(aa)
|
||||
bb2, err2 := json.Marshal(bb)
|
||||
if err1 != nil || err2 != nil {
|
||||
return false
|
||||
}
|
||||
return bytesEqualJSON(ab, bb2)
|
||||
}
|
||||
|
||||
func withSyncChanges(mapped map[string]any, changes []string) map[string]any {
|
||||
out := stripSyncChanges(mapped)
|
||||
if len(changes) == 0 {
|
||||
return out
|
||||
}
|
||||
tags := make([]any, 0, len(changes))
|
||||
seen := map[string]struct{}{}
|
||||
for _, c := range changes {
|
||||
c = strings.TrimSpace(strings.ToLower(c))
|
||||
if c == "" {
|
||||
continue
|
||||
}
|
||||
if _, ok := seen[c]; ok {
|
||||
continue
|
||||
}
|
||||
seen[c] = struct{}{}
|
||||
tags = append(tags, c)
|
||||
}
|
||||
if len(tags) > 0 {
|
||||
out[syncChangesMappedKey] = tags
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -0,0 +1,125 @@
|
||||
package feeds
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"testing"
|
||||
|
||||
"github.com/google/uuid"
|
||||
)
|
||||
|
||||
func TestDetectMappedChangesPriceStock(t *testing.T) {
|
||||
t.Parallel()
|
||||
old := []byte(`{"price":"10","stock":"5","title":"A"}`)
|
||||
changes := detectMappedChanges(old, map[string]any{"price": "12", "stock": "5", "title": "A"})
|
||||
if len(changes) != 1 || changes[0] != "price" {
|
||||
t.Fatalf("changes=%v", changes)
|
||||
}
|
||||
changes = detectMappedChanges(old, map[string]any{"price": "10", "stock": "0", "title": "A"})
|
||||
if len(changes) != 1 || changes[0] != "stock" {
|
||||
t.Fatalf("stock changes=%v", changes)
|
||||
}
|
||||
changes = detectMappedChanges(old, map[string]any{"price": "11", "stock": "1", "title": "B", "availability": "out"})
|
||||
want := map[string]bool{"price": true, "stock": true, "title": true, "availability": true}
|
||||
for _, c := range changes {
|
||||
if !want[c] {
|
||||
t.Fatalf("unexpected %s in %v", c, changes)
|
||||
}
|
||||
delete(want, c)
|
||||
}
|
||||
if len(want) != 0 {
|
||||
t.Fatalf("missing %v from %v", want, changes)
|
||||
}
|
||||
}
|
||||
|
||||
func TestClassifyUpsertOpsStampsSyncChanges(t *testing.T) {
|
||||
t.Parallel()
|
||||
idUpdate := uuid.MustParse("22222222-2222-2222-2222-222222222222")
|
||||
existing := map[string]existingProduct{
|
||||
"update": {ID: idUpdate, MappedData: []byte(`{"price":"10","title":"Old"}`)},
|
||||
}
|
||||
chunk := []pendingProduct{
|
||||
{GTIN: "new", RawData: map[string]string{"EAN": "new"}, MappedData: map[string]any{"title": "N"}},
|
||||
{GTIN: "update", RawData: map[string]string{"EAN": "update"}, MappedData: map[string]any{"price": "12", "title": "Old"}},
|
||||
}
|
||||
ops, skipped := classifyUpsertOps(chunk, existing)
|
||||
if skipped != 0 || len(ops) != 2 {
|
||||
t.Fatalf("ops=%d skipped=%d", len(ops), skipped)
|
||||
}
|
||||
if ops[0].kind != upsertOpInsert || len(ops[0].changes) != 1 || ops[0].changes[0] != "new" {
|
||||
t.Fatalf("insert op=%+v", ops[0])
|
||||
}
|
||||
var mapped map[string]any
|
||||
if err := json.Unmarshal(ops[0].mappedJSON, &mapped); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
tags, _ := mapped[syncChangesMappedKey].([]any)
|
||||
if len(tags) != 1 || tags[0] != "new" {
|
||||
t.Fatalf("insert mapped meta=%v", mapped[syncChangesMappedKey])
|
||||
}
|
||||
if ops[1].kind != upsertOpUpdate || len(ops[1].changes) != 1 || ops[1].changes[0] != "price" {
|
||||
t.Fatalf("update op=%+v", ops[1])
|
||||
}
|
||||
}
|
||||
|
||||
func TestSyncDeltaCountsAddChanges(t *testing.T) {
|
||||
t.Parallel()
|
||||
var d syncDeltaCounts
|
||||
d.addChanges([]string{"price", "stock"})
|
||||
d.addChanges([]string{"new"})
|
||||
d.addChanges(nil)
|
||||
if d.PriceChanged != 1 || d.StockChanged != 1 || d.New != 1 || d.OtherChanged != 1 {
|
||||
t.Fatalf("%+v", d)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPreserveSyncedExtrasKeepsCategory(t *testing.T) {
|
||||
t.Parallel()
|
||||
existing := []byte(`{"price":"10","category":"46","title":"Roborock"}`)
|
||||
mapped := map[string]any{"price": "12", "title": "Roborock"}
|
||||
got := preserveSyncedExtras(existing, mapped)
|
||||
if got["category"] != "46" {
|
||||
t.Fatalf("category=%v want 46", got["category"])
|
||||
}
|
||||
if got["price"] != "12" {
|
||||
t.Fatalf("price=%v want 12", got["price"])
|
||||
}
|
||||
// Explicit new category wins.
|
||||
mapped2 := map[string]any{"price": "12", "category": "120"}
|
||||
got2 := preserveSyncedExtras(existing, mapped2)
|
||||
if got2["category"] != "120" {
|
||||
t.Fatalf("category=%v want 120", got2["category"])
|
||||
}
|
||||
// Empty / none prior must not invent a category.
|
||||
got3 := preserveSyncedExtras([]byte(`{"category":"none"}`), map[string]any{"title": "X"})
|
||||
if _, ok := got3["category"]; ok {
|
||||
t.Fatalf("unexpected category=%v", got3["category"])
|
||||
}
|
||||
}
|
||||
|
||||
func TestClassifyUpsertOpsPreservesCategory(t *testing.T) {
|
||||
t.Parallel()
|
||||
idUpdate := uuid.MustParse("33333333-3333-3333-3333-333333333333")
|
||||
existing := map[string]existingProduct{
|
||||
"keep": {ID: idUpdate, MappedData: []byte(`{"price":"10","category":"46","title":"Old"}`)},
|
||||
}
|
||||
chunk := []pendingProduct{
|
||||
{GTIN: "keep", RawData: map[string]string{"EAN": "keep"}, MappedData: map[string]any{"price": "12", "title": "Old"}},
|
||||
}
|
||||
ops, skipped := classifyUpsertOps(chunk, existing)
|
||||
if skipped != 0 || len(ops) != 1 {
|
||||
t.Fatalf("ops=%d skipped=%d", len(ops), skipped)
|
||||
}
|
||||
if ops[0].kind != upsertOpUpdate {
|
||||
t.Fatalf("kind=%v want update", ops[0].kind)
|
||||
}
|
||||
var mapped map[string]any
|
||||
if err := json.Unmarshal(ops[0].mappedJSON, &mapped); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if mapped["category"] != "46" {
|
||||
t.Fatalf("category wiped: %v", mapped["category"])
|
||||
}
|
||||
if mapped["price"] != "12" {
|
||||
t.Fatalf("price=%v", mapped["price"])
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,94 @@
|
||||
package feeds
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/google/uuid"
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
)
|
||||
|
||||
func TestFindOrCreatePendingSyncJobDedupesSameFeed(t *testing.T) {
|
||||
dsn := strings.TrimSpace(os.Getenv("DATABASE_URL"))
|
||||
if dsn == "" {
|
||||
t.Skip("DATABASE_URL not set")
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer cancel()
|
||||
|
||||
pg, err := pgxpool.New(ctx, dsn)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer pg.Close()
|
||||
|
||||
companyID := uuid.New()
|
||||
_, err = pg.Exec(ctx, `INSERT INTO companies (id, name) VALUES ($1, $2)`,
|
||||
companyID, "sync-dedupe-"+companyID.String()[:8])
|
||||
if err != nil {
|
||||
t.Fatalf("seed company: %v", err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
_, _ = pg.Exec(context.Background(), `DELETE FROM companies WHERE id = $1`, companyID)
|
||||
})
|
||||
|
||||
var feedID uuid.UUID
|
||||
err = pg.QueryRow(ctx, `
|
||||
INSERT INTO input_feeds (company_id, name, url, feed_type, status, sync_interval_minutes, options)
|
||||
VALUES ($1, 'dedupe-feed', 'https://example.com/feed.csv', 'csv', 'active', 60, '{}'::jsonb)
|
||||
RETURNING id`, companyID).Scan(&feedID)
|
||||
if err != nil {
|
||||
t.Fatalf("seed feed: %v", err)
|
||||
}
|
||||
|
||||
svc := &Service{Pool: pg}
|
||||
first, err := svc.findOrCreatePendingSyncJob(ctx, companyID, feedID)
|
||||
if err != nil {
|
||||
t.Fatalf("first: %v", err)
|
||||
}
|
||||
second, err := svc.findOrCreatePendingSyncJob(ctx, companyID, feedID)
|
||||
if err != nil {
|
||||
t.Fatalf("second: %v", err)
|
||||
}
|
||||
if first != second {
|
||||
t.Fatalf("dedupe failed: first=%s second=%s", first, second)
|
||||
}
|
||||
|
||||
var pendingCount int
|
||||
if err := pg.QueryRow(ctx, `
|
||||
SELECT COUNT(*) FROM feed_sync_jobs
|
||||
WHERE feed_id = $1 AND company_id = $2 AND status = 'pending'`,
|
||||
feedID, companyID).Scan(&pendingCount); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if pendingCount != 1 {
|
||||
t.Fatalf("pending count=%d want 1", pendingCount)
|
||||
}
|
||||
|
||||
const n = 8
|
||||
ids := make([]uuid.UUID, n)
|
||||
var wg sync.WaitGroup
|
||||
wg.Add(n)
|
||||
for i := 0; i < n; i++ {
|
||||
go func(i int) {
|
||||
defer wg.Done()
|
||||
id, err := svc.findOrCreatePendingSyncJob(ctx, companyID, feedID)
|
||||
if err != nil {
|
||||
t.Errorf("concurrent findOrCreate: %v", err)
|
||||
return
|
||||
}
|
||||
ids[i] = id
|
||||
}(i)
|
||||
}
|
||||
wg.Wait()
|
||||
for _, id := range ids {
|
||||
if id != first {
|
||||
t.Fatalf("concurrent id=%s want %s", id, first)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,390 @@
|
||||
package feeds
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"net/url"
|
||||
"os"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/google/uuid"
|
||||
)
|
||||
|
||||
func TestParseCSVAndMappings(t *testing.T) {
|
||||
csv := "EAN,Title\n123,Widget\n456,\n"
|
||||
mappings := parseMappings([]any{
|
||||
map[string]any{"source": "EAN", "target": "gtin"},
|
||||
map[string]any{"column": "Title", "fieldName": "title"},
|
||||
})
|
||||
var rows []map[string]any
|
||||
n, err := parseCSV(strings.NewReader(csv), func(row feedRow) error {
|
||||
mapped, gtin := applyMappings(row, mappings)
|
||||
rows = append(rows, map[string]any{"gtin": gtin, "mapped": mapped})
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if n != 2 {
|
||||
t.Fatalf("rows=%d", n)
|
||||
}
|
||||
if rows[0]["gtin"] != "123" {
|
||||
t.Fatalf("gtin=%v", rows[0]["gtin"])
|
||||
}
|
||||
m := rows[0]["mapped"].(map[string]any)
|
||||
if m["title"] != "Widget" {
|
||||
t.Fatalf("mapped=%v", m)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseXMLItems(t *testing.T) {
|
||||
xmlBody := `<?xml version="1.0"?><products><item><gtin>999</gtin><title>X</title></item></products>`
|
||||
mappings := parseMappings(map[string]any{
|
||||
"gtin": map[string]any{"fieldName": "gtin"},
|
||||
"title": map[string]any{"fieldName": "title"},
|
||||
})
|
||||
var got string
|
||||
n, err := parseXMLItems(strings.NewReader(xmlBody), "item", func(row feedRow) error {
|
||||
_, gtin := applyMappings(row, mappings)
|
||||
got = gtin
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if n != 1 || got != "999" {
|
||||
t.Fatalf("n=%d gtin=%q", n, got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDownloadRejectsPrivateAndFTP(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
if _, err := downloadFeed(ctx, "ftp://example.com/a.csv"); err == nil {
|
||||
t.Fatal("expected ftp error")
|
||||
}
|
||||
if _, err := downloadFeed(ctx, "http://127.0.0.1/x"); err == nil {
|
||||
t.Fatal("expected private IP error")
|
||||
}
|
||||
if _, err := downloadFeed(ctx, "http://localhost/x"); err == nil {
|
||||
t.Fatal("expected localhost error")
|
||||
}
|
||||
}
|
||||
|
||||
func TestSSRFTransportDisablesEnvProxy(t *testing.T) {
|
||||
tr := ssrfTransport()
|
||||
if tr.Proxy != nil {
|
||||
t.Fatal("feed SSRF transport must not use ProxyFromEnvironment")
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateFeedURLRejectsPrivateAndFTP(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
if err := ValidateFeedURL(ctx, ""); err != nil {
|
||||
t.Fatalf("empty url should be ok: %v", err)
|
||||
}
|
||||
if err := ValidateFeedURL(ctx, "ftp://example.com/a.csv"); err == nil {
|
||||
t.Fatal("expected ftp error")
|
||||
}
|
||||
if err := ValidateFeedURL(ctx, "http://127.0.0.1/x"); err == nil {
|
||||
t.Fatal("expected private IP error")
|
||||
}
|
||||
if err := ValidateFeedURL(ctx, "http://169.254.169.254/latest"); err == nil {
|
||||
t.Fatal("expected metadata IP error")
|
||||
}
|
||||
if err := ValidateFeedURL(ctx, "https://8.8.8.8/feed.xml"); err != nil {
|
||||
t.Fatalf("public IP https should be ok: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDownloadPublicOKWithSizeCap(t *testing.T) {
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "text/csv")
|
||||
_, _ = w.Write([]byte("EAN,Title\n1,A\n"))
|
||||
}))
|
||||
t.Cleanup(srv.Close)
|
||||
|
||||
// httptest uses 127.0.0.1 — should be blocked by SSRF guard.
|
||||
_, err := downloadFeed(context.Background(), srv.URL)
|
||||
if err == nil || !strings.Contains(err.Error(), "private") && err != errURLPrivate {
|
||||
// allow either wrapped or direct
|
||||
if err == nil {
|
||||
t.Fatal("expected loopback blocked")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestDownloadAllowlistPrivateHost(t *testing.T) {
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "text/csv")
|
||||
_, _ = w.Write([]byte("EAN,Title\n1,A\n"))
|
||||
}))
|
||||
t.Cleanup(srv.Close)
|
||||
|
||||
u, err := url.Parse(srv.URL)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := ConfigurePrivateAllowlist([]string{u.Hostname()}, []string{"127.0.0.0/8"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
_ = ConfigurePrivateAllowlist(nil, nil)
|
||||
})
|
||||
|
||||
blob, err := downloadFeed(context.Background(), srv.URL)
|
||||
if err != nil {
|
||||
t.Fatalf("allowlisted download: %v", err)
|
||||
}
|
||||
t.Cleanup(func() { _ = blob.Close() })
|
||||
data, err := os.ReadFile(blob.path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !strings.Contains(string(data), "EAN") {
|
||||
t.Fatalf("body=%q ct=%s", data, blob.contentType)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDownloadStreamsToTempFile(t *testing.T) {
|
||||
payload := "EAN,Title\n1,A\n2,B\n"
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "text/csv")
|
||||
_, _ = w.Write([]byte(payload))
|
||||
}))
|
||||
t.Cleanup(srv.Close)
|
||||
|
||||
u, err := url.Parse(srv.URL)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := ConfigurePrivateAllowlist([]string{u.Hostname()}, []string{"127.0.0.0/8"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { _ = ConfigurePrivateAllowlist(nil, nil) })
|
||||
|
||||
blob, err := downloadFeed(context.Background(), srv.URL)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !blob.owned || blob.path == "" {
|
||||
t.Fatalf("expected owned temp path, got %+v", blob)
|
||||
}
|
||||
if _, err := os.Stat(blob.path); err != nil {
|
||||
t.Fatalf("temp missing: %v", err)
|
||||
}
|
||||
|
||||
hash, err := sha256HexFile(blob)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
wantHash := sha256Hex([]byte(payload))
|
||||
if hash != wantHash {
|
||||
t.Fatalf("hash=%s want=%s", hash, wantHash)
|
||||
}
|
||||
|
||||
f, err := blob.Open()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var rows int
|
||||
n, err := parseCSV(f, func(feedRow) error {
|
||||
rows++
|
||||
return nil
|
||||
})
|
||||
_ = f.Close()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if n != 2 || rows != 2 {
|
||||
t.Fatalf("n=%d rows=%d", n, rows)
|
||||
}
|
||||
|
||||
path := blob.path
|
||||
if err := blob.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := os.Stat(path); !os.IsNotExist(err) {
|
||||
t.Fatalf("temp should be removed after Close, err=%v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDetectFeedFormat(t *testing.T) {
|
||||
if detectFeedFormat("csv", "", "", nil) != "csv" {
|
||||
t.Fatal("csv")
|
||||
}
|
||||
if detectFeedFormat("", "application/xml", "", []byte("<a/>")) != "xml" {
|
||||
t.Fatal("xml")
|
||||
}
|
||||
}
|
||||
|
||||
func TestClassifyUpsertOpsInsertTouchUpdate(t *testing.T) {
|
||||
idTouch := uuid.MustParse("11111111-1111-1111-1111-111111111111")
|
||||
idUpdate := uuid.MustParse("22222222-2222-2222-2222-222222222222")
|
||||
existing := map[string]existingProduct{
|
||||
"touch": {ID: idTouch, MappedData: []byte(`{"title":"Same"}`)},
|
||||
"update": {ID: idUpdate, MappedData: []byte(`{"title":"Old"}`)},
|
||||
}
|
||||
chunk := []pendingProduct{
|
||||
{GTIN: "new", RawData: map[string]string{"EAN": "new"}, MappedData: map[string]any{"title": "N"}},
|
||||
{GTIN: "touch", RawData: map[string]string{"EAN": "touch"}, MappedData: map[string]any{"title": "Same"}},
|
||||
{GTIN: "update", RawData: map[string]string{"EAN": "update"}, MappedData: map[string]any{"title": "New"}},
|
||||
}
|
||||
ops, skipped := classifyUpsertOps(chunk, existing)
|
||||
if skipped != 0 {
|
||||
t.Fatalf("skipped=%d", skipped)
|
||||
}
|
||||
if len(ops) != 3 {
|
||||
t.Fatalf("ops=%d", len(ops))
|
||||
}
|
||||
if ops[0].kind != upsertOpInsert || ops[0].gtin != "new" {
|
||||
t.Fatalf("op0=%+v", ops[0])
|
||||
}
|
||||
if ops[1].kind != upsertOpTouch || ops[1].id != idTouch {
|
||||
t.Fatalf("op1=%+v", ops[1])
|
||||
}
|
||||
if ops[2].kind != upsertOpUpdate || ops[2].id != idUpdate {
|
||||
t.Fatalf("op2=%+v", ops[2])
|
||||
}
|
||||
|
||||
inserts, touches, updates := partitionUpsertOps(ops)
|
||||
if len(inserts) != 1 || len(touches) != 1 || len(updates) != 1 {
|
||||
t.Fatalf("partition inserts=%d touches=%d updates=%d", len(inserts), len(touches), len(updates))
|
||||
}
|
||||
if inserts[0].gtin != "new" || touches[0].id != idTouch || updates[0].id != idUpdate {
|
||||
t.Fatalf("partition payloads insert=%+v touch=%+v update=%+v", inserts[0], touches[0], updates[0])
|
||||
}
|
||||
}
|
||||
|
||||
func TestDedupePendingByGTINLastWins(t *testing.T) {
|
||||
chunk := []pendingProduct{
|
||||
{GTIN: "a", MappedData: map[string]any{"title": "first"}},
|
||||
{GTIN: "b", MappedData: map[string]any{"title": "only"}},
|
||||
{GTIN: "a", MappedData: map[string]any{"title": "last"}},
|
||||
}
|
||||
got := dedupePendingByGTIN(chunk)
|
||||
if len(got) != 2 {
|
||||
t.Fatalf("len=%d", len(got))
|
||||
}
|
||||
if got[0].GTIN != "a" || got[0].MappedData["title"] != "last" {
|
||||
t.Fatalf("got[0]=%+v", got[0])
|
||||
}
|
||||
if got[1].GTIN != "b" {
|
||||
t.Fatalf("got[1]=%+v", got[1])
|
||||
}
|
||||
if dedupePendingByGTIN(nil) != nil {
|
||||
t.Fatal("nil in")
|
||||
}
|
||||
single := []pendingProduct{{GTIN: "x"}}
|
||||
if out := dedupePendingByGTIN(single); len(out) != 1 || out[0].GTIN != "x" {
|
||||
t.Fatalf("single=%+v", out)
|
||||
}
|
||||
}
|
||||
|
||||
func TestUpsertSQLIsSetBased(t *testing.T) {
|
||||
for name, sql := range map[string]string{
|
||||
"insert": upsertSQLInsertSet,
|
||||
"touch": upsertSQLTouchSet,
|
||||
"update": upsertSQLUpdateSet,
|
||||
} {
|
||||
lower := strings.ToLower(sql)
|
||||
switch name {
|
||||
case "insert", "update":
|
||||
if !strings.Contains(lower, "unnest(") {
|
||||
t.Fatalf("%s missing unnest: %s", name, sql)
|
||||
}
|
||||
case "touch":
|
||||
if !strings.Contains(lower, "any(") {
|
||||
t.Fatalf("touch missing ANY: %s", sql)
|
||||
}
|
||||
}
|
||||
if strings.Contains(lower, "values ($1") {
|
||||
t.Fatalf("%s still per-row VALUES form", name)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseCSVRespectsMaxRows(t *testing.T) {
|
||||
old := maxParseRows
|
||||
maxParseRows = 2
|
||||
t.Cleanup(func() { maxParseRows = old })
|
||||
|
||||
body := "EAN\n1\n2\n3\n"
|
||||
n, err := parseCSV(strings.NewReader(body), func(feedRow) error { return nil })
|
||||
if err == nil || !errors.Is(err, errParseTooManyRows) {
|
||||
t.Fatalf("n=%d err=%v want errParseTooManyRows", n, err)
|
||||
}
|
||||
if !strings.Contains(err.Error(), "(2)") {
|
||||
t.Fatalf("err=%v want limit in message", err)
|
||||
}
|
||||
if msg, ok := ClientError(err); !ok || !strings.Contains(msg, "max row limit") {
|
||||
t.Fatalf("ClientError=%q ok=%v", msg, ok)
|
||||
}
|
||||
if n != 3 {
|
||||
t.Fatalf("count=%d want 3 (exceeded after 3rd)", n)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseXMLRespectsMaxRows(t *testing.T) {
|
||||
old := maxParseRows
|
||||
maxParseRows = 1
|
||||
t.Cleanup(func() { maxParseRows = old })
|
||||
|
||||
body := `<r><item><g>1</g></item><item><g>2</g></item></r>`
|
||||
n, err := parseXMLItems(strings.NewReader(body), "item", func(feedRow) error { return nil })
|
||||
if err == nil || !errors.Is(err, errParseTooManyRows) {
|
||||
t.Fatalf("n=%d err=%v want errParseTooManyRows", n, err)
|
||||
}
|
||||
if n != 2 {
|
||||
t.Fatalf("count=%d want 2", n)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDownloadRejectsOversizedBody(t *testing.T) {
|
||||
old := maxDownloadBytes
|
||||
maxDownloadBytes = 64
|
||||
t.Cleanup(func() { maxDownloadBytes = old })
|
||||
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "text/csv")
|
||||
_, _ = w.Write([]byte(strings.Repeat("x", int(maxDownloadBytes)+2)))
|
||||
}))
|
||||
t.Cleanup(srv.Close)
|
||||
|
||||
u, err := url.Parse(srv.URL)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := ConfigurePrivateAllowlist([]string{u.Hostname()}, []string{"127.0.0.0/8"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { _ = ConfigurePrivateAllowlist(nil, nil) })
|
||||
|
||||
_, err = downloadFeed(context.Background(), srv.URL)
|
||||
if !errors.Is(err, errDownloadTooLarge) {
|
||||
t.Fatalf("err=%v want errDownloadTooLarge", err)
|
||||
}
|
||||
// Tiny test caps report bytes; production (≥1 MiB) reports MiB.
|
||||
if !strings.Contains(err.Error(), "max 64 bytes") {
|
||||
t.Fatalf("err=%v want max bytes in message", err)
|
||||
}
|
||||
if msg, ok := ClientError(err); !ok || !strings.Contains(msg, "size limit") {
|
||||
t.Fatalf("ClientError=%q ok=%v", msg, ok)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDefaultFeedCaps(t *testing.T) {
|
||||
const wantDownloadBytes int64 = 256 << 20 // 256 MiB — within 200–500 MiB band
|
||||
const wantParseRows = 1_000_000
|
||||
if defaultMaxDownloadBytes != wantDownloadBytes {
|
||||
t.Fatalf("defaultMaxDownloadBytes=%d want %d", defaultMaxDownloadBytes, wantDownloadBytes)
|
||||
}
|
||||
if defaultMaxParseRows != wantParseRows {
|
||||
t.Fatalf("defaultMaxParseRows=%d want %d", defaultMaxParseRows, wantParseRows)
|
||||
}
|
||||
if defaultMaxDownloadBytes < 200<<20 || defaultMaxDownloadBytes > 500<<20 {
|
||||
t.Fatalf("defaultMaxDownloadBytes=%d outside 200–500 MiB guidance", defaultMaxDownloadBytes)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,112 @@
|
||||
package feeds
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"github.com/google/uuid"
|
||||
"github.com/jackc/pgx/v5"
|
||||
)
|
||||
|
||||
// requiredStandardField is an enabled, required company standard field used for sync preflight.
|
||||
type requiredStandardField struct {
|
||||
Key string
|
||||
Name string
|
||||
}
|
||||
|
||||
// activeMappings returns rows with a non-empty source and a real target (not "none").
|
||||
// Mirrors the frontend activeMappingRows filter.
|
||||
func activeMappings(mappings []FieldMapping) []FieldMapping {
|
||||
out := make([]FieldMapping, 0, len(mappings))
|
||||
for _, m := range mappings {
|
||||
src := m.sourceKey()
|
||||
tgt := m.targetKey()
|
||||
if src == "" || tgt == "" || strings.EqualFold(tgt, "none") {
|
||||
continue
|
||||
}
|
||||
out = append(out, m)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// validateMappingsForSync returns a ClientMsg when mappings are empty or required targets are missing.
|
||||
// Optional (non-required) standard fields may remain unmapped.
|
||||
func validateMappingsForSync(mappings []FieldMapping, required []requiredStandardField) error {
|
||||
active := activeMappings(mappings)
|
||||
if len(active) == 0 {
|
||||
return ClientMsg("Map at least one source field before syncing.")
|
||||
}
|
||||
if len(required) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
mapped := make(map[string]struct{}, len(active))
|
||||
for _, m := range active {
|
||||
mapped[m.targetKey()] = struct{}{}
|
||||
}
|
||||
|
||||
missing := make([]string, 0)
|
||||
for _, r := range required {
|
||||
if _, ok := mapped[r.Key]; ok {
|
||||
continue
|
||||
}
|
||||
label := strings.TrimSpace(r.Name)
|
||||
if label == "" {
|
||||
label = r.Key
|
||||
}
|
||||
missing = append(missing, label)
|
||||
}
|
||||
if len(missing) == 0 {
|
||||
return nil
|
||||
}
|
||||
return ClientMsg(fmt.Sprintf("Map required fields before syncing: %s.", strings.Join(missing, ", ")))
|
||||
}
|
||||
|
||||
// loadRequiredStandardFields returns enabled+required standard field keys for the company.
|
||||
func (s *Service) loadRequiredStandardFields(ctx context.Context, companyID uuid.UUID) ([]requiredStandardField, error) {
|
||||
if s == nil || s.Pool == nil {
|
||||
return nil, nil
|
||||
}
|
||||
rows, err := s.Pool.Query(ctx, `
|
||||
SELECT key, COALESCE(NULLIF(TRIM(name), ''), key) AS name
|
||||
FROM standard_fields
|
||||
WHERE company_id = $1 AND enabled = true AND is_required = true
|
||||
ORDER BY sort_order ASC, key ASC`, companyID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
out := make([]requiredStandardField, 0)
|
||||
for rows.Next() {
|
||||
var key, name string
|
||||
if err := rows.Scan(&key, &name); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
key = strings.TrimSpace(key)
|
||||
if key == "" {
|
||||
continue
|
||||
}
|
||||
out = append(out, requiredStandardField{Key: key, Name: strings.TrimSpace(name)})
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
// ensureMappingsReadyForSync loads active mappings and required standard fields, then validates.
|
||||
// Called before creating a sync job so clients get a clear 400 without a failed job row.
|
||||
func (s *Service) ensureMappingsReadyForSync(ctx context.Context, companyID, feedID uuid.UUID) error {
|
||||
raw, err := s.loadMappingsRaw(ctx, companyID, feedID)
|
||||
if err != nil {
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return ClientMsg("Map at least one source field before syncing.")
|
||||
}
|
||||
return err
|
||||
}
|
||||
required, err := s.loadRequiredStandardFields(ctx, companyID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return validateMappingsForSync(parseMappings(raw), required)
|
||||
}
|
||||
@@ -0,0 +1,130 @@
|
||||
package feeds
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestValidateMappingsForSyncEmpty(t *testing.T) {
|
||||
err := validateMappingsForSync(nil, nil)
|
||||
if err == nil {
|
||||
t.Fatal("expected error for empty mappings")
|
||||
}
|
||||
msg, ok := ClientError(err)
|
||||
if !ok || !strings.Contains(msg, "Map at least one source field") {
|
||||
t.Fatalf("got %v", err)
|
||||
}
|
||||
|
||||
err = validateMappingsForSync([]FieldMapping{
|
||||
{Source: "col", Target: "none"},
|
||||
{Source: "", Target: "gtin"},
|
||||
}, nil)
|
||||
if err == nil {
|
||||
t.Fatal("expected error when only inactive rows present")
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateMappingsForSyncOptionalUnmappedOK(t *testing.T) {
|
||||
err := validateMappingsForSync([]FieldMapping{
|
||||
{Source: "ean", Target: "gtin"},
|
||||
{Source: "name", Target: "title"},
|
||||
}, []requiredStandardField{
|
||||
{Key: "gtin", Name: "GTIN/EAN"},
|
||||
{Key: "title", Name: "Product name"},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("optional gaps should be allowed: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateMappingsForSyncMissingRequired(t *testing.T) {
|
||||
err := validateMappingsForSync([]FieldMapping{
|
||||
{Source: "ean", Target: "gtin"},
|
||||
}, []requiredStandardField{
|
||||
{Key: "gtin", Name: "GTIN/EAN"},
|
||||
{Key: "title", Name: "Product name"},
|
||||
{Key: "brand", Name: "Brand"},
|
||||
})
|
||||
if err == nil {
|
||||
t.Fatal("expected missing required error")
|
||||
}
|
||||
msg, ok := ClientError(err)
|
||||
if !ok {
|
||||
t.Fatalf("expected ClientMsg, got %v", err)
|
||||
}
|
||||
if !strings.Contains(msg, "Map required fields before syncing") {
|
||||
t.Fatalf("message=%q", msg)
|
||||
}
|
||||
if !strings.Contains(msg, "Product name") || !strings.Contains(msg, "Brand") {
|
||||
t.Fatalf("expected missing labels in %q", msg)
|
||||
}
|
||||
if strings.Contains(msg, "GTIN") {
|
||||
t.Fatalf("mapped required field should not appear: %q", msg)
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateMappingsForSyncNoRequiredConfigured(t *testing.T) {
|
||||
err := validateMappingsForSync([]FieldMapping{
|
||||
{Source: "sku", Target: "sku"},
|
||||
}, nil)
|
||||
if err != nil {
|
||||
t.Fatalf("empty required list should pass when mappings exist: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMappingDocIncomplete(t *testing.T) {
|
||||
required := []requiredStandardField{{Key: "gtin", Name: "GTIN"}}
|
||||
xmlItem := map[string]any{"feed_type": "xml", "options": map[string]any{}}
|
||||
csvItem := map[string]any{"feed_type": "csv"}
|
||||
|
||||
empty := []FieldMapping{}
|
||||
if !mappingDocIncomplete(xmlItem, map[string]any{"fields": empty}, empty, required) {
|
||||
t.Fatal("empty mappings should be incomplete")
|
||||
}
|
||||
|
||||
withGTIN := []FieldMapping{{Source: "id", Target: "gtin"}}
|
||||
rawNoPath := map[string]any{"fields": []any{map[string]any{"source": "id", "target": "gtin"}}}
|
||||
if !mappingDocIncomplete(xmlItem, rawNoPath, withGTIN, required) {
|
||||
t.Fatal("xml without item_path should be incomplete")
|
||||
}
|
||||
if mappingDocIncomplete(csvItem, rawNoPath, withGTIN, required) {
|
||||
t.Fatal("csv without item_path should be complete when required mapped")
|
||||
}
|
||||
|
||||
rawWithPath := map[string]any{
|
||||
"item_path": "rss/channel/item",
|
||||
"fields": []any{map[string]any{"source": "id", "target": "gtin"}},
|
||||
}
|
||||
if mappingDocIncomplete(xmlItem, rawWithPath, withGTIN, required) {
|
||||
t.Fatal("xml with item_path + required should be complete")
|
||||
}
|
||||
}
|
||||
|
||||
func TestActiveMappingsFiltersNone(t *testing.T) {
|
||||
got := activeMappings([]FieldMapping{
|
||||
{Source: "a", Target: "gtin"},
|
||||
{Source: "b", Target: "none"},
|
||||
{Source: "c", FieldName: "None"},
|
||||
{Column: "d", Field: "title"},
|
||||
})
|
||||
if len(got) != 2 {
|
||||
t.Fatalf("got %d want 2: %#v", len(got), got)
|
||||
}
|
||||
if got[0].targetKey() != "gtin" || got[1].targetKey() != "title" {
|
||||
t.Fatalf("unexpected: %#v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestApplyMappingsSkipsNoneTarget(t *testing.T) {
|
||||
row := map[string]string{"col": "value", "ean": "123"}
|
||||
mapped, gtin := applyMappings(row, []FieldMapping{
|
||||
{Source: "col", Target: "none"},
|
||||
{Source: "ean", Target: "gtin"},
|
||||
})
|
||||
if _, ok := mapped["none"]; ok {
|
||||
t.Fatalf("none target must not be applied: %#v", mapped)
|
||||
}
|
||||
if gtin != "123" || mapped["gtin"] != "123" {
|
||||
t.Fatalf("expected gtin mapping, got mapped=%#v gtin=%q", mapped, gtin)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user