greeneclipse
8580c996c3
Drop one-shot tmp/axe scripts and agent i18n scratch so the Gitea tree is deployable.
391 lines
11 KiB
Go
391 lines
11 KiB
Go
package feeds
|
||
|
||
import (
|
||
"context"
|
||
"errors"
|
||
"net/http"
|
||
"net/http/httptest"
|
||
"net/url"
|
||
"os"
|
||
"strings"
|
||
"testing"
|
||
|
||
"github.com/google/uuid"
|
||
)
|
||
|
||
func TestParseCSVAndMappings(t *testing.T) {
|
||
csv := "EAN,Title\n123,Widget\n456,\n"
|
||
mappings := parseMappings([]any{
|
||
map[string]any{"source": "EAN", "target": "gtin"},
|
||
map[string]any{"column": "Title", "fieldName": "title"},
|
||
})
|
||
var rows []map[string]any
|
||
n, err := parseCSV(strings.NewReader(csv), func(row feedRow) error {
|
||
mapped, gtin := applyMappings(row, mappings)
|
||
rows = append(rows, map[string]any{"gtin": gtin, "mapped": mapped})
|
||
return nil
|
||
})
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
if n != 2 {
|
||
t.Fatalf("rows=%d", n)
|
||
}
|
||
if rows[0]["gtin"] != "123" {
|
||
t.Fatalf("gtin=%v", rows[0]["gtin"])
|
||
}
|
||
m := rows[0]["mapped"].(map[string]any)
|
||
if m["title"] != "Widget" {
|
||
t.Fatalf("mapped=%v", m)
|
||
}
|
||
}
|
||
|
||
func TestParseXMLItems(t *testing.T) {
|
||
xmlBody := `<?xml version="1.0"?><products><item><gtin>999</gtin><title>X</title></item></products>`
|
||
mappings := parseMappings(map[string]any{
|
||
"gtin": map[string]any{"fieldName": "gtin"},
|
||
"title": map[string]any{"fieldName": "title"},
|
||
})
|
||
var got string
|
||
n, err := parseXMLItems(strings.NewReader(xmlBody), "item", func(row feedRow) error {
|
||
_, gtin := applyMappings(row, mappings)
|
||
got = gtin
|
||
return nil
|
||
})
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
if n != 1 || got != "999" {
|
||
t.Fatalf("n=%d gtin=%q", n, got)
|
||
}
|
||
}
|
||
|
||
func TestDownloadRejectsPrivateAndFTP(t *testing.T) {
|
||
ctx := context.Background()
|
||
if _, err := downloadFeed(ctx, "ftp://example.com/a.csv"); err == nil {
|
||
t.Fatal("expected ftp error")
|
||
}
|
||
if _, err := downloadFeed(ctx, "http://127.0.0.1/x"); err == nil {
|
||
t.Fatal("expected private IP error")
|
||
}
|
||
if _, err := downloadFeed(ctx, "http://localhost/x"); err == nil {
|
||
t.Fatal("expected localhost error")
|
||
}
|
||
}
|
||
|
||
func TestSSRFTransportDisablesEnvProxy(t *testing.T) {
|
||
tr := ssrfTransport()
|
||
if tr.Proxy != nil {
|
||
t.Fatal("feed SSRF transport must not use ProxyFromEnvironment")
|
||
}
|
||
}
|
||
|
||
func TestValidateFeedURLRejectsPrivateAndFTP(t *testing.T) {
|
||
ctx := context.Background()
|
||
if err := ValidateFeedURL(ctx, ""); err != nil {
|
||
t.Fatalf("empty url should be ok: %v", err)
|
||
}
|
||
if err := ValidateFeedURL(ctx, "ftp://example.com/a.csv"); err == nil {
|
||
t.Fatal("expected ftp error")
|
||
}
|
||
if err := ValidateFeedURL(ctx, "http://127.0.0.1/x"); err == nil {
|
||
t.Fatal("expected private IP error")
|
||
}
|
||
if err := ValidateFeedURL(ctx, "http://169.254.169.254/latest"); err == nil {
|
||
t.Fatal("expected metadata IP error")
|
||
}
|
||
if err := ValidateFeedURL(ctx, "https://8.8.8.8/feed.xml"); err != nil {
|
||
t.Fatalf("public IP https should be ok: %v", err)
|
||
}
|
||
}
|
||
|
||
func TestDownloadPublicOKWithSizeCap(t *testing.T) {
|
||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||
w.Header().Set("Content-Type", "text/csv")
|
||
_, _ = w.Write([]byte("EAN,Title\n1,A\n"))
|
||
}))
|
||
t.Cleanup(srv.Close)
|
||
|
||
// httptest uses 127.0.0.1 — should be blocked by SSRF guard.
|
||
_, err := downloadFeed(context.Background(), srv.URL)
|
||
if err == nil || !strings.Contains(err.Error(), "private") && err != errURLPrivate {
|
||
// allow either wrapped or direct
|
||
if err == nil {
|
||
t.Fatal("expected loopback blocked")
|
||
}
|
||
}
|
||
}
|
||
|
||
func TestDownloadAllowlistPrivateHost(t *testing.T) {
|
||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||
w.Header().Set("Content-Type", "text/csv")
|
||
_, _ = w.Write([]byte("EAN,Title\n1,A\n"))
|
||
}))
|
||
t.Cleanup(srv.Close)
|
||
|
||
u, err := url.Parse(srv.URL)
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
if err := ConfigurePrivateAllowlist([]string{u.Hostname()}, []string{"127.0.0.0/8"}); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
t.Cleanup(func() {
|
||
_ = ConfigurePrivateAllowlist(nil, nil)
|
||
})
|
||
|
||
blob, err := downloadFeed(context.Background(), srv.URL)
|
||
if err != nil {
|
||
t.Fatalf("allowlisted download: %v", err)
|
||
}
|
||
t.Cleanup(func() { _ = blob.Close() })
|
||
data, err := os.ReadFile(blob.path)
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
if !strings.Contains(string(data), "EAN") {
|
||
t.Fatalf("body=%q ct=%s", data, blob.contentType)
|
||
}
|
||
}
|
||
|
||
func TestDownloadStreamsToTempFile(t *testing.T) {
|
||
payload := "EAN,Title\n1,A\n2,B\n"
|
||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||
w.Header().Set("Content-Type", "text/csv")
|
||
_, _ = w.Write([]byte(payload))
|
||
}))
|
||
t.Cleanup(srv.Close)
|
||
|
||
u, err := url.Parse(srv.URL)
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
if err := ConfigurePrivateAllowlist([]string{u.Hostname()}, []string{"127.0.0.0/8"}); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
t.Cleanup(func() { _ = ConfigurePrivateAllowlist(nil, nil) })
|
||
|
||
blob, err := downloadFeed(context.Background(), srv.URL)
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
if !blob.owned || blob.path == "" {
|
||
t.Fatalf("expected owned temp path, got %+v", blob)
|
||
}
|
||
if _, err := os.Stat(blob.path); err != nil {
|
||
t.Fatalf("temp missing: %v", err)
|
||
}
|
||
|
||
hash, err := sha256HexFile(blob)
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
wantHash := sha256Hex([]byte(payload))
|
||
if hash != wantHash {
|
||
t.Fatalf("hash=%s want=%s", hash, wantHash)
|
||
}
|
||
|
||
f, err := blob.Open()
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
var rows int
|
||
n, err := parseCSV(f, func(feedRow) error {
|
||
rows++
|
||
return nil
|
||
})
|
||
_ = f.Close()
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
if n != 2 || rows != 2 {
|
||
t.Fatalf("n=%d rows=%d", n, rows)
|
||
}
|
||
|
||
path := blob.path
|
||
if err := blob.Close(); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
if _, err := os.Stat(path); !os.IsNotExist(err) {
|
||
t.Fatalf("temp should be removed after Close, err=%v", err)
|
||
}
|
||
}
|
||
|
||
func TestDetectFeedFormat(t *testing.T) {
|
||
if detectFeedFormat("csv", "", "", nil) != "csv" {
|
||
t.Fatal("csv")
|
||
}
|
||
if detectFeedFormat("", "application/xml", "", []byte("<a/>")) != "xml" {
|
||
t.Fatal("xml")
|
||
}
|
||
}
|
||
|
||
func TestClassifyUpsertOpsInsertTouchUpdate(t *testing.T) {
|
||
idTouch := uuid.MustParse("11111111-1111-1111-1111-111111111111")
|
||
idUpdate := uuid.MustParse("22222222-2222-2222-2222-222222222222")
|
||
existing := map[string]existingProduct{
|
||
"touch": {ID: idTouch, MappedData: []byte(`{"title":"Same"}`)},
|
||
"update": {ID: idUpdate, MappedData: []byte(`{"title":"Old"}`)},
|
||
}
|
||
chunk := []pendingProduct{
|
||
{GTIN: "new", RawData: map[string]string{"EAN": "new"}, MappedData: map[string]any{"title": "N"}},
|
||
{GTIN: "touch", RawData: map[string]string{"EAN": "touch"}, MappedData: map[string]any{"title": "Same"}},
|
||
{GTIN: "update", RawData: map[string]string{"EAN": "update"}, MappedData: map[string]any{"title": "New"}},
|
||
}
|
||
ops, skipped := classifyUpsertOps(chunk, existing)
|
||
if skipped != 0 {
|
||
t.Fatalf("skipped=%d", skipped)
|
||
}
|
||
if len(ops) != 3 {
|
||
t.Fatalf("ops=%d", len(ops))
|
||
}
|
||
if ops[0].kind != upsertOpInsert || ops[0].gtin != "new" {
|
||
t.Fatalf("op0=%+v", ops[0])
|
||
}
|
||
if ops[1].kind != upsertOpTouch || ops[1].id != idTouch {
|
||
t.Fatalf("op1=%+v", ops[1])
|
||
}
|
||
if ops[2].kind != upsertOpUpdate || ops[2].id != idUpdate {
|
||
t.Fatalf("op2=%+v", ops[2])
|
||
}
|
||
|
||
inserts, touches, updates := partitionUpsertOps(ops)
|
||
if len(inserts) != 1 || len(touches) != 1 || len(updates) != 1 {
|
||
t.Fatalf("partition inserts=%d touches=%d updates=%d", len(inserts), len(touches), len(updates))
|
||
}
|
||
if inserts[0].gtin != "new" || touches[0].id != idTouch || updates[0].id != idUpdate {
|
||
t.Fatalf("partition payloads insert=%+v touch=%+v update=%+v", inserts[0], touches[0], updates[0])
|
||
}
|
||
}
|
||
|
||
func TestDedupePendingByGTINLastWins(t *testing.T) {
|
||
chunk := []pendingProduct{
|
||
{GTIN: "a", MappedData: map[string]any{"title": "first"}},
|
||
{GTIN: "b", MappedData: map[string]any{"title": "only"}},
|
||
{GTIN: "a", MappedData: map[string]any{"title": "last"}},
|
||
}
|
||
got := dedupePendingByGTIN(chunk)
|
||
if len(got) != 2 {
|
||
t.Fatalf("len=%d", len(got))
|
||
}
|
||
if got[0].GTIN != "a" || got[0].MappedData["title"] != "last" {
|
||
t.Fatalf("got[0]=%+v", got[0])
|
||
}
|
||
if got[1].GTIN != "b" {
|
||
t.Fatalf("got[1]=%+v", got[1])
|
||
}
|
||
if dedupePendingByGTIN(nil) != nil {
|
||
t.Fatal("nil in")
|
||
}
|
||
single := []pendingProduct{{GTIN: "x"}}
|
||
if out := dedupePendingByGTIN(single); len(out) != 1 || out[0].GTIN != "x" {
|
||
t.Fatalf("single=%+v", out)
|
||
}
|
||
}
|
||
|
||
func TestUpsertSQLIsSetBased(t *testing.T) {
|
||
for name, sql := range map[string]string{
|
||
"insert": upsertSQLInsertSet,
|
||
"touch": upsertSQLTouchSet,
|
||
"update": upsertSQLUpdateSet,
|
||
} {
|
||
lower := strings.ToLower(sql)
|
||
switch name {
|
||
case "insert", "update":
|
||
if !strings.Contains(lower, "unnest(") {
|
||
t.Fatalf("%s missing unnest: %s", name, sql)
|
||
}
|
||
case "touch":
|
||
if !strings.Contains(lower, "any(") {
|
||
t.Fatalf("touch missing ANY: %s", sql)
|
||
}
|
||
}
|
||
if strings.Contains(lower, "values ($1") {
|
||
t.Fatalf("%s still per-row VALUES form", name)
|
||
}
|
||
}
|
||
}
|
||
|
||
func TestParseCSVRespectsMaxRows(t *testing.T) {
|
||
old := maxParseRows
|
||
maxParseRows = 2
|
||
t.Cleanup(func() { maxParseRows = old })
|
||
|
||
body := "EAN\n1\n2\n3\n"
|
||
n, err := parseCSV(strings.NewReader(body), func(feedRow) error { return nil })
|
||
if err == nil || !errors.Is(err, errParseTooManyRows) {
|
||
t.Fatalf("n=%d err=%v want errParseTooManyRows", n, err)
|
||
}
|
||
if !strings.Contains(err.Error(), "(2)") {
|
||
t.Fatalf("err=%v want limit in message", err)
|
||
}
|
||
if msg, ok := ClientError(err); !ok || !strings.Contains(msg, "max row limit") {
|
||
t.Fatalf("ClientError=%q ok=%v", msg, ok)
|
||
}
|
||
if n != 3 {
|
||
t.Fatalf("count=%d want 3 (exceeded after 3rd)", n)
|
||
}
|
||
}
|
||
|
||
func TestParseXMLRespectsMaxRows(t *testing.T) {
|
||
old := maxParseRows
|
||
maxParseRows = 1
|
||
t.Cleanup(func() { maxParseRows = old })
|
||
|
||
body := `<r><item><g>1</g></item><item><g>2</g></item></r>`
|
||
n, err := parseXMLItems(strings.NewReader(body), "item", func(feedRow) error { return nil })
|
||
if err == nil || !errors.Is(err, errParseTooManyRows) {
|
||
t.Fatalf("n=%d err=%v want errParseTooManyRows", n, err)
|
||
}
|
||
if n != 2 {
|
||
t.Fatalf("count=%d want 2", n)
|
||
}
|
||
}
|
||
|
||
func TestDownloadRejectsOversizedBody(t *testing.T) {
|
||
old := maxDownloadBytes
|
||
maxDownloadBytes = 64
|
||
t.Cleanup(func() { maxDownloadBytes = old })
|
||
|
||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||
w.Header().Set("Content-Type", "text/csv")
|
||
_, _ = w.Write([]byte(strings.Repeat("x", int(maxDownloadBytes)+2)))
|
||
}))
|
||
t.Cleanup(srv.Close)
|
||
|
||
u, err := url.Parse(srv.URL)
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
if err := ConfigurePrivateAllowlist([]string{u.Hostname()}, []string{"127.0.0.0/8"}); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
t.Cleanup(func() { _ = ConfigurePrivateAllowlist(nil, nil) })
|
||
|
||
_, err = downloadFeed(context.Background(), srv.URL)
|
||
if !errors.Is(err, errDownloadTooLarge) {
|
||
t.Fatalf("err=%v want errDownloadTooLarge", err)
|
||
}
|
||
// Tiny test caps report bytes; production (≥1 MiB) reports MiB.
|
||
if !strings.Contains(err.Error(), "max 64 bytes") {
|
||
t.Fatalf("err=%v want max bytes in message", err)
|
||
}
|
||
if msg, ok := ClientError(err); !ok || !strings.Contains(msg, "size limit") {
|
||
t.Fatalf("ClientError=%q ok=%v", msg, ok)
|
||
}
|
||
}
|
||
|
||
func TestDefaultFeedCaps(t *testing.T) {
|
||
const wantDownloadBytes int64 = 256 << 20 // 256 MiB — within 200–500 MiB band
|
||
const wantParseRows = 1_000_000
|
||
if defaultMaxDownloadBytes != wantDownloadBytes {
|
||
t.Fatalf("defaultMaxDownloadBytes=%d want %d", defaultMaxDownloadBytes, wantDownloadBytes)
|
||
}
|
||
if defaultMaxParseRows != wantParseRows {
|
||
t.Fatalf("defaultMaxParseRows=%d want %d", defaultMaxParseRows, wantParseRows)
|
||
}
|
||
if defaultMaxDownloadBytes < 200<<20 || defaultMaxDownloadBytes > 500<<20 {
|
||
t.Fatalf("defaultMaxDownloadBytes=%d outside 200–500 MiB guidance", defaultMaxDownloadBytes)
|
||
}
|
||
}
|