package feeds import ( "context" "errors" "fmt" "io" "net" "net/http" "net/url" "os" "strings" "sync" "time" ) const ( // defaultMaxDownloadBytes caps HTTP downloads and local upload sources. // Bodies stream to a temp file (not RAM), so hundreds of MiB are safe for // large merchant catalogs. Keep within ~200–500 MiB; raise carefully if disk // and downloadTimeout remain adequate. Exceeding returns downloadTooLarge(). defaultMaxDownloadBytes int64 = 256 << 20 // 256 MiB downloadTimeout = 60 * time.Second dialTimeout = 10 * time.Second maxRedirects = 5 defaultUserAgent = "DescrybeFeedSync/2.0" ) // maxDownloadBytes bounds feed downloads (temp file / local source size). Mutable for tests. var maxDownloadBytes = defaultMaxDownloadBytes var ( errURLRequired = errors.New("feed url required") errURLScheme = errors.New("url scheme must be http or https") errURLPrivate = errors.New("url resolves to a private or blocked address") errURLFTP = errors.New("ftp/ftps feed sync is not supported yet") errDownloadTooLarge = errors.New("feed download exceeds size limit") ) // downloadTooLarge returns errDownloadTooLarge with the active size limit for clients. func downloadTooLarge() error { if maxDownloadBytes >= 1<<20 { return fmt.Errorf("%w (max %d MiB)", errDownloadTooLarge, maxDownloadBytes>>20) } return fmt.Errorf("%w (max %d bytes)", errDownloadTooLarge, maxDownloadBytes) } var ( allowMu sync.RWMutex allowConfigured bool allowHosts map[string]struct{} allowCIDRs []*net.IPNet ) // ConfigurePrivateAllowlist sets hostnames and CIDRs that may bypass the private-IP SSRF block. // Intended for tests and optional startup wiring; production normally uses FEED_URL_PRIVATE_ALLOWLIST // and/or admin platform setting feeds.private_url_allowlist. func ConfigurePrivateAllowlist(hosts []string, cidrs []string) error { h := make(map[string]struct{}, len(hosts)) for _, raw := range hosts { raw = strings.ToLower(strings.TrimSpace(raw)) if raw != "" { h[raw] = struct{}{} } } nets := make([]*net.IPNet, 0, len(cidrs)) for _, raw := range cidrs { raw = strings.TrimSpace(raw) if raw == "" { continue } _, n, err := net.ParseCIDR(raw) if err != nil { return fmt.Errorf("invalid allowlist cidr %q: %w", raw, err) } nets = append(nets, n) } allowMu.Lock() allowHosts = h allowCIDRs = nets allowConfigured = true allowMu.Unlock() return nil } // ApplyPrivateAllowlistCSV merges env FEED_URL_PRIVATE_ALLOWLIST with an optional // admin/settings CSV (settings override by appending unique entries). func ApplyPrivateAllowlistCSV(settingsCSV string) { parts := make([]string, 0, 8) for _, src := range []string{os.Getenv("FEED_URL_PRIVATE_ALLOWLIST"), settingsCSV} { for _, part := range strings.Split(src, ",") { part = strings.TrimSpace(part) if part != "" { parts = append(parts, part) } } } hosts := make([]string, 0) cidrs := make([]string, 0) seen := map[string]struct{}{} for _, part := range parts { key := strings.ToLower(part) if _, ok := seen[key]; ok { continue } seen[key] = struct{}{} if strings.Contains(part, "/") { cidrs = append(cidrs, part) } else { hosts = append(hosts, part) } } _ = ConfigurePrivateAllowlist(hosts, cidrs) } func ensureAllowlist() { allowMu.RLock() done := allowConfigured allowMu.RUnlock() if done { return } allowMu.Lock() defer allowMu.Unlock() if allowConfigured { return } allowHosts = map[string]struct{}{} allowCIDRs = nil raw := strings.TrimSpace(os.Getenv("FEED_URL_PRIVATE_ALLOWLIST")) for _, part := range strings.Split(raw, ",") { part = strings.TrimSpace(part) if part == "" { continue } if strings.Contains(part, "/") { if _, n, err := net.ParseCIDR(part); err == nil { allowCIDRs = append(allowCIDRs, n) } continue } allowHosts[strings.ToLower(part)] = struct{}{} } allowConfigured = true } // ValidateFeedURL checks a feed source URL for SSRF before persist. // Empty URL is allowed (local upload sources). Does not fetch the URL. func ValidateFeedURL(ctx context.Context, rawURL string) error { ensureAllowlist() rawURL = strings.TrimSpace(rawURL) if rawURL == "" { return nil } if len(rawURL) > 2048 { return errURLScheme } lower := strings.ToLower(rawURL) if strings.HasPrefix(lower, "ftp://") || strings.HasPrefix(lower, "ftps://") { return errURLFTP } u, err := url.Parse(rawURL) if err != nil || u.Host == "" { return ClientMsg("invalid url") } if u.Scheme != "http" && u.Scheme != "https" { return errURLScheme } return assertPublicHost(ctx, u.Hostname()) } // downloadFeed streams an http(s) feed to a temp file with SSRF controls and a hard size cap. // Callers must Close the returned blob to remove the temp file. func downloadFeed(ctx context.Context, rawURL string) (*feedBlob, error) { ensureAllowlist() rawURL = strings.TrimSpace(rawURL) if rawURL == "" { return nil, errURLRequired } lower := strings.ToLower(rawURL) if strings.HasPrefix(lower, "ftp://") || strings.HasPrefix(lower, "ftps://") { return nil, errURLFTP } u, err := url.Parse(rawURL) if err != nil || u.Host == "" { return nil, ClientMsg("invalid url") } if u.Scheme != "http" && u.Scheme != "https" { return nil, errURLScheme } if err := assertPublicHost(ctx, u.Hostname()); err != nil { return nil, err } client := &http.Client{ Timeout: downloadTimeout, Transport: ssrfTransport(), CheckRedirect: func(req *http.Request, via []*http.Request) error { if len(via) >= maxRedirects { return ClientMsg("too many redirects") } if req.URL.Scheme != "http" && req.URL.Scheme != "https" { return errURLScheme } return assertPublicHost(req.Context(), req.URL.Hostname()) }, } req, err := http.NewRequestWithContext(ctx, http.MethodGet, u.String(), nil) if err != nil { return nil, err } req.Header.Set("User-Agent", defaultUserAgent) req.Header.Set("Accept", "text/csv,application/csv,application/xml,text/xml,application/atom+xml,*/*") res, err := client.Do(req) if err != nil { return nil, fmt.Errorf("download failed: %w", err) } defer res.Body.Close() if res.StatusCode < 200 || res.StatusCode >= 300 { return nil, fmt.Errorf("download status %d", res.StatusCode) } if res.ContentLength > maxDownloadBytes { return nil, downloadTooLarge() } tmp, err := os.CreateTemp("", "descrybe-feed-*") if err != nil { return nil, fmt.Errorf("temp file: %w", err) } tmpPath := tmp.Name() cleanup := true defer func() { _ = tmp.Close() if cleanup { _ = os.Remove(tmpPath) } }() limited := io.LimitReader(res.Body, maxDownloadBytes+1) written, err := io.Copy(tmp, limited) if err != nil { return nil, fmt.Errorf("read body: %w", err) } if written > maxDownloadBytes { return nil, downloadTooLarge() } if err := tmp.Close(); err != nil { return nil, fmt.Errorf("close temp: %w", err) } cleanup = false return &feedBlob{ path: tmpPath, contentType: res.Header.Get("Content-Type"), size: written, owned: true, }, nil } func ssrfTransport() *http.Transport { dialer := &net.Dialer{Timeout: dialTimeout, KeepAlive: 30 * time.Second} return &http.Transport{ // Never honor HTTP(S)_PROXY: dialing a proxy skips destination SSRF checks. Proxy: nil, DialContext: func(ctx context.Context, network, addr string) (net.Conn, error) { host, port, err := net.SplitHostPort(addr) if err != nil { return nil, err } if err := assertPublicHost(ctx, host); err != nil { return nil, err } ips, err := resolveHostIPs(ctx, host) if err != nil { return nil, err } var lastErr error for _, ip := range ips { if isBlockedIP(ip) && !isAllowlistedHostOrIP(host, ip) { lastErr = errURLPrivate continue } target := net.JoinHostPort(ip.String(), port) conn, err := dialer.DialContext(ctx, network, target) if err == nil { return conn, nil } lastErr = err } if lastErr == nil { lastErr = errURLPrivate } return nil, lastErr }, ForceAttemptHTTP2: true, MaxIdleConns: 10, IdleConnTimeout: 90 * time.Second, TLSHandshakeTimeout: 10 * time.Second, ExpectContinueTimeout: 1 * time.Second, ResponseHeaderTimeout: 30 * time.Second, } } func resolveHostIPs(ctx context.Context, host string) ([]net.IP, error) { if ip := net.ParseIP(host); ip != nil { return []net.IP{ip}, nil } addrs, err := net.DefaultResolver.LookupIPAddr(ctx, host) if err != nil { return nil, fmt.Errorf("dns lookup: %w", err) } out := make([]net.IP, 0, len(addrs)) for _, a := range addrs { out = append(out, a.IP) } return out, nil } func assertPublicHost(ctx context.Context, host string) error { ensureAllowlist() host = strings.TrimSpace(host) if host == "" { return errURLPrivate } lower := strings.ToLower(host) allowMu.RLock() _, hostAllowed := allowHosts[lower] allowMu.RUnlock() if hostAllowed { return nil } if lower == "localhost" || strings.HasSuffix(lower, ".localhost") || strings.HasSuffix(lower, ".local") { return errURLPrivate } if ip := net.ParseIP(host); ip != nil { if isBlockedIP(ip) && !isAllowlistedIP(ip) { return errURLPrivate } return nil } addrs, err := resolveHostIPs(ctx, host) if err != nil { return err } if len(addrs) == 0 { return errURLPrivate } for _, ip := range addrs { if isBlockedIP(ip) && !isAllowlistedIP(ip) { return errURLPrivate } } return nil } func isAllowlistedHostOrIP(host string, ip net.IP) bool { ensureAllowlist() allowMu.RLock() _, ok := allowHosts[strings.ToLower(strings.TrimSpace(host))] allowMu.RUnlock() if ok { return true } return isAllowlistedIP(ip) } func isAllowlistedIP(ip net.IP) bool { ensureAllowlist() if ip == nil { return false } allowMu.RLock() defer allowMu.RUnlock() for _, n := range allowCIDRs { if n.Contains(ip) { return true } } return false } func isBlockedIP(ip net.IP) bool { if ip == nil { return true } if ip.IsLoopback() || ip.IsPrivate() || ip.IsLinkLocalUnicast() || ip.IsLinkLocalMulticast() || ip.IsMulticast() || ip.IsUnspecified() { return true } // AWS/GCP/Azure metadata and CGNAT. if ip4 := ip.To4(); ip4 != nil { if ip4[0] == 169 && ip4[1] == 254 { return true } if ip4[0] == 100 && ip4[1] >= 64 && ip4[1] <= 127 { return true } } return false }