package feeds import ( "bytes" "encoding/csv" "encoding/xml" "errors" "fmt" "io" "strings" ) // defaultMaxParseRows caps CSV/XML product rows per sync. Parse is streaming // (one row at a time); the bound limits sync duration and DB write volume for // oversized catalogs. Exceeding returns parseTooManyRows() with the numeric limit. const defaultMaxParseRows = 1_000_000 // maxParseRows is the active row cap. Mutable for tests. var maxParseRows = defaultMaxParseRows // errParseTooManyRows is the sentinel for ClientError / errors.Is checks. var errParseTooManyRows = errors.New("feed exceeds max row limit") // parseTooManyRows returns errParseTooManyRows with the active row limit for clients. func parseTooManyRows() error { return fmt.Errorf("%w (%d)", errParseTooManyRows, maxParseRows) } // feedRow is a normalized flat record from CSV or XML. type feedRow map[string]string func detectFeedFormat(feedType, contentType, urlHint string, sample []byte) string { ft := strings.ToLower(strings.TrimSpace(feedType)) if ft == "csv" || ft == "xml" { return ft } ct := strings.ToLower(contentType) u := strings.ToLower(urlHint) switch { case strings.Contains(ct, "csv") || strings.HasSuffix(u, ".csv"): return "csv" case strings.Contains(ct, "xml") || strings.HasSuffix(u, ".xml"): return "xml" } trimmed := bytes.TrimSpace(sample) if len(trimmed) > 0 && trimmed[0] == '<' { return "xml" } return "csv" } // parseCSV streams rows via callback to avoid holding the full matrix when possible. // The CSV reader still tokenizes; we only keep one row at a time in the callback path. func parseCSV(r io.Reader, onRow func(feedRow) error) (int, error) { cr := csv.NewReader(r) cr.ReuseRecord = true cr.LazyQuotes = true cr.TrimLeadingSpace = true cr.FieldsPerRecord = -1 header, err := cr.Read() if err != nil { return 0, fmt.Errorf("csv header: %w", err) } cols := make([]string, len(header)) for i, h := range header { cols[i] = strings.TrimSpace(h) } count := 0 for { rec, err := cr.Read() if err == io.EOF { break } if err != nil { return count, fmt.Errorf("csv row %d: %w", count+1, err) } count++ if count > maxParseRows { return count, parseTooManyRows() } row := make(feedRow, len(cols)) for i, col := range cols { if col == "" { continue } if i < len(rec) { row[col] = rec[i] } else { row[col] = "" } } expandSpecificationFields(row) if err := onRow(row); err != nil { return count, err } } return count, nil } // parseXMLItems streams element-local text maps for repeating item tags. // When itemLocal is empty and r is seekable, a small prefix is sniffed then rewound. func parseXMLItems(r io.Reader, itemLocal string, onRow func(feedRow) error) (int, error) { itemLocal = strings.TrimSpace(itemLocal) if itemLocal == "" { var err error itemLocal, r, err = resolveXMLItemLocal(r) if err != nil { return 0, err } } dec := xml.NewDecoder(r) dec.Strict = false count := 0 for { tok, err := dec.Token() if err == io.EOF { break } if err != nil { return count, fmt.Errorf("xml: %w", err) } se, ok := tok.(xml.StartElement) if !ok { continue } if !localNameEquals(se.Name, itemLocal) { continue } row, err := readXMLElementMap(dec, se) if err != nil { return count, err } expandSpecificationFields(row) count++ if count > maxParseRows { return count, parseTooManyRows() } if err := onRow(row); err != nil { return count, err } } return count, nil } const xmlItemGuessBytes = 64 << 10 func resolveXMLItemLocal(r io.Reader) (string, io.Reader, error) { if rs, ok := r.(io.ReadSeeker); ok { sample := make([]byte, xmlItemGuessBytes) n, err := io.ReadFull(rs, sample) if err != nil && err != io.EOF && err != io.ErrUnexpectedEOF { return "", nil, err } sample = sample[:n] local := guessXMLItemLocal(sample) if _, err := rs.Seek(0, io.SeekStart); err != nil { return "", nil, err } return local, rs, nil } sample, err := io.ReadAll(io.LimitReader(r, xmlItemGuessBytes)) if err != nil { return "", nil, err } local := guessXMLItemLocal(sample) return local, io.MultiReader(bytes.NewReader(sample), r), nil } func guessXMLItemLocal(data []byte) string { sample := string(data) if len(sample) > xmlItemGuessBytes { sample = sample[:xmlItemGuessBytes] } lower := strings.ToLower(sample) for _, cand := range []string{"item", "product", "entry", "offer", "row"} { if strings.Contains(lower, "<"+cand) || strings.Contains(lower, ":"+cand) { return cand } } return "item" } func localNameEquals(n xml.Name, local string) bool { return strings.EqualFold(n.Local, local) } func readXMLElementMap(dec *xml.Decoder, start xml.StartElement) (feedRow, error) { row := make(feedRow) for _, a := range start.Attr { key := "@" + a.Name.Local row[key] = a.Value row[start.Name.Local+"/"+key] = a.Value } var path []string for { tok, err := dec.Token() if err != nil { return nil, err } switch t := tok.(type) { case xml.StartElement: path = append(path, t.Name.Local) for _, a := range t.Attr { key := strings.Join(path, "/") + "/@" + a.Name.Local row["@"+a.Name.Local] = a.Value row[key] = a.Value } case xml.EndElement: if len(path) == 0 { return row, nil } path = path[:len(path)-1] case xml.CharData: text := strings.TrimSpace(string(t)) if text == "" || len(path) == 0 { continue } leaf := path[len(path)-1] full := strings.Join(path, "/") // Prefer nested path as source of truth; only set bare leaf when // unique (no other nested field already owns this leaf name). if prev, ok := row[full]; ok && prev != "" && prev != text { row[full] = prev + " " + text } else { row[full] = text } if prev, ok := row[leaf]; !ok || prev == "" || prev == text || prev == row[full] { row[leaf] = row[full] } } } }