@@ -2,11 +2,14 @@ import { Readable } from 'node:stream'
22import { truncate } from '@sim/utils/string'
33import { parse as parseCsvStream } from 'csv-parse'
44import { CSV_PREVIEW_MAX_ROWS } from '@/lib/api/contracts/workspace-file-table'
5+ import { assertKnownSizeWithinLimit } from '@/lib/core/utils/stream-limits'
56import type { StorageContext } from '@/lib/uploads/config'
67import { downloadFileStream } from '@/lib/uploads/core/storage-service'
78
89/** Cap a single cell so one pathological field can't bloat the preview payload. */
910const MAX_CELL_LENGTH = 2_000
11+ const MAX_RECORD_SIZE = 256 * 1024
12+ const MAX_PREVIEW_BYTES = 8 * 1024 * 1024
1013
1114/** Read at most this many bytes while sniffing the first line for the delimiter. */
1215const DELIMITER_SNIFF_MAX_BYTES = 256 * 1024
@@ -44,7 +47,7 @@ function cell(value: unknown): string {
4447/**
4548 * Streams the first {@link CSV_PREVIEW_MAX_ROWS} rows of a CSV/TSV from storage without
4649 * ever buffering the whole file. The source stream is destroyed as soon as enough rows are
47- * read (one past the cap, to detect truncation), so a multi-GB file costs O(rows) of memory.
50+ * read (one past the cap, to detect truncation), with record and total-byte limits bounding parser memory and work .
4851 */
4952export async function getCsvPreviewSlice ( {
5053 key,
@@ -60,6 +63,17 @@ export async function getCsvPreviewSlice({
6063 signal ?. addEventListener ( 'abort' , onAbort , { once : true } )
6164
6265 const reader = source [ Symbol . asyncIterator ] ( )
66+ let bytesRead = 0
67+ let piped : Readable | undefined
68+ let parser : ReturnType < typeof parseCsvStream > | undefined
69+ async function nextChunk ( ) {
70+ const next = await reader . next ( )
71+ if ( ! next . done ) {
72+ bytesRead += Buffer . byteLength ( next . value )
73+ assertKnownSizeWithinLimit ( bytesRead , MAX_PREVIEW_BYTES , 'CSV preview' )
74+ }
75+ return next
76+ }
6377
6478 try {
6579 // Pull chunks until the first newline so the delimiter can be sniffed before parsing.
@@ -71,12 +85,14 @@ export async function getCsvPreviewSlice({
7185 let firstLine = ''
7286 let sniffedBytes = 0
7387 while ( true ) {
74- const { value, done } = await reader . next ( )
88+ const { value, done } = await nextChunk ( )
7589 if ( done ) break
7690 const chunk = Buffer . isBuffer ( value ) ? value : Buffer . from ( value )
7791 sniffed . push ( chunk )
7892 sniffedBytes += chunk . length
79- const text = chunk . toString ( 'utf-8' )
93+ const text = chunk
94+ . subarray ( 0 , Math . max ( 0 , DELIMITER_SNIFF_MAX_BYTES - sniffedBytes + chunk . length ) )
95+ . toString ( 'utf-8' )
8096 const nl = text . indexOf ( '\n' )
8197 if ( nl !== - 1 ) {
8298 firstLine += text . slice ( 0 , nl )
@@ -91,13 +107,13 @@ export async function getCsvPreviewSlice({
91107 }
92108
93109 const delimiter = detectDelimiter ( firstLine )
94- const parser = parseCsvStream ( {
110+ parser = parseCsvStream ( {
95111 columns : false ,
96112 skip_empty_lines : true ,
97113 trim : true ,
98114 relax_column_count : true ,
99115 relax_quotes : true ,
100- skip_records_with_error : true ,
116+ max_record_size : MAX_RECORD_SIZE ,
101117 cast : false ,
102118 bom : true ,
103119 delimiter,
@@ -107,13 +123,13 @@ export async function getCsvPreviewSlice({
107123 async function * rejoin ( ) {
108124 for ( const chunk of sniffed ) yield chunk
109125 while ( true ) {
110- const { value, done } = await reader . next ( )
126+ const { value, done } = await nextChunk ( )
111127 if ( done ) return
112128 yield value
113129 }
114130 }
115- const piped = Readable . from ( rejoin ( ) )
116- piped . on ( 'error' , ( err ) => parser . destroy ( err ) )
131+ piped = Readable . from ( rejoin ( ) )
132+ piped . on ( 'error' , ( err ) => parser ? .destroy ( err ) )
117133 piped . pipe ( parser )
118134
119135 let headers : string [ ] = [ ]
@@ -141,6 +157,8 @@ export async function getCsvPreviewSlice({
141157 if ( signal ?. aborted ) signal . throwIfAborted ( )
142158 throw error
143159 } finally {
160+ piped ?. destroy ( )
161+ parser ?. destroy ( )
144162 signal ?. removeEventListener ( 'abort' , onAbort )
145163 source . destroy ( )
146164 }
0 commit comments