From 764b401482bd83b3abf5de8be372a9128b55b7ea Mon Sep 17 00:00:00 2001 From: John Rogers Date: Fri, 4 Sep 2026 12:30:37 +0100 Subject: [PATCH] Parse uploaded files one at a time so large PDF batches don't time out A single /text/parse call carrying many large PDFs times out: Tika can take well over a minute per big or scanned PDF, and the client's 15s timeout with three blind retries then re-sent the whole batch while the server was still working on the first attempt. The user saw "Something went wrong" even though each file parses fine on its own. Upload.js now chains one /text/parse call per file, in series, with a progress toast ("Parsing 3 of 15: name.pdf"), shows each file as soon as it lands, and reports which files failed instead of failing the batch. DataContext.parse gets a generous per-file timeout and no retry, since re-sending a PDF that just timed out only piles more work on the server. The API caches parsed files by content, so a re-upload is instant. Fixes #61 Co-Authored-By: Claude Fable 5.1 --- src/components/Upload.js | 80 +++++++++++++++++++++++++------------ src/contexts/DataContext.js | 17 ++++++-- 2 files changed, 67 insertions(+), 30 deletions(-) diff --git a/src/components/Upload.js b/src/components/Upload.js index 0a3a1d6..d3a8ba9 100644 --- a/src/components/Upload.js +++ b/src/components/Upload.js @@ -217,34 +217,62 @@ export default function Upload({ ); }); Promise.all(frp) - .then((allFiles) => { - toast.promise( - new Promise((resolve, reject) => { - parse(allFiles) - .then((data) => { - const newFileInfos = [...fileInfos]; - // Load each returned file / instrument in the data - data.forEach((instrument) => { - newFileInfos.push(instrument); - }); - setFileInfos(newFileInfos); - resolve(true); - }) - .catch((e) => { - console.log(e); - setParseError(true); - reject("Parse Error"); - }) - .finally((_) => { - syncFileInfos(); + .then(async (allFiles) => { + // Parse ONE file per API call, in series (issue #61). A single + // /text/parse call carrying many large PDFs times out: Tika can take + // well over a minute per big or scanned PDF, and the old 15s timeout + // with 3 retries then re-sent the whole batch while the server was + // still working on the first attempt. One file per call keeps each + // request inside the proxy limits, shows progress, and lets the good + // files land even if one bad scan fails. The API caches parsed files + // by content, so re-uploading an already-parsed file is instant. + const progress = (i) => + allFiles.length === 1 + ? `Parsing ${allFiles[0].file_name} - this may take a while` + : `Parsing ${i + 1} of ${allFiles.length}: ${allFiles[i].file_name} - this may take a while`; + const toastId = toast.loading(progress(0)); + const newFileInfos = [...fileInfos]; + const failed = []; + let parsedCount = 0; + for (let i = 0; i < allFiles.length; i++) { + toast.update(toastId, { render: progress(i) }); + try { + const data = await parse([allFiles[i]]); + if (Array.isArray(data) && data.length) { + data.forEach((instrument) => { + newFileInfos.push(instrument); }); - }), - { - pending: "Parsing files - this may take a while", - success: "Success!", - error: "Something went wrong - please try again", + // Show each file as soon as it lands rather than at the end + setFileInfos([...newFileInfos]); + parsedCount++; + } else { + failed.push(`${allFiles[i].file_name} (no questions found)`); + } + } catch (e) { + console.log(e); + failed.push(allFiles[i].file_name); } - ); + } + syncFileInfos(); + if (failed.length === 0) { + toast.update(toastId, { + render: "Success!", + type: "success", + isLoading: false, + autoClose: 3000, + }); + } else { + setParseError(true); + toast.update(toastId, { + render: + parsedCount === 0 + ? "Something went wrong - please try again" + : `Parsed ${parsedCount} of ${allFiles.length} files - could not parse ${failed.join(", ")}`, + type: parsedCount === 0 ? "error" : "warning", + isLoading: false, + autoClose: 8000, + }); + } }) .catch((e) => { console.log(e); diff --git a/src/contexts/DataContext.js b/src/contexts/DataContext.js index 65ff6e0..a861be2 100644 --- a/src/contexts/DataContext.js +++ b/src/contexts/DataContext.js @@ -26,9 +26,13 @@ export function DataProvider({ children }) { framework: "huggingface", model: "sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2", }); - const retryablePostData = ({ url = "", data = {}, timeout = 8000 }) => { + const retryablePostData = ({ + url = "", + data = {}, + timeout = 8000, + retries = 3, + }) => { return new Promise(async (resolve, reject) => { - var retries = 3; var response; while (retries > 0) { try { @@ -125,11 +129,16 @@ export function DataProvider({ children }) { }); }; - const parse = (allFiles) => { + // Parse is called with ONE file at a time by Upload.js (see issue #61). + // Tika can take well over a minute on a large or scanned PDF, so the + // per-call timeout is generous and there is no blind retry: re-sending a + // big PDF that just timed out only piles more work onto the server. + const parse = (allFiles, timeout = 180000) => { return retryablePostData({ url: process.env.REACT_APP_API_PARSE, data: allFiles, - timeout: 15000, + timeout: timeout, + retries: 1, }); }; const match = useCallback(