Skip to content

Fix direct upload size limit validation and quota refresh #12706

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
sriramkanakam87 wants to merge 4 commits into
base: develop
Choose a base branch
from
+157 −15
Open

Fix direct upload size limit validation and quota refresh #12706

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
b8ef509
Fixed handling of failed storage quota validation for direct uploads …
sriramkanakam87 Sep 10, 2026
1a088f2
Fixed direct upload quota validation and refresh #12645
sriramkanakam87 Sep 14, 2026
f3ab952
Removed the redundant warning icon added by PrimeFaces #12645
sriramkanakam87 Sep 14, 2026
f76f5aa
Added tests for direct upload quota refresh #12645
sriramkanakam87 Sep 14, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
31 changes: 29 additions & 2 deletions src/main/java/edu/harvard/iq/dataverse/EditDatafilesPage.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -356,11 +356,13 @@ public boolean isUnlimitedUploadFileSize() {
}

public Long getMaxTotalUploadSizeInBytes() {
return maxTotalUploadSizeInBytes;
return uploadSessionQuota == null
? maxTotalUploadSizeInBytes
: uploadSessionQuota.getRemainingQuotaInBytes();
}

public String getHumanMaxTotalUploadSizeInBytes() {
return FileSizeChecker.bytesToHumanReadable(maxTotalUploadSizeInBytes);
return FileSizeChecker.bytesToHumanReadable(getMaxTotalUploadSizeInBytes());
}

public boolean isStorageQuotaEnforced() {
Expand Down Expand Up @@ -981,6 +983,8 @@ private void deleteFiles(List<FileMetadata> filesForDelete) {
// removing it from the fileMetadatas lists (above), we also remove it from
// the newFiles list and the dataset's files, so it never gets saved.

releaseUploadQuota(markedForDelete.getDataFile());

FileMetadataUtil.removeDataFileFromList(dataset.getFiles(), markedForDelete.getDataFile());
FileMetadataUtil.removeDataFileFromList(newFiles, markedForDelete.getDataFile());
FileUtil.deleteTempFile(markedForDelete.getDataFile(), dataset, ingestService);
Expand Down Expand Up @@ -1011,6 +1015,13 @@ private void deleteFiles(List<FileMetadata> filesForDelete) {
}
}

private void releaseUploadQuota(DataFile dataFile) {
if (uploadSessionQuota != null && dataFile.getFilesize() > 0) {
uploadSessionQuota.setTotalUsageInBytes(Math.max(0L,
uploadSessionQuota.getTotalUsageInBytes() - dataFile.getFilesize()));
}
}

/**
* Save for File Replace operations
*
Expand Down Expand Up @@ -2190,6 +2201,22 @@ public void handleExternalUpload() {
Optional.ofNullable(editDataFilesPageHelper.getHtmlErrorMessage(createDataFilesResult)).ifPresent(errorMessage -> errorMessages.add(errorMessage));
} catch (CommandException ex) {
logger.log(Level.SEVERE, "Error during ingest of file {0}", new Object[]{fileName});

// Direct upload has already stored the file in S3.
// Remove it if Dataverse rejects the upload.
try {
sio.delete();
} catch (IOException deleteEx) {
logger.log(
Level.WARNING,
"Failed to delete rejected direct upload {0}: {1}",
new Object[]{fileName, deleteEx.getMessage()}
);
}

// Show the actual command error to the user.
errorMessages.add(ex.getMessage());
return;
}

if (datafiles == null) {
Expand Down
37 changes: 31 additions & 6 deletions src/main/webapp/editFilesFragment.xhtml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -138,6 +138,31 @@
$(fileUpload).prepend('<div id="dragdropMsg">#{bundle['file.selectToAdd.dragdropMsg']}</div>');
}
}
function checkUploadLimits(files, maxFileSize, remainingQuota) {
var fileUploadWidget = PF('fileUploadWidget');
var totalSize = 0;
for (var i = 0; i < files.length; i++) {
if (maxFileSize !== null && files[i].size > maxFileSize) {
fileUploadWidget.clear();
fileUploadWidget.showMessage({summary: '#{bundle["file.edit.error.file_exceeds_limit"]}'});
fileUploadWidget.jq.find('.ui-messages-error-icon').remove();
return false;
}
totalSize += files[i].size;
}

if (remainingQuota !== null && totalSize > remainingQuota) {
var message = '#{bundle["file.addreplace.error.quota_exceeded"]}'
.replace('{0}', fileUploadWidget.formatSize(totalSize))
.replace('{1}', fileUploadWidget.formatSize(remainingQuota));
fileUploadWidget.clear();
fileUploadWidget.showMessage({summary: message});
fileUploadWidget.jq.find('.ui-messages-error-icon').remove();
return false;
}

return true;
}
function dropBoxUploadFinished(fileupload) {
$('button[id$="dropBoxUploadFinished"]').trigger('click');
}
Expand All @@ -147,7 +172,7 @@

$(document).ready(function () {
uploadWidgetDropMsg();
#{useDirectUpload ? 'setupDirectUpload(true);':''}
setupDirectUpload(#{useDirectUpload}, #{empty EditDatafilesPage.maxFileUploadSizeInBytes ? 'null' : EditDatafilesPage.maxFileUploadSizeInBytes}, #{empty EditDatafilesPage.maxTotalUploadSizeInBytes ? 'null' : EditDatafilesPage.maxTotalUploadSizeInBytes});
});
//]]>
</script>
Expand Down Expand Up @@ -348,9 +373,9 @@

<p:commandButton id="updateEditDataFilesButtonsForUpload" action="#{EditDatafilesPage.uploadFinished()}" update="datasetForm:editDataFilesButtons" rendered="#{showFileButtonUpdate}" style="display:none"/>
<p:commandButton id="updateEditDataFilesButtonsForDelete" action="#{EditDatafilesPage.deleteFilesCompleted()}" update="datasetForm:editDataFilesButtons" rendered="#{showFileButtonUpdate}" style="display:none"/>
<p:commandButton id="AllUploadsFinished" action="#{EditDatafilesPage.uploadFinished()}" update="@([id$=filesButtons]),datasetForm:contentOfHttpPanel,datasetForm:fileUpload,datasetForm:dropBoxUserButton,datasetForm:uploadMessage,datasetForm:rsyncPanel,datasetForm:filesCounts,datasetForm:filesTable" oncomplete="javascript:uploadWidgetDropMsg();" style="display:none"/>
<p:commandButton id="dropBoxUploadFinished" action="#{EditDatafilesPage.uploadFinished()}" update="@([id$=filesButtons]),datasetForm:fileUpload,datasetForm:dropBoxUserButton,datasetForm:dropBoxUploadMessage,datasetForm:rsyncPanel,datasetForm:filesCounts,datasetForm:filesTable" style="display:none"/>
<p:commandButton id="allDeletesFinished" action="#{EditDatafilesPage.deleteFilesCompleted()}" update="@([id$=filesButtons]),datasetForm:fileUpload,datasetForm:dropBoxUserButton,datasetForm:uploadMessage,datasetForm:rsyncPanel,datasetForm:filesCounts,datasetForm:filesTable" oncomplete="javascript:uploadWidgetDropMsg();" style="display:none"/>
<p:commandButton id="AllUploadsFinished" action="#{EditDatafilesPage.uploadFinished()}" update="@([id$=filesButtons]),datasetForm:contentOfHttpPanel,datasetForm:fileUpload,datasetForm:dropBoxUserButton,datasetForm:uploadMessage,datasetForm:rsyncPanel,datasetForm:filesCounts,datasetForm:filesTable" oncomplete="uploadWidgetDropMsg(); setupDirectUpload(#{useDirectUpload}, #{empty EditDatafilesPage.maxFileUploadSizeInBytes ? 'null' : EditDatafilesPage.maxFileUploadSizeInBytes}, #{empty EditDatafilesPage.maxTotalUploadSizeInBytes ? 'null' : EditDatafilesPage.maxTotalUploadSizeInBytes});" style="display:none"/>
<p:commandButton id="dropBoxUploadFinished" action="#{EditDatafilesPage.uploadFinished()}" update="@([id$=filesButtons]),datasetForm:fileUpload,datasetForm:dropBoxUserButton,datasetForm:dropBoxUploadMessage,datasetForm:rsyncPanel,datasetForm:filesCounts,datasetForm:filesTable" oncomplete="setupDirectUpload(#{useDirectUpload}, #{empty EditDatafilesPage.maxFileUploadSizeInBytes ? 'null' : EditDatafilesPage.maxFileUploadSizeInBytes}, #{empty EditDatafilesPage.maxTotalUploadSizeInBytes ? 'null' : EditDatafilesPage.maxTotalUploadSizeInBytes});" style="display:none"/>
<p:commandButton id="allDeletesFinished" action="#{EditDatafilesPage.deleteFilesCompleted()}" update="@([id$=filesButtons]),datasetForm:contentOfHttpPanel,datasetForm:fileUpload,datasetForm:dropBoxUserButton,datasetForm:uploadMessage,datasetForm:rsyncPanel,datasetForm:filesCounts,datasetForm:filesTable" oncomplete="uploadWidgetDropMsg(); setupDirectUpload(#{useDirectUpload}, #{empty EditDatafilesPage.maxFileUploadSizeInBytes ? 'null' : EditDatafilesPage.maxFileUploadSizeInBytes}, #{empty EditDatafilesPage.maxTotalUploadSizeInBytes ? 'null' : EditDatafilesPage.maxTotalUploadSizeInBytes});" style="display:none"/>
<!-- ????? TODO: How scalable are all these hidden update buttons ????? -->
<!-- ?????
datasetForm:fileUpload,
Expand Down Expand Up @@ -625,7 +650,7 @@
<p class="text-warning"><span class="glyphicon glyphicon-warning-sign"/> #{EditDatafilesPage.warningMessageForFileTypeDifferentPopUp}</p>
<div class="button-block">
<p:commandButton styleClass="btn btn-default" value="#{bundle['file.delete']}" onclick="PF('fileTypeDifferentPopup').hide()" oncomplete="uploadWidgetDropMsg();
#{useDirectUpload ? 'setupDirectUpload(true);': ''}"
setupDirectUpload(#{useDirectUpload}, #{empty EditDatafilesPage.maxFileUploadSizeInBytes ? 'null' : EditDatafilesPage.maxFileUploadSizeInBytes}, #{empty EditDatafilesPage.maxTotalUploadSizeInBytes ? 'null' : EditDatafilesPage.maxTotalUploadSizeInBytes});"
action="#{EditDatafilesPage.deleteFiles()}"
update=":#{p:resolveClientId('datasetForm:filesTable', view)},:messagePanel,:#{p:resolveClientId('datasetForm:fileUpload', view)},uploadMessage"/>
<button class="btn btn-default" onclick="PF('fileTypeDifferentPopup').hide();" type="button">
Expand All @@ -643,7 +668,7 @@
</h:outputFormat>
</p>
<div class="button-block">
<p:commandButton styleClass="btn btn-default" value="#{EditDatafilesPage.labelForDeleteFilesPopup}" onclick="PF('fileAlreadyExistsPopup').hide()" oncomplete="uploadWidgetDropMsg();"
<p:commandButton styleClass="btn btn-default" value="#{EditDatafilesPage.labelForDeleteFilesPopup}" onclick="PF('fileAlreadyExistsPopup').hide()" oncomplete="uploadWidgetDropMsg(); setupDirectUpload(#{useDirectUpload}, #{empty EditDatafilesPage.maxFileUploadSizeInBytes ? 'null' : EditDatafilesPage.maxFileUploadSizeInBytes}, #{empty EditDatafilesPage.maxTotalUploadSizeInBytes ? 'null' : EditDatafilesPage.maxTotalUploadSizeInBytes});"
action="#{EditDatafilesPage.deleteDuplicateFiles()}"
update=":#{p:resolveClientId('datasetForm:filesTable', view)},:messagePanel,:#{p:resolveClientId('datasetForm:fileUpload', view)},uploadMessage,@([id$=filesButtons])"/>
<button class="btn btn-default" onclick="PF('fileAlreadyExistsPopup').hide();" type="button">
Expand Down
1 change: 1 addition & 0 deletions src/main/webapp/editdatafiles.xhtml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -77,6 +77,7 @@
<ui:param name="bean" value="#{EditDatafilesPage}"/>
<ui:param name="restrictFileAction" value="restrictFiles"/>
<ui:param name="deleteFileAction" value="deleteFiles"/>
<ui:param name="deleteFileOncomplete" value="deleteFinished();"/>
<ui:param name="saveEmbargoAction" value="saveEmbargo"/>
<ui:param name="saveRetentionAction" value="saveRetention"/>
<ui:param name="updateElements" value=":datasetForm:embargoInputs"/>
Expand Down
2 changes: 1 addition & 1 deletion src/main/webapp/file-edit-popup-fragment.xhtml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -264,7 +264,7 @@
</ui:fragment>
<div class="button-block">
<p:commandButton styleClass="btn btn-default" value="#{bundle['file.delete']}" onclick="PF('deleteFileConfirmation').hide()"
action="#{bean[deleteFileAction]}"/>
action="#{bean[deleteFileAction]}" oncomplete="#{deleteFileOncomplete}"/>
<button class="btn btn-link" onclick="PF('deleteFileConfirmation').hide()" type="button">
#{bundle.cancel}
</button>
Expand Down
21 changes: 16 additions & 5 deletions src/main/webapp/resources/js/fileupload.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,7 @@ var finishFile = (function() {
})();


function setupDirectUpload(enabled) {
function setupDirectUpload(enabled, maxFileSize, remainingQuota) {
if (enabled) {
directUploadEnabled = true;
//An indicator as to which version is being used - should keep updated.
Expand Down Expand Up @@ -75,6 +75,16 @@ function setupDirectUpload(enabled) {
}
//Add support for drag and drop. Since the fileUploadForm is not replaced by PF, catching changes with a mutationobserver isn't needed
var fileDropWidget = document.getElementById('datasetForm:fileUpload');
fileDropWidget.addEventListener('change', function(event) {
if (!checkUploadLimits(event.target.files, maxFileSize, remainingQuota)) {
event.stopImmediatePropagation();
}
}, true);
fileDropWidget.addEventListener('drop', function(event) {
if (!checkUploadLimits(event.dataTransfer.files, maxFileSize, remainingQuota)) {
event.stopImmediatePropagation();
}
}, true);
fileDropWidget.addEventListener('drop', function(event) {
fileList = [];
for (var i = 0; i < event.dataTransfer.files.length; i++) {
Expand All @@ -87,11 +97,12 @@ function setupDirectUpload(enabled) {
mutations.forEach(function(mutation) {
for (i = 0; i < mutation.addedNodes.length; i++) {
//Add a listener on any replacement file 'select' widget
if (mutation.addedNodes[i].id === 'datasetForm:fileUpload_input') {
if (mutation.addedNodes[i].id === 'datasetForm:fileUpload_input') {
fileInput = mutation.addedNodes[i];
mutation.addedNodes[i].addEventListener('change', function(event) {
for (var j = 0; j < mutation.addedNodes[i].files.length; j++) {
queueFileForDirectUpload(mutation.addedNodes[i].files[j]);
var files = event.target.files;
for (var j = 0; j < files.length; j++) {
queueFileForDirectUpload(files[j]);
}
}, { once: false });
}
Expand Down Expand Up @@ -690,4 +701,4 @@ function getChecksum(blob, cbProgress) {
}
});
});
}
}
80 changes: 79 additions & 1 deletion src/test/java/edu/harvard/iq/dataverse/EditDatafilesPageTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,17 +1,26 @@
package edu.harvard.iq.dataverse;

import edu.harvard.iq.dataverse.storageuse.UploadSessionQuotaLimit;
import edu.harvard.iq.dataverse.util.FileUtil;
import edu.harvard.iq.dataverse.util.JsfHelper;
import edu.harvard.iq.dataverse.util.SystemConfig;
import org.junit.jupiter.api.Test;
import org.mockito.InjectMocks;
import org.mockito.Mock;
import org.mockito.MockedStatic;
import org.mockito.MockitoAnnotations;

import java.lang.reflect.Field;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.mockStatic;
import static org.mockito.Mockito.when;

class EditDatafilesPageTest {
Expand Down Expand Up @@ -61,4 +70,73 @@ void testPopulateHumanPerFormatTabularLimits_WithDefaultKey() {
assertTrue(result.contains("csv: 10.0 MB"), "Expected CSV limit in human-readable format, but got: " + result);
assertFalse(result.contains("default"), "Default key should be excluded from the output");
}
}

@Test
void testMaxTotalUploadSizeTracksCurrentUploadSessionQuota() throws Exception {
UploadSessionQuotaLimit quota = new UploadSessionQuotaLimit(100L, 80L);
Field quotaField = EditDatafilesPage.class.getDeclaredField("uploadSessionQuota");
quotaField.setAccessible(true);
quotaField.set(editDatafilesPage, quota);

assertEquals(20L, editDatafilesPage.getMaxTotalUploadSizeInBytes());
assertEquals("20 B", editDatafilesPage.getHumanMaxTotalUploadSizeInBytes());

quota.setTotalUsageInBytes(90L);

assertEquals(10L, editDatafilesPage.getMaxTotalUploadSizeInBytes());
assertEquals("10 B", editDatafilesPage.getHumanMaxTotalUploadSizeInBytes());
}

@Test
void testMaxTotalUploadSizeFallsBackToConfiguredLimitWithoutQuota() throws Exception {
Field limitField = EditDatafilesPage.class.getDeclaredField("maxTotalUploadSizeInBytes");
limitField.setAccessible(true);
limitField.set(editDatafilesPage, 25L);

assertEquals(25L, editDatafilesPage.getMaxTotalUploadSizeInBytes());
assertEquals("25 B", editDatafilesPage.getHumanMaxTotalUploadSizeInBytes());
}

@Test
void testDeletingUnsavedFileReleasesUploadQuota() throws Exception {
UploadSessionQuotaLimit quota = new UploadSessionQuotaLimit(100L, 90L);
Field quotaField = EditDatafilesPage.class.getDeclaredField("uploadSessionQuota");
quotaField.setAccessible(true);
quotaField.set(editDatafilesPage, quota);
DataFile dataFile = new DataFile();
dataFile.setFilesize(10L);
dataFile.setStorageIdentifier("test-storage-id");
FileMetadata fileMetadata = new FileMetadata();
fileMetadata.setDataFile(dataFile);
fileMetadata.setLabel("test.txt");
Dataset dataset = mock(Dataset.class);
DatasetVersion version = mock(DatasetVersion.class);
List<FileMetadata> displayedFiles = new ArrayList<>(List.of(fileMetadata));
List<FileMetadata> versionFiles = new ArrayList<>(List.of(fileMetadata));
List<DataFile> datasetFiles = new ArrayList<>(List.of(dataFile));
List<DataFile> newFiles = new ArrayList<>(List.of(dataFile));
when(dataset.getOrCreateEditVersion()).thenReturn(version);
when(version.getFileMetadatas()).thenReturn(versionFiles);
when(dataset.getFiles()).thenReturn(datasetFiles);
editDatafilesPage.setDataset(dataset);
editDatafilesPage.setMode(EditDatafilesPage.FileEditMode.UPLOAD);
editDatafilesPage.setFileMetadatas(displayedFiles);
editDatafilesPage.setSelectedFiles(List.of(fileMetadata));
Field newFilesField = EditDatafilesPage.class.getDeclaredField("newFiles");
newFilesField.setAccessible(true);
newFilesField.set(editDatafilesPage, newFiles);

try (MockedStatic<FileUtil> fileUtil = mockStatic(FileUtil.class);
MockedStatic<JsfHelper> jsfHelper = mockStatic(JsfHelper.class)) {
editDatafilesPage.deleteFiles();
fileUtil.verify(() -> FileUtil.deleteTempFile(dataFile, dataset, null));
}

assertEquals(20L, editDatafilesPage.getMaxTotalUploadSizeInBytes());
assertTrue(displayedFiles.isEmpty());
assertTrue(versionFiles.isEmpty());
assertTrue(datasetFiles.isEmpty());
assertTrue(newFiles.isEmpty());
}

}