diff --git a/CHANGES.rst b/CHANGES.rst index 88b983e4bf..1525e4ba9d 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -37,11 +37,20 @@ Compute Storage ~~~~~~~ +- [S3] Fix ``chunk_size`` argument being ignored by + ``download_object_as_stream`` and ``download_object_range_as_stream``. The + requested chunk size is now passed through to the underlying iterator + instead of the hardcoded default. + (GITHUB-1798) + + [Sanjay Santhanam - @Sanjays2402] + - [Azure Blobs] Fix ``chunk_size`` argument being ignored by ``download_object_as_stream`` and ``download_object_range_as_stream``. The requested chunk size is now forwarded to the underlying iterator instead of always using ``AZURE_DOWNLOAD_CHUNK_SIZE``. (GITHUB-1698) + [Sanjay Santhanam - @Sanjays2402] Changes in Apache Libcloud 3.9.1 diff --git a/libcloud/storage/drivers/s3.py b/libcloud/storage/drivers/s3.py index bd9212ea50..4b59e2df9b 100644 --- a/libcloud/storage/drivers/s3.py +++ b/libcloud/storage/drivers/s3.py @@ -521,7 +521,7 @@ def download_object_as_stream(self, obj, chunk_size=None): callback=read_in_chunks, response=response, callback_kwargs={ - "iterator": response.iter_content(CHUNK_SIZE), + "iterator": response.iter_content(chunk_size or CHUNK_SIZE), "chunk_size": chunk_size, }, success_status_code=httplib.OK, @@ -573,7 +573,7 @@ def download_object_range_as_stream(self, obj, start_bytes, end_bytes=None, chun callback=read_in_chunks, response=response, callback_kwargs={ - "iterator": response.iter_content(CHUNK_SIZE), + "iterator": response.iter_content(chunk_size or CHUNK_SIZE), "chunk_size": chunk_size, }, success_status_code=httplib.PARTIAL_CONTENT, diff --git a/libcloud/test/storage/test_s3.py b/libcloud/test/storage/test_s3.py index be70898c84..406a5fa356 100644 --- a/libcloud/test/storage/test_s3.py +++ b/libcloud/test/storage/test_s3.py @@ -861,6 +861,30 @@ def mock_get_object( finally: self.driver_type._get_object = old_func + def test_download_object_as_stream_uses_chunk_size(self): + # Regression test: the chunk_size passed by the caller must be + # forwarded to iter_content instead of the hardcoded CHUNK_SIZE. + container = Container(name="foo_bar_container", extra={}, driver=self.driver) + obj = Object( + name="foo_bar_object", + size=1000, + hash=None, + extra={}, + container=container, + meta_data=None, + driver=self.driver_type, + ) + + requested_chunk_size = CHUNK_SIZE * 2 + mock_response = Mock(name="mock response") + mock_response.iter_content.return_value = iter([b"a"]) + + with mock.patch.object(self.driver.connection, "request", return_value=mock_response): + with mock.patch.object(self.driver, "_get_object", side_effect=lambda **kw: kw): + self.driver.download_object_as_stream(obj=obj, chunk_size=requested_chunk_size) + + mock_response.iter_content.assert_called_once_with(requested_chunk_size) + def test_upload_object_invalid_ex_storage_class(self): # Invalid hash is detected on the amazon side and BAD_REQUEST is # returned