diff --git a/CHANGELOG.rst b/CHANGELOG.rst index cf4db41092..0df9b8fb64 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -8,6 +8,10 @@ Next release ``licensedcode-data``. https://github.com/aboutcode-org/scancode-toolkit/pull/5056 +- Fix a regression in PDF text extraction that silently stopped after the + first page, so copyright/license detection missed text on later pages. + https://github.com/aboutcode-org/scancode-toolkit/pull/5260 + - Improve copyright detection for statements with parens or trailing "authors" diff --git a/src/textcode/pdf.py b/src/textcode/pdf.py index 4e90c866f8..4beb11aea6 100644 --- a/src/textcode/pdf.py +++ b/src/textcode/pdf.py @@ -22,8 +22,9 @@ def get_text_lines(location, max_pages=5): """ - Return a list of unicode text lines extracted from a pdf file at - `location`. May raise exceptions. Extract up to `max_pages` pages. + Return a list of text lines, as bytes, extracted from a pdf file at + `location`. May raise exceptions. Extract up to `max_pages` pages, or + all pages if `max_pages` is 0. """ extracted_text = BytesIO() laparams = LAParams() @@ -43,5 +44,5 @@ def get_text_lines(location, max_pages=5): interpreter.process_page(page) if max_pages and page_num == max_pages: break - extracted_text.seek(0) - return extracted_text.readlines() + extracted_text.seek(0) + return extracted_text.readlines() diff --git a/tests/licensedcode/data/datadriven/lic4/should_detect_something_5.pdf.yml b/tests/licensedcode/data/datadriven/lic4/should_detect_something_5.pdf.yml index 8aa82b1c4c..39c8442d82 100644 --- a/tests/licensedcode/data/datadriven/lic4/should_detect_something_5.pdf.yml +++ b/tests/licensedcode/data/datadriven/lic4/should_detect_something_5.pdf.yml @@ -1,4 +1,2 @@ license_expressions: - sun-sissl-1.1 - - proprietary-license - - cpal-1.0 diff --git a/tests/textcode/data/pdf/multi_page.pdf b/tests/textcode/data/pdf/multi_page.pdf new file mode 100644 index 0000000000..d369663e18 Binary files /dev/null and b/tests/textcode/data/pdf/multi_page.pdf differ diff --git a/tests/textcode/test_pdf.py b/tests/textcode/test_pdf.py index 7941953568..cc7b2d05da 100644 --- a/tests/textcode/test_pdf.py +++ b/tests/textcode/test_pdf.py @@ -51,6 +51,33 @@ def get_text(location): assert result == expected + def test_get_text_lines_extracts_all_pages_up_to_max_pages(self): + # regression test: text extraction must not stop after the first page + test_file = self.get_test_loc('pdf/multi_page.pdf') + result = pdf.get_text_lines(test_file) + text = b''.join(result) + assert b'This is page 1 of a multi-page test document.' in text + assert b'This notice is on page 2.' in text + assert b'The last page 5 of this test document.' in text + # the default max_pages=5 must still be honored + assert b'Page 6' not in text + + def test_get_text_lines_returns_lines_when_max_pages_is_one(self): + # regression test: reaching max_pages must not return None + test_file = self.get_test_loc('pdf/multi_page.pdf') + result = pdf.get_text_lines(test_file, max_pages=1) + assert result + text = b''.join(result) + assert b'This is page 1 of a multi-page test document.' in text + assert b'page 2' not in text + + def test_get_text_lines_extracts_all_pages_when_max_pages_is_zero(self): + test_file = self.get_test_loc('pdf/multi_page.pdf') + result = pdf.get_text_lines(test_file, max_pages=0) + text = b''.join(result) + assert b'Page 6 is beyond the default max_pages limit.' in text + assert b'Page 7 is also beyond the default max_pages limit.' in text + def test_pdfminer_can_parse_faulty_broadcom_doc(self): # test for https://github.com/euske/pdfminer/issues/118 test_file = self.get_test_loc('pdf/pdfminer_bug_118/faulty.pdf')