From 350fa60bcd8569acff9f562b91217f7f7a8f108f Mon Sep 17 00:00:00 2001 From: Adnan Raza Date: Mon, 31 Aug 2026 00:58:18 +0530 Subject: [PATCH 1/2] Fix Poetry lock optional dependency parsing Signed-off-by: Adnan Raza --- AUTHORS.rst | 1 + CHANGELOG.rst | 3 ++ src/packagedcode/pypi.py | 2 +- .../univers-package-assembly-expected.json | 20 +++++------ .../poetry/univers-poetry.lock-expected.json | 10 +++--- tests/packagedcode/test_pypi.py | 34 ++++++++++++++++++- 6 files changed, 53 insertions(+), 17 deletions(-) diff --git a/AUTHORS.rst b/AUTHORS.rst index 9a8224dc7d..a5a420d5e0 100644 --- a/AUTHORS.rst +++ b/AUTHORS.rst @@ -2,6 +2,7 @@ The following organizations or individuals have contributed to ScanCode: - Abhigyan Kumar Singh @Abhigyankrsingh - Abhishek Kumar @Abhishek-Dev09 +- Adnan Raza @Mars-60 - Aditya Viki @adityaviki - Adrian Braemer @abraemer - Agni Bhattacharyya @PyAgni diff --git a/CHANGELOG.rst b/CHANGELOG.rst index d9a5a6b402..1c5158d7c7 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -4,6 +4,9 @@ Changelog Next release -------------- +- Fix incorrect optional dependency reporting when parsing Poetry lockfiles. + https://github.com/aboutcode-org/scancode-toolkit/issues/5294 + - Fix the optional ``licenses`` extra dependency typo to install ``licensedcode-data``. https://github.com/aboutcode-org/scancode-toolkit/pull/5056 diff --git a/src/packagedcode/pypi.py b/src/packagedcode/pypi.py index dcfd237590..00b29ab16a 100644 --- a/src/packagedcode/pypi.py +++ b/src/packagedcode/pypi.py @@ -806,7 +806,7 @@ def parse(cls, location, package_only=False): ) resolved_package = models.PackageData.from_data(package_data, package_only) - is_optional = package.get("is_optional") or True + is_optional = package.get("optional", False) dependency = models.DependentPackage( purl=resolved_package.purl, extracted_requirement=None, diff --git a/tests/packagedcode/data/pypi/poetry/univers-package-assembly-expected.json b/tests/packagedcode/data/pypi/poetry/univers-package-assembly-expected.json index 4f0b49841b..511c981f23 100644 --- a/tests/packagedcode/data/pypi/poetry/univers-package-assembly-expected.json +++ b/tests/packagedcode/data/pypi/poetry/univers-package-assembly-expected.json @@ -146,7 +146,7 @@ "extracted_requirement": null, "scope": null, "is_runtime": true, - "is_optional": true, + "is_optional": false, "is_pinned": true, "is_direct": false, "resolved_package": { @@ -579,7 +579,7 @@ "extracted_requirement": null, "scope": null, "is_runtime": true, - "is_optional": true, + "is_optional": false, "is_pinned": true, "is_direct": false, "resolved_package": { @@ -649,7 +649,7 @@ "extracted_requirement": null, "scope": null, "is_runtime": true, - "is_optional": true, + "is_optional": false, "is_pinned": true, "is_direct": false, "resolved_package": { @@ -730,7 +730,7 @@ "extracted_requirement": null, "scope": null, "is_runtime": true, - "is_optional": true, + "is_optional": false, "is_pinned": true, "is_direct": false, "resolved_package": { @@ -921,7 +921,7 @@ "extracted_requirement": null, "scope": null, "is_runtime": true, - "is_optional": true, + "is_optional": false, "is_pinned": true, "is_direct": false, "resolved_package": { @@ -1033,7 +1033,7 @@ "extracted_requirement": null, "scope": null, "is_runtime": true, - "is_optional": true, + "is_optional": false, "is_pinned": true, "is_direct": false, "resolved_package": { @@ -1462,7 +1462,7 @@ "extracted_requirement": null, "scope": null, "is_runtime": true, - "is_optional": true, + "is_optional": false, "is_pinned": true, "is_direct": false, "resolved_package": { @@ -1528,7 +1528,7 @@ "extracted_requirement": null, "scope": null, "is_runtime": true, - "is_optional": true, + "is_optional": false, "is_pinned": true, "is_direct": false, "resolved_package": { @@ -1605,7 +1605,7 @@ "extracted_requirement": null, "scope": null, "is_runtime": true, - "is_optional": true, + "is_optional": false, "is_pinned": true, "is_direct": false, "resolved_package": { @@ -1792,7 +1792,7 @@ "extracted_requirement": null, "scope": null, "is_runtime": true, - "is_optional": true, + "is_optional": false, "is_pinned": true, "is_direct": false, "resolved_package": { diff --git a/tests/packagedcode/data/pypi/poetry/univers-poetry.lock-expected.json b/tests/packagedcode/data/pypi/poetry/univers-poetry.lock-expected.json index bfe38cff67..a657cd5184 100644 --- a/tests/packagedcode/data/pypi/poetry/univers-poetry.lock-expected.json +++ b/tests/packagedcode/data/pypi/poetry/univers-poetry.lock-expected.json @@ -45,7 +45,7 @@ "extracted_requirement": null, "scope": null, "is_runtime": true, - "is_optional": true, + "is_optional": false, "is_pinned": true, "is_direct": false, "resolved_package": { @@ -474,7 +474,7 @@ "extracted_requirement": null, "scope": null, "is_runtime": true, - "is_optional": true, + "is_optional": false, "is_pinned": true, "is_direct": false, "resolved_package": { @@ -540,7 +540,7 @@ "extracted_requirement": null, "scope": null, "is_runtime": true, - "is_optional": true, + "is_optional": false, "is_pinned": true, "is_direct": false, "resolved_package": { @@ -617,7 +617,7 @@ "extracted_requirement": null, "scope": null, "is_runtime": true, - "is_optional": true, + "is_optional": false, "is_pinned": true, "is_direct": false, "resolved_package": { @@ -804,7 +804,7 @@ "extracted_requirement": null, "scope": null, "is_runtime": true, - "is_optional": true, + "is_optional": false, "is_pinned": true, "is_direct": false, "resolved_package": { diff --git a/tests/packagedcode/test_pypi.py b/tests/packagedcode/test_pypi.py index 20afae813d..6f7a1650f3 100644 --- a/tests/packagedcode/test_pypi.py +++ b/tests/packagedcode/test_pypi.py @@ -398,6 +398,39 @@ def test_parse_poetry_lock_univers(self): expected_loc = self.get_test_loc('pypi/poetry/univers-poetry.lock-expected.json') self.check_packages_data(package, expected_loc, regen=REGEN_TEST_FIXTURES) + def test_parse_poetry_lock_package_optional(self): + test_file = self.get_temp_file('poetry.lock') + with open(test_file, 'w') as lockfile: + lockfile.write('''\ +[[package]] +name = "required-dep" +version = "1.0.0" +description = "required dependency" +optional = false +python-versions = ">=3.8" + +[[package]] +name = "optional-dep" +version = "2.0.0" +description = "optional dependency" +optional = true +python-versions = ">=3.8" + +[metadata] +lock-version = "2.0" +python-versions = ">=3.8" +content-hash = "test" +''') + + package_data = list(pypi.PoetryLockHandler.parse(test_file))[0] + dependencies_by_purl = { + dependency['purl']: dependency + for dependency in package_data.dependencies + } + + assert dependencies_by_purl['pkg:pypi/required-dep@1.0.0']['is_optional'] is False + assert dependencies_by_purl['pkg:pypi/optional-dep@2.0.0']['is_optional'] is True + def test_parse_pyproject_toml_poetry_univers(self): test_file = self.get_test_loc('pypi/poetry/univers/pyproject.toml') package = pypi.PoetryPyprojectTomlHandler.parse(test_file) @@ -805,4 +838,3 @@ def test_parse_setup_py(test_loc): ) def test_parse_more_setup_py(test_loc): check_setup_py_parsing(test_loc) - From 7622155685a625ca6ac09c0e5099767e20fb4437 Mon Sep 17 00:00:00 2001 From: Adnan Raza Date: Wed, 2 Sep 2026 20:04:28 +0530 Subject: [PATCH 2/2] Fix Poetry optional dependency test Signed-off-by: Adnan Raza --- .../poetry/optional-poetry.lock-expected.json | 158 ++++++++++++++++++ .../data/pypi/poetry/optional/poetry.lock | 17 ++ tests/packagedcode/test_pypi.py | 35 +--- 3 files changed, 179 insertions(+), 31 deletions(-) create mode 100644 tests/packagedcode/data/pypi/poetry/optional-poetry.lock-expected.json create mode 100644 tests/packagedcode/data/pypi/poetry/optional/poetry.lock diff --git a/tests/packagedcode/data/pypi/poetry/optional-poetry.lock-expected.json b/tests/packagedcode/data/pypi/poetry/optional-poetry.lock-expected.json new file mode 100644 index 0000000000..55ea493506 --- /dev/null +++ b/tests/packagedcode/data/pypi/poetry/optional-poetry.lock-expected.json @@ -0,0 +1,158 @@ +[ + { + "type": "pypi", + "namespace": null, + "name": null, + "version": null, + "qualifiers": {}, + "subpath": null, + "primary_language": "Python", + "description": null, + "release_date": null, + "parties": [], + "keywords": [], + "homepage_url": null, + "download_url": null, + "size": null, + "sha1": null, + "md5": null, + "sha256": null, + "sha512": null, + "bug_tracking_url": null, + "code_view_url": null, + "vcs_url": null, + "copyright": null, + "holder": null, + "declared_license_expression": null, + "declared_license_expression_spdx": null, + "license_detections": [], + "other_license_expression": null, + "other_license_expression_spdx": null, + "other_license_detections": [], + "extracted_license_statement": null, + "notice_text": null, + "source_packages": [], + "file_references": [], + "is_private": false, + "is_virtual": false, + "extra_data": { + "python_version": ">=3.8", + "lock_version": "2.0" + }, + "dependencies": [ + { + "purl": "pkg:pypi/required-dep@1.0.0", + "extracted_requirement": null, + "scope": null, + "is_runtime": true, + "is_optional": false, + "is_pinned": true, + "is_direct": false, + "resolved_package": { + "type": "pypi", + "namespace": null, + "name": "required-dep", + "version": "1.0.0", + "qualifiers": {}, + "subpath": null, + "primary_language": "Python", + "description": null, + "release_date": null, + "parties": [], + "keywords": [], + "homepage_url": null, + "download_url": null, + "size": null, + "sha1": null, + "md5": null, + "sha256": null, + "sha512": null, + "bug_tracking_url": null, + "code_view_url": null, + "vcs_url": null, + "copyright": null, + "holder": null, + "declared_license_expression": null, + "declared_license_expression_spdx": null, + "license_detections": [], + "other_license_expression": null, + "other_license_expression_spdx": null, + "other_license_detections": [], + "extracted_license_statement": null, + "notice_text": null, + "source_packages": [], + "file_references": [], + "is_private": false, + "is_virtual": true, + "extra_data": {}, + "dependencies": [], + "repository_homepage_url": "https://pypi.org/project/required-dep", + "repository_download_url": "https://pypi.org/packages/source/r/required-dep/required-dep-1.0.0.tar.gz", + "api_data_url": "https://pypi.org/pypi/required-dep/1.0.0/json", + "datasource_id": "pypi_poetry_lock", + "purl": "pkg:pypi/required-dep@1.0.0" + }, + "extra_data": {} + }, + { + "purl": "pkg:pypi/optional-dep@2.0.0", + "extracted_requirement": null, + "scope": null, + "is_runtime": true, + "is_optional": true, + "is_pinned": true, + "is_direct": false, + "resolved_package": { + "type": "pypi", + "namespace": null, + "name": "optional-dep", + "version": "2.0.0", + "qualifiers": {}, + "subpath": null, + "primary_language": "Python", + "description": null, + "release_date": null, + "parties": [], + "keywords": [], + "homepage_url": null, + "download_url": null, + "size": null, + "sha1": null, + "md5": null, + "sha256": null, + "sha512": null, + "bug_tracking_url": null, + "code_view_url": null, + "vcs_url": null, + "copyright": null, + "holder": null, + "declared_license_expression": null, + "declared_license_expression_spdx": null, + "license_detections": [], + "other_license_expression": null, + "other_license_expression_spdx": null, + "other_license_detections": [], + "extracted_license_statement": null, + "notice_text": null, + "source_packages": [], + "file_references": [], + "is_private": false, + "is_virtual": true, + "extra_data": {}, + "dependencies": [], + "repository_homepage_url": "https://pypi.org/project/optional-dep", + "repository_download_url": "https://pypi.org/packages/source/o/optional-dep/optional-dep-2.0.0.tar.gz", + "api_data_url": "https://pypi.org/pypi/optional-dep/2.0.0/json", + "datasource_id": "pypi_poetry_lock", + "purl": "pkg:pypi/optional-dep@2.0.0" + }, + "extra_data": {} + } + ], + "repository_homepage_url": null, + "repository_download_url": null, + "api_data_url": null, + "datasource_id": "pypi_poetry_lock", + "purl": null + } +] \ No newline at end of file diff --git a/tests/packagedcode/data/pypi/poetry/optional/poetry.lock b/tests/packagedcode/data/pypi/poetry/optional/poetry.lock new file mode 100644 index 0000000000..8eafc2c716 --- /dev/null +++ b/tests/packagedcode/data/pypi/poetry/optional/poetry.lock @@ -0,0 +1,17 @@ +[[package]] +name = "required-dep" +version = "1.0.0" +description = "required dependency" +optional = false +python-versions = ">=3.8" + +[[package]] +name = "optional-dep" +version = "2.0.0" +description = "optional dependency" +optional = true +python-versions = ">=3.8" + +[metadata] +lock-version = "2.0" +python-versions = ">=3.8" \ No newline at end of file diff --git a/tests/packagedcode/test_pypi.py b/tests/packagedcode/test_pypi.py index 6f7a1650f3..dd911f34bf 100644 --- a/tests/packagedcode/test_pypi.py +++ b/tests/packagedcode/test_pypi.py @@ -399,37 +399,10 @@ def test_parse_poetry_lock_univers(self): self.check_packages_data(package, expected_loc, regen=REGEN_TEST_FIXTURES) def test_parse_poetry_lock_package_optional(self): - test_file = self.get_temp_file('poetry.lock') - with open(test_file, 'w') as lockfile: - lockfile.write('''\ -[[package]] -name = "required-dep" -version = "1.0.0" -description = "required dependency" -optional = false -python-versions = ">=3.8" - -[[package]] -name = "optional-dep" -version = "2.0.0" -description = "optional dependency" -optional = true -python-versions = ">=3.8" - -[metadata] -lock-version = "2.0" -python-versions = ">=3.8" -content-hash = "test" -''') - - package_data = list(pypi.PoetryLockHandler.parse(test_file))[0] - dependencies_by_purl = { - dependency['purl']: dependency - for dependency in package_data.dependencies - } - - assert dependencies_by_purl['pkg:pypi/required-dep@1.0.0']['is_optional'] is False - assert dependencies_by_purl['pkg:pypi/optional-dep@2.0.0']['is_optional'] is True + test_file = self.get_test_loc('pypi/poetry/optional/poetry.lock') + package = pypi.PoetryLockHandler.parse(test_file) + expected_loc = self.get_test_loc('pypi/poetry/optional-poetry.lock-expected.json') + self.check_packages_data(package, expected_loc, regen=REGEN_TEST_FIXTURES) def test_parse_pyproject_toml_poetry_univers(self): test_file = self.get_test_loc('pypi/poetry/univers/pyproject.toml')