diff --git a/.github/scripts/merge-licenses.py b/.github/scripts/merge-licenses.py new file mode 100644 index 0000000..d56d064 --- /dev/null +++ b/.github/scripts/merge-licenses.py @@ -0,0 +1,115 @@ +#!/usr/bin/env python3 +"""Merge pip-licenses output with a SBOM into one +THIRD_PARTY_LICENSES.json file. + +Python libary licences created with pip-licenses (incl. LicenseText, URL etc.), +all other system licences (apt/dpkg, apk, rpm, npm, gem, cargo, +go modules, ...) created via Syft SBOM. Python entries from +SBOM are discarded to avoid duplicates. + +Usage: + merge-licenses.py +""" +import json +import sys + +SOURCE_LABELS = { + "deb": "apt/dpkg (OS package)", + "apk": "apk (OS package)", + "rpm": "rpm (OS package)", + "npm": "npm", + "gemspec": "gem", + "go-module": "go module", + "rust-crate": "cargo", + "java-archive": "java (jar)", +} + +# Ecosystems that are ignored by the Syft SBOM because they are already +# covered by pip-licenses. +EXCLUDE_SYFT_TYPES = {"python"} + + +def load_pip_licenses(path): + with open(path) as f: + data = json.load(f) + entries = [] + for pkg in data: + entries.append({ + "Name": pkg.get("Name"), + "Version": pkg.get("Version"), + "License": pkg.get("License", "UNKNOWN"), + "Source": "python (pip)", + "URL": pkg.get("URL"), + "LicenseText": pkg.get("LicenseText"), + }) + return entries + + +def extract_license(licenses): + """Depending on the version, Syft’s ‘licence’ field takes different forms: + either a list of strings, or a list of objects with a 'value' field. + Both are handled here.""" + if not licenses: + return "UNKNOWN" + values = [] + for lic in licenses: + if isinstance(lic, dict): + values.append(lic.get("value") or lic.get("spdxExpression") or "UNKNOWN") + else: + values.append(str(lic)) + return ", ".join(sorted(set(values))) if values else "UNKNOWN" + + +def load_syft(path, exclude_types=EXCLUDE_SYFT_TYPES): + with open(path) as f: + data = json.load(f) + entries = [] + for artifact in data.get("artifacts", []): + pkg_type = artifact.get("type", "unknown") + if pkg_type in exclude_types: + continue + entries.append({ + "Name": artifact.get("name"), + "Version": artifact.get("version"), + "License": extract_license(artifact.get("licenses")), + "Source": SOURCE_LABELS.get(pkg_type, pkg_type), + }) + return entries + + +def dedupe(entries): + seen = set() + result = [] + for entry in entries: + key = (entry.get("Name"), entry.get("Version"), entry.get("Source")) + if key in seen: + continue + seen.add(key) + result.append(entry) + return result + + +def main(): + if len(sys.argv) != 4: + print( + "Usage: merge-licenses.py ", + file=sys.stderr, + ) + sys.exit(1) + + pip_path, syft_path, out_path = sys.argv[1:4] + + entries = [] + entries += load_pip_licenses(pip_path) + entries += load_syft(syft_path) + entries = dedupe(entries) + entries.sort(key=lambda e: (e.get("Source") or "", (e.get("Name") or "").lower())) + + with open(out_path, "w") as f: + json.dump(entries, f, indent=2) + + print(f"Wrote {len(entries)} license entries to {out_path}") + + +if __name__ == "__main__": + main() diff --git a/.github/workflows/third-party-licenses.yml b/.github/workflows/third-party-licenses.yml new file mode 100644 index 0000000..e3b0c1f --- /dev/null +++ b/.github/workflows/third-party-licenses.yml @@ -0,0 +1,223 @@ +name: Generate Third-Party Licenses + +on: + workflow_call: + inputs: + dockerfile: + description: "Path to the Dockerfile" + required: false + type: string + requirements: + description: "Path to the requirements.txt file" + required: false + type: string + pyproject: + description: "Path to the pyproject.toml file" + required: false + type: string + +jobs: + generate: + name: Generate THIRD_PARTY_LICENSES.json + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Validate inputs + id: validate + shell: bash + run: | + set -euo pipefail + + count=0 + source_type="" + source_path="" + + if [ -n "${{ inputs.dockerfile }}" ]; then + count=$((count + 1)) + source_type="dockerfile" + source_path="${{ inputs.dockerfile }}" + fi + if [ -n "${{ inputs.requirements }}" ]; then + count=$((count + 1)) + source_type="requirements" + source_path="${{ inputs.requirements }}" + fi + if [ -n "${{ inputs.pyproject }}" ]; then + count=$((count + 1)) + source_type="pyproject" + source_path="${{ inputs.pyproject }}" + fi + + if [ "$count" -eq 0 ]; then + echo "::error::One of the inputs 'dockerfile', 'requirements' or 'pyproject' has to be set." + exit 1 + fi + if [ "$count" -gt 1 ]; then + echo "::error::Only one of the inputs 'dockerfile', 'requirements' or 'pyproject' may be set." + exit 1 + fi + if [ ! -f "$source_path" ]; then + echo "::error::File '$source_path' not found." + exit 1 + fi + + echo "source-type=$source_type" >> "$GITHUB_OUTPUT" + echo "source-path=$source_path" >> "$GITHUB_OUTPUT" + + # --- Case 1: requirements.txt ------------------------------------- + - name: Licenses from requirements.txt + if: inputs.requirements != '' + shell: bash + run: | + set -euo pipefail + python -m venv .license-venv + source .license-venv/bin/activate + pip install --upgrade pip pip-licenses --quiet + pip install -r "${{ inputs.requirements }}" + pip-licenses --from=mixed --with-authors --with-urls --with-license-file --format=json --output-file=THIRD_PARTY_LICENSES.json + + # --- Case 2: pyproject.toml ---------------------------------------- + - name: Licenses from pyproject.toml + if: inputs.pyproject != '' + shell: bash + run: | + set -euo pipefail + python -m venv .license-venv + source .license-venv/bin/activate + pip install --upgrade pip pip-licenses --quiet + project_dir=$(dirname "${{ inputs.pyproject }}") + pip install "$project_dir" + pip-licenses --from=mixed --with-authors --with-urls --with-license-file --format=json --output-file=THIRD_PARTY_LICENSES.json + + # --- Case 3: Dockerfile --------------------------------------------- + # Build the image and executes pip-licenses inside the docker container + - name: Licenses from Dockerfile + if: inputs.dockerfile != '' + shell: bash + run: | + set -euo pipefail + docker build \ + -t license-scan-image \ + -f "${{ inputs.dockerfile }}" . + docker run --entrypoint sh --rm -v "$PWD":/output license-scan-image -c " + python -m pip install --upgrade pip pip-licenses && + pip-licenses --from=mixed --with-authors --with-urls --with-license-file --format=json --output-file=/output/pip-licenses.json + " + - name: Generate SBOM from Docker image + if: inputs.dockerfile != '' + uses: anchore/sbom-action@e22c389904149dbc22b58101806040fa8d37a610 # v0.24.0 + with: + image: license-scan-image + format: syft-json + output-file: sbom.json + upload-artifact: false + # upload-artifact: true + # artifact-name: sbom.json + - name: Merge licenses + if: inputs.dockerfile != '' + shell: bash + run: | + set -euo pipefail + wget -q https://raw.githubusercontent.com/mundialis/github-workflows/refs/heads/main/.github/scripts/merge-licenses.py + python3 merge-licenses.py pip-licenses.json sbom.json THIRD_PARTY_LICENSES.json + + - name: Validate Output + shell: bash + run: | + set -euo pipefail + if [ ! -s THIRD_PARTY_LICENSES.json ]; then + echo "::error::THIRD_PARTY_LICENSES.json not created or is empty." + exit 1 + fi + python3 -c "import json; json.load(open('THIRD_PARTY_LICENSES.json'))" + + - name: Upload THIRD_PARTY_LICENSES.json as artifact + uses: actions/upload-artifact@v4 + with: + name: third-party-license + path: THIRD_PARTY_LICENSES.json + + - name: Upload as release asset + uses: actions/upload-release-asset@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + upload_url: ${{ github.event.release.upload_url }} + asset_path: ./THIRD_PARTY_LICENSES.json + asset_name: THIRD_PARTY_LICENSES.json + asset_content_type: application/json + + license-scan: + runs-on: ubuntu-latest + needs: generate + continue-on-error: true + steps: + - uses: actions/checkout@v4 + + - name: Download THIRD_PARTY_LICENSES.json + uses: actions/download-artifact@v4 + with: + name: third-party-license + path: . + + - name: Scan THIRD_PARTY_LICENSES.json for UNKNOWN / GPL/AGPL/LGPL + run: | + set -e + + FILE="THIRD_PARTY_LICENSES.json" + + if [ ! -f "$FILE" ]; then + echo "::error file=$FILE::File $FILE not found." + exit 1 + fi + + # Python-Skript: scans JSON, raise warning und creates summary table + python3 - << 'PY' + import json + import sys + from pathlib import Path + + file_path = Path("THIRD_PARTY_LICENSES.json") + data = json.loads(file_path.read_text(encoding="utf-8")) + + # Expected format: List of objects with Name, Author, License, URL + entries = data if isinstance(data, list) else [] + + problem_entries = [] + + for e in entries: + category = None + if any(["UNKNOWN" in val.upper() for val in e.values()]): + category = "UNKNOWN" + license_val = e.get("License", "") + if license_val: + lic = str(license_val).strip() + is_gpl_like = any( + x in lic.upper() + for x in ["GPL", "AGPL", "LGPL"] + ) + category = "GPL/AGPL/LGPL" if is_gpl_like else category + if category is not None: + name = e.get("Name", "") + author = e.get("Author", "") + url = e.get("URL", "") + license_file = e.get("LicenseFile", "") + license_text = e.get("LicenseText", "") + + problem_entries.append({ + "Name": name, + "Author": author, + "License": lic, + "URL": url, + "Category": category, + }) + + # Warn-Annotationen for each entry + for e in problem_entries: + msg = f"{e['Category']}: {e['Name']} {e['Author']} ({e['License']})" + print(f"::warning title=License-Scan::{msg}") + if problem_entries: + sys.exit(1) + PY \ No newline at end of file diff --git a/README.md b/README.md index fa893ef..16e1a39 100644 --- a/README.md +++ b/README.md @@ -221,13 +221,66 @@ The calling job requires the following permissions: Optional inputs: - `fetch_depth`: Number of commits to fetch during checkout. Use `0` to fetch the full history and tags. Default: `1`. -- `fail-build`: Set to `true` if the workflow should fail when vulnerabilities above the severity +- `fail-build`: Set to `true` if the workflow should fail when vulnerabilities above the severity cutoff are found. Default: `false`. The generated Docker or Python SBOM is uploaded as a workflow artifact. The vulnerability results are available under **Security and quality** → **Code scanning**. +## Generate Third-Party-License list on release + +The workflow generates a json file with third-party-licenses as release asset. +The workflow can be used with a `Dockerfile`, a `requirements.txt` file, or a `pyproject.toml` file. + +You can use it e.g. like this: +```yaml +name: Generate Third-Party Licenses + +on: + release: + types: [published] + +jobs: + generate-third-party-licenses: + uses: mundialis/github-workflows/.github/workflows/third-party-licenses.yml@main + with: + # dockerfile: docker/actinia-core-alpine/Dockerfile + requirements: requirements.txt + # pyproject: pyproject.toml +``` + +Provide exactly one of the following inputs: + +- `dockerfile`: Path to the Dockerfile. +- `requirements`: Path to the requirements.txt file. +- `pyproject`: Path to the pyproject.toml file. + +The workflow contains two jobs: +1. `generate`: The generation of the THIRD_PARTY_LICENSES.json file +2. `license-scan`: A scan of the file where warnings will be given when a +license contains: + * "unknown": this should be fixed if possible, you can use following commands + to update the THIRD_PARTY_LICENSES.json + ```bash + # list all releases + gh release list + VERSION="0.0.0" + # view assets from release + gh release view ${VERSION} + # download assets + gh release download ${VERSION} + # TODO adjust THIRD_PARTY_LICENSES.json + # delete old THIRD_PARTY_LICENSES.json from release + gh release delete-asset ${VERSION} THIRD_PARTY_LICENSES.json + # upload adjusted THIRD_PARTY_LICENSES.json to release + gh release upload ${VERSION} THIRD_PARTY_LICENSES.json + ``` + * OR "GLP/AGPL/LPGL": these versions need to be checked for + compability with the other licenses, see: + * [Wiki licenses overview](https://en.wikipedia.org/wiki/Comparison_of_free_and_open-source_software_licenses#Approvals) + * [compatibility-checker](https://joinup.ec.europa.eu/collection/eupl/solution/joinup-licensing-assistant/jla-compatibility-checker) + # pre-commit ## Python Linting