diff --git a/.github/workflows/cla.yml b/.github/workflows/cla.yml index 0aa06e5..f6b10c0 100644 --- a/.github/workflows/cla.yml +++ b/.github/workflows/cla.yml @@ -1,4 +1,5 @@ -name: "CLA Assistant" +name: CLA + on: issue_comment: types: [created] @@ -10,88 +11,6 @@ permissions: statuses: write jobs: - CLAAssistant: - runs-on: ubuntu-24.04 - steps: - - name: Generate token from GitHub App - id: app-token - uses: actions/create-github-app-token@v3 - with: - app-id: ${{ secrets.APP_ID }} - private-key: ${{ secrets.APP_PRIVATE_KEY }} - owner: blacklanternsecurity - - - name: Check all committers against org and allowlist - id: cla-check - env: - GH_TOKEN: ${{ steps.app-token.outputs.token }} - run: | - if [ "${{ github.event_name }}" = "pull_request_target" ]; then - PR_NUM="${{ github.event.pull_request.number }}" - else - PR_NUM="${{ github.event.issue.number }}" - fi - - COMMITTERS=$(gh api "repos/${{ github.repository }}/pulls/$PR_NUM/commits" --paginate --jq '.[].author.login' | sort -u) - ALL_EXEMPT=true - - for LOGIN in $COMMITTERS; do - # treat commits with no associated GitHub login as non-exempt - if [ -z "$LOGIN" ] || [ "$LOGIN" = "null" ]; then - echo "Unknown committer (no GitHub login) — not exempt" - ALL_EXEMPT=false - break - fi - - EXEMPT=false - - # check if account type is Bot (GitHub App accounts) - AUTHOR_TYPE=$(gh api "users/${LOGIN}" --jq '.type' 2>/dev/null || echo "Unknown") - if [ "$AUTHOR_TYPE" = "Bot" ]; then - echo "$LOGIN is a Bot account — exempt" - EXEMPT=true - fi - - # check org membership - if [ "$EXEMPT" = "false" ]; then - if gh api "orgs/blacklanternsecurity/members/$LOGIN" > /dev/null 2>&1; then - echo "$LOGIN is an org member — exempt" - EXEMPT=true - fi - fi - - if [ "$EXEMPT" = "false" ]; then - echo "$LOGIN is not exempt — CLA required" - ALL_EXEMPT=false - break - fi - done - - echo "all_exempt=$ALL_EXEMPT" >> "$GITHUB_OUTPUT" - - - name: Skip CLA when all committers are exempt - if: steps.cla-check.outputs.all_exempt == 'true' && github.event_name == 'pull_request_target' - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - gh api --method POST "repos/${{ github.repository }}/statuses/${{ github.event.pull_request.head.sha }}" \ - -f state=success \ - -f context="CLAAssistant" \ - -f description="CLA check skipped — all committers are org members or bots" - - - name: "CLA Assistant" - if: | - (steps.cla-check.outputs.all_exempt != 'true') && - ((github.event.comment.body == 'recheck' || github.event.comment.body == 'I have read the CLA Document and I hereby sign the CLA') || github.event_name == 'pull_request_target') - uses: contributor-assistant/github-action@v2.6.1 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - PERSONAL_ACCESS_TOKEN: ${{ steps.app-token.outputs.token }} - with: - path-to-signatures: "signatures/version1/cla.json" - path-to-document: "https://github.com/blacklanternsecurity/CLA/blob/main/ICLA.md" - branch: "main" - allowlist: "dependabot[bot],github-actions[bot],renovate[bot]" - remote-organization-name: "blacklanternsecurity" - remote-repository-name: "CLA" - lock-pullrequest-aftermerge: "false" + cla: + uses: blacklanternsecurity/CLA/.github/workflows/cla-reusable.yml@4f85d3c525483f0f846ef4384a8ece24a3c74375 # 2026-09-29 + secrets: inherit diff --git a/.github/workflows/daily-update.yml b/.github/workflows/daily-update.yml index 192b0b8..194b50b 100644 --- a/.github/workflows/daily-update.yml +++ b/.github/workflows/daily-update.yml @@ -6,6 +6,9 @@ on: - cron: '0 6 * * *' workflow_dispatch: # Allow manual trigger +permissions: + contents: read + # Prevent concurrent runs from racing to commit/merge # cancel-in-progress: false means new runs wait instead of canceling in-progress ones concurrency: @@ -21,24 +24,19 @@ jobs: steps: - name: Generate app token id: app-token - uses: actions/create-github-app-token@v3 + uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 # v3.2.0 with: app-id: ${{ secrets.CLOUDCHECK_APP_ID }} private-key: ${{ secrets.CLOUDCHECK_APP_PRIVATE_KEY }} - name: Checkout stable branch - uses: actions/checkout@v7 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: ref: stable token: ${{ steps.app-token.outputs.token }} - - name: Set up Python - uses: actions/setup-python@v7 - with: - python-version: "3.12" - - name: Set up uv - uses: astral-sh/setup-uv@v7 + uses: astral-sh/setup-uv@c18668ad3cf93ea998bef934396af7bb5c839dc7 # v10.2.0 - name: Install dependencies run: uv sync diff --git a/.github/workflows/docker-tests.yml b/.github/workflows/docker-tests.yml deleted file mode 100644 index 0625a6f..0000000 --- a/.github/workflows/docker-tests.yml +++ /dev/null @@ -1,97 +0,0 @@ -name: Docker Tests - -on: - push: - branches: [ stable ] - pull_request: - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -jobs: - docker-test: - runs-on: ubuntu-24.04 - - steps: - - uses: actions/checkout@v7 - - - name: Set up Python - uses: actions/setup-python@v7 - with: - python-version: "3.11" - - - name: Install dependencies - run: | - pip install requests - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v4 - - - name: Run Docker tests - run: | - python test_docker.py - - docker-publish: - runs-on: ubuntu-24.04 - needs: docker-test - if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/stable' }} - - steps: - - name: Checkout code - uses: actions/checkout@v7 - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v4 - - - name: Log in to Docker Hub - uses: docker/login-action@v4 - with: - username: blacklanternsecurity - password: ${{ secrets.DOCKER_TOKEN }} - - - name: Extract metadata - id: meta - uses: docker/metadata-action@v6 - with: - images: blacklanternsecurity/cloudcheck - tags: | - type=ref,event=branch - type=ref,event=pr - type=sha,prefix=sha- - type=raw,value=latest,enable={{is_default_branch}} - type=raw,value=v11 - type=raw,value=v11.2 - type=raw,value=v11.2.0 - - - name: Build and push Docker image - uses: docker/build-push-action@v7 - with: - context: . - file: ./Dockerfile - push: true - tags: ${{ steps.meta.outputs.tags }} - labels: ${{ steps.meta.outputs.labels }} - cache-from: type=gha - cache-to: type=gha,mode=max - - - name: Clean up old Docker Hub tags (up to 50 most recent tags plus 'latest') - run: | - # Install jq for JSON processing - sudo apt-get update && sudo apt-get install -y jq - - echo "Cleaning up blacklanternsecurity/cloudcheck tags..." - - tags_response=$(curl -s -H "Authorization: Bearer ${{ secrets.DOCKER_TOKEN }}" \ - "https://hub.docker.com/v2/repositories/blacklanternsecurity/cloudcheck/tags/?page_size=100") - - tags_to_delete=$(echo "$tags_response" | jq -r '.results[] | select(.name != "latest") | [.last_updated, .name] | @tsv' | \ - sort -r | tail -n +11 | cut -f2) - - for tag in $tags_to_delete; do - echo "Deleting blacklanternsecurity/cloudcheck tag: $tag" - curl -X DELETE -H "Authorization: Bearer ${{ secrets.DOCKER_TOKEN }}" \ - "https://hub.docker.com/v2/repositories/blacklanternsecurity/cloudcheck/tags/$tag/" - done - - echo "Cleanup completed for blacklanternsecurity/cloudcheck. Kept 50 most recent tags plus 'latest'." diff --git a/.github/workflows/pipeline-tests.yml b/.github/workflows/pipeline-tests.yml deleted file mode 100644 index 0b9476f..0000000 --- a/.github/workflows/pipeline-tests.yml +++ /dev/null @@ -1,33 +0,0 @@ -name: Pipeline Tests - -on: - push: - branches: [ stable ] - pull_request: - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -jobs: - test: - runs-on: ubuntu-24.04 - - steps: - - uses: actions/checkout@v7 - - - name: Set up Python - uses: actions/setup-python@v7 - with: - python-version: 3.x - - - name: Set up uv - uses: astral-sh/setup-uv@v7 - - - name: Install dependencies - run: | - uv sync - - - name: Run tests - run: | - uv run pytest cloudcheck_update/test_cloudcheck_update.py -v diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml new file mode 100644 index 0000000..80a06bf --- /dev/null +++ b/.github/workflows/publish.yml @@ -0,0 +1,76 @@ +name: Publish + +on: + push: + tags: ["v*"] + +permissions: + contents: read + +concurrency: + group: publish-${{ github.ref }} + cancel-in-progress: false + +jobs: + check: + uses: blacklanternsecurity/CLA/.github/workflows/release-check.yml@e532142ee9e7322888f6edc80d9a89e2f8c0d96d # feat/shared-workflows + + tests: + needs: check + uses: ./.github/workflows/tests.yml + + wheels: + needs: tests + uses: blacklanternsecurity/CLA/.github/workflows/maturin-wheels.yml@e532142ee9e7322888f6edc80d9a89e2f8c0d96d # feat/shared-workflows + with: + # No s390x wheel has ever been published for cloudcheck. + targets: '{"linux": ["x86_64", "x86", "aarch64", "armv7", "ppc64le"], "musllinux": ["x86_64", "x86", "aarch64", "armv7"], "windows": ["x64", "x86"], "macos": ["x86_64", "aarch64"]}' + before-script-linux: scripts/wheel-container-setup.sh + + crates: + needs: wheels + uses: blacklanternsecurity/CLA/.github/workflows/crates.yml@e532142ee9e7322888f6edc80d9a89e2f8c0d96d # feat/shared-workflows + secrets: + CARGO_REGISTRY_TOKEN: ${{ secrets.CARGO_REGISTRY_TOKEN }} + + # Trusted publishing only matches a top-level job, so PyPI cannot move into a reusable workflow. + pypi: + needs: wheels + runs-on: ubuntu-24.04 + environment: pypi + permissions: + id-token: write + attestations: write + steps: + - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + pattern: ${{ needs.wheels.outputs.artifacts }} + path: dist + merge-multiple: true + - uses: actions/attest-build-provenance@4d101475d8b20a2381f78447822ac1eab6504dd8 # v4.2.2 + with: + subject-path: dist/* + - uses: pypa/gh-action-pypi-publish@dc37677b2e1c63e2034f94d8a5b11f265b73ba33 # v1.14.2 + + docker: + needs: [check, wheels] + uses: blacklanternsecurity/CLA/.github/workflows/docker.yml@e532142ee9e7322888f6edc80d9a89e2f8c0d96d # feat/shared-workflows + secrets: inherit + with: + image: ${{ github.repository }} + version: ${{ needs.check.outputs.version }} + prerelease: ${{ needs.check.outputs.prerelease }} + major: ${{ needs.check.outputs.major }} + minor: ${{ needs.check.outputs.minor }} + + release: + needs: [check, wheels, crates, pypi, docker] + permissions: + contents: write + uses: blacklanternsecurity/CLA/.github/workflows/publish.yml@e532142ee9e7322888f6edc80d9a89e2f8c0d96d # feat/shared-workflows + with: + version: ${{ needs.check.outputs.version }} + prerelease: ${{ needs.check.outputs.prerelease }} + images: ${{ needs.docker.outputs.images }} + artifacts: ${{ needs.wheels.outputs.artifacts }} + assets: dist/* diff --git a/.github/workflows/python-tests.yml b/.github/workflows/python-tests.yml deleted file mode 100644 index 956195b..0000000 --- a/.github/workflows/python-tests.yml +++ /dev/null @@ -1,297 +0,0 @@ -name: Python Tests - -on: - push: - branches: - - stable - pull_request: - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -jobs: - test: - runs-on: ubuntu-24.04 - strategy: - matrix: - python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"] - - steps: - - uses: actions/checkout@v7 - - - name: Set up Python - uses: actions/setup-python@v7 - with: - python-version: ${{ matrix.python-version }} - - - name: Set up uv - uses: astral-sh/setup-uv@v7 - - - name: Install Rust toolchain - uses: actions-rust-lang/setup-rust-toolchain@v2 - - - name: Install dependencies - run: | - uv sync - - - name: Build Python bindings - run: | - uv run maturin develop --release - - - name: Run Python tests - run: | - uv run pytest test_cloudcheck.py -v - - publish: - runs-on: ubuntu-24.04 - needs: test - if: github.event_name == 'push' && github.ref == 'refs/heads/stable' - permissions: - contents: write - steps: - - name: Generate app token - id: app-token - uses: actions/create-github-app-token@v3 - with: - app-id: ${{ secrets.CLOUDCHECK_APP_ID }} - private-key: ${{ secrets.CLOUDCHECK_APP_PRIVATE_KEY }} - - uses: actions/checkout@v7 - with: - fetch-depth: 0 - token: ${{ steps.app-token.outputs.token }} - - name: Set up Python - uses: actions/setup-python@v7 - with: - python-version: "3.x" - - name: Set up Rust - uses: dtolnay/rust-toolchain@6bed0761d98439e5a578e2877258200ad565ba87 # 2026-09-03, no release tag - - name: Set up uv - uses: astral-sh/setup-uv@v7 - - name: Get current version - id: get_version - run: | - VERSION=$(grep '^version' Cargo.toml | head -1 | sed 's/.*"\(.*\)".*/\1/') - echo "VERSION=v${VERSION}" >> $GITHUB_OUTPUT - - name: Check for version change - id: version_check - run: | - git fetch --tags - LATEST_TAG=$(git describe --tags $(git rev-list --tags --max-count=1) 2>/dev/null || echo "none") - CURRENT="${{ steps.get_version.outputs.VERSION }}" - if [ "$LATEST_TAG" = "$CURRENT" ]; then - echo "changed=false" >> $GITHUB_OUTPUT - else - echo "changed=true" >> $GITHUB_OUTPUT - fi - - name: Build PyPi package - run: uv run maturin build --release --out dist - - name: Publish PyPi package - run: uv run maturin publish --skip-existing --username __token__ --password ${{ secrets.PYPI_TOKEN }} - - name: Tag release - if: steps.version_check.outputs.changed == 'true' - run: | - git config user.name "github-actions[bot]" - git config user.email "github-actions[bot]@users.noreply.github.com" - git tag -a "${{ steps.get_version.outputs.VERSION }}" -m "Release ${{ steps.get_version.outputs.VERSION }}" - git push origin "refs/tags/${{ steps.get_version.outputs.VERSION }}" - linux: - runs-on: ${{ matrix.platform.runner }} - needs: publish - if: github.event_name == 'push' && github.ref == 'refs/heads/stable' - strategy: - matrix: - platform: - - runner: ubuntu-22.04 - target: x86_64 - - runner: ubuntu-22.04 - target: x86 - - runner: ubuntu-22.04 - target: aarch64 - - runner: ubuntu-22.04 - target: armv7 - # get rekt ibm - # - runner: ubuntu-22.04 - # target: s390x - - runner: ubuntu-22.04 - target: ppc64le - steps: - - uses: actions/checkout@v7 - - uses: actions/setup-python@v7 - with: - python-version: 3.x - - name: Build wheels - uses: PyO3/maturin-action@v1 - with: - target: ${{ matrix.platform.target }} - args: --release --out dist --find-interpreter - sccache: false - manylinux: auto - before-script-linux: | - if command -v apt-get >/dev/null 2>&1; then - # Debian-based - sudo apt-get update - sudo apt-get install -y pkg-config libssl-dev perl make gcc g++ binutils - elif command -v yum >/dev/null 2>&1; then - # CentOS-based - yum update -y - yum install -y openssl openssl-devel perl perl-core make gcc gcc-c++ binutils - else - echo "Error: Neither apt-get nor yum found" - exit 1 - fi - - name: Upload wheels - uses: actions/upload-artifact@v7 - with: - name: wheels-linux-${{ matrix.platform.target }} - path: dist - - musllinux: - runs-on: ${{ matrix.platform.runner }} - needs: publish - if: github.event_name == 'push' && github.ref == 'refs/heads/stable' - strategy: - matrix: - platform: - - runner: ubuntu-22.04 - target: x86_64 - - runner: ubuntu-22.04 - target: x86 - - runner: ubuntu-22.04 - target: aarch64 - - runner: ubuntu-22.04 - target: armv7 - steps: - - uses: actions/checkout@v7 - - uses: actions/setup-python@v7 - with: - python-version: 3.x - - name: Build wheels - uses: PyO3/maturin-action@v1 - with: - target: ${{ matrix.platform.target }} - args: --release --out dist --find-interpreter - sccache: false - manylinux: musllinux_1_2 - before-script-linux: | - if command -v apk >/dev/null 2>&1; then - # Alpine (musllinux) - apk add --no-cache openssl-dev pkgconfig perl make musl-dev - elif command -v apt-get >/dev/null 2>&1; then - # Debian-based - sudo apt-get update - sudo apt-get install -y pkg-config libssl-dev perl make gcc g++ binutils - elif command -v yum >/dev/null 2>&1; then - # CentOS-based - yum update -y - yum install -y openssl openssl-devel perl perl-core make gcc gcc-c++ binutils - else - echo "Error: No supported package manager found" - exit 1 - fi - - - name: Upload wheels - uses: actions/upload-artifact@v7 - with: - name: wheels-musllinux-${{ matrix.platform.target }} - path: dist - - windows: - runs-on: ${{ matrix.platform.runner }} - needs: publish - if: github.event_name == 'push' && github.ref == 'refs/heads/stable' - strategy: - matrix: - platform: - - runner: windows-latest - target: x64 - - runner: windows-latest - target: x86 - steps: - - uses: actions/checkout@v7 - - uses: actions/setup-python@v7 - with: - python-version: 3.x - architecture: ${{ matrix.platform.target }} - - name: Build wheels - uses: PyO3/maturin-action@v1 - with: - target: ${{ matrix.platform.target }} - args: --release --out dist --find-interpreter - sccache: false - - name: Upload wheels - uses: actions/upload-artifact@v7 - with: - name: wheels-windows-${{ matrix.platform.target }} - path: dist - - macos: - runs-on: ${{ matrix.platform.runner }} - needs: publish - if: github.event_name == 'push' && github.ref == 'refs/heads/stable' - strategy: - matrix: - platform: - - runner: macos-15-intel - target: x86_64 - - runner: macos-latest - target: aarch64 - steps: - - uses: actions/checkout@v7 - - uses: actions/setup-python@v7 - with: - python-version: 3.x - - name: Build wheels - uses: PyO3/maturin-action@v1 - with: - target: ${{ matrix.platform.target }} - args: --release --out dist --find-interpreter - sccache: false - - name: Upload wheels - uses: actions/upload-artifact@v7 - with: - name: wheels-macos-${{ matrix.platform.target }} - path: dist - - sdist: - runs-on: ubuntu-24.04 - needs: publish - if: github.event_name == 'push' && github.ref == 'refs/heads/stable' - steps: - - uses: actions/checkout@v7 - - name: Build sdist - uses: PyO3/maturin-action@v1 - with: - command: sdist - args: --out dist - - name: Upload sdist - uses: actions/upload-artifact@v7 - with: - name: wheels-sdist - path: dist - - release: - name: Release - runs-on: ubuntu-24.04 - needs: [linux, musllinux, windows, macos, sdist] - if: github.event_name == 'push' && github.ref == 'refs/heads/stable' - permissions: - # Use to sign the release artifacts - id-token: write - # Used to upload release artifacts - contents: write - # Used to generate artifact attestation - attestations: write - steps: - - uses: actions/download-artifact@v8 - - name: Generate artifact attestation - uses: actions/attest-build-provenance@v4 - with: - subject-path: 'wheels-*/*' - - name: Publish to PyPI - uses: PyO3/maturin-action@v1 - env: - MATURIN_PYPI_TOKEN: ${{ secrets.PYPI_TOKEN }} - with: - command: upload - args: --non-interactive --skip-existing wheels-*/* \ No newline at end of file diff --git a/.github/workflows/rust-tests.yml b/.github/workflows/rust-tests.yml deleted file mode 100644 index b45087b..0000000 --- a/.github/workflows/rust-tests.yml +++ /dev/null @@ -1,60 +0,0 @@ -name: Rust tests - -on: - push: - branches: [ stable ] - pull_request: - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -jobs: - test: - runs-on: ubuntu-24.04 - - steps: - - uses: actions/checkout@v7 - - - name: Install Rust toolchain - uses: actions-rust-lang/setup-rust-toolchain@v2 - - - name: Cache Rust dependencies - uses: actions/cache@v6 - with: - path: | - ~/.cargo/registry - ~/.cargo/git - target - key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }} - restore-keys: | - ${{ runner.os }}-cargo- - - - name: Check formatting - run: cargo fmt --all -- --check - - - name: Run clippy - run: cargo clippy --all-targets --all-features -- -D warnings - - - name: Run tests - run: cargo test --verbose --all-features - - publish: - needs: test - runs-on: ubuntu-24.04 - if: github.event_name == 'push' && github.ref == 'refs/heads/stable' - - steps: - - uses: actions/checkout@v7 - - - name: Install Rust toolchain - uses: actions-rust-lang/setup-rust-toolchain@v2 - - - name: Publish to crates.io - run: | - VERSION=$(cargo metadata --format-version 1 --no-deps | jq -r '.packages[0].version') - if cargo search cloudcheck --limit 1 | head -1 | grep -q "$VERSION"; then - echo "Version $VERSION already exists on crates.io, skipping publish" - else - cargo publish --token ${{ secrets.CARGO_REGISTRY_TOKEN }} - fi diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml new file mode 100644 index 0000000..52496c9 --- /dev/null +++ b/.github/workflows/tests.yml @@ -0,0 +1,34 @@ +name: Tests + +on: + pull_request: + push: + branches: [stable] + workflow_call: + +concurrency: + group: tests-${{ github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +permissions: + contents: read + +jobs: + rust: + uses: blacklanternsecurity/CLA/.github/workflows/rust-tests.yml@e532142ee9e7322888f6edc80d9a89e2f8c0d96d # feat/shared-workflows + with: + cargo-args: --all-features --locked + + python: + uses: blacklanternsecurity/CLA/.github/workflows/python-tests.yml@e532142ee9e7322888f6edc80d9a89e2f8c0d96d # feat/shared-workflows + with: + pytest-args: test_cloudcheck.py cloudcheck_update -v + + docker: + runs-on: ubuntu-24.04 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: astral-sh/setup-uv@c18668ad3cf93ea998bef934396af7bb5c839dc7 # v10.2.0 + - run: uv run --with requests python test_docker.py diff --git a/README.md b/README.md index 59c4ea2..243913b 100644 --- a/README.md +++ b/README.md @@ -89,10 +89,12 @@ pip install cloudcheck import asyncio from cloudcheck import CloudCheck + async def main(): cloudcheck = CloudCheck() results = await cloudcheck.lookup("8.8.8.8") - print(results) # [{'name': 'Google', 'tags': ['cloud']}] + print(results) # [{'name': 'Google', 'tags': ['cloud']}] + asyncio.run(main()) ``` @@ -103,6 +105,7 @@ asyncio.run(main()) import asyncio from cloudcheck import CloudCheck, CloudCheckError + async def main(): cloudcheck = CloudCheck() try: @@ -111,6 +114,7 @@ async def main(): except CloudCheckError as e: print(f"Error: {e}") + asyncio.run(main()) ``` @@ -120,17 +124,19 @@ asyncio.run(main()) import asyncio from cloudcheck import CloudCheck + async def main(): # Custom configuration cloudcheck = CloudCheck( signature_url="https://example.com/custom.json", # Custom signature URL - max_retries=5, # Max retry attempts (default: 10) - retry_delay_seconds=2, # Delay between retries (default: 1) - force_refresh=True # Force fresh fetch (default: False) + max_retries=5, # Max retry attempts (default: 10) + retry_delay_seconds=2, # Delay between retries (default: 1) + force_refresh=True, # Force fresh fetch (default: False) ) results = await cloudcheck.lookup("8.8.8.8") print(results) + asyncio.run(main()) ``` @@ -324,3 +330,7 @@ cargo fmt --all cargo clippy --all-targets --all-features -- -D warnings ``` + +## Releasing + +Releases are cut by pushing a `vMAJOR.MINOR.PATCH` (or `vMAJOR.MINOR.PATCH-rc.N`) tag with the org's [release.sh](https://github.com/blacklanternsecurity/CLA/blob/e532142ee9e7322888f6edc80d9a89e2f8c0d96d/scripts/release.sh), after the version bump is merged. The tag must equal the manifest version. `publish.yml` then tests, builds, publishes, and creates the GitHub release with its SBOM in the same run. diff --git a/pyproject.toml b/pyproject.toml index f6920d7..6537870 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "cloudcheck" -version = "11.2.0" +dynamic = ["version"] description = "Detailed database of cloud providers. Instantly look up a domain or IP address" readme = "README.md" requires-python = ">=3.10" @@ -24,3 +24,6 @@ dev = [ no-default-features = true features = ["py"] module-name = "cloudcheck" + +[tool.ruff.lint] +select = ["E4", "E7", "E9", "F"] diff --git a/scripts/wheel-container-setup.sh b/scripts/wheel-container-setup.sh new file mode 100644 index 0000000..8a83880 --- /dev/null +++ b/scripts/wheel-container-setup.sh @@ -0,0 +1,8 @@ +# Sourced by maturin-action inside the manylinux and musllinux containers; vendored OpenSSL needs perl and make. +if command -v apk >/dev/null 2>&1; then + apk add --no-cache perl make musl-dev +elif command -v yum >/dev/null 2>&1; then + yum install -y perl-core make gcc +elif command -v apt-get >/dev/null 2>&1; then + apt-get update && apt-get install -y perl make gcc +fi diff --git a/src/lib.rs b/src/lib.rs index 11330f8..b8d0e08 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -35,7 +35,7 @@ struct ProviderData { long_description: String, } -type ProvidersMap = HashMap>; +type ProvidersMap = HashMap>>; type Error = Box; #[derive(Clone)] @@ -294,66 +294,84 @@ impl CloudCheck { } } - /// Parses JSON and builds the radix tree and providers map. - /// For each provider, inserts all CIDRs and domains into the radix tree, - /// normalizing them in the process. Maps normalized values to provider lists. + /// Lower covers more. IPs and domains live in separate trees, so their scores never compare. + fn entry_breadth(entry: &str) -> u32 { + if let Some((addr, prefix)) = entry.split_once('/') + && addr.parse::().is_ok() + && let Ok(prefix_len) = prefix.parse::() + { + return prefix_len; + } + if let Ok(addr) = entry.parse::() { + return if addr.is_ipv6() { 128 } else { 32 }; + } + entry + .trim_matches('.') + .split('.') + .filter(|label| !label.is_empty()) + .count() as u32 + } + + /// `Normal` mode, not `Acl`: `Acl` collapses nested entries into their container, + /// which leaked a tenant's single bucket onto every host of the container. + /// Broadest first, so each entry inherits its ancestor's finished provider list. fn build_data_structures(json_data: &str) -> Result<(RadixTarget, ProvidersMap), Error> { let providers_data: HashMap = serde_json::from_str(json_data)?; - let mut radix = RadixTarget::new(&[], ScopeMode::Acl)?; + let mut owners: HashMap>> = HashMap::new(); + for provider in providers_data.values() { + let cloud_provider = Arc::new(CloudProvider { + name: provider.name.clone(), + tags: provider.tags.clone(), + short_description: provider.short_description.clone(), + long_description: provider.long_description.clone(), + }); + for entry in provider.cidrs.iter().chain(provider.domains.iter()) { + let entry_owners = owners.entry(entry.clone()).or_default(); + if !entry_owners.iter().any(|p| p.name == cloud_provider.name) { + entry_owners.push(Arc::clone(&cloud_provider)); + } + } + } + + let mut sorted: Vec<(String, Vec>)> = owners.into_iter().collect(); + sorted.sort_by(|(a, _), (b, _)| { + Self::entry_breadth(a) + .cmp(&Self::entry_breadth(b)) + .then_with(|| a.cmp(b)) + }); + + let mut radix = RadixTarget::new(&[], ScopeMode::Normal)?; let mut providers_map: ProvidersMap = HashMap::new(); - // here, we iterate twice to ensure similar domains get grouped together regardless of insert order - // this exists for a specific reason. a real world example is when github has a domain of - // blob.core.widnows.net, and azure has a domain of windows.net. if blob.core.windows.net gets inserted first, - // it gets blown away when windows.net is inserted. - // iterating twice ensures that on the second pass, a .get() for blob.core.windows.net will return the - // parent domain, allowing us to nest both cloud providers under the same key of windows.net. - for _ in 0..2 { - for provider in providers_data.values() { - let cloud_provider = CloudProvider { - name: provider.name.clone(), - tags: provider.tags.clone(), - short_description: provider.short_description.clone(), - long_description: provider.long_description.clone(), - }; - - // Insert all CIDRs for this provider - for cidr in &provider.cidrs { - let normalized = match radix.get(cidr) { - Some(n) => n, - None => match radix.insert(cidr) { - Ok(Some(n)) => n, - Ok(None) => continue, - Err(e) => { - log::warn!("Error inserting CIDR '{}': {}", cidr, e); - continue; - } - }, - }; - let providers_list = providers_map.entry(normalized.clone()).or_default(); - if !providers_list.iter().any(|p| p.name == cloud_provider.name) { - providers_list.push(cloud_provider.clone()); - } - } + for (entry, entry_owners) in sorted { + let inherited: Vec> = radix + .get(&entry) + .and_then(|ancestor| providers_map.get(&ancestor).cloned()) + .unwrap_or_default(); + + let novel: Vec> = entry_owners + .iter() + .filter(|p| !inherited.iter().any(|i| i.name == p.name)) + .cloned() + .collect(); + + // Same answer as its container, so the node would only cost memory. + if novel.is_empty() { + continue; + } - // Insert all domains for this provider - for domain in &provider.domains { - let normalized = match radix.get(domain) { - Some(n) => n, - None => match radix.insert(domain) { - Ok(Some(n)) => n, - Ok(None) => continue, - Err(e) => { - log::warn!("Error inserting domain '{}': {}", domain, e); - continue; - } - }, - }; - let providers_list = providers_map.entry(normalized.clone()).or_default(); - if !providers_list.iter().any(|p| p.name == cloud_provider.name) { - providers_list.push(cloud_provider.clone()); - } + let mut merged = inherited; + merged.extend(novel); + + match radix.insert(&entry) { + Ok(Some(normalized)) => { + providers_map.insert(normalized, merged); + } + Ok(None) => continue, + Err(e) => { + log::warn!("Error inserting entry '{}': {}", entry, e); + continue; } } } @@ -455,7 +473,10 @@ impl CloudCheck { if let Some(normalized) = radix.get(target) { debug!("Found normalized target: {} for {}", normalized, target); - let result = providers.get(&normalized).cloned().unwrap_or_default(); + let result: Vec = providers + .get(&normalized) + .map(|found| found.iter().map(|p| (**p).clone()).collect()) + .unwrap_or_default(); debug!("Returning {} providers", result.len()); Ok(result) } else { @@ -506,23 +527,155 @@ mod tests { ); } + async fn names_for(target: &str) -> Vec { + let cloudcheck = CloudCheck::new(); + let results = cloudcheck.lookup(target).await.unwrap(); + results.iter().map(|p| p.name.clone()).collect() + } + #[tokio::test] async fn test_lookup_windows_blob_domain() { - let cloudcheck = CloudCheck::new(); - let results = cloudcheck - .lookup("asdf.blob.core.windows.net") - .await - .unwrap(); - let names: Vec = results.iter().map(|p| p.name.clone()).collect(); + let names = names_for("asdf.blob.core.windows.net").await; assert!( - names.contains(&"GitHub".to_string()), - "Expected GitHub in results: {:?}", + names.contains(&"Microsoft".to_string()), + "Expected Microsoft in results: {:?}", names ); assert!( - names.contains(&"Microsoft".to_string()), - "Expected Microsoft in results: {:?}", + !names.contains(&"GitHub".to_string()), + "GitHub owns sibling hosts under windows.net, not this one: {:?}", + names + ); + } + + #[tokio::test] + async fn test_lookup_github_owned_blob_host() { + let names = names_for("copilotprodattachments.blob.core.windows.net").await; + for expected in ["GitHub", "Microsoft"] { + assert!( + names.contains(&expected.to_string()), + "Expected {} in results: {:?}", + expected, + names + ); + } + } + + #[tokio::test] + async fn test_lookup_amazonaws_no_tenant_leak() { + let names = names_for("foo.s3.amazonaws.com").await; + assert!( + names.contains(&"Amazon".to_string()), + "Expected Amazon in results: {:?}", + names + ); + for leaked in ["GitHub", "HPE", "Microsoft"] { + assert!( + !names.contains(&leaked.to_string()), + "{} leaked onto an unrelated amazonaws.com host: {:?}", + leaked, + names + ); + } + } + + #[tokio::test] + async fn test_lookup_tenant_bucket_keeps_both() { + let names = names_for("hpefonts.s3.amazonaws.com").await; + for expected in ["Amazon", "HPE"] { + assert!( + names.contains(&expected.to_string()), + "Expected {} in results: {:?}", + expected, + names + ); + } + } + + /// A single QUIC.cloud /32 inside AWS made bbot's portfilter treat whole AWS ranges as CDN. + #[tokio::test] + async fn test_lookup_aws_ip_no_cdn_leak() { + let names = names_for("18.195.165.195").await; + assert!( + names.contains(&"Amazon".to_string()), + "Expected Amazon in results: {:?}", names ); + assert!( + !names.contains(&"Quiccloud".to_string()), + "Quiccloud leaked onto an unrelated AWS address: {:?}", + names + ); + } + + #[test] + fn test_entry_breadth_ordering() { + assert!(CloudCheck::entry_breadth("10.0.0.0/8") < CloudCheck::entry_breadth("10.1.2.0/24")); + assert_eq!(CloudCheck::entry_breadth("1.2.3.4"), 32); + assert_eq!(CloudCheck::entry_breadth("::1"), 128); + assert!( + CloudCheck::entry_breadth("amazonaws.com") + < CloudCheck::entry_breadth("foo.s3.amazonaws.com") + ); + assert_eq!(CloudCheck::entry_breadth("example.com."), 2); + } + + const NESTED: &str = r#"{ + "bigcloud": {"name": "BigCloud", "tags": ["cloud"], "cidrs": ["10.0.0.0/8"], "domains": ["bigcloud.example"]}, + "tinycdn": {"name": "TinyCdn", "tags": ["cdn"], "cidrs": ["10.1.2.3/32"], "domains": ["node.bigcloud.example"]}, + "elsewhere": {"name": "Elsewhere", "tags": ["cloud"], "cidrs": ["192.0.2.0/24"], "domains": []} + }"#; + + const CHAIN: &str = r#"{ + "a": {"name": "A", "tags": ["cloud"], "cidrs": ["10.0.0.0/8"], "domains": ["a.example"]}, + "b": {"name": "B", "tags": ["cloud"], "cidrs": ["10.1.0.0/16"], "domains": ["b.a.example"]}, + "c": {"name": "C", "tags": ["cloud"], "cidrs": ["10.1.2.0/24"], "domains": ["c.b.a.example"]} + }"#; + + fn names(target: &str) -> Vec { + names_in(NESTED, target) + } + + fn names_in(fixture: &str, target: &str) -> Vec { + let (radix, providers) = CloudCheck::build_data_structures(fixture).unwrap(); + let mut names: Vec = radix + .get(target) + .and_then(|entry| providers.get(&entry).cloned()) + .unwrap_or_default() + .iter() + .map(|p| p.name.clone()) + .collect(); + names.sort(); + names + } + + #[test] + fn test_nested_provider_does_not_leak_to_container() { + assert_eq!(names("10.9.9.9"), ["BigCloud"]); + assert_eq!(names("10.1.2.4"), ["BigCloud"]); + assert_eq!(names("www.bigcloud.example"), ["BigCloud"]); + assert_eq!(names("bigcloud.example"), ["BigCloud"]); + } + + #[test] + fn test_nested_provider_matches_its_own_entry_and_ancestors() { + assert_eq!(names("10.1.2.3"), ["BigCloud", "TinyCdn"]); + assert_eq!(names("node.bigcloud.example"), ["BigCloud", "TinyCdn"]); + assert_eq!(names("a.node.bigcloud.example"), ["BigCloud", "TinyCdn"]); + } + + #[test] + fn test_unrelated_and_unknown_targets() { + assert_eq!(names("192.0.2.5"), ["Elsewhere"]); + assert!(names("203.0.113.1").is_empty()); + assert!(names("unknown.test").is_empty()); + } + + #[test] + fn test_inheritance_chains_through_intermediate_entries() { + assert_eq!(names_in(CHAIN, "10.9.9.9"), ["A"]); + assert_eq!(names_in(CHAIN, "10.1.9.9"), ["A", "B"]); + assert_eq!(names_in(CHAIN, "10.1.2.5"), ["A", "B", "C"]); + assert_eq!(names_in(CHAIN, "x.c.b.a.example"), ["A", "B", "C"]); } } diff --git a/uv.lock b/uv.lock index 6e52242..5b960f5 100644 --- a/uv.lock +++ b/uv.lock @@ -80,7 +80,6 @@ wheels = [ [[package]] name = "cloudcheck" -version = "11.2.0" source = { editable = "." } [package.dev-dependencies]