Integrate FileAutomation development into main

.gitattributes

@@ -1,2 +1,4 @@
 # Auto detect text files and perform LF normalization
 * text=auto
+# Shell scripts run under bash on Linux runners, which does not accept CRLF.
+*.sh text eol=lf

.github/dependabot.yml

@@ -6,16 +6,27 @@
 version: 2
 updates:
   - package-ecosystem: "pip" # See documentation for possible values
-    directory: "/" # Location of package manifests
+    # "/" holds requirements.txt and dev_requirements.txt. .github/requirements
+    # holds the hash-locked tools of the publish jobs (publish.in, publish.txt).
+    directories: # Locations of package manifests
+      - "/"
+      - "/.github/requirements"
     # Updates belong on dev: main is the released branch and merging into it
     # publishes, so a PR opened against main just sits there.
     target-branch: "dev"
     schedule:
       interval: "daily"
+    # A new release waits 7 days before Dependabot proposes it, so a
+    # compromised version is more likely to be found and yanked first.
+    # Dependabot's own default is 3 days. Security updates are not delayed.
+    cooldown:
+      default-days: 7
   # Workflow actions are pinned to commit SHAs with the version as a comment
   # (test_workflow_actions.py); Dependabot moves both together.
   - package-ecosystem: "github-actions"
     directory: "/"
     target-branch: "dev"
     schedule:
       interval: "weekly"
+    cooldown:
+      default-days: 7

.github/requirements/integration.in

@@ -0,0 +1,33 @@
+# Python 3.12 integration dependencies, locked for all runner platforms.
+# Regenerate: uv pip compile --universal --python-version 3.12 --generate-hashes
+# --only-binary :all: --exclude-newer 2026-10-02T00:00:00Z
+# .github/requirements/integration.in -o .github/requirements/integration.txt
+setuptools>=77
+wheel
+requests>=2.31.0
+tqdm>=4.66.0
+watchdog>=4.0.0
+cryptography>=50.0.0
+prometheus_client>=0.26.0
+defusedxml>=0.7.1
+je_action_core>=0.0.2
+PyYAML>=6.0.3
+tzdata>=2024.1; platform_system == 'Windows'
+opentelemetry-api>=1.44.0
+opentelemetry-sdk>=1.44.0
+tomli>=2.0.1; python_version<"3.11"
+boto3>=1.34.0
+azure-storage-blob>=12.19.0
+google-api-python-client>=2.100.0
+google-auth-httplib2>=0.2.0
+google-auth-oauthlib>=1.2.0
+dropbox>=11.36.2
+paramiko>=3.4.0
+smbprotocol>=1.13.0
+fsspec>=2024.2.0
+msal>=1.39.0
+boxsdk>=10.15.0,<11
+pyarrow>=25.0.1
+PySide6>=6.11.2,<6.12
+pytest>=8.0.0
+pytest-cov>=5.0.0

.github/requirements/lint.in

@@ -0,0 +1,6 @@
+# Regenerate with uv pip compile --generate-hashes --python-version 3.12
+# --only-binary :all: --exclude-newer 2026-10-02T00:00:00Z
+# .github/requirements/lint.in -o .github/requirements/lint.txt
+ruff
+mypy
+je_action_core>=0.0.2

.github/requirements/publish.in

@@ -0,0 +1,11 @@
+# Tools the two publish jobs run: publish-dev in ci-dev.yml and publish in publish.yml. Their version
+# scripts use the standard library only. publish.txt is generated from this file, for the Python 3.12
+# on Linux those jobs set up:
+#   uv pip compile .github/requirements/publish.in --generate-hashes --python-version 3.12 --python-platform x86_64-manylinux_2_28 --only-binary :all: --exclude-newer <a week ago, as 2026-09-24T00:00:00Z> -o .github/requirements/publish.txt
+# --exclude-newer leaves out releases under a week old, the wait dependabot.yml sets.
+build
+twine
+# The build backend. The jobs run `python -m build --no-isolation`, so the backend is this locked one
+# and not whatever is newest on PyPI when the job runs. It must satisfy `build-system.requires` in
+# stable.toml and dev.toml (tests/test_workflow_actions.py).
+setuptools

.github/workflows/ci-dev.yml

@@ -14,6 +14,7 @@ permissions:
 jobs:
   lint:
     runs-on: ubuntu-latest
+    timeout-minutes: 15  # about 3x the slowest recent run, at least 15
     steps:
       - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1  # v7.0.1
         with:
@@ -24,8 +25,8 @@ jobs:
           cache: pip
       - name: Install tooling
         run: |
-          python -m pip install --upgrade pip
-          pip install ruff mypy
+          # mypy reads the action engine types from the hash-locked tooling.
+          python -m pip install --require-hashes --only-binary :all: -r .github/requirements/lint.txt
       - name: Ruff check
         run: ruff check automation_file tests
       - name: Ruff format check
@@ -36,6 +37,7 @@ jobs:
   pytest:
     needs: lint
     runs-on: windows-latest
+    timeout-minutes: 15  # about 3x the slowest recent run, at least 15
     strategy:
       fail-fast: false
       matrix:
@@ -53,8 +55,8 @@ jobs:
         run: |
           python -m pip install --upgrade pip wheel
           Copy-Item dev.toml pyproject.toml -Force
-          pip install -e .
-          pip install pytest pytest-cov
+          # Every extra, so every backend's tests run. The minimal job runs without any.
+          pip install -e ".[all,test]"
       - name: Run pytest with coverage
         run: python -m pytest tests/ -v --tb=short --cov=automation_file --cov-report=term-missing --cov-report=xml
       - name: Upload coverage artifact
@@ -63,3 +65,124 @@ jobs:
         with:
           name: coverage-xml
           path: coverage.xml
+
+  minimal:
+    # The base install: no cloud SDK and no GUI toolkit. The tests of a backend whose extra is
+    # missing skip; everything else, the import guard included, has to pass.
+    needs: lint
+    runs-on: windows-latest
+    timeout-minutes: 15  # no run yet: the floor of 15 minutes, to revisit after the first runs
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1  # v7.0.1
+        with:
+          persist-credentials: false
+      - name: Set up Python
+        uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97  # v7.0.0
+        with:
+          python-version: "3.12"
+          cache: pip
+      - name: Install the base package
+        run: |
+          python -m pip install --upgrade pip wheel
+          Copy-Item dev.toml pyproject.toml -Force
+          pip install -e ".[test]"
+      - name: Run pytest
+        run: python -m pytest tests/ -v --tb=short
+
+  extras:
+    # Each extra on its own: it installs, and its backend's tests run with no other SDK present.
+    needs: lint
+    runs-on: windows-latest
+    timeout-minutes: 15  # no run yet: the floor of 15 minutes, to revisit after the first runs
+    strategy:
+      fail-fast: false
+      matrix:
+        extra: [ s3, azure, gdrive, dropbox, sftp, ftp, webdav, smb, fsspec, onedrive, box, parquet, gui ]
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1  # v7.0.1
+        with:
+          persist-credentials: false
+      - name: Set up Python
+        uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97  # v7.0.0
+        with:
+          python-version: "3.12"
+          cache: pip
+      - name: Install the package with one extra
+        run: |
+          python -m pip install --upgrade pip wheel
+          Copy-Item dev.toml pyproject.toml -Force
+          pip install -e ".[${{ matrix.extra }},test]"
+      - name: Run pytest
+        run: python -m pytest tests/ -v --tb=short
+
+  package:
+    # The sdist and the wheel build from dev.toml and their metadata renders, before anything is
+    # merged. It gates nothing: the publish job builds again from the locked tools.
+    needs: lint
+    runs-on: ubuntu-latest
+    timeout-minutes: 15  # no run yet: the floor of 15 minutes, to revisit after the first runs
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1  # v7.0.1
+        with:
+          persist-credentials: false
+      - name: Set up Python
+        uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97  # v7.0.0
+        with:
+          python-version: "3.12"
+          cache: pip
+      - name: Build the distribution and check its metadata
+        run: |
+          # The locked, wheels-only tools the publish jobs build with.
+          python -m pip install --require-hashes --only-binary :all: -r .github/requirements/publish.txt
+          cp dev.toml pyproject.toml
+          python -m build --no-isolation
+          twine check dist/*
+
+  publish-dev:
+    # The dev channel. A push to dev that passes lint and the tests is built from dev.toml and uploaded
+    # when it is still the tip of dev and ships something the newest automation_file_dev does not.
+    # scripts/dev_release.py picks the version from PyPI, so nothing is committed back.
+    name: Publish automation_file_dev to PyPI
+    needs: [lint, pytest, minimal, extras]
+    if: github.event_name == 'push' && github.ref == 'refs/heads/dev'
+    runs-on: ubuntu-latest
+    timeout-minutes: 15  # about 3x the slowest recent run, at least 15
+    concurrency:
+      group: publish-dev
+      cancel-in-progress: false
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1  # v7.0.1
+        with:
+          persist-credentials: false
+      - name: Set up Python
+        uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97  # v7.0.0
+        with:
+          python-version: "3.12"
+      # This job holds the PyPI token, so its tools are hash-locked and wheels only.
+      # .github/requirements/publish.in says how publish.txt is generated.
+      - name: Install build tools
+        run: |
+          python -m pip install --require-hashes --only-binary :all: -r .github/requirements/publish.txt
+      - name: Write pyproject.toml from dev.toml with the next version
+        run: python scripts/dev_release.py prepare
+      # --no-isolation: the backend is the setuptools locked in publish.txt, not a fresh download.
+      - name: Build distribution
+        run: python -m build --no-isolation
+      - name: Verify distribution metadata
+        run: python -m twine check dist/*
+      - name: Compare with the newest published wheel
+        id: compare
+        run: python scripts/dev_release.py changed dist
+      # A run that finishes after a newer push would otherwise publish older code as the newest release.
+      - name: Check that this commit is still the tip of dev
+        id: tip
+        run: |
+          tip="$(git ls-remote origin refs/heads/dev | cut -f1)"
+          if [ "$tip" = "$GITHUB_SHA" ]; then current=true; else current=false; fi
+          echo "current=$current" >> "$GITHUB_OUTPUT"
+      - name: Publish to PyPI
+        if: steps.compare.outputs.changed == 'true' && steps.tip.outputs.current == 'true'
+        env:
+          TWINE_USERNAME: __token__
+          TWINE_PASSWORD: ${{ secrets.PYPI_API_TOKEN }}
+        run: python -m twine upload --non-interactive dist/*

.github/workflows/ci-stable.yml

@@ -14,6 +14,7 @@ permissions:
 jobs:
   lint:
     runs-on: ubuntu-latest
+    timeout-minutes: 15  # about 3x the slowest recent run, at least 15
     steps:
       - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1  # v7.0.1
         with:
@@ -24,8 +25,8 @@ jobs:
           cache: pip
       - name: Install tooling
         run: |
-          python -m pip install --upgrade pip
-          pip install ruff mypy
+          # mypy reads the action engine types from the hash-locked tooling.
+          python -m pip install --require-hashes --only-binary :all: -r .github/requirements/lint.txt
       - name: Ruff check
         run: ruff check automation_file tests
       - name: Ruff format check
@@ -36,6 +37,7 @@ jobs:
   pytest:
     needs: lint
     runs-on: windows-latest
+    timeout-minutes: 15  # about 3x the slowest recent run, at least 15
     strategy:
       fail-fast: false
       matrix:
@@ -53,8 +55,8 @@ jobs:
         run: |
           python -m pip install --upgrade pip wheel
           Copy-Item stable.toml pyproject.toml -Force
-          pip install -e .
-          pip install pytest pytest-cov
+          # Every extra, so every backend's tests run. The minimal job runs without any.
+          pip install -e ".[all,test]"
       - name: Run pytest with coverage
         run: python -m pytest tests/ -v --tb=short --cov=automation_file --cov-report=term-missing --cov-report=xml
       - name: Upload coverage artifact
@@ -63,3 +65,75 @@ jobs:
         with:
           name: coverage-xml
           path: coverage.xml
+
+  minimal:
+    # The base install: no cloud SDK and no GUI toolkit. The tests of a backend whose extra is
+    # missing skip; everything else, the import guard included, has to pass.
+    needs: lint
+    runs-on: windows-latest
+    timeout-minutes: 15  # no run yet: the floor of 15 minutes, to revisit after the first runs
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1  # v7.0.1
+        with:
+          persist-credentials: false
+      - name: Set up Python
+        uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97  # v7.0.0
+        with:
+          python-version: "3.12"
+          cache: pip
+      - name: Install the base package
+        run: |
+          python -m pip install --upgrade pip wheel
+          Copy-Item stable.toml pyproject.toml -Force
+          pip install -e ".[test]"
+      - name: Run pytest
+        run: python -m pytest tests/ -v --tb=short
+
+  extras:
+    # Each extra on its own: it installs, and its backend's tests run with no other SDK present.
+    needs: lint
+    runs-on: windows-latest
+    timeout-minutes: 15  # no run yet: the floor of 15 minutes, to revisit after the first runs
+    strategy:
+      fail-fast: false
+      matrix:
+        extra: [ s3, azure, gdrive, dropbox, sftp, ftp, webdav, smb, fsspec, onedrive, box, parquet, gui ]
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1  # v7.0.1
+        with:
+          persist-credentials: false
+      - name: Set up Python
+        uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97  # v7.0.0
+        with:
+          python-version: "3.12"
+          cache: pip
+      - name: Install the package with one extra
+        run: |
+          python -m pip install --upgrade pip wheel
+          Copy-Item stable.toml pyproject.toml -Force
+          pip install -e ".[${{ matrix.extra }},test]"
+      - name: Run pytest
+        run: python -m pytest tests/ -v --tb=short
+
+  package:
+    # The sdist and the wheel build from stable.toml and their metadata renders, before anything is
+    # merged. It gates nothing: the publish job builds again from the locked tools.
+    needs: lint
+    runs-on: ubuntu-latest
+    timeout-minutes: 15  # no run yet: the floor of 15 minutes, to revisit after the first runs
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1  # v7.0.1
+        with:
+          persist-credentials: false
+      - name: Set up Python
+        uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97  # v7.0.0
+        with:
+          python-version: "3.12"
+          cache: pip
+      - name: Build the distribution and check its metadata
+        run: |
+          # The locked, wheels-only tools the publish jobs build with.
+          python -m pip install --require-hashes --only-binary :all: -r .github/requirements/publish.txt
+          cp stable.toml pyproject.toml
+          python -m build --no-isolation
+          twine check dist/*