diff --git a/.github/workflows/docs-check.yml b/.github/workflows/docs-check.yml new file mode 100644 index 000000000..10e499d73 --- /dev/null +++ b/.github/workflows/docs-check.yml @@ -0,0 +1,52 @@ +name: Docs + +# Builds the Sphinx docs on pull requests that change them, and fails on any warning, so +# broken markup, missing pages or bad docstrings are caught before they are merged + +on: + pull_request: + branches: + - master + paths: + - 'docs/**' + # the modules whose docstrings are part of the docs (autodoc) + - 'lithops/executors.py' + - 'lithops/concurrent/futures.py' + - 'lithops/retries.py' + - 'lithops/storage/storage.py' + - 'lithops/utils.py' + - 'pyproject.toml' + - '.github/workflows/docs-check.yml' + + workflow_dispatch: + # this allows to run the workflow manually through the github dashboard + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + + sphinx: + runs-on: ubuntu-latest + timeout-minutes: 15 + + steps: + - name: Clone Lithops repository + uses: actions/checkout@v5 + + - name: Install Python 3.12 + uses: actions/setup-python@v6 + with: + python-version: '3.12' + cache: 'pip' + cache-dependency-path: pyproject.toml + + - name: Install the docs dependencies + run: | + sudo apt-get update && sudo apt-get install --no-install-recommends -y pandoc + pip3 install '.[docs]' + + - name: Build the docs + # -W turns warnings into errors; --keep-going reports all of them, not just the first + run: make -C docs html SPHINXOPTS="-W --keep-going" diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml new file mode 100644 index 000000000..87ac92f5c --- /dev/null +++ b/.github/workflows/docs.yml @@ -0,0 +1,135 @@ +name: Publish docs + +# Builds the Sphinx docs and commits them to the docs/ folder of the GitHub Pages +# repository (/lithops-cloud.github.io), authored by whoever runs the workflow. +# +# It always builds master, and names the commit after the latest GitHub release +# ("Update docs to X"). The Release workflow calls it at the end of every release; run it by +# hand from the Actions tab (Publish docs -> Run workflow) to republish the docs without a +# release, e.g. after fixing them. +# +# Pushing to the docs repository needs a deploy key with write access: its private key goes +# in the DOCS_DEPLOY_KEY secret of this repository; without it the docs are only built. + +on: + workflow_dispatch: + workflow_call: + secrets: + DOCS_DEPLOY_KEY: + required: false + +jobs: + + docs: + runs-on: ubuntu-latest + timeout-minutes: 20 + # one push to the docs repository at a time + concurrency: + group: publish-docs + cancel-in-progress: false + permissions: + contents: read + env: + # the GitHub Pages repository of the same owner, e.g. lithops-cloud/lithops-cloud.github.io + DOCS_REPOSITORY: ${{ github.repository_owner }}/lithops-cloud.github.io + + steps: + - name: Check the deploy key of the docs repository + id: key + env: + DEPLOY_KEY: ${{ secrets.DOCS_DEPLOY_KEY }} + run: | + if [ -z "$DEPLOY_KEY" ]; then + echo "::warning::The DOCS_DEPLOY_KEY secret is not set, the docs are built but not published to $DOCS_REPOSITORY" + echo "publish=false" >> "$GITHUB_OUTPUT" + else + echo "publish=true" >> "$GITHUB_OUTPUT" + fi + + - name: Clone Lithops repository + uses: actions/checkout@v5 + with: + ref: master + + - name: Get the latest release + id: version + env: + GH_TOKEN: ${{ github.token }} + run: | + # a repository without releases answers 404, and gh prints its body on stdout + VERSION=$(gh api "repos/$GITHUB_REPOSITORY/releases/latest" --jq .tag_name 2>/dev/null) || VERSION="" + echo "Docs of release ${VERSION:-(none)}" + echo "version=$VERSION" >> "$GITHUB_OUTPUT" + + - name: Set the docs to the release version + # the docs show lithops.__version__ ("Lithops vX" in the logo), and master carries the + # next development version; set before installing, as conf.py imports the installed + # package. Without any release the version of master is kept + if: ${{ steps.version.outputs.version != '' }} + env: + VERSION: ${{ steps.version.outputs.version }} + run: | + sed -i -E "s/^__version__ = \".*\"/__version__ = \"$VERSION\"/" lithops/version.py + grep '^__version__' lithops/version.py + + - name: Install Python + uses: actions/setup-python@v6 + with: + python-version: '3.12' + + - name: Install the docs dependencies + run: | + sudo apt-get update && sudo apt-get install --no-install-recommends -y pandoc + pip3 install '.[docs]' + + - name: Build the docs + run: make -C docs html + + - name: Get the git identity of who runs the workflow + if: ${{ steps.key.outputs.publish == 'true' }} + id: author + env: + GH_TOKEN: ${{ github.token }} + LOGIN: ${{ github.actor }} + LOGIN_ID: ${{ github.actor_id }} + run: | + # Their GitHub username, and the email of their latest commit in this repository, + # which is the one their own commits use. Someone who never committed here gets + # their GitHub noreply address + # (gh prints the error body on stdout when the call fails, so drop it then) + email=$(gh api "repos/$GITHUB_REPOSITORY/commits?author=$LOGIN&per_page=1" \ + --jq '.[0].commit.author.email // empty' 2>/dev/null) || email="" + [ -n "$email" ] || email="$LOGIN_ID+$LOGIN@users.noreply.github.com" + echo "Committing as $LOGIN <$email>" + echo "name=$LOGIN" >> "$GITHUB_OUTPUT" + echo "email=$email" >> "$GITHUB_OUTPUT" + + - name: Clone the docs repository + if: ${{ steps.key.outputs.publish == 'true' }} + uses: actions/checkout@v5 + with: + repository: ${{ env.DOCS_REPOSITORY }} + ssh-key: ${{ secrets.DOCS_DEPLOY_KEY }} + path: site + + - name: Commit and push the docs + if: ${{ steps.key.outputs.publish == 'true' }} + env: + VERSION: ${{ steps.version.outputs.version }} + AUTHOR_NAME: ${{ steps.author.outputs.name }} + AUTHOR_EMAIL: ${{ steps.author.outputs.email }} + run: | + # replace everything under docs/ with the new build (dotfiles are kept, as the + # manual "rm -R docs/*; cp -R _build/html/* docs/" did) + rm -rf site/docs/* + cp -R docs/_build/html/* site/docs/ + cd site + git add -A docs + if git diff --cached --quiet; then + echo "The docs did not change" + exit 0 + fi + git config user.name "$AUTHOR_NAME" + git config user.email "$AUTHOR_EMAIL" + git commit -m "Update docs${VERSION:+ to $VERSION}" + git push diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index a9cd202d5..a0dde8b03 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -7,8 +7,11 @@ name: Release # 2. builds the sdist and wheel and publishes them (trusted publishing, no token) # 3. creates the GitHub release "Lithops-X" with the changelog section as notes # 4. bumps the branch to the next development version (X.Y.Z+1.dev0) +# 5. at the end, publishes the docs to the GitHub Pages repository as "Update docs to X" +# (the Publish docs workflow, docs.yml, which can also be run by hand to republish them) # Commits go to the branch the workflow runs on, authored by whoever runs it. -# With "dry run" checked it only does the edits and the build, and pushes or publishes nothing. +# With "dry run" checked it only does the edits and the package build, and pushes or +# publishes nothing. # # In lithops-cloud/lithops it releases from master to PyPI. In a fork it publishes to # TestPyPI instead, so the whole release can be tried out from a throwaway branch. @@ -66,8 +69,9 @@ jobs: # Their GitHub username, and the email of their latest commit in this repository, # which is the one their own commits use. Someone who never committed here gets # their GitHub noreply address + # (gh prints the error body on stdout when the call fails, so drop it then) email=$(gh api "repos/$GITHUB_REPOSITORY/commits?author=$LOGIN&per_page=1" \ - --jq '.[0].commit.author.email // empty' 2>/dev/null || true) + --jq '.[0].commit.author.email // empty' 2>/dev/null) || email="" [ -n "$email" ] || email="$LOGIN_ID+$LOGIN@users.noreply.github.com" echo "Committing as $LOGIN <$email>" echo "name=$LOGIN" >> "$GITHUB_OUTPUT" @@ -242,3 +246,12 @@ jobs: git config user.email "$AUTHOR_EMAIL" git commit -am "Bump version to $DEV_VERSION" git push origin "HEAD:$GITHUB_REF_NAME" + + docs: + # last, once the release is out and master bumped (so a dry run, which skips those, + # does not publish the docs either) + needs: [github-release, bump-dev] + permissions: + contents: read + uses: ./.github/workflows/docs.yml + secrets: inherit diff --git a/AGENTS.md b/AGENTS.md index 02e451597..5ea68eb15 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -38,7 +38,9 @@ loads the developer's own configuration (`~/.lithops/config`, `.lithops_config`, `LITHOPS_CONFIG_FILE`) and runs against whatever cloud account it points to. Some tests need a Redis server on `localhost:6379` and skip themselves when none is reachable. -Documentation is built with Sphinx from `docs/` (`make html`, see [docs/README.md](docs/README.md)). +Documentation is built with Sphinx from `docs/` (`pip3 install -e '.[docs]'`, then +`make -C docs html SPHINXOPTS="-W --keep-going"`, see [docs/README.md](docs/README.md)). Pull +requests that touch the docs must build without warnings: CI runs that same command. ## Repository map @@ -91,7 +93,8 @@ Documentation is built with Sphinx from `docs/` (`make html`, see [docs/README.m `ruff check .` clean with line length 120. Do not run `ruff format`: the code base is not formatted with it and it would rewrite almost every file. - Package metadata, dependencies and extras live in `pyproject.toml`. When adding a - dependency to an extra, also add it to the `all` extra. + dependency to an extra, also add it to the `all` extra (except the `dev` and `docs` tooling + extras). - Every bug fix includes a regression test; every feature includes tests of its behaviour. Tests must run on the localhost backend and storage; backend-specific code that cannot be exercised locally is tested with fakes or mocks. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 60b4b36a6..ddfac9ef4 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -116,8 +116,10 @@ Releases are made by the [Release workflow](.github/workflows/release.yml): *Act *Release* -> *Run workflow*, with the version to release (e.g. `3.7.1`). Before running it, review the development section at the top of `CHANGELOG.md`, which becomes the release notes. The workflow sets the version, tags it, publishes the sdist and wheel to PyPI, creates the -GitHub release and bumps `master` to the next development version. Check *dry run* to build -and check a release without pushing or publishing anything. +GitHub release, publishes the docs to the website repository and bumps `master` to the next +development version. Check *dry run* to build and check a release without pushing or +publishing anything. To republish the docs without a release, run the *Publish docs* workflow: +it builds `master` and publishes it as the docs of the latest release. ## AI coding agents diff --git a/docs/README.md b/docs/README.md index af6f68776..f301d2dd3 100644 --- a/docs/README.md +++ b/docs/README.md @@ -1,9 +1,14 @@ # Build Lithops documentation -1. Install [Sphinx](https://www.sphinx-doc.org/en/master/usage/installation.html) and all plugins: +The [Release workflow](../.github/workflows/release.yml) builds these docs and publishes them to +the website repository on every release. To republish them without a release (e.g. after fixing +them), run the [Publish docs workflow](../.github/workflows/docs.yml) from the *Actions* tab: it +builds `master` and publishes it as the docs of the latest release. The steps below are for building them locally. + +1. Install Lithops with [Sphinx](https://www.sphinx-doc.org/en/master/usage/installation.html) and all plugins, from the repository root: ```bash - python3 -m pip install sphinx myst-parser sphinx_copybutton jupyter ipykernel nbsphinx sphinx_book_theme + python3 -m pip install -e '.[docs]' ``` 2. Install [Pandoc](https://pandoc.org/installing.html). For debian/ubuntu: diff --git a/docs/source/compute_config/ibm_cf.md b/docs/source/compute_config/ibm_cf.md index 98bd4130f..b448f57e6 100644 --- a/docs/source/compute_config/ibm_cf.md +++ b/docs/source/compute_config/ibm_cf.md @@ -1,3 +1,7 @@ +--- +orphan: true +--- + # IBM Cloud Functions Lithops with *IBM Cloud Functions* as compute backend. diff --git a/docs/source/compute_config/kubernetes.md b/docs/source/compute_config/kubernetes.md index 081a082b8..9c8d0f970 100644 --- a/docs/source/compute_config/kubernetes.md +++ b/docs/source/compute_config/kubernetes.md @@ -118,4 +118,10 @@ You can view the function executions logs in your local machine using the *litho ```bash lithops logs poll -``` \ No newline at end of file +``` + +```{toctree} +:hidden: + +kubernetes_rabbitmq +``` diff --git a/docs/source/contributing.rst b/docs/source/contributing.rst index f51c3d0d9..086faeb37 100644 --- a/docs/source/contributing.rst +++ b/docs/source/contributing.rst @@ -136,8 +136,10 @@ Releases are made by the `Release workflow *Release* -> *Run workflow*, with the version to release (e.g. ``3.7.1``). Before running it, review the development section at the top of ``CHANGELOG.md``, which becomes the release notes. The workflow sets the version, tags it, publishes the sdist and wheel to PyPI, -creates the GitHub release and bumps ``master`` to the next development version. Check -*dry run* to build and check a release without pushing or publishing anything. +creates the GitHub release, publishes the docs to the website repository and bumps ``master`` +to the next development version. Check *dry run* to build and check a release without pushing +or publishing anything. To republish the docs without a release, run the *Publish docs* +workflow: it builds ``master`` and publishes it as the docs of the latest release. AI coding agents diff --git a/docs/source/metrics.rst b/docs/source/metrics.rst index c84c8e2c5..f9773d6e2 100644 --- a/docs/source/metrics.rst +++ b/docs/source/metrics.rst @@ -92,7 +92,7 @@ that. Lithops keeps a registry of cumulative metrics and replaces its Pushgateway group with it on every push. Installing Prometheus and the Pushgateway -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ The quickest way to get all three services up, including Grafana: @@ -194,7 +194,7 @@ OpenTelemetry endpoint: http://localhost:4318 Straight into Prometheus, without a collector -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Prometheus can receive OTLP itself, which means the ``otlp`` backend reaches it with no collector and no Pushgateway in between. Start Prometheus with diff --git a/docs/source/storage_backends.rst b/docs/source/storage_backends.rst index 008412d56..fe0a37729 100644 --- a/docs/source/storage_backends.rst +++ b/docs/source/storage_backends.rst @@ -1,11 +1,7 @@ Storage Backends ================ -.. toctree:: - :glob: - :maxdepth: 1 - - compute_config/localhost.md +* :doc:`compute_config/localhost` **Object Storage:** diff --git a/lithops/concurrent/futures.py b/lithops/concurrent/futures.py index 2d139af79..679d68d98 100644 --- a/lithops/concurrent/futures.py +++ b/lithops/concurrent/futures.py @@ -287,10 +287,30 @@ def done(self): return super().done() def result(self, timeout=None): + """ + Returns the result of the call, waiting for it to finish. + + :param timeout: Seconds to wait if the call is not done yet. ``None`` + waits without limit + :return: The value returned by the call + :raises concurrent.futures.CancelledError: If the future was cancelled + :raises TimeoutError: If the call did not finish within ``timeout`` + :raises Exception: The exception raised by the call, if it raised one + """ self._sync() return super().result(timeout) def exception(self, timeout=None): + """ + Returns the exception raised by the call, waiting for it to finish. + + :param timeout: Seconds to wait if the call is not done yet. ``None`` + waits without limit + :return: The exception raised by the call, or ``None`` if it returned + normally + :raises concurrent.futures.CancelledError: If the future was cancelled + :raises TimeoutError: If the call did not finish within ``timeout`` + """ self._sync() return super().exception(timeout) @@ -635,6 +655,17 @@ def _check_running(self): # -- concurrent.futures.Executor ---------------------------------------- def submit(self, fn, /, *args, **kwargs): + """ + Schedules ``fn(*args, **kwargs)`` to run on a Lithops worker. + + :param fn: The callable to run + :param args: Positional arguments for ``fn`` + :param kwargs: Keyword arguments for ``fn`` + :return: A :class:`Future` representing the call + :raises RuntimeError: If the executor has been shut down + :raises concurrent.futures.BrokenExecutor: If the executor stopped + working and can no longer run calls + """ # Shutdown must see every accepted submission in _pending before it # can release the native executor, including while submission blocks. with self._submission_lock: diff --git a/pyproject.toml b/pyproject.toml index 26254f584..04c541f46 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -19,8 +19,10 @@ keywords = [ "map-reduce", ] authors = [ - { name = "Gil Vernik", email = "gilv@ibm.com" }, - { name = "Josep Sampe", email = "josep.sampe@gmail.com" }, + { name = "Gil Vernik" }, + { name = "Josep Sampe" }, + { email = "gilv@ibm.com" }, + { email = "josep.sampe@gmail.com" }, ] license = "Apache-2.0" license-files = ["LICENSE"] @@ -144,7 +146,17 @@ dev = [ "ruff>=0.16,<0.17", "pre-commit", ] -# union of every extra above except dev; keep it in sync when adding a dependency +# to build the Sphinx docs (the notebooks also need the pandoc binary) +docs = [ + "sphinx", + "myst-parser", + "sphinx-copybutton", + "nbsphinx", + "ipykernel", + "sphinx-book-theme", + "sphinxcontrib-mermaid", +] +# union of every extra above except dev and docs; keep it in sync when adding a dependency all = [ "alibabacloud-fc20230330>=4.7.0", "azure-identity",