From 7429b01b2378f43da93a3263ed6c17e9dd255dd5 Mon Sep 17 00:00:00 2001 From: JosepSampe Date: Thu, 24 Sep 2026 12:35:46 +0200 Subject: [PATCH] Update project --- .editorconfig | 26 +++ .github/ISSUE_TEMPLATE/bug_report.md | 30 +++ .github/ISSUE_TEMPLATE/config.yml | 8 + .github/ISSUE_TEMPLATE/feature_request.md | 17 ++ ...UEST_TEMPLATE => PULL_REQUEST_TEMPLATE.md} | 18 +- .github/workflows/python-linting.yml | 10 +- .github/workflows/tests-all-os.yml | 2 +- .github/workflows/tests.yml | 4 +- .gitignore | 4 + .pre-commit-config.yaml | 12 + AGENTS.md | 112 +++++++++ CHANGELOG.md | 2 + CLAUDE.md | 18 ++ CONTRIBUTING.md | 136 ++++++++--- SECURITY.md | 30 +++ docs/Dockerfile | 4 +- docs/source/api_futures.rst | 3 +- docs/source/api_multiprocessing.rst | 5 - docs/source/contributing.rst | 141 +++++++++--- docs/source/monitoring.rst | 4 +- docs/source/notebooks/function_chaining.ipynb | 2 +- .../serverless/backends/aws_lambda/config.py | 3 +- .../azure_containers/azure_containers.py | 3 +- .../backends/gcp_cloudrun/cloudrun.py | 5 +- .../backends/gcp_functions/gcp_functions.py | 4 +- lithops/serverless/backends/k8s/k8s.py | 3 +- .../serverless/backends/knative/knative.py | 3 +- .../backends/singularity/singularity.py | 3 +- .../standalone/backends/aws_ec2/aws_ec2.py | 16 +- .../standalone/backends/ibm_vpc/ibm_vpc.py | 16 +- .../storage/backends/localhost/localhost.py | 2 +- .../storage/backends/oracle_oss/oracle_oss.py | 19 +- pyproject.toml | 217 ++++++++++++++++++ setup.cfg | 9 - setup.py | 147 ------------ 35 files changed, 774 insertions(+), 264 deletions(-) create mode 100644 .editorconfig create mode 100644 .github/ISSUE_TEMPLATE/bug_report.md create mode 100644 .github/ISSUE_TEMPLATE/config.yml create mode 100644 .github/ISSUE_TEMPLATE/feature_request.md rename .github/{PULL_REQUEST_TEMPLATE => PULL_REQUEST_TEMPLATE.md} (70%) create mode 100644 .pre-commit-config.yaml create mode 100644 AGENTS.md create mode 100644 CLAUDE.md create mode 100644 SECURITY.md create mode 100644 pyproject.toml delete mode 100644 setup.cfg delete mode 100644 setup.py diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 000000000..1289419ca --- /dev/null +++ b/.editorconfig @@ -0,0 +1,26 @@ +root = true + +[*] +charset = utf-8 +end_of_line = lf +insert_final_newline = true +trim_trailing_whitespace = true +indent_style = space +indent_size = 4 + +[*.py] +max_line_length = 120 + +[*.{yml,yaml,json}] +indent_size = 2 + +[*.md] +indent_size = 2 +# two trailing spaces are a hard line break in Markdown +trim_trailing_whitespace = false + +[*.rst] +indent_size = 3 + +[Makefile] +indent_style = tab diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md new file mode 100644 index 000000000..f75e0f9aa --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -0,0 +1,30 @@ +--- +name: Bug report +about: Something does not work as documented +labels: bug +--- + +**What happened** +A clear description of the bug and what you expected instead. + +**Minimal reproduction** +```python +# the smallest script that reproduces it +import lithops + +fexec = lithops.FunctionExecutor() +``` + +**Traceback / logs** +``` +paste the full traceback; if possible, run with log_level: DEBUG +``` + +**Environment** +- Lithops version (`lithops --version`): +- Compute backend (e.g. `localhost`, `aws_lambda`, `code_engine`, `aws_ec2`): +- Storage backend (e.g. `localhost`, `aws_s3`, `ibm_cos`): +- Runtime (default or custom image): +- Python version and OS of the client: + + diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml new file mode 100644 index 000000000..08fdc81a5 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -0,0 +1,8 @@ +blank_issues_enabled: true +contact_links: + - name: Questions and discussions + url: https://github.com/lithops-cloud/lithops/discussions + about: Usage questions, ideas and general discussion + - name: Security vulnerabilities + url: https://github.com/lithops-cloud/lithops/security/advisories/new + about: Report security issues privately, not as public issues (see SECURITY.md) diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md new file mode 100644 index 000000000..94c1fbf01 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature_request.md @@ -0,0 +1,17 @@ +--- +name: Feature request +about: Propose a new feature, backend or API change +labels: enhancement +--- + +**Use case** +What are you trying to run (workload, scale, compute / storage backend)? + +**Problem** +What is slow, expensive or impossible today? + +**Proposal** +The API or behaviour you would like. + +**Alternatives** +Workarounds you use today. diff --git a/.github/PULL_REQUEST_TEMPLATE b/.github/PULL_REQUEST_TEMPLATE.md similarity index 70% rename from .github/PULL_REQUEST_TEMPLATE rename to .github/PULL_REQUEST_TEMPLATE.md index e7eb9261c..1a9701c96 100644 --- a/.github/PULL_REQUEST_TEMPLATE +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -1,6 +1,21 @@ +## What and why + + +## How it was tested + +- [ ] `ruff check .` +- [ ] `pytest -v --backend localhost --storage localhost` (in `lithops/tests`) +- [ ] On a cloud backend: + +## Checklist + +- [ ] Tests cover the new behaviour / the fixed bug (regression test) +- [ ] Docs updated (`docs/`, `README.md`, `config/config_template.yaml` for new config keys) +- [ ] `CHANGELOG.md` entry under the development version + +--- - Developer's Certificate of Origin 1.1 By making a contribution to this project, I certify that: @@ -25,4 +40,3 @@ Developer's Certificate of Origin 1.1 personal information I submit with it, including my sign-off) is maintained indefinitely and may be redistributed consistent with this project or the open source license(s) involved. - diff --git a/.github/workflows/python-linting.yml b/.github/workflows/python-linting.yml index bbf84c444..def7c21f4 100644 --- a/.github/workflows/python-linting.yml +++ b/.github/workflows/python-linting.yml @@ -5,7 +5,7 @@ on: branches: - master paths: - - 'setup.py' + - 'pyproject.toml' - 'lithops/**' - '.github/workflows/python-linting.yml' @@ -18,7 +18,7 @@ concurrency: jobs: - flake8: + ruff: runs-on: ubuntu-latest timeout-minutes: 10 @@ -34,8 +34,8 @@ jobs: - name: Install dependencies run: | python3 -m pip install --upgrade pip - pip3 install -U flake8 + pip3 install ruff==0.16.8 # same version as .pre-commit-config.yaml - - name: Lint with flake8 + - name: Lint with ruff run: | - flake8 lithops --count --statistics + ruff check --output-format=github . diff --git a/.github/workflows/tests-all-os.yml b/.github/workflows/tests-all-os.yml index e27a3bc3b..1d3e8f1be 100644 --- a/.github/workflows/tests-all-os.yml +++ b/.github/workflows/tests-all-os.yml @@ -25,7 +25,7 @@ jobs: with: python-version: ${{ matrix.python-version }} cache: 'pip' - cache-dependency-path: setup.py + cache-dependency-path: pyproject.toml - name: Install Redis # There is no Redis build for Windows. The tests that need a server diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 4d4ec9e37..a623bf51b 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -5,7 +5,7 @@ on: branches: - master paths: - - 'setup.py' + - 'pyproject.toml' - 'lithops/**' - '.github/workflows/tests.yml' @@ -48,7 +48,7 @@ jobs: with: python-version: ${{ matrix.python-version }} cache: 'pip' - cache-dependency-path: setup.py + cache-dependency-path: pyproject.toml - name: Install Lithops run: | diff --git a/.gitignore b/.gitignore index 1817e5ee3..6c728f7d4 100644 --- a/.gitignore +++ b/.gitignore @@ -10,6 +10,10 @@ lithops_*.zip *.txt *.csv *.coverage* +.pytest_cache/ + +# Local Lithops config (may hold credentials) +.lithops_config # Virtual environments .env diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 000000000..3a4ca3fdc --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,12 @@ +repos: + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v6.0.0 + hooks: + - id: check-yaml + - id: check-merge-conflict + - id: check-added-large-files + - repo: https://github.com/astral-sh/ruff-pre-commit + rev: v0.16.8 + hooks: + # configured in pyproject.toml, same as the Python Linting workflow + - id: ruff-check diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 000000000..02e451597 --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,112 @@ +# AGENTS.md + +Instructions for AI coding agents working in this repository (OpenAI Codex, GitHub Copilot +coding agent, Cursor, Gemini CLI, Aider, Claude Code and others). Human contributors should +read [CONTRIBUTING.md](CONTRIBUTING.md); this file summarizes the same rules plus the +architectural context an agent needs. + +## Project overview + +Lithops is a Python framework that runs unmodified Python functions at scale on serverless +platforms, container and batch services, virtual machines and the local machine, behind one +small API (`FunctionExecutor.call_async()`, `map()`, `map_reduce()`, `wait()`, +`get_result()`, `Storage`). The architecture is described in +[docs/source/design.rst](docs/source/design.rst); read it before changing core code. + +- **Client:** where `import lithops` runs. Builds jobs, serializes code and data, invokes the + workers and collects results. All orchestration lives here. +- **Compute backend:** where the workers run, one call each. +- **Object storage:** the only communication bus between client and workers. The client + writes the pickled function and data; workers read their input and write their output and + status back. There are no direct client-worker connections. + +## Setup and commands + +```bash +pip3 install -e '.[tests,dev]' # editable install + test deps + ruff / pre-commit + # (backend extras: '.[aws]', '.[gcp]', '.[all]', ...) +ruff check . # lint; must pass (config in pyproject.toml, line length 120) + +cd lithops/tests +pytest -v --timeout=120 --timeout-method=thread --backend localhost --storage localhost +pytest -v --backend localhost --storage localhost -k test_map # a subset +pytest --collect-only # list the tests +``` + +**Always pass `--backend localhost --storage localhost`.** Without them the test session +loads the developer's own configuration (`~/.lithops/config`, `.lithops_config`, +`LITHOPS_CONFIG_FILE`) and runs against whatever cloud account it points to. Some tests need a +Redis server on `localhost:6379` and skip themselves when none is reachable. + +Documentation is built with Sphinx from `docs/` (`make html`, see [docs/README.md](docs/README.md)). + +## Repository map + +| Path | Content | +|---|---| +| `lithops/__init__.py` | Public API surface (`FunctionExecutor`, `Storage`, `wait`, `get_result`, ...) | +| `lithops/executors.py` | `FunctionExecutor` and `LocalhostExecutor` / `ServerlessExecutor` / `StandaloneExecutor` | +| `lithops/config.py` | Loads and merges configuration (dict, env vars, YAML), validates backends | +| `lithops/job/` | Job creation, function/module serialization (`cloudpickle`), data partitioning | +| `lithops/invokers.py` | `FaaSInvoker` (per-call invocations) and `BatchInvoker` (one submission, many tasks) | +| `lithops/future.py`, `wait.py`, `retries.py` | `ResponseFuture`, `wait()` / `get_result()`, `RetryingFunctionExecutor` | +| `lithops/worker/` | Code that runs on the compute backend: `handler.py` (`function_handler`) and `jobrunner.py` (`JobRunner`) | +| `lithops/serverless/backends//` | Serverless / batch compute backends (`.py`, `config.py`, `entry_point.py`) | +| `lithops/standalone/` | Standalone mode: master / worker VMs coordinated through Redis; `backends//` for VM providers | +| `lithops/localhost/` | Localhost compute backend (`v1`, `v2`) | +| `lithops/storage/` | `Storage` / `InternalStorage`, cloud file APIs; `backends//` for object stores | +| `lithops/monitoring/` | Job monitor and its pluggable backends (storage polling, RabbitMQ, Redis, queues) | +| `lithops/telemetry/` | Prometheus / OpenTelemetry metrics (off by default) | +| `lithops/multiprocessing/`, `concurrent/`, `util/joblib/` | Drop-in `multiprocessing`, `concurrent.futures` and joblib APIs | +| `lithops/scripts/` | `lithops` CLI (`cli.py`) and the temporary-data cleaner | +| `lithops/tests/` | pytest suite (`conftest.py` defines `--backend`, `--storage`, `--config`, `--region`) | +| `runtime//` | Dockerfiles and instructions to build runtimes for each backend | +| `config/` | `config_template.yaml` (every configuration key) and the configuration guide | +| `docs/` | Sphinx documentation; `docs/source/compute_config/` and `storage_config/` per backend | +| `examples/` | Single-file usage examples | + +## Architectural invariants + +1. **Storage is the bus.** Workers receive their input and return their results and status + through object storage (`func_key`, `agg_data`, `output_key`, `status_key`), never through + a direct connection to the client. +2. **Every backend looks the same to the core.** Backend-specific code stays in its backend + package; the executor, invoker, job and worker code must not special-case a backend by + name when an interface method or a config value can express it. +3. **The worker must tolerate the environment it runs in.** Worker code runs in minimal + runtimes (Lambda, containers, VMs, macOS/Windows localhost). Imports of optional + dependencies stay lazy, and a failure in the user's function is reported in its status + rather than crashing the worker. +4. **Configuration keys are documented where users look for them.** A new or changed key goes + into `config/config_template.yaml` and the backend's page in `docs/source/compute_config/` + or `docs/source/storage_config/` (general `lithops` keys also in + `docs/source/lithops_config_keys.csv`). +5. **Public API compatibility.** `FunctionExecutor`, `Storage`, futures and the + `multiprocessing` / `concurrent.futures` layers are used by existing programs; do not + change signatures or defaults without a deprecation path and a CHANGELOG entry. + +## Conventions + +- Python 3.10 - 3.14 (the versions CI tests). Match the style of the surrounding code; + `ruff check .` clean with line length 120. Do not run `ruff format`: the code base is not + formatted with it and it would rewrite almost every file. +- Package metadata, dependencies and extras live in `pyproject.toml`. When adding a + dependency to an extra, also add it to the `all` extra. +- Every bug fix includes a regression test; every feature includes tests of its behaviour. + Tests must run on the localhost backend and storage; backend-specific code that cannot be + exercised locally is tested with fakes or mocks. +- User-visible changes go into `CHANGELOG.md` under the topmost (development) version + heading, in the *Added / Changed / Fixed / Removed* section, prefixed with the component, + e.g. `- [Monitoring] ...`. Update `docs/` and `README.md` when behaviour changes. +- `CONTRIBUTING.md` and `docs/source/contributing.rst` carry the same content: change both. +- Pull requests target `master`. Keep changes small and focused; do not refactor unrelated + code. + +## Safety + +- Do not run tests, examples or `lithops` CLI commands against real cloud backends (AWS, GCP, + Azure, IBM Cloud, Aliyun, Oracle, Kubernetes clusters) and do not build or push runtime + images without the maintainer's explicit approval: it creates billable resources and uses + real credentials. +- Never commit credentials or configuration files (`~/.lithops/config`, `.lithops_config`). +- Do not commit or push unless asked to. diff --git a/CHANGELOG.md b/CHANGELOG.md index 3a05a826c..b6c925688 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -24,6 +24,8 @@ - [Storage] `CloudFileProxy.walk()` matches `os.walk` on a missing path; `cloud_open()` rejects an unsupported mode. - [CLI] `job list`, `worker list`, `image delete` and `image list` reject unknown flags. - [Joblib] `lithops_args` applied to the batch pool; upload/download pools capped at 32 threads. +- [Packaging] Package metadata moved from `setup.py` / `setup.cfg` to `pyproject.toml`; `python_requires` raised to `>=3.10`, matching the tested versions. +- [Development] Linting moved from flake8 to ruff (`ruff check .`, line length 120); new `dev` extra, pre-commit hooks, `.editorconfig`, `AGENTS.md`, `SECURITY.md` and issue / PR templates. ### Fixed - [Core] Serialization, executor IDs, `FuturesList`, module inspection, `chunksize=0`, packaging, ports, and Ctrl+C/`sys.exit()`. diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 000000000..2d4c14ce5 --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,18 @@ +# CLAUDE.md + +Claude Code reads this file automatically. The project instructions shared by all AI coding +agents live in [AGENTS.md](AGENTS.md) and are imported here, so there is a single source of +truth: + +@AGENTS.md + +## Claude Code specifics + +- Use the Python interpreter that has this checkout installed in editable mode + (`pip show lithops` shows the *Editable project location*). +- Run `ruff check .` and the localhost test suite (with `--backend localhost --storage + localhost`) before reporting a change as done. +- Long-running jobs (full test suite, Docker / runtime builds, docs builds) should run in the + background. +- Ask before any action that uses cloud accounts, builds or pushes runtime images, or + publishes content. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 27519bbfb..18ebbc51e 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,54 +1,122 @@ -Contributing to Lithops -=================== +# Contributing to Lithops -Reporting bugs and asking questions ------------------------------------ +Thanks for your interest in improving Lithops! This guide explains how to get help, report +problems, set up a development environment, and get a change merged. -You can post questions or issues or feedback through the following channels: +## Questions, bugs and ideas -1. [GitHub Discussions](https://github.com/lithops-cloud/lithops/discussions): For discussions about development, questions about usage, and feature requests. -2. [GitHub Issues](https://github.com/lithops-cloud/lithops/issues): For bug reports and feature requests. +- **Questions and ideas:** use [GitHub Discussions](https://github.com/lithops-cloud/lithops/discussions) + for usage questions, design discussions and feature ideas. +- **Bugs:** open a [GitHub issue](https://github.com/lithops-cloud/lithops/issues) with the bug + template. Include the Lithops version (`lithops --version`), the compute and storage + backends, a minimal script that reproduces the problem and the full traceback (ideally with + `log_level: DEBUG`). Remove credentials and account IDs from any configuration you paste. +- **Larger changes** (new backends, API changes, changes to the execution flow): open an issue + or discussion first, so we can agree on the approach before you write the code. +- **Security issues:** do **not** open a public issue, see [SECURITY.md](SECURITY.md). +## Development setup -To contribute a patch: ----------------------- +```bash +git clone https://github.com//lithops && cd lithops # your fork +python3 -m venv .venv && source .venv/bin/activate # optional, recommended +pip3 install -e '.[tests,dev]' # editable install + test deps + ruff and pre-commit +pip3 install -e '.[aws]' # add the extras of the backends you work on + # (aws, gcp, azure, ibm, aliyun, oracle, ... or all) +pre-commit install # optional: run ruff on every commit +``` -1. Break your work into small, single-purpose patches if possible. It's much - harder to merge in a large change with a lot of disjoint features. -2. Submit the patch as a GitHub pull request against the master branch. -3. Make sure that your code passes the tests. -4. Make sure that your code passes the linter. Install `flake8` with `pip3 install flake8` and run the following command until you see no linting errors: - ```bash - flake8 lithops --count --max-line-length=180 --statistics --ignore W605,W503 - ``` -5. Add new tests for your code. +The localhost backend needs no cloud account and no configuration, so it is all you need for +most changes. To work on a cloud backend, configure it as described in the +[configuration guide](config/README.md). +## Running the tests -Testing -------- +The tests live in `lithops/tests/` and are run from that folder: -To test that everything is working as expected, you must install `pytest`, navigate to the tests folder `lithops/tests/`, and execute: ```bash -pytest -v +cd lithops/tests +pytest -v --timeout=120 --timeout-method=thread --backend localhost --storage localhost ``` -If you made changes to a specific backend, please run tests on that backend. -For example, if you made changes to the AWS Lambda backend, execute the tests with: -```bash -pytest -v --backend aws_lambda --storage aws_s3 -``` +Always pass `--backend` and `--storage`: without them the tests use your own Lithops +configuration (`~/.lithops/config`, `.lithops_config` or `LITHOPS_CONFIG_FILE`) and run +against the cloud account it points to. Some tests need a Redis server on `localhost:6379` +(for example `docker run -d -p 6379:6379 redis:7`) and are skipped when none is reachable. + +Other useful invocations: -You can list all the available tests using: ```bash -pytest --collect-only +pytest -v --backend localhost --storage localhost -k test_map # a test or group of tests +pytest --collect-only # list all the tests +pytest -o log_cli=true --log-cli-level=DEBUG --backend localhost --storage localhost # with logs ``` -To run a specific test or group of tests, use the `-k` parameter, for example: +If you change a specific backend, also run the tests on that backend, for example: + ```bash -pytest -v --backend localhost --storage localhost -k test_map +pytest -v --backend aws_lambda --storage aws_s3 +pytest -v --config /path/to/config.yaml --backend code_engine --storage ibm_cos --region eu-de ``` -To view all the Lithops logs during the tests, and in DEBUG mode, execute: -```bash -pytest -o log_cli=true --log-cli-level=DEBUG --backend localhost --storage localhost +CI runs the localhost suite on Python 3.10 - 3.14 for every pull request; cloud backends are +not exercised in CI, so mention in your pull request which backend you tested on. + +## Code conventions + +- **Linting:** `ruff check .` must pass (`ruff check --fix .` fixes what it can). The + configuration (rules, line length 120) is in `pyproject.toml`; CI runs the same check. +- **Style:** follow the style of the surrounding code. The editor settings are in + `.editorconfig`. Do not run `ruff format`: the code base is not formatted with it. +- **Dependencies:** package metadata, dependencies and extras are declared in + `pyproject.toml`. A dependency added to an extra also goes into the `all` extra. +- **Tests:** every bug fix comes with a regression test, and every feature with tests of its + behaviour. Tests must pass on the localhost backend; code that only runs in a cloud + backend is tested with fakes or mocks. +- **Backends:** backend-specific code stays in its package (`lithops/serverless/backends/`, + `lithops/standalone/backends/`, `lithops/storage/backends/`). Import the provider SDKs + there, not in the core modules, so Lithops keeps working without the extras installed. +- **Configuration keys:** document new or changed keys in `config/config_template.yaml` and + in the backend's page under `docs/source/compute_config/` or `docs/source/storage_config/` + (general `lithops` keys in `docs/source/lithops_config_keys.csv`). +- **Public API:** keep `FunctionExecutor`, `Storage`, futures and the `multiprocessing` / + `concurrent.futures` APIs backwards compatible, or discuss the change in an issue first. + +## Documentation + +The documentation is written in reStructuredText and Markdown under `docs/` and built with +Sphinx; see [docs/README.md](docs/README.md) for how to build it locally. Update it together +with any user-visible change. + +## Changelog + +Add an entry for every user-visible change to [CHANGELOG.md](CHANGELOG.md), under the topmost +(development) version, in the *Added*, *Changed*, *Fixed* or *Removed* section, prefixed with +the component: + +```markdown +### Fixed +- [AWS Lambda] Short description of the fix. ``` + +## Pull requests + +1. Break your work into small, single-purpose pull requests. A large change with many + unrelated parts is much harder to review and merge. +2. Open the pull request against the `master` branch and fill in the template: what changed, + why, and how you tested it. +3. Make sure ruff and the localhost tests pass, and that tests, docs and the changelog are + updated. +4. By opening a pull request you certify the Developer's Certificate of Origin included in the + pull request template. + +## AI coding agents + +Instructions for AI coding agents (Claude Code, Codex, Copilot, Cursor, ...) are in +[AGENTS.md](AGENTS.md). Contributions made with their help follow the same rules as any other: +you are responsible for reviewing, testing and understanding the code you submit. + +## License + +By contributing you agree that your contributions are licensed under the +[Apache License 2.0](LICENSE). diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 000000000..6ba745bf4 --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,30 @@ +# Security policy + +## Reporting a vulnerability + +Please report security issues **privately** through +[GitHub security advisories](https://github.com/lithops-cloud/lithops/security/advisories/new) +instead of opening a public issue. You will get an acknowledgement within a few working days. + +## Security model (read before deploying) + +- **The storage bucket must be trusted.** Functions, their arguments, results and statuses + travel through the Lithops storage bucket as Python pickles (`cloudpickle`). Anyone who can + write to that bucket (and its `lithops.jobs` / runtime prefixes) can execute code on the + workers and on the client that reads the results. Restrict write access with bucket + policies / IAM. +- **Workers run your code with the permissions of the runtime identity** (Lambda execution + role, Batch job role, Code Engine / Kubernetes service account, VM instance profile). Grant + least privilege: only the buckets and services your functions need. +- **Credentials live in the Lithops configuration** (`~/.lithops/config`, `.lithops_config`, + `/etc/lithops/config`, `LITHOPS_CONFIG_FILE` or a config dict) or come from the cloud + provider's default credential chain. Keep configuration files out of version control and + readable only by their owner; prefer provider credential chains and short-lived tokens over + static keys. +- **Standalone mode** opens SSH connections to the VMs it creates and runs a master service + on them. Keep the security groups / firewall rules Lithops creates restricted to the client, + and delete idle VMs (`lithops clean`). +- **Runtime images** are built from the Dockerfiles in `runtime/` and the base image you + choose; keep them patched and pull them only from registries you control. +- Lithops is **not a sandbox**: do not run untrusted functions or data with privileged + credentials. diff --git a/docs/Dockerfile b/docs/Dockerfile index 0e0317f25..5af7c5963 100644 --- a/docs/Dockerfile +++ b/docs/Dockerfile @@ -11,9 +11,9 @@ RUN pip install --no-cache-dir sphinx myst-parser sphinx_copybutton jupyter ipyk ADD lithops lithops -COPY setup.py ./ +COPY pyproject.toml LICENSE ./ -RUN python setup.py install +RUN pip install --no-cache-dir . ADD docs docs diff --git a/docs/source/api_futures.rst b/docs/source/api_futures.rst index b6edd8864..e61cf6e15 100644 --- a/docs/source/api_futures.rst +++ b/docs/source/api_futures.rst @@ -56,7 +56,8 @@ Core API Reference :show-inheritance: ``map()`` and ``map_reduce()`` return a :class:`~lithops.utils.FuturesList`, which -is a list of futures that can be mapped over again. See +is a list of futures that can be mapped over again. A plain list or tuple of +those futures is a chain too, including a slice or a comprehension. See :doc:`Function chaining `. .. autoclass:: lithops.utils.FuturesList diff --git a/docs/source/api_multiprocessing.rst b/docs/source/api_multiprocessing.rst index 76aa98628..1cbbcec72 100644 --- a/docs/source/api_multiprocessing.rst +++ b/docs/source/api_multiprocessing.rst @@ -77,11 +77,6 @@ so a few things of it have no counterpart: * - ``RLock`` - Only re-entrant for the object that took it. A copy of it in another process, or one restored from a pickle, does not know the lock is held - * - ``Semaphore.acquire()``, ``Lock.acquire()`` - - Take ``block``, but no ``timeout`` - * - ``Condition.wait(timeout)`` - - A wait that timed out leaves its token behind, so the next - ``notify()`` may wake nobody. ``notify_all()`` is not affected * - ``RawArray('c', ...)`` - Not implemented. Use ``Array('c', ...)`` * - ``Process.close()`` diff --git a/docs/source/contributing.rst b/docs/source/contributing.rst index ba8eeb1c4..185d43c05 100644 --- a/docs/source/contributing.rst +++ b/docs/source/contributing.rst @@ -3,61 +3,142 @@ Contributing to Lithops ======================= -Reporting bugs and asking questions ------------------------------------ +Thanks for your interest in improving Lithops! This guide explains how to get help, report +problems, set up a development environment, and get a change merged. -You can post questions or issues or feedback through the following channels: -1. `GitHub Discussions `_: For discussions about development, questions about usage, and feature requests. -2. `GitHub Issues `_: For bug reports and feature requests. +Questions, bugs and ideas +------------------------- +- **Questions and ideas:** use `GitHub Discussions `_ + for usage questions, design discussions and feature ideas. +- **Bugs:** open a `GitHub issue `_ with the bug + template. Include the Lithops version (``lithops --version``), the compute and storage + backends, a minimal script that reproduces the problem and the full traceback (ideally with + ``log_level: DEBUG``). Remove credentials and account IDs from any configuration you paste. +- **Larger changes** (new backends, API changes, changes to the execution flow): open an issue + or discussion first, so we can agree on the approach before you write the code. +- **Security issues:** do **not** open a public issue, see + `SECURITY.md `_. -To contribute a patch ---------------------- -1. Break your work into small, single-purpose patches if possible. It's much - harder to merge in a large change with a lot of disjoint features. -2. Submit the patch as a GitHub pull request against the master branch. -3. Make sure that your code passes the tests. -4. Make sure that your code passes the linter. Install ``flake8`` with ``pip3 install flake8`` and run the following command until you don't see any linting errors: +Development setup +----------------- - .. code:: bash +.. code:: bash - flake8 lithops --count --max-line-length=180 --statistics --ignore W605,W503 + git clone https://github.com//lithops && cd lithops # your fork + python3 -m venv .venv && source .venv/bin/activate # optional, recommended + pip3 install -e '.[tests,dev]' # editable install + test deps + ruff and pre-commit + pip3 install -e '.[aws]' # add the extras of the backends you work on + # (aws, gcp, azure, ibm, aliyun, oracle, ... or all) + pre-commit install # optional: run ruff on every commit -5. Add new tests for your code. +The localhost backend needs no cloud account and no configuration, so it is all you need for +most changes. To work on a cloud backend, configure it as described in +:doc:`configuration`. -Testing -------- +Running the tests +----------------- -To test that all is working as expected, install ``pytest``, navigate to the tests folder ``lithops/tests/``, and execute: +The tests live in ``lithops/tests/`` and are run from that folder: .. code:: bash - pytest -v + cd lithops/tests + pytest -v --timeout=120 --timeout-method=thread --backend localhost --storage localhost + +Always pass ``--backend`` and ``--storage``: without them the tests use your own Lithops +configuration (``~/.lithops/config``, ``.lithops_config`` or ``LITHOPS_CONFIG_FILE``) and run +against the cloud account it points to. Some tests need a Redis server on ``localhost:6379`` +(for example ``docker run -d -p 6379:6379 redis:7``) and are skipped when none is reachable. -If you made changes to a specific backend, please run tests on that backend. -For example, if you made changes to the AWS Lambda backend, execute the tests with: +Other useful invocations: .. code:: bash - pytest -v --backend aws_lambda --storage aws_s3 + pytest -v --backend localhost --storage localhost -k test_map # a test or group of tests + pytest --collect-only # list all the tests + pytest -o log_cli=true --log-cli-level=DEBUG --backend localhost --storage localhost # with logs -You can list all the available tests using: +If you change a specific backend, also run the tests on that backend, for example: .. code:: bash - pytest --collect-only + pytest -v --backend aws_lambda --storage aws_s3 + pytest -v --config /path/to/config.yaml --backend code_engine --storage ibm_cos --region eu-de -To run a specific test or group of tests, use the ``-k`` parameter, for example: +CI runs the localhost suite on Python 3.10 - 3.14 for every pull request; cloud backends are +not exercised in CI, so mention in your pull request which backend you tested on. -.. code:: bash - pytest -v --backend localhost --storage localhost -k test_map +Code conventions +---------------- -To view all the Lithops logs during the tests in DEBUG mode, execute: +- **Linting:** ``ruff check .`` must pass (``ruff check --fix .`` fixes what it can). The + configuration (rules, line length 120) is in ``pyproject.toml``; CI runs the same check. +- **Style:** follow the style of the surrounding code. The editor settings are in + ``.editorconfig``. Do not run ``ruff format``: the code base is not formatted with it. +- **Dependencies:** package metadata, dependencies and extras are declared in + ``pyproject.toml``. A dependency added to an extra also goes into the ``all`` extra. +- **Tests:** every bug fix comes with a regression test, and every feature with tests of its + behaviour. Tests must pass on the localhost backend; code that only runs in a cloud + backend is tested with fakes or mocks. +- **Backends:** backend-specific code stays in its package (``lithops/serverless/backends/``, + ``lithops/standalone/backends/``, ``lithops/storage/backends/``). Import the provider SDKs + there, not in the core modules, so Lithops keeps working without the extras installed. +- **Configuration keys:** document new or changed keys in ``config/config_template.yaml`` and + in the backend's page under ``docs/source/compute_config/`` or ``docs/source/storage_config/`` + (general ``lithops`` keys in ``docs/source/lithops_config_keys.csv``). +- **Public API:** keep ``FunctionExecutor``, ``Storage``, futures and the ``multiprocessing`` / + ``concurrent.futures`` APIs backwards compatible, or discuss the change in an issue first. -.. code:: bash - pytest -o log_cli=true --log-cli-level=DEBUG --backend localhost --storage localhost +Documentation +------------- + +The documentation is written in reStructuredText and Markdown under ``docs/`` and built with +Sphinx; see `docs/README.md `_ for how to build it locally. Update it +together with any user-visible change. + + +Changelog +--------- + +Add an entry for every user-visible change to `CHANGELOG.md `_, under the +topmost (development) version, in the *Added*, *Changed*, *Fixed* or *Removed* section, prefixed +with the component: + +.. code:: markdown + + ### Fixed + - [AWS Lambda] Short description of the fix. + + +Pull requests +------------- + +1. Break your work into small, single-purpose pull requests. A large change with many + unrelated parts is much harder to review and merge. +2. Open the pull request against the ``master`` branch and fill in the template: what changed, + why, and how you tested it. +3. Make sure ruff and the localhost tests pass, and that tests, docs and the changelog are + updated. +4. By opening a pull request you certify the Developer's Certificate of Origin included in the + pull request template. + + +AI coding agents +---------------- + +Instructions for AI coding agents (Claude Code, Codex, Copilot, Cursor, ...) are in +`AGENTS.md `_. Contributions made with their help follow the same rules as any +other: you are responsible for reviewing, testing and understanding the code you submit. + + +License +------- + +By contributing you agree that your contributions are licensed under the +`Apache License 2.0 `_. diff --git a/docs/source/monitoring.rst b/docs/source/monitoring.rst index 9425d58b9..d2e23feee 100644 --- a/docs/source/monitoring.rst +++ b/docs/source/monitoring.rst @@ -90,8 +90,8 @@ finishes normally. A message backend is a faster path for the same information, **Resources are cleaned up on exit.** A message backend creates one queue, topic or list per executor, named after the executor id. It is created before the first function is invoked and deleted when the executor shuts down — when the -``with`` block ends, or on interpreter exit. A process killed hard enough to skip that leaves the resource behind, and -it has to be removed by hand. +``with`` block ends, or on interpreter exit. A process killed hard enough to skip that leaves the resource behind. +A RabbitMQ queue expires on its own after 24 hours; the others have to be removed by hand. **Nested executors work.** A function may create a ``FunctionExecutor`` of its own. Its call statuses reach every executor up the chain, so a client waiting on the outer job still sees the progress of the inner one. diff --git a/docs/source/notebooks/function_chaining.ipynb b/docs/source/notebooks/function_chaining.ipynb index adcddee7d..84d487b4a 100644 --- a/docs/source/notebooks/function_chaining.ipynb +++ b/docs/source/notebooks/function_chaining.ipynb @@ -271,7 +271,7 @@ "together with chaining and raises a `ValueError`. Return the extra values from the previous function instead, or\n", "get the results of the chain and start a new job with them.\n", "\n", - "Any list of futures of a previous job works as the input of the next one, not only the list `map()` returned, so\n", + "Any list or tuple of futures of a previous job works as the input of the next one, not only the list `map()` returned, so\n", "a slice or a comprehension chains the same way:" ] }, diff --git a/lithops/serverless/backends/aws_lambda/config.py b/lithops/serverless/backends/aws_lambda/config.py index 853b6e599..a1c9cfcb5 100644 --- a/lithops/serverless/backends/aws_lambda/config.py +++ b/lithops/serverless/backends/aws_lambda/config.py @@ -109,8 +109,9 @@ def load_config(config_data): "vpc/subnet' section".format(type(config_data['aws_lambda']['vpc']['subnets']))) if not isinstance(config_data['aws_lambda']['vpc']['security_groups'], list): + security_groups = config_data['aws_lambda']['vpc']['security_groups'] raise Exception("Unknown type {} for 'aws_lambda/" - "vpc/security_groups' section".format(type(config_data['aws_lambda']['vpc']['security_groups']))) + "vpc/security_groups' section".format(type(security_groups))) if not isinstance(config_data['aws_lambda']['efs'], list): raise Exception("Unknown type {} for " diff --git a/lithops/serverless/backends/azure_containers/azure_containers.py b/lithops/serverless/backends/azure_containers/azure_containers.py index f7a850db2..b64b34e49 100644 --- a/lithops/serverless/backends/azure_containers/azure_containers.py +++ b/lithops/serverless/backends/azure_containers/azure_containers.py @@ -282,7 +282,8 @@ def _create_app(self, runtime_name, memory, timeout): ca_template['properties']['environmentId'] = self._get_managed_environment_id() - cmd = f"az storage account show-connection-string -g {self.resource_group} --name {self.storage_account_name} --query connectionString --out json" + cmd = (f"az storage account show-connection-string -g {self.resource_group} " + f"--name {self.storage_account_name} --query connectionString --out json") queueconnection = self._run_az_command(cmd, return_result=True) ca_template['properties']['configuration']['secrets'][0]['value'] = queueconnection diff --git a/lithops/serverless/backends/gcp_cloudrun/cloudrun.py b/lithops/serverless/backends/gcp_cloudrun/cloudrun.py index d8d9667bb..16e3eb347 100644 --- a/lithops/serverless/backends/gcp_cloudrun/cloudrun.py +++ b/lithops/serverless/backends/gcp_cloudrun/cloudrun.py @@ -409,8 +409,9 @@ def _create_service(self, runtime_name, runtime_memory, timeout): svc_res['spec']['template']['spec']['containerConcurrency'] = 1 svc_res['spec']['template']['spec']['serviceAccountName'] = self.service_account svc_res['spec']['template']['metadata']['labels']['lithops-version'] = __version__.replace('.', '-') - svc_res['spec']['template']['metadata']['annotations']['autoscaling.knative.dev/minScale'] = str(self.cr_config['min_workers']) - svc_res['spec']['template']['metadata']['annotations']['autoscaling.knative.dev/maxScale'] = str(self.cr_config['max_workers']) + annotations = svc_res['spec']['template']['metadata']['annotations'] + annotations['autoscaling.knative.dev/minScale'] = str(self.cr_config['min_workers']) + annotations['autoscaling.knative.dev/maxScale'] = str(self.cr_config['max_workers']) container = svc_res['spec']['template']['spec']['containers'][0] container['image'] = img_name diff --git a/lithops/serverless/backends/gcp_functions/gcp_functions.py b/lithops/serverless/backends/gcp_functions/gcp_functions.py index 564e5877c..a4fc462d4 100644 --- a/lithops/serverless/backends/gcp_functions/gcp_functions.py +++ b/lithops/serverless/backends/gcp_functions/gcp_functions.py @@ -288,7 +288,9 @@ def _create_function(self, runtime_name, memory, timeout=60): if attempt < self.num_retries - 1: time.sleep(self.retry_sleep) else: - raise Exception(f"Failed to create Cloud Run function (v2) after {self.num_retries} attempts.") from e + raise Exception( + f"Failed to create Cloud Run function (v2) after {self.num_retries} attempts." + ) from e # Wait until the function is completely deployed logger.info('Waiting for the function to be deployed') diff --git a/lithops/serverless/backends/k8s/k8s.py b/lithops/serverless/backends/k8s/k8s.py index 6867b55c2..a39f7c1da 100644 --- a/lithops/serverless/backends/k8s/k8s.py +++ b/lithops/serverless/backends/k8s/k8s.py @@ -68,7 +68,8 @@ def __init__(self, k8s_config, internal_storage): context = None if self.kubecfg_context == 'default' else self.kubecfg_context load_kube_config(config_file=self.kubecfg_path, context=context) contexts, current_context = list_kube_config_contexts(config_file=self.kubecfg_path) - current_context = current_context if context is None else [it for it in contexts if it['name'] == context][0] + if context is not None: + current_context = [it for it in contexts if it['name'] == context][0] ctx_name = current_context.get('name') ctx_context = current_context.get('context') self.namespace = ctx_context.get('namespace') or self.namespace diff --git a/lithops/serverless/backends/knative/knative.py b/lithops/serverless/backends/knative/knative.py index 02eb5ab28..e0843a3aa 100644 --- a/lithops/serverless/backends/knative/knative.py +++ b/lithops/serverless/backends/knative/knative.py @@ -445,7 +445,8 @@ def _create_service(self, runtime_name, runtime_memory, timeout): svc_res['spec']['template']['spec']['timeoutSeconds'] = timeout svc_res['spec']['template']['spec']['containerConcurrency'] = 1 svc_res['spec']['template']['metadata']['labels']['lithops-version'] = __version__.replace('.', '-') - svc_res['spec']['template']['metadata']['annotations']['autoscaling.knative.dev/maxScale'] = str(self.kn_config['max_workers']) + annotations = svc_res['spec']['template']['metadata']['annotations'] + annotations['autoscaling.knative.dev/maxScale'] = str(self.kn_config['max_workers']) container = svc_res['spec']['template']['spec']['containers'][0] container['image'] = runtime_name diff --git a/lithops/serverless/backends/singularity/singularity.py b/lithops/serverless/backends/singularity/singularity.py index 79ed5f570..5a5408caf 100644 --- a/lithops/serverless/backends/singularity/singularity.py +++ b/lithops/serverless/backends/singularity/singularity.py @@ -85,7 +85,8 @@ def build_runtime(self, singularity_image_name, singularityfile, extra_args=[]): cmd = f'{singularity_path} build ' + ' '.join(extra_args) + f' {singularity_image_path} {singularityfile}' else: default_singularityfile = self._create_default_runtime() - cmd = f'{singularity_path} build ' + ' '.join(extra_args) + f' {singularity_image_path} {default_singularityfile}' + cmd = (f'{singularity_path} build ' + ' '.join(extra_args) + + f' {singularity_image_path} {default_singularityfile}') try: entry_point = os.path.join(os.path.dirname(__file__), 'entry_point.py') diff --git a/lithops/standalone/backends/aws_ec2/aws_ec2.py b/lithops/standalone/backends/aws_ec2/aws_ec2.py index e5fb6d381..a713d2a21 100644 --- a/lithops/standalone/backends/aws_ec2/aws_ec2.py +++ b/lithops/standalone/backends/aws_ec2/aws_ec2.py @@ -772,7 +772,9 @@ def build_image(self, image_name, script_file, overwrite, include, extra_args=[] script = get_host_setup_script(lithops_pip_spec='lithops[aws,redis]') build_vm.get_ssh_client().upload_data_to_file(script, remote_script) logger.debug("Executing Lithops installation script. Be patient, this process can take up to 3 minutes") - build_vm.get_ssh_client().run_remote_command(f"chmod 777 {remote_script}; sudo {remote_script}; rm {remote_script};") + build_vm.get_ssh_client().run_remote_command( + f"chmod 777 {remote_script}; sudo {remote_script}; rm {remote_script};" + ) logger.debug("Lithops installation script finsihed") for src_dst_file in include: @@ -787,7 +789,9 @@ def build_image(self, image_name, script_file, overwrite, include, extra_args=[] remote_script = "/tmp/install_user_lithops.sh" build_vm.get_ssh_client().upload_local_file(script, remote_script) logger.debug(f"Executing user script '{script_file}'") - build_vm.get_ssh_client().run_remote_command(f"chmod 777 {remote_script}; sudo {remote_script}; rm {remote_script};") + build_vm.get_ssh_client().run_remote_command( + f"chmod 777 {remote_script}; sudo {remote_script}; rm {remote_script};" + ) logger.debug(f"User script '{script_file}' finsihed") build_vm_id = build_vm.get_instance_id() @@ -1439,8 +1443,12 @@ def _create_instance(self, user_data=None): LaunchSpecification['MinCount'] = 1 LaunchSpecification['MaxCount'] = 1 - LaunchSpecification["TagSpecifications"] = [{"ResourceType": "instance", "Tags": [{'Key': 'Name', 'Value': self.name}]}] - LaunchSpecification["InstanceInitiatedShutdownBehavior"] = 'terminate' if self.delete_on_dismantle else 'stop' + LaunchSpecification["TagSpecifications"] = [ + {"ResourceType": "instance", "Tags": [{'Key': 'Name', 'Value': self.name}]} + ] + LaunchSpecification["InstanceInitiatedShutdownBehavior"] = ( + 'terminate' if self.delete_on_dismantle else 'stop' + ) if user_data: LaunchSpecification['UserData'] = user_data diff --git a/lithops/standalone/backends/ibm_vpc/ibm_vpc.py b/lithops/standalone/backends/ibm_vpc/ibm_vpc.py index cf6252a2d..9a96b1327 100644 --- a/lithops/standalone/backends/ibm_vpc/ibm_vpc.py +++ b/lithops/standalone/backends/ibm_vpc/ibm_vpc.py @@ -667,7 +667,9 @@ def build_image(self, image_name, script_file, overwrite, include, extra_args=[] script = get_host_setup_script(lithops_pip_spec='lithops[ibm,redis]') build_vm.get_ssh_client().upload_data_to_file(script, remote_script) logger.debug("Executing Lithops installation script. Be patient, this process can take up to 3 minutes") - build_vm.get_ssh_client().run_remote_command(f"chmod 777 {remote_script}; sudo {remote_script}; rm {remote_script};") + build_vm.get_ssh_client().run_remote_command( + f"chmod 777 {remote_script}; sudo {remote_script}; rm {remote_script};" + ) logger.debug("Lithops installation script finsihed") for src_dst_file in include: @@ -682,7 +684,9 @@ def build_image(self, image_name, script_file, overwrite, include, extra_args=[] remote_script = "/tmp/install_user_lithops.sh" build_vm.get_ssh_client().upload_local_file(script, remote_script) logger.debug(f"Executing user script '{script_file}'") - build_vm.get_ssh_client().run_remote_command(f"chmod 777 {remote_script}; sudo {remote_script}; rm {remote_script};") + build_vm.get_ssh_client().run_remote_command( + f"chmod 777 {remote_script}; sudo {remote_script}; rm {remote_script};" + ) logger.debug(f"User script '{script_file}' finsihed") build_vm.stop() @@ -701,7 +705,9 @@ def build_image(self, image_name, script_file, overwrite, include, extra_args=[] logger.debug("Be patient, VM imaging can take up to 6 minutes") while True: - images = self.vpc_cli.list_images(name=image_name, resource_group_id=self.config['resource_group_id']).result['images'] + images = self.vpc_cli.list_images( + name=image_name, resource_group_id=self.config['resource_group_id'] + ).result['images'] if len(images) > 0: logger.debug(f"VM Image is being created. Current status: {images[0]['status']}") if images[0]['status'] == 'available': @@ -1424,7 +1430,9 @@ def validate_capabilities(self): if char != '\n': sockets.add(char) if len(sockets) != 1: - raise LithopsValidationError(f'Not using single CPU socket as specified, using {len(sockets)} sockets instead') + raise LithopsValidationError( + f'Not using single CPU socket as specified, using {len(sockets)} sockets instead' + ) RETRIABLE = ['list_vpcs', diff --git a/lithops/storage/backends/localhost/localhost.py b/lithops/storage/backends/localhost/localhost.py index 899e24aaa..84e00582a 100644 --- a/lithops/storage/backends/localhost/localhost.py +++ b/lithops/storage/backends/localhost/localhost.py @@ -72,7 +72,7 @@ def put_object(self, bucket_name, key, data): file_path = os.path.join(LITHOPS_TEMP_DIR, bucket_name, key) os.makedirs(os.path.dirname(file_path), exist_ok=True) - if data_type == bytes: + if data_type is bytes: with open(file_path, "wb") as f: f.write(data) elif hasattr(data, 'read'): diff --git a/lithops/storage/backends/oracle_oss/oracle_oss.py b/lithops/storage/backends/oracle_oss/oracle_oss.py index 7d18ee681..35400d34f 100644 --- a/lithops/storage/backends/oracle_oss/oracle_oss.py +++ b/lithops/storage/backends/oracle_oss/oracle_oss.py @@ -101,7 +101,8 @@ def put_object(self, bucket_name, key, data): try: self.os_client.put_object(self.namespace, bucket_name, key, data) - logger.debug('OSS Object {} uploaded to bucket {} - Size: {}'.format(key, bucket_name, sizeof_fmt(len(data)))) + logger.debug('OSS Object {} uploaded to bucket {} - Size: {}'.format( + key, bucket_name, sizeof_fmt(len(data)))) except oci.exceptions.ServiceError as e: logger.debug("ServiceError in put_object: %s", str(e)) raise StorageNoSuchKeyError(bucket_name, key) @@ -161,7 +162,8 @@ def download_file(self, bucket, key, file_name=None, extra_args={}, config=None) Download a file from the specified bucket and key in the object storage. :param bucket: Name of the bucket :param key: The key or path of the file in the object storage - :param file_name: (Optional) The name of the file to be saved locally. If not provided, the key is used as the file name + :param file_name: (Optional) The name of the file to be saved locally. If not provided, + the key is used as the file name :param extra_args: (Optional) Additional arguments for the download process. :return: True if the file is downloaded successfully, False otherwise. :rtype: bool @@ -189,7 +191,8 @@ def head_object(self, bucket_name, key): :param bucket: Name of the bucket :param key: The key under which the file is stored - :param (Optional) file_name: The local file path where the downloaded file will be written. If None, the key is used + :param (Optional) file_name: The local file path where the downloaded file will be written. + If None, the key is used :param extra_args: Additional arguments that may be passed to the function :return: True if the file was successfully downloaded :rtype: bool @@ -212,7 +215,8 @@ def delete_object(self, bucket_name, key): def delete_objects(self, bucket_name, keys_list): ''' - Deletes multiple objects from OCI Object Storage. The objects are identified by a list of keys in a specified bucket. + Deletes multiple objects from OCI Object Storage. The objects are identified by a list of keys + in a specified bucket. :param bucket_name: Name of the bucket :param keys_list: A list of keys under which the objects are stored @@ -248,13 +252,16 @@ def list_objects(self, bucket_name, prefix=None, match_pattern=None): :param bucket_name: Name of the bucket :param prefix: (Optional) Prefix to filter object names. Default is None :param match_pattern: (Optional) Match pattern to further filter object names. Default is None - :return: A list of dictionaries containing the keys and sizes of the objects that match the given prefix and match pattern + :return: A list of dictionaries containing the keys and sizes of the objects that match the given + prefix and match pattern :rtype: list of dict :raises StorageNoSuchKeyError: If the specified bucket does not exist or there is a service error ''' prefix = '' if prefix is None else prefix try: - res = self.os_client.list_objects(self.namespace, bucket_name, prefix=prefix, limit=1000, fields="name,size") + res = self.os_client.list_objects( + self.namespace, bucket_name, prefix=prefix, limit=1000, fields="name,size" + ) obj_list = [{'Key': obj.name, 'Size': obj.size} for obj in res.data.objects] return obj_list diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 000000000..26254f584 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,217 @@ +[build-system] +requires = ["setuptools>=77"] +build-backend = "setuptools.build_meta" + +[project] +name = "lithops" +dynamic = ["version"] +description = "Run unmodified Python code at massive scale on any cloud, with no servers to manage" +keywords = [ + "serverless", + "distributed-computing", + "parallel-computing", + "multicloud", + "faas", + "kubernetes", + "hpc", + "big-data", + "aws-lambda", + "map-reduce", +] +authors = [ + { name = "Gil Vernik", email = "gilv@ibm.com" }, + { name = "Josep Sampe", email = "josep.sampe@gmail.com" }, +] +license = "Apache-2.0" +license-files = ["LICENSE"] +requires-python = ">=3.10" +classifiers = [ + "Development Status :: 5 - Production/Stable", + "Intended Audience :: Developers", + "Operating System :: OS Independent", + "Natural Language :: English", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", + "Programming Language :: Python :: 3.14", + "Topic :: Software Development :: Libraries :: Python Modules", + "Topic :: Scientific/Engineering", + "Topic :: System :: Distributed Computing", +] +dependencies = [ + "Click", + "tabulate", + "six", + "PyYAML", + "pika", + "tqdm", + "tblib", + "requests", + "paramiko", + "cloudpickle", + "ps-mem", + "psutil", +] + +[project.optional-dependencies] +ibm = [ + "ibm-cos-sdk", + "ibm-code-engine-sdk", + "ibm-vpc", + "kubernetes", +] +aws = [ + "boto3", +] +gcp = [ + "httplib2", + "google-cloud-storage", + "google-cloud-pubsub", + "google-api-python-client", + "google-auth", +] +azure = [ + "azure-mgmt-resource", + "azure-mgmt-compute", + "azure-mgmt-network", + "azure-identity", + "azure-storage-blob", + "azure-storage-queue", +] +aliyun = [ + "alibabacloud-fc20230330>=4.7.0", + "oss2", +] +ceph = [ + "boto3", +] +knative = [ + "kubernetes", +] +kubernetes = [ + "kubernetes", +] +minio = [ + "boto3", +] +redis = [ + "redis", +] +multiprocessing = [ + "redis", + "pynng", +] +joblib = [ + "joblib", + "diskcache", + "numpy", + "redis", +] +plotting = [ + "pandas", + "matplotlib", + "seaborn", + "numpy", +] +oracle = [ + "oci", +] +tests = [ + "pytest", + "pytest-timeout", + "kubernetes", + "pika", + "ibm-cos-sdk", + "ibm-cloud-sdk-core", + "joblib", + "diskcache", + "numpy", + "redis", + "pandas", + "matplotlib", + "seaborn", + "flask", + "gevent", + "scikit-learn", +] +telemetry = [ + "prometheus-client", + "opentelemetry-sdk", + "opentelemetry-exporter-otlp-proto-http", +] +dev = [ + "ruff>=0.16,<0.17", + "pre-commit", +] +# union of every extra above except dev; keep it in sync when adding a dependency +all = [ + "alibabacloud-fc20230330>=4.7.0", + "azure-identity", + "azure-mgmt-compute", + "azure-mgmt-network", + "azure-mgmt-resource", + "azure-storage-blob", + "azure-storage-queue", + "boto3", + "diskcache", + "flask", + "gevent", + "google-api-python-client", + "google-auth", + "google-cloud-pubsub", + "google-cloud-storage", + "httplib2", + "ibm-cloud-sdk-core", + "ibm-code-engine-sdk", + "ibm-cos-sdk", + "ibm-vpc", + "joblib", + "kubernetes", + "matplotlib", + "numpy", + "oci", + "opentelemetry-exporter-otlp-proto-http", + "opentelemetry-sdk", + "oss2", + "pandas", + "pika", + "prometheus-client", + "pynng", + "pytest", + "pytest-timeout", + "redis", + "scikit-learn", + "seaborn", +] + +[project.urls] +Homepage = "https://github.com/lithops-cloud/lithops" +Documentation = "https://lithops-cloud.github.io/docs/" +Source = "https://github.com/lithops-cloud/lithops" +Issues = "https://github.com/lithops-cloud/lithops/issues" +Changelog = "https://github.com/lithops-cloud/lithops/blob/master/CHANGELOG.md" + +[project.scripts] +lithops = "lithops.scripts.cli:lithops_cli" + +[tool.setuptools.dynamic] +version = { attr = "lithops.version.__version__" } + +[tool.setuptools.packages.find] +include = ["lithops*"] + +[tool.ruff] +line-length = 120 +target-version = "py310" +# same scope flake8 had: the package only, not the examples, docs or runtime files +extend-exclude = ["examples", "docs", "runtime", "config"] +# apply the excludes to files passed explicitly too (pre-commit, editors) +force-exclude = true + +[tool.ruff.lint] +# the pycodestyle and pyflakes rules flake8 checked +select = ["E", "W", "F"] +ignore = [ + "W605", # invalid escape sequences in regex / docstring strings +] diff --git a/setup.cfg b/setup.cfg deleted file mode 100644 index 02cc59ebe..000000000 --- a/setup.cfg +++ /dev/null @@ -1,9 +0,0 @@ -[flake8] -max-line-length = 180 -extend-ignore = W605, W503 -exclude = - .git, - __pycache__, - build, - dist, - *.egg-info diff --git a/setup.py b/setup.py deleted file mode 100644 index 8af234063..000000000 --- a/setup.py +++ /dev/null @@ -1,147 +0,0 @@ -#!/usr/bin/env python -from setuptools import setup, find_packages -from itertools import chain - - -install_requires = [ - 'Click', - 'tabulate', - 'six', - 'PyYAML', - 'pika', - 'tqdm', - 'tblib', - 'requests', - 'paramiko', - 'cloudpickle', - 'ps-mem', - 'psutil' -] - - -extras_require = { - 'ibm': [ - 'ibm-cos-sdk', - 'ibm-code-engine-sdk', - 'ibm-vpc', - 'kubernetes', - ], - 'aws': [ - 'boto3' - ], - 'gcp': [ - 'httplib2', - 'google-cloud-storage', - 'google-cloud-pubsub', - 'google-api-python-client', - 'google-auth' - ], - 'azure': [ - 'azure-mgmt-resource', - 'azure-mgmt-compute', - 'azure-mgmt-network', - 'azure-identity', - 'azure-storage-blob', - 'azure-storage-queue' - ], - 'aliyun': [ - 'alibabacloud-fc20230330>=4.7.0', - 'oss2', - ], - 'ceph': [ - 'boto3' - ], - 'knative': [ - 'kubernetes', - ], - 'kubernetes': [ - 'kubernetes', - ], - 'minio': [ - 'boto3' - ], - 'redis': [ - 'redis' - ], - 'multiprocessing': [ - 'redis', - 'pynng' - ], - 'joblib': [ - 'joblib', - 'diskcache', - 'numpy', - 'redis' - ], - 'plotting': [ - 'pandas', - 'matplotlib', - 'seaborn', - 'numpy' - ], - 'oracle': [ - 'oci', - ], - 'tests': [ - 'pytest', - 'pytest-timeout', - 'kubernetes', - 'pika', - 'ibm-cos-sdk', - 'ibm-cloud-sdk-core', - 'joblib', - 'diskcache', - 'numpy', - 'redis', - 'pandas', - 'matplotlib', - 'seaborn', - 'flask', - 'gevent', - 'scikit-learn', - ], - 'telemetry': [ - 'prometheus-client', - 'opentelemetry-sdk', - 'opentelemetry-exporter-otlp-proto-http', - ] -} - - -extras_require["all"] = list(set(chain.from_iterable(extras_require.values()))) - - -# how to get version info into the project -exec(open('lithops/version.py').read()) -setup( - name='lithops', - version=__version__, - url='https://github.com/lithops-cloud/lithops', - author='Gil Vernik, Josep Sampe', - description='Lithops lets you transparently run your Python applications in the Cloud', - author_email='gilv@ibm.com, josep.sampe@gmail.com', - packages=find_packages(), - install_requires=install_requires, - extras_require=extras_require, - include_package_data=True, - entry_points=''' - [console_scripts] - lithops=lithops.scripts.cli:lithops_cli - ''', - classifiers=[ - 'Development Status :: 5 - Production/Stable', - 'Intended Audience :: Developers', - 'Operating System :: OS Independent', - 'Natural Language :: English', - 'License :: OSI Approved :: Apache Software License', - 'Programming Language :: Python :: 3.10', - 'Programming Language :: Python :: 3.11', - 'Programming Language :: Python :: 3.12', - 'Programming Language :: Python :: 3.13', - 'Programming Language :: Python :: 3.14', - 'Topic :: Software Development :: Libraries :: Python Modules', - 'Topic :: Scientific/Engineering', - 'Topic :: System :: Distributed Computing', - ], - python_requires='>=3.6', -)