diff --git a/.github/workflows/documentation_builder.yml b/.github/workflows/documentation_builder.yml index 5bc9aab..259f5f9 100644 --- a/.github/workflows/documentation_builder.yml +++ b/.github/workflows/documentation_builder.yml @@ -40,7 +40,7 @@ jobs: if [[ ${{github.event_name}} == "release" ]] then echo "Deploy doc for release: ${{ github.event.release.tag_name }}" - mike deploy --push --update-aliases ${{ github.event.release.tag_name }} latest || mike deploy --push ${{ github.event.release.tag_name }} + mike deploy --push --alias-type=redirect --update-aliases ${{ github.event.release.tag_name }} latest || mike deploy --push ${{ github.event.release.tag_name }} mike set-default --push latest else echo "Deploy doc for main" diff --git a/.github/workflows/validation.yml b/.github/workflows/validation.yml index 9279df3..f628e6c 100644 --- a/.github/workflows/validation.yml +++ b/.github/workflows/validation.yml @@ -28,13 +28,11 @@ jobs: - name: Install Formatting run: | python -m pip install --upgrade pip - pip install flake8 - pip install isort + pip install ruff - name: Check Formatting run: | - flake8 onecode tests/unit/ --exclude=__init__.py,tests/data --max-line-length=100 - isort . -m3 --thirdparty . --check-only --skip tests/data + ruff check onecode tests testing: name: Check Testing @@ -42,7 +40,7 @@ jobs: fail-fast: false matrix: os: [ubuntu-latest, windows-latest, macos-latest] - python: ["3.10", "3.11", "3.12", "3.13", "3.14"] + python: ["3.11", "3.12", "3.13", "3.14"] runs-on: ${{ matrix.os }} timeout-minutes: 30 steps: @@ -60,12 +58,11 @@ jobs: - name: Check Testing Linux/MacOS if: ${{ matrix.os != 'windows-latest' }} run: | - ONECODE_DO_TYPECHECK=1 python -m pytest tests -n auto + python -m pytest tests -n auto - name: Check Testing Windows if: ${{ matrix.os == 'windows-latest' }} run: | - set ONECODE_DO_TYPECHECK=1 python -m pytest tests coverage: @@ -92,12 +89,11 @@ jobs: - name: Test coverage Linux if: ${{ matrix.os == 'ubuntu-latest' }} run: | - ONECODE_DO_TYPECHECK=1 coverage run -m pytest tests --cov=./ --cov-report=xml + coverage run -m pytest tests --cov=./ --cov-report=xml - name: Test coverage Windows if: ${{ matrix.os == 'windows-latest' }} run: | - set ONECODE_DO_TYPECHECK=1 coverage run -m pytest tests --cov=./ --cov-report=xml - name: Upload coverage reports to Codecov diff --git a/.github/workflows/vulnerability_monitor.yml b/.github/workflows/vulnerability_monitor.yml new file mode 100644 index 0000000..6c990b4 --- /dev/null +++ b/.github/workflows/vulnerability_monitor.yml @@ -0,0 +1,117 @@ +name: Vulnerability monitor + +on: + pull_request: + branches: + - "*.x" + schedule: + - cron: "0 5 * * *" + workflow_dispatch: + inputs: + backfill: + description: Attach SBOMs to releases that do not have one + type: boolean + default: false + release: + types: [published] + +jobs: + ranges: + if: github.event_name == 'pull_request' + name: Dependency ranges + runs-on: ubuntu-latest + permissions: + contents: read + steps: + - uses: actions/checkout@v4 + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.12" + + - name: Install packaging + run: python -m pip install packaging + + - name: Compare ranges with OSV + continue-on-error: true + run: python scripts/cra/scan_ranges.py + + releases: + if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' + name: Release SBOMs + runs-on: ubuntu-latest + permissions: + contents: read + issues: write + steps: + - uses: actions/checkout@v4 + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.12" + + - name: Install packaging + run: python -m pip install packaging + + - name: Rescan release SBOMs + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + CRA_NOTIFY: theweaklink + run: python scripts/cra/monitor_releases.py + + backfill: + if: github.event_name == 'workflow_dispatch' && inputs.backfill + name: Backfill release SBOMs + runs-on: ubuntu-latest + permissions: + contents: write + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.12" + + - name: Attach missing SBOMs + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: python scripts/cra/backfill_sboms.py + + sbom: + if: github.event_name == 'release' + name: Publish release SBOM + runs-on: ubuntu-latest + permissions: + contents: write + steps: + - name: Checkout SBOM scripts + uses: actions/checkout@v4 + with: + path: workflow-src + + - name: Checkout release + uses: actions/checkout@v4 + with: + ref: ${{ github.event.release.tag_name }} + path: release-src + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.12" + + - name: Generate CycloneDX + run: | + python workflow-src/scripts/cra/make_release_sbom.py \ + --pyproject release-src/pyproject.toml \ + --out-dir sbom + + - name: Upload CycloneDX + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + gh release upload "${{ github.event.release.tag_name }}" sbom/*.cdx.json \ + --clobber \ + --repo deeplime-io/onecode diff --git a/README.md b/README.md index c3a9384..51263d6 100644 --- a/README.md +++ b/README.md @@ -217,6 +217,10 @@ for your patience on that one. ## :wave: Getting Help +Security reports go through [private vulnerability reporting](https://github.com/deeplime-io/onecode/security/advisories/new). See [SECURITY.md](SECURITY.md). Security fixes are published for the maintained `1.x` line. + +Issues labeled [`cra-vulnerability`](https://github.com/deeplime-io/onecode/issues?q=is%3Aissue+is%3Aopen+label%3Acra-vulnerability) come from the dependency scan. They are not the place to disclose a new flaw. + If you are a OneCode customer, you may directly email our support team. Feel free as well to browse the [GitHub Issues](https://github.com/deeplime-io/onecode/issues) and reach out to the community by posting bug reports, questions and suggestions. diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 0000000..72e4c90 --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,19 @@ +# Security + +OneCode is open source under the MIT license and is maintained by DeepLime, not as a community project. Security fixes are published for the maintained `1.x` line. + +## Reporting a vulnerability + +Report a vulnerability in OneCode through [GitHub private vulnerability reporting](https://github.com/deeplime-io/onecode/security/advisories/new). + +Do not open a public issue, and do not file the report in the GitHub Advisory Database. A public report discloses the problem before a fix is available. + +DeepLime acknowledges the report, prepares a fix on the maintained line, and releases it. After that release, DeepLime publishes a GitHub Security Advisory. That publication is what enters the public advisory database and OSV. + +If the vulnerability is actively exploited, DeepLime also reports it through the CRA single reporting platform operated with ENISA. That duty applies from 11 September 2026. An early warning is due within 24 hours of learning that exploitation is underway, followed by the fuller notification the platform requires. + +## Dependency monitor + +Issues labeled [`cra-vulnerability`](https://github.com/deeplime-io/onecode/issues?q=is%3Aissue+is%3Aopen+label%3Acra-vulnerability) are opened by the scheduled scan of release SBOMs. They record published advisories that intersect a released dependency range. They are not the channel for reporting a new vulnerability in OneCode. + +Each published GitHub release carries a CycloneDX file named `onecode-.cdx.json`. diff --git a/codecov.yml b/codecov.yml index bb7be40..73a4b1b 100644 --- a/codecov.yml +++ b/codecov.yml @@ -1,3 +1,13 @@ +coverage: + status: + project: + default: + target: 95% + patch: + default: + target: 95% + ignore: - - "tests/**/*" - - "tests/*" + - "tests" + - "onecode/pycg" + - "scripts" diff --git a/docs/changelogs/1.3.0.md b/docs/changelogs/1.3.0.md new file mode 100644 index 0000000..c291707 --- /dev/null +++ b/docs/changelogs/1.3.0.md @@ -0,0 +1,30 @@ +## Bug Fixes + +:octicons-issue-opened-24: Issue Ref | :fontawesome-solid-thumbtack: Summary | :material-message-text: Description +-|-|- + + +## Enhancements + +:octicons-issue-opened-24: Issue Ref | :fontawesome-solid-thumbtack: Summary | :material-message-text: Description +-|-|- +[No Ref] | Meta-data of input element to match expections from OneCode cloud | `metadata()` to be consistent with OneCode Cloud usage. +[No Ref] | Remove the `typeguard` dependency | `is_type` now checks values with the standard library. Every item of a list is checked. +[No Ref] | Raise minimum dependency versions | PyArrow, pydash, pytest, and Material for MkDocs now start at versions that include published security fixes. Other runtime, test, and docs dependencies move off release lines that predate the supported Python range. +[No Ref] | Vendor the call-graph library | The `onecode-pycg` dependency is removed. The call-graph code OneCode uses now lives in `onecode.pycg` under the Apache License 2.0. +[No Ref] | Check style with Ruff | Formatting CI uses Ruff instead of flake8 and isort. +[No Ref] | Publish a dependency SBOM and scan released versions | Each GitHub release carries a CycloneDX SBOM of the direct runtime dependency ranges. Pull requests report advisories that intersect those ranges, and a daily job updates one issue per affected release. Security reports go through private vulnerability reporting. See `SECURITY.md`. +[No Ref] | Lock the manifest with the operating system | Concurrent appends to the output manifest now use `filelock` (`fcntl` on Unix, `msvcrt` on Windows). The `flufl.lock` dependency is removed. + + +## New Features + +:octicons-issue-opened-24: Issue Ref | :fontawesome-solid-thumbtack: Summary | :material-message-text: Description +-|-|- + + +## :warning: Breaking changes + +Python 3.10 and older are no longer supported. OneCode now requires Python 3.11, 3.12, 3.13, or 3.14. + +The `check_type` decorator and the `ONECODE_DO_TYPECHECK` environment variable are removed. `Env.ONECODE_DO_TYPECHECK` is no longer available, and Pydantic is no longer a developer dependency. diff --git a/docs/index.md b/docs/index.md index cffe8b5..65366e9 100644 --- a/docs/index.md +++ b/docs/index.md @@ -5,5 +5,5 @@ ## Credits Credits to all open-source libraries that helped build this project. Special thanks to: -- [PyCG](https://github.com/vitsalis/PyCG) and its contributors for making the Call Graph algos essential to the OneCode mechanism. +- [PyCG](https://github.com/vitsalis/PyCG) (Vitalis Salis) and its contributors for the call-graph algorithms used by OneCode. The DeepLime fork is vendored in `onecode.pycg` under the Apache License 2.0. See `onecode/pycg/LICENCE`. The rest of OneCode remains MIT. - [Geode Solutions](https://geode-solutions.com/) and [Spotlight Earth](https://spotlight-earth.com/) for testing the open-source library and providing invaluable feedback. diff --git a/mkdocs.yml b/mkdocs.yml index e9ad57e..eb561b3 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -36,6 +36,7 @@ nav: - Project: reference/base/project.md - FAQs: faq.md - Changelogs: + - 1.3.0: changelogs/1.3.0.md - 1.2.2: changelogs/1.2.2.md - 1.2.1: changelogs/1.2.1.md - 1.2.0: changelogs/1.2.0.md @@ -69,7 +70,6 @@ plugins: show_source: false show_root_toc_entry: false separate_signature: false - new_path_syntax: yes filters: - "^_" - "^" diff --git a/onecode/__init__.py b/onecode/__init__.py index 3bb034d..9f5bdc3 100644 --- a/onecode/__init__.py +++ b/onecode/__init__.py @@ -1,7 +1,7 @@ # SPDX-FileCopyrightText: 2023-2024 DeepLime # SPDX-License-Identifier: MIT -__version__ = "1.2.2" +__version__ = "1.3.0.dev" from .base import * diff --git a/onecode/base/__init__.py b/onecode/base/__init__.py index 91dfcb6..eb8d363 100644 --- a/onecode/base/__init__.py +++ b/onecode/base/__init__.py @@ -1,7 +1,6 @@ # SPDX-FileCopyrightText: 2023-2024 DeepLime # SPDX-License-Identifier: MIT -from .decorator import * from .enums import * from .logger import * from .project import * diff --git a/onecode/base/decorator.py b/onecode/base/decorator.py deleted file mode 100644 index fe1eee9..0000000 --- a/onecode/base/decorator.py +++ /dev/null @@ -1,31 +0,0 @@ -# SPDX-FileCopyrightText: 2023-2024 DeepLime -# SPDX-License-Identifier: MIT - -import ast -import os -from typing import Callable - -from .enums import Env - -_do_type_check = ( - Env.ONECODE_DO_TYPECHECK in os.environ and - bool(ast.literal_eval(os.environ[Env.ONECODE_DO_TYPECHECK])) -) - -if _do_type_check: - from pydantic import validate_call - - -def check_type(func: Callable): # pragma: no cover - """ - Decorator forcing type checking at runtime. To control it, run your process with - the environment variable `ONECODE_DO_TYPECHECK = 1` - - Args: - func: Function to apply type-checking. - - """ - if _do_type_check: - return validate_call(func, config=dict(arbitrary_types_allowed=True)) - - return func diff --git a/onecode/base/enums.py b/onecode/base/enums.py index 29165a0..216aa26 100644 --- a/onecode/base/enums.py +++ b/onecode/base/enums.py @@ -1,8 +1,7 @@ # SPDX-FileCopyrightText: 2023-2024 DeepLime # SPDX-License-Identifier: MIT -# strenum to allow for code backward-compat between Python 3.10 and Python 3.11+ -from strenum import StrEnum +from enum import StrEnum class Env(StrEnum): @@ -13,16 +12,13 @@ class Env(StrEnum): :octicons-arrow-both-24: `"ONECODE_PROJECT_DATA"` - `ONECODE_CONFIG_FILE`: name of the file containing OneCode project configurations :octicons-arrow-both-24: `".onecode.json"` - - `ONECODE_DO_TYPECHECK`: set to 1 to force runtime type-checking with Pydantic - :octicons-arrow-both-24: `"ONECODE_DO_TYPECHECK"` - `ONECODE_LOGGER_NAME`: base logger name to avoid logging conflict with other loggers :octicons-arrow-both-24: `|OneCode|` """ - ONECODE_PROJECT_DATA = "ONECODE_PROJECT_DATA" # noqa: E-221 - ONECODE_CONFIG_FILE = ".onecode.json" # noqa: E-221 - ONECODE_DO_TYPECHECK = "ONECODE_DO_TYPECHECK" # noqa: E-221 - ONECODE_LOGGER_NAME = "|OneCode|" # noqa: E-221 + ONECODE_PROJECT_DATA = "ONECODE_PROJECT_DATA" # noqa: E221 + ONECODE_CONFIG_FILE = ".onecode.json" # noqa: E221 + ONECODE_LOGGER_NAME = "|OneCode|" # noqa: E221 class ConfigOption(StrEnum): @@ -39,11 +35,11 @@ class ConfigOption(StrEnum): typically to handle code using display :octicons-arrow-both-24: `"CLOUD_ENV": False` """ - FLUSH_STDOUT = "FLUSH_STDOUT" # noqa: E-221 - LOGGER_COLOR = "LOGGER_COLOR" # noqa: E-221 - LOGGER_TIMESTAMP = "LOGGER_TIMESTAMP" # noqa: E-221 - CHECK_MODULES = "CHECK_MODULES" # noqa: E-221 - CLOUD_ENV = "CLOUD_ENV" # noqa: E-221 + FLUSH_STDOUT = "FLUSH_STDOUT" # noqa: E221 + LOGGER_COLOR = "LOGGER_COLOR" # noqa: E221 + LOGGER_TIMESTAMP = "LOGGER_TIMESTAMP" # noqa: E221 + CHECK_MODULES = "CHECK_MODULES" # noqa: E221 + CLOUD_ENV = "CLOUD_ENV" # noqa: E221 class Mode(StrEnum): @@ -91,12 +87,12 @@ def do_my_new_mode(onecode_project_path: str): ``` """ - CONSOLE = "_console" # noqa: E-221 - EXECUTE = "_execute" # noqa: E-221 - LOAD_THEN_EXECUTE = "_load_then_execute" # noqa: E-221 - EXTRACT = "_extract" # noqa: E-221 - EXTRACT_ALL = "_extract_all" # noqa: E-221 - BUILD_GUI = "_build_gui" # noqa: E-221 + CONSOLE = "_console" # noqa: E221 + EXECUTE = "_execute" # noqa: E221 + LOAD_THEN_EXECUTE = "_load_then_execute" # noqa: E221 + EXTRACT = "_extract" # noqa: E221 + EXTRACT_ALL = "_extract_all" # noqa: E221 + BUILD_GUI = "_build_gui" # noqa: E221 class ElementType(StrEnum): @@ -108,8 +104,8 @@ class ElementType(StrEnum): - `OUTPUT`: `OutputElement` Type :octicons-arrow-both-24: `"OUTPUT"` """ - INPUT = "INPUT" # noqa: E-221 - OUTPUT = "OUTPUT" # noqa: E-221 + INPUT = "INPUT" # noqa: E221 + OUTPUT = "OUTPUT" # noqa: E221 class FileFilter(tuple): @@ -124,7 +120,7 @@ class FileFilter(tuple): * `ZIP` :octicons-arrow-both-24: `("ZIP", ".zip .gz .tar.gz .7z")` """ - CSV = ("CSV", ".csv") # noqa: E-221 - PYTHON = ("Python", ".py") # noqa: E-221 - IMAGE = ("Image", ".jpg .png .jpeg") # noqa: E-221 - ZIP = ("ZIP", ".zip .gz .tar.gz .7z") # noqa: E-221 + CSV = ("CSV", ".csv") # noqa: E221 + PYTHON = ("Python", ".py") # noqa: E221 + IMAGE = ("Image", ".jpg .png .jpeg") # noqa: E221 + ZIP = ("ZIP", ".zip .gz .tar.gz .7z") # noqa: E221 diff --git a/onecode/base/logger.py b/onecode/base/logger.py index 19417d5..02857bf 100644 --- a/onecode/base/logger.py +++ b/onecode/base/logger.py @@ -1,24 +1,18 @@ # SPDX-FileCopyrightText: 2023-2024 DeepLime # SPDX-License-Identifier: MIT -import ast import inspect import logging import os import sys from typing import Any, Optional -from .decorator import check_type from .enums import ConfigOption, Env from .project import Project from .singleton import Singleton -_do_type_check = ( - Env.ONECODE_DO_TYPECHECK in os.environ and - bool(ast.literal_eval(os.environ[Env.ONECODE_DO_TYPECHECK])) -) -_logger_stack_level = 6 if _do_type_check else 2 -_stack_level = 4 if _do_type_check else 2 +_logger_stack_level = 2 +_stack_level = 2 class ColoredFormatter(logging.Formatter): @@ -44,7 +38,6 @@ class ColoredFormatter(logging.Formatter): BOLD_RED = "\x1b[31;1m" RESET = "\x1b[0m" - @check_type def __init__( self, color: bool = True @@ -52,7 +45,6 @@ def __init__( super().__init__() self._color = color - @check_type def format( self, record: logging.LogRecord @@ -150,7 +142,6 @@ def reset( base_logger.propagate = False self.set_level(logging.INFO) - @check_type def add_handler( self, handler: Optional[logging.Handler] = None, @@ -182,7 +173,6 @@ def add_handler( namespace = Env.ONECODE_LOGGER_NAME if not root_logger else None logging.getLogger(namespace).addHandler(handler) - @check_type def set_level( self, level: int @@ -198,7 +188,6 @@ def set_level( """ logging.getLogger(Env.ONECODE_LOGGER_NAME).setLevel(level) - @check_type def logger( self, stacklevel: int = 1, @@ -232,7 +221,6 @@ def _flush() -> None: sys.stdout.flush() @staticmethod - @check_type def debug(msg: Any) -> None: """ Convenience function to log a debug message. @@ -245,7 +233,6 @@ def debug(msg: Any) -> None: Logger._flush() @staticmethod - @check_type def info(msg: Any) -> None: """ Convenience function to log an info message. @@ -258,7 +245,6 @@ def info(msg: Any) -> None: Logger._flush() @staticmethod - @check_type def warning(msg: Any) -> None: """ Convenience function to log a warning message. @@ -271,7 +257,6 @@ def warning(msg: Any) -> None: Logger._flush() @staticmethod - @check_type def error(msg: Any) -> None: """ Convenience function to log an error message. @@ -284,7 +269,6 @@ def error(msg: Any) -> None: Logger._flush() @staticmethod - @check_type def critical(msg: Any) -> None: """ Convenience function to log a critical message. diff --git a/onecode/base/project.py b/onecode/base/project.py index 1234fe0..9240e68 100644 --- a/onecode/base/project.py +++ b/onecode/base/project.py @@ -8,9 +8,8 @@ from typing import Any, Dict, Optional, Set, Union import pydash -from flufl.lock import Lock +from filelock import FileLock -from .decorator import check_type from .enums import ConfigOption, Env, Mode from .singleton import Singleton @@ -108,7 +107,6 @@ def registered_elements(self) -> Set[str]: """ return self._registered_elements - @check_type def register_element( self, element_name: str @@ -190,7 +188,6 @@ def data_root(self) -> str: """ return self._data_root - @check_type def _set_data_root( self, data_path: str @@ -212,7 +209,6 @@ def _set_data_root( self._data_root = data_path - @check_type def get_input_path( self, filepath: str @@ -232,7 +228,6 @@ def get_input_path( return filepath if not filepath or os.path.isabs(filepath) \ else os.path.join(self.data_root, filepath) - @check_type def get_output_path( self, filepath: str @@ -318,7 +313,6 @@ def data( """ self._data = data - @check_type def add_data( self, key: str, @@ -343,7 +337,6 @@ def add_data( self._data[key] = value - @check_type def set_config( self, key: Union[ConfigOption, str], @@ -365,7 +358,6 @@ def set_config( self._config[key] = value - @check_type def get_config( self, key: Union[ConfigOption, str] @@ -385,7 +377,6 @@ def get_config( return self._config[key] - @check_type def write_output( self, output: Dict @@ -404,9 +395,11 @@ def write_output( output: Output data to write to the manifest file. """ - manifest_dir = os.path.dirname(self.get_output_manifest()) + manifest = self.get_output_manifest() + lock_path = os.path.join(os.path.dirname(manifest), '.locks', 'MANIFEST.lock') - # manage concurrent access in case of multiprocessing - with Lock(os.path.join(manifest_dir, '.locks', 'MANIFEST.lock'), lifetime=3): - with open(self.get_output_manifest(), "a") as f: - f.write(f'{json.dumps(output)}\n') + # OS lock (fcntl on Unix, msvcrt on Windows). Released when the holder + # exits, so a slow process is not interrupted by a timeout. + with FileLock(lock_path, timeout=-1, preserve_lock_file=True): + with open(manifest, "a") as handle: + handle.write(f'{json.dumps(output)}\n') diff --git a/onecode/cli/add.py b/onecode/cli/add.py index 8072b9b..75fa29a 100644 --- a/onecode/cli/add.py +++ b/onecode/cli/add.py @@ -9,12 +9,10 @@ from slugify import slugify from yaspin import yaspin -from ..base.decorator import check_type from ..base.enums import Env from .utils import _add_flow, _get_flow_choices -@check_type def add( project_path: str, name: str, diff --git a/onecode/cli/build.py b/onecode/cli/build.py index 2266f5e..c5d566b 100644 --- a/onecode/cli/build.py +++ b/onecode/cli/build.py @@ -9,14 +9,12 @@ import onecode # noqa -from ..base.decorator import check_type from ..base.enums import * # noqa from ..base.enums import ElementType, Mode from ..base.project import Project from .utils import process_call_graph -@check_type def process(calls: List[Dict[str, str]]) -> Dict: """ Evaluate the given calls such as: @@ -80,7 +78,6 @@ def process(calls: List[Dict[str, str]]) -> Dict: return params -@check_type def extract_gui( project_path: str, to_file: str, diff --git a/onecode/cli/create.py b/onecode/cli/create.py index 6cb19e1..9145299 100644 --- a/onecode/cli/create.py +++ b/onecode/cli/create.py @@ -10,11 +10,9 @@ from slugify import slugify from yaspin import yaspin -from ..base.decorator import check_type from .utils import _add_flow -@check_type def create( project_path: str, name: str, diff --git a/onecode/cli/extract.py b/onecode/cli/extract.py index cb9d0ab..e1ed6f8 100644 --- a/onecode/cli/extract.py +++ b/onecode/cli/extract.py @@ -11,7 +11,6 @@ import onecode # noqa -from ..base.decorator import check_type from ..base.enums import * # noqa from ..base.enums import ElementType, Mode from ..base.project import Project @@ -19,7 +18,6 @@ from .utils import process_call_graph -@check_type def process(calls: List[Dict[str, str]]) -> Dict: """ Evaluate the given calls such as: @@ -83,7 +81,6 @@ def process(calls: List[Dict[str, str]]) -> Dict: return params -@check_type def extract_json( project_path: str, to_file: str, @@ -113,7 +110,6 @@ def extract_json( json.dump(parameters, out, indent=4) -@check_type def main(cli: bool = True) -> None: # pragma: no cover """ ```bash diff --git a/onecode/cli/skeleton/main.py b/onecode/cli/skeleton/main.py index cd1a592..6019ad4 100644 --- a/onecode/cli/skeleton/main.py +++ b/onecode/cli/skeleton/main.py @@ -20,7 +20,7 @@ Project, check_modules, get_imported_modules, - register_ext_module + register_ext_module, ) diff --git a/onecode/cli/utils.py b/onecode/cli/utils.py index a6727f0..68cbb1d 100644 --- a/onecode/cli/utils.py +++ b/onecode/cli/utils.py @@ -9,17 +9,16 @@ import pydash from InquirerPy.base.control import Choice -from pycg.pycg import CallGraphGenerator -from pycg.utils.constants import CALL_GRAPH_OP from slugify import slugify -from ..base.decorator import check_type +from onecode.pycg.pycg import CallGraphGenerator as _CallGraphGenerator +from onecode.pycg.utils.constants import CALL_GRAPH_OP as _CALL_GRAPH_OP + from ..base.enums import Env from ..base.project import Project from ..utils.module import get_call_graph_entry_files -@check_type def get_flows(project_path: str) -> Dict: """ Get the flows configuration as stored at the OneCode project's root (filename is given by the @@ -41,7 +40,6 @@ def get_flows(project_path: str) -> Dict: return config -@check_type def _get_flow_choices(project_path: str) -> List[Choice]: # pragma: no cover """ Internal function for CLI commands to get the existing OneCode project's flows as interactive @@ -60,7 +58,6 @@ def _get_flow_choices(project_path: str) -> List[Choice]: # pragma: no cover return choices -@check_type def _add_flow( project_path: str, name: Optional[str], @@ -164,7 +161,6 @@ def _resolve_graph_key(name: str, graph: Dict) -> str: return name -# check_type decorator not compatible with recursive calls def extract_calls( entry_point: str, graph: Dict, @@ -179,9 +175,8 @@ def extract_calls( Args: entry_point: Call Graph function name from which to start the extraction from, e.g. `flows.my_flow.run`. - graph: Call Graph typically constructed by the DeepLime forked PyCG. Check out PyCG for - more information about the graph structure or directly the forked repository at - https://github.com/deeplime-io/PyCG/tree/onecode + graph: Enriched call graph produced by the vendored generator in `onecode.pycg` + (Apache-2.0, derived from PyCG). calls: List of calls as `{"func": , "loc": }` where results are aggregated. These `calls` are typically piped to the `process` functions for JSON extraction. @@ -219,7 +214,6 @@ def extract_calls( extract_calls(next_point, graph, calls) -@check_type def process_call_graph( project_path: str = None, verbose: bool = False @@ -249,11 +243,11 @@ def process_call_graph( statements = OrderedDict() entry_files = get_call_graph_entry_files(project_path) - cg = CallGraphGenerator( + cg = _CallGraphGenerator( entry_files, project_path, -1, - CALL_GRAPH_OP + _CALL_GRAPH_OP ) cg.analyze() flow_graph = cg.output_enriched() diff --git a/onecode/cli/zip.py b/onecode/cli/zip.py index f5b2135..d1f2458 100644 --- a/onecode/cli/zip.py +++ b/onecode/cli/zip.py @@ -6,11 +6,9 @@ import os import zipfile -from ..base.decorator import check_type from .utils import get_flows -@check_type def zip_output( project_path: str, data_path: str, diff --git a/onecode/elements/input/__init__.pyi b/onecode/elements/input/__init__.pyi index c9df0c3..c67c6c8 100644 --- a/onecode/elements/input/__init__.pyi +++ b/onecode/elements/input/__init__.pyi @@ -108,7 +108,7 @@ def csv_reader( ``` """ - + def dropdown( key: str, diff --git a/onecode/elements/input/checkbox.py b/onecode/elements/input/checkbox.py index f9e2062..9907609 100644 --- a/onecode/elements/input/checkbox.py +++ b/onecode/elements/input/checkbox.py @@ -3,12 +3,10 @@ from typing import Any, List, Optional, Union -from ...base.decorator import check_type from ..input_element import InputElement class Checkbox(InputElement): - @check_type def __init__( self, key: str, @@ -76,7 +74,6 @@ def _value_type(self) -> type: """ return bool - @check_type def _validate( self, value: bool diff --git a/onecode/elements/input/csv_reader.py b/onecode/elements/input/csv_reader.py index 773e2b4..f28ee31 100644 --- a/onecode/elements/input/csv_reader.py +++ b/onecode/elements/input/csv_reader.py @@ -2,17 +2,105 @@ # SPDX-License-Identifier: MIT import os -from typing import Any, Dict, List, Optional, Union +from typing import Any, BinaryIO, Dict, List, Optional, Union import pandas as pd +import pyarrow as pa +import pyarrow.compute as pc +import pyarrow.csv as pv -from ...base.decorator import check_type from ...base.project import Project from ..input_element import InputElement +MAX_UNIQUE = 100_000 + + +def _jsonable(value: Any) -> Any: + """Convert Arrow/Python scalars to JSON-friendly values.""" + if value is None: + return None + if hasattr(value, "as_py"): + try: + value = value.as_py() + except Exception: # noqa: BLE001 + pass + try: + if pd.isna(value): + return None + except (TypeError, ValueError): + pass + if hasattr(value, "item"): + try: + return value.item() + except (ValueError, AttributeError): + pass + return value + + +def _metadata_from_table(table: pa.Table) -> Dict[str, Any]: + """Build the expression-evaluator bag (legacy cloud CSV processor shape).""" + stats: Dict[str, Any] = { + ".columns": list(table.column_names), + "__len__()": int(table.num_rows), + } + + for column in table.column_names: + column_data = table[column] + col_key = f".{column}[]" + + if pa.types.is_floating(column_data.type): + stats[col_key] = { + "unique()": None, + "mode()": None, + "min()": _jsonable(pc.min(column_data)), + "max()": _jsonable(pc.max(column_data)), + "mean()": _jsonable(pc.mean(column_data)), + "count()": _jsonable(pc.count(column_data)), + "sum()": _jsonable(pc.sum(column_data)), + } + elif pa.types.is_integer(column_data.type): + uniq = None + most_freq = None + try: + counts = pc.value_counts(column_data) + uniq = pc.unique(column_data).to_pylist()[:MAX_UNIQUE] + if len(counts) > 0: + most_freq = counts[0][0].as_py() + except Exception: # noqa: BLE001 # pragma: no cover + pass # pragma: no cover + stats[col_key] = { + "unique()": [_jsonable(v) for v in uniq] if uniq is not None else None, + "mode()": _jsonable(most_freq), + "min()": _jsonable(pc.min(column_data)), + "max()": _jsonable(pc.max(column_data)), + "mean()": _jsonable(pc.mean(column_data)), + "count()": _jsonable(pc.count(column_data)), + "sum()": _jsonable(pc.sum(column_data)), + } + else: + uniq = None + most_freq = None + try: + counts = pc.value_counts(column_data) + uniq = pc.unique(column_data).to_pylist()[:MAX_UNIQUE] + if len(counts) > 0: + most_freq = counts[0][0].as_py() + except Exception: # noqa: BLE001 # pragma: no cover + pass # pragma: no cover + stats[col_key] = { + "unique()": list(uniq) if uniq is not None else None, + "mode()": _jsonable(most_freq), + "min()": None, + "max()": None, + "mean()": None, + "count()": _jsonable(pc.count(column_data)), + "sum()": None, + } + + return stats + class CsvReader(InputElement): - @check_type def __init__( self, key: str, @@ -87,28 +175,54 @@ def __init__( ) @staticmethod - def metadata(value: str) -> Dict: + def metadata(value: Union[str, BinaryIO], **options: Any) -> Dict: """ - Returns the metadata associated to the given CSV(s). + Build dynamic-UI / expression-evaluator metadata for a CSV. - Returns: - A dictionnary metadata for each CSV path provided: - ```py - { - "columns": df.columns.to_list(), - "stats": df.describe().to_dict() - } - ``` + Uses PyArrow (same approach as the legacy cloud CSV processor) so large files + can be read from a path or a **stream** (e.g. HTTP signed-URL body) without + loading via Pandas first. - """ - df = pd.read_csv(value) + Returns the evaluator bag shape: - meta = { - "columns": df.columns.to_list(), - "stats": df.describe().to_dict() + ```py + { + ".columns": ["A", "B"], + "__len__()": 123, + ".A[]": { + "unique()": [...], + "mode()": ..., + "min()": ..., + "max()": ..., + "mean()": ..., + "count()": ..., + "sum()": ..., + }, + ... } + ``` - return meta + Args: + value: Filesystem path, or a binary file-like / readable stream. + **options: Optional ``csv_options`` with ``read_options``, ``parse_options``, + ``convert_options`` (PyArrow CSV option dicts). + """ + csv_options = options.get("csv_options") or {} + read_options = dict(csv_options.get("read_options") or {}) + parse_options = dict(csv_options.get("parse_options") or {}) + convert_options = dict(csv_options.get("convert_options") or {}) + + # Drop null delimiter so PyArrow uses its default. + if parse_options.get("delimiter") is None: + parse_options.pop("delimiter", None) + + table = pv.read_csv( + value, + read_options=pv.ReadOptions(**read_options) if read_options else None, + parse_options=pv.ParseOptions(**parse_options) if parse_options else None, + convert_options=pv.ConvertOptions(**convert_options) if convert_options else None, + ) + return _metadata_from_table(table) @property def _value_type(self) -> type: @@ -148,7 +262,6 @@ def value(self) -> Optional[pd.DataFrame]: return None - @check_type def _validate( self, value: pd.DataFrame diff --git a/onecode/elements/input/dropdown.py b/onecode/elements/input/dropdown.py index 9b1be8c..a4b617d 100644 --- a/onecode/elements/input/dropdown.py +++ b/onecode/elements/input/dropdown.py @@ -3,12 +3,10 @@ from typing import Any, List, Optional, Union -from ...base.decorator import check_type from ..input_element import InputElement class Dropdown(InputElement): - @check_type def __init__( self, key: str, @@ -112,7 +110,6 @@ def _value_type(self) -> type: """ return List[Union[str, int, float]] if self.multiple else Union[str, int, float] - @check_type def _validate_option_value( self, value: Union[str, int, float] @@ -129,7 +126,6 @@ def _validate_option_value( if not isinstance(self.options, str) and value not in self.options: raise ValueError(f"[{self.key}] Not a valid choice: {value}") - @check_type def _validate( self, value: Union[List[str], str] diff --git a/onecode/elements/input/file_input.py b/onecode/elements/input/file_input.py index 0be4b25..302f733 100644 --- a/onecode/elements/input/file_input.py +++ b/onecode/elements/input/file_input.py @@ -4,14 +4,12 @@ import os from typing import Any, List, Optional, Tuple, Union -from ...base.decorator import check_type from ...base.project import Project from ...utils.typing import is_type from ..input_element import InputElement class FileInput(InputElement): - @check_type def __init__( self, key: str, @@ -120,7 +118,6 @@ def value(self) -> Optional[Union[List[str], str]]: return None - @check_type def _validate_file_value( self, value: str @@ -136,7 +133,6 @@ def _validate_file_value( elif not os.path.isfile(value): raise FileNotFoundError(f"[{self.key}] Path is not a file: {value}") - @check_type def _validate( self, value: Union[List[str], str] diff --git a/onecode/elements/input/number_input.py b/onecode/elements/input/number_input.py index 514609a..e1f6b3a 100644 --- a/onecode/elements/input/number_input.py +++ b/onecode/elements/input/number_input.py @@ -3,12 +3,10 @@ from typing import Any, List, Optional, Union -from ...base.decorator import check_type from ..input_element import InputElement class NumberInput(InputElement): - @check_type def __init__( self, key: str, @@ -86,7 +84,6 @@ def _value_type(self) -> type: """ return float - @check_type def _validate( self, value: float diff --git a/onecode/elements/input/radio_button.py b/onecode/elements/input/radio_button.py index 074830d..53cbc72 100644 --- a/onecode/elements/input/radio_button.py +++ b/onecode/elements/input/radio_button.py @@ -3,12 +3,10 @@ from typing import Any, List, Optional, Union -from ...base.decorator import check_type from ..input_element import InputElement class RadioButton(InputElement): - @check_type def __init__( self, key: str, @@ -102,7 +100,6 @@ def _value_type(self) -> type: """ return str - @check_type def _validate( self, value: str diff --git a/onecode/elements/input/slider.py b/onecode/elements/input/slider.py index 30cd1ff..620d677 100644 --- a/onecode/elements/input/slider.py +++ b/onecode/elements/input/slider.py @@ -3,7 +3,6 @@ from typing import Any, List, Optional, Union -from ...base.decorator import check_type from ..input_element import InputElement @@ -12,7 +11,6 @@ def _is_int(x): class Slider(InputElement): - @check_type def __init__( self, key: str, @@ -112,7 +110,6 @@ def value(self) -> Optional[Union[float, int, List[float], List[int]]]: return self._value - @check_type def _validate( self, value: Union[float, int] diff --git a/onecode/elements/input/text_input.py b/onecode/elements/input/text_input.py index 5046feb..ff176f2 100644 --- a/onecode/elements/input/text_input.py +++ b/onecode/elements/input/text_input.py @@ -3,12 +3,10 @@ from typing import Any, List, Optional, Union -from ...base.decorator import check_type from ..input_element import InputElement class TextInput(InputElement): - @check_type def __init__( self, key: str, @@ -85,7 +83,6 @@ def _value_type(self) -> type: """ return str - @check_type def _validate( self, value: str diff --git a/onecode/elements/input_element.py b/onecode/elements/input_element.py index 4f208e9..fed6a14 100644 --- a/onecode/elements/input_element.py +++ b/onecode/elements/input_element.py @@ -8,7 +8,6 @@ import pydash from slugify import slugify -from ..base.decorator import check_type from ..base.project import Project from ..utils.typing import is_type @@ -62,7 +61,6 @@ class InputElement(ABC): """ - @check_type def __init__( self, key: str, @@ -117,14 +115,22 @@ def __init__( self.__dict__.update(self._extra_args) @staticmethod - def metadata(value: Any) -> Dict: + def metadata(value: Any, **options: Any) -> Dict: """ - Re-implement this function to process the `value` and extract metadata from it. - By default, it returns an empty dictionnary, meaning no metadata. + Re-implement this function to process the `value` and extract metadata from it + (dynamic UI / expression evaluator bag). By default, it returns an empty + dictionnary, meaning no metadata. - Returns: - A dictionnary of metadata. + The GUI builder sets ``metadata: true`` when this method is overridden on the + concrete class. + + Args: + value: Element value (e.g. local path, or a stream when the cloud service + calls this). + **options: Element-specific options (e.g. ``csv_options``). + Returns: + A dictionnary of metadata (JSON-serializable). """ return {} @@ -265,7 +271,6 @@ def _validate( """ pass - @check_type def _prepare_and_validate( self, value: Optional[Any] diff --git a/onecode/elements/output/__init__.pyi b/onecode/elements/output/__init__.pyi index a498abc..949e0d3 100644 --- a/onecode/elements/output/__init__.pyi +++ b/onecode/elements/output/__init__.pyi @@ -2,7 +2,7 @@ # SPDX-License-Identifier: MIT # pyi required for VSCode instrospection -from typing import Any, List, Optional, Tuple, Union +from typing import Any, List, Optional def file_output( key: str, diff --git a/onecode/elements/output/file_output.py b/onecode/elements/output/file_output.py index 36f122a..fdde92c 100644 --- a/onecode/elements/output/file_output.py +++ b/onecode/elements/output/file_output.py @@ -5,13 +5,11 @@ import os from typing import Any, List, Optional -from ...base.decorator import check_type from ...base.project import Project from ..output_element import OutputElement class FileOutput(OutputElement): - @check_type def __init__( self, key: str, @@ -95,7 +93,6 @@ def value(self) -> str: """ return Project().get_output_path(self._value) - @check_type def _validate( self, value: str diff --git a/onecode/elements/output_element.py b/onecode/elements/output_element.py index 941eaa5..ef5a546 100644 --- a/onecode/elements/output_element.py +++ b/onecode/elements/output_element.py @@ -7,7 +7,6 @@ import pydash from slugify import slugify -from ..base.decorator import check_type from ..base.project import Project @@ -52,7 +51,6 @@ class OutputElement(ABC): """ - @check_type def __init__( self, key: str, diff --git a/onecode/pycg/LICENCE b/onecode/pycg/LICENCE new file mode 100644 index 0000000..261eeb9 --- /dev/null +++ b/onecode/pycg/LICENCE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/onecode/pycg/__init__.py b/onecode/pycg/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/onecode/pycg/machinery/__init__.py b/onecode/pycg/machinery/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/onecode/pycg/machinery/callgraph.py b/onecode/pycg/machinery/callgraph.py new file mode 100644 index 0000000..e263296 --- /dev/null +++ b/onecode/pycg/machinery/callgraph.py @@ -0,0 +1,68 @@ +# +# Copyright (c) 2020 Vitalis Salis. +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +class CallGraph(object): + def __init__(self): + self.cg = {} + self.modnames = {} + self.cg_enriched = {} + + def add_node(self, name, modname=""): + if not isinstance(name, str): + raise CallGraphError("Only string node names allowed") + if not name: + raise CallGraphError("Empty node name") + + if name not in self.cg: + self.cg[name] = set() + self.modnames[name] = modname + + if name in self.cg and not self.modnames[name]: + self.modnames[name] = modname + + def add_edge(self, src, dest, code=None): + self.add_node(src) + self.add_node(dest) + self.cg[src].add(dest) + if code is not None: + if src in self.cg_enriched: + self.cg_enriched[src].append({"normed": dest, "code": code.strip()}) + else: + self.cg_enriched[src] = [{"normed": dest, "code": code.strip()}] + + def get(self): + return self.cg + + def get_enriched(self): + return self.cg_enriched + + def get_edges(self): + output = [] + for src in self.cg: + for dst in self.cg[src]: + output.append([src, dst]) + return output + + def get_modules(self): + return self.modnames + + +class CallGraphError(Exception): + pass diff --git a/onecode/pycg/machinery/classes.py b/onecode/pycg/machinery/classes.py new file mode 100644 index 0000000..d9d46fe --- /dev/null +++ b/onecode/pycg/machinery/classes.py @@ -0,0 +1,78 @@ +# +# Copyright (c) 2020 Vitalis Salis. +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +class ClassManager: + def __init__(self): + self.names = {} + + def get(self, name): + if name in self.names: + return self.names[name] + + def create(self, name, module): + if name not in self.names: + cls = ClassNode(name, module) + self.names[name] = cls + return self.names[name] + + def get_classes(self): + return self.names + + +class ClassNode: + def __init__(self, ns, module): + self.ns = ns + self.module = module + self.mro = [ns] + + def add_parent(self, parent): + if isinstance(parent, str): + self.mro.append(parent) + elif isinstance(parent, list): + for item in parent: + self.mro.append(item) + self.fix_mro() + + def fix_mro(self): + new_mro = [] + for idx, item in enumerate(self.mro): + if self.mro[idx + 1 :].count(item) > 0: + continue + new_mro.append(item) + self.mro = new_mro + + def get_mro(self): + return self.mro + + def get_module(self): + return self.module + + def compute_mro(self): + res = [] + self.mro.reverse() + for parent in self.mro: + if parent not in res: + res.append(parent) + + res.reverse() + self.mro = res + + def clear_mro(self): + self.mro = [self.ns] diff --git a/onecode/pycg/machinery/definitions.py b/onecode/pycg/machinery/definitions.py new file mode 100644 index 0000000..4462d87 --- /dev/null +++ b/onecode/pycg/machinery/definitions.py @@ -0,0 +1,230 @@ +# +# Copyright (c) 2020 Vitalis Salis. +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +from onecode.pycg import utils +from onecode.pycg.machinery.pointers import LiteralPointer, NamePointer + + +class DefinitionManager(object): + def __init__(self): + self.defs = {} + + def create(self, ns, def_type): + if not ns or not isinstance(ns, str): + raise DefinitionError("Invalid namespace argument") + if def_type not in Definition.types: + raise DefinitionError("Invalid def type argument") + if self.get(ns): + raise DefinitionError("Definition already exists") + + self.defs[ns] = Definition(ns, def_type) + return self.defs[ns] + + def assign(self, ns, defi): + self.defs[ns] = Definition(ns, defi.get_type()) + self.defs[ns].merge(defi) + + # if it is a function def, we need to create a return pointer + if defi.is_function_def(): + return_ns = utils.join_ns(ns, utils.constants.RETURN_NAME) + self.defs[return_ns] = Definition(return_ns, utils.constants.NAME_DEF) + self.defs[return_ns].get_name_pointer().add( + utils.join_ns(defi.get_ns(), utils.constants.RETURN_NAME) + ) + + return self.defs[ns] + + def get(self, ns): + if ns in self.defs: + return self.defs[ns] + + def get_defs(self): + return self.defs + + def handle_function_def(self, parent_ns, fn_name): + full_ns = utils.join_ns(parent_ns, fn_name) + defi = self.get(full_ns) + if not defi: + defi = self.create(full_ns, utils.constants.FUN_DEF) + defi.decorator_names = set() + + return_ns = utils.join_ns(full_ns, utils.constants.RETURN_NAME) + if not self.get(return_ns): + self.create(return_ns, utils.constants.NAME_DEF) + + return defi + + def handle_class_def(self, parent_ns, cls_name): + full_ns = utils.join_ns(parent_ns, cls_name) + defi = self.get(full_ns) + if not defi: + defi = self.create(full_ns, utils.constants.CLS_DEF) + + return defi + + def transitive_closure(self): + closured = {} + + def dfs(defi): + name_pointer = defi.get_name_pointer() + new_set = set() + # bottom + if closured.get(defi.get_ns(), None) is not None: + return closured[defi.get_ns()] + + if not name_pointer.get(): + new_set.add(defi.get_ns()) + + closured[defi.get_ns()] = new_set + + for name in name_pointer.get(): + if not self.defs.get(name, None): + continue + items = dfs(self.defs[name]) + if not items: + items = set([name]) + new_set = new_set.union(items) + + closured[defi.get_ns()] = new_set + return closured[defi.get_ns()] + + for ns, current_def in self.defs.items(): + if closured.get(current_def, None) is None: + dfs(current_def) + + return closured + + def complete_definitions(self): + # THE MOST expensive part of this tool's process + # TODO: IMPROVE COMPLEXITY + def update_pointsto_args(pointsto_args, arg, name): + changed_something = False + if arg == pointsto_args: + return False + for pointsto_arg in pointsto_args: + if not self.defs.get(pointsto_arg, None): + continue + if pointsto_arg == name: + continue + pointsto_arg_def = self.defs[pointsto_arg].get_name_pointer() + if pointsto_arg_def == pointsto_args: + continue + + # sometimes we may end up with a cycle + if pointsto_arg in arg: + arg.remove(pointsto_arg) + + for item in arg: + if item not in pointsto_arg_def.get(): + if self.defs.get(item, None) is not None: + changed_something = True + # HACK: this check shouldn't be needed + # if we remove this the following breaks: + # x = lambda x: x + 1 + # x(1) + # since on line 184 we don't discriminate between + # literal values and name values + if not self.defs.get(item, None): + continue + pointsto_arg_def.add(item) + return changed_something + + for i in range(len(self.defs)): + changed_something = False + for ns, current_def in self.defs.items(): + # the name pointer of the definition we're currently iterating + current_name_pointer = current_def.get_name_pointer() + # iterate the names the current definition points to items + # for name in current_name_pointer.get(): + for name in current_name_pointer.get().copy(): + # get the name pointer of the points to name + if not self.defs.get(name, None): + continue + if name == ns: + continue + + pointsto_name_pointer = self.defs[name].get_name_pointer() + # iterate the arguments of the definition + # we're currently iterating + for arg_name, arg in current_name_pointer.get_args().items(): + pos = current_name_pointer.get_pos_of_name(arg_name) + if pos is not None: + pointsto_args = pointsto_name_pointer.get_pos_arg(pos) + if not pointsto_args: + pointsto_name_pointer.add_pos_arg(pos, None, arg) + continue + else: + pointsto_args = pointsto_name_pointer.get_arg(arg_name) + if not pointsto_args: + pointsto_name_pointer.add_arg(arg_name, arg) + continue + changed_something = changed_something or update_pointsto_args( + pointsto_args, arg, current_def.get_ns() + ) + + if not changed_something: + break + + +class Definition(object): + types = [ + utils.constants.FUN_DEF, + utils.constants.MOD_DEF, + utils.constants.NAME_DEF, + utils.constants.CLS_DEF, + utils.constants.EXT_DEF, + ] + + def __init__(self, fullns, def_type): + self.fullns = fullns + self.points_to = {"lit": LiteralPointer(), "name": NamePointer()} + self.def_type = def_type + + def get_type(self): + return self.def_type + + def is_function_def(self): + return self.def_type == utils.constants.FUN_DEF + + def is_ext_def(self): + return self.def_type == utils.constants.EXT_DEF + + def is_callable(self): + return self.is_function_def() or self.is_ext_def() + + def get_lit_pointer(self): + return self.points_to["lit"] + + def get_name_pointer(self): + return self.points_to["name"] + + def get_name(self): + return self.fullns.split(".")[-1] + + def get_ns(self): + return self.fullns + + def merge(self, to_merge): + for name, pointer in to_merge.points_to.items(): + self.points_to[name].merge(pointer) + + +class DefinitionError(Exception): + pass diff --git a/onecode/pycg/machinery/imports.py b/onecode/pycg/machinery/imports.py new file mode 100644 index 0000000..98f8d32 --- /dev/null +++ b/onecode/pycg/machinery/imports.py @@ -0,0 +1,235 @@ +# +# Copyright (c) 2020 Vitalis Salis. +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +import copy +import importlib +import os +import sys +from importlib import abc + +from onecode.pycg import utils + + +def get_custom_loader(ig_obj): + """ + Closure which returns a custom loader + that modifies an ImportManager object + """ + + class CustomLoader(abc.SourceLoader): + def __init__(self, fullname, path): + self.fullname = fullname + self.path = path + + ig_obj.create_edge(self.fullname) + if not ig_obj.get_node(self.fullname): + ig_obj.create_node(self.fullname) + ig_obj.set_filepath(self.fullname, self.path) + + def get_filename(self, fullname): + return self.path + + def get_data(self, filename): + return "" + + return CustomLoader + + +class ImportManager(object): + def __init__(self): + self.import_graph = dict() + self.current_module = "" + self.input_file = "" + self.mod_dir = None + self.old_path_hooks = None + self.old_path = None + + def set_pkg(self, input_pkg): + self.mod_dir = input_pkg + + def get_mod_dir(self): + return self.mod_dir + + def get_node(self, name): + if name in self.import_graph: + return self.import_graph[name] + + def create_node(self, name): + if not name or not isinstance(name, str): + raise ImportManagerError("Invalid node name") + + if self.get_node(name): + raise ImportManagerError("Can't create a node a second time") + + self.import_graph[name] = {"filename": "", "imports": set()} + return self.import_graph[name] + + def create_edge(self, dest): + if not dest or not isinstance(dest, str): + raise ImportManagerError("Invalid node name") + + node = self.get_node(self._get_module_path()) + if not node: + raise ImportManagerError("Can't add edge to a non existing node") + + node["imports"].add(dest) + + def _clear_caches(self): + # importlib.invalidate_caches() + sys.path_importer_cache.clear() + # TODO: maybe not do that since it empties the whole cache + for name in self.import_graph: + if name in sys.modules: + del sys.modules[name] + + def _get_module_path(self): + return self.current_module + + def set_current_mod(self, name, fname): + self.current_module = name + self.input_file = os.path.abspath(fname) + + def get_filepath(self, modname): + if modname in self.import_graph: + return self.import_graph[modname]["filename"] + + def set_filepath(self, node_name, filename): + if not filename or not isinstance(filename, str): + raise ImportManagerError("Invalid node name") + + node = self.get_node(node_name) + if not node: + raise ImportManagerError("Node does not exist") + + node["filename"] = os.path.abspath(filename) + + def get_imports(self, modname): + if modname not in self.import_graph: + return [] + return self.import_graph[modname]["imports"] + + def _is_init_file(self): + return self.input_file.endswith("__init__.py") + + def _handle_import_level(self, name, level): + # add a dot for each level + package = self._get_module_path().split(".") + if level > len(package): + raise ImportError("Attempting import beyond top level package") + + mod_name = ("." * level) + name + # When an __init__ file is analyzed, + # then the module name doesn't contain + # the __init__ part in it, + # so special care must be taken for levels. + if self._is_init_file() and level >= 1: + if level != 1: + level -= 1 + package = package[:-level] + else: + package = package[:-level] + + return mod_name, ".".join(package) + + def _do_import(self, mod_name, package): + if mod_name in sys.modules: + self.create_edge(mod_name) + return sys.modules[mod_name] + + try: + module_spec = importlib.util.find_spec(mod_name, package=package) + except ModuleNotFoundError: + module_spec = None + + if module_spec is None: + return importlib.import_module(mod_name, package=package) + + return importlib.util.module_from_spec(module_spec) + + def handle_import(self, name, level): + # We currently don't support builtin modules because they're frozen. + # Add an edge and continue. + # TODO: identify a way to include frozen modules + root = name.split(".")[0] + if root in sys.builtin_module_names: + self.create_edge(root) + return + + # Import the module + try: + mod_name, package = self._handle_import_level(name, level) + except ImportError: + return + + parent = ".".join(mod_name.split(".")[:-1]) + parent_name = ".".join(name.split(".")[:-1]) + combos = [ + (mod_name, package), + (parent, package), + (utils.join_ns(package, name), ""), + (utils.join_ns(package, parent_name), ""), + ] + + mod = None + for mn, pkg in combos: + try: + mod = self._do_import(mn, pkg) + break + except Exception: + continue + + if not mod: + return + + if not hasattr(mod, "__file__") or not mod.__file__: + return + if self.mod_dir not in mod.__file__: + return + fname = mod.__file__ + if fname.endswith("__init__.py"): + fname = os.path.split(fname)[0] + + return utils.to_mod_name(os.path.relpath(fname, self.mod_dir)) + + def get_import_graph(self): + return self.import_graph + + def install_hooks(self): + loader = get_custom_loader(self) + self.old_path_hooks = copy.deepcopy(sys.path_hooks) + self.old_path = copy.deepcopy(sys.path) + + loader_details = loader, importlib.machinery.all_suffixes() + sys.path_hooks.insert( + 0, importlib.machinery.FileFinder.path_hook(loader_details) + ) + sys.path.insert(0, os.path.abspath(self.mod_dir)) + + self._clear_caches() + + def remove_hooks(self): + sys.path_hooks = self.old_path_hooks + sys.path = self.old_path + + self._clear_caches() + + +class ImportManagerError(Exception): + pass diff --git a/onecode/pycg/machinery/modules.py b/onecode/pycg/machinery/modules.py new file mode 100644 index 0000000..5905ee3 --- /dev/null +++ b/onecode/pycg/machinery/modules.py @@ -0,0 +1,64 @@ +# +# Copyright (c) 2020 Vitalis Salis. +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +class ModuleManager: + def __init__(self): + self.internal = {} + self.external = {} + + def create(self, name, fname, external=False): + mod = Module(name, fname) + if external: + self.external[name] = mod + else: + self.internal[name] = mod + return mod + + def get(self, name): + if name in self.internal: + return self.internal[name] + if name in self.external: + return self.external[name] + + def get_internal_modules(self): + return self.internal + + def get_external_modules(self): + return self.external + + +class Module: + def __init__(self, name, filename): + self.name = name + self.filename = filename + self.methods = dict() + + def get_name(self): + return self.name + + def get_filename(self): + return self.filename + + def get_methods(self): + return self.methods + + def add_method(self, method, first=None, last=None): + if not self.methods.get(method, None): + self.methods[method] = dict(name=method, first=first, last=last) diff --git a/onecode/pycg/machinery/pointers.py b/onecode/pycg/machinery/pointers.py new file mode 100644 index 0000000..cf78b4b --- /dev/null +++ b/onecode/pycg/machinery/pointers.py @@ -0,0 +1,149 @@ +# +# Copyright (c) 2020 Vitalis Salis. +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +class Pointer(object): + def __init__(self): + self.values = set() + + def add(self, item): + self.values.add(item) + + def add_set(self, s): + self.values = self.values.union(s) + + def get(self): + return self.values + + def merge(self, pointer): + self.values = self.values.union(pointer.values) + + +class LiteralPointer(Pointer): + STR_LIT = "STRING" + INT_LIT = "INTEGER" + UNK_LIT = "UNKNOWN" + + # no need to add the actual item + def add(self, item): + if isinstance(item, str): + self.values.add(item) + elif isinstance(item, int): + self.values.add(item) + else: + self.values.add(self.UNK_LIT) + + +class NamePointer(Pointer): + def __init__(self): + super().__init__() + self.pos_to_name = {} + self.name_to_pos = {} + self.args = {} + + def _sanitize_pos(self, pos): + try: + int(pos) + except ValueError: + raise PointerError("Invalid position for argument") + + return pos + + def get_or_create(self, name): + if name not in self.args: + self.args[name] = set() + return self.args[name] + + def add_arg(self, name, item): + self.get_or_create(name) + if isinstance(item, str): + self.args[name].add(item) + elif isinstance(item, set): + self.args[name] = self.args[name].union(item) + else: + raise Exception() + + def add_lit_arg(self, name, item): + arg = self.get_or_create(name) + if isinstance(item, str): + arg.add(LiteralPointer.STR_LIT) + elif isinstance(item, int): + arg.add(LiteralPointer.INT_LIT) + else: + arg.add(LiteralPointer.UNK_LIT) + + def add_pos_arg(self, pos, name, item): + pos = self._sanitize_pos(pos) + if not name: + if self.pos_to_name.get(pos, None): + name = self.pos_to_name[pos] + else: + name = str(pos) + self.pos_to_name[pos] = name + self.name_to_pos[name] = pos + + self.add_arg(name, item) + + def add_name_arg(self, name, item): + self.add_arg(name, item) + + def add_pos_lit_arg(self, pos, name, item): + pos = self._sanitize_pos(pos) + if not name: + name = str(pos) + self.pos_to_name[pos] = name + self.name_to_pos[name] = pos + self.add_lit_arg(name, item) + + def get_pos_arg(self, pos): + pos = self._sanitize_pos(pos) + name = self.pos_to_name.get(pos, None) + return self.get_arg(name) + + def get_arg(self, name): + if self.args.get(name, None): + return self.args[name] + + def get_args(self): + return self.args + + def get_pos_args(self): + args = {} + for pos, name in self.pos_to_name.items(): + args[pos] = self.args[name] + return args + + def get_pos_of_name(self, name): + if name in self.name_to_pos: + return self.name_to_pos[name] + + def get_pos_names(self): + return self.pos_to_name + + def merge(self, pointer): + super().merge(pointer) + if hasattr(pointer, "get_pos_names"): + for pos, name in pointer.get_pos_names().items(): + self.pos_to_name[pos] = name + for name, arg in pointer.get_args().items(): + self.add_arg(name, arg) + + +class PointerError(Exception): + pass diff --git a/onecode/pycg/machinery/scopes.py b/onecode/pycg/machinery/scopes.py new file mode 100644 index 0000000..5078b76 --- /dev/null +++ b/onecode/pycg/machinery/scopes.py @@ -0,0 +1,154 @@ +# +# Copyright (c) 2020 Vitalis Salis. +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +import symtable + +from onecode.pycg import utils + + +class ScopeManager(object): + """Manages the scope entries""" + + def __init__(self): + self.scopes = {} + + def handle_module(self, modulename, filename, contents): + functions = [] + classes = [] + + def process(namespace, parent, table): + if table.get_name() == "top" and table.get_lineno() == 0: + name = "" + else: + name = table.get_name() + + if name: + fullns = utils.join_ns(namespace, name) + else: + fullns = namespace + + if table.get_type() == "function": + functions.append(fullns) + + if table.get_type() == "class": + classes.append(fullns) + + sc = self.create_scope(fullns, parent) + + for t in table.get_children(): + process(fullns, sc, t) + + process( + modulename, None, symtable.symtable(contents, filename, compile_type="exec") + ) + return {"functions": functions, "classes": classes} + + def handle_assign(self, ns, target, defi): + scope = self.get_scope(ns) + if scope: + scope.add_def(target, defi) + + def get_def(self, current_ns, var_name): + current_scope = self.get_scope(current_ns) + while current_scope: + defi = current_scope.get_def(var_name) + if defi: + return defi + current_scope = current_scope.parent + + def get_scope(self, namespace): + if namespace in self.get_scopes(): + return self.get_scopes()[namespace] + + def create_scope(self, namespace, parent): + if namespace not in self.scopes: + sc = ScopeItem(namespace, parent) + self.scopes[namespace] = sc + return self.scopes[namespace] + + def get_scopes(self): + return self.scopes + + +class ScopeItem(object): + def __init__(self, fullns, parent): + if parent and not isinstance(parent, ScopeItem): + raise ScopeError("Parent must be a ScopeItem instance") + + if not isinstance(fullns, str): + raise ScopeError("Namespace should be a string") + + self.parent = parent + self.defs = {} + self.lambda_counter = 0 + self.dict_counter = 0 + self.list_counter = 0 + self.fullns = fullns + + def get_ns(self): + return self.fullns + + def get_defs(self): + return self.defs + + def get_def(self, name): + defs = self.get_defs() + if name in defs: + return defs[name] + + def get_lambda_counter(self): + return self.lambda_counter + + def get_dict_counter(self): + return self.dict_counter + + def get_list_counter(self): + return self.list_counter + + def inc_lambda_counter(self, val=1): + self.lambda_counter += val + return self.lambda_counter + + def inc_dict_counter(self, val=1): + self.dict_counter += val + return self.dict_counter + + def inc_list_counter(self, val=1): + self.list_counter += val + return self.list_counter + + def reset_counters(self): + self.lambda_counter = 0 + self.dict_counter = 0 + self.list_counter = 0 + + def add_def(self, name, defi): + self.defs[name] = defi + + def merge_def(self, name, to_merge): + if name not in self.defs: + self.defs[name] = to_merge + return + + self.defs[name].merge_points_to(to_merge.get_points_to()) + + +class ScopeError(Exception): + pass diff --git a/onecode/pycg/processing/__init__.py b/onecode/pycg/processing/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/onecode/pycg/processing/base.py b/onecode/pycg/processing/base.py new file mode 100644 index 0000000..bb18b73 --- /dev/null +++ b/onecode/pycg/processing/base.py @@ -0,0 +1,547 @@ +# +# Copyright (c) 2020 Vitalis Salis. +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +import ast +import os + +from onecode.pycg import utils +from onecode.pycg.machinery.definitions import Definition + + +class ProcessingBase(ast.NodeVisitor): + def __init__(self, filename, modname, modules_analyzed): + self.modname = modname + + self.modules_analyzed = modules_analyzed + self.modules_analyzed.add(self.modname) + + self.filename = os.path.abspath(filename) + + with open(filename, "rt", errors="replace") as f: + self.contents = f.read() + + self.name_stack = [] + self.method_stack = [] + self.last_called_names = None + + def get_modules_analyzed(self): + return self.modules_analyzed + + def merge_modules_analyzed(self, analyzed): + self.modules_analyzed = self.modules_analyzed.union(analyzed) + + @property + def current_ns(self): + return ".".join(self.name_stack) + + @property + def current_method(self): + return ".".join(self.method_stack) + + def visit_Module(self, node): + self.name_stack.append(self.modname) + self.method_stack.append(self.modname) + self.scope_manager.get_scope(self.modname).reset_counters() + self.generic_visit(node) + self.method_stack.pop() + self.name_stack.pop() + + def visit_FunctionDef(self, node): + self.name_stack.append(node.name) + self.method_stack.append(node.name) + if self.scope_manager.get_scope(self.current_ns): + self.scope_manager.get_scope(self.current_ns).reset_counters() + for stmt in node.body: + self.visit(stmt) + self.method_stack.pop() + self.name_stack.pop() + + def visit_Lambda(self, node, lambda_name=None): + lambda_ns = utils.join_ns(self.current_ns, lambda_name) + if not self.scope_manager.get_scope(lambda_ns): + self.scope_manager.create_scope( + lambda_ns, self.scope_manager.get_scope(self.current_ns) + ) + self.name_stack.append(lambda_name) + self.method_stack.append(lambda_name) + self.visit(node.body) + self.method_stack.pop() + self.name_stack.pop() + + def visit_For(self, node): + for item in node.body: + self.visit(item) + + def visit_Dict(self, node): + counter = self.scope_manager.get_scope(self.current_ns).inc_dict_counter() + dict_name = utils.get_dict_name(counter) + + sc = self.scope_manager.get_scope(utils.join_ns(self.current_ns, dict_name)) + if not sc: + return + self.name_stack.append(dict_name) + sc.reset_counters() + for key, val in zip(node.keys, node.values): + if key: + self.visit(key) + if val: + self.visit(val) + self.name_stack.pop() + + def visit_List(self, node): + counter = self.scope_manager.get_scope(self.current_ns).inc_list_counter() + list_name = utils.get_list_name(counter) + + sc = self.scope_manager.get_scope(utils.join_ns(self.current_ns, list_name)) + if not sc: + return + self.name_stack.append(list_name) + sc.reset_counters() + for elt in node.elts: + self.visit(elt) + self.name_stack.pop() + + def visit_BinOp(self, node): + self.visit(node.left) + self.visit(node.right) + + def visit_ClassDef(self, node): + self.name_stack.append(node.name) + self.method_stack.append(node.name) + if self.scope_manager.get_scope(self.current_ns): + self.scope_manager.get_scope(self.current_ns).reset_counters() + for stmt in node.body: + self.visit(stmt) + self.method_stack.pop() + self.name_stack.pop() + + def visit_Tuple(self, node): + for elt in node.elts: + self.visit(elt) + + def _handle_assign(self, targetns, decoded): + defi = self.def_manager.get(targetns) + if not defi: + defi = self.def_manager.create(targetns, utils.constants.NAME_DEF) + + # check if decoded is iterable + try: + iter(decoded) + except TypeError: + return defi + + for d in decoded: + if isinstance(d, Definition): + defi.get_name_pointer().add(d.get_ns()) + else: + defi.get_lit_pointer().add(d) + return defi + + def _visit_return(self, node): + if not node or not node.value: + return + + self.visit(node.value) + + return_ns = utils.join_ns(self.current_ns, utils.constants.RETURN_NAME) + self._handle_assign(return_ns, self.decode_node(node.value)) + + def _get_target_ns(self, target): + if isinstance(target, ast.Name): + return [utils.join_ns(self.current_ns, target.id)] + if isinstance(target, ast.Attribute): + bases = self._retrieve_base_names(target) + res = [] + for base in bases: + res.append(utils.join_ns(base, target.attr)) + return res + if isinstance(target, ast.Subscript): + return self.retrieve_subscript_names(target) + return [] + + def _visit_assign(self, value, targets): + self.visit(value) + + decoded = self.decode_node(value) + + def do_assign(decoded, target): + self.visit(target) + if isinstance(target, ast.Tuple): + for pos, elt in enumerate(target.elts): + if not isinstance(decoded, Definition) and pos < len(decoded): + do_assign(decoded[pos], elt) + else: + targetns = self._get_target_ns(target) + for tns in targetns: + if not tns: + continue + defi = self._handle_assign(tns, decoded) + splitted = tns.split(".") + self.scope_manager.handle_assign( + ".".join(splitted[:-1]), splitted[-1], defi + ) + + for target in targets: + do_assign(decoded, target) + + def decode_node(self, node): + if isinstance(node, ast.Name): + return [self.scope_manager.get_def(self.current_ns, node.id)] + elif isinstance(node, ast.Call): + decoded = self.decode_node(node.func) + return_defs = [] + for called_def in decoded: + if not isinstance(called_def, Definition): + continue + + return_ns = utils.constants.INVALID_NAME + if called_def.get_type() == utils.constants.FUN_DEF: + return_ns = utils.join_ns( + called_def.get_ns(), utils.constants.RETURN_NAME + ) + elif ( + called_def.get_type() == utils.constants.CLS_DEF + or called_def.get_type() == utils.constants.EXT_DEF + ): + return_ns = called_def.get_ns() + defi = self.def_manager.get(return_ns) + if defi: + return_defs.append(defi) + + return return_defs + elif isinstance(node, ast.Lambda): + lambda_counter = self.scope_manager.get_scope( + self.current_ns + ).get_lambda_counter() + lambda_name = utils.get_lambda_name(lambda_counter) + return [self.scope_manager.get_def(self.current_ns, lambda_name)] + elif isinstance(node, ast.Tuple): + decoded = [] + for elt in node.elts: + decoded.append(self.decode_node(elt)) + return decoded + elif isinstance(node, ast.BinOp): + decoded_left = self.decode_node(node.left) + decoded_right = self.decode_node(node.right) + # return the non definition types if we're talking about a binop + # since we only care about the type of the return (num, str, etc) + if not isinstance(decoded_left, Definition): + return decoded_left + if not isinstance(decoded_right, Definition): + return decoded_right + elif isinstance(node, ast.Attribute): + names = self._retrieve_attribute_names(node) + defis = [] + for name in names: + defi = self.def_manager.get(name) + if defi: + defis.append(defi) + return defis + elif isinstance(node, ast.Constant): + return [node.value] + elif isinstance(node, ast.Constant): + return [node.value] + elif self._is_literal(node): + return [node] + elif isinstance(node, ast.Dict): + dict_counter = self.scope_manager.get_scope( + self.current_ns + ).get_dict_counter() + dict_name = utils.get_dict_name(dict_counter) + scope_def = self.scope_manager.get_def(self.current_ns, dict_name) + return [scope_def] + elif isinstance(node, ast.List): + list_counter = self.scope_manager.get_scope( + self.current_ns + ).get_list_counter() + list_name = utils.get_list_name(list_counter) + scope_def = self.scope_manager.get_def(self.current_ns, list_name) + return [scope_def] + elif isinstance(node, ast.Subscript): + names = self.retrieve_subscript_names(node) + defis = [] + for name in names: + defi = self.def_manager.get(name) + if defi: + defis.append(defi) + return defis + + return [] + + def _is_literal(self, item): + return isinstance(item, int) or isinstance(item, str) or isinstance(item, float) + + def _retrieve_base_names(self, node): + if not isinstance(node, ast.Attribute): + raise Exception("The node is not an attribute") + + if not getattr(self, "closured", None): + return set() + + decoded = self.decode_node(node.value) + if not decoded: + return set() + + names = set() + for name in decoded: + if not name or not isinstance(name, Definition): + continue + + for base in self.closured.get(name.get_ns(), []): + cls = self.class_manager.get(base) + if not cls: + continue + + for item in cls.get_mro(): + names.add(item) + return names + + def _retrieve_parent_names(self, node): + if not isinstance(node, ast.Attribute): + raise Exception("The node is not an attribute") + + decoded = self.decode_node(node.value) + if not decoded: + return set() + + names = set() + for parent in decoded: + if not parent or not isinstance(parent, Definition): + continue + if getattr(self, "closured", None) and self.closured.get( + parent.get_ns(), None + ): + names = names.union(self.closured.get(parent.get_ns())) + else: + names.add(parent.get_ns()) + return names + + def _retrieve_attribute_names(self, node): + if not getattr(self, "closured", None): + return set() + + parent_names = self._retrieve_parent_names(node) + names = set() + for parent_name in parent_names: + for name in self.closured.get(parent_name, []): + defi = self.def_manager.get(name) + if not defi: + continue + if defi.get_type() == utils.constants.CLS_DEF: + cls_names = self.find_cls_fun_ns(defi.get_ns(), node.attr) + if cls_names: + names = names.union(cls_names) + if defi.get_type() in [ + utils.constants.FUN_DEF, + utils.constants.MOD_DEF, + ]: + names.add(utils.join_ns(name, node.attr)) + if defi.get_type() == utils.constants.EXT_DEF: + # HACK: extenral attributes can lead to infinite loops + # Identify them here + if node.attr in name: + continue + ext_name = utils.join_ns(name, node.attr) + if not self.def_manager.get(ext_name): + self.def_manager.create(ext_name, utils.constants.EXT_DEF) + names.add(ext_name) + return names + + def iterate_call_args(self, defi, node): + for pos, arg in enumerate(node.args): + self.visit(arg) + decoded = self.decode_node(arg) + if defi.is_function_def(): + pos_arg_names = defi.get_name_pointer().get_pos_arg(pos) + # if arguments for this position exist update their namespace + if not pos_arg_names: + continue + for name in pos_arg_names: + arg_def = self.def_manager.get(name) + if not arg_def: + continue + for d in decoded: + if isinstance(d, Definition): + arg_def.get_name_pointer().add(d.get_ns()) + else: + arg_def.get_lit_pointer().add(d) + else: + for d in decoded: + if isinstance(d, Definition): + defi.get_name_pointer().add_pos_arg(pos, None, d.get_ns()) + else: + defi.get_name_pointer().add_pos_lit_arg(pos, None, d) + + for keyword in node.keywords: + self.visit(keyword.value) + decoded = self.decode_node(keyword.value) + if defi.is_function_def(): + arg_names = defi.get_name_pointer().get_arg(keyword.arg) + if not arg_names: + continue + for name in arg_names: + arg_def = self.def_manager.get(name) + if not arg_def: + continue + for d in decoded: + if isinstance(d, Definition): + arg_def.get_name_pointer().add(d.get_ns()) + else: + arg_def.get_lit_pointer().add(d) + else: + for d in decoded: + if isinstance(d, Definition): + defi.get_name_pointer().add_arg(keyword.arg, d.get_ns()) + else: + defi.get_name_pointer().add_lit_arg(keyword.arg, d) + + def retrieve_subscript_names(self, node): + if not isinstance(node, ast.Subscript): + raise Exception("The node is not an subcript") + + if not getattr(self, "closured", None): + return set() + + if getattr(node.slice, "value", None) and self._is_literal(node.slice.value): + sl_names = [node.slice.value] + else: + sl_names = self.decode_node(node.slice) + + val_names = self.decode_node(node.value) + + decoded_vals = set() + keys = set() + full_names = set() + # get all names associated with this variable name + for n in val_names: + if n and isinstance(n, Definition) and self.closured.get(n.get_ns(), None): + decoded_vals |= self.closured.get(n.get_ns()) + for s in sl_names: + if isinstance(s, Definition) and self.closured.get(s.get_ns(), None): + # we care about the literals pointed by the name + # not the namespaces, so retrieve the literals pointed + for name in self.closured.get(s.get_ns()): + defi = self.def_manager.get(name) + if not defi: + continue + keys |= defi.get_lit_pointer().get() + elif isinstance(s, str): + keys.add(s) + elif isinstance(s, int): + keys.add(utils.get_int_name(s)) + + for d in decoded_vals: + for key in keys: + # check for existence of var name and key combination + str_key = str(key) + if isinstance(key, int): + str_key = utils.get_int_name(key) + full_ns = utils.join_ns(d, str_key) + full_names.add(full_ns) + + return full_names + + def retrieve_call_names(self, node): + names = set() + if isinstance(node.func, ast.Name): + defi = self.scope_manager.get_def(self.current_ns, node.func.id) + if defi: + names = self.closured.get(defi.get_ns(), None) + elif isinstance(node.func, ast.Call) and self.last_called_names: + for name in self.last_called_names: + return_ns = utils.join_ns(name, utils.constants.RETURN_NAME) + returns = self.closured.get(return_ns) + if not returns: + continue + for ret in returns: + defi = self.def_manager.get(ret) + names.add(defi.get_ns()) + elif isinstance(node.func, ast.Attribute): + names = self._retrieve_attribute_names(node.func) + elif isinstance(node.func, ast.Subscript): + # Calls can be performed only on single indices, not ranges + full_names = self.retrieve_subscript_names(node.func) + for n in full_names: + if self.closured.get(n, None): + names |= self.closured.get(n) + + return names + + def analyze_submodules(self, cls, *args, **kwargs): + imports = self.import_manager.get_imports(self.modname) + + for imp in imports: + self.analyze_submodule(cls, imp, *args, **kwargs) + + def analyze_submodule(self, cls, imp, *args, **kwargs): + if imp in self.get_modules_analyzed(): + return + + fname = self.import_manager.get_filepath(imp) + + if ( + not fname + or not fname.endswith(".py") + or self.import_manager.get_mod_dir() not in fname + ): + return + + self.import_manager.set_current_mod(imp, fname) + + visitor = cls(fname, imp, *args, **kwargs) + visitor.analyze() + self.merge_modules_analyzed(visitor.get_modules_analyzed()) + + self.import_manager.set_current_mod(self.modname, self.filename) + + def find_cls_fun_ns(self, cls_name, fn): + cls = self.class_manager.get(cls_name) + if not cls: + return set() + + ext_names = set() + for item in cls.get_mro(): + ns = utils.join_ns(item, fn) + names = set() + if getattr(self, "closured", None) and self.closured.get(ns, None): + names = self.closured[ns] + else: + names.add(ns) + + if self.def_manager.get(ns): + return names + + parent = self.def_manager.get(item) + if parent and parent.get_type() == utils.constants.EXT_DEF: + ext_names.add(ns) + + for name in ext_names: + self.def_manager.create(name, utils.constants.EXT_DEF) + self.add_ext_mod_node(name) + return ext_names + + def add_ext_mod_node(self, name): + ext_modname = name.split(".")[0] + ext_mod = self.module_manager.get(ext_modname) + if not ext_mod: + ext_mod = self.module_manager.create(ext_modname, None, external=True) + ext_mod.add_method(ext_modname) + + ext_mod.add_method(name) diff --git a/onecode/pycg/processing/cgprocessor.py b/onecode/pycg/processing/cgprocessor.py new file mode 100644 index 0000000..8a2f3aa --- /dev/null +++ b/onecode/pycg/processing/cgprocessor.py @@ -0,0 +1,253 @@ +# +# Copyright (c) 2020 Vitalis Salis. +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +import ast +import os + +from onecode.pycg import utils +from onecode.pycg.machinery.definitions import Definition +from onecode.pycg.processing.base import ProcessingBase + + +class CallGraphProcessor(ProcessingBase): + def __init__( + self, + filename, + modname, + import_manager, + scope_manager, + def_manager, + class_manager, + module_manager, + call_graph=None, + modules_analyzed=None, + ): + super().__init__(filename, modname, modules_analyzed) + # parent directory of file + self.parent_dir = os.path.dirname(filename) + + self.import_manager = import_manager + self.scope_manager = scope_manager + self.def_manager = def_manager + self.class_manager = class_manager + self.module_manager = module_manager + + self.call_graph = call_graph + + self.closured = self.def_manager.transitive_closure() + + def visit_Module(self, node): + self.call_graph.add_node(self.modname, self.modname) + super().visit_Module(node) + + def visit_For(self, node): + self.visit(node.iter) + self.visit(node.target) + # assign target.id to the return value of __next__ of node.iter.it + # we need to have a visit for on the postprocessor also + iter_decoded = self.decode_node(node.iter) + for item in iter_decoded: + if not isinstance(item, Definition): + continue + names = self.closured.get(item.get_ns(), []) + for name in names: + iter_ns = utils.join_ns(name, utils.constants.ITER_METHOD) + next_ns = utils.join_ns(name, utils.constants.NEXT_METHOD) + if self.def_manager.get(iter_ns): + self.call_graph.add_edge(self.current_method, iter_ns) + if self.def_manager.get(next_ns): + self.call_graph.add_edge(self.current_method, next_ns) + + super().visit_For(node) + + def visit_Lambda(self, node): + counter = self.scope_manager.get_scope(self.current_ns).inc_lambda_counter() + lambda_name = utils.get_lambda_name(counter) + lambda_fullns = utils.join_ns(self.current_ns, lambda_name) + + self.call_graph.add_node(lambda_fullns, self.modname) + + super().visit_Lambda(node, lambda_name) + + def visit_Raise(self, node): + if not node.exc: + return + self.visit(node.exc) + decoded = self.decode_node(node.exc) + for d in decoded: + if not isinstance(d, Definition): + continue + names = self.closured.get(d.get_ns(), []) + for name in names: + pointer_def = self.def_manager.get(name) + if pointer_def.get_type() == utils.constants.CLS_DEF: + init_ns = self.find_cls_fun_ns(name, utils.constants.CLS_INIT) + for ns in init_ns: + self.call_graph.add_edge(self.current_method, ns) + if pointer_def.get_type() == utils.constants.EXT_DEF: + self.call_graph.add_edge(self.current_method, name) + + def visit_AsyncFunctionDef(self, node): + self.visit_FunctionDef(node) + + def visit_FunctionDef(self, node): + for decorator in node.decorator_list: + self.visit(decorator) + decoded = self.decode_node(decorator) + for d in decoded: + if not isinstance(d, Definition): + continue + names = self.closured.get(d.get_ns(), []) + for name in names: + self.call_graph.add_edge(self.current_method, name) + + self.call_graph.add_node( + utils.join_ns(self.current_ns, node.name), self.modname + ) + super().visit_FunctionDef(node) + + def visit_Call(self, node): + def create_ext_edge(name, ext_modname, code): + self.add_ext_mod_node(name) + self.call_graph.add_node(name, ext_modname) + self.call_graph.add_edge(self.current_method, name, code) + + code = ast.unparse(node) + # First visit the child function so that on the case of + # func()()() + # we first visit the call to func and then the other calls + for arg in node.args: + self.visit(arg) + + for keyword in node.keywords: + self.visit(keyword.value) + + self.visit(node.func) + + names = self.retrieve_call_names(node) + if not names: + if isinstance(node.func, ast.Attribute) and self.has_ext_parent(node.func): + # TODO: This doesn't work for cases + # where there is an assignment of an attribute + # i.e. import os; lala = os.path; lala.dirname() + for name in self.get_full_attr_names(node.func): + ext_modname = name.split(".")[0] + create_ext_edge(name, ext_modname, code) + elif getattr(node.func, "id", None) and self.is_builtin(node.func.id): + name = utils.join_ns(utils.constants.BUILTIN_NAME, node.func.id) + create_ext_edge(name, utils.constants.BUILTIN_NAME, code) + return + + self.last_called_names = names + for pointer in names: + pointer_def = self.def_manager.get(pointer) + if not pointer_def or not isinstance(pointer_def, Definition): + continue + if pointer_def.is_callable(): + if pointer_def.get_type() == utils.constants.EXT_DEF: + ext_modname = pointer.split(".")[0] + create_ext_edge(pointer, ext_modname, code) + continue + self.call_graph.add_edge(self.current_method, pointer, code) + + # TODO: This doesn't work + # and leads to calls from the decorators + # themselves to the function, + # creating edges to the first decorator + # for decorator in pointer_def.decorator_names: + # dec_names = self.closured.get(decorator, []) + # for dec_name in dec_names: + # if self.def_manager.get(dec_name). + # get_type() == utils.constants.FUN_DEF: + # self.call_graph.add_edge(self.current_ns, dec_name) + + if pointer_def.get_type() == utils.constants.CLS_DEF: + init_ns = self.find_cls_fun_ns(pointer, utils.constants.CLS_INIT) + + for ns in init_ns: + self.call_graph.add_edge(self.current_method, ns, code) + + def analyze_submodules(self): + super().analyze_submodules( + CallGraphProcessor, + self.import_manager, + self.scope_manager, + self.def_manager, + self.class_manager, + self.module_manager, + call_graph=self.call_graph, + modules_analyzed=self.get_modules_analyzed(), + ) + + def analyze(self): + self.visit(ast.parse(self.contents, self.filename)) + self.analyze_submodules() + + def get_all_reachable_functions(self): + reachable = set() + names = set() + current_scope = self.scope_manager.get_scope(self.current_ns) + while current_scope: + for name, defi in current_scope.get_defs().items(): + if defi.is_function_def() and name not in names: + closured = self.closured.get(defi.get_ns()) + for item in closured: + reachable.add(item) + names.add(name) + current_scope = current_scope.parent + + return reachable + + def has_ext_parent(self, node): + if not isinstance(node, ast.Attribute): + return False + + while isinstance(node, ast.Attribute): + parents = self._retrieve_parent_names(node) + for parent in parents: + for name in self.closured.get(parent, []): + defi = self.def_manager.get(name) + if defi and defi.is_ext_def(): + return True + node = node.value + return False + + def get_full_attr_names(self, node): + name = "" + while isinstance(node, ast.Attribute): + if not name: + name = node.attr + else: + name = node.attr + "." + name + node = node.value + + names = [] + if getattr(node, "id", None) is None: + return names + + defi = self.scope_manager.get_def(self.current_ns, node.id) + if defi and self.closured.get(defi.get_ns()): + for id in self.closured.get(defi.get_ns()): + names.append(id + "." + name) + + return names + + def is_builtin(self, name): + return name in __builtins__ diff --git a/onecode/pycg/processing/postprocessor.py b/onecode/pycg/processing/postprocessor.py new file mode 100644 index 0000000..47fb778 --- /dev/null +++ b/onecode/pycg/processing/postprocessor.py @@ -0,0 +1,342 @@ +# +# Copyright (c) 2020 Vitalis Salis. +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +import ast + +from onecode.pycg import utils +from onecode.pycg.machinery.definitions import Definition +from onecode.pycg.processing.base import ProcessingBase + + +class PostProcessor(ProcessingBase): + def __init__( + self, + input_file, + modname, + import_manager, + scope_manager, + def_manager, + class_manager, + module_manager, + modules_analyzed=None, + ): + super().__init__(input_file, modname, modules_analyzed) + self.import_manager = import_manager + self.scope_manager = scope_manager + self.def_manager = def_manager + self.class_manager = class_manager + self.module_manager = module_manager + self.closured = self.def_manager.transitive_closure() + + def visit_Lambda(self, node): + counter = self.scope_manager.get_scope(self.current_ns).inc_lambda_counter() + lambda_name = utils.get_lambda_name(counter) + super().visit_Lambda(node, lambda_name) + + def visit_Call(self, node): + self.visit(node.func) + + names = self.retrieve_call_names(node) + if not names: + return + + self.last_called_names = names + + for name in names: + defi = self.def_manager.get(name) + if not defi: + continue + if defi.get_type() == utils.constants.CLS_DEF: + self.update_parent_classes(defi) + defi = self.def_manager.get( + utils.join_ns(defi.get_ns(), utils.constants.CLS_INIT) + ) + if not defi: + continue + self.iterate_call_args(defi, node) + + def visit_Assign(self, node): + self._visit_assign(node.value, node.targets) + + # Redefined in line 118, 121 + # def visit_Return(self, node): + # self._visit_return(node) + + # def visit_Yield(self, node): + # self._visit_return(node) + + def visit_For(self, node): + # only handle name targets + if isinstance(node.target, ast.Name): + target_def = self.def_manager.get( + utils.join_ns(self.current_ns, node.target.id) + ) + # if the target definition exists + if target_def: + iter_decoded = self.decode_node(node.iter) + # assign the target to the return value + # of the next function + for item in iter_decoded: + if not isinstance(item, Definition): + continue + # return value for generators + for name in self.closured.get(item.get_ns(), []): + # If there exists a next method on the iterable + # and if yes, add a pointer to it + next_defi = self.def_manager.get( + utils.join_ns( + name, + utils.constants.NEXT_METHOD, + utils.constants.RETURN_NAME, + ) + ) + if next_defi: + for name in self.closured.get(next_defi.get_ns(), []): + target_def.get_name_pointer().add(name) + else: # otherwise, add a pointer to the name + # (e.g. a yield) + target_def.get_name_pointer().add(name) + + super().visit_For(node) + + def visit_Return(self, node): + self._visit_return(node) + + def visit_Yield(self, node): + self._visit_return(node) + + def visit_AsyncFunctionDef(self, node): + self.visit_FunctionDef(node) + + def visit_FunctionDef(self, node): + # here we iterate decorators + if node.decorator_list: + fn_def = self.def_manager.get(utils.join_ns(self.current_ns, node.name)) + reversed_decorators = list(reversed(node.decorator_list)) + + # add to the name pointer of the function definition + # the return value of the first decorator + # since, now the function is a namespace to that point + if hasattr(fn_def, "decorator_names") and reversed_decorators: + last_decoded = self.decode_node(reversed_decorators[-1]) + for d in last_decoded: + if not isinstance(d, Definition): + continue + fn_def.decorator_names.add( + utils.join_ns(d.get_ns(), utils.constants.RETURN_NAME) + ) + + previous_names = self.closured.get(fn_def.get_ns(), set()) + for decorator in reversed_decorators: + # assign the previous_def + # as the first parameter of the decorator + decoded = self.decode_node(decorator) + new_previous_names = set() + for d in decoded: + if not isinstance(d, Definition): + continue + for name in self.closured.get(d.get_ns(), []): + return_ns = utils.join_ns(name, utils.constants.RETURN_NAME) + + if self.closured.get(return_ns, None) is None: + continue + + new_previous_names = new_previous_names.union( + self.closured.get(return_ns) + ) + + for prev_name in previous_names: + pos_arg_names = d.get_name_pointer().get_pos_arg(0) + if not pos_arg_names: + continue + for name in pos_arg_names: + arg_def = self.def_manager.get(name) + arg_def.get_name_pointer().add(prev_name) + previous_names = new_previous_names + + super().visit_FunctionDef(node) + + def visit_ClassDef(self, node): + # create a definition for the class (node.name) + cls_def = self.def_manager.handle_class_def(self.current_ns, node.name) + + # iterate bases to compute MRO for the class + cls = self.class_manager.get(cls_def.get_ns()) + if not cls: + cls = self.class_manager.create(cls_def.get_ns(), self.modname) + + cls.clear_mro() + for base in node.bases: + # all bases are of the type ast.Name + self.visit(base) + + bases = self.decode_node(base) + for base_def in bases: + if not isinstance(base_def, Definition): + continue + names = set() + if base_def.get_name_pointer().get(): + names = base_def.get_name_pointer().get() + else: + names.add(base_def.get_ns()) + for name in names: + # add the base as a parent + cls.add_parent(name) + + # add the base's parents + parent_cls = self.class_manager.get(name) + if parent_cls: + cls.add_parent(parent_cls.get_mro()) + + cls.compute_mro() + + super().visit_ClassDef(node) + + def visit_List(self, node): + # Works similarly with dicts + current_scope = self.scope_manager.get_scope(self.current_ns) + list_counter = current_scope.inc_list_counter() + list_name = utils.get_list_name(list_counter) + list_full_ns = utils.join_ns(self.current_ns, list_name) + + # create a scope for the list + self.scope_manager.create_scope(list_full_ns, current_scope) + + # create a list definition + list_def = self.def_manager.get(list_full_ns) + if not list_def: + list_def = self.def_manager.create(list_full_ns, utils.constants.NAME_DEF) + current_scope.add_def(list_name, list_def) + + self.name_stack.append(list_name) + for idx, elt in enumerate(node.elts): + self.visit(elt) + key_full_ns = utils.join_ns(list_def.get_ns(), utils.get_int_name(idx)) + key_def = self.def_manager.get(key_full_ns) + if not key_def: + key_def = self.def_manager.create(key_full_ns, utils.constants.NAME_DEF) + + decoded_elt = self.decode_node(elt) + for v in decoded_elt: + if isinstance(v, Definition): + key_def.get_name_pointer().add(v.get_ns()) + else: + key_def.get_lit_pointer().add(v) + + self.name_stack.pop() + + def visit_Dict(self, node): + # 1. create a scope using a counter + # 2. Iterate keys and add them as children of the scope + # 3. Iterate values and makes a points to connection with the keys + current_scope = self.scope_manager.get_scope(self.current_ns) + dict_counter = current_scope.inc_dict_counter() + dict_name = utils.get_dict_name(dict_counter) + dict_full_ns = utils.join_ns(self.current_ns, dict_name) + + # create a scope for the dict + dict_scope = self.scope_manager.create_scope(dict_full_ns, current_scope) + + # Create a dict definition + dict_def = self.def_manager.get(dict_full_ns) + if not dict_def: + dict_def = self.def_manager.create(dict_full_ns, utils.constants.NAME_DEF) + # add it to the current scope + current_scope.add_def(dict_name, dict_def) + + self.name_stack.append(dict_name) + for key, value in zip(node.keys, node.values): + if key: + self.visit(key) + if value: + self.visit(value) + decoded_key = self.decode_node(key) + decoded_value = self.decode_node(value) + + # iterate decoded keys and values + # to do the assignment operation + for k in decoded_key: + if isinstance(k, Definition): + # get literal pointer + names = k.get_lit_pointer().get() + else: + names = set() + if isinstance(k, list): + continue + names.add(k) + for name in names: + # create a definition for the key + if isinstance(name, int): + name = utils.get_int_name(name) + key_full_ns = utils.join_ns(dict_def.get_ns(), str(name)) + key_def = self.def_manager.get(key_full_ns) + if not key_def: + key_def = self.def_manager.create( + key_full_ns, utils.constants.NAME_DEF + ) + dict_scope.add_def(str(name), key_def) + for v in decoded_value: + if isinstance(v, Definition): + key_def.get_name_pointer().add(v.get_ns()) + else: + key_def.get_lit_pointer().add(v) + self.name_stack.pop() + + def update_parent_classes(self, defi): + cls = self.class_manager.get(defi.get_ns()) + if not cls: + return + current_scope = self.scope_manager.get_scope(defi.get_ns()) + for parent in cls.get_mro(): + parent_def = self.def_manager.get(parent) + if not parent_def: + continue + parent_scope = self.scope_manager.get_scope(parent) + if not parent_scope: + continue + list(parent_scope.get_defs().keys()) + for key, child_def in current_scope.get_defs().items(): + if key == "__init__": + continue + # resolve name from the parent_def + names = self.find_cls_fun_ns(parent_def.get_ns(), key) + + new_ns = utils.join_ns(parent_def.get_ns(), key) + new_def = self.def_manager.get(new_ns) + if not new_def: + new_def = self.def_manager.create(new_ns, utils.constants.NAME_DEF) + + new_def.get_name_pointer().add_set(names) + new_def.get_name_pointer().add(child_def.get_ns()) + + def analyze_submodules(self): + super().analyze_submodules( + PostProcessor, + self.import_manager, + self.scope_manager, + self.def_manager, + self.class_manager, + self.module_manager, + modules_analyzed=self.get_modules_analyzed(), + ) + + def analyze(self): + self.visit(ast.parse(self.contents, self.filename)) + self.analyze_submodules() diff --git a/onecode/pycg/processing/preprocessor.py b/onecode/pycg/processing/preprocessor.py new file mode 100644 index 0000000..685ad6f --- /dev/null +++ b/onecode/pycg/processing/preprocessor.py @@ -0,0 +1,425 @@ +# +# Copyright (c) 2020 Vitalis Salis. +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +import ast + +from onecode.pycg import utils +from onecode.pycg.machinery.definitions import Definition +from onecode.pycg.processing.base import ProcessingBase + + +class PreProcessor(ProcessingBase): + def __init__( + self, + filename, + modname, + import_manager, + scope_manager, + def_manager, + class_manager, + module_manager, + modules_analyzed=None, + ): + super().__init__(filename, modname, modules_analyzed) + + self.modname = modname + self.mod_dir = "/".join(self.filename.split("/")[:-1]) + + self.import_manager = import_manager + self.scope_manager = scope_manager + self.def_manager = def_manager + self.class_manager = class_manager + self.module_manager = module_manager + + def _get_fun_defaults(self, node): + defaults = {} + pos_args = node.args.posonlyargs + node.args.args + start = len(pos_args) - len(node.args.defaults) + for cnt, d in enumerate(node.args.defaults, start=start): + if not d: + continue + + self.visit(d) + defaults[pos_args[cnt].arg] = self.decode_node(d) + + start = len(node.args.kwonlyargs) - len(node.args.kw_defaults) + for cnt, d in enumerate(node.args.kw_defaults, start=start): + if not d: + continue + self.visit(d) + defaults[node.args.kwonlyargs[cnt].arg] = self.decode_node(d) + + return defaults + + def analyze_submodule(self, modname): + super().analyze_submodule( + PreProcessor, + modname, + self.import_manager, + self.scope_manager, + self.def_manager, + self.class_manager, + self.module_manager, + modules_analyzed=self.get_modules_analyzed(), + ) + + def visit_Module(self, node): + def iterate_mod_items(items, const): + for item in items: + defi = self.def_manager.get(item) + if not defi: + defi = self.def_manager.create(item, const) + + splitted = item.split(".") + name = splitted[-1] + parentns = ".".join(splitted[:-1]) + self.scope_manager.get_scope(parentns).add_def(name, defi) + + self.import_manager.set_current_mod(self.modname, self.filename) + + mod = self.module_manager.create(self.modname, self.filename) + + first = 1 + last = len(self.contents.splitlines()) + if last == 0: + first = 0 + mod.add_method(self.modname, first, last) + + root_sc = self.scope_manager.get_scope(self.modname) + if not root_sc: + # initialize module scopes + items = self.scope_manager.handle_module( + self.modname, self.filename, self.contents + ) + + root_sc = self.scope_manager.get_scope(self.modname) + root_defi = self.def_manager.get(self.modname) + if not root_defi: + root_defi = self.def_manager.create( + self.modname, utils.constants.MOD_DEF + ) + root_sc.add_def(self.modname.split(".")[-1], root_defi) + + # create function and class defs and add them to their scope + # we do this here, because scope_manager doesn't have an + # interface with def_manager, and we want function definitions + # to have the correct points_to set + iterate_mod_items(items["functions"], utils.constants.FUN_DEF) + iterate_mod_items(items["classes"], utils.constants.CLS_DEF) + + defi = self.def_manager.get(self.modname) + if not defi: + defi = self.def_manager.create(self.modname, utils.constants.MOD_DEF) + + super().visit_Module(node) + + def visit_Import(self, node, prefix="", level=0): + """ + For imports of the form + `from something import anything` + prefix is set to "something". + For imports of the form + `from .relative import anything` + level is set to a number indicating the number + of parent directories (e.g. in this case level=1) + """ + + def handle_src_name(name): + # Get the module name and prepend prefix if necessary + src_name = name + if prefix: + src_name = prefix + "." + src_name + return src_name + + def handle_scopes(imp_name, tgt_name, modname): + def create_def(scope, name, imported_def): + if name not in scope.get_defs(): + def_ns = utils.join_ns(scope.get_ns(), name) + defi = self.def_manager.get(def_ns) + if not defi: + defi = self.def_manager.assign(def_ns, imported_def) + defi.get_name_pointer().add(imported_def.get_ns()) + current_scope.add_def(name, defi) + + current_scope = self.scope_manager.get_scope(self.current_ns) + imported_scope = self.scope_manager.get_scope(modname) + if tgt_name == "*": + for name, defi in imported_scope.get_defs().items(): + create_def(current_scope, name, defi) + current_scope.get_def(name).get_name_pointer().add(defi.get_ns()) + else: + # if it exists in the imported scope then copy it + defi = imported_scope.get_def(imp_name) + if not defi: + # maybe its a full namespace + defi = self.def_manager.get(imp_name) + + if defi: + create_def(current_scope, tgt_name, defi) + current_scope.get_def(tgt_name).get_name_pointer().add( + defi.get_ns() + ) + + def add_external_def(name, target): + # In case we encounter an external import in the form of: + # "import package.module.module... + # we want to treat it as: "import package" + # and save it as such in the definition manager, + # so that we will be able to later map it + # with its corresponding calls + if (name == target) & (len(name.split(".")) > 1): + name = name.split(".")[0] + target = target.split(".")[0] + # add an external def for the name + defi = self.def_manager.get(name) + if not defi: + defi = self.def_manager.create(name, utils.constants.EXT_DEF) + scope = self.scope_manager.get_scope(self.current_ns) + if target != "*": + # add a def for the target that points to the name + tgt_ns = utils.join_ns(scope.get_ns(), target) + tgt_defi = self.def_manager.get(tgt_ns) + if not tgt_defi: + tgt_defi = self.def_manager.create(tgt_ns, utils.constants.EXT_DEF) + tgt_defi.get_name_pointer().add(defi.get_ns()) + scope.add_def(target, tgt_defi) + + for import_item in node.names: + src_name = handle_src_name(import_item.name) + tgt_name = import_item.asname if import_item.asname else import_item.name + imported_name = self.import_manager.handle_import(src_name, level) + + if not imported_name: + add_external_def(src_name, tgt_name) + continue + + fname = self.import_manager.get_filepath(imported_name) + if not fname: + add_external_def(src_name, tgt_name) + continue + # only analyze modules under the current directory + if self.import_manager.get_mod_dir() in fname: + if imported_name not in self.modules_analyzed: + self.analyze_submodule(imported_name) + handle_scopes(import_item.name, tgt_name, imported_name) + else: + add_external_def(src_name, tgt_name) + + # handle all modules that were not analyzed + for modname in self.import_manager.get_imports(self.modname): + fname = self.import_manager.get_filepath(modname) + + if not fname: + continue + # only analyze modules under the current directory + if ( + self.import_manager.get_mod_dir() in fname + and modname not in self.modules_analyzed + ): + self.analyze_submodule(modname) + + def visit_ImportFrom(self, node): + self.visit_Import(node, prefix=node.module, level=node.level) + + def _get_last_line(self, node): + lines = sorted( + list(ast.walk(node)), + key=lambda x: x.lineno if hasattr(x, "lineno") else 0, + reverse=True, + ) + if not lines: + return node.lineno + + last = getattr(lines[0], "lineno", node.lineno) + if last < node.lineno: + return node.lineno + + return last + + def _handle_function_def(self, node, fn_name): + current_def = self.def_manager.get(self.current_ns) + + defaults = self._get_fun_defaults(node) + + fn_def = self.def_manager.handle_function_def(self.current_ns, fn_name) + + mod = self.module_manager.get(self.modname) + if not mod: + mod = self.module_manager.create(self.modname, self.filename) + mod.add_method(fn_def.get_ns(), node.lineno, self._get_last_line(node)) + + defs_to_create = [] + name_pointer = fn_def.get_name_pointer() + + # TODO: static methods can be created using + # the staticmethod() function too + is_static_method = False + if hasattr(node, "decorator_list"): + for decorator in node.decorator_list: + if ( + isinstance(decorator, ast.Name) + and decorator.id == utils.constants.STATIC_METHOD + ): + is_static_method = True + + if ( + current_def.get_type() == utils.constants.CLS_DEF + and not is_static_method + and node.args.args + ): + arg_ns = utils.join_ns(fn_def.get_ns(), node.args.args[0].arg) + arg_def = self.def_manager.get(arg_ns) + if not arg_def: + arg_def = self.def_manager.create(arg_ns, utils.constants.NAME_DEF) + arg_def.get_name_pointer().add(current_def.get_ns()) + + self.scope_manager.handle_assign( + fn_def.get_ns(), arg_def.get_name(), arg_def + ) + node.args.args = node.args.args[1:] + + for pos, arg in enumerate(node.args.args): + arg_ns = utils.join_ns(fn_def.get_ns(), arg.arg) + name_pointer.add_pos_arg(pos, arg.arg, arg_ns) + defs_to_create.append(arg_ns) + + for arg in node.args.kwonlyargs: + arg_ns = utils.join_ns(fn_def.get_ns(), arg.arg) + # TODO: add_name_arg function + name_pointer.add_name_arg(arg.arg, arg_ns) + defs_to_create.append(arg_ns) + + # TODO: Add support for kwargs and varargs + # if node.args.kwarg: + # pass + # if node.args.vararg: + # pass + + for arg_ns in defs_to_create: + arg_def = self.def_manager.get(arg_ns) + if not arg_def: + arg_def = self.def_manager.create(arg_ns, utils.constants.NAME_DEF) + + self.scope_manager.handle_assign( + fn_def.get_ns(), arg_def.get_name(), arg_def + ) + + # has a default + arg_name = arg_ns.split(".")[-1] + if defaults.get(arg_name, None): + for default in defaults[arg_name]: + if isinstance(default, Definition): + arg_def.get_name_pointer().add(default.get_ns()) + if default.is_function_def(): + arg_def.get_name_pointer().add(default.get_ns()) + else: + arg_def.merge(default) + else: + arg_def.get_lit_pointer().add(default) + return fn_def + + def visit_AsyncFunctionDef(self, node): + self.visit_FunctionDef(node) + + def visit_FunctionDef(self, node): + self._handle_function_def(node, node.name) + + super().visit_FunctionDef(node) + + def visit_For(self, node): + # just create the definition for target + if isinstance(node.target, ast.Name): + target_ns = utils.join_ns(self.current_ns, node.target.id) + if not self.def_manager.get(target_ns): + defi = self.def_manager.create(target_ns, utils.constants.NAME_DEF) + self.scope_manager.get_scope(self.current_ns).add_def( + node.target.id, defi + ) + super().visit_For(node) + + def visit_Assign(self, node): + self._visit_assign(node.value, node.targets) + + def visit_Return(self, node): + self._visit_return(node) + + def visit_Yield(self, node): + self._visit_return(node) + + def visit_Call(self, node): + self.visit(node.func) + # if it is not a name there's nothing we can do here + # ModuleVisitor will be able to resolve those calls + # since it'll have the name tracking information + if not isinstance(node.func, ast.Name): + return + + utils.join_ns(self.current_ns, node.func.id) + + defi = self.scope_manager.get_def(self.current_ns, node.func.id) + if not defi: + return + + if defi.get_type() == utils.constants.CLS_DEF: + defi = self.def_manager.get( + utils.join_ns(defi.get_ns(), utils.constants.CLS_INIT) + ) + if not defi: + return + + self.iterate_call_args(defi, node) + + def visit_Lambda(self, node): + # The name of a lambda is defined by the counter of the current scope + current_scope = self.scope_manager.get_scope(self.current_ns) + lambda_counter = current_scope.inc_lambda_counter() + lambda_name = utils.get_lambda_name(lambda_counter) + lambda_full_ns = utils.join_ns(self.current_ns, lambda_name) + + # create a scope for the lambda + self.scope_manager.create_scope(lambda_full_ns, current_scope) + lambda_def = self._handle_function_def(node, lambda_name) + # add it to the current scope + current_scope.add_def(lambda_name, lambda_def) + + super().visit_Lambda(node, lambda_name) + + def visit_ClassDef(self, node): + # create a definition for the class (node.name) + cls_def = self.def_manager.handle_class_def(self.current_ns, node.name) + + mod = self.module_manager.get(self.modname) + if not mod: + mod = self.module_manager.create(self.modname, self.filename) + mod.add_method(cls_def.get_ns(), node.lineno, self._get_last_line(node)) + + # iterate bases to compute MRO for the class + cls = self.class_manager.get(cls_def.get_ns()) + if not cls: + cls = self.class_manager.create(cls_def.get_ns(), self.modname) + + super().visit_ClassDef(node) + + def analyze(self): + if not self.import_manager.get_node(self.modname): + self.import_manager.create_node(self.modname) + self.import_manager.set_filepath(self.modname, self.filename) + + self.visit(ast.parse(self.contents, self.filename)) diff --git a/onecode/pycg/pycg.py b/onecode/pycg/pycg.py new file mode 100644 index 0000000..a09e140 --- /dev/null +++ b/onecode/pycg/pycg.py @@ -0,0 +1,246 @@ +# +# Copyright (c) 2020 Vitalis Salis. +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +import os + +from onecode.pycg import utils +from onecode.pycg.machinery.callgraph import CallGraph +from onecode.pycg.machinery.classes import ClassManager +from onecode.pycg.machinery.definitions import DefinitionManager +from onecode.pycg.machinery.imports import ImportManager +from onecode.pycg.machinery.modules import ModuleManager +from onecode.pycg.machinery.scopes import ScopeManager +from onecode.pycg.processing.cgprocessor import CallGraphProcessor +from onecode.pycg.processing.postprocessor import PostProcessor +from onecode.pycg.processing.preprocessor import PreProcessor + + +class CallGraphGenerator(object): + def __init__(self, entry_points, package, max_iter, operation): + self.entry_points = entry_points + self.package = package + self.state = None + self.max_iter = max_iter + self.operation = operation + self.setUp() + + def setUp(self): + self.import_manager = ImportManager() + self.scope_manager = ScopeManager() + self.def_manager = DefinitionManager() + self.class_manager = ClassManager() + self.module_manager = ModuleManager() + self.cg = CallGraph() + + def extract_state(self): + state = {} + state["defs"] = {} + for key, defi in self.def_manager.get_defs().items(): + state["defs"][key] = { + "names": defi.get_name_pointer().get().copy(), + "lit": defi.get_lit_pointer().get().copy(), + } + + state["scopes"] = {} + for key, scope in self.scope_manager.get_scopes().items(): + state["scopes"][key] = set([ + x.get_ns() for (_, x) in scope.get_defs().items() + ]) + + state["classes"] = {} + for key, ch in self.class_manager.get_classes().items(): + state["classes"][key] = ch.get_mro().copy() + return state + + def reset_counters(self): + for key, scope in self.scope_manager.get_scopes().items(): + scope.reset_counters() + + def has_converged(self): + if not self.state: + return False + + curr_state = self.extract_state() + + # check defs + for key, defi in curr_state["defs"].items(): + if key not in self.state["defs"]: + return False + if defi["names"] != self.state["defs"][key]["names"]: + return False + if defi["lit"] != self.state["defs"][key]["lit"]: + return False + + # check scopes + for key, scope in curr_state["scopes"].items(): + if key not in self.state["scopes"]: + return False + if scope != self.state["scopes"][key]: + return False + + # check classes + for key, ch in curr_state["classes"].items(): + if key not in self.state["classes"]: + return False + if ch != self.state["classes"][key]: + return False + + return True + + def remove_import_hooks(self): + self.import_manager.remove_hooks() + + def tearDown(self): + self.remove_import_hooks() + + def _get_mod_name(self, entry, pkg): + # We do this because we want __init__ modules to + # only contain the parent module + # since pycg can't differentiate between functions + # coming from __init__ files. + + input_mod = utils.to_mod_name(os.path.relpath(entry, pkg)) + if input_mod.endswith("__init__"): + input_mod = ".".join(input_mod.split(".")[:-1]) + + return input_mod + + def do_pass(self, cls, install_hooks=False, *args, **kwargs): + modules_analyzed = set() + for entry_point in self.entry_points: + input_pkg = self.package + input_mod = self._get_mod_name(entry_point, input_pkg) + input_file = os.path.abspath(entry_point) + + if not input_mod: + continue + + if not input_pkg: + input_pkg = os.path.dirname(input_file) + + if input_mod not in modules_analyzed: + if install_hooks: + self.import_manager.set_pkg(input_pkg) + self.import_manager.install_hooks() + + processor = cls( + input_file, + input_mod, + modules_analyzed=modules_analyzed, + *args, + **kwargs, + ) + processor.analyze() + modules_analyzed = modules_analyzed.union( + processor.get_modules_analyzed() + ) + + if install_hooks: + self.remove_import_hooks() + + def analyze(self): + self.do_pass( + PreProcessor, + True, + self.import_manager, + self.scope_manager, + self.def_manager, + self.class_manager, + self.module_manager, + ) + self.def_manager.complete_definitions() + + iter_cnt = 0 + while (self.max_iter < 0 or iter_cnt < self.max_iter) and ( + not self.has_converged() + ): + self.state = self.extract_state() + self.reset_counters() + self.do_pass( + PostProcessor, + False, + self.import_manager, + self.scope_manager, + self.def_manager, + self.class_manager, + self.module_manager, + ) + + self.def_manager.complete_definitions() + iter_cnt += 1 + + self.reset_counters() + if self.operation == utils.constants.CALL_GRAPH_OP: + self.do_pass( + CallGraphProcessor, + False, + self.import_manager, + self.scope_manager, + self.def_manager, + self.class_manager, + self.module_manager, + call_graph=self.cg, + ) + else: + raise Exception("Invalid operation: " + self.operation) + + def output(self): + return self.cg.get() + + def output_enriched(self): + return self.cg.get_enriched() + + def output_edges(self): + return self.cg.get_edges() + + def _generate_mods(self, mods): + res = {} + for mod, node in mods.items(): + res[mod] = { + "filename": ( + os.path.relpath(node.get_filename(), self.package) + if node.get_filename() + else None + ), + "methods": node.get_methods(), + } + return res + + def output_internal_mods(self): + return self._generate_mods(self.module_manager.get_internal_modules()) + + def output_external_mods(self): + return self._generate_mods(self.module_manager.get_external_modules()) + + def output_functions(self): + functions = [] + for ns, defi in self.def_manager.get_defs().items(): + if defi.is_function_def(): + functions.append(ns) + return functions + + def output_classes(self): + classes = {} + for cls, node in self.class_manager.get_classes().items(): + classes[cls] = {"mro": node.get_mro(), "module": node.get_module()} + return classes + + def get_as_graph(self): + return self.def_manager.get_defs().items() diff --git a/onecode/pycg/utils/__init__.py b/onecode/pycg/utils/__init__.py new file mode 100644 index 0000000..b873773 --- /dev/null +++ b/onecode/pycg/utils/__init__.py @@ -0,0 +1,22 @@ +# +# Copyright (c) 2020 Vitalis Salis. +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +from . import constants # noqa: F401 +from .common import * # noqa diff --git a/onecode/pycg/utils/common.py b/onecode/pycg/utils/common.py new file mode 100644 index 0000000..6346198 --- /dev/null +++ b/onecode/pycg/utils/common.py @@ -0,0 +1,45 @@ +# +# Copyright (c) 2020 Vitalis Salis. +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +import os + + +def get_lambda_name(counter): + return "".format(counter) + + +def get_dict_name(counter): + return "".format(counter) + + +def get_list_name(counter): + return "".format(counter) + + +def get_int_name(counter): + return "".format(counter) + + +def join_ns(*args): + return ".".join([arg for arg in args]) + + +def to_mod_name(name, package=None): + return os.path.splitext(name)[0].replace("/", ".") diff --git a/onecode/pycg/utils/constants.py b/onecode/pycg/utils/constants.py new file mode 100644 index 0000000..0a1da88 --- /dev/null +++ b/onecode/pycg/utils/constants.py @@ -0,0 +1,41 @@ +# +# Copyright (c) 2020 Vitalis Salis. +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +RETURN_NAME = "" +LAMBDA_NAME = "" # needs to be formatted +BUILTIN_NAME = "" +EXT_NAME = "" + +FUN_DEF = "FUNCTIONDEF" +NAME_DEF = "NAMEDEF" +MOD_DEF = "MODULEDEF" +CLS_DEF = "CLASSDEF" +EXT_DEF = "EXTERNALDEF" + +OBJECT_BASE = "object" + +CLS_INIT = "__init__" +ITER_METHOD = "__iter__" +NEXT_METHOD = "__next__" +STATIC_METHOD = "staticmethod" + +INVALID_NAME = "<**INVALID**>" + +CALL_GRAPH_OP = "call-graph" diff --git a/onecode/utils/import_input.py b/onecode/utils/import_input.py index 57d5036..5e11289 100644 --- a/onecode/utils/import_input.py +++ b/onecode/utils/import_input.py @@ -8,12 +8,10 @@ import pydash -from ..base.decorator import check_type from ..base.enums import ElementType from ..base.project import Project -@check_type def import_input( init_file: str, module_name: str diff --git a/onecode/utils/import_output.py b/onecode/utils/import_output.py index 8269ff2..0f3a534 100644 --- a/onecode/utils/import_output.py +++ b/onecode/utils/import_output.py @@ -8,12 +8,10 @@ import pydash -from ..base.decorator import check_type from ..base.enums import ElementType from ..base.project import Project -@check_type def import_output( init_file: str, module_name: str diff --git a/onecode/utils/module.py b/onecode/utils/module.py index 6669cf4..2abf8ac 100644 --- a/onecode/utils/module.py +++ b/onecode/utils/module.py @@ -13,14 +13,13 @@ import requirements from packaging.specifiers import SpecifierSet -from pycg.pycg import CallGraphGenerator -from pycg.utils.constants import CALL_GRAPH_OP -from ..base.decorator import check_type +from onecode.pycg.pycg import CallGraphGenerator as _CallGraphGenerator +from onecode.pycg.utils.constants import CALL_GRAPH_OP as _CALL_GRAPH_OP + from ..base.enums import Env -@check_type def register_ext_module( project_path: str = os.getcwd(), module_name: str = "onecode_ext", @@ -108,7 +107,6 @@ def _collect_flow_helper_files(flow_files: List[str]) -> List[str]: return helpers -@check_type def get_call_graph_entry_files(project_path: str) -> List[str]: """ Return the Python entry files used for static call-graph analysis. @@ -153,7 +151,6 @@ def get_call_graph_entry_files(project_path: str) -> List[str]: return entry_files -@check_type def get_imported_modules(scripts_folder: str) -> List[str]: """ Get the names of all modules imported by the Python scripts present in the given folder. @@ -166,18 +163,17 @@ def get_imported_modules(scripts_folder: str) -> List[str]: """ entry_files = get_call_graph_entry_files(scripts_folder) - cg = CallGraphGenerator( + cg = _CallGraphGenerator( entry_files, scripts_folder, 0, - CALL_GRAPH_OP + _CALL_GRAPH_OP ) cg.analyze() return list(cg.output_external_mods().keys() - {''}) -@check_type def _find_version(dist_name: str) -> Optional[str]: """ Find the version of the distribution package if found. @@ -196,7 +192,6 @@ def _find_version(dist_name: str) -> Optional[str]: return dist.version if dist is not None else None -@check_type def check_modules( modules: List[str], requirements_file: Optional[str] = None @@ -261,7 +256,6 @@ def check_modules( return mods -@check_type def write_requirements( to_file: str, scripts_folder: str = os.getcwd(), diff --git a/onecode/utils/typing.py b/onecode/utils/typing.py index f4624e8..89a9e51 100644 --- a/onecode/utils/typing.py +++ b/onecode/utils/typing.py @@ -1,9 +1,8 @@ # SPDX-FileCopyrightText: 2023-2024 DeepLime # SPDX-License-Identifier: MIT -from typing import Any - -from typeguard import check_type +from types import UnionType +from typing import Any, Union, get_args, get_origin def is_type( @@ -22,9 +21,22 @@ def is_type( True if the object match the type, otherwise False. """ - try: - check_type(value=obj, expected_type=t) + if t is Any: return True - except Exception: + origin = get_origin(t) + if origin is Union or origin is UnionType: + return any(is_type(obj, arg) for arg in get_args(t)) + + if origin is list: + if not isinstance(obj, list): + return False + args = get_args(t) + if not args or args == (Any,): + return True + return all(is_type(item, args[0]) for item in obj) + + try: + return isinstance(obj, t) + except TypeError: return False diff --git a/pyproject.toml b/pyproject.toml index b3d1ef0..e6c6ded 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "onecode" -version = "1.2.2" +version = "1.3.0.dev" description = "Python skeleton and library for OneCode projects" readme = "README.md" authors = ["DeepLime "] @@ -18,57 +18,57 @@ classifiers = [ "Operating System :: Unix", "Operating System :: Microsoft :: Windows", "Programming Language :: Python", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", + "Programming Language :: Python :: 3.14", "Topic :: Scientific/Engineering", "Topic :: Software Development :: Code Generators", ] +[tool.poetry.urls] +SBOM = "https://github.com/deeplime-io/onecode/releases/latest" + [tool.poetry.dependencies] -python = ">=3.8, <3.15" -'flufl.lock' = ">=7.1.1,<9" -pandas = ">=1.4.0,<3" -pydash = ">=5.1.0,<8" -python-slugify = ">=6.1.2,<9" -strenum = ">=0.4.6,<1" -typeguard = ">=3,<5" +python = ">=3.11, <3.15" +filelock = ">=3.30,<5" +pandas = ">=2.3,<4" +pyarrow = ">=22,<26" +pydash = ">=7,<9" +python-slugify = ">=8,<10" requirements-parser = ">=0.10.2,<1" # cli inquirerpy = ">=0.3.3,<1" -onecode-pycg = ">=0.0.7,<2" -yaspin = ">=2.1.0,<4" +yaspin = ">=3,<4" # docs -griffe = { version = "^0", optional = true } -mike = { version = "~1.1", optional = true } -mkdocs = { version = "^1.5", optional = true } -mkdocs-material = { version = "^9.5", optional = true } -mkdocstrings = { version = "^0", optional = true } -mkdocstrings-python = { version = "^1", optional = true } +griffe = { version = ">=2,<3", optional = true } +mike = { version = ">=2,<3", optional = true } +mkdocs = { version = ">=1.6,<2", optional = true } +mkdocs-material = { version = ">=9.7.7,<10", optional = true } +mkdocstrings = { version = ">=1,<2", optional = true } +mkdocstrings-python = { version = ">=2,<3", optional = true } # developer datatest = { version = ">=0.11.1,<1", optional = true } -pydantic = { version = ">=2,<3", optional = true } -pytest = { version = ">=7,<8", optional = true } -pytest-cov = { version = ">=2.10.1,<5", optional = true } -pytest-xdist = { version = ">=3.1.0,<4", optional = true } -toml = { version = ">=0.10.2,<1", optional = true } +pytest = { version = ">=9.0.3,<10", optional = true } +pytest-cov = { version = ">=6,<8", optional = true } +pytest-xdist = { version = ">=3.6,<4", optional = true } [tool.poetry.extras] developer = [ "datatest", - "pydantic", "pytest", "pytest-cov", "pytest-xdist", - "toml", ] docs = [ "griffe", - "markdown-katex", "mike", "mkdocs", - "mkdocs-jupyter", "mkdocs-material", "mkdocstrings", "mkdocstrings-python", @@ -87,3 +87,22 @@ onecode-check = 'onecode.cli.check:main' [build-system] requires = ["poetry-core>=1.0.0"] build-backend = "poetry.core.masonry.api" + +[tool.ruff] +line-length = 100 +target-version = "py311" +src = ["."] +extend-exclude = ["tests/data", "site"] + +[tool.ruff.lint] +select = ["E", "F", "W", "I"] +# pycodestyle 2.15 (current flake8) does not emit E721. +ignore = ["E721"] + +[tool.ruff.lint.isort] +known-first-party = ["onecode", "tests"] + +[tool.ruff.lint.per-file-ignores] +"**/__init__.py" = ["E", "F", "W"] +"onecode/pycg/**" = ["E501"] +"tests/unit/pycg/**" = ["E501"] diff --git a/scripts/cra/__init__.py b/scripts/cra/__init__.py new file mode 100644 index 0000000..129d48e --- /dev/null +++ b/scripts/cra/__init__.py @@ -0,0 +1 @@ +"""CRA SBOM generation and vulnerability monitoring.""" diff --git a/scripts/cra/backfill_sboms.py b/scripts/cra/backfill_sboms.py new file mode 100644 index 0000000..d924c8e --- /dev/null +++ b/scripts/cra/backfill_sboms.py @@ -0,0 +1,68 @@ +#!/usr/bin/env python3 +"""Attach CycloneDX SBOMs to published GitHub releases that do not have one. + +Only existing releases are updated. Tags without a GitHub release are left alone. +""" + +from __future__ import annotations + +import subprocess +import sys +import tempfile +from pathlib import Path + +_SCRIPTS = Path(__file__).resolve().parent.parent +if str(_SCRIPTS) not in sys.path: + sys.path.insert(0, str(_SCRIPTS)) + +from cra.make_release_sbom import build # noqa: E402 +from cra.monitor_releases import REPO, _gh, list_releases, release_has_sbom # noqa: E402 + + +def pyproject_at_tag(tag: str, dest: Path) -> None: + result = subprocess.run( + ["git", "show", f"{tag}:pyproject.toml"], + check=True, + capture_output=True, + ) + dest.write_bytes(result.stdout) + + +def backfill() -> list[str]: + """Upload a missing SBOM for each published release. Return tags that failed.""" + failed: list[str] = [] + for release in list_releases(): + tag = release["tag_name"] + if release_has_sbom(release): + print(f"{tag}: SBOM already attached") + continue + try: + with tempfile.TemporaryDirectory() as tmp: + folder = Path(tmp) + pyproject = folder / "pyproject.toml" + pyproject_at_tag(tag, pyproject) + path = build(pyproject, folder) + _gh( + "release", + "upload", + tag, + str(path), + "--clobber", + "--repo", + REPO, + ) + print(f"{tag}: uploaded {path.name}") + except Exception as exc: # noqa: BLE001 — keep going so one old tag does not stop the rest + failed.append(f"{tag}: {exc}") + print(f"{tag}: failed: {exc}", file=sys.stderr) + return failed + + +def main() -> None: + failed = backfill() + if failed: + raise SystemExit(f"{len(failed)} release(s) could not be updated") + + +if __name__ == "__main__": + main() diff --git a/scripts/cra/cyclonedx.py b/scripts/cra/cyclonedx.py new file mode 100644 index 0000000..9a19b6d --- /dev/null +++ b/scripts/cra/cyclonedx.py @@ -0,0 +1,53 @@ +"""Minimal CycloneDX 1.6 JSON documents.""" + +from __future__ import annotations + +import json +import uuid +from datetime import datetime, timezone +from pathlib import Path +from typing import Any + +SPEC_VERSION = "1.6" +TOOL_NAME = "deeplime-cra-sbom" +TOOL_VERSION = "1" + + +def new_bom( + root: dict[str, Any], + components: list[dict[str, Any]], + dependencies: list[dict[str, Any]], + extra_tools: list[dict[str, Any]] | None = None, +) -> dict[str, Any]: + tools = [ + { + "type": "application", + "name": TOOL_NAME, + "version": TOOL_VERSION, + }, + *(extra_tools or []), + ] + return { + "$schema": f"http://cyclonedx.org/schema/bom-{SPEC_VERSION}.schema.json", + "bomFormat": "CycloneDX", + "specVersion": SPEC_VERSION, + "serialNumber": f"urn:uuid:{uuid.uuid4()}", + "version": 1, + "metadata": { + "timestamp": datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"), + "tools": {"components": tools}, + "component": root, + }, + "components": components, + "dependencies": dependencies, + } + + +def write_bom(document: dict[str, Any], path: Path) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(document, indent=2) + "\n") + + +def add_property(component: dict[str, Any], name: str, value: str) -> None: + properties = component.setdefault("properties", []) + properties.append({"name": name, "value": value}) diff --git a/scripts/cra/findings.py b/scripts/cra/findings.py new file mode 100644 index 0000000..92a7e5d --- /dev/null +++ b/scripts/cra/findings.py @@ -0,0 +1,428 @@ +"""Match dependency constraints and exact versions to OSV advisories.""" + +from __future__ import annotations + +import json +import math +import re +import urllib.request +from dataclasses import dataclass +from typing import Any, Callable + +from packaging.specifiers import InvalidSpecifier, SpecifierSet +from packaging.version import InvalidVersion, Version + +OSV_QUERY = "https://api.osv.dev/v1/query" +OSV_QUERYBATCH = "https://api.osv.dev/v1/querybatch" +OSV_VULN = "https://api.osv.dev/v1/vulns" +SEVERITY_RANK = {"critical": 0, "high": 1, "medium": 2, "moderate": 2, "low": 3, "unknown": 4} +_CVSS_V3_AV = {"N": 0.85, "A": 0.62, "L": 0.55, "P": 0.2} +_CVSS_V3_AC = {"L": 0.77, "H": 0.44} +_CVSS_V3_UI = {"N": 0.85, "R": 0.62} +_CVSS_V3_CIA = {"H": 0.56, "L": 0.22, "N": 0.0} +_CVSS_V3_PR = { + "U": {"N": 0.85, "L": 0.62, "H": 0.27}, + "C": {"N": 0.85, "L": 0.68, "H": 0.5}, +} + + +@dataclass(frozen=True) +class Finding: + vuln_id: str + package: str + version: str + summary: str + severity: str + known_exploited: bool + ecosystem: str + + def key(self) -> tuple[str, str, str]: + return (self.vuln_id, self.package, self.version) + + +def is_constraint(version: str) -> bool: + return bool(re.search(r"[<>=!~^*]|\s|,", version)) + + +def _normalize(constraint: str) -> str: + return ",".join("".join(part.split()) for part in constraint.split(",") if part.strip()) + + +def _bounds(constraint: str) -> tuple[Version | None, bool, Version | None, bool] | None: + """Return an inclusive/exclusive interval, or None when the constraint matches nothing.""" + normalized = _normalize(constraint) + if not normalized or normalized == "*": + return None, True, None, False + try: + spec = SpecifierSet(normalized) + except (InvalidSpecifier, InvalidVersion): + return None + equals = [Version(item.version) for item in spec if item.operator == "=="] + if equals: + point = equals[0] + rest = ",".join(str(item) for item in spec if item.operator != "==") + if rest and not SpecifierSet(rest).contains(str(point), prereleases=True): + return None + return point, True, point, True + + lower: Version | None = None + lower_inclusive = True + upper: Version | None = None + upper_inclusive = False + for item in spec: + if item.operator == "!=": + continue + version = Version(item.version) + if item.operator == ">=": + if lower is None or version > lower: + lower, lower_inclusive = version, True + elif item.operator == ">": + if lower is None or version > lower or (version == lower and lower_inclusive): + lower, lower_inclusive = version, False + elif item.operator == "<": + if upper is None or version < upper: + upper, upper_inclusive = version, False + elif item.operator == "<=": + if upper is None or version < upper or (version == upper and not upper_inclusive): + upper, upper_inclusive = version, True + elif item.operator == "~=": + if lower is None or version > lower: + lower, lower_inclusive = version, True + release = version.release + if len(release) >= 2: + prefix = list(release[:-1]) + prefix[-1] += 1 + cap = Version(".".join(str(part) for part in prefix)) + if upper is None or cap < upper: + upper, upper_inclusive = cap, False + if lower is not None and upper is not None and ( + lower > upper or (lower == upper and not (lower_inclusive and upper_inclusive)) + ): + return None + return lower, lower_inclusive, upper, upper_inclusive + + +def _parse_version(raw: str) -> Version | None: + if raw in {"0", "0.0.0"}: + return Version("0") + try: + return Version(raw) + except InvalidVersion: + return None + + +def _ends_before( + end: Version | None, + end_inclusive: bool, + start: Version | None, + start_inclusive: bool, +) -> bool: + if end is None or start is None: + return False + if end < start: + return True + if end > start: + return False + return not (end_inclusive and start_inclusive) + + +def _intervals_overlap( + left: tuple[Version | None, bool, Version | None, bool], + right: tuple[Version | None, bool, Version | None, bool], +) -> bool: + left_start, left_start_inc, left_end, left_end_inc = left + right_start, right_start_inc, right_end, right_end_inc = right + if _ends_before(left_end, left_end_inc, right_start, right_start_inc): + return False + if _ends_before(right_end, right_end_inc, left_start, left_start_inc): + return False + return True + + +def _osv_intervals( + ranges: list[dict[str, Any]], +) -> list[tuple[Version | None, bool, Version | None, bool]]: + intervals = [] + for item in ranges: + if item.get("type") == "GIT": + continue + start: Version | None = None + started = False + for event in item.get("events") or []: + if "introduced" in event: + start = _parse_version(str(event["introduced"])) + started = start is not None or str(event["introduced"]) in {"0", "0.0.0"} + if str(event["introduced"]) in {"0", "0.0.0"}: + start = Version("0") + elif started and "fixed" in event: + fixed = _parse_version(str(event["fixed"])) + if fixed is not None: + intervals.append((start, True, fixed, False)) + started = False + elif started and "last_affected" in event: + last = _parse_version(str(event["last_affected"])) + if last is not None: + intervals.append((start, True, last, True)) + started = False + if started: + intervals.append((start, True, None, False)) + return intervals + + +def _affected_package(entry: dict[str, Any], package: str, ecosystem: str) -> bool: + info = entry.get("package") or {} + if ecosystem and info.get("ecosystem") and info["ecosystem"].lower() != ecosystem.lower(): + return False + return str(info.get("name", "")).lower().replace("_", "-") == package.lower().replace("_", "-") + + +def constraint_overlaps( + constraint: str, + vuln: dict[str, Any], + package: str, + ecosystem: str, +) -> bool: + open_ended = constraint.strip() in {"", "*"} + bounds = _bounds(constraint) + if bounds is None and not open_ended: + return False + interval = bounds if bounds is not None else (None, True, None, False) + for entry in vuln.get("affected") or []: + if not _affected_package(entry, package, ecosystem): + continue + versions = entry.get("versions") or [] + if versions and bounds is not None: + try: + spec = SpecifierSet(_normalize(constraint)) + except (InvalidSpecifier, InvalidVersion): + spec = None + for version in versions if spec is not None else []: + try: + if spec.contains(str(version), prereleases=True): + return True + except InvalidVersion: + continue + ranges = entry.get("ranges") or [] + if not ranges and not versions: + return True + for osv_interval in _osv_intervals(ranges): + if _intervals_overlap(interval, osv_interval): + return True + return False + + +def known_exploited(vuln: dict[str, Any]) -> bool: + specific = vuln.get("database_specific") or {} + if isinstance(specific, dict) and ( + specific.get("known_exploited") or specific.get("cisa_known_exploited") + ): + return True + for ref in vuln.get("references") or []: + url = str(ref.get("url") or "").lower() + if "known_exploited" in url or ("cisa.gov" in url and "kev" in url): + return True + return False + + +def advisory_url(vuln_id: str) -> str: + if vuln_id.startswith("GHSA-"): + return f"https://github.com/advisories/{vuln_id}" + if vuln_id.startswith("CVE-"): + return f"https://nvd.nist.gov/vuln/detail/{vuln_id}" + return f"https://osv.dev/vulnerability/{vuln_id}" + + +def advisory_link(vuln_id: str) -> str: + return f"[{vuln_id}]({advisory_url(vuln_id)})" + + +def _cvss_roundup(score: float) -> float: + """CVSS v3 Roundup: smallest one-decimal value greater than or equal to score.""" + scaled = round(score * 100000) + if scaled % 10000 == 0: + return scaled / 100000.0 + return (math.floor(scaled / 10000) + 1) / 10.0 + + +def _cvss_v3_severity(vector: str) -> str | None: + if not vector.startswith(("CVSS:3.0/", "CVSS:3.1/")): + return None + metrics = {} + for part in vector.split("/")[1:]: + if ":" not in part: + continue + key, value = part.split(":", 1) + metrics[key] = value + try: + scope = metrics["S"] + iss = 1 - ( + (1 - _CVSS_V3_CIA[metrics["C"]]) + * (1 - _CVSS_V3_CIA[metrics["I"]]) + * (1 - _CVSS_V3_CIA[metrics["A"]]) + ) + exploitability = ( + 8.22 + * _CVSS_V3_AV[metrics["AV"]] + * _CVSS_V3_AC[metrics["AC"]] + * _CVSS_V3_PR[scope][metrics["PR"]] + * _CVSS_V3_UI[metrics["UI"]] + ) + except KeyError: + return None + if scope == "U": + impact = 6.42 * iss + raw = impact + exploitability if impact > 0 else 0 + elif scope == "C": + impact = 7.52 * (iss - 0.029) - 3.25 * (iss - 0.02) ** 15 + raw = 1.08 * (impact + exploitability) if impact > 0 else 0 + else: + return None + score = _cvss_roundup(min(raw, 10)) + if score >= 9.0: + return "critical" + if score >= 7.0: + return "high" + if score >= 4.0: + return "moderate" + if score > 0: + return "low" + return "low" + + +def osv_severity(vuln: dict[str, Any]) -> str: + specific = vuln.get("database_specific") or {} + if isinstance(specific, dict) and specific.get("severity"): + return str(specific["severity"]).lower() + for item in vuln.get("severity") or []: + if not isinstance(item, dict): + continue + mapped = _cvss_v3_severity(str(item.get("score") or "")) + if mapped: + return mapped + return "unknown" + + +def _fetch_vuln(vuln_id: str) -> dict[str, Any] | None: + request = urllib.request.Request( + f"{OSV_VULN}/{vuln_id}", + headers={"User-Agent": "onecode-cra"}, + ) + try: + with urllib.request.urlopen(request, timeout=30) as response: + payload = json.load(response) + except (OSError, json.JSONDecodeError, TimeoutError, ValueError): + return None + return payload if isinstance(payload, dict) else None + + +def _severity_from_aliases(vuln: dict[str, Any], known: dict[str, str]) -> str: + for alias in vuln.get("aliases") or []: + alias_id = str(alias) + inherited = known.get(alias_id, "unknown") + if inherited != "unknown": + return inherited + fetched = _fetch_vuln(alias_id) + if fetched is None: + continue + inherited = osv_severity(fetched) + if inherited == "unknown": + continue + known[alias_id] = inherited + return inherited + return "unknown" + + +def _post(url: str, payload: dict[str, Any]) -> dict[str, Any]: + request = urllib.request.Request( + url, + data=json.dumps(payload).encode(), + headers={"Content-Type": "application/json"}, + ) + with urllib.request.urlopen(request, timeout=60) as response: + return json.load(response) + + +def query_package(name: str, ecosystem: str) -> dict[str, Any]: + return _post(OSV_QUERY, {"package": {"name": name, "ecosystem": ecosystem}}) + + +def query_exact(name: str, ecosystem: str, version: str) -> dict[str, Any]: + return _post( + OSV_QUERY, + {"package": {"name": name, "ecosystem": ecosystem}, "version": version}, + ) + + +def query_batch(queries: list[dict[str, Any]]) -> list[dict[str, Any]]: + results: list[dict[str, Any]] = [] + for start in range(0, len(queries), 500): + chunk = queries[start : start + 500] + payload = _post(OSV_QUERYBATCH, {"queries": chunk}) + results.extend(payload.get("results") or []) + return results + + +def findings_for_vulns( + package: str, + version: str, + ecosystem: str, + vulns: list[dict[str, Any]], + overlaps: Callable[[dict[str, Any]], bool] | None = None, +) -> list[Finding]: + found: list[Finding] = [] + seen: set[tuple[str, str, str]] = set() + known_severity = { + str(vuln.get("id")): osv_severity(vuln) + for vuln in vulns + if vuln.get("id") + } + for vuln in vulns: + if overlaps is not None and not overlaps(vuln): + continue + vuln_id = str(vuln.get("id")) + severity = known_severity.get(vuln_id) or osv_severity(vuln) + if severity == "unknown": + severity = _severity_from_aliases(vuln, known_severity) + known_severity[vuln_id] = severity + finding = Finding( + vuln_id=vuln_id, + package=package, + version=version, + summary=str(vuln.get("summary") or vuln.get("details") or "").replace("\n", " "), + severity=severity, + known_exploited=known_exploited(vuln), + ecosystem=ecosystem, + ) + if finding.key() not in seen: + seen.add(finding.key()) + found.append(finding) + return found + + +def match_constraint( + package: str, + constraint: str, + ecosystem: str, + query: dict[str, Any], +) -> list[Finding]: + vulns = query.get("vulns") or [] + return findings_for_vulns( + package, + constraint, + ecosystem, + vulns, + overlaps=lambda vuln: constraint_overlaps(constraint, vuln, package, ecosystem), + ) + + +def sort_findings(findings: list[Finding]) -> list[Finding]: + return sorted( + findings, + key=lambda item: ( + not item.known_exploited, + SEVERITY_RANK.get(item.severity.lower(), 4), + item.vuln_id, + item.package, + item.version, + ), + ) diff --git a/scripts/cra/make_release_sbom.py b/scripts/cra/make_release_sbom.py new file mode 100644 index 0000000..e0f35a0 --- /dev/null +++ b/scripts/cra/make_release_sbom.py @@ -0,0 +1,43 @@ +#!/usr/bin/env python3 +"""Build a CycloneDX 1.6 SBOM for an OneCode release. + +Python dependencies keep the version ranges declared in pyproject.toml. +""" + +from __future__ import annotations + +import argparse +import sys +from pathlib import Path + +_SCRIPTS = Path(__file__).resolve().parent.parent +if str(_SCRIPTS) not in sys.path: + sys.path.insert(0, str(_SCRIPTS)) + +from cra.cyclonedx import write_bom # noqa: E402 +from cra.python_deps import python_sbom, sbom_filename # noqa: E402 + + +def build(pyproject: Path, out_dir: Path) -> Path: + document = python_sbom(pyproject) + version = document["metadata"]["component"]["version"] + out_dir.mkdir(parents=True, exist_ok=True) + path = out_dir / sbom_filename(version) + write_bom(document, path) + return path + + +def main() -> None: + if sys.version_info < (3, 11): + raise SystemExit("make_release_sbom.py requires Python 3.11 or newer") + parser = argparse.ArgumentParser( + description="Write a CycloneDX 1.6 SBOM for an OneCode release", + ) + parser.add_argument("--pyproject", default="pyproject.toml") + parser.add_argument("--out-dir", required=True) + args = parser.parse_args() + print(build(Path(args.pyproject), Path(args.out_dir))) + + +if __name__ == "__main__": + main() diff --git a/scripts/cra/monitor_releases.py b/scripts/cra/monitor_releases.py new file mode 100644 index 0000000..819f325 --- /dev/null +++ b/scripts/cra/monitor_releases.py @@ -0,0 +1,323 @@ +#!/usr/bin/env python3 +"""Rescan release SBOMs for newly published advisories and update one issue per release.""" + +from __future__ import annotations + +import hashlib +import json +import os +import re +import subprocess +import sys +import tempfile +from pathlib import Path +from typing import Any + +_SCRIPTS = Path(__file__).resolve().parent.parent +if str(_SCRIPTS) not in sys.path: + sys.path.insert(0, str(_SCRIPTS)) + +from cra.findings import ( # noqa: E402 + Finding, + advisory_link, + findings_for_vulns, + is_constraint, + match_constraint, + query_exact, + query_package, + sort_findings, +) +from cra.python_deps import is_sbom_asset, pypi_name # noqa: E402 + +REPO = "deeplime-io/onecode" +LABEL = "cra-vulnerability" +FINGERPRINT_RE = re.compile(r"") + + +def _prop(component: dict[str, Any], name: str) -> str | None: + for item in component.get("properties") or []: + if item.get("name") == name: + return str(item.get("value")) + return None + + +def fingerprint(findings: list[Finding]) -> str: + encoded = [list(item.key()) for item in sort_findings(findings)] + payload = json.dumps(encoded, separators=(",", ":")) + return hashlib.sha256(payload.encode()).hexdigest() + + +def notification_body(tag: str, findings: list[Finding], logins: list[str]) -> str: + mentions = " ".join(f"@{login}" for login in logins) + exploited = sum(1 for item in findings if item.known_exploited) + return ( + f"{mentions} {len(findings)} advisories match release `{tag}` " + f"({exploited} known-exploited).\n" + ) + + +def render_run_summary(scanned: list[tuple[str, list[Finding]]]) -> str: + affected = [(tag, findings) for tag, findings in scanned if findings] + lines = [ + "## CRA vulnerability scan", + "", + f"Scanned {len(scanned)} releases. {len(affected)} " + f"{'has' if len(affected) == 1 else 'have'} matching advisories.", + "", + "| Release | Advisories | Known exploited |", + "| --- | ---: | ---: |", + ] + if not affected: + lines.append("| none | 0 | 0 |") + for tag, findings in affected: + exploited = sum(1 for item in findings if item.known_exploited) + lines.append(f"| {tag} | {len(findings)} | {exploited} |") + lines.append("") + return "\n".join(lines) + "\n" + + +def publish_run_summary(text: str) -> None: + print(text, end="") + path = os.environ.get("GITHUB_STEP_SUMMARY") + if not path: + return + with open(path, "a", encoding="utf-8") as handle: + handle.write(text) + + +def _notify_logins() -> list[str]: + logins: list[str] = [] + for item in os.environ.get("CRA_NOTIFY", "").split(","): + login = item.strip().lstrip("@") + if re.fullmatch(r"[A-Za-z0-9-]+", login): + logins.append(login) + return logins + + +def render_body(findings: list[Finding]) -> str: + ordered = sort_findings(findings) + marker = f"" + if not ordered: + return f"{marker}\n\nNo advisories currently match this release SBOM.\n" + exploited = sum(1 for item in ordered if item.known_exploited) + lines = [ + marker, + "", + f"Known-exploited advisories affecting this release: {exploited}", + "", + "A match means the advisory's affected range intersects the SBOM entry.", + "It is not an exploitability decision.", + "", + "| Advisory | Package | Version or constraint | Severity | Known exploited | Ecosystem |", + "| --- | --- | --- | --- | --- | --- |", + ] + for item in ordered: + summary = item.summary[:120].replace("|", "/") + lines.append( + f"| {advisory_link(item.vuln_id)} | {item.package} | `{item.version}` | " + f"{item.severity} | {'yes' if item.known_exploited else 'no'} | {item.ecosystem} |", + ) + if summary: + lines.append(f"| | | {summary} | | | |") + lines.append("") + body = "\n".join(lines) + "\n" + if len(body) > 60000: + body = body[:60000] + "\n\nTruncated.\n" + return body + + +def _gh(*args: str, check: bool = True) -> subprocess.CompletedProcess[str]: + return subprocess.run(["gh", *args], check=check, capture_output=True, text=True) + + +def list_releases() -> list[dict[str, Any]]: + result = _gh("api", f"repos/{REPO}/releases?per_page=100") + releases = json.loads(result.stdout) + return [release for release in releases if not release.get("draft")] + + +def release_has_sbom(release: dict[str, Any]) -> bool: + return any(is_sbom_asset(asset.get("name") or "") for asset in release.get("assets") or []) + + +def _python_findings(component: dict[str, Any]) -> list[Finding]: + if component.get("name") == "python": + return [] + constraint = _prop(component, "deeplime:version-constraint") or str( + component.get("version") or "", + ) + package = pypi_name(str(component.get("name") or "")) + if is_constraint(constraint): + return match_constraint(package, constraint, "PyPI", query_package(package, "PyPI")) + payload = query_exact(package, "PyPI", constraint) + return findings_for_vulns(package, constraint, "PyPI", payload.get("vulns") or []) + + +def scan_product(document: dict[str, Any]) -> list[Finding]: + components = list(document.get("components") or []) + root = (document.get("metadata") or {}).get("component") + if root: + components.append(root) + findings: list[Finding] = [] + for component in components: + if _prop(component, "deeplime:ecosystem") != "python": + continue + findings.extend(_python_findings(component)) + return sort_findings(findings) + + +def _ensure_label() -> None: + _gh( + "label", + "create", + LABEL, + "--repo", + REPO, + "--color", + "B60205", + "--description", + "CRA vulnerability monitor", + check=False, + ) + + +def _issues() -> list[dict[str, Any]]: + result = _gh( + "issue", + "list", + "--repo", + REPO, + "--state", + "all", + "--limit", + "200", + "--label", + LABEL, + "--json", + "number,title,state,body", + ) + return json.loads(result.stdout or "[]") + + +def _issue_with_title(issues: list[dict[str, Any]], title: str) -> dict[str, Any] | None: + for issue in issues: + if issue.get("title") == title: + return issue + return None + + +def _write_issue( + title: str, + body: str, + existing: dict[str, Any] | None, + reopen: bool, + close: bool, +) -> str: + with tempfile.NamedTemporaryFile("w", suffix=".md", delete=False) as handle: + handle.write(body) + body_path = handle.name + try: + if existing is None: + created = _gh( + "issue", + "create", + "--repo", + REPO, + "--title", + title, + "--label", + LABEL, + "--body-file", + body_path, + ) + match = re.search(r"/issues/(\d+)", created.stdout) + if match is None: + raise RuntimeError(f"could not read the new issue number from {created.stdout!r}") + return match.group(1) + number = str(existing["number"]) + _gh("issue", "edit", number, "--repo", REPO, "--body-file", body_path) + state = str(existing.get("state") or "").upper() + if reopen and state == "CLOSED": + _gh("issue", "reopen", number, "--repo", REPO) + if close and state == "OPEN": + _gh("issue", "close", number, "--repo", REPO, "--reason", "completed") + return number + finally: + Path(body_path).unlink(missing_ok=True) + + +def _comment(number: str, body: str) -> None: + with tempfile.NamedTemporaryFile("w", suffix=".md", delete=False) as handle: + handle.write(body) + body_path = handle.name + try: + _gh("issue", "comment", number, "--repo", REPO, "--body-file", body_path) + finally: + Path(body_path).unlink(missing_ok=True) + + +def _download(tag: str, name: str, dest: Path) -> Path: + dest.mkdir(parents=True, exist_ok=True) + subprocess.run( + [ + "gh", + "release", + "download", + tag, + "--repo", + REPO, + "--pattern", + name, + "--dir", + str(dest), + "--clobber", + ], + check=True, + ) + return dest / name + + +def sync_release(release: dict[str, Any], issues: list[dict[str, Any]]) -> list[Finding]: + tag = release["tag_name"] + names = [ + asset["name"] + for asset in release.get("assets") or [] + if is_sbom_asset(asset.get("name") or "") + ] + if not names: + return [] + findings: list[Finding] = [] + with tempfile.TemporaryDirectory() as tmp: + folder = Path(tmp) + for name in names: + path = _download(tag, name, folder) + findings.extend(scan_product(json.loads(path.read_text()))) + findings = sort_findings(findings) + title = f"CRA vulnerabilities in {tag}" + body = render_body(findings) + marker = fingerprint(findings) + existing = _issue_with_title(issues, title) + match = FINGERPRINT_RE.search((existing or {}).get("body") or "") if existing else None + if match and match.group(1) == marker: + return findings + if not findings and existing is None: + return findings + number = _write_issue(title, body, existing, reopen=bool(findings), close=not findings) + logins = _notify_logins() + if findings and logins: + _comment(number, notification_body(tag, findings, logins)) + return findings + + +def main() -> None: + _ensure_label() + issues = _issues() + scanned: list[tuple[str, list[Finding]]] = [] + for release in list_releases(): + scanned.append((release["tag_name"], sync_release(release, issues))) + issues = _issues() + publish_run_summary(render_run_summary(scanned)) + + +if __name__ == "__main__": + main() diff --git a/scripts/cra/python_deps.py b/scripts/cra/python_deps.py new file mode 100644 index 0000000..49a14bf --- /dev/null +++ b/scripts/cra/python_deps.py @@ -0,0 +1,85 @@ +"""Python SBOM from Poetry ranges in pyproject.toml. + +Direct runtime ranges only. Optional docs and test extras are omitted. +""" + +from __future__ import annotations + +import tomllib +from pathlib import Path +from typing import Any + +from cra.cyclonedx import add_property, new_bom + +DECLARED_SCOPE = "installed-alongside-wheel" + + +def pypi_name(name: str) -> str: + return name.lower().replace("_", "-") + + +def sbom_filename(version: str) -> str: + return f"onecode-{version}.cdx.json" + + +def is_sbom_asset(name: str) -> bool: + return name.startswith("onecode-") and name.endswith(".cdx.json") + + +def runtime_dependencies(pyproject: Path) -> tuple[str, str, list[tuple[str, str]]]: + """Return package name, version, and direct runtime (name, constraint) pairs.""" + data = tomllib.loads(pyproject.read_text()) + poetry = data["tool"]["poetry"] + dependencies: list[tuple[str, str]] = [] + for name, spec in poetry["dependencies"].items(): + if isinstance(spec, str): + dependencies.append((name, spec)) + continue + if isinstance(spec, dict) and not spec.get("optional") and "version" in spec: + dependencies.append((name, str(spec["version"]))) + return poetry["name"], poetry["version"], dependencies + + +def python_sbom(pyproject: Path) -> dict[str, Any]: + name, version, dependencies = runtime_dependencies(pyproject) + root_ref = f"pkg:pypi/{pypi_name(name)}@{version}" + root: dict[str, Any] = { + "bom-ref": root_ref, + "type": "library", + "name": name, + "version": version, + "purl": root_ref, + "supplier": {"name": "DeepLime"}, + "licenses": [{"license": {"id": "MIT"}}], + } + components: list[dict[str, Any]] = [] + depends_on: list[str] = [] + for dep_name, constraint in dependencies: + if dep_name == "python": + ref = "pkg:generic/python" + component: dict[str, Any] = { + "bom-ref": ref, + "type": "platform", + "name": "python", + "version": constraint, + "purl": ref, + } + else: + ref = f"pkg:pypi/{pypi_name(dep_name)}" + component = { + "bom-ref": ref, + "type": "library", + "name": dep_name, + "version": constraint, + "purl": ref, + } + add_property(component, "deeplime:ecosystem", "python") + add_property(component, "deeplime:install-scope", DECLARED_SCOPE) + add_property(component, "deeplime:version-constraint", constraint) + components.append(component) + depends_on.append(ref) + return new_bom( + root, + components, + [{"ref": root_ref, "dependsOn": depends_on}], + ) diff --git a/scripts/cra/scan_ranges.py b/scripts/cra/scan_ranges.py new file mode 100644 index 0000000..0bf0675 --- /dev/null +++ b/scripts/cra/scan_ranges.py @@ -0,0 +1,71 @@ +#!/usr/bin/env python3 +"""Compare direct pyproject.toml ranges with OSV. Never fails the pull request for findings.""" + +from __future__ import annotations + +import os +import sys +from pathlib import Path + +_SCRIPTS = Path(__file__).resolve().parent.parent +if str(_SCRIPTS) not in sys.path: + sys.path.insert(0, str(_SCRIPTS)) + +from cra.findings import advisory_link, match_constraint, query_package, sort_findings # noqa: E402 +from cra.python_deps import pypi_name, runtime_dependencies # noqa: E402 + + +def render(findings) -> str: + if not findings: + return "No OSV advisories intersect the direct dependency ranges in pyproject.toml.\n" + lines = [ + "## Dependency range advisories", + "", + "These are presence matches against declared ranges, not exploitability.", + "", + "| Advisory | Package | Constraint | Severity | Known exploited |", + "| --- | --- | --- | --- | --- |", + ] + for finding in findings: + lines.append( + f"| {advisory_link(finding.vuln_id)} | {finding.package} | `{finding.version}` | " + f"{finding.severity} | " + f"{'yes' if finding.known_exploited else 'no'} |", + ) + lines.append("") + return "\n".join(lines) + "\n" + + +def scan(pyproject: Path) -> str: + _name, _version, dependencies = runtime_dependencies(pyproject) + findings = [] + for dep_name, constraint in dependencies: + if dep_name == "python": + continue + package = pypi_name(dep_name) + payload = query_package(package, "PyPI") + findings.extend(match_constraint(package, constraint, "PyPI", payload)) + return render(sort_findings(findings)) + + +def main() -> None: + pyproject = Path(sys.argv[1]) if len(sys.argv) > 1 else Path("pyproject.toml") + try: + report = scan(pyproject) + except Exception as exc: # noqa: BLE001 — a scanner bug should be visible, findings must not fail CI + report = f"Dependency range scan failed: {exc}\n" + print(report, file=sys.stderr) + summary = os.environ.get("GITHUB_STEP_SUMMARY") + if summary: + with open(summary, "a", encoding="utf-8") as handle: + handle.write(report) + raise + print(report) + summary = os.environ.get("GITHUB_STEP_SUMMARY") + if summary: + with open(summary, "a", encoding="utf-8") as handle: + handle.write(report) + + +if __name__ == "__main__": + main() diff --git a/tests/conftest.py b/tests/conftest.py index faf619b..fbcd3fd 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -15,23 +15,3 @@ def clear_project(): logger = logging.getLogger(Env.ONECODE_LOGGER_NAME) logger.propagate = True Project().mode = Mode.EXECUTE - - -def pytest_addoption(parser): - parser.addoption( - "--with-emulations", action="store_true", default=False, help="run emulations tests" - ) - - -def pytest_configure(config): - config.addinivalue_line("markers", "emulations: mark test as emulations needed to run") - - -def pytest_collection_modifyitems(config, items): - if config.getoption("--with-emulations"): - return - - skip_emulation = pytest.mark.skip(reason="need --with-emulations option to run") - for item in items: - if "emulations" in item.keywords: - item.add_marker(skip_emulation) diff --git a/tests/emulations/dev/no_test_run_extend_input_element.py b/tests/emulations/dev/no_test_run_extend_input_element.py deleted file mode 100644 index 4bfc0f0..0000000 --- a/tests/emulations/dev/no_test_run_extend_input_element.py +++ /dev/null @@ -1,69 +0,0 @@ -import os -import shutil - -import pytest - -from onecode import Env -from onecode.cli.create import create -from tests.utils.flow_cli import _clean_flow, _generate_flow_name - - -@pytest.mark.emulations -def test_execute_single_count(): - flow_name, flow_folder, flow_id = _generate_flow_name() - - tmp = _clean_flow(flow_folder) - create(tmp, flow_name, cli=False) - - flow_dir = os.path.join(tmp, flow_folder) - - with open( - os.path.join(flow_dir, 'flows', 'onecode_ext', 'input_elements', 'my_input_element.py'), - 'w' - ) as f: - f.write(""" -from typing import Any, Union - -from onecode import InputElement - - -class MyInputElement(InputElement): - def __init__( - self, - key: str, - value: Union[float, int] - ): - super().__init__(key, value) - - @property - def value(self) -> Union[float, int]: - return 3 * self._value + 5.5 - - @property - def _value_type(self) -> type: - return Union[float, int] - - def _validate( - self, - value: Any - ) -> None: - pass - -""") - - with open(os.path.join(flow_dir, 'flows', f'{flow_id}.py'), 'w') as f: - f.write(""" -from onecode_ext import * - -def run(): - x = my_input_element('My Element', 14.3) - with open("stdout.txt", 'w') as f: - f.write(str(x)) - """) - - os.system(f'cd "{flow_dir}" && {Env.ONECODE_DO_TYPECHECK}=1 python main.py') - - with open(os.path.join(flow_dir, 'stdout.txt')) as f: - assert f.read() == f"{3 * 14.3 + 5.5 }" - - shutil.rmtree(flow_dir) diff --git a/tests/emulations/dev/no_test_run_extend_output_element.py b/tests/emulations/dev/no_test_run_extend_output_element.py deleted file mode 100644 index ddf2819..0000000 --- a/tests/emulations/dev/no_test_run_extend_output_element.py +++ /dev/null @@ -1,77 +0,0 @@ -import json -import os -import shutil - -import pytest - -from onecode import Env -from onecode.cli.create import create -from tests.utils.flow_cli import _clean_flow, _generate_flow_name - - -@pytest.mark.emulations -def test_execute_single_count(): - flow_name, flow_folder, flow_id = _generate_flow_name() - - tmp = _clean_flow(flow_folder) - create(tmp, flow_name, cli=False) - - flow_dir = os.path.join(tmp, flow_folder) - - with open( - os.path.join(flow_dir, 'flows', 'onecode_ext', 'input_elements', 'my_output_element.py'), - 'w' - ) as f: - f.write(""" -from typing import Any, Union - -from onecode import OutputElement - - -class MyOutputElement(OutputElement): - def __init__( - self, - key: str, - value: Union[float, int], - label: str, - extra: int - ): - super().__init__(key, value, label, extra=extra) - - @property - def value(self) -> Union[float, int]: - return 3 * self._value + 5.5 - - def _validate( - self, - value: Any - ) -> None: - pass - -""") - - with open(os.path.join(flow_dir, 'flows', f'{flow_id}.py'), 'w') as f: - f.write(""" -from onecode_ext import * - -def run(): - x = my_output_element('My Element', 14.3, None, 56) - with open("stdout.txt", 'w') as f: - f.write(str(x)) - """) - - os.system(f'cd "{flow_dir}" && {Env.ONECODE_DO_TYPECHECK}=1 python main.py') - - with open(os.path.join(flow_dir, 'stdout.txt')) as f: - assert f.read() == f"{3 * 14.3 + 5.5 }" - - with open(os.path.join(flow_dir, 'data', 'outputs', flow_id, 'MANIFEST.txt')) as f: - assert json.load(f) == { - "key": "my_element", - "label": "My Element", - "value": 3 * 14.3 + 5.5, - "kind": "MyOutputElement", - "extra": 56 - } - - shutil.rmtree(flow_dir) diff --git a/tests/emulations/elements/test_run_checkbox.py b/tests/emulations/elements/test_run_checkbox.py deleted file mode 100644 index 3ddaab8..0000000 --- a/tests/emulations/elements/test_run_checkbox.py +++ /dev/null @@ -1,249 +0,0 @@ -import json -import os -import shutil - -import pytest - -from onecode import Env -from onecode.cli.create import create -from tests.utils.flow_cli import _clean_flow, _generate_flow_name - - -@pytest.mark.emulations -def test_execute_single_count(): - flow_name, flow_folder, flow_id = _generate_flow_name() - - tmp = _clean_flow(flow_folder) - create(tmp, flow_name, cli=False) - - flow_dir = os.path.join(tmp, flow_folder) - - with open(os.path.join(flow_dir, 'flows', f'{flow_id}.py'), 'a') as f: - f.write(""" - x = onecode.checkbox('my_checkbox', True) - with open("stdout.txt", 'w') as f: - f.write(str(x)) - """) - - os.system(f'cd "{flow_dir}" && {Env.ONECODE_DO_TYPECHECK}=1 python main.py') - - with open(os.path.join(flow_dir, 'stdout.txt')) as f: - assert f.read() == "True" - - shutil.rmtree(flow_dir) - - -@pytest.mark.emulations -def test_execute_multiple_count(): - flow_name, flow_folder, flow_id = _generate_flow_name() - - tmp = _clean_flow(flow_folder) - create(tmp, flow_name, cli=False) - - flow_dir = os.path.join(tmp, flow_folder) - - # count is ignored at loading & execution as it is only used in UI mode - with open(os.path.join(flow_dir, 'flows', f'{flow_id}.py'), 'a') as f: - f.write(""" - x = onecode.checkbox('my_checkbox', [False, False], count=3) - with open("stdout.txt", 'w') as f: - f.write(str(x)) - """) - - os.system(f'cd "{flow_dir}" && {Env.ONECODE_DO_TYPECHECK}=1 python main.py') - - with open(os.path.join(flow_dir, 'stdout.txt')) as f: - assert f.read() == "[False, False]" - - shutil.rmtree(flow_dir) - - -@pytest.mark.emulations -def test_load_then_execute_single_count(): - flow_name, flow_folder, flow_id = _generate_flow_name() - - tmp = _clean_flow(flow_folder) - create(tmp, flow_name, cli=False) - - flow_dir = os.path.join(tmp, flow_folder) - - with open(os.path.join(flow_dir, 'flows', f'{flow_id}.py'), 'a') as f: - f.write(""" - x = onecode.checkbox('my_checkbox', True) - with open("stdout.txt", 'w') as f: - f.write(str(x)) - """) - - params = os.path.join(flow_dir, 'params.json') - with open(params, 'w') as f: - json.dump({"my_checkbox": False}, f) - - os.system(f'cd "{flow_dir}" && {Env.ONECODE_DO_TYPECHECK}=1 python main.py {params}') - - with open(os.path.join(flow_dir, 'stdout.txt')) as f: - assert f.read() == "False" - - shutil.rmtree(flow_dir) - - -@pytest.mark.emulations -def test_load_then_execute_multiple_count(): - flow_name, flow_folder, flow_id = _generate_flow_name() - - tmp = _clean_flow(flow_folder) - create(tmp, flow_name, cli=False) - - flow_dir = os.path.join(tmp, flow_folder) - - # count is ignored at loading & execution as it is only used in UI mode - with open(os.path.join(flow_dir, 'flows', f'{flow_id}.py'), 'a') as f: - f.write(""" - x = onecode.checkbox('my_checkbox', False, count=4) - with open("stdout.txt", 'w') as f: - f.write(str(x)) - """) - - params = os.path.join(flow_dir, 'params.json') - with open(params, 'w') as f: - json.dump({"my_checkbox": [True, False, True]}, f) - - os.system(f'cd "{flow_dir}" && {Env.ONECODE_DO_TYPECHECK}=1 python main.py {params}') - - with open(os.path.join(flow_dir, 'stdout.txt')) as f: - assert f.read() == "[True, False, True]" - - shutil.rmtree(flow_dir) - - -@pytest.mark.emulations -def test_execute_optional_value(): - flow_name, flow_folder, flow_id = _generate_flow_name() - - tmp = _clean_flow(flow_folder) - create(tmp, flow_name, cli=False) - - flow_dir = os.path.join(tmp, flow_folder) - - with open(os.path.join(flow_dir, 'flows', f'{flow_id}.py'), 'a') as f: - f.write(""" - x = onecode.checkbox('my_checkbox', None, optional=True) - with open("stdout.txt", 'w') as f: - f.write(str(x)) - """) - - os.system(f'cd "{flow_dir}" && {Env.ONECODE_DO_TYPECHECK}=1 python main.py') - - with open(os.path.join(flow_dir, 'stdout.txt')) as f: - assert f.read() == "None" - - shutil.rmtree(flow_dir) - - -@pytest.mark.emulations -def test_load_then_execute_optional_value(): - flow_name, flow_folder, flow_id = _generate_flow_name() - - tmp = _clean_flow(flow_folder) - create(tmp, flow_name, cli=False) - - flow_dir = os.path.join(tmp, flow_folder) - - with open(os.path.join(flow_dir, 'flows', f'{flow_id}.py'), 'a') as f: - f.write(""" - x = onecode.checkbox('my_checkbox', False, optional=True) - with open("stdout.txt", 'w') as f: - f.write(str(x)) - """) - - params = os.path.join(flow_dir, 'params.json') - with open(params, 'w') as f: - json.dump({"my_checkbox": None}, f) - - os.system(f'cd "{flow_dir}" && {Env.ONECODE_DO_TYPECHECK}=1 python main.py {params}') - - with open(os.path.join(flow_dir, 'stdout.txt')) as f: - assert f.read() == "None" - - shutil.rmtree(flow_dir) - - -@pytest.mark.emulations -def test_execute_invalid_optional_value(): - flow_name, flow_folder, flow_id = _generate_flow_name() - - tmp = _clean_flow(flow_folder) - create(tmp, flow_name, cli=False) - - flow_dir = os.path.join(tmp, flow_folder) - - with open(os.path.join(flow_dir, 'flows', f'{flow_id}.py'), 'a') as f: - f.write(""" - try: - x = onecode.checkbox('my_checkbox', None, optional=False) - except ValueError as err: - with open("stdout.txt", 'w') as f: - f.write(str(err)) - """) - - os.system(f'cd "{flow_dir}" && {Env.ONECODE_DO_TYPECHECK}=1 python main.py') - - with open(os.path.join(flow_dir, 'stdout.txt')) as f: - assert f.read() == "[my_checkbox] Value is required: None provided" - - shutil.rmtree(flow_dir) - - -@pytest.mark.emulations -def test_load_then_execute_invalid_optional_value(): - flow_name, flow_folder, flow_id = _generate_flow_name() - - tmp = _clean_flow(flow_folder) - create(tmp, flow_name, cli=False) - - flow_dir = os.path.join(tmp, flow_folder) - - with open(os.path.join(flow_dir, 'flows', f'{flow_id}.py'), 'a') as f: - f.write(""" - try: - x = onecode.checkbox('my_checkbox', True, optional=False) - except ValueError as err: - with open("stdout.txt", 'w') as f: - f.write(str(err)) - """) - - params = os.path.join(flow_dir, 'params.json') - with open(params, 'w') as f: - json.dump({"my_checkbox": None}, f) - - os.system(f'cd "{flow_dir}" && {Env.ONECODE_DO_TYPECHECK}=1 python main.py {params}') - - with open(os.path.join(flow_dir, 'stdout.txt')) as f: - assert f.read() == "[my_checkbox] Value is required: None provided" - - shutil.rmtree(flow_dir) - - -@pytest.mark.emulations -def test_invalid_name_key(): - flow_name, flow_folder, flow_id = _generate_flow_name() - - tmp = _clean_flow(flow_folder) - create(tmp, flow_name, cli=False) - - flow_dir = os.path.join(tmp, flow_folder) - - with open(os.path.join(flow_dir, 'flows', f'{flow_id}.py'), 'a') as f: - f.write(""" - try: - x = onecode.checkbox('_my_checkbox', True) - except ValueError as err: - with open("stdout.txt", 'w') as f: - f.write(str(err)) - """) - - os.system(f'cd "{flow_dir}" && {Env.ONECODE_DO_TYPECHECK}=1 python main.py') - - with open(os.path.join(flow_dir, 'stdout.txt')) as f: - assert f.read() == 'Key starting with "_" are reserved: _my_checkbox' - - shutil.rmtree(flow_dir) diff --git a/tests/emulations/elements/test_run_csv_reader.py b/tests/emulations/elements/test_run_csv_reader.py deleted file mode 100644 index 0a21d45..0000000 --- a/tests/emulations/elements/test_run_csv_reader.py +++ /dev/null @@ -1,340 +0,0 @@ -import json -import os -import shutil - -import pytest - -from onecode import Env -from onecode.cli.create import create -from tests.utils.flow_cli import ( - _clean_flow, - _generate_csv_file, - _generate_flow_name -) - - -@pytest.mark.emulations -def test_execute_single_count(): - flow_name, flow_folder, flow_id = _generate_flow_name() - - tmp = _clean_flow(flow_folder) - create(tmp, flow_name, cli=False) - - flow_dir = os.path.join(tmp, flow_folder) - _generate_csv_file(flow_dir, 'test.csv') - - with open(os.path.join(flow_dir, 'flows', f'{flow_id}.py'), 'a') as f: - f.write(""" - import os - import pandas as pd - - x = onecode.csv_reader('my_csv_reader', 'test.csv') - with open("stdout.txt", 'w') as f: - df = pd.read_csv(os.path.join(onecode.Project().data_root, 'test.csv')) - f.write(str(df.equals(x))) - """) - - os.system(f'cd "{flow_dir}" && {Env.ONECODE_DO_TYPECHECK}=1 python main.py') - - with open(os.path.join(flow_dir, 'stdout.txt')) as f: - assert f.read() == "True" - - shutil.rmtree(flow_dir) - - -@pytest.mark.emulations -def test_execute_multiple_count(): - flow_name, flow_folder, flow_id = _generate_flow_name() - - tmp = _clean_flow(flow_folder) - create(tmp, flow_name, cli=False) - - flow_dir = os.path.join(tmp, flow_folder) - _generate_csv_file(flow_dir, 'test.csv') - - # count is ignored at loading & execution as it is only used in UI mode - with open(os.path.join(flow_dir, 'flows', f'{flow_id}.py'), 'a') as f: - f.write(""" - import os - import pandas as pd - - x = onecode.csv_reader('my_csv_reader', ['test.csv', 'test.csv'], count=3) - with open("stdout.txt", 'w') as f: - df = pd.read_csv(os.path.join(onecode.Project().data_root, 'test.csv')) - f.write(str([df.equals(x_df) for x_df in x])) - """) - - os.system(f'cd "{flow_dir}" && {Env.ONECODE_DO_TYPECHECK}=1 python main.py') - - with open(os.path.join(flow_dir, 'stdout.txt')) as f: - assert f.read() == "[True, True]" - - shutil.rmtree(flow_dir) - - -@pytest.mark.emulations -def test_load_then_execute_single_count(): - flow_name, flow_folder, flow_id = _generate_flow_name() - - tmp = _clean_flow(flow_folder) - create(tmp, flow_name, cli=False) - - flow_dir = os.path.join(tmp, flow_folder) - _generate_csv_file(flow_dir, 'test.csv') - - with open(os.path.join(flow_dir, 'flows', f'{flow_id}.py'), 'a') as f: - f.write(""" - import os - import pandas as pd - - x = onecode.csv_reader('my_csv_reader', 'test.csv') - with open("stdout.txt", 'w') as f: - df = pd.read_csv(os.path.join(onecode.Project().data_root, 'test.csv')) - f.write(str(df.equals(x))) - """) - - params = os.path.join(flow_dir, 'params.json') - with open(params, 'w') as f: - json.dump({"my_csv_reader": "test.csv"}, f) - - os.system(f'cd "{flow_dir}" && {Env.ONECODE_DO_TYPECHECK}=1 python main.py {params}') - - with open(os.path.join(flow_dir, 'stdout.txt')) as f: - assert f.read() == "True" - - shutil.rmtree(flow_dir) - - -@pytest.mark.emulations -def test_load_then_execute_multiple_count(): - flow_name, flow_folder, flow_id = _generate_flow_name() - - tmp = _clean_flow(flow_folder) - create(tmp, flow_name, cli=False) - - flow_dir = os.path.join(tmp, flow_folder) - _generate_csv_file(flow_dir, 'test.csv') - - # count is ignored at loading & execution as it is only used in UI mode - with open(os.path.join(flow_dir, 'flows', f'{flow_id}.py'), 'a') as f: - f.write(""" - import os - import pandas as pd - - x = onecode.csv_reader('my_csv_reader', 'test.csv', count=4) - with open("stdout.txt", 'w') as f: - df = pd.read_csv(os.path.join(onecode.Project().data_root, 'test.csv')) - f.write(str([df.equals(x_df) for x_df in x])) - """) - - params = os.path.join(flow_dir, 'params.json') - with open(params, 'w') as f: - json.dump({"my_csv_reader": ["test.csv", "test.csv", "test.csv"]}, f) - - os.system(f'cd "{flow_dir}" && {Env.ONECODE_DO_TYPECHECK}=1 python main.py {params}') - - with open(os.path.join(flow_dir, 'stdout.txt')) as f: - assert f.read() == "[True, True, True]" - - shutil.rmtree(flow_dir) - - -@pytest.mark.emulations -def test_execute_invalid_csv(): - flow_name, flow_folder, flow_id = _generate_flow_name() - - tmp = _clean_flow(flow_folder) - create(tmp, flow_name, cli=False) - - flow_dir = os.path.join(tmp, flow_folder) - with open(os.path.join(flow_dir, 'data', 'test.csv'), 'w') as f: - f.write("A,B,C\n") - - with open(os.path.join(flow_dir, 'flows', f'{flow_id}.py'), 'a') as f: - f.write(""" - import os - import pandas as pd - - try: - x = onecode.csv_reader('my_csv_reader', 'test.csv') - except ValueError as err: - with open("stdout.txt", 'w') as f: - f.write(str(err)) - """) - - os.system(f'cd "{flow_dir}" && {Env.ONECODE_DO_TYPECHECK}=1 python main.py') - - with open(os.path.join(flow_dir, 'stdout.txt')) as f: - assert f.read() == "[my_csv_reader] Empty dataframe" - - shutil.rmtree(flow_dir) - - -@pytest.mark.emulations -def test_load_then_execute_invalid_csv(): - flow_name, flow_folder, flow_id = _generate_flow_name() - - tmp = _clean_flow(flow_folder) - create(tmp, flow_name, cli=False) - - flow_dir = os.path.join(tmp, flow_folder) - _generate_csv_file(flow_dir, 'valid.csv') - with open(os.path.join(flow_dir, 'data', 'test.csv'), 'w') as f: - f.write("A,B,C\n") - - with open(os.path.join(flow_dir, 'flows', f'{flow_id}.py'), 'a') as f: - f.write(""" - import os - import pandas as pd - - try: - x = onecode.csv_reader('my_csv_reader', 'valid.csv') - except ValueError as err: - with open("stdout.txt", 'w') as f: - f.write(str(err)) - """) - - params = os.path.join(flow_dir, 'params.json') - with open(params, 'w') as f: - json.dump({"my_csv_reader": "test.csv"}, f) - - os.system(f'cd "{flow_dir}" && {Env.ONECODE_DO_TYPECHECK}=1 python main.py {params}') - - with open(os.path.join(flow_dir, 'stdout.txt')) as f: - assert f.read() == "[my_csv_reader] Empty dataframe" - - shutil.rmtree(flow_dir) - - -@pytest.mark.emulations -def test_execute_optional_value(): - flow_name, flow_folder, flow_id = _generate_flow_name() - - tmp = _clean_flow(flow_folder) - create(tmp, flow_name, cli=False) - - flow_dir = os.path.join(tmp, flow_folder) - - with open(os.path.join(flow_dir, 'flows', f'{flow_id}.py'), 'a') as f: - f.write(""" - x = onecode.csv_reader('my_csv_reader', None, optional=True) - with open("stdout.txt", 'w') as f: - f.write(str(x)) - """) - - os.system(f'cd "{flow_dir}" && {Env.ONECODE_DO_TYPECHECK}=1 python main.py') - - with open(os.path.join(flow_dir, 'stdout.txt')) as f: - assert f.read() == "None" - - shutil.rmtree(flow_dir) - - -@pytest.mark.emulations -def test_load_then_execute_optional_value(): - flow_name, flow_folder, flow_id = _generate_flow_name() - - tmp = _clean_flow(flow_folder) - create(tmp, flow_name, cli=False) - - flow_dir = os.path.join(tmp, flow_folder) - - with open(os.path.join(flow_dir, 'flows', f'{flow_id}.py'), 'a') as f: - f.write(""" - x = onecode.csv_reader('my_csv_reader', 'test.csv', optional=True) - with open("stdout.txt", 'w') as f: - f.write(str(x)) - """) - - params = os.path.join(flow_dir, 'params.json') - with open(params, 'w') as f: - json.dump({"my_csv_reader": None}, f) - - os.system(f'cd "{flow_dir}" && {Env.ONECODE_DO_TYPECHECK}=1 python main.py {params}') - - with open(os.path.join(flow_dir, 'stdout.txt')) as f: - assert f.read() == "None" - - shutil.rmtree(flow_dir) - - -@pytest.mark.emulations -def test_execute_invalid_optional_value(): - flow_name, flow_folder, flow_id = _generate_flow_name() - - tmp = _clean_flow(flow_folder) - create(tmp, flow_name, cli=False) - - flow_dir = os.path.join(tmp, flow_folder) - - with open(os.path.join(flow_dir, 'flows', f'{flow_id}.py'), 'a') as f: - f.write(""" - try: - x = onecode.slider('my_csv_reader', None, optional=False) - except ValueError as err: - with open("stdout.txt", 'w') as f: - f.write(str(err)) - """) - - os.system(f'cd "{flow_dir}" && {Env.ONECODE_DO_TYPECHECK}=1 python main.py') - - with open(os.path.join(flow_dir, 'stdout.txt')) as f: - assert f.read() == "[my_csv_reader] Value is required: None provided" - - shutil.rmtree(flow_dir) - - -@pytest.mark.emulations -def test_load_then_execute_invalid_optional_value(): - flow_name, flow_folder, flow_id = _generate_flow_name() - - tmp = _clean_flow(flow_folder) - create(tmp, flow_name, cli=False) - - flow_dir = os.path.join(tmp, flow_folder) - - with open(os.path.join(flow_dir, 'flows', f'{flow_id}.py'), 'a') as f: - f.write(""" - try: - x = onecode.csv_reader('my_csv_reader', 'test.csv', optional=False) - except ValueError as err: - with open("stdout.txt", 'w') as f: - f.write(str(err)) - """) - - params = os.path.join(flow_dir, 'params.json') - with open(params, 'w') as f: - json.dump({"my_csv_reader": None}, f) - - os.system(f'cd "{flow_dir}" && {Env.ONECODE_DO_TYPECHECK}=1 python main.py {params}') - - with open(os.path.join(flow_dir, 'stdout.txt')) as f: - assert f.read() == "[my_csv_reader] Value is required: None provided" - - shutil.rmtree(flow_dir) - - -@pytest.mark.emulations -def test_invalid_name_key(): - flow_name, flow_folder, flow_id = _generate_flow_name() - - tmp = _clean_flow(flow_folder) - create(tmp, flow_name, cli=False) - - flow_dir = os.path.join(tmp, flow_folder) - - with open(os.path.join(flow_dir, 'flows', f'{flow_id}.py'), 'a') as f: - f.write(""" - try: - x = onecode.csv_reader('_my_csv_reader', 'test.csv') - except ValueError as err: - with open("stdout.txt", 'w') as f: - f.write(str(err)) - """) - - os.system(f'cd "{flow_dir}" && {Env.ONECODE_DO_TYPECHECK}=1 python main.py') - - with open(os.path.join(flow_dir, 'stdout.txt')) as f: - assert f.read() == 'Key starting with "_" are reserved: _my_csv_reader' - - shutil.rmtree(flow_dir) diff --git a/tests/emulations/elements/test_run_dropdown.py b/tests/emulations/elements/test_run_dropdown.py deleted file mode 100644 index 738ddbe..0000000 --- a/tests/emulations/elements/test_run_dropdown.py +++ /dev/null @@ -1,516 +0,0 @@ -import json -import os -import shutil - -import pytest - -from onecode import Env -from onecode.cli.create import create -from tests.utils.flow_cli import _clean_flow, _generate_flow_name - - -@pytest.mark.emulations -def test_execute_single_value_single_count(): - flow_name, flow_folder, flow_id = _generate_flow_name() - - tmp = _clean_flow(flow_folder) - create(tmp, flow_name, cli=False) - - flow_dir = os.path.join(tmp, flow_folder) - - with open(os.path.join(flow_dir, 'flows', f'{flow_id}.py'), 'a') as f: - f.write(""" - x = onecode.dropdown('my_dropdown', 'B', options=['A', 'B', 'C']) - with open("stdout.txt", 'w') as f: - f.write(x) - """) - - os.system(f'cd "{flow_dir}" && {Env.ONECODE_DO_TYPECHECK}=1 python main.py') - - with open(os.path.join(flow_dir, 'stdout.txt')) as f: - assert f.read() == "B" - - shutil.rmtree(flow_dir) - - -@pytest.mark.emulations -def test_execute_multiple_values_single_count(): - flow_name, flow_folder, flow_id = _generate_flow_name() - - tmp = _clean_flow(flow_folder) - create(tmp, flow_name, cli=False) - - flow_dir = os.path.join(tmp, flow_folder) - - with open(os.path.join(flow_dir, 'flows', f'{flow_id}.py'), 'a') as f: - f.write(""" - x = onecode.dropdown('my_dropdown', ['B', 'C'], options=['A', 'B', 'C'], multiple=True) - with open("stdout.txt", 'w') as f: - f.write(str(x)) - """) - - os.system(f'cd "{flow_dir}" && {Env.ONECODE_DO_TYPECHECK}=1 python main.py') - - with open(os.path.join(flow_dir, 'stdout.txt')) as f: - assert f.read() == "['B', 'C']" - - shutil.rmtree(flow_dir) - - -@pytest.mark.emulations -def test_execute_single_value_multiple_count(): - flow_name, flow_folder, flow_id = _generate_flow_name() - - tmp = _clean_flow(flow_folder) - create(tmp, flow_name, cli=False) - - flow_dir = os.path.join(tmp, flow_folder) - - # count is ignored at loading & execution as it is only used in UI mode - with open(os.path.join(flow_dir, 'flows', f'{flow_id}.py'), 'a') as f: - f.write(""" - x = onecode.dropdown('my_dropdown', ['B', 'B'], options=['A', 'B', 'C'], count=3) - with open("stdout.txt", 'w') as f: - f.write(str(x)) - """) - - os.system(f'cd "{flow_dir}" && {Env.ONECODE_DO_TYPECHECK}=1 python main.py') - - with open(os.path.join(flow_dir, 'stdout.txt')) as f: - assert f.read() == "['B', 'B']" - - shutil.rmtree(flow_dir) - - -@pytest.mark.emulations -def test_execute_multiple_values_multiple_count(): - flow_name, flow_folder, flow_id = _generate_flow_name() - - tmp = _clean_flow(flow_folder) - create(tmp, flow_name, cli=False) - - flow_dir = os.path.join(tmp, flow_folder) - - # count is ignored at loading & execution as it is only used in UI mode - with open(os.path.join(flow_dir, 'flows', f'{flow_id}.py'), 'a') as f: - f.write(""" - x = onecode.dropdown( - 'my_dropdown', - [['B', 'C']], - options=['A', 'B', 'C'], - multiple=True, - count=3 - ) - with open("stdout.txt", 'w') as f: - f.write(str(x)) - """) - - os.system(f'cd "{flow_dir}" && {Env.ONECODE_DO_TYPECHECK}=1 python main.py') - - with open(os.path.join(flow_dir, 'stdout.txt')) as f: - assert f.read() == "[['B', 'C']]" - - shutil.rmtree(flow_dir) - - -@pytest.mark.emulations -def test_load_then_execute_single_value_single_count(): - flow_name, flow_folder, flow_id = _generate_flow_name() - - tmp = _clean_flow(flow_folder) - create(tmp, flow_name, cli=False) - - flow_dir = os.path.join(tmp, flow_folder) - - with open(os.path.join(flow_dir, 'flows', f'{flow_id}.py'), 'a') as f: - f.write(""" - x = onecode.dropdown('my_dropdown', 'B', options=['A', 'B', 'C']) - with open("stdout.txt", 'w') as f: - f.write(x) - """) - - params = os.path.join(flow_dir, 'params.json') - with open(params, 'w') as f: - json.dump({"my_dropdown": "B"}, f) - - os.system(f'cd "{flow_dir}" && {Env.ONECODE_DO_TYPECHECK}=1 python main.py {params}') - - with open(os.path.join(flow_dir, 'stdout.txt')) as f: - assert f.read() == "B" - - shutil.rmtree(flow_dir) - - -@pytest.mark.emulations -def test_load_then_execute_multiple_values_single_count(): - flow_name, flow_folder, flow_id = _generate_flow_name() - - tmp = _clean_flow(flow_folder) - create(tmp, flow_name, cli=False) - - flow_dir = os.path.join(tmp, flow_folder) - - with open(os.path.join(flow_dir, 'flows', f'{flow_id}.py'), 'a') as f: - f.write(""" - x = onecode.dropdown('my_dropdown', ['B', 'C'], options=['A', 'B', 'C'], multiple=True) - with open("stdout.txt", 'w') as f: - f.write(str(x)) - """) - - params = os.path.join(flow_dir, 'params.json') - with open(params, 'w') as f: - json.dump({"my_dropdown": ["B", "C"]}, f) - - os.system(f'cd "{flow_dir}" && {Env.ONECODE_DO_TYPECHECK}=1 python main.py {params}') - - with open(os.path.join(flow_dir, 'stdout.txt')) as f: - assert f.read() == "['B', 'C']" - - shutil.rmtree(flow_dir) - - -@pytest.mark.emulations -def test_load_then_execute_single_value_multiple_count(): - flow_name, flow_folder, flow_id = _generate_flow_name() - - tmp = _clean_flow(flow_folder) - create(tmp, flow_name, cli=False) - - flow_dir = os.path.join(tmp, flow_folder) - - # count is ignored at loading & execution as it is only used in UI mode - with open(os.path.join(flow_dir, 'flows', f'{flow_id}.py'), 'a') as f: - f.write(""" - x = onecode.dropdown('my_dropdown', ['B', 'B', 'B'], options=['A', 'B', 'C'], count=4) - with open("stdout.txt", 'w') as f: - f.write(str(x)) - """) - - params = os.path.join(flow_dir, 'params.json') - with open(params, 'w') as f: - json.dump({"my_dropdown": ["B", "B"]}, f) - - os.system(f'cd "{flow_dir}" && {Env.ONECODE_DO_TYPECHECK}=1 python main.py {params}') - - with open(os.path.join(flow_dir, 'stdout.txt')) as f: - assert f.read() == "['B', 'B']" - - shutil.rmtree(flow_dir) - - -@pytest.mark.emulations -def test_load_then_execute_multiple_values_multiple_count(): - flow_name, flow_folder, flow_id = _generate_flow_name() - - tmp = _clean_flow(flow_folder) - create(tmp, flow_name, cli=False) - - flow_dir = os.path.join(tmp, flow_folder) - - # count is ignored at loading & execution as it is only used in UI mode - with open(os.path.join(flow_dir, 'flows', f'{flow_id}.py'), 'a') as f: - f.write(""" - x = onecode.dropdown( - 'my_dropdown', - ['B', 'C'], - options=['A', 'B', 'C'], - multiple=True, - count=4 - ) - with open("stdout.txt", 'w') as f: - f.write(str(x)) - """) - - params = os.path.join(flow_dir, 'params.json') - with open(params, 'w') as f: - json.dump({"my_dropdown": [["B", "C"], ["B", "C"], ["B", "C"]]}, f) - - os.system(f'cd "{flow_dir}" && {Env.ONECODE_DO_TYPECHECK}=1 python main.py {params}') - - with open(os.path.join(flow_dir, 'stdout.txt')) as f: - assert f.read() == "[['B', 'C'], ['B', 'C'], ['B', 'C']]" - - shutil.rmtree(flow_dir) - - -@pytest.mark.emulations -def test_execute_invalid_single_value(): - flow_name, flow_folder, flow_id = _generate_flow_name() - - tmp = _clean_flow(flow_folder) - create(tmp, flow_name, cli=False) - - flow_dir = os.path.join(tmp, flow_folder) - - with open(os.path.join(flow_dir, 'flows', f'{flow_id}.py'), 'a') as f: - f.write(""" - try: - x = onecode.dropdown('my_dropdown', 'D', options=['A', 'B', 'C']) - except ValueError as err: - with open("stdout.txt", 'w') as f: - f.write(str(err)) - """) - - os.system(f'cd "{flow_dir}" && {Env.ONECODE_DO_TYPECHECK}=1 python main.py') - - with open(os.path.join(flow_dir, 'stdout.txt')) as f: - assert f.read() == "[my_dropdown] Not a valid choice: D" - - shutil.rmtree(flow_dir) - - -@pytest.mark.emulations -def test_load_then_execute_invalid_single_value(): - flow_name, flow_folder, flow_id = _generate_flow_name() - - tmp = _clean_flow(flow_folder) - create(tmp, flow_name, cli=False) - - flow_dir = os.path.join(tmp, flow_folder) - - with open(os.path.join(flow_dir, 'flows', f'{flow_id}.py'), 'a') as f: - f.write(""" - try: - x = onecode.dropdown('my_dropdown', 'B', options=['A', 'B', 'C']) - except ValueError as err: - with open("stdout.txt", 'w') as f: - f.write(str(err)) - """) - - params = os.path.join(flow_dir, 'params.json') - with open(params, 'w') as f: - json.dump({"my_dropdown": "D"}, f) - - os.system(f'cd "{flow_dir}" && {Env.ONECODE_DO_TYPECHECK}=1 python main.py {params}') - - with open(os.path.join(flow_dir, 'stdout.txt')) as f: - assert f.read() == "[my_dropdown] Not a valid choice: D" - - shutil.rmtree(flow_dir) - - -@pytest.mark.emulations -def test_execute_invalid_multiple_values(): - flow_name, flow_folder, flow_id = _generate_flow_name() - - tmp = _clean_flow(flow_folder) - create(tmp, flow_name, cli=False) - - flow_dir = os.path.join(tmp, flow_folder) - - with open(os.path.join(flow_dir, 'flows', f'{flow_id}.py'), 'a') as f: - f.write(""" - try: - x = onecode.dropdown('my_dropdown', ['B', 'C', 'D'], options=['A', 'B', 'C'], multiple=True) - except ValueError as err: - with open("stdout.txt", 'w') as f: - f.write(str(err)) - """) - - os.system(f'cd "{flow_dir}" && {Env.ONECODE_DO_TYPECHECK}=1 python main.py') - - with open(os.path.join(flow_dir, 'stdout.txt')) as f: - assert f.read() == "[my_dropdown] Not a valid choice: D" - - shutil.rmtree(flow_dir) - - -@pytest.mark.emulations -def test_load_then_execute_invalid_multiple_values(): - flow_name, flow_folder, flow_id = _generate_flow_name() - - tmp = _clean_flow(flow_folder) - create(tmp, flow_name, cli=False) - - flow_dir = os.path.join(tmp, flow_folder) - - with open(os.path.join(flow_dir, 'flows', f'{flow_id}.py'), 'a') as f: - f.write(""" - try: - x = onecode.dropdown('my_dropdown', ['B', 'C'], options=['A', 'B', 'C'], multiple=True) - except ValueError as err: - with open("stdout.txt", 'w') as f: - f.write(str(err)) - """) - - params = os.path.join(flow_dir, 'params.json') - with open(params, 'w') as f: - json.dump({"my_dropdown": ["B", "C", "D"]}, f) - - os.system(f'cd "{flow_dir}" && {Env.ONECODE_DO_TYPECHECK}=1 python main.py {params}') - - with open(os.path.join(flow_dir, 'stdout.txt')) as f: - assert f.read() == "[my_dropdown] Not a valid choice: D" - - shutil.rmtree(flow_dir) - - -@pytest.mark.emulations -def test_load_then_execute_invalid_multiple_values_with_count(): - flow_name, flow_folder, flow_id = _generate_flow_name() - - tmp = _clean_flow(flow_folder) - create(tmp, flow_name, cli=False) - - flow_dir = os.path.join(tmp, flow_folder) - - # count is ignored at loading & execution as it is only used in UI mode - with open(os.path.join(flow_dir, 'flows', f'{flow_id}.py'), 'a') as f: - f.write(""" - try: - x = onecode.dropdown( - 'my_dropdown', - ['B', 'C'], - options=['A', 'B', 'C'], - multiple=True, - count=4 - ) - except ValueError as err: - with open("stdout.txt", 'w') as f: - f.write(str(err)) - """) - - params = os.path.join(flow_dir, 'params.json') - with open(params, 'w') as f: - json.dump({"my_dropdown": [["B", "C"], ["B", "C"], ["B", "C", "D"]]}, f) - - os.system(f'cd "{flow_dir}" && {Env.ONECODE_DO_TYPECHECK}=1 python main.py {params}') - - with open(os.path.join(flow_dir, 'stdout.txt')) as f: - assert f.read() == "[my_dropdown] Not a valid choice: D" - - shutil.rmtree(flow_dir) - - -@pytest.mark.emulations -def test_execute_optional_value(): - flow_name, flow_folder, flow_id = _generate_flow_name() - - tmp = _clean_flow(flow_folder) - create(tmp, flow_name, cli=False) - - flow_dir = os.path.join(tmp, flow_folder) - - with open(os.path.join(flow_dir, 'flows', f'{flow_id}.py'), 'a') as f: - f.write(""" - x = onecode.dropdown('my_dropdown', None, options=['A', 'B', 'C'], optional=True) - with open("stdout.txt", 'w') as f: - f.write(str(x)) - """) - - os.system(f'cd "{flow_dir}" && {Env.ONECODE_DO_TYPECHECK}=1 python main.py') - - with open(os.path.join(flow_dir, 'stdout.txt')) as f: - assert f.read() == "None" - - shutil.rmtree(flow_dir) - - -@pytest.mark.emulations -def test_load_then_execute_optional_value(): - flow_name, flow_folder, flow_id = _generate_flow_name() - - tmp = _clean_flow(flow_folder) - create(tmp, flow_name, cli=False) - - flow_dir = os.path.join(tmp, flow_folder) - - with open(os.path.join(flow_dir, 'flows', f'{flow_id}.py'), 'a') as f: - f.write(""" - x = onecode.dropdown('my_dropdown', 'B', options=['A', 'B', 'C'], optional=True) - with open("stdout.txt", 'w') as f: - f.write(str(x)) - """) - - params = os.path.join(flow_dir, 'params.json') - with open(params, 'w') as f: - json.dump({"my_dropdown": None}, f) - - os.system(f'cd "{flow_dir}" && {Env.ONECODE_DO_TYPECHECK}=1 python main.py {params}') - - with open(os.path.join(flow_dir, 'stdout.txt')) as f: - assert f.read() == "None" - - shutil.rmtree(flow_dir) - - -@pytest.mark.emulations -def test_execute_invalid_optional_value(): - flow_name, flow_folder, flow_id = _generate_flow_name() - - tmp = _clean_flow(flow_folder) - create(tmp, flow_name, cli=False) - - flow_dir = os.path.join(tmp, flow_folder) - - with open(os.path.join(flow_dir, 'flows', f'{flow_id}.py'), 'a') as f: - f.write(""" - try: - x = onecode.dropdown('my_dropdown', None, options=['A', 'B', 'C'], optional=False) - except ValueError as err: - with open("stdout.txt", 'w') as f: - f.write(str(err)) - """) - - os.system(f'cd "{flow_dir}" && {Env.ONECODE_DO_TYPECHECK}=1 python main.py') - - with open(os.path.join(flow_dir, 'stdout.txt')) as f: - assert f.read() == "[my_dropdown] Value is required: None provided" - - shutil.rmtree(flow_dir) - - -@pytest.mark.emulations -def test_load_then_execute_invalid_optional_value(): - flow_name, flow_folder, flow_id = _generate_flow_name() - - tmp = _clean_flow(flow_folder) - create(tmp, flow_name, cli=False) - - flow_dir = os.path.join(tmp, flow_folder) - - with open(os.path.join(flow_dir, 'flows', f'{flow_id}.py'), 'a') as f: - f.write(""" - try: - x = onecode.dropdown('my_dropdown', 'B', options=['A', 'B', 'C'], optional=False) - except ValueError as err: - with open("stdout.txt", 'w') as f: - f.write(str(err)) - """) - - params = os.path.join(flow_dir, 'params.json') - with open(params, 'w') as f: - json.dump({"my_dropdown": None}, f) - - os.system(f'cd "{flow_dir}" && {Env.ONECODE_DO_TYPECHECK}=1 python main.py {params}') - - with open(os.path.join(flow_dir, 'stdout.txt')) as f: - assert f.read() == "[my_dropdown] Value is required: None provided" - - shutil.rmtree(flow_dir) - - -@pytest.mark.emulations -def test_invalid_name_key(): - flow_name, flow_folder, flow_id = _generate_flow_name() - - tmp = _clean_flow(flow_folder) - create(tmp, flow_name, cli=False) - - flow_dir = os.path.join(tmp, flow_folder) - - with open(os.path.join(flow_dir, 'flows', f'{flow_id}.py'), 'a') as f: - f.write(""" - try: - x = onecode.dropdown('_my_dropdown', 'B', options=['A', 'B', 'C']) - except ValueError as err: - with open("stdout.txt", 'w') as f: - f.write(str(err)) - """) - - os.system(f'cd "{flow_dir}" && {Env.ONECODE_DO_TYPECHECK}=1 python main.py') - - with open(os.path.join(flow_dir, 'stdout.txt')) as f: - assert f.read() == 'Key starting with "_" are reserved: _my_dropdown' - - shutil.rmtree(flow_dir) diff --git a/tests/emulations/elements/test_run_file_input.py b/tests/emulations/elements/test_run_file_input.py deleted file mode 100644 index 4bbdfcf..0000000 --- a/tests/emulations/elements/test_run_file_input.py +++ /dev/null @@ -1,719 +0,0 @@ -import json -import os -import shutil - -import pytest - -from onecode import Env -from onecode.cli.create import create -from tests.utils.flow_cli import ( - _clean_flow, - _generate_csv_file, - _generate_flow_name -) - - -@pytest.mark.emulations -def test_execute_single_value_single_count(): - flow_name, flow_folder, flow_id = _generate_flow_name() - - tmp = _clean_flow(flow_folder) - create(tmp, flow_name, cli=False) - - flow_dir = os.path.join(tmp, flow_folder) - flow_data = os.path.join(flow_dir, 'data') - _generate_csv_file(flow_dir, 'input/test_file.txt') - - with open(os.path.join(flow_dir, 'flows', f'{flow_id}.py'), 'a') as f: - f.write(""" - x = onecode.file_input('my_file_input', 'input/test_file.txt') - with open("stdout.txt", 'w') as f: - f.write(x) - """) - - os.system(f'cd "{flow_dir}" && {Env.ONECODE_DO_TYPECHECK}=1 python main.py') - - with open(os.path.join(flow_dir, 'stdout.txt')) as f: - assert f.read() == f"{flow_data}/input/test_file.txt" - - shutil.rmtree(flow_dir) - - -@pytest.mark.emulations -def test_execute_multiple_values_single_count(): - flow_name, flow_folder, flow_id = _generate_flow_name() - - tmp = _clean_flow(flow_folder) - create(tmp, flow_name, cli=False) - - flow_dir = os.path.join(tmp, flow_folder) - flow_data = os.path.join(flow_dir, 'data') - _generate_csv_file(flow_dir, 'input/test_file_1.txt') - _generate_csv_file(flow_dir, 'input/test_file_2.txt') - - with open(os.path.join(flow_dir, 'flows', f'{flow_id}.py'), 'a') as f: - f.write(""" - x = onecode.file_input( - 'my_file_input', - ['input/test_file_1.txt', 'input/test_file_2.txt'], - multiple=True - ) - with open("stdout.txt", 'w') as f: - f.write(str(x)) - """) - - os.system(f'cd "{flow_dir}" && {Env.ONECODE_DO_TYPECHECK}=1 python main.py') - - with open(os.path.join(flow_dir, 'stdout.txt')) as f: - assert f.read() == f"['{flow_data}/input/test_file_1.txt'," \ - f" '{flow_data}/input/test_file_2.txt']" - - shutil.rmtree(flow_dir) - - -@pytest.mark.emulations -def test_execute_single_value_multiple_count(): - flow_name, flow_folder, flow_id = _generate_flow_name() - - tmp = _clean_flow(flow_folder) - create(tmp, flow_name, cli=False) - - flow_dir = os.path.join(tmp, flow_folder) - flow_data = os.path.join(flow_dir, 'data') - _generate_csv_file(flow_dir, 'input/test_file.txt') - - # count is ignored at loading & execution as it is only used in UI mode - with open(os.path.join(flow_dir, 'flows', f'{flow_id}.py'), 'a') as f: - f.write(""" - x = onecode.file_input('my_file_input', ['input/test_file.txt', 'input/test_file.txt'], count=3) - with open("stdout.txt", 'w') as f: - f.write(str(x)) - """) - - os.system(f'cd "{flow_dir}" && {Env.ONECODE_DO_TYPECHECK}=1 python main.py') - - with open(os.path.join(flow_dir, 'stdout.txt')) as f: - assert f.read() == f"['{flow_data}/input/test_file.txt'," \ - f" '{flow_data}/input/test_file.txt']" - - shutil.rmtree(flow_dir) - - -@pytest.mark.emulations -def test_execute_multiple_values_multiple_count(): - flow_name, flow_folder, flow_id = _generate_flow_name() - - tmp = _clean_flow(flow_folder) - create(tmp, flow_name, cli=False) - - flow_dir = os.path.join(tmp, flow_folder) - flow_data = os.path.join(flow_dir, 'data') - _generate_csv_file(flow_dir, 'input/test_file_1.txt') - _generate_csv_file(flow_dir, 'input/test_file_2.txt') - - # count is ignored at loading & execution as it is only used in UI mode - with open(os.path.join(flow_dir, 'flows', f'{flow_id}.py'), 'a') as f: - f.write(""" - x = onecode.file_input( - 'my_file_input', - [['input/test_file_1.txt', 'input/test_file_2.txt']], - multiple=True, - count=3 - ) - with open("stdout.txt", 'w') as f: - f.write(str(x)) - """) - - os.system(f'cd "{flow_dir}" && {Env.ONECODE_DO_TYPECHECK}=1 python main.py') - - with open(os.path.join(flow_dir, 'stdout.txt')) as f: - assert f.read() == \ - f"[['{flow_data}/input/test_file_1.txt', '{flow_data}/input/test_file_2.txt']]" - - shutil.rmtree(flow_dir) - - -@pytest.mark.emulations -def test_load_then_execute_single_value_single_count(): - flow_name, flow_folder, flow_id = _generate_flow_name() - - tmp = _clean_flow(flow_folder) - create(tmp, flow_name, cli=False) - - flow_dir = os.path.join(tmp, flow_folder) - flow_data = os.path.join(flow_dir, 'data') - _generate_csv_file(flow_dir, 'input/test_file.txt') - - with open(os.path.join(flow_dir, 'flows', f'{flow_id}.py'), 'a') as f: - f.write(""" - x = onecode.file_input('my_file_input', 'input/test_file.txt') - with open("stdout.txt", 'w') as f: - f.write(x) - """) - - params = os.path.join(flow_dir, 'params.json') - with open(params, 'w') as f: - json.dump({"my_file_input": "input/test_file.txt"}, f) - - os.system(f'cd "{flow_dir}" && {Env.ONECODE_DO_TYPECHECK}=1 python main.py {params}') - - with open(os.path.join(flow_dir, 'stdout.txt')) as f: - assert f.read() == f"{flow_data}/input/test_file.txt" - - shutil.rmtree(flow_dir) - - -@pytest.mark.emulations -def test_load_then_execute_multiple_values_single_count(): - flow_name, flow_folder, flow_id = _generate_flow_name() - - tmp = _clean_flow(flow_folder) - create(tmp, flow_name, cli=False) - - flow_dir = os.path.join(tmp, flow_folder) - flow_data = os.path.join(flow_dir, 'data') - _generate_csv_file(flow_dir, 'input/test_file_1.txt') - _generate_csv_file(flow_dir, 'input/test_file_2.txt') - - with open(os.path.join(flow_dir, 'flows', f'{flow_id}.py'), 'a') as f: - f.write(""" - x = onecode.file_input( - 'my_file_input', - ['input/test_file_1.txt', 'input/test_file_2.txt'], - multiple=True - ) - with open("stdout.txt", 'w') as f: - f.write(str(x)) - """) - - params = os.path.join(flow_dir, 'params.json') - with open(params, 'w') as f: - json.dump({"my_file_input": ['input/test_file_1.txt', 'input/test_file_2.txt']}, f) - - os.system(f'cd "{flow_dir}" && {Env.ONECODE_DO_TYPECHECK}=1 python main.py {params}') - - with open(os.path.join(flow_dir, 'stdout.txt')) as f: - assert f.read() == f"['{flow_data}/input/test_file_1.txt'," \ - f" '{flow_data}/input/test_file_2.txt']" - - shutil.rmtree(flow_dir) - - -@pytest.mark.emulations -def test_load_then_execute_single_value_multiple_count(): - flow_name, flow_folder, flow_id = _generate_flow_name() - - tmp = _clean_flow(flow_folder) - create(tmp, flow_name, cli=False) - - flow_dir = os.path.join(tmp, flow_folder) - flow_data = os.path.join(flow_dir, 'data') - _generate_csv_file(flow_dir, 'input/test_file.txt') - - # count is ignored at loading & execution as it is only used in UI mode - with open(os.path.join(flow_dir, 'flows', f'{flow_id}.py'), 'a') as f: - f.write(""" - x = onecode.file_input('my_file_input', 'input/test_file.txt', count=4) - with open("stdout.txt", 'w') as f: - f.write(str(x)) - """) - - params = os.path.join(flow_dir, 'params.json') - with open(params, 'w') as f: - json.dump({ - "my_file_input": ["input/test_file.txt", "input/test_file.txt", "input/test_file.txt"] - }, f) - - os.system(f'cd "{flow_dir}" && {Env.ONECODE_DO_TYPECHECK}=1 python main.py {params}') - - with open(os.path.join(flow_dir, 'stdout.txt')) as f: - assert f.read() == f"['{flow_data}/input/test_file.txt'," \ - f" '{flow_data}/input/test_file.txt'," \ - f" '{flow_data}/input/test_file.txt']" - - shutil.rmtree(flow_dir) - - -@pytest.mark.emulations -def test_load_then_execute_multiple_values_multiple_count(): - flow_name, flow_folder, flow_id = _generate_flow_name() - - tmp = _clean_flow(flow_folder) - create(tmp, flow_name, cli=False) - - flow_dir = os.path.join(tmp, flow_folder) - flow_data = os.path.join(flow_dir, 'data') - _generate_csv_file(flow_dir, 'input/test_file_1.txt') - _generate_csv_file(flow_dir, 'input/test_file_2.txt') - - # count is ignored at loading & execution as it is only used in UI mode - with open(os.path.join(flow_dir, 'flows', f'{flow_id}.py'), 'a') as f: - f.write(""" - x = onecode.file_input( - 'my_file_input', - ['input/test_file_1.txt', 'input/test_file_2.txt'], - multiple=True, - count=3 - ) - with open("stdout.txt", 'w') as f: - f.write(str(x)) - """) - - params = os.path.join(flow_dir, 'params.json') - with open(params, 'w') as f: - json.dump({ - "my_file_input": [ - ['input/test_file_1.txt', 'input/test_file_2.txt'], - ['input/test_file_1.txt', 'input/test_file_2.txt'] - ] - }, - f - ) - - os.system(f'cd "{flow_dir}" && {Env.ONECODE_DO_TYPECHECK}=1 python main.py {params}') - - with open(os.path.join(flow_dir, 'stdout.txt')) as f: - assert f.read() == \ - f"[['{flow_data}/input/test_file_1.txt', '{flow_data}/input/test_file_2.txt']," \ - f" ['{flow_data}/input/test_file_1.txt', '{flow_data}/input/test_file_2.txt']]" - - shutil.rmtree(flow_dir) - - -@pytest.mark.emulations -def test_execute_file_not_found_single_value(): - flow_name, flow_folder, flow_id = _generate_flow_name() - - tmp = _clean_flow(flow_folder) - create(tmp, flow_name, cli=False) - - flow_dir = os.path.join(tmp, flow_folder) - flow_data = os.path.join(flow_dir, 'data') - - with open(os.path.join(flow_dir, 'flows', f'{flow_id}.py'), 'a') as f: - f.write(""" - try: - x = onecode.file_input('my_file_input', 'input/test_file.txt') - except FileNotFoundError as err: - with open("stdout.txt", 'w') as f: - f.write(str(err)) - """) - - os.system(f'cd "{flow_dir}" && {Env.ONECODE_DO_TYPECHECK}=1 python main.py') - - with open(os.path.join(flow_dir, 'stdout.txt')) as f: - assert f.read() == f"[my_file_input] File not found: {flow_data}/input/test_file.txt" - - shutil.rmtree(flow_dir) - - -@pytest.mark.emulations -def test_load_execute_file_not_found_single_value(): - flow_name, flow_folder, flow_id = _generate_flow_name() - - tmp = _clean_flow(flow_folder) - create(tmp, flow_name, cli=False) - - flow_dir = os.path.join(tmp, flow_folder) - flow_data = os.path.join(flow_dir, 'data') - _generate_csv_file(flow_dir, 'input/test_file.txt') - - with open(os.path.join(flow_dir, 'flows', f'{flow_id}.py'), 'a') as f: - f.write(""" - try: - x = onecode.file_input('my_file_input', 'input/test_file.txt') - except FileNotFoundError as err: - with open("stdout.txt", 'w') as f: - f.write(str(err)) - """) - - params = os.path.join(flow_dir, 'params.json') - with open(params, 'w') as f: - json.dump({"my_file_input": "input/test_file_1.txt"}, f) - - os.system(f'cd "{flow_dir}" && {Env.ONECODE_DO_TYPECHECK}=1 python main.py {params}') - - with open(os.path.join(flow_dir, 'stdout.txt')) as f: - assert f.read() == f"[my_file_input] File not found: {flow_data}/input/test_file_1.txt" - - shutil.rmtree(flow_dir) - - -@pytest.mark.emulations -def test_execute_not_a_file_single_value(): - flow_name, flow_folder, flow_id = _generate_flow_name() - - tmp = _clean_flow(flow_folder) - create(tmp, flow_name, cli=False) - - flow_dir = os.path.join(tmp, flow_folder) - flow_data = os.path.join(flow_dir, 'data') - os.makedirs(os.path.join(flow_data, 'input', 'test_file.txt')) - - with open(os.path.join(flow_dir, 'flows', f'{flow_id}.py'), 'a') as f: - f.write(""" - try: - x = onecode.file_input('my_file_input', 'input/test_file.txt') - except FileNotFoundError as err: - with open("stdout.txt", 'w') as f: - f.write(str(err)) - """) - - os.system(f'cd "{flow_dir}" && {Env.ONECODE_DO_TYPECHECK}=1 python main.py') - - with open(os.path.join(flow_dir, 'stdout.txt')) as f: - assert f.read() == f"[my_file_input] Path is not a file: {flow_data}/input/test_file.txt" - - shutil.rmtree(flow_dir) - - -@pytest.mark.emulations -def test_load_execute_not_a_file_single_value(): - flow_name, flow_folder, flow_id = _generate_flow_name() - - tmp = _clean_flow(flow_folder) - create(tmp, flow_name, cli=False) - - flow_dir = os.path.join(tmp, flow_folder) - flow_data = os.path.join(flow_dir, 'data') - _generate_csv_file(flow_dir, 'input/test_file.txt') - os.makedirs(os.path.join(flow_data, 'input', 'test_file_1.txt')) - - with open(os.path.join(flow_dir, 'flows', f'{flow_id}.py'), 'a') as f: - f.write(""" - try: - x = onecode.file_input('my_file_input', 'input/test_file.txt') - except FileNotFoundError as err: - with open("stdout.txt", 'w') as f: - f.write(str(err)) - """) - - params = os.path.join(flow_dir, 'params.json') - with open(params, 'w') as f: - json.dump({"my_file_input": "input/test_file_1.txt"}, f) - - os.system(f'cd "{flow_dir}" && {Env.ONECODE_DO_TYPECHECK}=1 python main.py {params}') - - with open(os.path.join(flow_dir, 'stdout.txt')) as f: - assert f.read() == f"[my_file_input] Path is not a file: {flow_data}/input/test_file_1.txt" - - shutil.rmtree(flow_dir) - - -@pytest.mark.emulations -def test_execute_file_not_found_multiple_value(): - flow_name, flow_folder, flow_id = _generate_flow_name() - - tmp = _clean_flow(flow_folder) - create(tmp, flow_name, cli=False) - - flow_dir = os.path.join(tmp, flow_folder) - flow_data = os.path.join(flow_dir, 'data') - _generate_csv_file(flow_dir, 'input/test_file_1.txt') - - with open(os.path.join(flow_dir, 'flows', f'{flow_id}.py'), 'a') as f: - f.write(""" - try: - x = onecode.file_input( - 'my_file_input', - ['input/test_file_1.txt', 'input/test_file_2.txt'], - multiple=True - ) - except FileNotFoundError as err: - with open("stdout.txt", 'w') as f: - f.write(str(err)) - """) - - os.system(f'cd "{flow_dir}" && {Env.ONECODE_DO_TYPECHECK}=1 python main.py') - - with open(os.path.join(flow_dir, 'stdout.txt')) as f: - assert f.read() == f"[my_file_input] File not found: {flow_data}/input/test_file_2.txt" - - shutil.rmtree(flow_dir) - - -@pytest.mark.emulations -def test_load_execute_file_not_found_multiple_value(): - flow_name, flow_folder, flow_id = _generate_flow_name() - - tmp = _clean_flow(flow_folder) - create(tmp, flow_name, cli=False) - - flow_dir = os.path.join(tmp, flow_folder) - flow_data = os.path.join(flow_dir, 'data') - _generate_csv_file(flow_dir, 'input/test_file_1.txt') - _generate_csv_file(flow_dir, 'input/test_file_2.txt') - - with open(os.path.join(flow_dir, 'flows', f'{flow_id}.py'), 'a') as f: - f.write(""" - try: - x = onecode.file_input( - 'my_file_input', - ['input/test_file_1.txt', 'input/test_file_2.txt'], - multiple=True - ) - except FileNotFoundError as err: - with open("stdout.txt", 'w') as f: - f.write(str(err)) - """) - - params = os.path.join(flow_dir, 'params.json') - with open(params, 'w') as f: - json.dump({"my_file_input": ['input/test_file_1.txt', 'input/test_file_3.txt']}, f) - - os.system(f'cd "{flow_dir}" && {Env.ONECODE_DO_TYPECHECK}=1 python main.py {params}') - - with open(os.path.join(flow_dir, 'stdout.txt')) as f: - assert f.read() == f"[my_file_input] File not found: {flow_data}/input/test_file_3.txt" - - shutil.rmtree(flow_dir) - - -@pytest.mark.emulations -def test_execute_not_a_file_multiple_value(): - flow_name, flow_folder, flow_id = _generate_flow_name() - - tmp = _clean_flow(flow_folder) - create(tmp, flow_name, cli=False) - - flow_dir = os.path.join(tmp, flow_folder) - flow_data = os.path.join(flow_dir, 'data') - _generate_csv_file(flow_dir, 'input/test_file_1.txt') - _generate_csv_file(flow_dir, 'input/test_file_2.txt') - os.makedirs(os.path.join(flow_data, 'input', 'test_file.txt')) - - with open(os.path.join(flow_dir, 'flows', f'{flow_id}.py'), 'a') as f: - f.write(""" - try: - x = onecode.file_input( - 'my_file_input', - ['input/test_file_1.txt', 'input/test_file.txt'], - multiple=True - ) - except FileNotFoundError as err: - with open("stdout.txt", 'w') as f: - f.write(str(err)) - """) - - os.system(f'cd "{flow_dir}" && {Env.ONECODE_DO_TYPECHECK}=1 python main.py') - - with open(os.path.join(flow_dir, 'stdout.txt')) as f: - assert f.read() == f"[my_file_input] Path is not a file: {flow_data}/input/test_file.txt" - - shutil.rmtree(flow_dir) - - -@pytest.mark.emulations -def test_load_execute_not_a_file_multiple_value(): - flow_name, flow_folder, flow_id = _generate_flow_name() - - tmp = _clean_flow(flow_folder) - create(tmp, flow_name, cli=False) - - flow_dir = os.path.join(tmp, flow_folder) - flow_data = os.path.join(flow_dir, 'data') - _generate_csv_file(flow_dir, 'input/test_file.txt') - os.makedirs(os.path.join(flow_data, 'input', 'test_file_1.txt')) - - with open(os.path.join(flow_dir, 'flows', f'{flow_id}.py'), 'a') as f: - f.write(""" - try: - x = onecode.file_input( - 'my_file_input', - ['input/test_file.txt'], - multiple=True - ) - except FileNotFoundError as err: - with open("stdout.txt", 'w') as f: - f.write(str(err)) - """) - - params = os.path.join(flow_dir, 'params.json') - with open(params, 'w') as f: - json.dump({"my_file_input": ['input/test_file.txt', 'input/test_file_1.txt']}, f) - - os.system(f'cd "{flow_dir}" && {Env.ONECODE_DO_TYPECHECK}=1 python main.py {params}') - - with open(os.path.join(flow_dir, 'stdout.txt')) as f: - assert f.read() == f"[my_file_input] Path is not a file: {flow_data}/input/test_file_1.txt" - - shutil.rmtree(flow_dir) - - -@pytest.mark.emulations -def test_load_execute_not_a_file_multiple_value_with_count(): - flow_name, flow_folder, flow_id = _generate_flow_name() - - tmp = _clean_flow(flow_folder) - create(tmp, flow_name, cli=False) - - flow_dir = os.path.join(tmp, flow_folder) - flow_data = os.path.join(flow_dir, 'data') - _generate_csv_file(flow_dir, 'input/test_file_1.txt') - os.makedirs(os.path.join(flow_data, 'input', 'test_file.txt')) - - # count is ignored at loading & execution as it is only used in UI mode - with open(os.path.join(flow_dir, 'flows', f'{flow_id}.py'), 'a') as f: - f.write(""" - try: - x = onecode.file_input( - 'my_file_input', - ['input/test_file_1.txt'], - multiple=True, - count=3 - ) - except FileNotFoundError as err: - with open("stdout.txt", 'w') as f: - f.write(str(err)) - """) - - params = os.path.join(flow_dir, 'params.json') - with open(params, 'w') as f: - json.dump({ - "my_file_input": [ - ['input/test_file_1.txt'], - ['input/test_file_1.txt'], - ['input/test_file_1.txt', 'input/test_file.txt'], - ]}, - f - ) - os.system(f'cd "{flow_dir}" && {Env.ONECODE_DO_TYPECHECK}=1 python main.py {params}') - - with open(os.path.join(flow_dir, 'stdout.txt')) as f: - assert f.read() == f"[my_file_input] Path is not a file: {flow_data}/input/test_file.txt" - - shutil.rmtree(flow_dir) - - -@pytest.mark.emulations -def test_execute_optional_value(): - flow_name, flow_folder, flow_id = _generate_flow_name() - - tmp = _clean_flow(flow_folder) - create(tmp, flow_name, cli=False) - - flow_dir = os.path.join(tmp, flow_folder) - - with open(os.path.join(flow_dir, 'flows', f'{flow_id}.py'), 'a') as f: - f.write(""" - x = onecode.file_input('my_file_input', None, optional=True) - with open("stdout.txt", 'w') as f: - f.write(str(x)) - """) - - os.system(f'cd "{flow_dir}" && {Env.ONECODE_DO_TYPECHECK}=1 python main.py') - - with open(os.path.join(flow_dir, 'stdout.txt')) as f: - assert f.read() == "None" - - shutil.rmtree(flow_dir) - - -@pytest.mark.emulations -def test_load_then_execute_optional_value(): - flow_name, flow_folder, flow_id = _generate_flow_name() - - tmp = _clean_flow(flow_folder) - create(tmp, flow_name, cli=False) - - flow_dir = os.path.join(tmp, flow_folder) - - with open(os.path.join(flow_dir, 'flows', f'{flow_id}.py'), 'a') as f: - f.write(""" - x = onecode.file_input('my_file_input', 'input/test_file.txt', optional=True) - with open("stdout.txt", 'w') as f: - f.write(str(x)) - """) - - params = os.path.join(flow_dir, 'params.json') - with open(params, 'w') as f: - json.dump({"my_file_input": None}, f) - - os.system(f'cd "{flow_dir}" && {Env.ONECODE_DO_TYPECHECK}=1 python main.py {params}') - - with open(os.path.join(flow_dir, 'stdout.txt')) as f: - assert f.read() == "None" - - shutil.rmtree(flow_dir) - - -@pytest.mark.emulations -def test_execute_invalid_optional_value(): - flow_name, flow_folder, flow_id = _generate_flow_name() - - tmp = _clean_flow(flow_folder) - create(tmp, flow_name, cli=False) - - flow_dir = os.path.join(tmp, flow_folder) - - with open(os.path.join(flow_dir, 'flows', f'{flow_id}.py'), 'a') as f: - f.write(""" - try: - x = onecode.file_input('my_file_input', None, optional=False) - except ValueError as err: - with open("stdout.txt", 'w') as f: - f.write(str(err)) - """) - - os.system(f'cd "{flow_dir}" && {Env.ONECODE_DO_TYPECHECK}=1 python main.py') - - with open(os.path.join(flow_dir, 'stdout.txt')) as f: - assert f.read() == "[my_file_input] Value is required: None provided" - - shutil.rmtree(flow_dir) - - -@pytest.mark.emulations -def test_load_then_execute_invalid_optional_value(): - flow_name, flow_folder, flow_id = _generate_flow_name() - - tmp = _clean_flow(flow_folder) - create(tmp, flow_name, cli=False) - - flow_dir = os.path.join(tmp, flow_folder) - - with open(os.path.join(flow_dir, 'flows', f'{flow_id}.py'), 'a') as f: - f.write(""" - try: - x = onecode.file_input('my_file_input', 'input/test_file.txt', optional=False) - except ValueError as err: - with open("stdout.txt", 'w') as f: - f.write(str(err)) - """) - - params = os.path.join(flow_dir, 'params.json') - with open(params, 'w') as f: - json.dump({"my_file_input": None}, f) - - os.system(f'cd "{flow_dir}" && {Env.ONECODE_DO_TYPECHECK}=1 python main.py {params}') - - with open(os.path.join(flow_dir, 'stdout.txt')) as f: - assert f.read() == "[my_file_input] Value is required: None provided" - - shutil.rmtree(flow_dir) - - -@pytest.mark.emulations -def test_invalid_name_key(): - flow_name, flow_folder, flow_id = _generate_flow_name() - - tmp = _clean_flow(flow_folder) - create(tmp, flow_name, cli=False) - - flow_dir = os.path.join(tmp, flow_folder) - - with open(os.path.join(flow_dir, 'flows', f'{flow_id}.py'), 'a') as f: - f.write(""" - try: - x = onecode.file_input('_my_file_input', 'input/test_file.txt') - except ValueError as err: - with open("stdout.txt", 'w') as f: - f.write(str(err)) - """) - - os.system(f'cd "{flow_dir}" && {Env.ONECODE_DO_TYPECHECK}=1 python main.py') - - with open(os.path.join(flow_dir, 'stdout.txt')) as f: - assert f.read() == 'Key starting with "_" are reserved: _my_file_input' - - shutil.rmtree(flow_dir) diff --git a/tests/emulations/elements/test_run_file_output.py b/tests/emulations/elements/test_run_file_output.py deleted file mode 100644 index 17a01ec..0000000 --- a/tests/emulations/elements/test_run_file_output.py +++ /dev/null @@ -1,70 +0,0 @@ -import json -import os -import shutil - -import pytest - -from onecode import Env -from onecode.cli.create import create -from tests.utils.flow_cli import _clean_flow, _generate_flow_name - - -@pytest.mark.emulations -def test_execute(): - flow_name, flow_folder, flow_id = _generate_flow_name() - - tmp = _clean_flow(flow_folder) - create(tmp, flow_name, cli=False) - - flow_dir = os.path.join(tmp, flow_folder) - flow_data = os.path.join(flow_dir, 'data') - - with open(os.path.join(flow_dir, 'flows', f'{flow_id}.py'), 'a') as f: - f.write(""" - x = onecode.file_output('My File Output', 'test_file.txt', label="My Label", tags=['Core']) - with open("stdout.txt", 'w') as f: - f.write(x) - """) - - os.system(f'cd "{flow_dir}" && {Env.ONECODE_DO_TYPECHECK}=1 python main.py') - - with open(os.path.join(flow_dir, 'stdout.txt')) as f: - assert f.read() == f"{flow_data}/outputs/test_file.txt" - - with open(os.path.join(flow_data, "outputs", flow_id, "MANIFEST.txt")) as f: - assert json.loads(f.read()) == { - "key": "my_file_output", - "label": "My Label", - "value": f"{flow_data}/outputs/test_file.txt", - "mimetype": "text/plain", - "tags": ["Core"], - "kind": "FileOutput" - } - - shutil.rmtree(flow_dir) - - -@pytest.mark.emulations -def test_invalid_name_key(): - flow_name, flow_folder, flow_id = _generate_flow_name() - - tmp = _clean_flow(flow_folder) - create(tmp, flow_name, cli=False) - - flow_dir = os.path.join(tmp, flow_folder) - - with open(os.path.join(flow_dir, 'flows', f'{flow_id}.py'), 'a') as f: - f.write(""" - try: - x = onecode.file_output('_my_file_output', 'test_file.txt', tags=['Core']) - except ValueError as err: - with open("stdout.txt", 'w') as f: - f.write(str(err)) - """) - - os.system(f'cd "{flow_dir}" && {Env.ONECODE_DO_TYPECHECK}=1 python main.py') - - with open(os.path.join(flow_dir, 'stdout.txt')) as f: - assert f.read() == 'Key starting with "_" are reserved: _my_file_output' - - shutil.rmtree(flow_dir) diff --git a/tests/emulations/elements/test_run_number_input.py b/tests/emulations/elements/test_run_number_input.py deleted file mode 100644 index 77bcf34..0000000 --- a/tests/emulations/elements/test_run_number_input.py +++ /dev/null @@ -1,475 +0,0 @@ -import json -import os -import shutil - -import pytest - -from onecode import Env -from onecode.cli.create import create -from tests.utils.flow_cli import _clean_flow, _generate_flow_name - - -@pytest.mark.emulations -def test_execute_single_count(): - flow_name, flow_folder, flow_id = _generate_flow_name() - - tmp = _clean_flow(flow_folder) - create(tmp, flow_name, cli=False) - - flow_dir = os.path.join(tmp, flow_folder) - - with open(os.path.join(flow_dir, 'flows', f'{flow_id}.py'), 'a') as f: - f.write(""" - x = onecode.number_input('my_number_input', 0.5) - with open("stdout.txt", 'w') as f: - f.write(str(x)) - """) - - os.system(f'cd "{flow_dir}" && {Env.ONECODE_DO_TYPECHECK}=1 python main.py') - - with open(os.path.join(flow_dir, 'stdout.txt')) as f: - assert f.read() == "0.5" - - shutil.rmtree(flow_dir) - - -@pytest.mark.emulations -def test_execute_multiple_count(): - flow_name, flow_folder, flow_id = _generate_flow_name() - - tmp = _clean_flow(flow_folder) - create(tmp, flow_name, cli=False) - - flow_dir = os.path.join(tmp, flow_folder) - - # count is ignored at loading & execution as it is only used in UI mode - with open(os.path.join(flow_dir, 'flows', f'{flow_id}.py'), 'a') as f: - f.write(""" - x = onecode.number_input('my_number_input', [0.5, 0.5], count=3) - with open("stdout.txt", 'w') as f: - f.write(str(x)) - """) - - os.system(f'cd "{flow_dir}" && {Env.ONECODE_DO_TYPECHECK}=1 python main.py') - - with open(os.path.join(flow_dir, 'stdout.txt')) as f: - assert f.read() == "[0.5, 0.5]" - - shutil.rmtree(flow_dir) - - -@pytest.mark.emulations -def test_load_then_execute_single_count(): - flow_name, flow_folder, flow_id = _generate_flow_name() - - tmp = _clean_flow(flow_folder) - create(tmp, flow_name, cli=False) - - flow_dir = os.path.join(tmp, flow_folder) - - with open(os.path.join(flow_dir, 'flows', f'{flow_id}.py'), 'a') as f: - f.write(""" - x = onecode.number_input('my_number_input', 0.5) - with open("stdout.txt", 'w') as f: - f.write(str(x)) - """) - - params = os.path.join(flow_dir, 'params.json') - with open(params, 'w') as f: - json.dump({"my_number_input": 0.6}, f) - - os.system(f'cd "{flow_dir}" && {Env.ONECODE_DO_TYPECHECK}=1 python main.py {params}') - - with open(os.path.join(flow_dir, 'stdout.txt')) as f: - assert f.read() == "0.6" - - shutil.rmtree(flow_dir) - - -@pytest.mark.emulations -def test_load_then_execute_multiple_count(): - flow_name, flow_folder, flow_id = _generate_flow_name() - - tmp = _clean_flow(flow_folder) - create(tmp, flow_name, cli=False) - - flow_dir = os.path.join(tmp, flow_folder) - - # count is ignored at loading & execution as it is only used in UI mode - with open(os.path.join(flow_dir, 'flows', f'{flow_id}.py'), 'a') as f: - f.write(""" - x = onecode.number_input('my_number_input', 0.5, count=4) - with open("stdout.txt", 'w') as f: - f.write(str(x)) - """) - - params = os.path.join(flow_dir, 'params.json') - with open(params, 'w') as f: - json.dump({"my_number_input": [0.6, 0.4, 0.3]}, f) - - os.system(f'cd "{flow_dir}" && {Env.ONECODE_DO_TYPECHECK}=1 python main.py {params}') - - with open(os.path.join(flow_dir, 'stdout.txt')) as f: - assert f.read() == "[0.6, 0.4, 0.3]" - - shutil.rmtree(flow_dir) - - -@pytest.mark.emulations -def test_execute_invalid_bounds(): - flow_name, flow_folder, flow_id = _generate_flow_name() - - tmp = _clean_flow(flow_folder) - create(tmp, flow_name, cli=False) - - flow_dir = os.path.join(tmp, flow_folder) - - with open(os.path.join(flow_dir, 'flows', f'{flow_id}.py'), 'a') as f: - f.write(""" - try: - x = onecode.number_input('my_number_input', 0.5, min=0.8, max=0.4) - except ValueError as err: - with open("stdout.txt", 'w') as f: - f.write(str(err)) - """) - - os.system(f'cd "{flow_dir}" && {Env.ONECODE_DO_TYPECHECK}=1 python main.py') - - with open(os.path.join(flow_dir, 'stdout.txt')) as f: - assert f.read() == "[my_number_input] Minimum cannot be greater than maximum: 0.8 > 0.4" - - shutil.rmtree(flow_dir) - - -@pytest.mark.emulations -def test_load_then_execute_invalid_bounds(): - flow_name, flow_folder, flow_id = _generate_flow_name() - - tmp = _clean_flow(flow_folder) - create(tmp, flow_name, cli=False) - - flow_dir = os.path.join(tmp, flow_folder) - - with open(os.path.join(flow_dir, 'flows', f'{flow_id}.py'), 'a') as f: - f.write(""" - try: - x = onecode.number_input('my_number_input', 0.5, min=0.8, max=0.4) - except ValueError as err: - with open("stdout.txt", 'w') as f: - f.write(str(err)) - """) - - params = os.path.join(flow_dir, 'params.json') - with open(params, 'w') as f: - json.dump({"my_number_input": 0.5}, f) - - os.system(f'cd "{flow_dir}" && {Env.ONECODE_DO_TYPECHECK}=1 python main.py {params}') - - with open(os.path.join(flow_dir, 'stdout.txt')) as f: - assert f.read() == "[my_number_input] Minimum cannot be greater than maximum: 0.8 > 0.4" - - shutil.rmtree(flow_dir) - - -@pytest.mark.emulations -def test_execute_invalid_min(): - flow_name, flow_folder, flow_id = _generate_flow_name() - - tmp = _clean_flow(flow_folder) - create(tmp, flow_name, cli=False) - - flow_dir = os.path.join(tmp, flow_folder) - - with open(os.path.join(flow_dir, 'flows', f'{flow_id}.py'), 'a') as f: - f.write(""" - try: - x = onecode.number_input('my_number_input', 0.5, min=0.51, max=0.8) - except ValueError as err: - with open("stdout.txt", 'w') as f: - f.write(str(err)) - """) - - os.system(f'cd "{flow_dir}" && {Env.ONECODE_DO_TYPECHECK}=1 python main.py') - - with open(os.path.join(flow_dir, 'stdout.txt')) as f: - assert f.read() == "[my_number_input] Value lower than minimum: 0.5 < 0.51" - - shutil.rmtree(flow_dir) - - -@pytest.mark.emulations -def test_load_then_execute_invalid_min(): - flow_name, flow_folder, flow_id = _generate_flow_name() - - tmp = _clean_flow(flow_folder) - create(tmp, flow_name, cli=False) - - flow_dir = os.path.join(tmp, flow_folder) - - with open(os.path.join(flow_dir, 'flows', f'{flow_id}.py'), 'a') as f: - f.write(""" - try: - x = onecode.number_input('my_number_input', 0.6, min=0.51, max=0.8) - except ValueError as err: - with open("stdout.txt", 'w') as f: - f.write(str(err)) - """) - - params = os.path.join(flow_dir, 'params.json') - with open(params, 'w') as f: - json.dump({"my_number_input": 0.5}, f) - - os.system(f'cd "{flow_dir}" && {Env.ONECODE_DO_TYPECHECK}=1 python main.py {params}') - - with open(os.path.join(flow_dir, 'stdout.txt')) as f: - assert f.read() == "[my_number_input] Value lower than minimum: 0.5 < 0.51" - - shutil.rmtree(flow_dir) - - -@pytest.mark.emulations -def test_execute_invalid_max(): - flow_name, flow_folder, flow_id = _generate_flow_name() - - tmp = _clean_flow(flow_folder) - create(tmp, flow_name, cli=False) - - flow_dir = os.path.join(tmp, flow_folder) - - with open(os.path.join(flow_dir, 'flows', f'{flow_id}.py'), 'a') as f: - f.write(""" - try: - x = onecode.number_input('my_number_input', 0.5, min=0.4, max=0.49) - except ValueError as err: - with open("stdout.txt", 'w') as f: - f.write(str(err)) - """) - - os.system(f'cd "{flow_dir}" && {Env.ONECODE_DO_TYPECHECK}=1 python main.py') - - with open(os.path.join(flow_dir, 'stdout.txt')) as f: - assert f.read() == "[my_number_input] Value greater than maximum: 0.5 > 0.49" - - shutil.rmtree(flow_dir) - - -@pytest.mark.emulations -def test_load_then_execute_invalid_max(): - flow_name, flow_folder, flow_id = _generate_flow_name() - - tmp = _clean_flow(flow_folder) - create(tmp, flow_name, cli=False) - - flow_dir = os.path.join(tmp, flow_folder) - - with open(os.path.join(flow_dir, 'flows', f'{flow_id}.py'), 'a') as f: - f.write(""" - try: - x = onecode.number_input('my_number_input', 0.45, min=0.4, max=0.49) - except ValueError as err: - with open("stdout.txt", 'w') as f: - f.write(str(err)) - """) - - params = os.path.join(flow_dir, 'params.json') - with open(params, 'w') as f: - json.dump({"my_number_input": 0.5}, f) - - os.system(f'cd "{flow_dir}" && {Env.ONECODE_DO_TYPECHECK}=1 python main.py {params}') - - with open(os.path.join(flow_dir, 'stdout.txt')) as f: - assert f.read() == "[my_number_input] Value greater than maximum: 0.5 > 0.49" - - shutil.rmtree(flow_dir) - - -@pytest.mark.emulations -def test_execute_invalid_with_count(): - flow_name, flow_folder, flow_id = _generate_flow_name() - - tmp = _clean_flow(flow_folder) - create(tmp, flow_name, cli=False) - - flow_dir = os.path.join(tmp, flow_folder) - - # count is ignored at loading & execution as it is only used in UI mode - with open(os.path.join(flow_dir, 'flows', f'{flow_id}.py'), 'a') as f: - f.write(""" - try: - x = onecode.number_input('my_number_input', [0.5, 0.45, 0.45], min=0.4, max=0.49, count=3) - except ValueError as err: - with open("stdout.txt", 'w') as f: - f.write(str(err)) - """) - - os.system(f'cd "{flow_dir}" && {Env.ONECODE_DO_TYPECHECK}=1 python main.py') - - with open(os.path.join(flow_dir, 'stdout.txt')) as f: - assert f.read() == "[my_number_input] Value greater than maximum: 0.5 > 0.49" - - shutil.rmtree(flow_dir) - - -@pytest.mark.emulations -def test_load_then_execute_invalid_with_count(): - flow_name, flow_folder, flow_id = _generate_flow_name() - - tmp = _clean_flow(flow_folder) - create(tmp, flow_name, cli=False) - - flow_dir = os.path.join(tmp, flow_folder) - - # count is ignored at loading & execution as it is only used in UI mode - with open(os.path.join(flow_dir, 'flows', f'{flow_id}.py'), 'a') as f: - f.write(""" - try: - x = onecode.number_input('my_number_input', 0.45, min=0.4, max=0.49, count=4) - except ValueError as err: - with open("stdout.txt", 'w') as f: - f.write(str(err)) - """) - - params = os.path.join(flow_dir, 'params.json') - with open(params, 'w') as f: - json.dump({"my_number_input": [0.45, 0.5, 0.45]}, f) - - os.system(f'cd "{flow_dir}" && {Env.ONECODE_DO_TYPECHECK}=1 python main.py {params}') - - with open(os.path.join(flow_dir, 'stdout.txt')) as f: - assert f.read() == "[my_number_input] Value greater than maximum: 0.5 > 0.49" - - shutil.rmtree(flow_dir) - - -@pytest.mark.emulations -def test_execute_optional_value(): - flow_name, flow_folder, flow_id = _generate_flow_name() - - tmp = _clean_flow(flow_folder) - create(tmp, flow_name, cli=False) - - flow_dir = os.path.join(tmp, flow_folder) - - with open(os.path.join(flow_dir, 'flows', f'{flow_id}.py'), 'a') as f: - f.write(""" - x = onecode.number_input('my_number_input', None, optional=True) - with open("stdout.txt", 'w') as f: - f.write(str(x)) - """) - - os.system(f'cd "{flow_dir}" && {Env.ONECODE_DO_TYPECHECK}=1 python main.py') - - with open(os.path.join(flow_dir, 'stdout.txt')) as f: - assert f.read() == "None" - - shutil.rmtree(flow_dir) - - -@pytest.mark.emulations -def test_load_then_execute_optional_value(): - flow_name, flow_folder, flow_id = _generate_flow_name() - - tmp = _clean_flow(flow_folder) - create(tmp, flow_name, cli=False) - - flow_dir = os.path.join(tmp, flow_folder) - - with open(os.path.join(flow_dir, 'flows', f'{flow_id}.py'), 'a') as f: - f.write(""" - x = onecode.number_input('my_number_input', 0.5, optional=True) - with open("stdout.txt", 'w') as f: - f.write(str(x)) - """) - - params = os.path.join(flow_dir, 'params.json') - with open(params, 'w') as f: - json.dump({"my_number_input": None}, f) - - os.system(f'cd "{flow_dir}" && {Env.ONECODE_DO_TYPECHECK}=1 python main.py {params}') - - with open(os.path.join(flow_dir, 'stdout.txt')) as f: - assert f.read() == "None" - - shutil.rmtree(flow_dir) - - -@pytest.mark.emulations -def test_execute_invalid_optional_value(): - flow_name, flow_folder, flow_id = _generate_flow_name() - - tmp = _clean_flow(flow_folder) - create(tmp, flow_name, cli=False) - - flow_dir = os.path.join(tmp, flow_folder) - - with open(os.path.join(flow_dir, 'flows', f'{flow_id}.py'), 'a') as f: - f.write(""" - try: - x = onecode.number_input('my_number_input', None, optional=False) - except ValueError as err: - with open("stdout.txt", 'w') as f: - f.write(str(err)) - """) - - os.system(f'cd "{flow_dir}" && {Env.ONECODE_DO_TYPECHECK}=1 python main.py') - - with open(os.path.join(flow_dir, 'stdout.txt')) as f: - assert f.read() == "[my_number_input] Value is required: None provided" - - shutil.rmtree(flow_dir) - - -@pytest.mark.emulations -def test_load_then_execute_invalid_optional_value(): - flow_name, flow_folder, flow_id = _generate_flow_name() - - tmp = _clean_flow(flow_folder) - create(tmp, flow_name, cli=False) - - flow_dir = os.path.join(tmp, flow_folder) - - with open(os.path.join(flow_dir, 'flows', f'{flow_id}.py'), 'a') as f: - f.write(""" - try: - x = onecode.number_input('my_number_input', 0.5, optional=False) - except ValueError as err: - with open("stdout.txt", 'w') as f: - f.write(str(err)) - """) - - params = os.path.join(flow_dir, 'params.json') - with open(params, 'w') as f: - json.dump({"my_number_input": None}, f) - - os.system(f'cd "{flow_dir}" && {Env.ONECODE_DO_TYPECHECK}=1 python main.py {params}') - - with open(os.path.join(flow_dir, 'stdout.txt')) as f: - assert f.read() == "[my_number_input] Value is required: None provided" - - shutil.rmtree(flow_dir) - - -@pytest.mark.emulations -def test_invalid_name_key(): - flow_name, flow_folder, flow_id = _generate_flow_name() - - tmp = _clean_flow(flow_folder) - create(tmp, flow_name, cli=False) - - flow_dir = os.path.join(tmp, flow_folder) - - with open(os.path.join(flow_dir, 'flows', f'{flow_id}.py'), 'a') as f: - f.write(""" - try: - x = onecode.number_input('_my_number_input', 0.5) - except ValueError as err: - with open("stdout.txt", 'w') as f: - f.write(str(err)) - """) - - os.system(f'cd "{flow_dir}" && {Env.ONECODE_DO_TYPECHECK}=1 python main.py') - - with open(os.path.join(flow_dir, 'stdout.txt')) as f: - assert f.read() == 'Key starting with "_" are reserved: _my_number_input' - - shutil.rmtree(flow_dir) diff --git a/tests/emulations/elements/test_run_radio_button.py b/tests/emulations/elements/test_run_radio_button.py deleted file mode 100644 index b0ad69e..0000000 --- a/tests/emulations/elements/test_run_radio_button.py +++ /dev/null @@ -1,305 +0,0 @@ -import json -import os -import shutil - -import pytest - -from onecode import Env -from onecode.cli.create import create -from tests.utils.flow_cli import _clean_flow, _generate_flow_name - - -@pytest.mark.emulations -def test_execute_single_count(): - flow_name, flow_folder, flow_id = _generate_flow_name() - - tmp = _clean_flow(flow_folder) - create(tmp, flow_name, cli=False) - - flow_dir = os.path.join(tmp, flow_folder) - - with open(os.path.join(flow_dir, 'flows', f'{flow_id}.py'), 'a') as f: - f.write(""" - x = onecode.radio_button('my_radio', 'B', options=['A', 'B', 'C']) - with open("stdout.txt", 'w') as f: - f.write(x) - """) - - os.system(f'cd "{flow_dir}" && {Env.ONECODE_DO_TYPECHECK}=1 python main.py') - - with open(os.path.join(flow_dir, 'stdout.txt')) as f: - assert f.read() == "B" - - shutil.rmtree(flow_dir) - - -@pytest.mark.emulations -def test_execute_multiple_count(): - flow_name, flow_folder, flow_id = _generate_flow_name() - - tmp = _clean_flow(flow_folder) - create(tmp, flow_name, cli=False) - - flow_dir = os.path.join(tmp, flow_folder) - - # count is ignored at loading & execution as it is only used in UI mode - with open(os.path.join(flow_dir, 'flows', f'{flow_id}.py'), 'a') as f: - f.write(""" - x = onecode.radio_button('my_radio', ['B', 'B'], options=['A', 'B', 'C'], count=3) - with open("stdout.txt", 'w') as f: - f.write(str(x)) - """) - - os.system(f'cd "{flow_dir}" && {Env.ONECODE_DO_TYPECHECK}=1 python main.py') - - with open(os.path.join(flow_dir, 'stdout.txt')) as f: - assert f.read() == "['B', 'B']" - - shutil.rmtree(flow_dir) - - -@pytest.mark.emulations -def test_load_then_execute_single_count(): - flow_name, flow_folder, flow_id = _generate_flow_name() - - tmp = _clean_flow(flow_folder) - create(tmp, flow_name, cli=False) - - flow_dir = os.path.join(tmp, flow_folder) - - with open(os.path.join(flow_dir, 'flows', f'{flow_id}.py'), 'a') as f: - f.write(""" - x = onecode.radio_button('my_radio', 'B', options=['A', 'B', 'C']) - with open("stdout.txt", 'w') as f: - f.write(x) - """) - - params = os.path.join(flow_dir, 'params.json') - with open(params, 'w') as f: - json.dump({"my_radio": "B"}, f) - - os.system(f'cd "{flow_dir}" && {Env.ONECODE_DO_TYPECHECK}=1 python main.py {params}') - - with open(os.path.join(flow_dir, 'stdout.txt')) as f: - assert f.read() == "B" - - shutil.rmtree(flow_dir) - - -@pytest.mark.emulations -def test_load_then_execute_multiple_count(): - flow_name, flow_folder, flow_id = _generate_flow_name() - - tmp = _clean_flow(flow_folder) - create(tmp, flow_name, cli=False) - - flow_dir = os.path.join(tmp, flow_folder) - - # count is ignored at loading & execution as it is only used in UI mode - with open(os.path.join(flow_dir, 'flows', f'{flow_id}.py'), 'a') as f: - f.write(""" - x = onecode.radio_button('my_radio', 'B', options=['A', 'B', 'C'], count=4) - with open("stdout.txt", 'w') as f: - f.write(str(x)) - """) - - params = os.path.join(flow_dir, 'params.json') - with open(params, 'w') as f: - json.dump({"my_radio": ["B", "B", "B"]}, f) - - os.system(f'cd "{flow_dir}" && {Env.ONECODE_DO_TYPECHECK}=1 python main.py {params}') - - with open(os.path.join(flow_dir, 'stdout.txt')) as f: - assert f.read() == "['B', 'B', 'B']" - - shutil.rmtree(flow_dir) - - -@pytest.mark.emulations -def test_execute_invalid(): - flow_name, flow_folder, flow_id = _generate_flow_name() - - tmp = _clean_flow(flow_folder) - create(tmp, flow_name, cli=False) - - flow_dir = os.path.join(tmp, flow_folder) - - with open(os.path.join(flow_dir, 'flows', f'{flow_id}.py'), 'a') as f: - f.write(""" - try: - x = onecode.radio_button('my_radio', 'D', options=['A', 'B', 'C']) - except ValueError as err: - with open("stdout.txt", 'w') as f: - f.write(str(err)) - """) - - os.system(f'cd "{flow_dir}" && {Env.ONECODE_DO_TYPECHECK}=1 python main.py') - - with open(os.path.join(flow_dir, 'stdout.txt')) as f: - assert f.read() == "[my_radio] Not a valid choice: D" - - shutil.rmtree(flow_dir) - - -@pytest.mark.emulations -def test_load_then_execute_invalid(): - flow_name, flow_folder, flow_id = _generate_flow_name() - - tmp = _clean_flow(flow_folder) - create(tmp, flow_name, cli=False) - - flow_dir = os.path.join(tmp, flow_folder) - - with open(os.path.join(flow_dir, 'flows', f'{flow_id}.py'), 'a') as f: - f.write(""" - try: - x = onecode.radio_button('my_radio', 'B', options=['A', 'B', 'C']) - except ValueError as err: - with open("stdout.txt", 'w') as f: - f.write(str(err)) - """) - - params = os.path.join(flow_dir, 'params.json') - with open(params, 'w') as f: - json.dump({"my_radio": "D"}, f) - - os.system(f'cd "{flow_dir}" && {Env.ONECODE_DO_TYPECHECK}=1 python main.py {params}') - - with open(os.path.join(flow_dir, 'stdout.txt')) as f: - assert f.read() == "[my_radio] Not a valid choice: D" - - shutil.rmtree(flow_dir) - - -@pytest.mark.emulations -def test_execute_optional_value(): - flow_name, flow_folder, flow_id = _generate_flow_name() - - tmp = _clean_flow(flow_folder) - create(tmp, flow_name, cli=False) - - flow_dir = os.path.join(tmp, flow_folder) - - with open(os.path.join(flow_dir, 'flows', f'{flow_id}.py'), 'a') as f: - f.write(""" - x = onecode.radio_button('my_radio', None, options=['A', 'B', 'C'], optional=True) - with open("stdout.txt", 'w') as f: - f.write(str(x)) - """) - - os.system(f'cd "{flow_dir}" && {Env.ONECODE_DO_TYPECHECK}=1 python main.py') - - with open(os.path.join(flow_dir, 'stdout.txt')) as f: - assert f.read() == "None" - - shutil.rmtree(flow_dir) - - -@pytest.mark.emulations -def test_load_then_execute_optional_value(): - flow_name, flow_folder, flow_id = _generate_flow_name() - - tmp = _clean_flow(flow_folder) - create(tmp, flow_name, cli=False) - - flow_dir = os.path.join(tmp, flow_folder) - - with open(os.path.join(flow_dir, 'flows', f'{flow_id}.py'), 'a') as f: - f.write(""" - x = onecode.radio_button('my_radio', 'B', options=['A', 'B', 'C'], optional=True) - with open("stdout.txt", 'w') as f: - f.write(str(x)) - """) - - params = os.path.join(flow_dir, 'params.json') - with open(params, 'w') as f: - json.dump({"my_radio": None}, f) - - os.system(f'cd "{flow_dir}" && {Env.ONECODE_DO_TYPECHECK}=1 python main.py {params}') - - with open(os.path.join(flow_dir, 'stdout.txt')) as f: - assert f.read() == "None" - - shutil.rmtree(flow_dir) - - -@pytest.mark.emulations -def test_execute_invalid_optional_value(): - flow_name, flow_folder, flow_id = _generate_flow_name() - - tmp = _clean_flow(flow_folder) - create(tmp, flow_name, cli=False) - - flow_dir = os.path.join(tmp, flow_folder) - - with open(os.path.join(flow_dir, 'flows', f'{flow_id}.py'), 'a') as f: - f.write(""" - try: - x = onecode.radio_button('my_radio', None, options=['A', 'B', 'C'], optional=False) - except ValueError as err: - with open("stdout.txt", 'w') as f: - f.write(str(err)) - """) - - os.system(f'cd "{flow_dir}" && {Env.ONECODE_DO_TYPECHECK}=1 python main.py') - - with open(os.path.join(flow_dir, 'stdout.txt')) as f: - assert f.read() == "[my_radio] Value is required: None provided" - - shutil.rmtree(flow_dir) - - -@pytest.mark.emulations -def test_load_then_execute_invalid_optional_value(): - flow_name, flow_folder, flow_id = _generate_flow_name() - - tmp = _clean_flow(flow_folder) - create(tmp, flow_name, cli=False) - - flow_dir = os.path.join(tmp, flow_folder) - - with open(os.path.join(flow_dir, 'flows', f'{flow_id}.py'), 'a') as f: - f.write(""" - try: - x = onecode.radio_button('my_radio', 'B', options=['A', 'B', 'C'], optional=False) - except ValueError as err: - with open("stdout.txt", 'w') as f: - f.write(str(err)) - """) - - params = os.path.join(flow_dir, 'params.json') - with open(params, 'w') as f: - json.dump({"my_radio": None}, f) - - os.system(f'cd "{flow_dir}" && {Env.ONECODE_DO_TYPECHECK}=1 python main.py {params}') - - with open(os.path.join(flow_dir, 'stdout.txt')) as f: - assert f.read() == "[my_radio] Value is required: None provided" - - shutil.rmtree(flow_dir) - - -@pytest.mark.emulations -def test_invalid_name_key(): - flow_name, flow_folder, flow_id = _generate_flow_name() - - tmp = _clean_flow(flow_folder) - create(tmp, flow_name, cli=False) - - flow_dir = os.path.join(tmp, flow_folder) - - with open(os.path.join(flow_dir, 'flows', f'{flow_id}.py'), 'a') as f: - f.write(""" - try: - x = onecode.radio_button('_my_radio', 'B', options=['A', 'B', 'C']) - except ValueError as err: - with open("stdout.txt", 'w') as f: - f.write(str(err)) - """) - - os.system(f'cd "{flow_dir}" && {Env.ONECODE_DO_TYPECHECK}=1 python main.py') - - with open(os.path.join(flow_dir, 'stdout.txt')) as f: - assert f.read() == 'Key starting with "_" are reserved: _my_radio' - - shutil.rmtree(flow_dir) diff --git a/tests/emulations/elements/test_run_slider.py b/tests/emulations/elements/test_run_slider.py deleted file mode 100644 index 68f426c..0000000 --- a/tests/emulations/elements/test_run_slider.py +++ /dev/null @@ -1,475 +0,0 @@ -import json -import os -import shutil - -import pytest - -from onecode import Env -from onecode.cli.create import create -from tests.utils.flow_cli import _clean_flow, _generate_flow_name - - -@pytest.mark.emulations -def test_execute_single_count(): - flow_name, flow_folder, flow_id = _generate_flow_name() - - tmp = _clean_flow(flow_folder) - create(tmp, flow_name, cli=False) - - flow_dir = os.path.join(tmp, flow_folder) - - with open(os.path.join(flow_dir, 'flows', f'{flow_id}.py'), 'a') as f: - f.write(""" - x = onecode.slider('my_slider', 0.5) - with open("stdout.txt", 'w') as f: - f.write(str(x)) - """) - - os.system(f'cd "{flow_dir}" && {Env.ONECODE_DO_TYPECHECK}=1 python main.py') - - with open(os.path.join(flow_dir, 'stdout.txt')) as f: - assert f.read() == "0.5" - - shutil.rmtree(flow_dir) - - -@pytest.mark.emulations -def test_execute_multiple_count(): - flow_name, flow_folder, flow_id = _generate_flow_name() - - tmp = _clean_flow(flow_folder) - create(tmp, flow_name, cli=False) - - flow_dir = os.path.join(tmp, flow_folder) - - # count is ignored at loading & execution as it is only used in UI mode - with open(os.path.join(flow_dir, 'flows', f'{flow_id}.py'), 'a') as f: - f.write(""" - x = onecode.slider('my_slider', [0.5, 0.5], count=3) - with open("stdout.txt", 'w') as f: - f.write(str(x)) - """) - - os.system(f'cd "{flow_dir}" && {Env.ONECODE_DO_TYPECHECK}=1 python main.py') - - with open(os.path.join(flow_dir, 'stdout.txt')) as f: - assert f.read() == "[0.5, 0.5]" - - shutil.rmtree(flow_dir) - - -@pytest.mark.emulations -def test_load_then_execute_single_count(): - flow_name, flow_folder, flow_id = _generate_flow_name() - - tmp = _clean_flow(flow_folder) - create(tmp, flow_name, cli=False) - - flow_dir = os.path.join(tmp, flow_folder) - - with open(os.path.join(flow_dir, 'flows', f'{flow_id}.py'), 'a') as f: - f.write(""" - x = onecode.slider('my_slider', 0.5) - with open("stdout.txt", 'w') as f: - f.write(str(x)) - """) - - params = os.path.join(flow_dir, 'params.json') - with open(params, 'w') as f: - json.dump({"my_slider": 0.6}, f) - - os.system(f'cd "{flow_dir}" && {Env.ONECODE_DO_TYPECHECK}=1 python main.py {params}') - - with open(os.path.join(flow_dir, 'stdout.txt')) as f: - assert f.read() == "0.6" - - shutil.rmtree(flow_dir) - - -@pytest.mark.emulations -def test_load_then_execute_multiple_count(): - flow_name, flow_folder, flow_id = _generate_flow_name() - - tmp = _clean_flow(flow_folder) - create(tmp, flow_name, cli=False) - - flow_dir = os.path.join(tmp, flow_folder) - - # count is ignored at loading & execution as it is only used in UI mode - with open(os.path.join(flow_dir, 'flows', f'{flow_id}.py'), 'a') as f: - f.write(""" - x = onecode.slider('my_slider', 0.5, count=4) - with open("stdout.txt", 'w') as f: - f.write(str(x)) - """) - - params = os.path.join(flow_dir, 'params.json') - with open(params, 'w') as f: - json.dump({"my_slider": [0.6, 0.4, 0.3]}, f) - - os.system(f'cd "{flow_dir}" && {Env.ONECODE_DO_TYPECHECK}=1 python main.py {params}') - - with open(os.path.join(flow_dir, 'stdout.txt')) as f: - assert f.read() == "[0.6, 0.4, 0.3]" - - shutil.rmtree(flow_dir) - - -@pytest.mark.emulations -def test_execute_invalid_bounds(): - flow_name, flow_folder, flow_id = _generate_flow_name() - - tmp = _clean_flow(flow_folder) - create(tmp, flow_name, cli=False) - - flow_dir = os.path.join(tmp, flow_folder) - - with open(os.path.join(flow_dir, 'flows', f'{flow_id}.py'), 'a') as f: - f.write(""" - try: - x = onecode.slider('my_slider', 0.5, min=0.8, max=0.4) - except ValueError as err: - with open("stdout.txt", 'w') as f: - f.write(str(err)) - """) - - os.system(f'cd "{flow_dir}" && {Env.ONECODE_DO_TYPECHECK}=1 python main.py') - - with open(os.path.join(flow_dir, 'stdout.txt')) as f: - assert f.read() == "[my_slider] Minimum cannot be greater than maximum: 0.8 > 0.4" - - shutil.rmtree(flow_dir) - - -@pytest.mark.emulations -def test_load_then_execute_invalid_bounds(): - flow_name, flow_folder, flow_id = _generate_flow_name() - - tmp = _clean_flow(flow_folder) - create(tmp, flow_name, cli=False) - - flow_dir = os.path.join(tmp, flow_folder) - - with open(os.path.join(flow_dir, 'flows', f'{flow_id}.py'), 'a') as f: - f.write(""" - try: - x = onecode.slider('my_slider', 0.5, min=0.8, max=0.4) - except ValueError as err: - with open("stdout.txt", 'w') as f: - f.write(str(err)) - """) - - params = os.path.join(flow_dir, 'params.json') - with open(params, 'w') as f: - json.dump({"my_slider": 0.5}, f) - - os.system(f'cd "{flow_dir}" && {Env.ONECODE_DO_TYPECHECK}=1 python main.py {params}') - - with open(os.path.join(flow_dir, 'stdout.txt')) as f: - assert f.read() == "[my_slider] Minimum cannot be greater than maximum: 0.8 > 0.4" - - shutil.rmtree(flow_dir) - - -@pytest.mark.emulations -def test_execute_invalid_min(): - flow_name, flow_folder, flow_id = _generate_flow_name() - - tmp = _clean_flow(flow_folder) - create(tmp, flow_name, cli=False) - - flow_dir = os.path.join(tmp, flow_folder) - - with open(os.path.join(flow_dir, 'flows', f'{flow_id}.py'), 'a') as f: - f.write(""" - try: - x = onecode.slider('my_slider', 0.5, min=0.51, max=0.8) - except ValueError as err: - with open("stdout.txt", 'w') as f: - f.write(str(err)) - """) - - os.system(f'cd "{flow_dir}" && {Env.ONECODE_DO_TYPECHECK}=1 python main.py') - - with open(os.path.join(flow_dir, 'stdout.txt')) as f: - assert f.read() == "[my_slider] Value lower than minimum: 0.5 < 0.51" - - shutil.rmtree(flow_dir) - - -@pytest.mark.emulations -def test_load_then_execute_invalid_min(): - flow_name, flow_folder, flow_id = _generate_flow_name() - - tmp = _clean_flow(flow_folder) - create(tmp, flow_name, cli=False) - - flow_dir = os.path.join(tmp, flow_folder) - - with open(os.path.join(flow_dir, 'flows', f'{flow_id}.py'), 'a') as f: - f.write(""" - try: - x = onecode.slider('my_slider', 0.6, min=0.51, max=0.8) - except ValueError as err: - with open("stdout.txt", 'w') as f: - f.write(str(err)) - """) - - params = os.path.join(flow_dir, 'params.json') - with open(params, 'w') as f: - json.dump({"my_slider": 0.5}, f) - - os.system(f'cd "{flow_dir}" && {Env.ONECODE_DO_TYPECHECK}=1 python main.py {params}') - - with open(os.path.join(flow_dir, 'stdout.txt')) as f: - assert f.read() == "[my_slider] Value lower than minimum: 0.5 < 0.51" - - shutil.rmtree(flow_dir) - - -@pytest.mark.emulations -def test_execute_invalid_max(): - flow_name, flow_folder, flow_id = _generate_flow_name() - - tmp = _clean_flow(flow_folder) - create(tmp, flow_name, cli=False) - - flow_dir = os.path.join(tmp, flow_folder) - - with open(os.path.join(flow_dir, 'flows', f'{flow_id}.py'), 'a') as f: - f.write(""" - try: - x = onecode.slider('my_slider', 0.5, min=0.4, max=0.49) - except ValueError as err: - with open("stdout.txt", 'w') as f: - f.write(str(err)) - """) - - os.system(f'cd "{flow_dir}" && {Env.ONECODE_DO_TYPECHECK}=1 python main.py') - - with open(os.path.join(flow_dir, 'stdout.txt')) as f: - assert f.read() == "[my_slider] Value greater than maximum: 0.5 > 0.49" - - shutil.rmtree(flow_dir) - - -@pytest.mark.emulations -def test_load_then_execute_invalid_max(): - flow_name, flow_folder, flow_id = _generate_flow_name() - - tmp = _clean_flow(flow_folder) - create(tmp, flow_name, cli=False) - - flow_dir = os.path.join(tmp, flow_folder) - - with open(os.path.join(flow_dir, 'flows', f'{flow_id}.py'), 'a') as f: - f.write(""" - try: - x = onecode.slider('my_slider', 0.45, min=0.4, max=0.49) - except ValueError as err: - with open("stdout.txt", 'w') as f: - f.write(str(err)) - """) - - params = os.path.join(flow_dir, 'params.json') - with open(params, 'w') as f: - json.dump({"my_slider": 0.5}, f) - - os.system(f'cd "{flow_dir}" && {Env.ONECODE_DO_TYPECHECK}=1 python main.py {params}') - - with open(os.path.join(flow_dir, 'stdout.txt')) as f: - assert f.read() == "[my_slider] Value greater than maximum: 0.5 > 0.49" - - shutil.rmtree(flow_dir) - - -@pytest.mark.emulations -def test_execute_invalid_with_count(): - flow_name, flow_folder, flow_id = _generate_flow_name() - - tmp = _clean_flow(flow_folder) - create(tmp, flow_name, cli=False) - - flow_dir = os.path.join(tmp, flow_folder) - - # count is ignored at loading & execution as it is only used in UI mode - with open(os.path.join(flow_dir, 'flows', f'{flow_id}.py'), 'a') as f: - f.write(""" - try: - x = onecode.slider('my_slider', [0.5, 0.45, 0.45], min=0.4, max=0.49, count=3) - except ValueError as err: - with open("stdout.txt", 'w') as f: - f.write(str(err)) - """) - - os.system(f'cd "{flow_dir}" && {Env.ONECODE_DO_TYPECHECK}=1 python main.py') - - with open(os.path.join(flow_dir, 'stdout.txt')) as f: - assert f.read() == "[my_slider] Value greater than maximum: 0.5 > 0.49" - - shutil.rmtree(flow_dir) - - -@pytest.mark.emulations -def test_load_then_execute_invalid_with_count(): - flow_name, flow_folder, flow_id = _generate_flow_name() - - tmp = _clean_flow(flow_folder) - create(tmp, flow_name, cli=False) - - flow_dir = os.path.join(tmp, flow_folder) - - # count is ignored at loading & execution as it is only used in UI mode - with open(os.path.join(flow_dir, 'flows', f'{flow_id}.py'), 'a') as f: - f.write(""" - try: - x = onecode.slider('my_slider', 0.45, min=0.4, max=0.49, count=4) - except ValueError as err: - with open("stdout.txt", 'w') as f: - f.write(str(err)) - """) - - params = os.path.join(flow_dir, 'params.json') - with open(params, 'w') as f: - json.dump({"my_slider": [0.45, 0.5, 0.45]}, f) - - os.system(f'cd "{flow_dir}" && {Env.ONECODE_DO_TYPECHECK}=1 python main.py {params}') - - with open(os.path.join(flow_dir, 'stdout.txt')) as f: - assert f.read() == "[my_slider] Value greater than maximum: 0.5 > 0.49" - - shutil.rmtree(flow_dir) - - -@pytest.mark.emulations -def test_execute_optional_value(): - flow_name, flow_folder, flow_id = _generate_flow_name() - - tmp = _clean_flow(flow_folder) - create(tmp, flow_name, cli=False) - - flow_dir = os.path.join(tmp, flow_folder) - - with open(os.path.join(flow_dir, 'flows', f'{flow_id}.py'), 'a') as f: - f.write(""" - x = onecode.slider('my_slider', None, optional=True) - with open("stdout.txt", 'w') as f: - f.write(str(x)) - """) - - os.system(f'cd "{flow_dir}" && {Env.ONECODE_DO_TYPECHECK}=1 python main.py') - - with open(os.path.join(flow_dir, 'stdout.txt')) as f: - assert f.read() == "None" - - shutil.rmtree(flow_dir) - - -@pytest.mark.emulations -def test_load_then_execute_optional_value(): - flow_name, flow_folder, flow_id = _generate_flow_name() - - tmp = _clean_flow(flow_folder) - create(tmp, flow_name, cli=False) - - flow_dir = os.path.join(tmp, flow_folder) - - with open(os.path.join(flow_dir, 'flows', f'{flow_id}.py'), 'a') as f: - f.write(""" - x = onecode.slider('my_slider', 0.5, optional=True) - with open("stdout.txt", 'w') as f: - f.write(str(x)) - """) - - params = os.path.join(flow_dir, 'params.json') - with open(params, 'w') as f: - json.dump({"my_slider": None}, f) - - os.system(f'cd "{flow_dir}" && {Env.ONECODE_DO_TYPECHECK}=1 python main.py {params}') - - with open(os.path.join(flow_dir, 'stdout.txt')) as f: - assert f.read() == "None" - - shutil.rmtree(flow_dir) - - -@pytest.mark.emulations -def test_execute_invalid_optional_value(): - flow_name, flow_folder, flow_id = _generate_flow_name() - - tmp = _clean_flow(flow_folder) - create(tmp, flow_name, cli=False) - - flow_dir = os.path.join(tmp, flow_folder) - - with open(os.path.join(flow_dir, 'flows', f'{flow_id}.py'), 'a') as f: - f.write(""" - try: - x = onecode.slider('my_slider', None, optional=False) - except ValueError as err: - with open("stdout.txt", 'w') as f: - f.write(str(err)) - """) - - os.system(f'cd "{flow_dir}" && {Env.ONECODE_DO_TYPECHECK}=1 python main.py') - - with open(os.path.join(flow_dir, 'stdout.txt')) as f: - assert f.read() == "[my_slider] Value is required: None provided" - - shutil.rmtree(flow_dir) - - -@pytest.mark.emulations -def test_load_then_execute_invalid_optional_value(): - flow_name, flow_folder, flow_id = _generate_flow_name() - - tmp = _clean_flow(flow_folder) - create(tmp, flow_name, cli=False) - - flow_dir = os.path.join(tmp, flow_folder) - - with open(os.path.join(flow_dir, 'flows', f'{flow_id}.py'), 'a') as f: - f.write(""" - try: - x = onecode.slider('my_slider', 0.5, optional=False) - except ValueError as err: - with open("stdout.txt", 'w') as f: - f.write(str(err)) - """) - - params = os.path.join(flow_dir, 'params.json') - with open(params, 'w') as f: - json.dump({"my_slider": None}, f) - - os.system(f'cd "{flow_dir}" && {Env.ONECODE_DO_TYPECHECK}=1 python main.py {params}') - - with open(os.path.join(flow_dir, 'stdout.txt')) as f: - assert f.read() == "[my_slider] Value is required: None provided" - - shutil.rmtree(flow_dir) - - -@pytest.mark.emulations -def test_invalid_name_key(): - flow_name, flow_folder, flow_id = _generate_flow_name() - - tmp = _clean_flow(flow_folder) - create(tmp, flow_name, cli=False) - - flow_dir = os.path.join(tmp, flow_folder) - - with open(os.path.join(flow_dir, 'flows', f'{flow_id}.py'), 'a') as f: - f.write(""" - try: - x = onecode.slider('_my_slider', 0.5) - except ValueError as err: - with open("stdout.txt", 'w') as f: - f.write(str(err)) - """) - - os.system(f'cd "{flow_dir}" && {Env.ONECODE_DO_TYPECHECK}=1 python main.py') - - with open(os.path.join(flow_dir, 'stdout.txt')) as f: - assert f.read() == 'Key starting with "_" are reserved: _my_slider' - - shutil.rmtree(flow_dir) diff --git a/tests/emulations/elements/test_run_text_input.py b/tests/emulations/elements/test_run_text_input.py deleted file mode 100644 index b4d11f7..0000000 --- a/tests/emulations/elements/test_run_text_input.py +++ /dev/null @@ -1,249 +0,0 @@ -import json -import os -import shutil - -import pytest - -from onecode import Env -from onecode.cli.create import create -from tests.utils.flow_cli import _clean_flow, _generate_flow_name - - -@pytest.mark.emulations -def test_execute_single_count(): - flow_name, flow_folder, flow_id = _generate_flow_name() - - tmp = _clean_flow(flow_folder) - create(tmp, flow_name, cli=False) - - flow_dir = os.path.join(tmp, flow_folder) - - with open(os.path.join(flow_dir, 'flows', f'{flow_id}.py'), 'a') as f: - f.write(""" - x = onecode.text_input('my_text', "OneCode rocks!") - with open("stdout.txt", 'w') as f: - f.write(str(x)) - """) - - os.system(f'cd "{flow_dir}" && {Env.ONECODE_DO_TYPECHECK}=1 python main.py') - - with open(os.path.join(flow_dir, 'stdout.txt')) as f: - assert f.read() == "OneCode rocks!" - - shutil.rmtree(flow_dir) - - -@pytest.mark.emulations -def test_execute_multiple_count(): - flow_name, flow_folder, flow_id = _generate_flow_name() - - tmp = _clean_flow(flow_folder) - create(tmp, flow_name, cli=False) - - flow_dir = os.path.join(tmp, flow_folder) - - # count is ignored at loading & execution as it is only used in UI mode - with open(os.path.join(flow_dir, 'flows', f'{flow_id}.py'), 'a') as f: - f.write(""" - x = onecode.text_input('my_text', ["OneCode rocks!", "OneCode rocks!"], count=3) - with open("stdout.txt", 'w') as f: - f.write(str(x)) - """) - - os.system(f'cd "{flow_dir}" && {Env.ONECODE_DO_TYPECHECK}=1 python main.py') - - with open(os.path.join(flow_dir, 'stdout.txt')) as f: - assert f.read() == "['OneCode rocks!', 'OneCode rocks!']" - - shutil.rmtree(flow_dir) - - -@pytest.mark.emulations -def test_load_then_execute_single_count(): - flow_name, flow_folder, flow_id = _generate_flow_name() - - tmp = _clean_flow(flow_folder) - create(tmp, flow_name, cli=False) - - flow_dir = os.path.join(tmp, flow_folder) - - with open(os.path.join(flow_dir, 'flows', f'{flow_id}.py'), 'a') as f: - f.write(""" - x = onecode.text_input('my_text', "Nope!") - with open("stdout.txt", 'w') as f: - f.write(str(x)) - """) - - params = os.path.join(flow_dir, 'params.json') - with open(params, 'w') as f: - json.dump({"my_text": "OneCode rocks!"}, f) - - os.system(f'cd "{flow_dir}" && {Env.ONECODE_DO_TYPECHECK}=1 python main.py {params}') - - with open(os.path.join(flow_dir, 'stdout.txt')) as f: - assert f.read() == "OneCode rocks!" - - shutil.rmtree(flow_dir) - - -@pytest.mark.emulations -def test_load_then_execute_multiple_count(): - flow_name, flow_folder, flow_id = _generate_flow_name() - - tmp = _clean_flow(flow_folder) - create(tmp, flow_name, cli=False) - - flow_dir = os.path.join(tmp, flow_folder) - - # count is ignored at loading & execution as it is only used in UI mode - with open(os.path.join(flow_dir, 'flows', f'{flow_id}.py'), 'a') as f: - f.write(""" - x = onecode.text_input('my_text', "Nope", count=4) - with open("stdout.txt", 'w') as f: - f.write(str(x)) - """) - - params = os.path.join(flow_dir, 'params.json') - with open(params, 'w') as f: - json.dump({"my_text": ["OneCode", "rocks", "!"]}, f) - - os.system(f'cd "{flow_dir}" && {Env.ONECODE_DO_TYPECHECK}=1 python main.py {params}') - - with open(os.path.join(flow_dir, 'stdout.txt')) as f: - assert f.read() == "['OneCode', 'rocks', '!']" - - shutil.rmtree(flow_dir) - - -@pytest.mark.emulations -def test_execute_optional_value(): - flow_name, flow_folder, flow_id = _generate_flow_name() - - tmp = _clean_flow(flow_folder) - create(tmp, flow_name, cli=False) - - flow_dir = os.path.join(tmp, flow_folder) - - with open(os.path.join(flow_dir, 'flows', f'{flow_id}.py'), 'a') as f: - f.write(""" - x = onecode.text_input('my_text', None, optional=True) - with open("stdout.txt", 'w') as f: - f.write(str(x)) - """) - - os.system(f'cd "{flow_dir}" && {Env.ONECODE_DO_TYPECHECK}=1 python main.py') - - with open(os.path.join(flow_dir, 'stdout.txt')) as f: - assert f.read() == "None" - - shutil.rmtree(flow_dir) - - -@pytest.mark.emulations -def test_load_then_execute_optional_value(): - flow_name, flow_folder, flow_id = _generate_flow_name() - - tmp = _clean_flow(flow_folder) - create(tmp, flow_name, cli=False) - - flow_dir = os.path.join(tmp, flow_folder) - - with open(os.path.join(flow_dir, 'flows', f'{flow_id}.py'), 'a') as f: - f.write(""" - x = onecode.text_input('my_text', 'OneCode rocks!', optional=True) - with open("stdout.txt", 'w') as f: - f.write(str(x)) - """) - - params = os.path.join(flow_dir, 'params.json') - with open(params, 'w') as f: - json.dump({"my_text": None}, f) - - os.system(f'cd "{flow_dir}" && {Env.ONECODE_DO_TYPECHECK}=1 python main.py {params}') - - with open(os.path.join(flow_dir, 'stdout.txt')) as f: - assert f.read() == "None" - - shutil.rmtree(flow_dir) - - -@pytest.mark.emulations -def test_execute_invalid_optional_value(): - flow_name, flow_folder, flow_id = _generate_flow_name() - - tmp = _clean_flow(flow_folder) - create(tmp, flow_name, cli=False) - - flow_dir = os.path.join(tmp, flow_folder) - - with open(os.path.join(flow_dir, 'flows', f'{flow_id}.py'), 'a') as f: - f.write(""" - try: - x = onecode.text_input('my_text', None, optional=False) - except ValueError as err: - with open("stdout.txt", 'w') as f: - f.write(str(err)) - """) - - os.system(f'cd "{flow_dir}" && {Env.ONECODE_DO_TYPECHECK}=1 python main.py') - - with open(os.path.join(flow_dir, 'stdout.txt')) as f: - assert f.read() == "[my_text] Value is required: None provided" - - shutil.rmtree(flow_dir) - - -@pytest.mark.emulations -def test_load_then_execute_invalid_optional_value(): - flow_name, flow_folder, flow_id = _generate_flow_name() - - tmp = _clean_flow(flow_folder) - create(tmp, flow_name, cli=False) - - flow_dir = os.path.join(tmp, flow_folder) - - with open(os.path.join(flow_dir, 'flows', f'{flow_id}.py'), 'a') as f: - f.write(""" - try: - x = onecode.text_input('my_text', 'OneCode rocks!', optional=False) - except ValueError as err: - with open("stdout.txt", 'w') as f: - f.write(str(err)) - """) - - params = os.path.join(flow_dir, 'params.json') - with open(params, 'w') as f: - json.dump({"my_text": None}, f) - - os.system(f'cd "{flow_dir}" && {Env.ONECODE_DO_TYPECHECK}=1 python main.py {params}') - - with open(os.path.join(flow_dir, 'stdout.txt')) as f: - assert f.read() == "[my_text] Value is required: None provided" - - shutil.rmtree(flow_dir) - - -@pytest.mark.emulations -def test_invalid_name_key(): - flow_name, flow_folder, flow_id = _generate_flow_name() - - tmp = _clean_flow(flow_folder) - create(tmp, flow_name, cli=False) - - flow_dir = os.path.join(tmp, flow_folder) - - with open(os.path.join(flow_dir, 'flows', f'{flow_id}.py'), 'a') as f: - f.write(""" - try: - x = onecode.text_input('_my_text', None, optional=True) - except ValueError as err: - with open("stdout.txt", 'w') as f: - f.write(str(err)) - """) - - os.system(f'cd "{flow_dir}" && {Env.ONECODE_DO_TYPECHECK}=1 python main.py') - - with open(os.path.join(flow_dir, 'stdout.txt')) as f: - assert f.read() == 'Key starting with "_" are reserved: _my_text' - - shutil.rmtree(flow_dir) diff --git a/tests/emulations/base/test_run_project.py b/tests/unit/base/test_run_project.py similarity index 70% rename from tests/emulations/base/test_run_project.py rename to tests/unit/base/test_run_project.py index 02fb44c..a743c37 100644 --- a/tests/emulations/base/test_run_project.py +++ b/tests/unit/base/test_run_project.py @@ -1,16 +1,11 @@ import os import shutil -import pytest -from datatest import working_directory - from onecode import Env from onecode.cli.create import create -from tests.utils.flow_cli import _clean_flow, _generate_flow_name +from tests.utils.flow_cli import _clean_flow, _generate_flow_name, run_project -@pytest.mark.emulations -@working_directory(__file__) def test_default_data_path(): flow_name, flow_folder, flow_id = _generate_flow_name() @@ -26,7 +21,7 @@ def test_default_data_path(): f.write(onecode.Project().data_root) """) - os.system(f'cd "{flow_dir}" && {Env.ONECODE_DO_TYPECHECK}=1 python main.py') + run_project(flow_dir) with open(os.path.join(flow_dir, 'stdout.txt')) as f: assert f.read() == flow_data @@ -34,8 +29,6 @@ def test_default_data_path(): shutil.rmtree(flow_dir) -@pytest.mark.emulations -@working_directory(__file__) def test_env_data_path(): flow_name, flow_folder, flow_id = _generate_flow_name() @@ -51,11 +44,8 @@ def test_env_data_path(): f.write(onecode.Project().data_root) """) - os.system( - f'cd "{flow_dir}" && ' - f'{Env.ONECODE_PROJECT_DATA}="{flow_data}" ' - f'{Env.ONECODE_DO_TYPECHECK}=1 python main.py' - ) + os.environ[Env.ONECODE_PROJECT_DATA] = flow_data + run_project(flow_dir) with open(os.path.join(flow_dir, 'stdout.txt')) as f: assert f.read() == flow_data @@ -63,7 +53,6 @@ def test_env_data_path(): shutil.rmtree(os.path.join(tmp, flow_folder)) -@pytest.mark.emulations def test_output_multiprocess(): flow_name, flow_folder, flow_id = _generate_flow_name() @@ -85,16 +74,28 @@ def write_output(data, flow_name): with open(os.path.join(flow_dir, 'flows', f'{flow_id}.py'), 'w') as f: f.write(""" -from multiprocessing import Process +import multiprocessing from flows.utils import write_output import onecode +def _process_class(): + # Linux 3.14's default fork server keeps the environment from the first + # worker, so a later run still sees a deleted data path. Fork sees the + # current ONECODE_PROJECT_DATA. Every other platform keeps Process, which + # is the start method these tests have always used. + ctx = multiprocessing.get_context() + if type(ctx).__name__ == "ForkServerContext": + return multiprocessing.get_context("fork").Process + return multiprocessing.Process + + def run(): names = ['b', 'a', 'c'] * 10 procs = [] parent_flow = onecode.Project().current_flow - + Process = _process_class() + for name in names: proc = Process(target=write_output, args=(name, parent_flow)) procs.append(proc) @@ -102,13 +103,12 @@ def run(): for proc in procs: proc.join() + if proc.exitcode: + raise RuntimeError(f"worker exited with {proc.exitcode}") """) - os.system( - f'cd "{flow_dir}" && ' - f'{Env.ONECODE_PROJECT_DATA}="{flow_data}" ' - f'{Env.ONECODE_DO_TYPECHECK}=1 python main.py' - ) + os.environ[Env.ONECODE_PROJECT_DATA] = flow_data + run_project(flow_dir) with open(os.path.join(flow_data, 'outputs', flow_id, 'MANIFEST.txt')) as f: data = f.read() @@ -121,7 +121,6 @@ def run(): shutil.rmtree(os.path.join(tmp, flow_folder)) -@pytest.mark.emulations def test_manifest_cleaning(): flow_name, flow_folder, flow_id = _generate_flow_name() @@ -143,15 +142,27 @@ def write_output(data, flow_name): with open(os.path.join(flow_dir, 'flows', f'{flow_id}.py'), 'w') as f: f.write(""" -from multiprocessing import Process +import multiprocessing from flows.utils import write_output import onecode - + +def _process_class(): + # Linux 3.14's default fork server keeps the environment from the first + # worker, so a later run still sees a deleted data path. Fork sees the + # current ONECODE_PROJECT_DATA. Every other platform keeps Process, which + # is the start method these tests have always used. + ctx = multiprocessing.get_context() + if type(ctx).__name__ == "ForkServerContext": + return multiprocessing.get_context("fork").Process + return multiprocessing.Process + + def run(): names = ['b', 'a', 'c'] * 10 procs = [] parent_flow = onecode.Project().current_flow + Process = _process_class() for name in names: proc = Process(target=write_output, args=(name, parent_flow)) @@ -160,21 +171,14 @@ def run(): for proc in procs: proc.join() + if proc.exitcode: + raise RuntimeError(f"worker exited with {proc.exitcode}") """) - os.system( - f'cd "{flow_dir}" && ' - f'{Env.ONECODE_PROJECT_DATA}="{flow_data}" ' - f'{Env.ONECODE_DO_TYPECHECK}=1 python main.py' - ) - os.system( - f'{Env.ONECODE_PROJECT_DATA}="{flow_data}" ' - f'{Env.ONECODE_DO_TYPECHECK}=1 python main.py' - ) - os.system( - f'{Env.ONECODE_PROJECT_DATA}="{flow_data}" ' - f'{Env.ONECODE_DO_TYPECHECK}=1 python main.py' - ) + os.environ[Env.ONECODE_PROJECT_DATA] = flow_data + run_project(flow_dir) + run_project(flow_dir) + run_project(flow_dir) with open(os.path.join(flow_data, 'outputs', flow_id, 'MANIFEST.txt')) as f: data = f.read() diff --git a/tests/unit/base/test_version.py b/tests/unit/base/test_version.py index f298d01..4d76ef9 100644 --- a/tests/unit/base/test_version.py +++ b/tests/unit/base/test_version.py @@ -1,4 +1,5 @@ -import toml +import tomllib + from datatest import working_directory import onecode @@ -6,6 +7,6 @@ @working_directory(__file__) def test_version(): - with open('../../../pyproject.toml', 'r') as f: - parsed_toml = toml.load(f) + with open('../../../pyproject.toml', 'rb') as f: + parsed_toml = tomllib.load(f) assert onecode.__version__ == parsed_toml['tool']['poetry']['version'] diff --git a/tests/unit/cli/test_check.py b/tests/unit/cli/test_check.py index fb09e0f..395bbdc 100644 --- a/tests/unit/cli/test_check.py +++ b/tests/unit/cli/test_check.py @@ -76,7 +76,7 @@ def test_check_modules(): 'msg': None } - assert modules['onecode']['version'].split('.dev')[0] == "1.2.2" + assert modules['onecode']['version'].split('.dev')[0] == "1.3.0" assert modules['onecode']['in_env'] is True assert modules['onecode']['builtin'] is False assert modules['onecode']['dist_name'] == 'onecode' diff --git a/tests/emulations/cli/test_execution.py b/tests/unit/cli/test_execution.py similarity index 90% rename from tests/emulations/cli/test_execution.py rename to tests/unit/cli/test_execution.py index 4210d0c..0713751 100644 --- a/tests/emulations/cli/test_execution.py +++ b/tests/unit/cli/test_execution.py @@ -2,15 +2,11 @@ import os import shutil -import pytest - -from onecode import Env from onecode.cli.add import add from onecode.cli.create import create -from tests.utils.flow_cli import _clean_flow, _generate_flow_name +from tests.utils.flow_cli import _clean_flow, _generate_flow_name, run_project -@pytest.mark.emulations def test_single_flow_execution(): flow_name, flow_folder, flow_id = _generate_flow_name() @@ -25,7 +21,7 @@ def test_single_flow_execution(): f.write("Flow just executed!") """) - os.system(f'cd "{flow_dir}" && {Env.ONECODE_DO_TYPECHECK}=1 python main.py') + run_project(flow_dir) with open(os.path.join(flow_dir, 'stdout.txt')) as f: assert f.read() == "Flow just executed!" @@ -33,7 +29,6 @@ def test_single_flow_execution(): shutil.rmtree(flow_dir) -@pytest.mark.emulations def test_multiple_flows_execution(): flow_name, flow_folder, flow_id = _generate_flow_name() @@ -68,7 +63,7 @@ def test_multiple_flows_execution(): f.write("Flow 2 just executed!") """) - os.system(f'cd "{flow_dir}" && {Env.ONECODE_DO_TYPECHECK}=1 python main.py') + run_project(flow_dir) with open(os.path.join(flow_dir, 'stdout.txt')) as f: assert f.read() == ( @@ -81,7 +76,6 @@ def test_multiple_flows_execution(): shutil.rmtree(flow_dir) -@pytest.mark.emulations def test_single_flow_execution_in_multi_flows(): flow_name, flow_folder, flow_id = _generate_flow_name() @@ -116,7 +110,7 @@ def test_single_flow_execution_in_multi_flows(): f.write("Flow 2 just executed!") """) - os.system(f'cd "{flow_dir}" && {Env.ONECODE_DO_TYPECHECK}=1 python main.py --flow another_flow') + run_project(flow_dir, ["--flow", "another_flow"]) with open(os.path.join(flow_dir, 'stdout.txt')) as f: assert f.read() == ( @@ -126,7 +120,6 @@ def test_single_flow_execution_in_multi_flows(): shutil.rmtree(flow_dir) -@pytest.mark.emulations def test_single_flow_execution_in_multi_flows_with_params(): flow_name, flow_folder, flow_id = _generate_flow_name() @@ -166,10 +159,7 @@ def test_single_flow_execution_in_multi_flows_with_params(): f.write("Flow 2 just executed!") """) - os.system( - f'cd "{flow_dir}" && ' - f'{Env.ONECODE_DO_TYPECHECK}=1 python main.py --flow another_flow params.json' - ) + run_project(flow_dir, ["--flow", "another_flow", "params.json"]) with open(os.path.join(flow_dir, 'stdout.txt')) as f: assert f.read() == ( diff --git a/tests/unit/cra/conftest.py b/tests/unit/cra/conftest.py new file mode 100644 index 0000000..fd96e7e --- /dev/null +++ b/tests/unit/cra/conftest.py @@ -0,0 +1,4 @@ +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[3] / "scripts")) diff --git a/tests/unit/cra/test_sbom.py b/tests/unit/cra/test_sbom.py new file mode 100644 index 0000000..10a31d7 --- /dev/null +++ b/tests/unit/cra/test_sbom.py @@ -0,0 +1,196 @@ +import tempfile +import unittest +from pathlib import Path + +from cra.cyclonedx import SPEC_VERSION +from cra.findings import ( + Finding, + advisory_link, + constraint_overlaps, + findings_for_vulns, + osv_severity, +) +from cra.make_release_sbom import build +from cra.monitor_releases import ( + fingerprint, + notification_body, + release_has_sbom, + render_body, + render_run_summary, +) +from cra.python_deps import is_sbom_asset, python_sbom, runtime_dependencies + + +def _vuln(ranges, versions=None, name="numpy"): + entry = {"package": {"ecosystem": "PyPI", "name": name}} + if ranges is not None: + entry["ranges"] = [{"type": "ECOSYSTEM", "events": ranges}] + if versions is not None: + entry["versions"] = versions + return {"id": "GHSA-test", "affected": [entry]} + + +class RangeOverlapTest(unittest.TestCase): + def test_lower_bound_inside_fixed_range(self): + vuln = _vuln([{"introduced": "0"}, {"fixed": "1.26"}]) + self.assertTrue(constraint_overlaps(">= 1.25, < 2", vuln, "numpy", "PyPI")) + + def test_lower_bound_at_fix_is_clear(self): + vuln = _vuln([{"introduced": "0"}, {"fixed": "1.26"}]) + self.assertFalse(constraint_overlaps(">= 1.26, < 2", vuln, "numpy", "PyPI")) + + def test_range_above_fix_is_clear(self): + vuln = _vuln([{"introduced": "0"}, {"fixed": "1.5"}]) + self.assertFalse(constraint_overlaps(">= 2.3.3, < 3", vuln, "numpy", "PyPI")) + + def test_explicit_version_list(self): + vuln = _vuln(None, versions=["1.2.3"]) + self.assertTrue(constraint_overlaps("==1.2.3", vuln, "numpy", "PyPI")) + + def test_last_affected_is_inclusive(self): + vuln = _vuln([{"introduced": "0"}, {"last_affected": "1.5"}]) + self.assertTrue(constraint_overlaps(">= 1.5, < 2", vuln, "numpy", "PyPI")) + + def test_other_package_is_ignored(self): + vuln = _vuln([{"introduced": "0"}, {"fixed": "9"}]) + self.assertFalse(constraint_overlaps(">= 1, < 2", vuln, "pandas", "PyPI")) + + +class AdvisoryReportTest(unittest.TestCase): + def test_pysec_links_to_osv_and_ghsa_links_to_github(self): + self.assertEqual( + advisory_link("PYSEC-2024-161"), + "[PYSEC-2024-161](https://osv.dev/vulnerability/PYSEC-2024-161)", + ) + self.assertIn( + "https://github.com/advisories/GHSA-5wvp-7f3h-6wmm", + advisory_link("GHSA-5wvp-7f3h-6wmm"), + ) + + def test_cvss_vector_supplies_severity_when_database_specific_is_missing(self): + critical = { + "severity": [ + {"type": "CVSS_V3", "score": "CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:H/I:H/A:H"}, + ], + } + moderate = { + "severity": [ + {"type": "CVSS_V3", "score": "CVSS:3.1/AV:L/AC:H/PR:N/UI:N/S:C/C:L/I:L/A:N"}, + ], + } + self.assertEqual(osv_severity(critical), "critical") + self.assertEqual(osv_severity(moderate), "moderate") + + def test_database_specific_severity_wins(self): + vuln = { + "database_specific": {"severity": "HIGH"}, + "severity": [ + {"type": "CVSS_V3", "score": "CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:H/I:H/A:H"}, + ], + } + self.assertEqual(osv_severity(vuln), "high") + + def test_pysec_inherits_alias_severity_without_a_lookup(self): + ghsa = { + "id": "GHSA-aaaa-bbbb-cccc", + "aliases": ["PYSEC-2026-1"], + "database_specific": {"severity": "HIGH"}, + "summary": "example", + } + pysec = {"id": "PYSEC-2026-1", "aliases": ["GHSA-aaaa-bbbb-cccc"]} + + def fail_fetch(_vuln_id: str): + raise AssertionError("alias lookup") + + import cra.findings as findings + + original = findings._fetch_vuln + findings._fetch_vuln = fail_fetch + try: + found = findings_for_vulns("pyarrow", ">= 12", "PyPI", [ghsa, pysec]) + finally: + findings._fetch_vuln = original + by_id = {item.vuln_id: item.severity for item in found} + self.assertEqual(by_id["PYSEC-2026-1"], "high") + + +class PythonSbomTest(unittest.TestCase): + def test_ranges_are_kept_and_optional_deps_are_omitted(self): + with tempfile.TemporaryDirectory() as tmp: + path = Path(tmp) / "pyproject.toml" + path.write_text( + """ +[tool.poetry] +name = "onecode" +version = "1.2.2" + +[tool.poetry.dependencies] +python = ">=3.11, <3.15" +pandas = ">=2.3,<4" +datatest = { version = ">=0.11.1,<1", optional = true } +""".strip(), + ) + _name, _version, deps = runtime_dependencies(path) + self.assertEqual( + deps, + [("python", ">=3.11, <3.15"), ("pandas", ">=2.3,<4")], + ) + document = python_sbom(path) + self.assertEqual(document["specVersion"], SPEC_VERSION) + root = document["metadata"]["component"] + self.assertEqual(root["licenses"], [{"license": {"id": "MIT"}}]) + self.assertEqual(root["supplier"], {"name": "DeepLime"}) + by_name = {component["name"]: component for component in document["components"]} + self.assertEqual(by_name["pandas"]["version"], ">=2.3,<4") + self.assertEqual(by_name["pandas"]["purl"], "pkg:pypi/pandas") + self.assertNotIn("datatest", by_name) + written = build(path, Path(tmp)) + self.assertEqual(written.name, "onecode-1.2.2.cdx.json") + + def test_repository_pyproject_keeps_runtime_deps_only(self): + root = Path(__file__).resolve().parents[3] + _name, version, deps = runtime_dependencies(root / "pyproject.toml") + names = {name for name, _constraint in deps} + self.assertIn("pandas", names) + self.assertIn("pyarrow", names) + self.assertNotIn("pytest", names) + self.assertNotIn("mkdocs", names) + self.assertEqual(version, "1.3.0.dev") + + +class ReleaseAssetTest(unittest.TestCase): + def test_sbom_asset_name(self): + self.assertTrue(is_sbom_asset("onecode-1.2.2.cdx.json")) + self.assertFalse(is_sbom_asset("onecode-1.2.2.whl")) + + def test_release_without_sbom_is_selected_for_backfill(self): + release = {"assets": [{"name": "onecode-1.2.2.whl"}]} + self.assertFalse(release_has_sbom(release)) + release["assets"].append({"name": "onecode-1.2.2.cdx.json"}) + self.assertTrue(release_has_sbom(release)) + + +class IssueBodyTest(unittest.TestCase): + def test_fingerprint_is_stable(self): + finding = Finding("GHSA-1", "pandas", ">=2.3,<4", "summary", "high", False, "PyPI") + body = render_body([finding]) + self.assertIn(fingerprint([finding]), body) + self.assertTrue(body.startswith("