From 72a81f1ad670bd875dfbe416ad9cfaef9bb4e787 Mon Sep 17 00:00:00 2001 From: Chris Hagglund Date: Mon, 13 Jul 2026 13:05:58 -0600 Subject: [PATCH 01/25] wip --- .github/workflows/pull_request.yml | 46 +++- .../client/http/models/service_method.py | 2 +- .../client/http/models/upsert_user_request.py | 42 +++- tests/integration/README.md | 84 ++++--- tests/integration/client/test_async.py | 10 +- .../integration/test_async_lease_extension.py | 223 +++++++++--------- .../test_authorization_client_intg.py | 11 +- .../test_authorization_complete.py | 11 +- .../workflow/test_workflow_execution.py | 52 ++-- 9 files changed, 288 insertions(+), 193 deletions(-) diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index 7115cfda1..d71609dca 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -1,6 +1,13 @@ name: Continuous Integration -on: [pull_request, workflow_dispatch] +on: + push: + branches: + - main + pull_request: + branches: + - main + workflow_dispatch: concurrency: group: ${{ github.workflow }}-${{ github.ref }} @@ -46,9 +53,6 @@ jobs: id: unit_tests continue-on-error: true env: - CONDUCTOR_AUTH_KEY: ${{ secrets.CONDUCTOR_AUTH_KEY }} - CONDUCTOR_AUTH_SECRET: ${{ secrets.CONDUCTOR_AUTH_SECRET }} - CONDUCTOR_SERVER_URL: ${{ secrets.CONDUCTOR_SERVER_URL }} COVERAGE_FILE: ${{ env.COVERAGE_DIR }}/.coverage.unit run: | coverage run -m pytest tests/unit -v @@ -57,9 +61,6 @@ jobs: id: bc_tests continue-on-error: true env: - CONDUCTOR_AUTH_KEY: ${{ secrets.CONDUCTOR_AUTH_KEY }} - CONDUCTOR_AUTH_SECRET: ${{ secrets.CONDUCTOR_AUTH_SECRET }} - CONDUCTOR_SERVER_URL: ${{ secrets.CONDUCTOR_SERVER_URL }} COVERAGE_FILE: ${{ env.COVERAGE_DIR }}/.coverage.bc run: | coverage run -m pytest tests/backwardcompatibility -v @@ -68,9 +69,6 @@ jobs: id: serdeser_tests continue-on-error: true env: - CONDUCTOR_AUTH_KEY: ${{ secrets.CONDUCTOR_AUTH_KEY }} - CONDUCTOR_AUTH_SECRET: ${{ secrets.CONDUCTOR_AUTH_SECRET }} - CONDUCTOR_SERVER_URL: ${{ secrets.CONDUCTOR_SERVER_URL }} COVERAGE_FILE: ${{ env.COVERAGE_DIR }}/.coverage.serdeser run: | coverage run -m pytest tests/serdesertest -v @@ -103,4 +101,30 @@ jobs: - name: Check test results if: steps.unit_tests.outcome == 'failure' || steps.bc_tests.outcome == 'failure' || steps.serdeser_tests.outcome == 'failure' || steps.agent_base_import.outcome == 'failure' - run: exit 1 \ No newline at end of file + run: exit 1 + + integration-test: + runs-on: ubuntu-latest + timeout-minutes: 30 + env: + CONDUCTOR_SERVER_URL: ${{ vars.SDKDEV_V5_SERVER_URL }} + CONDUCTOR_AUTH_KEY: ${{ vars.SDKDEV_V5_AUTH_KEY }} + CONDUCTOR_AUTH_SECRET: ${{ secrets.SDKDEV_V5_AUTH_SECRET }} + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.12' + cache: 'pip' + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -e . + pip install pytest + + - name: Run integration tests + run: bash scripts/run_integration_tests.sh diff --git a/src/conductor/client/http/models/service_method.py b/src/conductor/client/http/models/service_method.py index df03f5502..3f6f4b23e 100644 --- a/src/conductor/client/http/models/service_method.py +++ b/src/conductor/client/http/models/service_method.py @@ -15,7 +15,7 @@ class ServiceMethod: 'input_type': 'str', 'output_type': 'str', 'request_params': 'list[RequestParam]', - 'example_input': 'dict' + 'example_input': 'object' } attribute_map = { diff --git a/src/conductor/client/http/models/upsert_user_request.py b/src/conductor/client/http/models/upsert_user_request.py index 9d455be0e..89305c7f7 100644 --- a/src/conductor/client/http/models/upsert_user_request.py +++ b/src/conductor/client/http/models/upsert_user_request.py @@ -2,7 +2,7 @@ import re # noqa: F401 import six from dataclasses import dataclass, field, InitVar -from typing import List, Optional +from typing import Dict, List, Optional from enum import Enum @@ -30,41 +30,50 @@ class UpsertUserRequest: name: InitVar[Optional[str]] = None roles: InitVar[Optional[List[str]]] = None groups: InitVar[Optional[List[str]]] = None + contact_information: InitVar[Optional[Dict[str, str]]] = None _name: str = field(default=None, init=False) _roles: List[str] = field(default=None, init=False) _groups: List[str] = field(default=None, init=False) + _contact_information: Dict[str, str] = field(default=None, init=False) swagger_types = { 'name': 'str', 'roles': 'list[str]', - 'groups': 'list[str]' + 'groups': 'list[str]', + 'contact_information': 'dict(str, str)' } attribute_map = { 'name': 'name', 'roles': 'roles', - 'groups': 'groups' + 'groups': 'groups', + 'contact_information': 'contactInformation' } - def __init__(self, name=None, roles=None, groups=None): # noqa: E501 + def __init__(self, name=None, roles=None, groups=None, contact_information=None): # noqa: E501 """UpsertUserRequest - a model defined in Swagger""" # noqa: E501 self._name = None self._roles = None self._groups = None + self._contact_information = None self.discriminator = None self.name = name if roles is not None: self.roles = roles if groups is not None: self.groups = groups + if contact_information is not None: + self.contact_information = contact_information - def __post_init__(self, name, roles, groups): + def __post_init__(self, name, roles, groups, contact_information): self.name = name if roles is not None: self.roles = roles if groups is not None: self.groups = groups + if contact_information is not None: + self.contact_information = contact_information @property def name(self): @@ -139,6 +148,29 @@ def groups(self, groups): self._groups = groups + @property + def contact_information(self): + """Gets the contact_information of this UpsertUserRequest. # noqa: E501 + + User's contact information, e.g. {"email": "user@example.com"} # noqa: E501 + + :return: The contact_information of this UpsertUserRequest. # noqa: E501 + :rtype: dict(str, str) + """ + return self._contact_information + + @contact_information.setter + def contact_information(self, contact_information): + """Sets the contact_information of this UpsertUserRequest. + + User's contact information, e.g. {"email": "user@example.com"} # noqa: E501 + + :param contact_information: The contact_information of this UpsertUserRequest. # noqa: E501 + :type: dict(str, str) + """ + + self._contact_information = contact_information + def to_dict(self): """Returns the model properties as a dict""" result = {} diff --git a/tests/integration/README.md b/tests/integration/README.md index ac00666e0..58d0abcab 100644 --- a/tests/integration/README.md +++ b/tests/integration/README.md @@ -38,8 +38,36 @@ export CONDUCTOR_AUTH_SECRET="your-secret" ## Running Tests +### Run the CI suite locally (recommended) + +Use the helper script to run exactly what CI runs (the `integration-test` job in +[`.github/workflows/pull_request.yml`](../../.github/workflows/pull_request.yml)). +It excludes the AI/agentic tests (which need a dedicated AI-enabled server) and +the slow performance test, so you don't have to remember the `--ignore` flags: + +```bash +export CONDUCTOR_SERVER_URL="http://localhost:8080/api" +# For Orkes / authenticated servers also set: +# export CONDUCTOR_AUTH_KEY="your-key" +# export CONDUCTOR_AUTH_SECRET="your-secret" + +./scripts/run_integration_tests.sh + +# Also run the performance test (test_update_task_v2_perf.py, ~1000 workflows, +# several minutes): +./scripts/run_integration_tests.sh --with-perf +``` + +Any extra arguments pass straight through to pytest, which is handy for +targeting a subset of tests or getting more detail on failures. See additional +options and examples in the comments at the top of +[`scripts/run_integration_tests.sh`](../../scripts/run_integration_tests.sh). + ### Run All Integration Tests +This includes the AI/agentic tests, which require an AI-enabled server (see +[Tests excluded by default](#tests-excluded-by-default) below): + ```bash python3 -m pytest tests/integration/ -v -s ``` @@ -442,46 +470,42 @@ To add more test scenarios: --- -## CI/CD Integration +## Tests excluded by default -### GitHub Actions Example +`scripts/run_integration_tests.sh` (and CI) skip a few tests by default. -```yaml -name: Integration Tests +**AI/agentic tests** need a dedicated AI-enabled server: -on: [push, pull_request] +- `test_ai_task_types.py` and `test_ai_examples.py` hardcode + `http://localhost:7001/api`. +- `test_agentic_workflows.py` needs an `openai` LLM provider (model + `gpt-4o-mini`) configured on the server. -jobs: - integration: - runs-on: ubuntu-latest +Run them only against a suitably configured AI-enabled server, e.g.: - services: - conductor: - image: conductoross/conductor-standalone:3.15.0 - ports: - - 8080:8080 - - 5000:5000 +```bash +python3 -m pytest tests/integration/test_ai_task_types.py -v -s +``` - steps: - - uses: actions/checkout@v2 +**Performance test** (`test_update_task_v2_perf.py`) submits ~1000 workflows and +takes several minutes. Include it with the `--with-perf` flag: - - name: Set up Python - uses: actions/setup-python@v2 - with: - python-version: '3.9' +```bash +./scripts/run_integration_tests.sh --with-perf +``` - - name: Install dependencies - run: pip install -e . +--- + +## CI/CD Integration - - name: Wait for Conductor - run: | - timeout 60 bash -c 'until curl -f http://localhost:8080/api/health; do sleep 2; done' +Integration tests run in CI via the `integration-test` job in +[`.github/workflows/pull_request.yml`](../../.github/workflows/pull_request.yml) +on pushes to `main`, PRs targeting `main`, and manual dispatch. The job invokes +`scripts/run_integration_tests.sh` (excluding the AI tests and the performance +test) and reads the server from the `SDKDEV_V5_*` repository variables/secret. - - name: Run integration tests - env: - CONDUCTOR_SERVER_URL: http://localhost:8080/api - run: python3 -m pytest tests/integration/ -v -s -``` +To reproduce the CI run locally, use the script documented in +[Running Tests](#running-tests) above. --- diff --git a/tests/integration/client/test_async.py b/tests/integration/client/test_async.py index 8efe4fc87..c83278a74 100644 --- a/tests/integration/client/test_async.py +++ b/tests/integration/client/test_async.py @@ -1,10 +1,18 @@ from conductor.client.http.api.metadata_resource_api import MetadataResourceApi from conductor.client.http.api_client import ApiClient +from conductor.client.http.models.task_def import TaskDef + +TASK_NAME = 'python_integration_test_task' def test_async_method(api_client: ApiClient): metadata_client = MetadataResourceApi(api_client) + + # Ensure the task def exists so the async lookup has something to return, + # regardless of test ordering. + metadata_client.register_task_def(body=[TaskDef(name=TASK_NAME)]) + thread = metadata_client.get_task_def( - async_req=True, tasktype='python_integration_test_task') + async_req=True, tasktype=TASK_NAME) thread.wait() assert thread.get() is not None diff --git a/tests/integration/test_async_lease_extension.py b/tests/integration/test_async_lease_extension.py index 9a73c88b1..ddd27ce6e 100644 --- a/tests/integration/test_async_lease_extension.py +++ b/tests/integration/test_async_lease_extension.py @@ -24,12 +24,11 @@ import os import sys import time -import threading import unittest sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))) -from conductor.client.automator.task_handler import TaskHandler +from conductor.client.automator.task_handler import TaskHandler, get_registered_workers from conductor.client.configuration.configuration import Configuration from conductor.client.worker.worker_task import worker_task from conductor.client.http.models.workflow_def import WorkflowDef @@ -141,6 +140,16 @@ async def async_lease_fast_no_hb(job_id: str) -> dict: class TestAsyncLeaseExtension(unittest.TestCase): + # Only the workers defined in THIS module. Used to scope the TaskHandler so + # it doesn't spin up every @worker_task registered across the imported test + # suite (that was ~24 worker processes started/torn down per test). + WORKER_TASK_NAMES = { + 'async_lease_heartbeat_task', + 'async_lease_no_heartbeat_task', + 'async_lease_fast_with_hb', + 'async_lease_fast_no_hb', + } + @classmethod def setUpClass(cls): from tests.integration.conftest import skip_if_server_unavailable @@ -150,6 +159,27 @@ def setUpClass(cls): cls.metadata_client = OrkesMetadataClient(cls.config) cls.workflow_client = OrkesWorkflowClient(cls.config) + # Start workers ONCE for the whole class (not per test) and only the + # four workers this module needs. Both changes cut the repeated + # process-scan/start/stop overhead that dominated the runtime. + workers = [ + w for w in get_registered_workers() + if w.get_task_definition_name() in cls.WORKER_TASK_NAMES + ] + cls._task_handler = TaskHandler( + workers=workers, + configuration=cls.config, + scan_for_annotated_workers=False, + ) + cls._task_handler.start_processes() + time.sleep(3) # let workers start (once, not per test) + + @classmethod + def tearDownClass(cls): + handler = getattr(cls, '_task_handler', None) + if handler is not None: + handler.stop_processes() + def _register_workflow(self, wf_name, task_names): """Register a workflow with one or more tasks in sequence.""" workflow = WorkflowDef(name=wf_name, version=1) @@ -186,23 +216,6 @@ def _wait_for_workflow(self, wf_id, timeout_seconds=90): time.sleep(1) return self.workflow_client.get_workflow(wf_id, include_tasks=True) - def _run_workers_in_background(self, duration_seconds=90): - """Start workers in a background thread, return stop function.""" - handler = TaskHandler( - configuration=self.config, - scan_for_annotated_workers=True, - ) - handler.start_processes() - - def stop(): - handler.stop_processes() - - timer = threading.Timer(duration_seconds, stop) - timer.daemon = True - timer.start() - - return stop - # -- Tests ---------------------------------------------------------------- def test_01_async_with_heartbeat_completes(self): @@ -215,30 +228,24 @@ def test_01_async_with_heartbeat_completes(self): wf_name = 'test_async_lease_heartbeat' self._register_workflow(wf_name, 'async_lease_heartbeat_task') - stop_workers = self._run_workers_in_background(duration_seconds=90) - time.sleep(3) # let workers start + wf_id = self._start_workflow(wf_name, 'ASYNC-HB-001') + wf = self._wait_for_workflow(wf_id, timeout_seconds=80) - try: - wf_id = self._start_workflow(wf_name, 'ASYNC-HB-001') - wf = self._wait_for_workflow(wf_id, timeout_seconds=80) + print(f"\n Workflow ID: {wf_id}") + print(f" Final status: {wf.status}") + for task in (wf.tasks or []): + print(f" Task {task.task_def_name}: {task.status}") - print(f"\n Workflow ID: {wf_id}") - print(f" Final status: {wf.status}") - for task in (wf.tasks or []): - print(f" Task {task.task_def_name}: {task.status}") + self.assertEqual(wf.status, 'COMPLETED', + f"Workflow should COMPLETE with heartbeat, got {wf.status}") - self.assertEqual(wf.status, 'COMPLETED', - f"Workflow should COMPLETE with heartbeat, got {wf.status}") - - tasks_by_ref = {t.reference_task_name: t for t in wf.tasks} - task = tasks_by_ref.get('async_lease_heartbeat_task_ref') - self.assertIsNotNone(task) - self.assertEqual(task.status, 'COMPLETED') - self.assertEqual(task.output_data.get('job_id'), 'ASYNC-HB-001') - self.assertEqual(task.output_data.get('slept'), TASK_SLEEP_SECONDS) - print("\n PASS: Async task completed with heartbeat keeping lease alive") - finally: - stop_workers() + tasks_by_ref = {t.reference_task_name: t for t in wf.tasks} + task = tasks_by_ref.get('async_lease_heartbeat_task_ref') + self.assertIsNotNone(task) + self.assertEqual(task.status, 'COMPLETED') + self.assertEqual(task.output_data.get('job_id'), 'ASYNC-HB-001') + self.assertEqual(task.output_data.get('slept'), TASK_SLEEP_SECONDS) + print("\n PASS: Async task completed with heartbeat keeping lease alive") def test_02_async_without_heartbeat_times_out(self): """Async task WITHOUT lease_extend_enabled times out when sleep > responseTimeout.""" @@ -250,29 +257,23 @@ def test_02_async_without_heartbeat_times_out(self): wf_name = 'test_async_lease_no_heartbeat' self._register_workflow(wf_name, 'async_lease_no_heartbeat_task') - stop_workers = self._run_workers_in_background(duration_seconds=90) - time.sleep(3) + wf_id = self._start_workflow(wf_name, 'ASYNC-NOHB-001') + wf = self._wait_for_workflow(wf_id, timeout_seconds=80) - try: - wf_id = self._start_workflow(wf_name, 'ASYNC-NOHB-001') - wf = self._wait_for_workflow(wf_id, timeout_seconds=80) - - print(f"\n Workflow ID: {wf_id}") - print(f" Final status: {wf.status}") - for task in (wf.tasks or []): - print(f" Task {task.task_def_name}: {task.status}") - - self.assertIn(wf.status, ('FAILED', 'TIMED_OUT'), - f"Workflow should FAIL/TIMEOUT without heartbeat, got {wf.status}") - - tasks_by_ref = {t.reference_task_name: t for t in wf.tasks} - task = tasks_by_ref.get('async_lease_no_heartbeat_task_ref') - self.assertIsNotNone(task) - self.assertIn(task.status, ('TIMED_OUT', 'FAILED', 'CANCELED'), - f"Task should be TIMED_OUT/FAILED, got {task.status}") - print("\n PASS: Async task timed out as expected without heartbeat") - finally: - stop_workers() + print(f"\n Workflow ID: {wf_id}") + print(f" Final status: {wf.status}") + for task in (wf.tasks or []): + print(f" Task {task.task_def_name}: {task.status}") + + self.assertIn(wf.status, ('FAILED', 'TIMED_OUT'), + f"Workflow should FAIL/TIMEOUT without heartbeat, got {wf.status}") + + tasks_by_ref = {t.reference_task_name: t for t in wf.tasks} + task = tasks_by_ref.get('async_lease_no_heartbeat_task_ref') + self.assertIsNotNone(task) + self.assertIn(task.status, ('TIMED_OUT', 'FAILED', 'CANCELED'), + f"Task should be TIMED_OUT/FAILED, got {task.status}") + print("\n PASS: Async task timed out as expected without heartbeat") def test_03_no_performance_overhead(self): """Heartbeat tracking adds no meaningful overhead to fast async tasks.""" @@ -286,58 +287,52 @@ def test_03_no_performance_overhead(self): self._register_workflow(wf_with_hb, 'async_lease_fast_with_hb') self._register_workflow(wf_no_hb, 'async_lease_fast_no_hb') - stop_workers = self._run_workers_in_background(duration_seconds=120) - time.sleep(3) - - try: - # Run tasks WITH heartbeat tracking - hb_workflow_ids = [] - for i in range(PERF_TASK_COUNT): - wf_id = self._start_workflow(wf_with_hb, f'PERF-HB-{i:03d}') - hb_workflow_ids.append(wf_id) - - # Run tasks WITHOUT heartbeat tracking - no_hb_workflow_ids = [] - for i in range(PERF_TASK_COUNT): - wf_id = self._start_workflow(wf_no_hb, f'PERF-NOHB-{i:03d}') - no_hb_workflow_ids.append(wf_id) - - # Wait for all to complete - hb_times = [] - for wf_id in hb_workflow_ids: - wf = self._wait_for_workflow(wf_id, timeout_seconds=30) - self.assertEqual(wf.status, 'COMPLETED', - f"Fast HB task should complete, got {wf.status}") - task = wf.tasks[0] - duration_ms = task.end_time - task.start_time - hb_times.append(duration_ms) - - no_hb_times = [] - for wf_id in no_hb_workflow_ids: - wf = self._wait_for_workflow(wf_id, timeout_seconds=30) - self.assertEqual(wf.status, 'COMPLETED', - f"Fast no-HB task should complete, got {wf.status}") - task = wf.tasks[0] - duration_ms = task.end_time - task.start_time - no_hb_times.append(duration_ms) - - avg_hb = sum(hb_times) / len(hb_times) - avg_no_hb = sum(no_hb_times) / len(no_hb_times) - overhead_ms = avg_hb - avg_no_hb - overhead_pct = (overhead_ms / avg_no_hb * 100) if avg_no_hb > 0 else 0 - - print(f"\n With heartbeat: avg {avg_hb:.0f}ms {hb_times}") - print(f" Without heartbeat: avg {avg_no_hb:.0f}ms {no_hb_times}") - print(f" Overhead: {overhead_ms:+.0f}ms ({overhead_pct:+.1f}%)") - - # Heartbeat tracking should add < 500ms overhead per task - # (LeaseManager.track is just a dict insert + set add) - self.assertLess(overhead_ms, 500, - f"Heartbeat overhead too high: {overhead_ms:.0f}ms") - - print("\n PASS: No meaningful performance overhead from heartbeat tracking") - finally: - stop_workers() + # Run tasks WITH heartbeat tracking + hb_workflow_ids = [] + for i in range(PERF_TASK_COUNT): + wf_id = self._start_workflow(wf_with_hb, f'PERF-HB-{i:03d}') + hb_workflow_ids.append(wf_id) + + # Run tasks WITHOUT heartbeat tracking + no_hb_workflow_ids = [] + for i in range(PERF_TASK_COUNT): + wf_id = self._start_workflow(wf_no_hb, f'PERF-NOHB-{i:03d}') + no_hb_workflow_ids.append(wf_id) + + # Wait for all to complete + hb_times = [] + for wf_id in hb_workflow_ids: + wf = self._wait_for_workflow(wf_id, timeout_seconds=30) + self.assertEqual(wf.status, 'COMPLETED', + f"Fast HB task should complete, got {wf.status}") + task = wf.tasks[0] + duration_ms = task.end_time - task.start_time + hb_times.append(duration_ms) + + no_hb_times = [] + for wf_id in no_hb_workflow_ids: + wf = self._wait_for_workflow(wf_id, timeout_seconds=30) + self.assertEqual(wf.status, 'COMPLETED', + f"Fast no-HB task should complete, got {wf.status}") + task = wf.tasks[0] + duration_ms = task.end_time - task.start_time + no_hb_times.append(duration_ms) + + avg_hb = sum(hb_times) / len(hb_times) + avg_no_hb = sum(no_hb_times) / len(no_hb_times) + overhead_ms = avg_hb - avg_no_hb + overhead_pct = (overhead_ms / avg_no_hb * 100) if avg_no_hb > 0 else 0 + + print(f"\n With heartbeat: avg {avg_hb:.0f}ms {hb_times}") + print(f" Without heartbeat: avg {avg_no_hb:.0f}ms {no_hb_times}") + print(f" Overhead: {overhead_ms:+.0f}ms ({overhead_pct:+.1f}%)") + + # Heartbeat tracking should add < 500ms overhead per task + # (LeaseManager.track is just a dict insert + set add) + self.assertLess(overhead_ms, 500, + f"Heartbeat overhead too high: {overhead_ms:.0f}ms") + + print("\n PASS: No meaningful performance overhead from heartbeat tracking") if __name__ == '__main__': diff --git a/tests/integration/test_authorization_client_intg.py b/tests/integration/test_authorization_client_intg.py index 8e9146a6f..62f9ecfc6 100644 --- a/tests/integration/test_authorization_client_intg.py +++ b/tests/integration/test_authorization_client_intg.py @@ -16,6 +16,7 @@ from conductor.client.http.models.upsert_user_request import UpsertUserRequest from conductor.client.orkes.models.access_type import AccessType from conductor.client.orkes.models.metadata_tag import MetadataTag +from conductor.client.http.rest import ApiException from conductor.client.orkes.orkes_authorization_client import OrkesAuthorizationClient logger = logging.getLogger( @@ -50,7 +51,7 @@ def setUpClass(cls): cls.test_app_name = f"test_app_{cls.timestamp}" cls.test_user_id = f"test_user_{cls.timestamp}@example.com" cls.test_group_id = f"test_group_{cls.timestamp}" - cls.test_role_name = f"test_role_{cls.timestamp}" + cls.test_role_name = f"TEST_ROLE_{cls.timestamp}" cls.test_gateway_config_id = None # Store created resource IDs for cleanup @@ -343,6 +344,7 @@ def test_19_upsert_group(self): request = UpsertGroupRequest() request.description = "Test Group" + request.roles = [] group = self.client.upsert_group(request, self.test_group_id) @@ -585,7 +587,12 @@ def test_42_list_gateway_auth_configs(self): """Test: list_gateway_auth_configs""" logger.info('TEST: list_gateway_auth_configs') - configs = self.client.list_gateway_auth_configs() + try: + configs = self.client.list_gateway_auth_configs() + except ApiException as e: + if e.status == 404: + self.skipTest('API Gateway auth-config endpoint not available on this server') + raise self.assertIsNotNone(configs) self.assertIsInstance(configs, list) diff --git a/tests/integration/test_authorization_complete.py b/tests/integration/test_authorization_complete.py index 2ba4bcf0d..6cbad5b83 100644 --- a/tests/integration/test_authorization_complete.py +++ b/tests/integration/test_authorization_complete.py @@ -27,7 +27,7 @@ from conductor.client.http.models.authentication_config import AuthenticationConfig from conductor.client.orkes.models.access_type import AccessType from conductor.client.orkes.models.metadata_tag import MetadataTag -from conductor.client.http.rest import RestException +from conductor.client.http.rest import ApiException, RestException @pytest.fixture(scope="module") @@ -239,7 +239,7 @@ def test_access_key_lifecycle(self, auth_client, test_run_id, cleanup_tracker): # Get app by access key (Method 6) found_app = auth_client.get_app_by_access_key_id(created_key.id) - assert found_app == app.id + assert found_app['id'] == app.id # Delete access key (Method 15) - handled in cleanup @@ -469,7 +469,12 @@ def test_gateway_auth_config(self, auth_client, test_run_id, cleanup_tracker): auth_config.api_keys = ["test-key"] auth_config.fallback_to_default_auth = False - created = auth_client.create_gateway_auth_config(auth_config) + try: + created = auth_client.create_gateway_auth_config(auth_config) + except ApiException as e: + if e.status == 404: + pytest.skip('API Gateway auth-config endpoint not available on this server') + raise cleanup_tracker['auth_configs'].append(config_id) assert created.get('id') == config_id diff --git a/tests/integration/workflow/test_workflow_execution.py b/tests/integration/workflow/test_workflow_execution.py index 2583ca3f8..951c88f1a 100644 --- a/tests/integration/workflow/test_workflow_execution.py +++ b/tests/integration/workflow/test_workflow_execution.py @@ -46,21 +46,21 @@ def run_workflow_execution_tests(configuration: Configuration, workflow_executor ) task_handler.start_processes() try: - test_get_workflow_by_correlation_ids(workflow_executor) + scenario_get_workflow_by_correlation_ids(workflow_executor) logger.debug('finished workflow correlation ids test') - test_workflow_registration(workflow_executor) + scenario_workflow_registration(workflow_executor) logger.debug('finished workflow registration tests') - test_workflow_execution( + scenario_workflow_execution( workflow_quantity=6, workflow_name=WORKFLOW_NAME, workflow_executor=workflow_executor, workflow_completion_timeout=5.0 ) - test_decorated_workers(workflow_executor) + scenario_decorated_workers(workflow_executor) logger.debug('finished decorated workers tests') - test_execute_workflow_async_features(workflow_executor) + scenario_execute_workflow_async_features(workflow_executor) logger.debug('finished execute_workflow reactive features tests') - test_execute_workflow_error_handling(workflow_executor) + scenario_execute_workflow_error_handling(workflow_executor) logger.debug('finished execute_workflow error handling tests') run_signal_tests(configuration, workflow_executor) logger.debug('finished signal API tests') @@ -86,7 +86,7 @@ def generate_tasks_defs(): return [python_simple_task_from_code] -def test_get_workflow_by_correlation_ids(workflow_executor: WorkflowExecutor): +def scenario_get_workflow_by_correlation_ids(workflow_executor: WorkflowExecutor): _run_with_retry_attempt( workflow_executor.get_by_correlation_ids, { @@ -98,7 +98,7 @@ def test_get_workflow_by_correlation_ids(workflow_executor: WorkflowExecutor): ) -def test_workflow_registration(workflow_executor: WorkflowExecutor): +def scenario_workflow_registration(workflow_executor: WorkflowExecutor): workflow = generate_workflow(workflow_executor) try: workflow_executor.metadata_client.unregister_workflow_def_with_http_info( @@ -113,7 +113,7 @@ def test_workflow_registration(workflow_executor: WorkflowExecutor): ) -def test_decorated_workers( +def scenario_decorated_workers( workflow_executor: WorkflowExecutor, workflow_name: str = 'TestPythonDecoratedWorkerWf', ) -> None: @@ -154,7 +154,7 @@ def test_decorated_workers( workflow_executor.metadata_client.unregister_workflow_def(wf.name, wf.version) -def test_workflow_execution( +def scenario_workflow_execution( workflow_quantity: int, workflow_name: str, workflow_executor: WorkflowExecutor, @@ -229,7 +229,7 @@ def _run_with_retry_attempt(f, params, retries=4) -> None: raise e sleep(1 << attempt) -def test_execute_workflow_async_features(workflow_executor: WorkflowExecutor): +def scenario_execute_workflow_async_features(workflow_executor: WorkflowExecutor): """Test the execute_workflow method with reactive features (consistency and return_strategy)""" logger.debug('Starting execute_workflow reactive features tests') @@ -367,7 +367,7 @@ def test_execute_workflow_async_features(workflow_executor: WorkflowExecutor): logger.debug('All execute_workflow reactive features tests passed!') -def test_execute_workflow_error_handling(workflow_executor: WorkflowExecutor): +def scenario_execute_workflow_error_handling(workflow_executor: WorkflowExecutor): """Test error handling in execute_workflow with invalid parameters""" logger.debug('Starting execute_workflow error handling tests') @@ -433,19 +433,19 @@ def run_signal_tests(configuration: Configuration, workflow_executor: WorkflowEx _register_signal_test_workflows(workflow_executor) # Test sync signal with different return strategies - test_signal_target_workflow(workflow_executor) - test_signal_blocking_workflow(workflow_executor) - test_signal_blocking_task(workflow_executor) - test_signal_blocking_task_input(workflow_executor) + scenario_signal_target_workflow(workflow_executor) + scenario_signal_blocking_workflow(workflow_executor) + scenario_signal_blocking_task(workflow_executor) + scenario_signal_blocking_task_input(workflow_executor) # Test default return strategy - test_signal_default_strategy(workflow_executor) + scenario_signal_default_strategy(workflow_executor) # Test async signal - test_signal_async(workflow_executor) + scenario_signal_async(workflow_executor) # Test to_dict fix - test_signal_to_dict_fix(workflow_executor) + scenario_signal_to_dict_fix(workflow_executor) logger.info('All signal tests completed successfully') @@ -569,7 +569,7 @@ def _complete_workflow(workflow_executor: WorkflowExecutor, workflow_id: str): raise -def test_signal_target_workflow(workflow_executor: WorkflowExecutor): +def scenario_signal_target_workflow(workflow_executor: WorkflowExecutor): """Test signal with TARGET_WORKFLOW return strategy""" logger.info('Testing signal with TARGET_WORKFLOW strategy...') @@ -640,7 +640,7 @@ def test_signal_target_workflow(workflow_executor: WorkflowExecutor): logger.info('TARGET_WORKFLOW strategy test completed') -def test_signal_blocking_workflow(workflow_executor: WorkflowExecutor): +def scenario_signal_blocking_workflow(workflow_executor: WorkflowExecutor): """Test signal with BLOCKING_WORKFLOW return strategy""" logger.info('Testing signal with BLOCKING_WORKFLOW strategy...') @@ -669,7 +669,7 @@ def test_signal_blocking_workflow(workflow_executor: WorkflowExecutor): logger.info('BLOCKING_WORKFLOW strategy test completed') -def test_signal_blocking_task(workflow_executor: WorkflowExecutor): +def scenario_signal_blocking_task(workflow_executor: WorkflowExecutor): """Test signal with BLOCKING_TASK return strategy""" logger.info('Testing signal with BLOCKING_TASK strategy...') @@ -699,7 +699,7 @@ def test_signal_blocking_task(workflow_executor: WorkflowExecutor): logger.info('BLOCKING_TASK strategy test completed') -def test_signal_blocking_task_input(workflow_executor: WorkflowExecutor): +def scenario_signal_blocking_task_input(workflow_executor: WorkflowExecutor): """Test signal with BLOCKING_TASK_INPUT return strategy""" logger.info('Testing signal with BLOCKING_TASK_INPUT strategy...') @@ -730,7 +730,7 @@ def test_signal_blocking_task_input(workflow_executor: WorkflowExecutor): logger.info('BLOCKING_TASK_INPUT strategy test completed') -def test_signal_default_strategy(workflow_executor: WorkflowExecutor): +def scenario_signal_default_strategy(workflow_executor: WorkflowExecutor): """Test signal with default return strategy""" logger.info('Testing signal with default strategy...') @@ -754,7 +754,7 @@ def test_signal_default_strategy(workflow_executor: WorkflowExecutor): logger.info('Default strategy test completed') -def test_signal_async(workflow_executor: WorkflowExecutor): +def scenario_signal_async(workflow_executor: WorkflowExecutor): """Test async signal""" logger.info('Testing async signal...') @@ -775,7 +775,7 @@ def test_signal_async(workflow_executor: WorkflowExecutor): logger.info('Async signal test completed') -def test_signal_to_dict_fix(workflow_executor: WorkflowExecutor): +def scenario_signal_to_dict_fix(workflow_executor: WorkflowExecutor): """Test that to_dict() returns actual values, not property objects""" logger.info('Testing to_dict() method fix...') From 7a99befe7c0d38373db27269902cf2f1171469d6 Mon Sep 17 00:00:00 2001 From: Chris Hagglund Date: Mon, 13 Jul 2026 13:06:07 -0600 Subject: [PATCH 02/25] wip --- scripts/run_integration_tests.sh | 69 ++++++++++++++++++++++++++++++++ 1 file changed, 69 insertions(+) create mode 100755 scripts/run_integration_tests.sh diff --git a/scripts/run_integration_tests.sh b/scripts/run_integration_tests.sh new file mode 100755 index 000000000..d673470c0 --- /dev/null +++ b/scripts/run_integration_tests.sh @@ -0,0 +1,69 @@ +#!/usr/bin/env bash +# +# Run the SDK integration test suite locally, mirroring the `integration-test` +# job in .github/workflows/pull_request.yml. +# +# The AI/agentic integration tests are excluded on purpose: they target a +# dedicated AI-enabled server (test_ai_task_types.py and test_ai_examples.py +# hardcode http://localhost:7001/api, and test_agentic_workflows.py needs an +# `openai` LLM provider configured on the server), so they don't run against +# the standard test server this suite targets. +# +# The performance test (test_update_task_v2_perf.py) is also excluded by +# default: it submits ~1000 workflows and takes several minutes. Pass +# --with-perf to include it. +# +# Server connection is read from the environment (see +# src/conductor/client/configuration/configuration.py): +# CONDUCTOR_SERVER_URL (required; defaults to http://localhost:8080/api) +# CONDUCTOR_AUTH_KEY (optional; needed for Orkes/authenticated servers) +# CONDUCTOR_AUTH_SECRET (optional; needed for Orkes/authenticated servers) +# +# Usage: +# export CONDUCTOR_SERVER_URL="http://localhost:8080/api" +# ./scripts/run_integration_tests.sh +# ./scripts/run_integration_tests.sh --with-perf # also run the perf test +# +# Any other arguments are passed straight through to pytest, which is handy for +# targeting a subset of tests or getting more detail on failures: +# +# # run a subset (by keyword or path) with live output +# ./scripts/run_integration_tests.sh -s -k lease +# ./scripts/run_integration_tests.sh tests/integration/test_lease_extension.py +# +# # short tracebacks + a one-line reason for every failure/skip, with live logs +# ./scripts/run_integration_tests.sh -ra --tb=short --log-cli-level=INFO +# +# # stop at the first failure instead of waiting for the whole suite +# ./scripts/run_integration_tests.sh -x --tb=long +# +# # re-run just specific tests, e.g. the ones that failed +# ./scripts/run_integration_tests.sh -s -k "upsert_group or create_role" +# +# # show the 15 slowest tests (setup/call/teardown timings) after the run +# ./scripts/run_integration_tests.sh --durations=15 + +set -euo pipefail + +REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +cd "$REPO_ROOT" + +# The perf test is skipped by default; --with-perf opts back in. +perf_ignore=(--ignore=tests/integration/test_update_task_v2_perf.py) +pytest_args=() +for arg in "$@"; do + if [[ "$arg" == "--with-perf" ]]; then + perf_ignore=() + else + pytest_args+=("$arg") + fi +done + +# Note: the "${arr[@]+"${arr[@]}"}" form is required so empty arrays don't trip +# "unbound variable" under `set -u` on bash 3.2 (the default macOS bash). +exec python3 -m pytest tests/integration -v \ + --ignore=tests/integration/test_ai_task_types.py \ + --ignore=tests/integration/test_ai_examples.py \ + --ignore=tests/integration/test_agentic_workflows.py \ + ${perf_ignore[@]+"${perf_ignore[@]}"} \ + ${pytest_args[@]+"${pytest_args[@]}"} From c7d34d6a5e75cd3f0fa47075476bb28a89054a76 Mon Sep 17 00:00:00 2001 From: Chris Hagglund Date: Mon, 13 Jul 2026 13:52:00 -0600 Subject: [PATCH 03/25] wip, testing/fixes --- .github/workflows/pull_request.yml | 7 +- pyproject.toml | 6 ++ scripts/run_integration_tests.sh | 77 ++++++++++++++++--- tests/integration/README.md | 19 +++++ .../client/orkes/test_orkes_clients.py | 23 +++++- .../metadata/test_workflow_definition.py | 20 ++++- .../integration/test_async_lease_extension.py | 3 + .../test_authorization_client_intg.py | 7 +- .../test_authorization_complete.py | 4 +- tests/integration/test_comprehensive_e2e.py | 37 +++++---- tests/integration/test_lease_extension.py | 3 + .../integration/test_workflow_client_intg.py | 3 + 12 files changed, 175 insertions(+), 34 deletions(-) diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index d71609dca..391edca20 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -106,6 +106,11 @@ jobs: integration-test: runs-on: ubuntu-latest timeout-minutes: 30 + strategy: + fail-fast: false + matrix: + bucket: [test-all, long-sync, long-async, core] + name: integration-test (${{ matrix.bucket }}) env: CONDUCTOR_SERVER_URL: ${{ vars.SDKDEV_V5_SERVER_URL }} CONDUCTOR_AUTH_KEY: ${{ vars.SDKDEV_V5_AUTH_KEY }} @@ -127,4 +132,4 @@ jobs: pip install pytest - name: Run integration tests - run: bash scripts/run_integration_tests.sh + run: bash scripts/run_integration_tests.sh --bucket=${{ matrix.bucket }} diff --git a/pyproject.toml b/pyproject.toml index 8e949e27a..b9c8e74c1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -192,11 +192,17 @@ line-ending = "auto" [tool.pytest.ini_options] pythonpath = ["src"] markers = [ +<<<<<<< HEAD "integration: requires a live Agentspan-compatible server", "e2e: requires a live server and AGENTSPAN_SERVER_URL", "sse: requires SSE streaming (AGENTSPAN_STREAMING_ENABLED=true)", "agent_correctness: marks tests as agent correctness tests", "semantic: marks tests that use an LLM judge for semantic assertions", +======= + "slow_sync: long-running sync lease-extension tests (~90s)", + "slow_async: long-running async lease-extension tests (~90s)", + "slow_test_all: long-running aggregate workflow-client test_all (~83s)", +>>>>>>> 973c79cf (wip, testing/fixes) ] [tool.coverage.run] diff --git a/scripts/run_integration_tests.sh b/scripts/run_integration_tests.sh index d673470c0..88d73f892 100755 --- a/scripts/run_integration_tests.sh +++ b/scripts/run_integration_tests.sh @@ -24,6 +24,19 @@ # ./scripts/run_integration_tests.sh # ./scripts/run_integration_tests.sh --with-perf # also run the perf test # +# Buckets (--bucket=): the slowest tests are split into their own buckets +# so they can run as separate parallel CI jobs and are skipped by default +# locally. Each slow bucket is path-scoped so it only imports its own module. +# core (default) everything except the slow buckets below +# long-sync sync lease-extension tests (test_lease_extension.py, ~90s) +# long-async async lease-extension tests (test_async_lease_extension.py, ~90s) +# test-all aggregate workflow-client test_all (test_workflow_client_intg.py, ~83s) +# all the full suite (no bucket filtering) — for a complete local run +# +# ./scripts/run_integration_tests.sh # fast: skips slow buckets +# ./scripts/run_integration_tests.sh --bucket=long-sync +# ./scripts/run_integration_tests.sh --bucket=all # run everything +# # Any other arguments are passed straight through to pytest, which is handy for # targeting a subset of tests or getting more detail on failures: # @@ -48,22 +61,66 @@ set -euo pipefail REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" cd "$REPO_ROOT" +# Slow-test files that get their own buckets / CI jobs (see --bucket above). +LEASE_SYNC=tests/integration/test_lease_extension.py +LEASE_ASYNC=tests/integration/test_async_lease_extension.py +TEST_ALL_FILE=tests/integration/test_workflow_client_intg.py + # The perf test is skipped by default; --with-perf opts back in. perf_ignore=(--ignore=tests/integration/test_update_task_v2_perf.py) +bucket="core" pytest_args=() for arg in "$@"; do - if [[ "$arg" == "--with-perf" ]]; then - perf_ignore=() - else - pytest_args+=("$arg") - fi + case "$arg" in + --with-perf) perf_ignore=() ;; + --bucket=*) bucket="${arg#*=}" ;; + *) pytest_args+=("$arg") ;; + esac done +# The AI/agentic tests always target a dedicated server (see header) and are +# never part of these buckets. +ai_ignore=( + --ignore=tests/integration/test_ai_task_types.py + --ignore=tests/integration/test_ai_examples.py + --ignore=tests/integration/test_agentic_workflows.py +) + +# Build the target paths + selection for the chosen bucket. The slow buckets are +# path-scoped so each job only imports its own module: this keeps +# scan_for_annotated_workers from starting unrelated workers and lets the four +# buckets run in parallel against one server without stealing each other's tasks. +case "$bucket" in + core) + targets=(tests/integration) + select=("${ai_ignore[@]}" ${perf_ignore[@]+"${perf_ignore[@]}"} \ + --ignore="$LEASE_SYNC" --ignore="$LEASE_ASYNC" --ignore="$TEST_ALL_FILE") + ;; + all) + targets=(tests/integration) + select=("${ai_ignore[@]}" ${perf_ignore[@]+"${perf_ignore[@]}"}) + ;; + long-sync) + targets=("$LEASE_SYNC") + select=(-m slow_sync) + ;; + long-async) + targets=("$LEASE_ASYNC") + select=(-m slow_async) + ;; + test-all) + targets=("$TEST_ALL_FILE") + select=(-m slow_test_all) + ;; + *) + echo "Unknown --bucket='$bucket' (expected: core, long-sync, long-async, test-all, all)" >&2 + exit 2 + ;; +esac + # Note: the "${arr[@]+"${arr[@]}"}" form is required so empty arrays don't trip # "unbound variable" under `set -u` on bash 3.2 (the default macOS bash). -exec python3 -m pytest tests/integration -v \ - --ignore=tests/integration/test_ai_task_types.py \ - --ignore=tests/integration/test_ai_examples.py \ - --ignore=tests/integration/test_agentic_workflows.py \ - ${perf_ignore[@]+"${perf_ignore[@]}"} \ +exec python3 -m pytest -v \ + "${targets[@]}" \ + ${select[@]+"${select[@]}"} \ ${pytest_args[@]+"${pytest_args[@]}"} diff --git a/tests/integration/README.md b/tests/integration/README.md index 58d0abcab..bc99d312d 100644 --- a/tests/integration/README.md +++ b/tests/integration/README.md @@ -58,6 +58,25 @@ export CONDUCTOR_SERVER_URL="http://localhost:8080/api" ./scripts/run_integration_tests.sh --with-perf ``` +By default this runs the fast `core` bucket and **skips the slowest tests** +(a few tests deliberately sleep ~50-90s to exercise lease-extension timeouts and +one aggregate `test_all`). Those live in their own buckets, each of which runs as +a separate parallel CI job. Select one with `--bucket=`: + +| Bucket | What it runs | +| --- | --- | +| `core` (default) | everything except the slow buckets below | +| `long-sync` | sync lease-extension tests (`test_lease_extension.py`, ~90s) | +| `long-async` | async lease-extension tests (`test_async_lease_extension.py`, ~90s) | +| `test-all` | aggregate workflow-client `test_all` (`test_workflow_client_intg.py`, ~83s) | +| `all` | the full suite (no bucket filtering) — for a complete local run | + +```bash +./scripts/run_integration_tests.sh # fast: skips the slow buckets +./scripts/run_integration_tests.sh --bucket=long-sync +./scripts/run_integration_tests.sh --bucket=all # run everything +``` + Any extra arguments pass straight through to pytest, which is handy for targeting a subset of tests or getting more detail on failures. See additional options and examples in the comments at the top of diff --git a/tests/integration/client/orkes/test_orkes_clients.py b/tests/integration/client/orkes/test_orkes_clients.py index 56e22ae4c..f00adb28f 100644 --- a/tests/integration/client/orkes/test_orkes_clients.py +++ b/tests/integration/client/orkes/test_orkes_clients.py @@ -1,4 +1,5 @@ import json +import time from shortuuid import uuid @@ -38,6 +39,19 @@ TEST_IP_JSON = 'tests/integration/resources/test_data/loan_workflow_input.json' +def _retry_on_404(func, *args, retries=5, **kwargs): + # Updating a task by ref name can transiently 404 while the server is still + # scheduling the referenced task. Retry with backoff to tolerate that race. + for attempt in range(retries): + try: + return func(*args, **kwargs) + except ApiException as e: + if e.status == 404 and attempt < retries - 1: + time.sleep(1 << attempt) + continue + raise + + class TestOrkesClients: def __init__(self, configuration: Configuration): self.api_client = ApiClient(configuration) @@ -576,7 +590,8 @@ def __test_task_execution_lifecycle(self): polledTask = batchPolledTasks[0] # Update first task of second workflow - self.task_client.update_task_by_ref_name( + _retry_on_404( + self.task_client.update_task_by_ref_name, workflow_uuid_2, polledTask.reference_task_name, "COMPLETED", @@ -584,7 +599,8 @@ def __test_task_execution_lifecycle(self): ) # Update second task of first workflow - self.task_client.update_task_by_ref_name( + _retry_on_404( + self.task_client.update_task_by_ref_name, workflow_uuid_2, "simple_task_ref_2", "COMPLETED", "task 2 op 1st wf" ) @@ -593,7 +609,8 @@ def __test_task_execution_lifecycle(self): polledTask = self.task_client.poll_task(TASK_TYPE) # Update second task of second workflow - self.task_client.update_task_sync( + _retry_on_404( + self.task_client.update_task_sync, workflow_uuid, "simple_task_ref_2", "COMPLETED", "task 1 op 2nd wf" ) diff --git a/tests/integration/metadata/test_workflow_definition.py b/tests/integration/metadata/test_workflow_definition.py index f513973b4..95f388271 100644 --- a/tests/integration/metadata/test_workflow_definition.py +++ b/tests/integration/metadata/test_workflow_definition.py @@ -1,6 +1,8 @@ +import time from typing import List from conductor.client.http.models import TaskDef +from conductor.client.http.rest import ApiException from conductor.client.http.models.start_workflow_request import StartWorkflowRequest from conductor.client.workflow.conductor_workflow import ConductorWorkflow from conductor.client.workflow.executor.workflow_executor import WorkflowExecutor @@ -68,7 +70,23 @@ def test_kitchensink_workflow_registration(workflow_executor: WorkflowExecutor) if type(workflow_id) != str or workflow_id == '': raise Exception(f'failed to start workflow, name: {WORKFLOW_NAME}') - workflow_executor.terminate(workflow_id=workflow_id, reason="End test") + _terminate_with_retry(workflow_executor, workflow_id, reason="End test") + + +def _terminate_with_retry(workflow_executor: WorkflowExecutor, workflow_id: str, + reason: str, retries: int = 5) -> None: + # Terminating a workflow that the server is still evaluating can transiently + # return 423 Locked ("Unable to acquire lock"). Retry with backoff so the + # test tolerates that race instead of failing. + for attempt in range(retries): + try: + workflow_executor.terminate(workflow_id=workflow_id, reason=reason) + return + except ApiException as e: + if e.status == 423 and attempt < retries - 1: + time.sleep(1 << attempt) + continue + raise def generate_simple_task(id: int) -> SimpleTask: diff --git a/tests/integration/test_async_lease_extension.py b/tests/integration/test_async_lease_extension.py index ddd27ce6e..bec8f7712 100644 --- a/tests/integration/test_async_lease_extension.py +++ b/tests/integration/test_async_lease_extension.py @@ -26,6 +26,8 @@ import time import unittest +import pytest + sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))) from conductor.client.automator.task_handler import TaskHandler, get_registered_workers @@ -138,6 +140,7 @@ async def async_lease_fast_no_hb(job_id: str) -> dict: # -- Test class -------------------------------------------------------------- +@pytest.mark.slow_async class TestAsyncLeaseExtension(unittest.TestCase): # Only the workers defined in THIS module. Used to scope the TaskHandler so diff --git a/tests/integration/test_authorization_client_intg.py b/tests/integration/test_authorization_client_intg.py index 62f9ecfc6..da0569c6f 100644 --- a/tests/integration/test_authorization_client_intg.py +++ b/tests/integration/test_authorization_client_intg.py @@ -532,7 +532,9 @@ def test_38_create_role(self): request = CreateOrUpdateRoleRequest() request.name = self.test_role_name - request.permissions = ["workflow:read"] + # Permission strings must be in the server's ACCESS_RESOURCE form + # (e.g. READ_WORKFLOW_DEF); "workflow:read" is rejected as invalid. + request.permissions = ["READ_WORKFLOW_DEF"] result = self.client.create_role(request) @@ -553,7 +555,8 @@ def test_40_update_role(self): request = CreateOrUpdateRoleRequest() request.name = self.test_role_name - request.permissions = ["workflow:read", "workflow:execute"] + # Valid ACCESS_RESOURCE permission strings (see test_38_create_role). + request.permissions = ["READ_WORKFLOW_DEF", "EXECUTE_WORKFLOW_DEF"] result = self.client.update_role(self.test_role_name, request) diff --git a/tests/integration/test_authorization_complete.py b/tests/integration/test_authorization_complete.py index 6cbad5b83..1ede72ffe 100644 --- a/tests/integration/test_authorization_complete.py +++ b/tests/integration/test_authorization_complete.py @@ -292,7 +292,9 @@ def test_user_lifecycle(self, auth_client, test_run_id, cleanup_tracker): target_type="WORKFLOW_DEF", target_id="test-workflow" ) - assert isinstance(result, bool) + # The server returns a per-access permission map + # (e.g. {'CREATE': True, 'READ': True, ...}), not a single bool. + assert isinstance(result, dict) class TestGroupManagement: diff --git a/tests/integration/test_comprehensive_e2e.py b/tests/integration/test_comprehensive_e2e.py index f300b8c99..5c4acb722 100644 --- a/tests/integration/test_comprehensive_e2e.py +++ b/tests/integration/test_comprehensive_e2e.py @@ -28,7 +28,6 @@ import os import sys import time -import threading import unittest from dataclasses import dataclass from typing import Optional, List, Dict, Union @@ -144,6 +143,7 @@ def setUpClass(cls): ) cls.workers_started = False + cls.task_handler = None def test_01_create_workflow(self): """Create test workflow.""" @@ -174,22 +174,23 @@ def test_01_create_workflow(self): def test_02_start_workers(self): """Start workers and verify they initialize.""" print("\n" + "="*80 + "\nTEST 2: Start Workers\n" + "="*80) - - def run_workers(): - with TaskHandler( - configuration=self.config, - metrics_settings=self.metrics_settings, - scan_for_annotated_workers=True, - event_listeners=[self.event_collector] - ) as handler: - handler.start_processes() - time.sleep(90) # Run for test duration - handler.stop_processes() - - thread = threading.Thread(target=run_workers, daemon=True) - thread.start() + + # Start the workers and keep the handler on the class so it stays alive + # for the remaining tests and is stopped deterministically in + # tearDownClass. (A previous version ran the handler in a daemon thread + # with a fixed 90s sleep; if the suite finished sooner, the worker + # processes were never stopped and the interpreter hung at exit joining + # them.) + handler = TaskHandler( + configuration=self.config, + metrics_settings=self.metrics_settings, + scan_for_annotated_workers=True, + event_listeners=[self.event_collector] + ) + handler.start_processes() + self.__class__.task_handler = handler time.sleep(5) # Wait for startup - + print("✓ Workers started") self.__class__.workers_started = True self.assertTrue(self.workers_started) @@ -422,6 +423,10 @@ def test_08_summary_assertions(self): @classmethod def tearDownClass(cls): + handler = getattr(cls, 'task_handler', None) + if handler is not None: + handler.stop_processes() + cls.task_handler = None if os.path.exists(cls.metrics_dir): import shutil shutil.rmtree(cls.metrics_dir) diff --git a/tests/integration/test_lease_extension.py b/tests/integration/test_lease_extension.py index 404f2f020..ca7d5434c 100644 --- a/tests/integration/test_lease_extension.py +++ b/tests/integration/test_lease_extension.py @@ -24,6 +24,8 @@ import threading import unittest +import pytest + sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))) from conductor.client.automator.task_handler import TaskHandler @@ -96,6 +98,7 @@ def lease_no_heartbeat_task(job_id: str) -> dict: # -- Test class -------------------------------------------------------------- +@pytest.mark.slow_sync class TestLeaseExtension(unittest.TestCase): @classmethod diff --git a/tests/integration/test_workflow_client_intg.py b/tests/integration/test_workflow_client_intg.py index d38ca0802..4afceb316 100644 --- a/tests/integration/test_workflow_client_intg.py +++ b/tests/integration/test_workflow_client_intg.py @@ -2,6 +2,8 @@ import os import unittest +import pytest + from tests.integration.client.orkes.test_orkes_clients import TestOrkesClients from conductor.client.configuration.configuration import Configuration from conductor.client.configuration.settings.authentication_settings import AuthenticationSettings @@ -40,6 +42,7 @@ def setUpClass(cls): cls.workflow_client = OrkesWorkflowClient(cls.config) logger.info(f'setting up TestOrkesWorkflowClientIntg with config {cls.config}') + @pytest.mark.slow_test_all def test_all(self): logger.info('START: integration tests') configuration = self.config From a5cdf5aa834a39c8bc1ecb61eda03e7b608ac192 Mon Sep 17 00:00:00 2001 From: Chris Hagglund Date: Mon, 13 Jul 2026 14:00:45 -0600 Subject: [PATCH 04/25] target to run this now --- .github/workflows/pull_request.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index 391edca20..afa82f5d8 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -4,13 +4,16 @@ on: push: branches: - main + # Temporary: run CI on this branch's pushes until it's exercised via a + # real PR targeting main. Remove once that PR validates the workflow. + - integ-test-on-pr pull_request: branches: - main workflow_dispatch: concurrency: - group: ${{ github.workflow }}-${{ github.ref }} + group: ${{ github.workflow }}-${{ github.head_ref || github.ref }} cancel-in-progress: true jobs: From 8da59909f0a96bb9896cab19bf9faa768c78cedf Mon Sep 17 00:00:00 2001 From: Chris Hagglund Date: Tue, 14 Jul 2026 09:18:17 -0600 Subject: [PATCH 05/25] correct task polling sequence assumption error in test --- scripts/run_integration_tests.sh | 4 ++ .../client/orkes/test_orkes_clients.py | 63 ++++++++++--------- 2 files changed, 39 insertions(+), 28 deletions(-) diff --git a/scripts/run_integration_tests.sh b/scripts/run_integration_tests.sh index 88d73f892..cce260e3a 100755 --- a/scripts/run_integration_tests.sh +++ b/scripts/run_integration_tests.sh @@ -100,6 +100,10 @@ case "$bucket" in targets=(tests/integration) select=("${ai_ignore[@]}" ${perf_ignore[@]+"${perf_ignore[@]}"}) ;; + # The slow buckets below target a single specific file, so the perf test is + # never in scope and perf_ignore is unnecessary here. This also means + # --with-perf has no effect on these buckets; the perf test is only reachable + # via the core/all buckets. long-sync) targets=("$LEASE_SYNC") select=(-m slow_sync) diff --git a/tests/integration/client/orkes/test_orkes_clients.py b/tests/integration/client/orkes/test_orkes_clients.py index f00adb28f..2060da258 100644 --- a/tests/integration/client/orkes/test_orkes_clients.py +++ b/tests/integration/client/orkes/test_orkes_clients.py @@ -575,7 +575,7 @@ def __test_task_execution_lifecycle(self): assert self.task_client.get_queue_size_for_task(TASK_TYPE) == 1 taskResult = TaskResult( - workflow_instance_id=workflow_uuid, + workflow_instance_id=polledTask.workflow_instance_id, task_id=polledTask.task_id, status=TaskResultStatus.COMPLETED ) @@ -585,34 +585,41 @@ def __test_task_execution_lifecycle(self): task = self.task_client.get_task(polledTask.task_id) assert task.status == TaskResultStatus.COMPLETED - batchPolledTasks = self.task_client.batch_poll_tasks(TASK_TYPE) - assert len(batchPolledTasks) == 1 - - polledTask = batchPolledTasks[0] - # Update first task of second workflow - _retry_on_404( - self.task_client.update_task_by_ref_name, - workflow_uuid_2, - polledTask.reference_task_name, - "COMPLETED", - "task 2 op 2nd wf" - ) - - # Update second task of first workflow - _retry_on_404( - self.task_client.update_task_by_ref_name, - workflow_uuid_2, "simple_task_ref_2", "COMPLETED", "task 2 op 1st wf" - ) + # The second task of the first workflow and both tasks of the second + # workflow all share TASK_TYPE and land in the same queue, so a poll can + # return a task from either workflow in a non-deterministic order. Drive + # every update from the polled task's own workflow id and reference name + # instead of assuming which workflow/ref we got back (the previous + # hardcoded workflow_uuid_2 / "simple_task_ref_2" pairing produced + # spurious 404s whenever the queue handed back the other workflow's + # task). We still exercise update_task_by_ref_name and update_task_sync. - # # Second task of second workflow is in the queue - # assert self.task_client.getQueueSizeForTask(TASK_TYPE) == 1 - polledTask = self.task_client.poll_task(TASK_TYPE) - - # Update second task of second workflow - _retry_on_404( - self.task_client.update_task_sync, - workflow_uuid, "simple_task_ref_2", "COMPLETED", "task 1 op 2nd wf" - ) + # Three task executions remain (we completed one above); drain them all. + batchPolledTasks = self.task_client.batch_poll_tasks(TASK_TYPE) + remaining = 3 + completed = 0 + for polledTask in batchPolledTasks: + _retry_on_404( + self.task_client.update_task_by_ref_name, + polledTask.workflow_instance_id, + polledTask.reference_task_name, + "COMPLETED", + f"task op {completed + 1} (by ref name)" + ) + completed += 1 + + while completed < remaining: + polledTask = self.task_client.poll_task(TASK_TYPE) + assert polledTask is not None, \ + "expected a task to be available to poll while draining the queue" + _retry_on_404( + self.task_client.update_task_sync, + polledTask.workflow_instance_id, + polledTask.reference_task_name, + "COMPLETED", + f"task op {completed + 1} (sync)" + ) + completed += 1 queue_size = self.task_client.get_queue_size_for_task(TASK_TYPE) print(f'queue size for {TASK_TYPE} is {queue_size}') From dc14a2947cb9f463c02b2942fbe8ba69087ea108 Mon Sep 17 00:00:00 2001 From: Chris Hagglund Date: Tue, 14 Jul 2026 09:40:16 -0600 Subject: [PATCH 06/25] wait for wf more robustly in test --- tests/integration/test_comprehensive_e2e.py | 66 +++++++++++++++------ 1 file changed, 48 insertions(+), 18 deletions(-) diff --git a/tests/integration/test_comprehensive_e2e.py b/tests/integration/test_comprehensive_e2e.py index 5c4acb722..dd79a45b6 100644 --- a/tests/integration/test_comprehensive_e2e.py +++ b/tests/integration/test_comprehensive_e2e.py @@ -195,36 +195,66 @@ def test_02_start_workers(self): self.__class__.workers_started = True self.assertTrue(self.workers_started) - def test_03_execute_workflow(self): - """Execute workflow and verify completion.""" - print("\n" + "="*80 + "\nTEST 3: Execute Workflow\n" + "="*80) - - self.assertTrue(self.workers_started, "Workers must be started first") - - workflow_client = OrkesWorkflowClient(self.config) + EXPECTED_TASK_COUNT = 5 + + def _run_workflow_to_terminal(self, workflow_client, timeout_s=90): + """Start the e2e workflow and wait until it is genuinely terminal with + all expected tasks present. Returns (wf_id, workflow_or_None). A None + workflow (or a non-terminal / short task list) signals a timed-out run + the caller can retry, rather than asserting against a half-scheduled + workflow (which is what produced the flaky "4 != 5 tasks" failure). + """ req = StartWorkflowRequest() req.name = 'e2e_comprehensive_test' req.version = 1 req.input = {} - + wf_id = workflow_client.start_workflow(start_workflow_request=req) print(f"✓ Started workflow: {wf_id}") - - # Wait for completion - for i in range(30): + + deadline = time.time() + timeout_s + wf = None + while time.time() < deadline: wf = workflow_client.get_workflow(wf_id, include_tasks=True) - print(f" Status: {wf.status} - ({i*2}s)") - if wf.status in ['COMPLETED', 'FAILED']: - break - for task in wf.tasks: + print(f" Status: {wf.status} - tasks={len(wf.tasks or [])}") + if wf.status in ('COMPLETED', 'FAILED') \ + and len(wf.tasks or []) == self.EXPECTED_TASK_COUNT: + return wf_id, wf + for task in (wf.tasks or []): print(f'task {task.task_def_name} : {task.status}') time.sleep(1) + + return wf_id, wf + + def test_03_execute_workflow(self): + """Execute workflow and verify completion.""" + print("\n" + "="*80 + "\nTEST 3: Execute Workflow\n" + "="*80) - final_wf = workflow_client.get_workflow(wf_id, include_tasks=True) + self.assertTrue(self.workers_started, "Workers must be started first") + workflow_client = OrkesWorkflowClient(self.config) + + # Each attempt starts a fresh workflow, so retrying a pathologically + # slow run (cold workers / slow CI) is safe and self-contained. This + # keeps a one-off timeout from forcing a manual CI job re-run. + final_wf = None + for attempt in range(3): + wf_id, wf = self._run_workflow_to_terminal(workflow_client, timeout_s=90) + if wf is not None and wf.status in ('COMPLETED', 'FAILED') \ + and len(wf.tasks or []) == self.EXPECTED_TASK_COUNT: + final_wf = wf + break + print(f"attempt {attempt + 1}: wf={wf_id} " + f"status={getattr(wf, 'status', None)} " + f"tasks={len(getattr(wf, 'tasks', []) or [])}; retrying") + # Assertions - self.assertIsNotNone(final_wf) - self.assertEqual(len(final_wf.tasks), 5, "Should have 5 tasks") + self.assertIsNotNone( + final_wf, + "workflow never reached a terminal state with all " + f"{self.EXPECTED_TASK_COUNT} tasks after retries") + self.assertEqual(len(final_wf.tasks), self.EXPECTED_TASK_COUNT, + f"Should have {self.EXPECTED_TASK_COUNT} tasks") # Verify each task tasks_by_ref = {t.reference_task_name: t for t in final_wf.tasks} From cce31267edf773cbf1629f0f8eacd90406a95de7 Mon Sep 17 00:00:00 2001 From: Chris Hagglund Date: Tue, 14 Jul 2026 10:01:37 -0600 Subject: [PATCH 07/25] additional test diagnostic output --- tests/integration/test_comprehensive_e2e.py | 56 +++++++++++++++++++++ 1 file changed, 56 insertions(+) diff --git a/tests/integration/test_comprehensive_e2e.py b/tests/integration/test_comprehensive_e2e.py index dd79a45b6..672c69f92 100644 --- a/tests/integration/test_comprehensive_e2e.py +++ b/tests/integration/test_comprehensive_e2e.py @@ -121,6 +121,18 @@ def failing_task(should_fail: bool) -> dict: # Main test class class TestComprehensiveE2E(unittest.TestCase): + # Annotated workers this suite relies on. Every one must be discovered by + # the scan AND have a live process, otherwise its task silently stays + # SCHEDULED forever (which is exactly how a non-starting task_in_progress + # worker manifested as a "4 != 5 tasks" failure in CI). + EXPECTED_WORKERS = ( + 'sync_basic', + 'async_basic', + 'complex_schema', + 'task_in_progress', + 'failing_task', + ) + @classmethod def setUpClass(cls): from tests.integration.conftest import skip_if_server_unavailable @@ -191,12 +203,55 @@ def test_02_start_workers(self): self.__class__.task_handler = handler time.sleep(5) # Wait for startup + # Confirm every expected annotated worker was discovered by the scan AND + # its child process is actually alive. Assert each one individually so a + # failure names the exact worker that didn't come up, instead of the + # failure surfacing later (and opaquely) as a task stuck in SCHEDULED. + started = {} + for worker, process in zip(handler.workers, handler.task_runner_processes): + started[worker.get_task_definition_name()] = process.is_alive() + print(f"Discovered {len(started)} annotated worker(s): {sorted(started)}") + + for name in self.EXPECTED_WORKERS: + self.assertIn( + name, started, + f"worker '{name}' was not discovered by the annotated-worker scan; " + f"discovered={sorted(started)}") + self.assertTrue( + started[name], + f"worker '{name}' was discovered but its process is not alive") + print("✓ Workers started") self.__class__.workers_started = True self.assertTrue(self.workers_started) EXPECTED_TASK_COUNT = 5 + def _dump_stuck_task_diagnostics(self, wf): + """When a run fails to complete, print server-side poll data and queue + sizes so CI shows *why* a task is stuck (e.g. no worker polling its + queue) rather than just a bare task-count assertion. Poll data is + server-side, so it survives regardless of worker child-process log + capture. + """ + from conductor.client.orkes.orkes_task_client import OrkesTaskClient + task_client = OrkesTaskClient(self.config) + + stuck = [t.task_def_name for t in (getattr(wf, 'tasks', None) or []) + if t.status not in ('COMPLETED', 'FAILED', 'FAILED_WITH_TERMINAL_ERROR')] + print(f" [diag] non-terminal tasks: {stuck}") + + # Include the expected workers so we can compare a stuck queue against a + # known-good one (e.g. task_in_progress vs sync_basic). + for task_type in sorted(set(stuck) | set(self.EXPECTED_WORKERS)): + try: + queue_size = task_client.get_queue_size_for_task(task_type) + poll_data = task_client.get_task_poll_data(task_type) or [] + pollers = [(p.worker_id, p.domain, p.last_poll_time) for p in poll_data] + print(f" [diag] {task_type}: queue_size={queue_size} pollers={pollers}") + except Exception as e: # diagnostics must never mask the real failure + print(f" [diag] {task_type}: failed to fetch poll data: {e!r}") + def _run_workflow_to_terminal(self, workflow_client, timeout_s=90): """Start the e2e workflow and wait until it is genuinely terminal with all expected tasks present. Returns (wf_id, workflow_or_None). A None @@ -247,6 +302,7 @@ def test_03_execute_workflow(self): print(f"attempt {attempt + 1}: wf={wf_id} " f"status={getattr(wf, 'status', None)} " f"tasks={len(getattr(wf, 'tasks', []) or [])}; retrying") + self._dump_stuck_task_diagnostics(wf) # Assertions self.assertIsNotNone( From b051e449c15abd54f7667fefc65818cee3149518 Mon Sep 17 00:00:00 2001 From: Chris Hagglund Date: Tue, 14 Jul 2026 10:07:25 -0600 Subject: [PATCH 08/25] additional diagnostics info --- .../integration/test_async_lease_extension.py | 48 +++++++++++++++++++ 1 file changed, 48 insertions(+) diff --git a/tests/integration/test_async_lease_extension.py b/tests/integration/test_async_lease_extension.py index bec8f7712..a29555382 100644 --- a/tests/integration/test_async_lease_extension.py +++ b/tests/integration/test_async_lease_extension.py @@ -177,6 +177,20 @@ def setUpClass(cls): cls._task_handler.start_processes() time.sleep(3) # let workers start (once, not per test) + # Confirm every worker this module needs was actually started with a + # live process. If one silently fails to start, its task sits in + # SCHEDULED forever and surfaces later as an opaque "workflow still + # RUNNING" assertion; fail loudly here naming the exact worker instead. + started = {} + for worker, process in zip(cls._task_handler.workers, + cls._task_handler.task_runner_processes): + started[worker.get_task_definition_name()] = process.is_alive() + logger.info("Started workers (name -> alive): %s", started) + missing = [n for n in cls.WORKER_TASK_NAMES + if not started.get(n, False)] + assert not missing, ( + f"workers not started/alive: {missing}; started={started}") + @classmethod def tearDownClass(cls): handler = getattr(cls, '_task_handler', None) @@ -219,6 +233,37 @@ def _wait_for_workflow(self, wf_id, timeout_seconds=90): time.sleep(1) return self.workflow_client.get_workflow(wf_id, include_tasks=True) + def _dump_workflow_diagnostics(self, wf): + """Print task statuses plus server-side poll data / queue size so a + non-terminal workflow shows *why* (e.g. a task stuck in SCHEDULED with + no poller = the worker isn't consuming it), rather than only a bare + status assertion. Poll data is server-side, so it survives regardless + of worker child-process log capture. + """ + from conductor.client.orkes.orkes_task_client import OrkesTaskClient + task_client = OrkesTaskClient(self.config) + + # Worker processes can die *after* setUpClass; report current liveness + # so we can tell "worker crashed mid-suite" apart from "worker alive but + # not polling / server not timing out". + handler = getattr(self, '_task_handler', None) + if handler is not None: + alive = {} + for worker, process in zip(handler.workers, handler.task_runner_processes): + alive[worker.get_task_definition_name()] = process.is_alive() + print(f" [diag] worker liveness: {alive}") + + for task in (getattr(wf, 'tasks', None) or []): + print(f" [diag] task {task.task_def_name}: {task.status}") + try: + queue_size = task_client.get_queue_size_for_task(task.task_def_name) + poll_data = task_client.get_task_poll_data(task.task_def_name) or [] + pollers = [(p.worker_id, p.domain, p.last_poll_time) for p in poll_data] + print(f" [diag] {task.task_def_name}: " + f"queue_size={queue_size} pollers={pollers}") + except Exception as e: # diagnostics must never mask the real failure + print(f" [diag] {task.task_def_name}: failed to fetch poll data: {e!r}") + # -- Tests ---------------------------------------------------------------- def test_01_async_with_heartbeat_completes(self): @@ -268,6 +313,9 @@ def test_02_async_without_heartbeat_times_out(self): for task in (wf.tasks or []): print(f" Task {task.task_def_name}: {task.status}") + if wf.status not in ('FAILED', 'TIMED_OUT'): + self._dump_workflow_diagnostics(wf) + self.assertIn(wf.status, ('FAILED', 'TIMED_OUT'), f"Workflow should FAIL/TIMEOUT without heartbeat, got {wf.status}") From aabb539e90315634bc96d4b2e01f04cc641685c8 Mon Sep 17 00:00:00 2001 From: Chris Hagglund Date: Tue, 14 Jul 2026 10:28:33 -0600 Subject: [PATCH 09/25] further measures to ensure no contention between test runs --- .../integration/test_async_lease_extension.py | 75 +++++++++++------- tests/integration/test_comprehensive_e2e.py | 56 +++++++++----- tests/integration/test_lease_extension.py | 76 +++++++++++++++---- .../integration/test_workflow_client_intg.py | 39 +++++++++- 4 files changed, 180 insertions(+), 66 deletions(-) diff --git a/tests/integration/test_async_lease_extension.py b/tests/integration/test_async_lease_extension.py index a29555382..0c03fd326 100644 --- a/tests/integration/test_async_lease_extension.py +++ b/tests/integration/test_async_lease_extension.py @@ -25,6 +25,7 @@ import sys import time import unittest +from uuid import uuid4 import pytest @@ -57,15 +58,25 @@ # Number of fast tasks for performance comparison PERF_TASK_COUNT = 5 +# Per-run suffix so this suite's task/workflow names don't collide with other +# runs (or other PRs/developers) on the shared dev server. With fixed names, +# concurrent runs poll the same queues and steal/strand each other's tasks, +# producing non-deterministic SCHEDULED/RUNNING failures. +RUN_ID = uuid4().hex[:8] +HEARTBEAT_TASK = f'async_lease_heartbeat_task_{RUN_ID}' +NO_HEARTBEAT_TASK = f'async_lease_no_heartbeat_task_{RUN_ID}' +FAST_HB_TASK = f'async_lease_fast_with_hb_{RUN_ID}' +FAST_NO_HB_TASK = f'async_lease_fast_no_hb_{RUN_ID}' + # -- Async Workers ----------------------------------------------------------- @worker_task( - task_definition_name='async_lease_heartbeat_task', + task_definition_name=HEARTBEAT_TASK, lease_extend_enabled=True, register_task_def=True, task_def=TaskDef( - name='async_lease_heartbeat_task', + name=HEARTBEAT_TASK, response_timeout_seconds=RESPONSE_TIMEOUT_SECONDS, timeout_seconds=180, retry_count=0, @@ -82,11 +93,11 @@ async def async_lease_heartbeat_task(job_id: str) -> dict: @worker_task( - task_definition_name='async_lease_no_heartbeat_task', + task_definition_name=NO_HEARTBEAT_TASK, lease_extend_enabled=False, register_task_def=True, task_def=TaskDef( - name='async_lease_no_heartbeat_task', + name=NO_HEARTBEAT_TASK, response_timeout_seconds=RESPONSE_TIMEOUT_SECONDS, timeout_seconds=120, retry_count=0, @@ -103,11 +114,11 @@ async def async_lease_no_heartbeat_task(job_id: str) -> dict: @worker_task( - task_definition_name='async_lease_fast_with_hb', + task_definition_name=FAST_HB_TASK, lease_extend_enabled=True, register_task_def=True, task_def=TaskDef( - name='async_lease_fast_with_hb', + name=FAST_HB_TASK, response_timeout_seconds=60, timeout_seconds=120, retry_count=0, @@ -121,11 +132,11 @@ async def async_lease_fast_with_hb(job_id: str) -> dict: @worker_task( - task_definition_name='async_lease_fast_no_hb', + task_definition_name=FAST_NO_HB_TASK, lease_extend_enabled=False, register_task_def=True, task_def=TaskDef( - name='async_lease_fast_no_hb', + name=FAST_NO_HB_TASK, response_timeout_seconds=60, timeout_seconds=120, retry_count=0, @@ -147,10 +158,10 @@ class TestAsyncLeaseExtension(unittest.TestCase): # it doesn't spin up every @worker_task registered across the imported test # suite (that was ~24 worker processes started/torn down per test). WORKER_TASK_NAMES = { - 'async_lease_heartbeat_task', - 'async_lease_no_heartbeat_task', - 'async_lease_fast_with_hb', - 'async_lease_fast_no_hb', + HEARTBEAT_TASK, + NO_HEARTBEAT_TASK, + FAST_HB_TASK, + FAST_NO_HB_TASK, } @classmethod @@ -224,14 +235,25 @@ def _start_workflow(self, wf_name, job_id): logger.info("Started workflow %s: %s", wf_name, wf_id) return wf_id + TERMINAL_STATES = ('COMPLETED', 'FAILED', 'TIMED_OUT', 'TERMINATED') + def _wait_for_workflow(self, wf_id, timeout_seconds=90): - """Poll until workflow reaches a terminal state.""" + """Poll until workflow reaches a terminal state. If it doesn't within + the budget, dump server-side diagnostics so the ensuing assertion shows + *why* (e.g. a task stuck in SCHEDULED with no poller) rather than only a + bare status mismatch. + """ for _ in range(timeout_seconds): wf = self.workflow_client.get_workflow(wf_id, include_tasks=True) - if wf.status in ('COMPLETED', 'FAILED', 'TIMED_OUT', 'TERMINATED'): + if wf.status in self.TERMINAL_STATES: return wf time.sleep(1) - return self.workflow_client.get_workflow(wf_id, include_tasks=True) + wf = self.workflow_client.get_workflow(wf_id, include_tasks=True) + if wf.status not in self.TERMINAL_STATES: + print(f" [diag] workflow {wf_id} still {wf.status} " + f"after {timeout_seconds}s") + self._dump_workflow_diagnostics(wf) + return wf def _dump_workflow_diagnostics(self, wf): """Print task statuses plus server-side poll data / queue size so a @@ -273,8 +295,8 @@ def test_01_async_with_heartbeat_completes(self): print(f" responseTimeoutSeconds={RESPONSE_TIMEOUT_SECONDS}s, task sleeps {TASK_SLEEP_SECONDS}s") print("=" * 80) - wf_name = 'test_async_lease_heartbeat' - self._register_workflow(wf_name, 'async_lease_heartbeat_task') + wf_name = f'test_async_lease_heartbeat_{RUN_ID}' + self._register_workflow(wf_name, HEARTBEAT_TASK) wf_id = self._start_workflow(wf_name, 'ASYNC-HB-001') wf = self._wait_for_workflow(wf_id, timeout_seconds=80) @@ -288,7 +310,7 @@ def test_01_async_with_heartbeat_completes(self): f"Workflow should COMPLETE with heartbeat, got {wf.status}") tasks_by_ref = {t.reference_task_name: t for t in wf.tasks} - task = tasks_by_ref.get('async_lease_heartbeat_task_ref') + task = tasks_by_ref.get(f'{HEARTBEAT_TASK}_ref') self.assertIsNotNone(task) self.assertEqual(task.status, 'COMPLETED') self.assertEqual(task.output_data.get('job_id'), 'ASYNC-HB-001') @@ -302,8 +324,8 @@ def test_02_async_without_heartbeat_times_out(self): print(f" responseTimeoutSeconds={RESPONSE_TIMEOUT_SECONDS}s, task sleeps {TASK_SLEEP_SECONDS}s") print("=" * 80) - wf_name = 'test_async_lease_no_heartbeat' - self._register_workflow(wf_name, 'async_lease_no_heartbeat_task') + wf_name = f'test_async_lease_no_heartbeat_{RUN_ID}' + self._register_workflow(wf_name, NO_HEARTBEAT_TASK) wf_id = self._start_workflow(wf_name, 'ASYNC-NOHB-001') wf = self._wait_for_workflow(wf_id, timeout_seconds=80) @@ -313,14 +335,11 @@ def test_02_async_without_heartbeat_times_out(self): for task in (wf.tasks or []): print(f" Task {task.task_def_name}: {task.status}") - if wf.status not in ('FAILED', 'TIMED_OUT'): - self._dump_workflow_diagnostics(wf) - self.assertIn(wf.status, ('FAILED', 'TIMED_OUT'), f"Workflow should FAIL/TIMEOUT without heartbeat, got {wf.status}") tasks_by_ref = {t.reference_task_name: t for t in wf.tasks} - task = tasks_by_ref.get('async_lease_no_heartbeat_task_ref') + task = tasks_by_ref.get(f'{NO_HEARTBEAT_TASK}_ref') self.assertIsNotNone(task) self.assertIn(task.status, ('TIMED_OUT', 'FAILED', 'CANCELED'), f"Task should be TIMED_OUT/FAILED, got {task.status}") @@ -333,10 +352,10 @@ def test_03_no_performance_overhead(self): print(f" Running {PERF_TASK_COUNT} tasks each, sleep={FAST_TASK_SLEEP_SECONDS}s") print("=" * 80) - wf_with_hb = 'test_async_perf_with_hb' - wf_no_hb = 'test_async_perf_no_hb' - self._register_workflow(wf_with_hb, 'async_lease_fast_with_hb') - self._register_workflow(wf_no_hb, 'async_lease_fast_no_hb') + wf_with_hb = f'test_async_perf_with_hb_{RUN_ID}' + wf_no_hb = f'test_async_perf_no_hb_{RUN_ID}' + self._register_workflow(wf_with_hb, FAST_HB_TASK) + self._register_workflow(wf_no_hb, FAST_NO_HB_TASK) # Run tasks WITH heartbeat tracking hb_workflow_ids = [] diff --git a/tests/integration/test_comprehensive_e2e.py b/tests/integration/test_comprehensive_e2e.py index 672c69f92..55c2f355c 100644 --- a/tests/integration/test_comprehensive_e2e.py +++ b/tests/integration/test_comprehensive_e2e.py @@ -32,6 +32,7 @@ from dataclasses import dataclass from typing import Optional, List, Dict, Union from collections import defaultdict +from uuid import uuid4 from conductor.client.worker.exception import NonRetryableException @@ -70,15 +71,28 @@ def on_task_execution_failure(self, e): self.events['exec_failed'].append(e) def on_task_update_failure(self, e): self.events['update_failed'].append(e) +# Per-run suffix so this suite's task/workflow names don't collide with other +# runs (or other PRs/developers) on the shared dev server. With fixed names, +# concurrent runs poll the same queues and steal/strand each other's tasks, +# producing non-deterministic SCHEDULED failures. +RUN_ID = uuid4().hex[:8] +SYNC_BASIC = f'sync_basic_{RUN_ID}' +ASYNC_BASIC = f'async_basic_{RUN_ID}' +COMPLEX_SCHEMA = f'complex_schema_{RUN_ID}' +TASK_IN_PROGRESS = f'task_in_progress_{RUN_ID}' +FAILING_TASK = f'failing_task_{RUN_ID}' +WF_NAME = f'e2e_comprehensive_test_{RUN_ID}' + + # Test workers covering all scenarios -@worker_task(task_definition_name='sync_basic', thread_count=5, register_task_def=True) +@worker_task(task_definition_name=SYNC_BASIC, thread_count=5, register_task_def=True) def sync_basic(value: str, count: int) -> dict: ctx = get_task_context() ctx.add_log(f"Processing {value}") return {'value': value, 'count': count, 'worker': 'sync'} -@worker_task(task_definition_name='async_basic', thread_count=10, register_task_def=True) +@worker_task(task_definition_name=ASYNC_BASIC, thread_count=10, register_task_def=True) async def async_basic(message: str) -> dict: await asyncio.sleep(0.1) return {'message': message, 'worker': 'async'} @@ -92,17 +106,17 @@ class OrderData: @worker_task( - task_definition_name='complex_schema', + task_definition_name=COMPLEX_SCHEMA, register_task_def=True, strict_schema=True, - task_def=TaskDef(name='complex_schema', retry_count=2, timeout_policy='RETRY') + task_def=TaskDef(name=COMPLEX_SCHEMA, retry_count=2, timeout_policy='RETRY') ) def complex_schema(data: OrderData, optional: Optional[str]) -> dict: assert data.id is not None return {'id': data.id, 'amount': data.amount, 'tag_count': len(data.tags)} -@worker_task(task_definition_name='task_in_progress') +@worker_task(task_definition_name=TASK_IN_PROGRESS) def task_in_progress(job_id: str) -> Union[dict, TaskInProgress]: ctx = get_task_context() polls = ctx.get_poll_count() @@ -111,7 +125,7 @@ def task_in_progress(job_id: str) -> Union[dict, TaskInProgress]: return {'job_id': job_id, 'polls': polls} -@worker_task(task_definition_name='failing_task') +@worker_task(task_definition_name=FAILING_TASK) def failing_task(should_fail: bool) -> dict: if should_fail: raise NonRetryableException("Test failure") @@ -126,11 +140,11 @@ class TestComprehensiveE2E(unittest.TestCase): # SCHEDULED forever (which is exactly how a non-starting task_in_progress # worker manifested as a "4 != 5 tasks" failure in CI). EXPECTED_WORKERS = ( - 'sync_basic', - 'async_basic', - 'complex_schema', - 'task_in_progress', - 'failing_task', + SYNC_BASIC, + ASYNC_BASIC, + COMPLEX_SCHEMA, + TASK_IN_PROGRESS, + FAILING_TASK, ) @classmethod @@ -163,18 +177,18 @@ def test_01_create_workflow(self): metadata_client = OrkesMetadataClient(self.config) - workflow = WorkflowDef(name='e2e_comprehensive_test', version=1) + workflow = WorkflowDef(name=WF_NAME, version=1) tasks = [ - WorkflowTask(name='sync_basic', task_reference_name='sync_1', + WorkflowTask(name=SYNC_BASIC, task_reference_name='sync_1', input_parameters={'value': 'test', 'count': 1}), - WorkflowTask(name='async_basic', task_reference_name='async_1', + WorkflowTask(name=ASYNC_BASIC, task_reference_name='async_1', input_parameters={'message': 'hello'}), - WorkflowTask(name='complex_schema', task_reference_name='complex_1', + WorkflowTask(name=COMPLEX_SCHEMA, task_reference_name='complex_1', input_parameters={'data': {'id': '123', 'amount': 99.99, 'tags': ['a', 'b']}, 'optional': None}), - WorkflowTask(name='task_in_progress', task_reference_name='tip_1', + WorkflowTask(name=TASK_IN_PROGRESS, task_reference_name='tip_1', input_parameters={'job_id': 'JOB1'}), - WorkflowTask(name='failing_task', task_reference_name='fail_1', + WorkflowTask(name=FAILING_TASK, task_reference_name='fail_1', input_parameters={'should_fail': True}), ] workflow._tasks = tasks @@ -260,7 +274,7 @@ def _run_workflow_to_terminal(self, workflow_client, timeout_s=90): workflow (which is what produced the flaky "4 != 5 tasks" failure). """ req = StartWorkflowRequest() - req.name = 'e2e_comprehensive_test' + req.name = WF_NAME req.version = 1 req.input = {} @@ -375,7 +389,7 @@ def test_05_verify_task_definitions(self): metadata_client = OrkesMetadataClient(self.config) - tasks_to_check = ['sync_basic', 'async_basic', 'complex_schema'] + tasks_to_check = [SYNC_BASIC, ASYNC_BASIC, COMPLEX_SCHEMA] for task_name in tasks_to_check: task_def = metadata_client.get_task_def(task_name) @@ -384,7 +398,7 @@ def test_05_verify_task_definitions(self): print(f"✓ Task definition exists: {task_name}") # Check schemas if they should exist - if task_name in ['sync_basic', 'async_basic', 'complex_schema']: + if task_name in tasks_to_check: # These have type hints, should have schemas if hasattr(task_def, 'input_schema') and task_def.input_schema: print(f" ✓ Has input schema") @@ -392,7 +406,7 @@ def test_05_verify_task_definitions(self): print(f" ✓ Has output schema") # Check complex_schema has TaskDef configuration - complex_def = metadata_client.get_task_def('complex_schema') + complex_def = metadata_client.get_task_def(COMPLEX_SCHEMA) self.assertEqual(complex_def.retry_count, 2, "Retry count from task_def should be applied") self.assertEqual(complex_def.timeout_policy, 'RETRY', "Timeout policy should be set") diff --git a/tests/integration/test_lease_extension.py b/tests/integration/test_lease_extension.py index ca7d5434c..8c05a157e 100644 --- a/tests/integration/test_lease_extension.py +++ b/tests/integration/test_lease_extension.py @@ -23,6 +23,7 @@ import time import threading import unittest +from uuid import uuid4 import pytest @@ -49,16 +50,24 @@ # ~30s) catches the expired task BEFORE the worker completes. TASK_SLEEP_SECONDS = 50 +# Per-run suffix so this suite's task/workflow names don't collide with other +# runs (or other PRs/developers) on the shared dev server. With fixed names, +# concurrent runs poll the same queues and steal/strand each other's tasks, +# producing non-deterministic SCHEDULED/RUNNING failures. +RUN_ID = uuid4().hex[:8] +HEARTBEAT_TASK = f'lease_heartbeat_task_{RUN_ID}' +NO_HEARTBEAT_TASK = f'lease_no_heartbeat_task_{RUN_ID}' + # -- Workers ----------------------------------------------------------------- # Worker WITH lease extension enabled — heartbeats keep it alive @worker_task( - task_definition_name='lease_heartbeat_task', + task_definition_name=HEARTBEAT_TASK, lease_extend_enabled=True, register_task_def=True, task_def=TaskDef( - name='lease_heartbeat_task', + name=HEARTBEAT_TASK, response_timeout_seconds=RESPONSE_TIMEOUT_SECONDS, timeout_seconds=180, retry_count=0, @@ -76,11 +85,11 @@ def lease_heartbeat_task(job_id: str) -> dict: # Worker WITHOUT lease extension — will time out @worker_task( - task_definition_name='lease_no_heartbeat_task', + task_definition_name=NO_HEARTBEAT_TASK, lease_extend_enabled=False, register_task_def=True, task_def=TaskDef( - name='lease_no_heartbeat_task', + name=NO_HEARTBEAT_TASK, response_timeout_seconds=RESPONSE_TIMEOUT_SECONDS, timeout_seconds=120, retry_count=0, @@ -135,15 +144,52 @@ def _start_workflow(self, wf_name, job_id): logger.info("Started workflow %s: %s", wf_name, wf_id) return wf_id + TERMINAL_STATES = ('COMPLETED', 'FAILED', 'TIMED_OUT', 'TERMINATED') + def _wait_for_workflow(self, wf_id, timeout_seconds=60): - """Poll until workflow reaches a terminal state.""" + """Poll until workflow reaches a terminal state. If it doesn't within + the budget, dump server-side diagnostics so the ensuing assertion shows + *why* (e.g. a task stuck in SCHEDULED with no poller) rather than only a + bare status mismatch. + """ for i in range(timeout_seconds): wf = self.workflow_client.get_workflow(wf_id, include_tasks=True) - if wf.status in ('COMPLETED', 'FAILED', 'TIMED_OUT', 'TERMINATED'): + if wf.status in self.TERMINAL_STATES: return wf time.sleep(1) - # Return whatever state it's in after timeout - return self.workflow_client.get_workflow(wf_id, include_tasks=True) + wf = self.workflow_client.get_workflow(wf_id, include_tasks=True) + if wf.status not in self.TERMINAL_STATES: + print(f" [diag] workflow {wf_id} still {wf.status} " + f"after {timeout_seconds}s") + self._dump_workflow_diagnostics(wf) + return wf + + def _dump_workflow_diagnostics(self, wf): + """Print task statuses plus server-side poll data / queue size so a + non-terminal workflow shows *why* (e.g. a task stuck in SCHEDULED with + no poller = the worker isn't consuming it). Poll data is server-side, + so it survives regardless of worker child-process log capture. + """ + from conductor.client.orkes.orkes_task_client import OrkesTaskClient + task_client = OrkesTaskClient(self.config) + + handler = getattr(self, '_active_handler', None) + if handler is not None: + alive = {} + for worker, process in zip(handler.workers, handler.task_runner_processes): + alive[worker.get_task_definition_name()] = process.is_alive() + print(f" [diag] worker liveness: {alive}") + + for task in (getattr(wf, 'tasks', None) or []): + print(f" [diag] task {task.task_def_name}: {task.status}") + try: + queue_size = task_client.get_queue_size_for_task(task.task_def_name) + poll_data = task_client.get_task_poll_data(task.task_def_name) or [] + pollers = [(p.worker_id, p.domain, p.last_poll_time) for p in poll_data] + print(f" [diag] {task.task_def_name}: " + f"queue_size={queue_size} pollers={pollers}") + except Exception as e: # diagnostics must never mask the real failure + print(f" [diag] {task.task_def_name}: failed to fetch poll data: {e!r}") def _run_workers_in_background(self, duration_seconds=60): """Start workers in a background thread, return stop function.""" @@ -152,6 +198,8 @@ def _run_workers_in_background(self, duration_seconds=60): scan_for_annotated_workers=True, ) handler.start_processes() + # Expose the handler so _dump_workflow_diagnostics can report liveness. + self.__class__._active_handler = handler def stop(): handler.stop_processes() @@ -170,8 +218,8 @@ def test_01_with_heartbeat_completes(self): print(f" responseTimeoutSeconds={RESPONSE_TIMEOUT_SECONDS}s, task sleeps {TASK_SLEEP_SECONDS}s") print("=" * 80) - wf_name = 'test_lease_heartbeat' - self._register_workflow(wf_name, 'lease_heartbeat_task') + wf_name = f'test_lease_heartbeat_{RUN_ID}' + self._register_workflow(wf_name, HEARTBEAT_TASK) stop_workers = self._run_workers_in_background(duration_seconds=90) time.sleep(3) # let workers start @@ -189,7 +237,7 @@ def test_01_with_heartbeat_completes(self): # Verify task output tasks_by_ref = {t.reference_task_name: t for t in wf.tasks} - task = tasks_by_ref.get('lease_heartbeat_task_ref') + task = tasks_by_ref.get(f'{HEARTBEAT_TASK}_ref') self.assertIsNotNone(task) self.assertEqual(task.status, 'COMPLETED') self.assertEqual(task.output_data.get('job_id'), 'HEARTBEAT-001') @@ -205,8 +253,8 @@ def test_02_without_heartbeat_times_out(self): print(f" responseTimeoutSeconds={RESPONSE_TIMEOUT_SECONDS}s, task sleeps {TASK_SLEEP_SECONDS}s") print("=" * 80) - wf_name = 'test_lease_no_heartbeat' - self._register_workflow(wf_name, 'lease_no_heartbeat_task') + wf_name = f'test_lease_no_heartbeat_{RUN_ID}' + self._register_workflow(wf_name, NO_HEARTBEAT_TASK) stop_workers = self._run_workers_in_background(duration_seconds=90) time.sleep(3) # let workers start @@ -224,7 +272,7 @@ def test_02_without_heartbeat_times_out(self): f"Workflow should FAIL/TIMEOUT without heartbeat, got {wf.status}") tasks_by_ref = {t.reference_task_name: t for t in wf.tasks} - task = tasks_by_ref.get('lease_no_heartbeat_task_ref') + task = tasks_by_ref.get(f'{NO_HEARTBEAT_TASK}_ref') self.assertIsNotNone(task) self.assertIn(task.status, ('TIMED_OUT', 'FAILED', 'CANCELED'), f"Task should be TIMED_OUT/FAILED, got {task.status}") diff --git a/tests/integration/test_workflow_client_intg.py b/tests/integration/test_workflow_client_intg.py index 4afceb316..960c16f3d 100644 --- a/tests/integration/test_workflow_client_intg.py +++ b/tests/integration/test_workflow_client_intg.py @@ -1,5 +1,6 @@ import logging import os +import time import unittest import pytest @@ -7,6 +8,7 @@ from tests.integration.client.orkes.test_orkes_clients import TestOrkesClients from conductor.client.configuration.configuration import Configuration from conductor.client.configuration.settings.authentication_settings import AuthenticationSettings +from conductor.client.http.rest import ApiException from conductor.client.orkes.orkes_workflow_client import OrkesWorkflowClient from conductor.client.workflow.executor.workflow_executor import WorkflowExecutor from tests.integration.metadata.test_workflow_definition import run_workflow_definition_tests @@ -31,6 +33,31 @@ def get_configuration(): return configuration +def _run_tolerating_transient(label, func, *args, retries=3, **kwargs): + """Run a sub-suite, retrying only on a transient (status 0) transport blip + against the shared dev server. + + A `(0)` ApiException is a raw connection/protocol hiccup (stale keep-alive, + HTTP/2 GOAWAY race, client closed by a fork cleanup, etc.) — not a real + assertion or server failure — and was observed flaking the `test-all` + bucket. Retrying absorbs that network noise while still surfacing genuine + failures immediately (any non-transient error, or a transient one on the + final attempt, re-raises). + """ + for attempt in range(retries): + try: + return func(*args, **kwargs) + except ApiException as e: + transient = getattr(e, 'transient', False) or e.status in (0, None) + if transient and attempt < retries - 1: + logger.warning( + 'transient (%s) API error in %s (attempt %d/%d): %s; retrying', + e.status, label, attempt + 1, retries, e) + time.sleep(2 ** attempt) + continue + raise + + class TestOrkesWorkflowClientIntg(unittest.TestCase): @classmethod @@ -49,7 +76,13 @@ def test_all(self): workflow_executor = WorkflowExecutor(configuration) # test_async.test_async_method(api_client) - run_workflow_definition_tests(workflow_executor) - run_workflow_execution_tests(configuration, workflow_executor) - TestOrkesClients(configuration=configuration).run() + _run_tolerating_transient( + 'workflow_definition_tests', + run_workflow_definition_tests, workflow_executor) + _run_tolerating_transient( + 'workflow_execution_tests', + run_workflow_execution_tests, configuration, workflow_executor) + _run_tolerating_transient( + 'orkes_clients', + TestOrkesClients(configuration=configuration).run) logger.info('END: integration tests') From d4ef268efe8280e0ea814a9d5b143c07ad713189 Mon Sep 17 00:00:00 2001 From: Chris Hagglund Date: Tue, 14 Jul 2026 11:20:43 -0600 Subject: [PATCH 10/25] Additional hardening for transient exceptions, fix intercepted exception (try/finally instead of try/catch/reraise) --- .../integration/test_async_lease_extension.py | 49 +++++++++++++++++-- tests/integration/test_lease_extension.py | 49 +++++++++++++++++-- .../integration/test_workflow_client_intg.py | 41 ++++++++++++---- .../workflow/test_workflow_execution.py | 11 +++-- 4 files changed, 126 insertions(+), 24 deletions(-) diff --git a/tests/integration/test_async_lease_extension.py b/tests/integration/test_async_lease_extension.py index 0c03fd326..a87f031cc 100644 --- a/tests/integration/test_async_lease_extension.py +++ b/tests/integration/test_async_lease_extension.py @@ -38,12 +38,39 @@ from conductor.client.http.models.task_def import TaskDef from conductor.client.http.models.workflow_task import WorkflowTask from conductor.client.http.models.start_workflow_request import StartWorkflowRequest +from conductor.client.http.rest import ApiException from conductor.client.orkes.orkes_workflow_client import OrkesWorkflowClient from conductor.client.orkes.orkes_metadata_client import OrkesMetadataClient logging.basicConfig(level=logging.INFO) logger = logging.getLogger(__name__) + +def _is_transient(e): + """A transient (status 0) ApiException is a raw transport blip against the + shared dev server (read timeout, connection reset, HTTP/2 GOAWAY, stale + keep-alive) — not a real failure. status 0 means no HTTP response was + received at all.""" + return isinstance(e, ApiException) and ( + getattr(e, 'transient', False) or e.status in (0, None)) + + +def _retry_on_transient(func, *args, retries=4, base_delay=1, **kwargs): + """Retry a workflow/metadata client call on a transient (0) transport blip. + These `(0)` errors flake the lease suites on a loaded shared server; retrying + absorbs the noise. Non-transient errors (real 4xx/5xx) raise immediately.""" + for attempt in range(retries): + try: + return func(*args, **kwargs) + except ApiException as e: + if _is_transient(e) and attempt < retries - 1: + logger.warning( + 'transient (%s) API error (attempt %d/%d): %s; retrying', + e.status, attempt + 1, retries, e) + time.sleep(base_delay * (2 ** attempt)) + continue + raise + # Short response timeout — task must heartbeat to stay alive RESPONSE_TIMEOUT_SECONDS = 10 @@ -220,9 +247,11 @@ def _register_workflow(self, wf_name, task_names): )) workflow._tasks = tasks try: - self.metadata_client.update_workflow_def(workflow, overwrite=True) + _retry_on_transient(self.metadata_client.update_workflow_def, + workflow, overwrite=True) except Exception: - self.metadata_client.register_workflow_def(workflow, overwrite=True) + _retry_on_transient(self.metadata_client.register_workflow_def, + workflow, overwrite=True) logger.info("Registered workflow: %s", wf_name) def _start_workflow(self, wf_name, job_id): @@ -231,7 +260,8 @@ def _start_workflow(self, wf_name, job_id): req.name = wf_name req.version = 1 req.input = {'job_id': job_id} - wf_id = self.workflow_client.start_workflow(start_workflow_request=req) + wf_id = _retry_on_transient(self.workflow_client.start_workflow, + start_workflow_request=req) logger.info("Started workflow %s: %s", wf_name, wf_id) return wf_id @@ -244,11 +274,20 @@ def _wait_for_workflow(self, wf_id, timeout_seconds=90): bare status mismatch. """ for _ in range(timeout_seconds): - wf = self.workflow_client.get_workflow(wf_id, include_tasks=True) + try: + wf = self.workflow_client.get_workflow(wf_id, include_tasks=True) + except ApiException as e: + # A transient blip on a single poll shouldn't abort the wait; + # keep polling until the budget is exhausted. + if _is_transient(e): + time.sleep(1) + continue + raise if wf.status in self.TERMINAL_STATES: return wf time.sleep(1) - wf = self.workflow_client.get_workflow(wf_id, include_tasks=True) + wf = _retry_on_transient(self.workflow_client.get_workflow, + wf_id, include_tasks=True) if wf.status not in self.TERMINAL_STATES: print(f" [diag] workflow {wf_id} still {wf.status} " f"after {timeout_seconds}s") diff --git a/tests/integration/test_lease_extension.py b/tests/integration/test_lease_extension.py index 8c05a157e..a216c1009 100644 --- a/tests/integration/test_lease_extension.py +++ b/tests/integration/test_lease_extension.py @@ -36,12 +36,39 @@ from conductor.client.http.models.task_def import TaskDef from conductor.client.http.models.workflow_task import WorkflowTask from conductor.client.http.models.start_workflow_request import StartWorkflowRequest +from conductor.client.http.rest import ApiException from conductor.client.orkes.orkes_workflow_client import OrkesWorkflowClient from conductor.client.orkes.orkes_metadata_client import OrkesMetadataClient logging.basicConfig(level=logging.INFO) logger = logging.getLogger(__name__) + +def _is_transient(e): + """A transient (status 0) ApiException is a raw transport blip against the + shared dev server (read timeout, connection reset, HTTP/2 GOAWAY, stale + keep-alive) — not a real failure. status 0 means no HTTP response was + received at all.""" + return isinstance(e, ApiException) and ( + getattr(e, 'transient', False) or e.status in (0, None)) + + +def _retry_on_transient(func, *args, retries=4, base_delay=1, **kwargs): + """Retry a workflow/metadata client call on a transient (0) transport blip. + These `(0)` errors flake the lease suites on a loaded shared server; retrying + absorbs the noise. Non-transient errors (real 4xx/5xx) raise immediately.""" + for attempt in range(retries): + try: + return func(*args, **kwargs) + except ApiException as e: + if _is_transient(e) and attempt < retries - 1: + logger.warning( + 'transient (%s) API error (attempt %d/%d): %s; retrying', + e.status, attempt + 1, retries, e) + time.sleep(base_delay * (2 ** attempt)) + continue + raise + # Short response timeout — task must heartbeat to stay alive RESPONSE_TIMEOUT_SECONDS = 10 @@ -129,9 +156,11 @@ def _register_workflow(self, wf_name, task_name): ) workflow._tasks = [task] try: - self.metadata_client.update_workflow_def(workflow, overwrite=True) + _retry_on_transient(self.metadata_client.update_workflow_def, + workflow, overwrite=True) except Exception: - self.metadata_client.register_workflow_def(workflow, overwrite=True) + _retry_on_transient(self.metadata_client.register_workflow_def, + workflow, overwrite=True) logger.info("Registered workflow: %s", wf_name) def _start_workflow(self, wf_name, job_id): @@ -140,7 +169,8 @@ def _start_workflow(self, wf_name, job_id): req.name = wf_name req.version = 1 req.input = {'job_id': job_id} - wf_id = self.workflow_client.start_workflow(start_workflow_request=req) + wf_id = _retry_on_transient(self.workflow_client.start_workflow, + start_workflow_request=req) logger.info("Started workflow %s: %s", wf_name, wf_id) return wf_id @@ -153,11 +183,20 @@ def _wait_for_workflow(self, wf_id, timeout_seconds=60): bare status mismatch. """ for i in range(timeout_seconds): - wf = self.workflow_client.get_workflow(wf_id, include_tasks=True) + try: + wf = self.workflow_client.get_workflow(wf_id, include_tasks=True) + except ApiException as e: + # A transient blip on a single poll shouldn't abort the wait; + # keep polling until the budget is exhausted. + if _is_transient(e): + time.sleep(1) + continue + raise if wf.status in self.TERMINAL_STATES: return wf time.sleep(1) - wf = self.workflow_client.get_workflow(wf_id, include_tasks=True) + wf = _retry_on_transient(self.workflow_client.get_workflow, + wf_id, include_tasks=True) if wf.status not in self.TERMINAL_STATES: print(f" [diag] workflow {wf_id} still {wf.status} " f"after {timeout_seconds}s") diff --git a/tests/integration/test_workflow_client_intg.py b/tests/integration/test_workflow_client_intg.py index 960c16f3d..96dda8df0 100644 --- a/tests/integration/test_workflow_client_intg.py +++ b/tests/integration/test_workflow_client_intg.py @@ -33,26 +33,47 @@ def get_configuration(): return configuration +def _first_transient_api_exception(exc): + """Walk the exception chain (cause/context) and return the first transient + ApiException (status 0 / flagged transient), if any. + + Inner test helpers sometimes catch an ApiException and re-raise it as a bare + ``Exception`` (losing the type), so we can't rely on the outermost exception + type alone. Implicit chaining still records the original on ``__context__`` + (or ``__cause__`` when ``raise ... from`` is used), so we follow that chain. + """ + seen = set() + cur = exc + while cur is not None and id(cur) not in seen: + seen.add(id(cur)) + if isinstance(cur, ApiException) and ( + getattr(cur, 'transient', False) or cur.status in (0, None)): + return cur + cur = cur.__cause__ or cur.__context__ + return None + + def _run_tolerating_transient(label, func, *args, retries=3, **kwargs): """Run a sub-suite, retrying only on a transient (status 0) transport blip against the shared dev server. - A `(0)` ApiException is a raw connection/protocol hiccup (stale keep-alive, - HTTP/2 GOAWAY race, client closed by a fork cleanup, etc.) — not a real - assertion or server failure — and was observed flaking the `test-all` - bucket. Retrying absorbs that network noise while still surfacing genuine - failures immediately (any non-transient error, or a transient one on the - final attempt, re-raises). + A `(0)` ApiException is a raw connection/protocol hiccup (read timeout, + stale keep-alive, HTTP/2 GOAWAY race, client closed by a fork cleanup, etc.) + — not a real assertion or server failure — and was observed flaking the + `test-all` bucket. Retrying absorbs that network noise while still surfacing + genuine failures immediately (any non-transient error, or a transient one on + the final attempt, re-raises). We inspect the whole exception chain because + sub-suites may wrap the ApiException in a generic Exception. """ for attempt in range(retries): try: return func(*args, **kwargs) - except ApiException as e: - transient = getattr(e, 'transient', False) or e.status in (0, None) - if transient and attempt < retries - 1: + except Exception as e: + transient = _first_transient_api_exception(e) + if transient is not None and attempt < retries - 1: logger.warning( 'transient (%s) API error in %s (attempt %d/%d): %s; retrying', - e.status, label, attempt + 1, retries, e) + transient.status, label, attempt + 1, retries, transient) time.sleep(2 ** attempt) continue raise diff --git a/tests/integration/workflow/test_workflow_execution.py b/tests/integration/workflow/test_workflow_execution.py index 951c88f1a..e19c3c75f 100644 --- a/tests/integration/workflow/test_workflow_execution.py +++ b/tests/integration/workflow/test_workflow_execution.py @@ -45,6 +45,12 @@ def run_workflow_execution_tests(configuration: Configuration, workflow_executor import_modules=['tests.integration.resources.worker.python.python_worker'] ) task_handler.start_processes() + # Use try/finally (not try/except+re-raise): the sole purpose here is to + # stop the workers on the way out. Re-raising as a bare `Exception` used to + # discard the original type and traceback, which hid transient + # ApiException(status=0) transport blips from the caller's retry logic + # (they'd surface as an opaque generic Exception instead). finally cleans up + # on both success and failure while letting the original error propagate. try: scenario_get_workflow_by_correlation_ids(workflow_executor) logger.debug('finished workflow correlation ids test') @@ -64,11 +70,8 @@ def run_workflow_execution_tests(configuration: Configuration, workflow_executor logger.debug('finished execute_workflow error handling tests') run_signal_tests(configuration, workflow_executor) logger.debug('finished signal API tests') - - except Exception as e: + finally: task_handler.stop_processes() - raise Exception(f'failed integration tests, reason: {e}') - task_handler.stop_processes() def generate_tasks_defs(): From 416f8f9d6f3dd7f5dae004e8d825ad2e0af5e87a Mon Sep 17 00:00:00 2001 From: Chris Hagglund Date: Tue, 14 Jul 2026 11:42:17 -0600 Subject: [PATCH 11/25] probably a red-herring but i want to try with http2 disabled --- .github/workflows/pull_request.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index afa82f5d8..7b4e75f99 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -118,6 +118,11 @@ jobs: CONDUCTOR_SERVER_URL: ${{ vars.SDKDEV_V5_SERVER_URL }} CONDUCTOR_AUTH_KEY: ${{ vars.SDKDEV_V5_AUTH_KEY }} CONDUCTOR_AUTH_SECRET: ${{ secrets.SDKDEV_V5_AUTH_SECRET }} + # Force HTTP/1.1 for integration tests: long-lived HTTP/2 connections to + # the shared dev server (through its proxy/LB) intermittently stall + # mid-stream, surfacing as httpcore.ReadTimeout -> ApiException(0). HTTP/1.1 + # uses one request per pooled connection, avoiding that failure class. + CONDUCTOR_HTTP2_ENABLED: "false" steps: - name: Checkout code uses: actions/checkout@v4 From 6f676764e1d33e8e6691042584045d385cf12ffd Mon Sep 17 00:00:00 2001 From: Chris Hagglund Date: Tue, 14 Jul 2026 12:25:16 -0600 Subject: [PATCH 12/25] additional resilience --- .../workflow/test_workflow_execution.py | 46 +++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/tests/integration/workflow/test_workflow_execution.py b/tests/integration/workflow/test_workflow_execution.py index e19c3c75f..af9430b20 100644 --- a/tests/integration/workflow/test_workflow_execution.py +++ b/tests/integration/workflow/test_workflow_execution.py @@ -22,6 +22,13 @@ SUB_WF_1_NAME = 'complex_wf_signal_test_subworkflow_1' SUB_WF_2_NAME = 'complex_wf_signal_test_subworkflow_2' +# Max time to wait for the batch of simple workflows to reach a terminal state. +# These normally finish in seconds, but on a loaded shared server we've observed +# them take ~30s+; the old ~12s budget (5s sleep + 1+2+4 retry backoff) produced +# confirmed false-negative timeouts on workflows that did complete. Poll up to +# this budget instead. +WORKFLOW_COMPLETION_MAX_WAIT_SECONDS = 120 + logger = logging.getLogger( Configuration.get_logging_formatted_name( __name__ @@ -167,8 +174,14 @@ def scenario_workflow_execution( for i in range(workflow_quantity): start_workflow_requests[i] = StartWorkflowRequest(name=workflow_name) workflow_ids = workflow_executor.start_workflows(*start_workflow_requests) + # Brief head start, then poll each workflow until it's terminal. The + # workflows were all started together, so we share one wall-clock deadline + # across the batch (bounding total wait to ~WORKFLOW_COMPLETION_MAX_WAIT_SECONDS + # even on a genuine failure) rather than giving each its own long budget. sleep(workflow_completion_timeout) + deadline = time.time() + WORKFLOW_COMPLETION_MAX_WAIT_SECONDS for workflow_id in workflow_ids: + _wait_for_workflow_terminal(workflow_id, workflow_executor, deadline) _run_with_retry_attempt( validate_workflow_status, { @@ -178,6 +191,39 @@ def scenario_workflow_execution( ) +def _wait_for_workflow_terminal(workflow_id, workflow_executor, deadline, + poll_interval=2): + """Poll until the workflow reaches a terminal state or the shared deadline + passes, logging progress (wf id + elapsed) so a slow-but-eventually-complete + run is visible instead of surfacing as a bare timeout. A transient poll error + is logged and retried. Returns the last observed status; the caller still + runs validate_workflow_status for the actual assertion. + """ + terminal = ('COMPLETED', 'FAILED', 'TIMED_OUT', 'TERMINATED') + start = time.time() + status = None + while True: + try: + workflow = workflow_executor.get_workflow( + workflow_id=workflow_id, include_tasks=False) + status = workflow.status + except Exception as e: # transient blip against the shared server + logger.warning('error polling workflow %s (%.0fs elapsed): %s', + workflow_id, time.time() - start, e) + if status in terminal: + logger.info('workflow %s reached %s after %.0fs', + workflow_id, status, time.time() - start) + return status + if time.time() >= deadline: + logger.warning( + 'workflow %s still %s after %.0fs; giving up wait', + workflow_id, status, time.time() - start) + return status + logger.info('workflow %s still %s after %.0fs; waiting', + workflow_id, status, time.time() - start) + sleep(poll_interval) + + def generate_workflow(workflow_executor: WorkflowExecutor, workflow_name: str = WORKFLOW_NAME, task_name: str = TASK_NAME) -> ConductorWorkflow: return ConductorWorkflow( From 2eeee4f72036c607dd74b201b6b29434d321b849 Mon Sep 17 00:00:00 2001 From: Chris Hagglund Date: Wed, 15 Jul 2026 09:08:41 -0600 Subject: [PATCH 13/25] attempt to make the lease extension tests more reliable --- .github/workflows/pull_request.yml | 5 +- scripts/run_integration_tests.sh | 9 +++ .../integration/test_async_lease_extension.py | 51 +++++++++---- tests/integration/test_lease_extension.py | 71 ++++++++++++++----- 4 files changed, 101 insertions(+), 35 deletions(-) diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index 7b4e75f99..57416c0cc 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -140,4 +140,7 @@ jobs: pip install pytest - name: Run integration tests - run: bash scripts/run_integration_tests.sh --bucket=${{ matrix.bucket }} + run: >- + bash scripts/run_integration_tests.sh --bucket=${{ matrix.bucket }} + -s --log-cli-level=INFO + --log-cli-format='%(asctime)s %(levelname)s %(name)s: %(message)s' diff --git a/scripts/run_integration_tests.sh b/scripts/run_integration_tests.sh index cce260e3a..05832c297 100755 --- a/scripts/run_integration_tests.sh +++ b/scripts/run_integration_tests.sh @@ -47,6 +47,15 @@ # # short tracebacks + a one-line reason for every failure/skip, with live logs # ./scripts/run_integration_tests.sh -ra --tb=short --log-cli-level=INFO # +# # watch a slow bucket live: -s streams print() and the worker/child-process +# # task logs; --log-cli-level=INFO streams the main-process logger.info lines +# ./scripts/run_integration_tests.sh --bucket=long-sync -s --log-cli-level=INFO +# +# # same, but with timestamped live logs (what CI uses) +# ./scripts/run_integration_tests.sh --bucket=long-sync -s \ +# --log-cli-level=INFO \ +# --log-cli-format='%(asctime)s %(levelname)s %(name)s: %(message)s' +# # # stop at the first failure instead of waiting for the whole suite # ./scripts/run_integration_tests.sh -x --tb=long # diff --git a/tests/integration/test_async_lease_extension.py b/tests/integration/test_async_lease_extension.py index a87f031cc..de022b119 100644 --- a/tests/integration/test_async_lease_extension.py +++ b/tests/integration/test_async_lease_extension.py @@ -74,11 +74,26 @@ def _retry_on_transient(func, *args, retries=4, base_delay=1, **kwargs): # Short response timeout — task must heartbeat to stay alive RESPONSE_TIMEOUT_SECONDS = 10 -# Task sleeps longer than the response timeout to prove heartbeat works. -# Must be long enough that the server's workflow sweeper catches the expired -# task BEFORE the worker completes. +# The heartbeat task sleeps longer than the response timeout so that, without +# heartbeats, its lease would expire mid-execution. Heartbeats keep it alive and +# it still completes. This only needs to exceed RESPONSE_TIMEOUT_SECONDS. TASK_SLEEP_SECONDS = 50 +# The no-heartbeat task must NOT report COMPLETED before the server times its +# expired lease out; otherwise the completion "wins" the race and the task ends +# up COMPLETED (the historical flake on a loaded shared server, where the +# sweeper runs late). Holding the task far longer than any server timeout path +# guarantees the server always times it out first. The test doesn't wait this +# long: it polls for the terminal state and returns as soon as the timeout +# lands, and teardown force-terminates the still-sleeping worker process. +NO_HEARTBEAT_HOLD_SECONDS = 600 + +# Poll budget/interval for waiting on the workflow to reach the expected +# terminal state. Generous ceiling so slow-server variance is absorbed; the +# happy path returns in well under a minute. +POLL_TIMEOUT_SECONDS = 600 +POLL_INTERVAL_SECONDS = 5 + # Short task duration for performance test — well within timeout FAST_TASK_SLEEP_SECONDS = 2 @@ -132,12 +147,16 @@ async def async_lease_heartbeat_task(job_id: str) -> dict: overwrite_task_def=True, ) async def async_lease_no_heartbeat_task(job_id: str) -> dict: - """Async long-running task without heartbeat — should time out.""" - logger.info("[async_no_heartbeat] Starting job %s, sleeping %ss (timeout=%ss)", - job_id, TASK_SLEEP_SECONDS, RESPONSE_TIMEOUT_SECONDS) - await asyncio.sleep(TASK_SLEEP_SECONDS) + """Async long-running task without heartbeat — should time out. + + Holds the task well past every server timeout path so the server always + times the expired lease out before this ever tries to report completion. + """ + logger.info("[async_no_heartbeat] Starting job %s, holding %ss (timeout=%ss)", + job_id, NO_HEARTBEAT_HOLD_SECONDS, RESPONSE_TIMEOUT_SECONDS) + await asyncio.sleep(NO_HEARTBEAT_HOLD_SECONDS) logger.info("[async_no_heartbeat] Completed job %s", job_id) - return {'job_id': job_id, 'status': 'completed', 'slept': TASK_SLEEP_SECONDS} + return {'job_id': job_id, 'status': 'completed', 'slept': NO_HEARTBEAT_HOLD_SECONDS} @worker_task( @@ -267,25 +286,27 @@ def _start_workflow(self, wf_name, job_id): TERMINAL_STATES = ('COMPLETED', 'FAILED', 'TIMED_OUT', 'TERMINATED') - def _wait_for_workflow(self, wf_id, timeout_seconds=90): + def _wait_for_workflow(self, wf_id, timeout_seconds=POLL_TIMEOUT_SECONDS, + poll_interval=POLL_INTERVAL_SECONDS): """Poll until workflow reaches a terminal state. If it doesn't within the budget, dump server-side diagnostics so the ensuing assertion shows *why* (e.g. a task stuck in SCHEDULED with no poller) rather than only a bare status mismatch. """ - for _ in range(timeout_seconds): + deadline = time.monotonic() + timeout_seconds + while time.monotonic() < deadline: try: wf = self.workflow_client.get_workflow(wf_id, include_tasks=True) except ApiException as e: # A transient blip on a single poll shouldn't abort the wait; # keep polling until the budget is exhausted. if _is_transient(e): - time.sleep(1) + time.sleep(poll_interval) continue raise if wf.status in self.TERMINAL_STATES: return wf - time.sleep(1) + time.sleep(poll_interval) wf = _retry_on_transient(self.workflow_client.get_workflow, wf_id, include_tasks=True) if wf.status not in self.TERMINAL_STATES: @@ -338,7 +359,7 @@ def test_01_async_with_heartbeat_completes(self): self._register_workflow(wf_name, HEARTBEAT_TASK) wf_id = self._start_workflow(wf_name, 'ASYNC-HB-001') - wf = self._wait_for_workflow(wf_id, timeout_seconds=80) + wf = self._wait_for_workflow(wf_id) print(f"\n Workflow ID: {wf_id}") print(f" Final status: {wf.status}") @@ -360,14 +381,14 @@ def test_02_async_without_heartbeat_times_out(self): """Async task WITHOUT lease_extend_enabled times out when sleep > responseTimeout.""" print("\n" + "=" * 80) print("TEST: Async without heartbeat — task should TIME OUT") - print(f" responseTimeoutSeconds={RESPONSE_TIMEOUT_SECONDS}s, task sleeps {TASK_SLEEP_SECONDS}s") + print(f" responseTimeoutSeconds={RESPONSE_TIMEOUT_SECONDS}s, task holds {NO_HEARTBEAT_HOLD_SECONDS}s") print("=" * 80) wf_name = f'test_async_lease_no_heartbeat_{RUN_ID}' self._register_workflow(wf_name, NO_HEARTBEAT_TASK) wf_id = self._start_workflow(wf_name, 'ASYNC-NOHB-001') - wf = self._wait_for_workflow(wf_id, timeout_seconds=80) + wf = self._wait_for_workflow(wf_id) print(f"\n Workflow ID: {wf_id}") print(f" Final status: {wf.status}") diff --git a/tests/integration/test_lease_extension.py b/tests/integration/test_lease_extension.py index a216c1009..2971d46ba 100644 --- a/tests/integration/test_lease_extension.py +++ b/tests/integration/test_lease_extension.py @@ -72,11 +72,27 @@ def _retry_on_transient(func, *args, retries=4, base_delay=1, **kwargs): # Short response timeout — task must heartbeat to stay alive RESPONSE_TIMEOUT_SECONDS = 10 -# Task sleeps longer than the response timeout to prove heartbeat works. -# Must be long enough that the server's workflow sweeper (which runs every -# ~30s) catches the expired task BEFORE the worker completes. +# The heartbeat task sleeps longer than the response timeout so that, without +# heartbeats, its lease would expire mid-execution. Heartbeats keep it alive and +# it still completes. This only needs to exceed RESPONSE_TIMEOUT_SECONDS. TASK_SLEEP_SECONDS = 50 +# The no-heartbeat task must NOT report COMPLETED before the server times its +# expired lease out; otherwise the completion "wins" the race and the task ends +# up COMPLETED (the historical flake on a loaded shared server, where the +# sweeper runs late). Holding the task far longer than any server timeout path +# (responseTimeout + sweeper, and the task's own timeoutSeconds) guarantees the +# server always times it out first. The test itself doesn't wait this long: it +# polls for the terminal state and returns as soon as the timeout lands, and +# teardown force-terminates the still-sleeping worker process. +NO_HEARTBEAT_HOLD_SECONDS = 600 + +# Poll budget/interval for waiting on the workflow to reach the expected +# terminal state. Generous ceiling so slow-server variance is absorbed; the +# happy path returns in well under a minute. +POLL_TIMEOUT_SECONDS = 600 +POLL_INTERVAL_SECONDS = 5 + # Per-run suffix so this suite's task/workflow names don't collide with other # runs (or other PRs/developers) on the shared dev server. With fixed names, # concurrent runs poll the same queues and steal/strand each other's tasks, @@ -124,12 +140,16 @@ def lease_heartbeat_task(job_id: str) -> dict: overwrite_task_def=True, ) def lease_no_heartbeat_task(job_id: str) -> dict: - """Long-running task without heartbeat — should time out.""" - logger.info("[no_heartbeat_task] Starting job %s, sleeping %ss (timeout=%ss)", - job_id, TASK_SLEEP_SECONDS, RESPONSE_TIMEOUT_SECONDS) - time.sleep(TASK_SLEEP_SECONDS) + """Long-running task without heartbeat — should time out. + + Holds the task well past every server timeout path so the server always + times the expired lease out before this ever tries to report completion. + """ + logger.info("[no_heartbeat_task] Starting job %s, holding %ss (timeout=%ss)", + job_id, NO_HEARTBEAT_HOLD_SECONDS, RESPONSE_TIMEOUT_SECONDS) + time.sleep(NO_HEARTBEAT_HOLD_SECONDS) logger.info("[no_heartbeat_task] Completed job %s", job_id) - return {'job_id': job_id, 'status': 'completed', 'slept': TASK_SLEEP_SECONDS} + return {'job_id': job_id, 'status': 'completed', 'slept': NO_HEARTBEAT_HOLD_SECONDS} # -- Test class -------------------------------------------------------------- @@ -176,25 +196,27 @@ def _start_workflow(self, wf_name, job_id): TERMINAL_STATES = ('COMPLETED', 'FAILED', 'TIMED_OUT', 'TERMINATED') - def _wait_for_workflow(self, wf_id, timeout_seconds=60): + def _wait_for_workflow(self, wf_id, timeout_seconds=POLL_TIMEOUT_SECONDS, + poll_interval=POLL_INTERVAL_SECONDS): """Poll until workflow reaches a terminal state. If it doesn't within the budget, dump server-side diagnostics so the ensuing assertion shows *why* (e.g. a task stuck in SCHEDULED with no poller) rather than only a bare status mismatch. """ - for i in range(timeout_seconds): + deadline = time.monotonic() + timeout_seconds + while time.monotonic() < deadline: try: wf = self.workflow_client.get_workflow(wf_id, include_tasks=True) except ApiException as e: # A transient blip on a single poll shouldn't abort the wait; # keep polling until the budget is exhausted. if _is_transient(e): - time.sleep(1) + time.sleep(poll_interval) continue raise if wf.status in self.TERMINAL_STATES: return wf - time.sleep(1) + time.sleep(poll_interval) wf = _retry_on_transient(self.workflow_client.get_workflow, wf_id, include_tasks=True) if wf.status not in self.TERMINAL_STATES: @@ -230,8 +252,14 @@ def _dump_workflow_diagnostics(self, wf): except Exception as e: # diagnostics must never mask the real failure print(f" [diag] {task.task_def_name}: failed to fetch poll data: {e!r}") - def _run_workers_in_background(self, duration_seconds=60): - """Start workers in a background thread, return stop function.""" + def _run_workers_in_background(self, duration_seconds=POLL_TIMEOUT_SECONDS + 60): + """Start workers in a background thread, return stop function. + + Workers stay alive across the whole poll window; the test's `finally` + calls the returned stop() as soon as it finishes (usually in well under + a minute), and the timer is only a backstop. stop() is idempotent so the + backstop firing after the test already stopped is harmless. + """ handler = TaskHandler( configuration=self.config, scan_for_annotated_workers=True, @@ -240,7 +268,12 @@ def _run_workers_in_background(self, duration_seconds=60): # Expose the handler so _dump_workflow_diagnostics can report liveness. self.__class__._active_handler = handler + stopped = threading.Event() + def stop(): + if stopped.is_set(): + return + stopped.set() handler.stop_processes() # Auto-stop after duration @@ -260,12 +293,12 @@ def test_01_with_heartbeat_completes(self): wf_name = f'test_lease_heartbeat_{RUN_ID}' self._register_workflow(wf_name, HEARTBEAT_TASK) - stop_workers = self._run_workers_in_background(duration_seconds=90) + stop_workers = self._run_workers_in_background() time.sleep(3) # let workers start try: wf_id = self._start_workflow(wf_name, 'HEARTBEAT-001') - wf = self._wait_for_workflow(wf_id, timeout_seconds=80) + wf = self._wait_for_workflow(wf_id) print(f"\n Final status: {wf.status}") for task in (wf.tasks or []): @@ -289,18 +322,18 @@ def test_02_without_heartbeat_times_out(self): """Task WITHOUT lease_extend_enabled times out when sleep > responseTimeout.""" print("\n" + "=" * 80) print("TEST: Without heartbeat — task should TIME OUT") - print(f" responseTimeoutSeconds={RESPONSE_TIMEOUT_SECONDS}s, task sleeps {TASK_SLEEP_SECONDS}s") + print(f" responseTimeoutSeconds={RESPONSE_TIMEOUT_SECONDS}s, task holds {NO_HEARTBEAT_HOLD_SECONDS}s") print("=" * 80) wf_name = f'test_lease_no_heartbeat_{RUN_ID}' self._register_workflow(wf_name, NO_HEARTBEAT_TASK) - stop_workers = self._run_workers_in_background(duration_seconds=90) + stop_workers = self._run_workers_in_background() time.sleep(3) # let workers start try: wf_id = self._start_workflow(wf_name, 'NO-HEARTBEAT-001') - wf = self._wait_for_workflow(wf_id, timeout_seconds=80) + wf = self._wait_for_workflow(wf_id) print(f"\n Final status: {wf.status}") for task in (wf.tasks or []): From 612763cd057cfa13f7458863fcf75a5eb169ae70 Mon Sep 17 00:00:00 2001 From: Chris Hagglund Date: Wed, 15 Jul 2026 09:43:12 -0600 Subject: [PATCH 14/25] repeated empirical evidence suggests that the server will time out the task when it feels like it, not on our timeline --- pyproject.toml | 4 +--- scripts/run_integration_tests.sh | 13 +++++++++++-- tests/integration/test_async_lease_extension.py | 9 +++++++++ tests/integration/test_lease_extension.py | 9 +++++++++ 4 files changed, 30 insertions(+), 5 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index b9c8e74c1..3c4dca238 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -192,17 +192,15 @@ line-ending = "auto" [tool.pytest.ini_options] pythonpath = ["src"] markers = [ -<<<<<<< HEAD "integration: requires a live Agentspan-compatible server", "e2e: requires a live server and AGENTSPAN_SERVER_URL", "sse: requires SSE streaming (AGENTSPAN_STREAMING_ENABLED=true)", "agent_correctness: marks tests as agent correctness tests", "semantic: marks tests that use an LLM judge for semantic assertions", -======= "slow_sync: long-running sync lease-extension tests (~90s)", "slow_async: long-running async lease-extension tests (~90s)", "slow_test_all: long-running aggregate workflow-client test_all (~83s)", ->>>>>>> 973c79cf (wip, testing/fixes) + "server_timeout_unreliable: depends on server-side task timeout firing in a bounded window; excluded from CI", ] [tool.coverage.run] diff --git a/scripts/run_integration_tests.sh b/scripts/run_integration_tests.sh index 05832c297..689a9ef22 100755 --- a/scripts/run_integration_tests.sh +++ b/scripts/run_integration_tests.sh @@ -33,6 +33,11 @@ # test-all aggregate workflow-client test_all (test_workflow_client_intg.py, ~83s) # all the full suite (no bucket filtering) — for a complete local run # +# The long-* buckets deselect the no-heartbeat test_02 cases (marker +# server_timeout_unreliable): those assert a server-side task timeout that the +# sdkdev server doesn't reliably fire on a CI-bounded timeline. Use --bucket=all +# (or target the test directly) to run them anyway. +# # ./scripts/run_integration_tests.sh # fast: skips slow buckets # ./scripts/run_integration_tests.sh --bucket=long-sync # ./scripts/run_integration_tests.sh --bucket=all # run everything @@ -113,13 +118,17 @@ case "$bucket" in # never in scope and perf_ignore is unnecessary here. This also means # --with-perf has no effect on these buckets; the perf test is only reachable # via the core/all buckets. + # The no-heartbeat "server should time the task out" cases (test_02) are + # deselected here: the sdkdev server does not reliably time a task out on a + # CI-bounded timeline, so they flake for reasons unrelated to the SDK. They + # carry the server_timeout_unreliable marker and still run under --bucket=all. long-sync) targets=("$LEASE_SYNC") - select=(-m slow_sync) + select=(-m "slow_sync and not server_timeout_unreliable") ;; long-async) targets=("$LEASE_ASYNC") - select=(-m slow_async) + select=(-m "slow_async and not server_timeout_unreliable") ;; test-all) targets=("$TEST_ALL_FILE") diff --git a/tests/integration/test_async_lease_extension.py b/tests/integration/test_async_lease_extension.py index de022b119..0a4350018 100644 --- a/tests/integration/test_async_lease_extension.py +++ b/tests/integration/test_async_lease_extension.py @@ -377,6 +377,15 @@ def test_01_async_with_heartbeat_completes(self): self.assertEqual(task.output_data.get('slept'), TASK_SLEEP_SECONDS) print("\n PASS: Async task completed with heartbeat keeping lease alive") + # NOTE: excluded from the long-* CI buckets. This scenario asserts a purely + # server-side mechanic — the server timing out the task's expired lease on + # its own. The sdkdev server does not seem to consistently/reliably time out + # the task by itself on a timeline that works with integration testing, so + # gating CI on it produces flakes unrelated to the SDK. The SDK behaviour + # (heartbeats keeping a task alive) is still covered by test_01. + # Deselected via `-m "... and not server_timeout_unreliable"`; still runs + # under --bucket=all or when targeted directly. + @pytest.mark.server_timeout_unreliable def test_02_async_without_heartbeat_times_out(self): """Async task WITHOUT lease_extend_enabled times out when sleep > responseTimeout.""" print("\n" + "=" * 80) diff --git a/tests/integration/test_lease_extension.py b/tests/integration/test_lease_extension.py index 2971d46ba..d60ea8eba 100644 --- a/tests/integration/test_lease_extension.py +++ b/tests/integration/test_lease_extension.py @@ -318,6 +318,15 @@ def test_01_with_heartbeat_completes(self): finally: stop_workers() + # NOTE: excluded from the long-* CI buckets. This scenario asserts a purely + # server-side mechanic — the server timing out the task's expired lease on + # its own. The sdkdev server does not seem to consistently/reliably time out + # the task by itself on a timeline that works with integration testing, so + # gating CI on it produces flakes unrelated to the SDK. The SDK behaviour + # (heartbeats keeping a task alive) is still covered by test_01. + # Deselected via `-m "... and not server_timeout_unreliable"`; still runs + # under --bucket=all or when targeted directly. + @pytest.mark.server_timeout_unreliable def test_02_without_heartbeat_times_out(self): """Task WITHOUT lease_extend_enabled times out when sleep > responseTimeout.""" print("\n" + "=" * 80) From 10dd0f3cfa4fdac2547adf9cbbe831f41b0d0994 Mon Sep 17 00:00:00 2001 From: Chris Hagglund Date: Wed, 15 Jul 2026 10:59:13 -0600 Subject: [PATCH 15/25] resilience --- .../client/orkes/test_orkes_clients.py | 48 ++++++++++--- .../metadata/test_workflow_definition.py | 7 +- .../integration/test_async_lease_extension.py | 9 +++ .../integration/test_workflow_client_intg.py | 68 +++---------------- .../workflow/test_workflow_execution.py | 68 +++++++++++++------ 5 files changed, 112 insertions(+), 88 deletions(-) diff --git a/tests/integration/client/orkes/test_orkes_clients.py b/tests/integration/client/orkes/test_orkes_clients.py index 2060da258..6551bb4f0 100644 --- a/tests/integration/client/orkes/test_orkes_clients.py +++ b/tests/integration/client/orkes/test_orkes_clients.py @@ -26,6 +26,7 @@ from conductor.client.workflow.conductor_workflow import ConductorWorkflow from conductor.client.workflow.executor.workflow_executor import WorkflowExecutor from conductor.client.workflow.task.simple_task import SimpleTask +from tests.integration.retry_helpers import retry_scenario SUFFIX = str(uuid()) WORKFLOW_NAME = 'IntegrationTestOrkesClientsWf_' + SUFFIX @@ -66,7 +67,7 @@ def __init__(self, configuration: Configuration): self.authorization_client = orkes_clients.get_authorization_client() self.workflow_id = None - def run(self) -> None: + def run(self, deadline=None) -> None: workflow = ConductorWorkflow( executor=self.workflow_executor, name=WORKFLOW_NAME, @@ -77,13 +78,24 @@ def run(self) -> None: workflow >> SimpleTask("simple_task", "simple_task_ref") workflowDef = workflow.to_workflow_def() - self.test_workflow_lifecycle(workflowDef, workflow) - self.test_task_lifecycle() - self.test_secret_lifecycle() - self.test_scheduler_lifecycle(workflowDef) - self.test_application_lifecycle() - self.__test_unit_test_workflow() - self.test_user_group_permissions_lifecycle(workflowDef) + # Each lifecycle is a scenario: on a transient (status 0) blip against + # the shared dev server it retries from the top until the shared deadline + # passes (see retry_helpers.retry_scenario); real failures raise at once. + retry_scenario('test_workflow_lifecycle', self.test_workflow_lifecycle, + workflowDef, workflow, deadline=deadline) + retry_scenario('test_task_lifecycle', self.test_task_lifecycle, + deadline=deadline) + retry_scenario('test_secret_lifecycle', self.test_secret_lifecycle, + deadline=deadline) + retry_scenario('test_scheduler_lifecycle', self.test_scheduler_lifecycle, + workflowDef, deadline=deadline) + retry_scenario('test_application_lifecycle', self.test_application_lifecycle, + deadline=deadline) + retry_scenario('__test_unit_test_workflow', self.__test_unit_test_workflow, + deadline=deadline) + retry_scenario('test_user_group_permissions_lifecycle', + self.test_user_group_permissions_lifecycle, workflowDef, + deadline=deadline) def test_workflow_lifecycle(self, workflowDef, workflow): self.__test_register_workflow_definition(workflowDef) @@ -382,6 +394,22 @@ def __test_unit_test_workflow(self): execution = self.workflow_client.test_workflow(testRequest) assert execution != None + print( + f"[test_workflow] workflow_id={getattr(execution, 'workflow_id', None)} " + f"status={execution.status} " + f"task_count={len(execution.tasks or [])}" + ) + for t in (execution.tasks or []): + print( + f"[test_workflow] task ref={t.reference_task_name} " + f"type={t.task_type} status={t.status} " + f"retried={getattr(t, 'retry_count', None)}" + ) + if execution.status != "COMPLETED": + print( + f"[test_workflow] non-terminal output={getattr(execution, 'output', None)}" + ) + # Ensure workflow is completed successfully assert execution.status == "COMPLETED" @@ -569,9 +597,9 @@ def __test_task_execution_lifecycle(self): self.task_client.add_task_log(polledTask.task_id, "Polled task...") taskExecLogs = self.task_client.get_task_logs(polledTask.task_id) - taskExecLogs[0].log == "Polled task..." + assert taskExecLogs[0].log == "Polled task..." - # First task of second workflow is in the queue + # First task of second workflow is still in the queue assert self.task_client.get_queue_size_for_task(TASK_TYPE) == 1 taskResult = TaskResult( diff --git a/tests/integration/metadata/test_workflow_definition.py b/tests/integration/metadata/test_workflow_definition.py index 95f388271..df4bece56 100644 --- a/tests/integration/metadata/test_workflow_definition.py +++ b/tests/integration/metadata/test_workflow_definition.py @@ -17,14 +17,17 @@ from conductor.client.workflow.task.sub_workflow_task import SubWorkflowTask, InlineSubWorkflowTask from conductor.client.workflow.task.switch_task import SwitchTask from conductor.client.workflow.task.terminate_task import TerminateTask, WorkflowStatus +from tests.integration.retry_helpers import retry_scenario WORKFLOW_NAME = 'python_test_workflow' TASK_NAME = 'python_test_simple_task' WORKFLOW_OWNER_EMAIL = "test@test" -def run_workflow_definition_tests(workflow_executor: WorkflowExecutor) -> None: - test_kitchensink_workflow_registration(workflow_executor) +def run_workflow_definition_tests(workflow_executor: WorkflowExecutor, deadline=None) -> None: + retry_scenario('test_kitchensink_workflow_registration', + test_kitchensink_workflow_registration, workflow_executor, + deadline=deadline) def generate_tasks_defs() -> List[TaskDef]: diff --git a/tests/integration/test_async_lease_extension.py b/tests/integration/test_async_lease_extension.py index 0a4350018..ef0f12e27 100644 --- a/tests/integration/test_async_lease_extension.py +++ b/tests/integration/test_async_lease_extension.py @@ -414,6 +414,15 @@ def test_02_async_without_heartbeat_times_out(self): f"Task should be TIMED_OUT/FAILED, got {task.status}") print("\n PASS: Async task timed out as expected without heartbeat") + # NOTE: excluded from the long-* CI buckets (reusing the + # server_timeout_unreliable marker). This "overhead" test compares + # server-side task durations (end_time - start_time) on the shared sdkdev + # server, so it mostly measures queue/scheduling latency rather than the + # SDK's LeaseManager.track() bookkeeping (a dict insert). On a loaded server + # a fast task can stay non-terminal past the wait budget, so it flakes for + # reasons unrelated to the SDK. A deterministic in-process micro-benchmark + # would be the right home for the no-overhead claim if we want to keep it. + @pytest.mark.server_timeout_unreliable def test_03_no_performance_overhead(self): """Heartbeat tracking adds no meaningful overhead to fast async tasks.""" print("\n" + "=" * 80) diff --git a/tests/integration/test_workflow_client_intg.py b/tests/integration/test_workflow_client_intg.py index 96dda8df0..2f909e454 100644 --- a/tests/integration/test_workflow_client_intg.py +++ b/tests/integration/test_workflow_client_intg.py @@ -8,11 +8,11 @@ from tests.integration.client.orkes.test_orkes_clients import TestOrkesClients from conductor.client.configuration.configuration import Configuration from conductor.client.configuration.settings.authentication_settings import AuthenticationSettings -from conductor.client.http.rest import ApiException from conductor.client.orkes.orkes_workflow_client import OrkesWorkflowClient from conductor.client.workflow.executor.workflow_executor import WorkflowExecutor from tests.integration.metadata.test_workflow_definition import run_workflow_definition_tests from tests.integration.workflow.test_workflow_execution import run_workflow_execution_tests +from tests.integration.retry_helpers import DEFAULT_OVERALL_DEADLINE_SECONDS WORKFLOW_NAME = 'ut_wf' WORKFLOW_UUID = 'ut_wf_uuid' @@ -33,52 +33,6 @@ def get_configuration(): return configuration -def _first_transient_api_exception(exc): - """Walk the exception chain (cause/context) and return the first transient - ApiException (status 0 / flagged transient), if any. - - Inner test helpers sometimes catch an ApiException and re-raise it as a bare - ``Exception`` (losing the type), so we can't rely on the outermost exception - type alone. Implicit chaining still records the original on ``__context__`` - (or ``__cause__`` when ``raise ... from`` is used), so we follow that chain. - """ - seen = set() - cur = exc - while cur is not None and id(cur) not in seen: - seen.add(id(cur)) - if isinstance(cur, ApiException) and ( - getattr(cur, 'transient', False) or cur.status in (0, None)): - return cur - cur = cur.__cause__ or cur.__context__ - return None - - -def _run_tolerating_transient(label, func, *args, retries=3, **kwargs): - """Run a sub-suite, retrying only on a transient (status 0) transport blip - against the shared dev server. - - A `(0)` ApiException is a raw connection/protocol hiccup (read timeout, - stale keep-alive, HTTP/2 GOAWAY race, client closed by a fork cleanup, etc.) - — not a real assertion or server failure — and was observed flaking the - `test-all` bucket. Retrying absorbs that network noise while still surfacing - genuine failures immediately (any non-transient error, or a transient one on - the final attempt, re-raises). We inspect the whole exception chain because - sub-suites may wrap the ApiException in a generic Exception. - """ - for attempt in range(retries): - try: - return func(*args, **kwargs) - except Exception as e: - transient = _first_transient_api_exception(e) - if transient is not None and attempt < retries - 1: - logger.warning( - 'transient (%s) API error in %s (attempt %d/%d): %s; retrying', - transient.status, label, attempt + 1, retries, transient) - time.sleep(2 ** attempt) - continue - raise - - class TestOrkesWorkflowClientIntg(unittest.TestCase): @classmethod @@ -96,14 +50,14 @@ def test_all(self): configuration = self.config workflow_executor = WorkflowExecutor(configuration) - # test_async.test_async_method(api_client) - _run_tolerating_transient( - 'workflow_definition_tests', - run_workflow_definition_tests, workflow_executor) - _run_tolerating_transient( - 'workflow_execution_tests', - run_workflow_execution_tests, configuration, workflow_executor) - _run_tolerating_transient( - 'orkes_clients', - TestOrkesClients(configuration=configuration).run) + # One shared wall-clock budget for the whole aggregate suite. Each + # sub-suite runs its scenarios once; a scenario that hits a transient + # (status 0) transport blip against the shared dev server retries until + # this deadline passes, with capped exponential backoff (see + # tests/integration/retry_helpers.py). Real failures raise immediately. + deadline = time.monotonic() + DEFAULT_OVERALL_DEADLINE_SECONDS + + run_workflow_definition_tests(workflow_executor, deadline=deadline) + run_workflow_execution_tests(configuration, workflow_executor, deadline=deadline) + TestOrkesClients(configuration=configuration).run(deadline=deadline) logger.info('END: integration tests') diff --git a/tests/integration/workflow/test_workflow_execution.py b/tests/integration/workflow/test_workflow_execution.py index af9430b20..2136d89f8 100644 --- a/tests/integration/workflow/test_workflow_execution.py +++ b/tests/integration/workflow/test_workflow_execution.py @@ -12,6 +12,7 @@ from conductor.client.workflow.executor.workflow_executor import WorkflowExecutor from conductor.client.workflow.task.simple_task import SimpleTask from tests.integration.resources.worker.python.python_worker import * +from tests.integration.retry_helpers import retry_scenario WORKFLOW_NAME = "sdk_python_integration_test_workflow" WORKFLOW_DESCRIPTION = "Python SDK Integration Test" @@ -36,7 +37,8 @@ ) -def run_workflow_execution_tests(configuration: Configuration, workflow_executor: WorkflowExecutor): +def run_workflow_execution_tests(configuration: Configuration, workflow_executor: WorkflowExecutor, + deadline=None): workers = [ ClassWorker(TASK_NAME), ClassWorkerWithDomain(TASK_NAME), @@ -59,23 +61,35 @@ def run_workflow_execution_tests(configuration: Configuration, workflow_executor # (they'd surface as an opaque generic Exception instead). finally cleans up # on both success and failure while letting the original error propagate. try: - scenario_get_workflow_by_correlation_ids(workflow_executor) + retry_scenario('scenario_get_workflow_by_correlation_ids', + scenario_get_workflow_by_correlation_ids, workflow_executor, + deadline=deadline) logger.debug('finished workflow correlation ids test') - scenario_workflow_registration(workflow_executor) + retry_scenario('scenario_workflow_registration', + scenario_workflow_registration, workflow_executor, + deadline=deadline) logger.debug('finished workflow registration tests') - scenario_workflow_execution( + retry_scenario( + 'scenario_workflow_execution', scenario_workflow_execution, workflow_quantity=6, workflow_name=WORKFLOW_NAME, workflow_executor=workflow_executor, - workflow_completion_timeout=5.0 + workflow_completion_timeout=5.0, + deadline=deadline, ) - scenario_decorated_workers(workflow_executor) + retry_scenario('scenario_decorated_workers', + scenario_decorated_workers, workflow_executor, + deadline=deadline) logger.debug('finished decorated workers tests') - scenario_execute_workflow_async_features(workflow_executor) + retry_scenario('scenario_execute_workflow_async_features', + scenario_execute_workflow_async_features, workflow_executor, + deadline=deadline) logger.debug('finished execute_workflow reactive features tests') - scenario_execute_workflow_error_handling(workflow_executor) + retry_scenario('scenario_execute_workflow_error_handling', + scenario_execute_workflow_error_handling, workflow_executor, + deadline=deadline) logger.debug('finished execute_workflow error handling tests') - run_signal_tests(configuration, workflow_executor) + run_signal_tests(configuration, workflow_executor, deadline=deadline) logger.debug('finished signal API tests') finally: task_handler.stop_processes() @@ -473,28 +487,44 @@ def _wait_for_workflow_completion(workflow_executor: WorkflowExecutor, workflow_ # ===== SIGNAL TESTS ===== -def run_signal_tests(configuration: Configuration, workflow_executor: WorkflowExecutor): - """Run all signal API tests using WorkflowExecutor methods""" +def run_signal_tests(configuration: Configuration, workflow_executor: WorkflowExecutor, + deadline=None): + """Run all signal API tests using WorkflowExecutor methods. + + Each scenario is retried at the scenario level on a transient blip (see + retry_scenario): a retry starts a fresh workflow and issues a fresh sync + signal, so the asserted SignalResponse is always from a signal this attempt + actually sent — no double-signalling of a single workflow. + """ logger.info('START: Signal API tests using WorkflowExecutor') try: # Register signal test workflows (same as original test) - _register_signal_test_workflows(workflow_executor) + retry_scenario('_register_signal_test_workflows', + _register_signal_test_workflows, workflow_executor, + deadline=deadline) # Test sync signal with different return strategies - scenario_signal_target_workflow(workflow_executor) - scenario_signal_blocking_workflow(workflow_executor) - scenario_signal_blocking_task(workflow_executor) - scenario_signal_blocking_task_input(workflow_executor) + retry_scenario('scenario_signal_target_workflow', + scenario_signal_target_workflow, workflow_executor, deadline=deadline) + retry_scenario('scenario_signal_blocking_workflow', + scenario_signal_blocking_workflow, workflow_executor, deadline=deadline) + retry_scenario('scenario_signal_blocking_task', + scenario_signal_blocking_task, workflow_executor, deadline=deadline) + retry_scenario('scenario_signal_blocking_task_input', + scenario_signal_blocking_task_input, workflow_executor, deadline=deadline) # Test default return strategy - scenario_signal_default_strategy(workflow_executor) + retry_scenario('scenario_signal_default_strategy', + scenario_signal_default_strategy, workflow_executor, deadline=deadline) # Test async signal - scenario_signal_async(workflow_executor) + retry_scenario('scenario_signal_async', + scenario_signal_async, workflow_executor, deadline=deadline) # Test to_dict fix - scenario_signal_to_dict_fix(workflow_executor) + retry_scenario('scenario_signal_to_dict_fix', + scenario_signal_to_dict_fix, workflow_executor, deadline=deadline) logger.info('All signal tests completed successfully') From 6ac2ccdbeb18e46bd5ffe5728963e7068b7bb952 Mon Sep 17 00:00:00 2001 From: Chris Hagglund Date: Wed, 15 Jul 2026 11:00:16 -0600 Subject: [PATCH 16/25] add missing file --- tests/integration/retry_helpers.py | 102 +++++++++++++++++++++++++++++ 1 file changed, 102 insertions(+) create mode 100644 tests/integration/retry_helpers.py diff --git a/tests/integration/retry_helpers.py b/tests/integration/retry_helpers.py new file mode 100644 index 000000000..95eae529d --- /dev/null +++ b/tests/integration/retry_helpers.py @@ -0,0 +1,102 @@ +"""Shared transient-retry helper for the aggregate ``test_all`` integration bucket. + +The bucket runs against a shared dev server whose transport occasionally blips +(read timeout, "server disconnected without a response", HTTP/2 GOAWAY, stale +keep-alive) and surfaces as ``ApiException(status=0)``. Those are not real test +failures. + +Rather than re-running the whole suite on such a blip, ``test_all`` establishes +a single overall wall-clock deadline once, and each *scenario* is retried on a +transient blip until that deadline passes, with capped exponential backoff. Real +errors (assertion failures, genuine 4xx/5xx) raise immediately. + +Scenario-level (rather than per-request) granularity is deliberate: a retried +scenario starts from scratch (fresh workflows, fresh signals), which keeps +otherwise non-idempotent operations — notably ``start_workflow`` and the sync +``signal`` calls whose returned ``SignalResponse`` the tests assert on — safe to +re-run without duplicating side effects against a single workflow instance. +""" + +import logging +import time + +from conductor.client.http.rest import ApiException + +logger = logging.getLogger(__name__) + +# Default overall budget for the whole aggregate suite (all sub-suites share it). +DEFAULT_OVERALL_DEADLINE_SECONDS = 600 # 10 minutes + +# Backoff bounds between scenario retries. +DEFAULT_BASE_DELAY_SECONDS = 1.0 +DEFAULT_MAX_DELAY_SECONDS = 30.0 + + +def first_transient_api_exception(exc): + """Walk the exception chain (``__cause__`` / ``__context__``) and return the + first transient ``ApiException`` (flagged transient, or status 0/None), or + ``None`` if there isn't one. + + Inner test helpers sometimes catch an ApiException and re-raise it as a bare + ``Exception`` (losing the type), so we can't rely on the outermost exception + type alone; implicit chaining still records the original on ``__context__`` + (or ``__cause__`` when ``raise ... from`` is used). + """ + seen = set() + cur = exc + while cur is not None and id(cur) not in seen: + seen.add(id(cur)) + if isinstance(cur, ApiException) and ( + getattr(cur, 'transient', False) or cur.status in (0, None)): + return cur + cur = cur.__cause__ or cur.__context__ + return None + + +def retry_scenario(label, func, *args, deadline=None, + base_delay=DEFAULT_BASE_DELAY_SECONDS, + max_delay=DEFAULT_MAX_DELAY_SECONDS, **kwargs): + """Run ``func(*args, **kwargs)``, retrying only on a transient (status 0) + transport blip until the shared ``deadline`` passes. + + Args: + label: Human-readable scenario name for logs. + func: The scenario callable to run. + deadline: ``time.monotonic()`` value after which we stop retrying. When + ``None`` (e.g. the standalone ``main.py`` runner), the scenario runs + exactly once with no transient retry — preserving prior behaviour. + base_delay / max_delay: capped exponential backoff bounds (seconds). + + Non-transient errors (assertion failures, genuine 4xx/5xx) raise + immediately. A transient blip at/after the deadline re-raises so the suite + fails cleanly rather than hanging past the CI job timeout. + """ + if deadline is None: + logger.info('running scenario %s (attempt 1, no transient retry)', label) + return func(*args, **kwargs) + + attempt = 0 + while True: + logger.info('running scenario %s (attempt %d)', label, attempt + 1) + try: + return func(*args, **kwargs) + except Exception as e: + transient = first_transient_api_exception(e) + if transient is None: + raise + now = time.monotonic() + if now >= deadline: + logger.error( + 'transient (%s) in %s but overall deadline exceeded by ' + '%.0fs; giving up: %s', + transient.status, label, now - deadline, transient) + raise + delay = min(base_delay * (2 ** attempt), max_delay) + delay = min(delay, max(0.0, deadline - now)) + logger.warning( + 'transient (%s) in %s (attempt %d); retrying in %.1fs ' + '(%.0fs left in overall budget): %s', + transient.status, label, attempt + 1, delay, + deadline - now, transient) + time.sleep(delay) + attempt += 1 From 8c0453feb1577999c1a05e761cd496928b30ce3c Mon Sep 17 00:00:00 2001 From: Chris Hagglund Date: Wed, 15 Jul 2026 11:10:21 -0600 Subject: [PATCH 17/25] consolidate retry helper stuff --- tests/integration/retry_helpers.py | 48 +++++++++++++++++-- .../integration/test_async_lease_extension.py | 34 +++---------- tests/integration/test_lease_extension.py | 33 +++---------- .../workflow/test_workflow_execution.py | 27 ++--------- 4 files changed, 62 insertions(+), 80 deletions(-) diff --git a/tests/integration/retry_helpers.py b/tests/integration/retry_helpers.py index 95eae529d..52fe40910 100644 --- a/tests/integration/retry_helpers.py +++ b/tests/integration/retry_helpers.py @@ -1,6 +1,12 @@ -"""Shared transient-retry helper for the aggregate ``test_all`` integration bucket. +"""Shared transient-retry / flakiness helpers for the integration suites. -The bucket runs against a shared dev server whose transport occasionally blips +Originally scoped to the aggregate ``test_all`` bucket, this module is now the +single home for the retry/poll primitives shared across the integration tests: +``is_transient`` (the one definition of "transient transport blip"), +``TERMINAL_WORKFLOW_STATES``, the per-request ``retry_on_transient``, and the +scenario-level ``retry_scenario``. + +The suites run against a shared dev server whose transport occasionally blips (read timeout, "server disconnected without a response", HTTP/2 GOAWAY, stale keep-alive) and surfaces as ``ApiException(status=0)``. Those are not real test failures. @@ -31,6 +37,19 @@ DEFAULT_BASE_DELAY_SECONDS = 1.0 DEFAULT_MAX_DELAY_SECONDS = 30.0 +# Canonical set of terminal workflow statuses, shared by the poll-to-terminal +# helpers across the integration suites so the set can't drift file to file. +TERMINAL_WORKFLOW_STATES = ('COMPLETED', 'FAILED', 'TIMED_OUT', 'TERMINATED') + + +def is_transient(exc): + """True when ``exc`` is a transient transport blip against the shared dev + server (read timeout, connection reset, HTTP/2 GOAWAY, stale keep-alive) + rather than a real failure. status 0/None means no HTTP response arrived. + """ + return isinstance(exc, ApiException) and ( + getattr(exc, 'transient', False) or exc.status in (0, None)) + def first_transient_api_exception(exc): """Walk the exception chain (``__cause__`` / ``__context__``) and return the @@ -46,13 +65,34 @@ def first_transient_api_exception(exc): cur = exc while cur is not None and id(cur) not in seen: seen.add(id(cur)) - if isinstance(cur, ApiException) and ( - getattr(cur, 'transient', False) or cur.status in (0, None)): + if is_transient(cur): return cur cur = cur.__cause__ or cur.__context__ return None +def retry_on_transient(func, *args, retries=4, base_delay=1, **kwargs): + """Retry ``func(*args, **kwargs)`` on a transient (status 0) transport blip + with capped-free exponential backoff. Non-transient errors (real 4xx/5xx, + assertion failures) raise immediately. + + This is *per-request* retry, suited to idempotent calls (get/register). + For non-idempotent scenarios (start_workflow, signal) use ``retry_scenario``, + which re-runs the whole scenario from scratch instead. + """ + for attempt in range(retries): + try: + return func(*args, **kwargs) + except ApiException as e: + if is_transient(e) and attempt < retries - 1: + logger.warning( + 'transient (%s) API error (attempt %d/%d): %s; retrying', + e.status, attempt + 1, retries, e) + time.sleep(base_delay * (2 ** attempt)) + continue + raise + + def retry_scenario(label, func, *args, deadline=None, base_delay=DEFAULT_BASE_DELAY_SECONDS, max_delay=DEFAULT_MAX_DELAY_SECONDS, **kwargs): diff --git a/tests/integration/test_async_lease_extension.py b/tests/integration/test_async_lease_extension.py index ef0f12e27..c920ae198 100644 --- a/tests/integration/test_async_lease_extension.py +++ b/tests/integration/test_async_lease_extension.py @@ -42,35 +42,15 @@ from conductor.client.orkes.orkes_workflow_client import OrkesWorkflowClient from conductor.client.orkes.orkes_metadata_client import OrkesMetadataClient +from tests.integration.retry_helpers import ( + TERMINAL_WORKFLOW_STATES, + is_transient as _is_transient, + retry_on_transient as _retry_on_transient, +) + logging.basicConfig(level=logging.INFO) logger = logging.getLogger(__name__) - -def _is_transient(e): - """A transient (status 0) ApiException is a raw transport blip against the - shared dev server (read timeout, connection reset, HTTP/2 GOAWAY, stale - keep-alive) — not a real failure. status 0 means no HTTP response was - received at all.""" - return isinstance(e, ApiException) and ( - getattr(e, 'transient', False) or e.status in (0, None)) - - -def _retry_on_transient(func, *args, retries=4, base_delay=1, **kwargs): - """Retry a workflow/metadata client call on a transient (0) transport blip. - These `(0)` errors flake the lease suites on a loaded shared server; retrying - absorbs the noise. Non-transient errors (real 4xx/5xx) raise immediately.""" - for attempt in range(retries): - try: - return func(*args, **kwargs) - except ApiException as e: - if _is_transient(e) and attempt < retries - 1: - logger.warning( - 'transient (%s) API error (attempt %d/%d): %s; retrying', - e.status, attempt + 1, retries, e) - time.sleep(base_delay * (2 ** attempt)) - continue - raise - # Short response timeout — task must heartbeat to stay alive RESPONSE_TIMEOUT_SECONDS = 10 @@ -284,7 +264,7 @@ def _start_workflow(self, wf_name, job_id): logger.info("Started workflow %s: %s", wf_name, wf_id) return wf_id - TERMINAL_STATES = ('COMPLETED', 'FAILED', 'TIMED_OUT', 'TERMINATED') + TERMINAL_STATES = TERMINAL_WORKFLOW_STATES def _wait_for_workflow(self, wf_id, timeout_seconds=POLL_TIMEOUT_SECONDS, poll_interval=POLL_INTERVAL_SECONDS): diff --git a/tests/integration/test_lease_extension.py b/tests/integration/test_lease_extension.py index d60ea8eba..fdb206e81 100644 --- a/tests/integration/test_lease_extension.py +++ b/tests/integration/test_lease_extension.py @@ -40,35 +40,16 @@ from conductor.client.orkes.orkes_workflow_client import OrkesWorkflowClient from conductor.client.orkes.orkes_metadata_client import OrkesMetadataClient +from tests.integration.retry_helpers import ( + TERMINAL_WORKFLOW_STATES, + is_transient as _is_transient, + retry_on_transient as _retry_on_transient, +) + logging.basicConfig(level=logging.INFO) logger = logging.getLogger(__name__) -def _is_transient(e): - """A transient (status 0) ApiException is a raw transport blip against the - shared dev server (read timeout, connection reset, HTTP/2 GOAWAY, stale - keep-alive) — not a real failure. status 0 means no HTTP response was - received at all.""" - return isinstance(e, ApiException) and ( - getattr(e, 'transient', False) or e.status in (0, None)) - - -def _retry_on_transient(func, *args, retries=4, base_delay=1, **kwargs): - """Retry a workflow/metadata client call on a transient (0) transport blip. - These `(0)` errors flake the lease suites on a loaded shared server; retrying - absorbs the noise. Non-transient errors (real 4xx/5xx) raise immediately.""" - for attempt in range(retries): - try: - return func(*args, **kwargs) - except ApiException as e: - if _is_transient(e) and attempt < retries - 1: - logger.warning( - 'transient (%s) API error (attempt %d/%d): %s; retrying', - e.status, attempt + 1, retries, e) - time.sleep(base_delay * (2 ** attempt)) - continue - raise - # Short response timeout — task must heartbeat to stay alive RESPONSE_TIMEOUT_SECONDS = 10 @@ -194,7 +175,7 @@ def _start_workflow(self, wf_name, job_id): logger.info("Started workflow %s: %s", wf_name, wf_id) return wf_id - TERMINAL_STATES = ('COMPLETED', 'FAILED', 'TIMED_OUT', 'TERMINATED') + TERMINAL_STATES = TERMINAL_WORKFLOW_STATES def _wait_for_workflow(self, wf_id, timeout_seconds=POLL_TIMEOUT_SECONDS, poll_interval=POLL_INTERVAL_SECONDS): diff --git a/tests/integration/workflow/test_workflow_execution.py b/tests/integration/workflow/test_workflow_execution.py index 2136d89f8..1e1fe32bc 100644 --- a/tests/integration/workflow/test_workflow_execution.py +++ b/tests/integration/workflow/test_workflow_execution.py @@ -12,7 +12,7 @@ from conductor.client.workflow.executor.workflow_executor import WorkflowExecutor from conductor.client.workflow.task.simple_task import SimpleTask from tests.integration.resources.worker.python.python_worker import * -from tests.integration.retry_helpers import retry_scenario +from tests.integration.retry_helpers import retry_scenario, TERMINAL_WORKFLOW_STATES WORKFLOW_NAME = "sdk_python_integration_test_workflow" WORKFLOW_DESCRIPTION = "Python SDK Integration Test" @@ -213,7 +213,6 @@ def _wait_for_workflow_terminal(workflow_id, workflow_executor, deadline, is logged and retried. Returns the last observed status; the caller still runs validate_workflow_status for the actual assertion. """ - terminal = ('COMPLETED', 'FAILED', 'TIMED_OUT', 'TERMINATED') start = time.time() status = None while True: @@ -224,7 +223,7 @@ def _wait_for_workflow_terminal(workflow_id, workflow_executor, deadline, except Exception as e: # transient blip against the shared server logger.warning('error polling workflow %s (%.0fs elapsed): %s', workflow_id, time.time() - start, e) - if status in terminal: + if status in TERMINAL_WORKFLOW_STATES: logger.info('workflow %s reached %s after %.0fs', workflow_id, status, time.time() - start) return status @@ -475,7 +474,7 @@ def _wait_for_workflow_completion(workflow_executor: WorkflowExecutor, workflow_ while time.time() - start_time < max_wait_seconds: workflow = workflow_executor.get_workflow(workflow_id, True) - if workflow.status in ['COMPLETED', 'FAILED', 'TERMINATED', 'TIMED_OUT']: + if workflow.status in TERMINAL_WORKFLOW_STATES: logger.debug(f'Workflow {workflow_id} finished with status: {workflow.status}') return workflow @@ -888,22 +887,4 @@ def scenario_signal_to_dict_fix(workflow_executor: WorkflowExecutor): _wait_for_workflow_completion(workflow_executor, workflow_id) - logger.info('to_dict() method test completed') - - -def _wait_for_workflow_completion(workflow_executor: WorkflowExecutor, workflow_id: str, timeout: int = 10): - """Wait for workflow to complete with timeout""" - max_iterations = timeout * 10 # Check every 0.1 seconds - - for i in range(max_iterations): - try: - workflow = workflow_executor.get_workflow(workflow_id, include_tasks=False) - if workflow.status in ["COMPLETED", "FAILED", "TERMINATED"]: - logger.debug(f'Workflow {workflow_id} completed with status: {workflow.status}') - return - except Exception as e: - logger.warning(f'Error checking workflow status: {e}') - - time.sleep(0.1) - - logger.warning(f'Workflow {workflow_id} did not complete within {timeout} seconds') \ No newline at end of file + logger.info('to_dict() method test completed') \ No newline at end of file From a5dcae6febda8df50f0c1eecf8bd2fc45e203282 Mon Sep 17 00:00:00 2001 From: Chris Hagglund Date: Wed, 15 Jul 2026 11:33:26 -0600 Subject: [PATCH 18/25] test workflow endpoint doesn't seem to always send it back as complete but we can poll for that --- .../client/orkes/test_orkes_clients.py | 61 ++++++++++++++----- 1 file changed, 47 insertions(+), 14 deletions(-) diff --git a/tests/integration/client/orkes/test_orkes_clients.py b/tests/integration/client/orkes/test_orkes_clients.py index 6551bb4f0..7cbcb5580 100644 --- a/tests/integration/client/orkes/test_orkes_clients.py +++ b/tests/integration/client/orkes/test_orkes_clients.py @@ -26,7 +26,7 @@ from conductor.client.workflow.conductor_workflow import ConductorWorkflow from conductor.client.workflow.executor.workflow_executor import WorkflowExecutor from conductor.client.workflow.task.simple_task import SimpleTask -from tests.integration.retry_helpers import retry_scenario +from tests.integration.retry_helpers import retry_scenario, TERMINAL_WORKFLOW_STATES SUFFIX = str(uuid()) WORKFLOW_NAME = 'IntegrationTestOrkesClientsWf_' + SUFFIX @@ -394,24 +394,28 @@ def __test_unit_test_workflow(self): execution = self.workflow_client.test_workflow(testRequest) assert execution != None - print( - f"[test_workflow] workflow_id={getattr(execution, 'workflow_id', None)} " - f"status={execution.status} " - f"task_count={len(execution.tasks or [])}" - ) - for t in (execution.tasks or []): - print( - f"[test_workflow] task ref={t.reference_task_name} " - f"type={t.task_type} status={t.status} " - f"retried={getattr(t, 'retry_count', None)}" - ) + # There appears to be no guarantee about it actually coming back + # complete, because it happens (often) that it does not. So we accept a + # COMPLETED result immediately, but if it isn't COMPLETED yet we don't + # fail: we poll the workflow and wait for it to reach a terminal state. if execution.status != "COMPLETED": + #need to fix the print statement below to have the wf id and the status print( - f"[test_workflow] non-terminal output={getattr(execution, 'output', None)}" + f"[test_workflow] workflow_id={getattr(execution, 'workflow_id', None)} status={execution.status} (was expecting COMPLETED - but will poll for that now)" ) + workflow_id = getattr(execution, "workflow_id", None) + polled = ( + self.__poll_workflow_until_complete(workflow_id) + if workflow_id else None + ) + if polled is not None: + execution = polled # Ensure workflow is completed successfully - assert execution.status == "COMPLETED" + assert execution.status == "COMPLETED", ( + f"workflow expected to be COMPLETED, but received {execution.status}, " + f"workflow_id: {getattr(execution, 'workflow_id', None)}" + ) # Ensure the inputs were captured correctly assert execution.input["loanAmount"] == testRequest.input["loanAmount"] @@ -463,6 +467,35 @@ def __test_unit_test_workflow(self): # Workflow output takes the latest iteration output of a loopOver task. assert execution.output["phoneNumberValid"] + def __poll_workflow_until_complete(self, workflow_id, timeout_seconds=60, + poll_interval=2): + """Poll ``workflow_id`` until it reaches a terminal state or the timeout + passes, returning the last observed Workflow (or None if it could never + be fetched). Transient poll errors are logged and retried within the + timeout so a slow-but-eventually-complete run isn't reported as a bare + failure. + """ + deadline = time.monotonic() + timeout_seconds + workflow = None + while True: + try: + workflow = self.workflow_client.get_workflow( + workflow_id, include_tasks=True) + except Exception as e: # transient blip against the shared server + print(f"[test_workflow] error polling {workflow_id}: {e}") + status = getattr(workflow, "status", None) + if status in TERMINAL_WORKFLOW_STATES: + print(f"[test_workflow] {workflow_id} reached {status}") + return workflow + if time.monotonic() >= deadline: + print( + f"[test_workflow] {workflow_id} still {status} after " + f"{timeout_seconds}s; giving up wait" + ) + return workflow + print(f"[test_workflow] {workflow_id} still {status}; waiting") + time.sleep(poll_interval) + def __test_unregister_workflow_definition(self): self.metadata_client.unregister_workflow_def(WORKFLOW_NAME, 1) From eaa64a1ebd9ce0878029e7423ee4757e24f419a4 Mon Sep 17 00:00:00 2001 From: Chris Hagglund Date: Wed, 15 Jul 2026 11:47:57 -0600 Subject: [PATCH 19/25] refactored some of the retry stuff to dedupe logic --- .../client/orkes/test_orkes_clients.py | 31 ++----- tests/integration/retry_helpers.py | 80 +++++++++++++++++++ .../integration/test_async_lease_extension.py | 28 +++---- tests/integration/test_comprehensive_e2e.py | 23 ++++-- tests/integration/test_lease_extension.py | 28 +++---- .../workflow/test_workflow_execution.py | 64 +++++---------- 6 files changed, 145 insertions(+), 109 deletions(-) diff --git a/tests/integration/client/orkes/test_orkes_clients.py b/tests/integration/client/orkes/test_orkes_clients.py index 7cbcb5580..781a90879 100644 --- a/tests/integration/client/orkes/test_orkes_clients.py +++ b/tests/integration/client/orkes/test_orkes_clients.py @@ -26,7 +26,7 @@ from conductor.client.workflow.conductor_workflow import ConductorWorkflow from conductor.client.workflow.executor.workflow_executor import WorkflowExecutor from conductor.client.workflow.task.simple_task import SimpleTask -from tests.integration.retry_helpers import retry_scenario, TERMINAL_WORKFLOW_STATES +from tests.integration.retry_helpers import retry_scenario, wait_for_workflow_terminal SUFFIX = str(uuid()) WORKFLOW_NAME = 'IntegrationTestOrkesClientsWf_' + SUFFIX @@ -471,30 +471,15 @@ def __poll_workflow_until_complete(self, workflow_id, timeout_seconds=60, poll_interval=2): """Poll ``workflow_id`` until it reaches a terminal state or the timeout passes, returning the last observed Workflow (or None if it could never - be fetched). Transient poll errors are logged and retried within the + be fetched). Transient poll errors are swallowed and retried within the timeout so a slow-but-eventually-complete run isn't reported as a bare - failure. + failure. Thin wrapper over the shared ``wait_for_workflow_terminal``. """ - deadline = time.monotonic() + timeout_seconds - workflow = None - while True: - try: - workflow = self.workflow_client.get_workflow( - workflow_id, include_tasks=True) - except Exception as e: # transient blip against the shared server - print(f"[test_workflow] error polling {workflow_id}: {e}") - status = getattr(workflow, "status", None) - if status in TERMINAL_WORKFLOW_STATES: - print(f"[test_workflow] {workflow_id} reached {status}") - return workflow - if time.monotonic() >= deadline: - print( - f"[test_workflow] {workflow_id} still {status} after " - f"{timeout_seconds}s; giving up wait" - ) - return workflow - print(f"[test_workflow] {workflow_id} still {status}; waiting") - time.sleep(poll_interval) + return wait_for_workflow_terminal( + self.workflow_client, workflow_id, + timeout_seconds=timeout_seconds, poll_interval=poll_interval, + include_tasks=True, swallow='all', + log=lambda msg: print(f"[test_workflow] {msg}")) def __test_unregister_workflow_definition(self): self.metadata_client.unregister_workflow_def(WORKFLOW_NAME, 1) diff --git a/tests/integration/retry_helpers.py b/tests/integration/retry_helpers.py index 52fe40910..273ebb6c2 100644 --- a/tests/integration/retry_helpers.py +++ b/tests/integration/retry_helpers.py @@ -140,3 +140,83 @@ def retry_scenario(label, func, *args, deadline=None, deadline - now, transient) time.sleep(delay) attempt += 1 + + +def wait_for_workflow_terminal( + client, workflow_id, *, + timeout_seconds=None, deadline=None, poll_interval=2, + include_tasks=False, terminal_states=TERMINAL_WORKFLOW_STATES, + is_terminal=None, swallow='transient', + log=None, on_poll=None, on_giveup=None): + """Poll ``client.get_workflow(workflow_id, include_tasks=...)`` until the + workflow reaches a terminal state (or the wall-clock budget is exhausted), + returning the last observed ``Workflow`` (or ``None`` if it could never be + fetched). This is the single shared "poll a workflow to terminal" primitive + for the integration suites — callers wrap it to preserve their own return + shape / side effects. + + Args: + client: anything exposing ``get_workflow(workflow_id, include_tasks=...)`` + (both ``OrkesWorkflowClient`` and ``WorkflowExecutor`` qualify). + timeout_seconds: relative budget; an internal ``time.monotonic()`` + deadline is derived from it. Ignored when ``deadline`` is given. + deadline: absolute ``time.monotonic()`` value to stop at. When both this + and ``timeout_seconds`` are ``None`` the workflow is polled exactly + once (no wait). + poll_interval: seconds between polls. + include_tasks: passed through to ``get_workflow``. + terminal_states: statuses considered terminal (default + ``TERMINAL_WORKFLOW_STATES``); used by the default predicate. + is_terminal: optional ``predicate(workflow) -> bool`` overriding the + default "status in terminal_states" check (e.g. to also gate on an + expected task count). + swallow: error policy for a failing poll — ``'transient'`` swallows only + transient blips (status 0/None) and re-raises real errors; + ``'all'`` swallows every exception and keeps polling; ``'none'`` lets + any exception propagate. + log: optional ``callable(str)`` for progress lines (e.g. ``logger.info`` + or ``print``). Defaults to ``logger.debug``. + on_poll: optional ``callable(workflow)`` invoked after each successful + poll, before the terminal check (e.g. to print per-task diagnostics). + on_giveup: optional ``callable(workflow)`` invoked once when the budget + is exhausted without reaching terminal (e.g. to dump diagnostics). + """ + if log is None: + log = logger.debug + if is_terminal is None: + def is_terminal(wf): + return getattr(wf, 'status', None) in terminal_states + if deadline is None and timeout_seconds is not None: + deadline = time.monotonic() + timeout_seconds + + start = time.monotonic() + workflow = None + while True: + try: + workflow = client.get_workflow( + workflow_id, include_tasks=include_tasks) + except Exception as e: + if swallow == 'none': + raise + if swallow == 'transient' and not is_transient(e): + raise + log('error polling workflow %s (%.0fs elapsed): %s' % ( + workflow_id, time.monotonic() - start, e)) + else: + if on_poll is not None: + on_poll(workflow) + if is_terminal(workflow): + log('workflow %s reached %s after %.0fs' % ( + workflow_id, getattr(workflow, 'status', None), + time.monotonic() - start)) + return workflow + if deadline is None or time.monotonic() >= deadline: + log('workflow %s still %s after %.0fs; giving up wait' % ( + workflow_id, getattr(workflow, 'status', None), + time.monotonic() - start)) + if on_giveup is not None: + on_giveup(workflow) + return workflow + log('workflow %s still %s; waiting' % ( + workflow_id, getattr(workflow, 'status', None))) + time.sleep(poll_interval) diff --git a/tests/integration/test_async_lease_extension.py b/tests/integration/test_async_lease_extension.py index c920ae198..d384ac4d0 100644 --- a/tests/integration/test_async_lease_extension.py +++ b/tests/integration/test_async_lease_extension.py @@ -38,14 +38,13 @@ from conductor.client.http.models.task_def import TaskDef from conductor.client.http.models.workflow_task import WorkflowTask from conductor.client.http.models.start_workflow_request import StartWorkflowRequest -from conductor.client.http.rest import ApiException from conductor.client.orkes.orkes_workflow_client import OrkesWorkflowClient from conductor.client.orkes.orkes_metadata_client import OrkesMetadataClient from tests.integration.retry_helpers import ( TERMINAL_WORKFLOW_STATES, - is_transient as _is_transient, retry_on_transient as _retry_on_transient, + wait_for_workflow_terminal, ) logging.basicConfig(level=logging.INFO) @@ -271,22 +270,17 @@ def _wait_for_workflow(self, wf_id, timeout_seconds=POLL_TIMEOUT_SECONDS, """Poll until workflow reaches a terminal state. If it doesn't within the budget, dump server-side diagnostics so the ensuing assertion shows *why* (e.g. a task stuck in SCHEDULED with no poller) rather than only a - bare status mismatch. + bare status mismatch. Delegates the polling loop to the shared + ``wait_for_workflow_terminal`` (transient blips swallowed, real errors + raised), then does a definitive final fetch + diagnostics on give-up. """ - deadline = time.monotonic() + timeout_seconds - while time.monotonic() < deadline: - try: - wf = self.workflow_client.get_workflow(wf_id, include_tasks=True) - except ApiException as e: - # A transient blip on a single poll shouldn't abort the wait; - # keep polling until the budget is exhausted. - if _is_transient(e): - time.sleep(poll_interval) - continue - raise - if wf.status in self.TERMINAL_STATES: - return wf - time.sleep(poll_interval) + wf = wait_for_workflow_terminal( + self.workflow_client, wf_id, + timeout_seconds=timeout_seconds, poll_interval=poll_interval, + include_tasks=True, terminal_states=self.TERMINAL_STATES, + swallow='transient', log=lambda _msg: None) + if wf is not None and wf.status in self.TERMINAL_STATES: + return wf wf = _retry_on_transient(self.workflow_client.get_workflow, wf_id, include_tasks=True) if wf.status not in self.TERMINAL_STATES: diff --git a/tests/integration/test_comprehensive_e2e.py b/tests/integration/test_comprehensive_e2e.py index 55c2f355c..4580a9174 100644 --- a/tests/integration/test_comprehensive_e2e.py +++ b/tests/integration/test_comprehensive_e2e.py @@ -56,6 +56,7 @@ TaskExecutionStarted, TaskExecutionCompleted, TaskExecutionFailure, TaskUpdateFailure ) +from tests.integration.retry_helpers import wait_for_workflow_terminal # Event collector class EventCollector(TaskRunnerEventsListener): @@ -281,18 +282,24 @@ def _run_workflow_to_terminal(self, workflow_client, timeout_s=90): wf_id = workflow_client.start_workflow(start_workflow_request=req) print(f"✓ Started workflow: {wf_id}") - deadline = time.time() + timeout_s - wf = None - while time.time() < deadline: - wf = workflow_client.get_workflow(wf_id, include_tasks=True) + # "Terminal" here is stricter than the usual terminal-status check: we + # also require the full expected task set to be present, so the caller + # never asserts against a half-scheduled workflow (the flaky "4 != 5 + # tasks" case). Delegates polling to the shared wait_for_workflow_terminal. + def _fully_materialized(wf): + return getattr(wf, 'status', None) in ('COMPLETED', 'FAILED') \ + and len(wf.tasks or []) == self.EXPECTED_TASK_COUNT + + def _show(wf): print(f" Status: {wf.status} - tasks={len(wf.tasks or [])}") - if wf.status in ('COMPLETED', 'FAILED') \ - and len(wf.tasks or []) == self.EXPECTED_TASK_COUNT: - return wf_id, wf for task in (wf.tasks or []): print(f'task {task.task_def_name} : {task.status}') - time.sleep(1) + wf = wait_for_workflow_terminal( + workflow_client, wf_id, + timeout_seconds=timeout_s, poll_interval=1, + include_tasks=True, is_terminal=_fully_materialized, + swallow='none', log=lambda _msg: None, on_poll=_show) return wf_id, wf def test_03_execute_workflow(self): diff --git a/tests/integration/test_lease_extension.py b/tests/integration/test_lease_extension.py index fdb206e81..3b986409f 100644 --- a/tests/integration/test_lease_extension.py +++ b/tests/integration/test_lease_extension.py @@ -36,14 +36,13 @@ from conductor.client.http.models.task_def import TaskDef from conductor.client.http.models.workflow_task import WorkflowTask from conductor.client.http.models.start_workflow_request import StartWorkflowRequest -from conductor.client.http.rest import ApiException from conductor.client.orkes.orkes_workflow_client import OrkesWorkflowClient from conductor.client.orkes.orkes_metadata_client import OrkesMetadataClient from tests.integration.retry_helpers import ( TERMINAL_WORKFLOW_STATES, - is_transient as _is_transient, retry_on_transient as _retry_on_transient, + wait_for_workflow_terminal, ) logging.basicConfig(level=logging.INFO) @@ -182,22 +181,17 @@ def _wait_for_workflow(self, wf_id, timeout_seconds=POLL_TIMEOUT_SECONDS, """Poll until workflow reaches a terminal state. If it doesn't within the budget, dump server-side diagnostics so the ensuing assertion shows *why* (e.g. a task stuck in SCHEDULED with no poller) rather than only a - bare status mismatch. + bare status mismatch. Delegates the polling loop to the shared + ``wait_for_workflow_terminal`` (transient blips swallowed, real errors + raised), then does a definitive final fetch + diagnostics on give-up. """ - deadline = time.monotonic() + timeout_seconds - while time.monotonic() < deadline: - try: - wf = self.workflow_client.get_workflow(wf_id, include_tasks=True) - except ApiException as e: - # A transient blip on a single poll shouldn't abort the wait; - # keep polling until the budget is exhausted. - if _is_transient(e): - time.sleep(poll_interval) - continue - raise - if wf.status in self.TERMINAL_STATES: - return wf - time.sleep(poll_interval) + wf = wait_for_workflow_terminal( + self.workflow_client, wf_id, + timeout_seconds=timeout_seconds, poll_interval=poll_interval, + include_tasks=True, terminal_states=self.TERMINAL_STATES, + swallow='transient', log=lambda _msg: None) + if wf is not None and wf.status in self.TERMINAL_STATES: + return wf wf = _retry_on_transient(self.workflow_client.get_workflow, wf_id, include_tasks=True) if wf.status not in self.TERMINAL_STATES: diff --git a/tests/integration/workflow/test_workflow_execution.py b/tests/integration/workflow/test_workflow_execution.py index 1e1fe32bc..49e534bf7 100644 --- a/tests/integration/workflow/test_workflow_execution.py +++ b/tests/integration/workflow/test_workflow_execution.py @@ -12,7 +12,7 @@ from conductor.client.workflow.executor.workflow_executor import WorkflowExecutor from conductor.client.workflow.task.simple_task import SimpleTask from tests.integration.resources.worker.python.python_worker import * -from tests.integration.retry_helpers import retry_scenario, TERMINAL_WORKFLOW_STATES +from tests.integration.retry_helpers import retry_scenario, wait_for_workflow_terminal WORKFLOW_NAME = "sdk_python_integration_test_workflow" WORKFLOW_DESCRIPTION = "Python SDK Integration Test" @@ -207,34 +207,19 @@ def scenario_workflow_execution( def _wait_for_workflow_terminal(workflow_id, workflow_executor, deadline, poll_interval=2): - """Poll until the workflow reaches a terminal state or the shared deadline - passes, logging progress (wf id + elapsed) so a slow-but-eventually-complete - run is visible instead of surfacing as a bare timeout. A transient poll error - is logged and retried. Returns the last observed status; the caller still - runs validate_workflow_status for the actual assertion. + """Poll until the workflow reaches a terminal state or the shared ``deadline`` + (a ``time.time()`` wall-clock value shared across a batch) passes, logging + progress so a slow-but-eventually-complete run is visible instead of a bare + timeout. A transient poll error is logged and retried. Returns the last + observed status; the caller still runs validate_workflow_status for the + actual assertion. Thin wrapper over the shared ``wait_for_workflow_terminal``. """ - start = time.time() - status = None - while True: - try: - workflow = workflow_executor.get_workflow( - workflow_id=workflow_id, include_tasks=False) - status = workflow.status - except Exception as e: # transient blip against the shared server - logger.warning('error polling workflow %s (%.0fs elapsed): %s', - workflow_id, time.time() - start, e) - if status in TERMINAL_WORKFLOW_STATES: - logger.info('workflow %s reached %s after %.0fs', - workflow_id, status, time.time() - start) - return status - if time.time() >= deadline: - logger.warning( - 'workflow %s still %s after %.0fs; giving up wait', - workflow_id, status, time.time() - start) - return status - logger.info('workflow %s still %s after %.0fs; waiting', - workflow_id, status, time.time() - start) - sleep(poll_interval) + workflow = wait_for_workflow_terminal( + workflow_executor, workflow_id, + timeout_seconds=max(0.0, deadline - time.time()), + poll_interval=poll_interval, include_tasks=False, + swallow='all', log=logger.info) + return getattr(workflow, 'status', None) def generate_workflow(workflow_executor: WorkflowExecutor, workflow_name: str = WORKFLOW_NAME, @@ -467,22 +452,13 @@ def scenario_execute_workflow_error_handling(workflow_executor: WorkflowExecutor def _wait_for_workflow_completion(workflow_executor: WorkflowExecutor, workflow_id: str, max_wait_seconds: int = 60): - """Helper function to wait for workflow completion""" - import time - start_time = time.time() - - while time.time() - start_time < max_wait_seconds: - workflow = workflow_executor.get_workflow(workflow_id, True) - - if workflow.status in TERMINAL_WORKFLOW_STATES: - logger.debug(f'Workflow {workflow_id} finished with status: {workflow.status}') - return workflow - - logger.debug(f'Waiting for workflow {workflow_id}... Status: {workflow.status}') - time.sleep(2) - - # Return final state even if not completed - return workflow_executor.get_workflow(workflow_id, True) + """Helper function to wait for workflow completion. Thin wrapper over the + shared ``wait_for_workflow_terminal``; returns the last observed Workflow. + """ + return wait_for_workflow_terminal( + workflow_executor, workflow_id, + timeout_seconds=max_wait_seconds, poll_interval=2, + include_tasks=True, swallow='none', log=logger.debug) # ===== SIGNAL TESTS ===== From eca49865555d37db1cde6d82f45449d856de7e7d Mon Sep 17 00:00:00 2001 From: Chris Hagglund Date: Wed, 15 Jul 2026 12:02:34 -0600 Subject: [PATCH 20/25] additional retry for stuff that is seeing transient errors/flakiness --- .../metadata/test_schema_service.py | 24 ++++++++++------ tests/integration/retry_helpers.py | 28 +++++++++++++++++++ tests/integration/test_v2_fallback_intg.py | 15 ++++++++-- 3 files changed, 56 insertions(+), 11 deletions(-) diff --git a/tests/integration/metadata/test_schema_service.py b/tests/integration/metadata/test_schema_service.py index e44ca8712..50d4e36a7 100644 --- a/tests/integration/metadata/test_schema_service.py +++ b/tests/integration/metadata/test_schema_service.py @@ -5,6 +5,7 @@ from conductor.client.http.api.schema_resource_api import SchemaResourceApi from conductor.client.http.models.schema_def import SchemaDef, SchemaType from conductor.client.orkes.orkes_schema_client import OrkesSchemaClient +from tests.integration.retry_helpers import retry_on_transient, retry_on_status SCHEMA_NAME = 'ut_schema' SCHEMA_VERSION = 1 @@ -41,33 +42,40 @@ def test_init(self): self.assertIsInstance(self.schema_client.schemaApi, SchemaResourceApi, message) def test_registerSchema(self): - self.schema_client.register_schema(self.schemaDef) - response = self.schema_client.schemaApi.get_schema_by_name_and_version(name=SCHEMA_NAME, version=SCHEMA_VERSION) + retry_on_transient(self.schema_client.register_schema, self.schemaDef) + # A GET right after register can briefly 404 until the write propagates + # on the shared dev server; retry the read rather than fail the test. + response = retry_on_status( + self.schema_client.schemaApi.get_schema_by_name_and_version, + name=SCHEMA_NAME, version=SCHEMA_VERSION) self.assertEqual(response.name, SCHEMA_NAME) self.assertEqual(response.version, SCHEMA_VERSION) self.assertEqual(response.type, SchemaType.JSON) def test_getSchema(self): - self.schema_client.register_schema(self.schemaDef) - schema = self.schema_client.get_schema(SCHEMA_NAME, SCHEMA_VERSION) + retry_on_transient(self.schema_client.register_schema, self.schemaDef) + # A GET right after register can briefly 404 until the write propagates + # on the shared dev server; retry the read rather than fail the test. + schema = retry_on_status(self.schema_client.get_schema, + SCHEMA_NAME, SCHEMA_VERSION) self.assertEqual(schema.name, SCHEMA_NAME) self.assertEqual(schema.version, SCHEMA_VERSION) def test_getAllSchemas(self): schemaDef2 = SchemaDef(name='ut_schema_2', version=1, type=SchemaType.JSON, data=schema, external_ref='http://example.com/2') - self.schema_client.register_schema(self.schemaDef) - self.schema_client.register_schema(schemaDef2) + retry_on_transient(self.schema_client.register_schema, self.schemaDef) + retry_on_transient(self.schema_client.register_schema, schemaDef2) schemas = self.schema_client.get_all_schemas() self.assertGreaterEqual(len(schemas), 2) def test_deleteSchema(self): - self.schema_client.register_schema(self.schemaDef) + retry_on_transient(self.schema_client.register_schema, self.schemaDef) self.schema_client.delete_schema(SCHEMA_NAME, SCHEMA_VERSION) with self.assertRaises(Exception): self.schema_client.get_schema(SCHEMA_NAME, SCHEMA_VERSION) def test_deleteSchemaByName(self): - self.schema_client.register_schema(self.schemaDef) + retry_on_transient(self.schema_client.register_schema, self.schemaDef) self.schema_client.delete_schema_by_name(SCHEMA_NAME) with self.assertRaises(Exception): self.schema_client.get_schema(SCHEMA_NAME, SCHEMA_VERSION) diff --git a/tests/integration/retry_helpers.py b/tests/integration/retry_helpers.py index 273ebb6c2..f1d120dd3 100644 --- a/tests/integration/retry_helpers.py +++ b/tests/integration/retry_helpers.py @@ -93,6 +93,34 @@ def retry_on_transient(func, *args, retries=4, base_delay=1, **kwargs): raise +def retry_on_status(func, *args, statuses=(404,), retries=5, base_delay=1.0, + max_delay=None, **kwargs): + """Retry ``func(*args, **kwargs)`` on an ``ApiException`` whose status is in + ``statuses`` (in addition to transient status-0 blips), with capped + exponential backoff. Any other error raises immediately. + + Intended for read-after-write races against the shared dev server: a GET + issued right after a register/update can briefly 404 until the write + propagates. This is *per-request* retry, so use it only for idempotent reads + (or writes that are safe to repeat). + """ + for attempt in range(retries): + try: + return func(*args, **kwargs) + except ApiException as e: + retryable = is_transient(e) or e.status in statuses + if retryable and attempt < retries - 1: + delay = base_delay * (2 ** attempt) + if max_delay is not None: + delay = min(delay, max_delay) + logger.warning( + 'retryable (%s) API error (attempt %d/%d): %s; retrying in ' + '%.1fs', e.status, attempt + 1, retries, e, delay) + time.sleep(delay) + continue + raise + + def retry_scenario(label, func, *args, deadline=None, base_delay=DEFAULT_BASE_DELAY_SECONDS, max_delay=DEFAULT_MAX_DELAY_SECONDS, **kwargs): diff --git a/tests/integration/test_v2_fallback_intg.py b/tests/integration/test_v2_fallback_intg.py index 4f2258cfa..b265da79d 100644 --- a/tests/integration/test_v2_fallback_intg.py +++ b/tests/integration/test_v2_fallback_intg.py @@ -25,6 +25,7 @@ from conductor.client.orkes.orkes_metadata_client import OrkesMetadataClient from conductor.client.orkes.orkes_workflow_client import OrkesWorkflowClient from conductor.client.worker.worker_task import worker_task +from tests.integration.retry_helpers import retry_on_transient logger = logging.getLogger(__name__) @@ -84,10 +85,14 @@ def test_0_register_workflow(self): workflow = WorkflowDef(name=WORKFLOW_NAME, version=WORKFLOW_VERSION) workflow._tasks = tasks + # Retry registration on a transient (status 0) transport blip against the + # shared dev server so a dropped connection doesn't fail the suite. try: - self.metadata_client.update_workflow_def(workflow, overwrite=True) + retry_on_transient(self.metadata_client.update_workflow_def, + workflow, overwrite=True) except Exception: - self.metadata_client.register_workflow_def(workflow, overwrite=True) + retry_on_transient(self.metadata_client.register_workflow_def, + workflow, overwrite=True) print(f"\n Registered workflow '{WORKFLOW_NAME}' with {len(tasks)} tasks") def test_1_workflows_complete_with_v2_or_fallback(self): @@ -128,7 +133,11 @@ def _run_workers(): req.name = WORKFLOW_NAME req.version = WORKFLOW_VERSION req.input = {"run_index": i} - wf_id = self.workflow_client.start_workflow(start_workflow_request=req) + # A status-0 blip here means no response arrived, so no id was + # returned; retrying gets a fresh attempt (any orphaned run just + # completes untracked and doesn't affect the tracked-id count). + wf_id = retry_on_transient(self.workflow_client.start_workflow, + start_workflow_request=req) workflow_ids.append(wf_id) print(f"\n Submitted {len(workflow_ids)} workflows") From 36042ac2d7adb49154bee50b8d07656c94a7994b Mon Sep 17 00:00:00 2001 From: Chris Hagglund Date: Wed, 15 Jul 2026 12:28:39 -0600 Subject: [PATCH 21/25] add nginx temporary fail codes to transient exception ignorification/retry --- scripts/run_integration_tests.sh | 11 ++++++-- tests/integration/retry_helpers.py | 43 +++++++++++++++++++++--------- 2 files changed, 39 insertions(+), 15 deletions(-) diff --git a/scripts/run_integration_tests.sh b/scripts/run_integration_tests.sh index 689a9ef22..d2722accc 100755 --- a/scripts/run_integration_tests.sh +++ b/scripts/run_integration_tests.sh @@ -7,7 +7,10 @@ # dedicated AI-enabled server (test_ai_task_types.py and test_ai_examples.py # hardcode http://localhost:7001/api, and test_agentic_workflows.py needs an # `openai` LLM provider configured on the server), so they don't run against -# the standard test server this suite targets. +# the standard test server this suite targets. The entire tests/integration/ai/ +# suite is excluded for the same reason: it drives the agent runtime (POST +# /api/agent/start plus an LLM provider), which the standard server doesn't +# expose (it 404s /api/agent/start). # # The performance test (test_update_task_v2_perf.py) is also excluded by # default: it submits ~1000 workflows and takes several minutes. Pass @@ -93,11 +96,15 @@ for arg in "$@"; do done # The AI/agentic tests always target a dedicated server (see header) and are -# never part of these buckets. +# never part of these buckets. The tests/integration/ai/ suite drives the agent +# runtime (POST /api/agent/start, an LLM provider, SSE streaming): the standard +# sdkdev server has no agent API and returns 404 for every one of them, so the +# whole directory is excluded here rather than file-by-file. ai_ignore=( --ignore=tests/integration/test_ai_task_types.py --ignore=tests/integration/test_ai_examples.py --ignore=tests/integration/test_agentic_workflows.py + --ignore=tests/integration/ai ) # Build the target paths + selection for the chosen bucket. The slow buckets are diff --git a/tests/integration/retry_helpers.py b/tests/integration/retry_helpers.py index f1d120dd3..4f69ed792 100644 --- a/tests/integration/retry_helpers.py +++ b/tests/integration/retry_helpers.py @@ -8,8 +8,9 @@ The suites run against a shared dev server whose transport occasionally blips (read timeout, "server disconnected without a response", HTTP/2 GOAWAY, stale -keep-alive) and surfaces as ``ApiException(status=0)``. Those are not real test -failures. +keep-alive) and surfaces as ``ApiException(status=0)``, or whose proxy/LB +briefly returns a gateway-class 5xx (502/503/504) while the upstream is +unreachable or restarting. Those are not real test failures. Rather than re-running the whole suite on such a blip, ``test_all`` establishes a single overall wall-clock deadline once, and each *scenario* is retried on a @@ -41,14 +42,28 @@ # helpers across the integration suites so the set can't drift file to file. TERMINAL_WORKFLOW_STATES = ('COMPLETED', 'FAILED', 'TIMED_OUT', 'TERMINATED') +# Gateway-class 5xx statuses. On the shared dev server these come from the +# proxy/LB in front of Conductor (nginx "502 Bad Gateway" / "503 Service +# Temporarily Unavailable" / "504 Gateway Timeout") when the upstream is briefly +# unreachable or restarting, not from the Conductor app itself — so they're +# transient infrastructure blips, not real test failures. 500 is deliberately +# excluded (more likely a genuine app error) and so is 429 (wants Retry-After +# handling rather than blind backoff). +GATEWAY_STATUSES = (502, 503, 504) + def is_transient(exc): - """True when ``exc`` is a transient transport blip against the shared dev - server (read timeout, connection reset, HTTP/2 GOAWAY, stale keep-alive) - rather than a real failure. status 0/None means no HTTP response arrived. + """True when ``exc`` is a transient blip against the shared dev server + rather than a real failure. Covers both transport-level hiccups where no + HTTP response arrived (read timeout, connection reset, HTTP/2 GOAWAY, stale + keep-alive — surfaced as status 0/None or the ``transient`` flag) and + gateway-class 5xx (502/503/504) returned by the proxy/LB in front of the + server (see ``GATEWAY_STATUSES``). """ return isinstance(exc, ApiException) and ( - getattr(exc, 'transient', False) or exc.status in (0, None)) + getattr(exc, 'transient', False) + or exc.status in (0, None) + or exc.status in GATEWAY_STATUSES) def first_transient_api_exception(exc): @@ -72,9 +87,10 @@ def first_transient_api_exception(exc): def retry_on_transient(func, *args, retries=4, base_delay=1, **kwargs): - """Retry ``func(*args, **kwargs)`` on a transient (status 0) transport blip - with capped-free exponential backoff. Non-transient errors (real 4xx/5xx, - assertion failures) raise immediately. + """Retry ``func(*args, **kwargs)`` on a transient blip (see ``is_transient``: + status 0/None transport hiccups plus gateway-class 502/503/504) with + capped-free exponential backoff. Non-transient errors (real 4xx, app-level + 500, assertion failures) raise immediately. This is *per-request* retry, suited to idempotent calls (get/register). For non-idempotent scenarios (start_workflow, signal) use ``retry_scenario``, @@ -96,8 +112,8 @@ def retry_on_transient(func, *args, retries=4, base_delay=1, **kwargs): def retry_on_status(func, *args, statuses=(404,), retries=5, base_delay=1.0, max_delay=None, **kwargs): """Retry ``func(*args, **kwargs)`` on an ``ApiException`` whose status is in - ``statuses`` (in addition to transient status-0 blips), with capped - exponential backoff. Any other error raises immediately. + ``statuses`` (in addition to transient blips — see ``is_transient``), with + capped exponential backoff. Any other error raises immediately. Intended for read-after-write races against the shared dev server: a GET issued right after a register/update can briefly 404 until the write @@ -124,8 +140,9 @@ def retry_on_status(func, *args, statuses=(404,), retries=5, base_delay=1.0, def retry_scenario(label, func, *args, deadline=None, base_delay=DEFAULT_BASE_DELAY_SECONDS, max_delay=DEFAULT_MAX_DELAY_SECONDS, **kwargs): - """Run ``func(*args, **kwargs)``, retrying only on a transient (status 0) - transport blip until the shared ``deadline`` passes. + """Run ``func(*args, **kwargs)``, retrying only on a transient blip (see + ``is_transient``: status 0/None transport hiccups plus gateway-class + 502/503/504) until the shared ``deadline`` passes. Args: label: Human-readable scenario name for logs. From 35d62479f7f42c0af22da1aa2ee9253aecdf09a7 Mon Sep 17 00:00:00 2001 From: Chris Hagglund Date: Wed, 15 Jul 2026 12:29:48 -0600 Subject: [PATCH 22/25] remove no longer needd branch target --- .github/workflows/pull_request.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index 57416c0cc..9fe1d4906 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -4,9 +4,6 @@ on: push: branches: - main - # Temporary: run CI on this branch's pushes until it's exercised via a - # real PR targeting main. Remove once that PR validates the workflow. - - integ-test-on-pr pull_request: branches: - main From 514280c27fa475717c74c2b3f37cd6402b5a0302 Mon Sep 17 00:00:00 2001 From: Chris Hagglund Date: Wed, 15 Jul 2026 12:42:15 -0600 Subject: [PATCH 23/25] access schedule props as object not dict --- tests/integration/client/orkes/test_orkes_clients.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/integration/client/orkes/test_orkes_clients.py b/tests/integration/client/orkes/test_orkes_clients.py index 781a90879..b179217cb 100644 --- a/tests/integration/client/orkes/test_orkes_clients.py +++ b/tests/integration/client/orkes/test_orkes_clients.py @@ -184,7 +184,7 @@ def test_scheduler_lifecycle(self, workflowDef): schedule = self.scheduler_client.get_schedule(SCHEDULE_NAME) - assert schedule['name'] == SCHEDULE_NAME + assert schedule.name == SCHEDULE_NAME self.scheduler_client.pause_schedule(SCHEDULE_NAME) @@ -195,7 +195,7 @@ def test_scheduler_lifecycle(self, workflowDef): self.scheduler_client.resume_schedule(SCHEDULE_NAME) schedule = self.scheduler_client.get_schedule(SCHEDULE_NAME) - assert not schedule['paused'] + assert not schedule.paused times = self.scheduler_client.get_next_few_schedule_execution_times("0 */5 * ? * *", limit=1) assert (len(times) == 1) From c197c5e2a786b0843c86d13d81f6dad0b5f8b008 Mon Sep 17 00:00:00 2001 From: Chris Hagglund Date: Wed, 15 Jul 2026 12:53:22 -0600 Subject: [PATCH 24/25] remove comment --- tests/integration/client/orkes/test_orkes_clients.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/integration/client/orkes/test_orkes_clients.py b/tests/integration/client/orkes/test_orkes_clients.py index b179217cb..2811d4926 100644 --- a/tests/integration/client/orkes/test_orkes_clients.py +++ b/tests/integration/client/orkes/test_orkes_clients.py @@ -399,7 +399,6 @@ def __test_unit_test_workflow(self): # COMPLETED result immediately, but if it isn't COMPLETED yet we don't # fail: we poll the workflow and wait for it to reach a terminal state. if execution.status != "COMPLETED": - #need to fix the print statement below to have the wf id and the status print( f"[test_workflow] workflow_id={getattr(execution, 'workflow_id', None)} status={execution.status} (was expecting COMPLETED - but will poll for that now)" ) From 4ffe01b96e0ba02d7ea9324c35571766d5f6a5af Mon Sep 17 00:00:00 2001 From: Chris Hagglund Date: Wed, 15 Jul 2026 13:12:09 -0600 Subject: [PATCH 25/25] test with additional resilience --- tests/integration/test_comprehensive_e2e.py | 51 ++++++++++++++++++++- 1 file changed, 50 insertions(+), 1 deletion(-) diff --git a/tests/integration/test_comprehensive_e2e.py b/tests/integration/test_comprehensive_e2e.py index 4580a9174..4fcfac8e6 100644 --- a/tests/integration/test_comprehensive_e2e.py +++ b/tests/integration/test_comprehensive_e2e.py @@ -24,6 +24,7 @@ """ import asyncio +import functools import logging import os import sys @@ -56,7 +57,11 @@ TaskExecutionStarted, TaskExecutionCompleted, TaskExecutionFailure, TaskUpdateFailure ) -from tests.integration.retry_helpers import wait_for_workflow_terminal +from tests.integration.retry_helpers import ( + DEFAULT_OVERALL_DEADLINE_SECONDS, + retry_scenario, + wait_for_workflow_terminal, +) # Event collector class EventCollector(TaskRunnerEventsListener): @@ -134,6 +139,29 @@ def failing_task(should_fail: bool) -> dict: # Main test class +def _retry_on_transient_until_deadline(test_method): + """Wrap a ``test_0xx`` method so a transient (status-0) transport blip against + the shared dev server re-runs the *whole* test until the class-wide deadline, + instead of failing the suite on a one-off dropped connection. + + This is the same model ``test_all`` uses via ``retry_scenario``: only a + transient ``ApiException(status=0)`` (no HTTP response arrived) is retried; + real failures (assertions, genuine 4xx/5xx) raise immediately, and a blip + at/after the deadline re-raises so the suite still fails cleanly rather than + hanging past the CI job timeout. Each test here starts from the shared + class state (workflow registered in test_01, workers up from test_02), so a + retry re-reads/re-executes against that same state safely. + """ + @functools.wraps(test_method) + def wrapper(self, *args, **kwargs): + deadline = getattr(self.__class__, '_retry_deadline', None) + return retry_scenario( + test_method.__name__, + lambda: test_method(self, *args, **kwargs), + deadline=deadline) + return wrapper + + class TestComprehensiveE2E(unittest.TestCase): # Annotated workers this suite relies on. Every one must be discovered by @@ -172,6 +200,12 @@ def setUpClass(cls): cls.workers_started = False cls.task_handler = None + # One shared wall-clock budget for the whole suite: a test that hits a + # transient (status-0) transport blip retries until this deadline passes + # (see _retry_on_transient_until_deadline). Real failures raise at once. + cls._retry_deadline = time.monotonic() + DEFAULT_OVERALL_DEADLINE_SECONDS + + @_retry_on_transient_until_deadline def test_01_create_workflow(self): """Create test workflow.""" print("\n" + "="*80 + "\nTEST 1: Create Workflow\n" + "="*80) @@ -198,10 +232,19 @@ def test_01_create_workflow(self): print("✓ Workflow registered") self.assertTrue(True) # Workflow registration succeeded + @_retry_on_transient_until_deadline def test_02_start_workers(self): """Start workers and verify they initialize.""" print("\n" + "="*80 + "\nTEST 2: Start Workers\n" + "="*80) + # If a prior attempt started a handler (this test can be retried on a + # transient blip), stop it first so a retry doesn't orphan its worker + # processes. + prior = getattr(self.__class__, 'task_handler', None) + if prior is not None: + prior.stop_processes() + self.__class__.task_handler = None + # Start the workers and keep the handler on the class so it stays alive # for the remaining tests and is stopped deterministically in # tearDownClass. (A previous version ran the handler in a daemon thread @@ -302,6 +345,7 @@ def _show(wf): swallow='none', log=lambda _msg: None, on_poll=_show) return wf_id, wf + @_retry_on_transient_until_deadline def test_03_execute_workflow(self): """Execute workflow and verify completion.""" print("\n" + "="*80 + "\nTEST 3: Execute Workflow\n" + "="*80) @@ -363,6 +407,7 @@ def test_03_execute_workflow(self): print("✓ All task assertions passed") + @_retry_on_transient_until_deadline def test_04_verify_events(self): """Verify event system works (via metrics and task execution).""" print("\n" + "="*80 + "\nTEST 4: Event System\n" + "="*80) @@ -390,6 +435,7 @@ def test_04_verify_events(self): print("✓ Event system architecture verified") print(" (Events are process-local - actual event testing done in unit tests)") + @_retry_on_transient_until_deadline def test_05_verify_task_definitions(self): """Verify task definitions and schemas were registered.""" print("\n" + "="*80 + "\nTEST 5: Task Registration & Schemas\n" + "="*80) @@ -419,6 +465,7 @@ def test_05_verify_task_definitions(self): print("✓ All task definition assertions passed") + @_retry_on_transient_until_deadline def test_06_verify_metrics(self): """Verify metrics were collected.""" print("\n" + "="*80 + "\nTEST 6: Metrics Collection\n" + "="*80) @@ -464,6 +511,7 @@ def test_06_verify_metrics(self): print("✓ Metrics system verified and operational") + @_retry_on_transient_until_deadline def test_07_configuration_assertions(self): """Verify configuration system works.""" print("\n" + "="*80 + "\nTEST 7: Configuration System\n" + "="*80) @@ -485,6 +533,7 @@ def test_07_configuration_assertions(self): self.assertTrue(True) + @_retry_on_transient_until_deadline def test_08_summary_assertions(self): """Final comprehensive assertions.""" print("\n" + "="*80 + "\nTEST 8: Summary & Final Assertions\n" + "="*80)