From db091b5771e4ad14a16be5bd7903eae80cccab21 Mon Sep 17 00:00:00 2001 From: evgeny Date: Mon, 14 Sep 2026 13:53:32 +0100 Subject: [PATCH 01/71] packaging: rename distribution to ably-pubsub-server MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Publish the SDK as `ably-pubsub-server` rather than `ably`, installing into `ably/pubsub` so that further per-product distributions can share the `ably` namespace. All existing code moves to `ably/pubsub` unchanged; the public API is now reached through `ably.pubsub.server`. Neither `ably` nor `ably.pubsub` carries an `__init__.py` — both are PEP 420 namespace packages. This matters for the next distribution: a regular package wins outright over any namespace portion during import resolution, so an `ably/pubsub/__init__.py` here would silently shadow an `ably/pubsub/client` installed under a different sys.path entry, which is exactly what an editable checkout of this repo alongside a released sibling looks like. Consequences of having no package root to put things in: - `api_version` and `lib_version` move to `ably/pubsub/version.py`, as `http/httputils.py` and `types/options.py` read them and core code must not depend on the server package. - The NullHandler silencing the library's loggers moves to `ably/pubsub/util/__init__.py`, the one module every entry point imports. - unasync writes the sync tree to `ably/pubsub/sync` and maps `ably.pubsub.server` onto the hand-written `ably.pubsub.server.sync`. Also fix `HttpUtils.default_headers()`, which read `api_version` and `lib_version` off the `ably` namespace and so raised AttributeError. Only the integration tests cover the Ably-Agent header, so the unit suite was silent on it. Tests import from `ably.pubsub.server`; UPDATING.md covers the 4.0.0 migration. Co-Authored-By: Claude Opus 5 (1M context) --- .claude/skills/release/SKILL.md | 2 +- .github/workflows/release.yml | 14 ++--- .gitignore | 2 +- CONTRIBUTING.md | 2 +- LONG_DESCRIPTION.rst | 15 +++-- README.md | 23 +++++++- UPDATING.md | 51 ++++++++++++++++ ably/__init__.py | 24 -------- ably/{ => pubsub}/http/__init__.py | 0 ably/{ => pubsub}/http/http.py | 12 ++-- ably/{ => pubsub}/http/httputils.py | 6 +- ably/{ => pubsub}/http/paginatedresult.py | 4 +- ably/{ => pubsub}/realtime/__init__.py | 0 ably/{ => pubsub}/realtime/annotations.py | 18 +++--- ably/{ => pubsub}/realtime/channel.py | 40 ++++++------- ably/{ => pubsub}/realtime/connection.py | 12 ++-- .../realtime/connectionmanager.py | 24 ++++---- ably/{ => pubsub}/realtime/presence.py | 16 ++--- ably/{ => pubsub}/realtime/presencemap.py | 2 +- ably/{ => pubsub}/realtime/realtime.py | 6 +- ably/{ => pubsub}/rest/__init__.py | 0 ably/{ => pubsub}/rest/annotations.py | 10 ++-- ably/{ => pubsub}/rest/auth.py | 20 +++---- ably/{ => pubsub}/rest/channel.py | 16 ++--- ably/{ => pubsub}/rest/push.py | 6 +- ably/{ => pubsub}/rest/rest.py | 18 +++--- ably/{ => pubsub}/scripts/__init__.py | 0 ably/{ => pubsub}/scripts/unasync.py | 31 ++++++---- ably/pubsub/server/__init__.py | 59 +++++++++++++++++++ ably/pubsub/server/sync.py | 54 +++++++++++++++++ ably/{ => pubsub}/transport/__init__.py | 0 ably/{ => pubsub}/transport/defaults.py | 0 .../transport/websockettransport.py | 16 ++--- ably/{ => pubsub}/types/__init__.py | 0 ably/{ => pubsub}/types/annotation.py | 10 ++-- ably/{ => pubsub}/types/authoptions.py | 2 +- ably/{ => pubsub}/types/capability.py | 0 ably/{ => pubsub}/types/channeldetails.py | 0 ably/{ => pubsub}/types/channelmode.py | 2 +- ably/{ => pubsub}/types/channeloptions.py | 6 +- ably/{ => pubsub}/types/channelstate.py | 2 +- .../{ => pubsub}/types/channelsubscription.py | 2 +- ably/{ => pubsub}/types/connectiondetails.py | 0 ably/{ => pubsub}/types/connectionerrors.py | 4 +- ably/{ => pubsub}/types/connectionstate.py | 2 +- ably/{ => pubsub}/types/device.py | 2 +- ably/{ => pubsub}/types/flags.py | 0 ably/{ => pubsub}/types/message.py | 12 ++-- ably/{ => pubsub}/types/mixins.py | 4 +- ably/{ => pubsub}/types/operations.py | 0 ably/{ => pubsub}/types/options.py | 8 +-- ably/{ => pubsub}/types/presence.py | 10 ++-- ably/{ => pubsub}/types/stats.py | 0 ably/{ => pubsub}/types/testoptions.py | 2 +- ably/{ => pubsub}/types/tokendetails.py | 2 +- ably/{ => pubsub}/types/tokenrequest.py | 0 ably/{ => pubsub}/types/typedbuffer.py | 0 ably/pubsub/util/__init__.py | 10 ++++ ably/{ => pubsub}/util/case.py | 0 ably/{ => pubsub}/util/clock.py | 2 +- ably/{ => pubsub}/util/crypto.py | 4 +- ably/{ => pubsub}/util/encoding.py | 4 +- ably/{ => pubsub}/util/eventemitter.py | 2 +- ably/{ => pubsub}/util/exceptions.py | 0 ably/{ => pubsub}/util/helper.py | 2 +- ably/{ => pubsub}/util/nocrypto.py | 0 ably/{util => pubsub/vcdiff}/__init__.py | 0 .../vcdiff/defaultvcdiffdecoder.py | 6 +- ably/pubsub/version.py | 9 +++ ably/vcdiff/__init__.py | 0 pyproject.toml | 12 ++-- test/ably/realtime/deltadecoding_test.py | 12 ++-- test/ably/realtime/eventemitter_test.py | 2 +- test/ably/realtime/presencemap_test.py | 4 +- .../ably/realtime/realtimeannotations_test.py | 10 ++-- test/ably/realtime/realtimeauth_test.py | 10 ++-- .../realtime/realtimechannel_publish_test.py | 22 +++---- test/ably/realtime/realtimechannel_test.py | 12 ++-- .../realtime/realtimechannel_vcdiff_test.py | 8 +-- .../realtimechannelmutablemessages_test.py | 6 +- test/ably/realtime/realtimeconnection_test.py | 8 +-- test/ably/realtime/realtimeinit_test.py | 6 +- test/ably/realtime/realtimepresence_test.py | 8 +-- test/ably/realtime/realtimeresume_test.py | 6 +- test/ably/rest/encoders_test.py | 44 +++++++------- test/ably/rest/restannotations_test.py | 6 +- test/ably/rest/restauth_test.py | 22 +++---- test/ably/rest/restcapability_test.py | 4 +- test/ably/rest/restchannelhistory_test.py | 4 +- .../rest/restchannelmutablemessages_test.py | 6 +- test/ably/rest/restchannelpublish_test.py | 16 ++--- test/ably/rest/restchannels_test.py | 6 +- test/ably/rest/restcrypto_test.py | 6 +- test/ably/rest/resthttp_test.py | 10 ++-- test/ably/rest/restinit_test.py | 10 ++-- test/ably/rest/restpaginatedresult_test.py | 2 +- test/ably/rest/restpresence_test.py | 4 +- test/ably/rest/restpush_test.py | 4 +- test/ably/rest/restrequest_test.py | 6 +- test/ably/rest/reststats_test.py | 6 +- test/ably/rest/resttime_test.py | 2 +- test/ably/rest/resttoken_test.py | 24 ++++---- test/ably/testapp.py | 12 ++-- test/ably/utils.py | 2 +- test/unit/annotation_test.py | 10 ++-- test/unit/clock_test.py | 8 +-- test/unit/http_test.py | 6 +- test/unit/message_test.py | 8 +-- test/unit/mutable_message_test.py | 6 +- test/unit/options_test.py | 4 +- test/unit/websockettransport_test.py | 6 +- test/uts/helpers/client.py | 10 ++-- test/uts/helpers/clock.py | 2 +- test/uts/helpers/mock_http_test.py | 6 +- test/uts/helpers/mock_websocket.py | 2 +- test/uts/helpers/mock_websocket_test.py | 4 +- test/uts/helpers/presence.py | 8 +-- test/uts/helpers/sandbox.py | 2 +- .../integration/auth/token_renewal_test.py | 2 +- .../integration/auth/token_request_test.py | 2 +- test/uts/realtime/integration/auth_test.py | 2 +- .../integration/channel_history_test.py | 2 +- .../channels/channel_attach_test.py | 6 +- .../channels/channel_publish_test.py | 4 +- .../channels/channel_subscribe_test.py | 2 +- .../connection/connection_failures_test.py | 2 +- .../integration/connection_lifecycle_test.py | 2 +- .../integration/delta_decoding_test.py | 12 ++-- .../integration/mutable_messages_test.py | 18 +++--- .../presence/presence_sync_test.py | 4 +- .../integration/presence_lifecycle_test.py | 4 +- .../integration/proxy/auth_reauth_test.py | 2 +- .../integration/proxy/channel_faults_test.py | 6 +- .../proxy/connection_open_failures_test.py | 4 +- .../proxy/connection_resume_test.py | 4 +- .../integration/proxy/heartbeat_test.py | 2 +- .../proxy/presence_reentry_test.py | 4 +- .../integration/proxy/rest_faults_test.py | 8 +-- .../unit/auth/auth_callback_errors_test.py | 8 +-- .../unit/auth/connection_auth_test.py | 8 +-- .../unit/auth/realtime_authorize_test.py | 10 ++-- .../auth/token_expiry_non_renewable_test.py | 2 +- .../channel_additional_attached_test.py | 8 +-- .../unit/channels/channel_annotations_test.py | 18 +++--- .../unit/channels/channel_attach_test.py | 14 ++--- .../unit/channels/channel_attributes_test.py | 8 +-- .../channels/channel_connection_state_test.py | 8 +-- .../channels/channel_delta_decoding_test.py | 8 +-- .../unit/channels/channel_detach_test.py | 8 +-- .../unit/channels/channel_error_test.py | 8 +-- .../unit/channels/channel_get_message_test.py | 2 +- .../unit/channels/channel_history_test.py | 10 ++-- .../channels/channel_message_versions_test.py | 4 +- .../unit/channels/channel_options_test.py | 26 ++++---- .../unit/channels/channel_properties_test.py | 8 +-- .../channels/channel_publish_pending_test.py | 10 ++-- .../unit/channels/channel_publish_test.py | 12 ++-- .../channel_server_initiated_detach_test.py | 8 +-- .../channels/channel_state_events_test.py | 10 ++-- .../unit/channels/channel_subscribe_test.py | 12 ++-- .../channel_update_delete_message_test.py | 10 ++-- .../unit/channels/channel_when_state_test.py | 6 +- .../unit/channels/channels_collection_test.py | 10 ++-- .../channels/message_field_population_test.py | 4 +- .../unit/client/realtime_client_test.py | 14 ++--- .../unit/client/realtime_stats_test.py | 2 +- .../unit/client/realtime_timeouts_test.py | 10 ++-- .../unit/connection/auto_connect_test.py | 2 +- .../unit/connection/backoff_jitter_test.py | 6 +- .../connection/connection_failures_test.py | 4 +- .../unit/connection/connection_id_key_test.py | 4 +- .../connection_open_failures_test.py | 4 +- .../unit/connection/connection_ping_test.py | 4 +- .../connection/connection_recovery_test.py | 6 +- .../unit/connection/error_reason_test.py | 4 +- .../unit/connection/fallback_hosts_test.py | 6 +- .../connection/forwards_compatibility_test.py | 6 +- .../unit/connection/heartbeat_test.py | 2 +- .../server_initiated_reauth_test.py | 6 +- .../unit/connection/update_events_test.py | 4 +- .../unit/connection/when_state_test.py | 2 +- .../unit/presence/local_presence_map_test.py | 4 +- .../unit/presence/presence_map_test.py | 2 +- .../unit/presence/presence_sync_test.py | 2 +- .../realtime_presence_channel_state_test.py | 14 ++--- .../presence/realtime_presence_enter_test.py | 12 ++-- .../presence/realtime_presence_get_test.py | 10 ++-- .../realtime_presence_history_test.py | 8 +-- .../realtime_presence_reentry_test.py | 8 +-- .../realtime_presence_subscribe_test.py | 12 ++-- test/uts/rest/integration/auth_test.py | 4 +- .../rest/integration/batch_presence_test.py | 2 +- .../rest/integration/mutable_messages_test.py | 10 ++-- test/uts/rest/integration/pagination_test.py | 4 +- test/uts/rest/integration/presence_test.py | 10 ++-- .../integration/proxy/rest_fallback_test.py | 4 +- test/uts/rest/integration/publish_test.py | 4 +- test/uts/rest/integration/push_admin_test.py | 4 +- .../rest/integration/push_channels_test.py | 6 +- .../rest/integration/revoke_tokens_test.py | 4 +- test/uts/rest/integration/time_stats_test.py | 6 +- test/uts/rest/unit/auth/auth_callback_test.py | 8 +-- test/uts/rest/unit/auth/auth_scheme_test.py | 6 +- test/uts/rest/unit/auth/authorize_test.py | 6 +- test/uts/rest/unit/auth/client_id_test.py | 4 +- test/uts/rest/unit/auth/revoke_tokens_test.py | 2 +- test/uts/rest/unit/auth/token_details_test.py | 8 +-- test/uts/rest/unit/auth/token_renewal_test.py | 4 +- test/uts/rest/unit/batch_presence_test.py | 4 +- test/uts/rest/unit/batch_publish_test.py | 4 +- .../uts/rest/unit/channel/annotations_test.py | 8 +-- .../uts/rest/unit/channel/get_message_test.py | 4 +- test/uts/rest/unit/channel/history_test.py | 4 +- .../uts/rest/unit/channel/idempotency_test.py | 2 +- .../unit/channel/message_versions_test.py | 4 +- .../rest/unit/channel/publish_result_test.py | 4 +- test/uts/rest/unit/channel/publish_test.py | 4 +- .../channel/rest_channel_attributes_test.py | 4 +- .../channel/update_delete_message_test.py | 6 +- .../uts/rest/unit/channels_collection_test.py | 2 +- .../unit/encoding/message_encoding_test.py | 2 +- .../unit/encoding/msgpack_interop_test.py | 2 +- test/uts/rest/unit/fallback_test.py | 2 +- .../rest/unit/presence/rest_presence_test.py | 8 +-- .../rest/unit/push/push_admin_publish_test.py | 4 +- .../push/push_channel_subscriptions_test.py | 6 +- test/uts/rest/unit/push/push_channels_test.py | 10 ++-- .../push/push_device_registrations_test.py | 6 +- test/uts/rest/unit/rest_client_test.py | 4 +- test/uts/rest/unit/stats_test.py | 4 +- test/uts/rest/unit/time_test.py | 2 +- test/uts/rest/unit/types/error_types_test.py | 2 +- .../uts/rest/unit/types/message_types_test.py | 2 +- .../unit/types/mutable_message_types_test.py | 6 +- .../uts/rest/unit/types/options_types_test.py | 10 ++-- .../rest/unit/types/paginated_result_test.py | 6 +- .../unit/types/presence_message_types_test.py | 2 +- test/uts/rest/unit/types/token_types_test.py | 4 +- uv.lock | 2 +- 239 files changed, 966 insertions(+), 770 deletions(-) delete mode 100644 ably/__init__.py rename ably/{ => pubsub}/http/__init__.py (100%) rename ably/{ => pubsub}/http/http.py (97%) rename ably/{ => pubsub}/http/httputils.py (90%) rename ably/{ => pubsub}/http/paginatedresult.py (98%) rename ably/{ => pubsub}/realtime/__init__.py (100%) rename ably/{ => pubsub}/realtime/annotations.py (94%) rename ably/{ => pubsub}/realtime/channel.py (96%) rename ably/{ => pubsub}/realtime/connection.py (90%) rename ably/{ => pubsub}/realtime/connectionmanager.py (97%) rename ably/{ => pubsub}/realtime/presence.py (98%) rename ably/{ => pubsub}/realtime/presencemap.py (99%) rename ably/{ => pubsub}/realtime/realtime.py (97%) rename ably/{ => pubsub}/rest/__init__.py (100%) rename ably/{ => pubsub}/rest/annotations.py (96%) rename ably/{ => pubsub}/rest/auth.py (96%) rename ably/{ => pubsub}/rest/channel.py (97%) rename ably/{ => pubsub}/rest/push.py (96%) rename ably/{ => pubsub}/rest/rest.py (91%) rename ably/{ => pubsub}/scripts/__init__.py (100%) rename ably/{ => pubsub}/scripts/unasync.py (87%) create mode 100644 ably/pubsub/server/__init__.py create mode 100644 ably/pubsub/server/sync.py rename ably/{ => pubsub}/transport/__init__.py (100%) rename ably/{ => pubsub}/transport/defaults.py (100%) rename ably/{ => pubsub}/transport/websockettransport.py (96%) rename ably/{ => pubsub}/types/__init__.py (100%) rename ably/{ => pubsub}/types/annotation.py (98%) rename ably/{ => pubsub}/types/authoptions.py (98%) rename ably/{ => pubsub}/types/capability.py (100%) rename ably/{ => pubsub}/types/channeldetails.py (100%) rename ably/{ => pubsub}/types/channelmode.py (96%) rename ably/{ => pubsub}/types/channeloptions.py (94%) rename ably/{ => pubsub}/types/channelstate.py (89%) rename ably/{ => pubsub}/types/channelsubscription.py (98%) rename ably/{ => pubsub}/types/connectiondetails.py (100%) rename ably/{ => pubsub}/types/connectionerrors.py (84%) rename ably/{ => pubsub}/types/connectionstate.py (93%) rename ably/{ => pubsub}/types/device.py (98%) rename ably/{ => pubsub}/types/flags.py (100%) rename ably/{ => pubsub}/types/message.py (97%) rename ably/{ => pubsub}/types/mixins.py (98%) rename ably/{ => pubsub}/types/operations.py (100%) rename ably/{ => pubsub}/types/options.py (98%) rename ably/{ => pubsub}/types/presence.py (96%) rename ably/{ => pubsub}/types/stats.py (100%) rename ably/{ => pubsub}/types/testoptions.py (93%) rename ably/{ => pubsub}/types/tokendetails.py (97%) rename ably/{ => pubsub}/types/tokenrequest.py (100%) rename ably/{ => pubsub}/types/typedbuffer.py (100%) create mode 100644 ably/pubsub/util/__init__.py rename ably/{ => pubsub}/util/case.py (100%) rename ably/{ => pubsub}/util/clock.py (97%) rename ably/{ => pubsub}/util/crypto.py (98%) rename ably/{ => pubsub}/util/encoding.py (91%) rename ably/{ => pubsub}/util/eventemitter.py (99%) rename ably/{ => pubsub}/util/exceptions.py (100%) rename ably/{ => pubsub}/util/helper.py (98%) rename ably/{ => pubsub}/util/nocrypto.py (100%) rename ably/{util => pubsub/vcdiff}/__init__.py (100%) rename ably/{ => pubsub}/vcdiff/defaultvcdiffdecoder.py (92%) create mode 100644 ably/pubsub/version.py delete mode 100644 ably/vcdiff/__init__.py diff --git a/.claude/skills/release/SKILL.md b/.claude/skills/release/SKILL.md index 01539b45..985df013 100644 --- a/.claude/skills/release/SKILL.md +++ b/.claude/skills/release/SKILL.md @@ -17,7 +17,7 @@ Then perform these steps in order: 1. Run `git checkout -b release/NEW_VERSION` 2. Replace `OLD_VERSION` with `NEW_VERSION` in all of the following files: - `pyproject.toml` — the `version` property - - `ably/__init__.py` — lib_version value + - `ably/pubsub/version.py` — lib_version value 3.Run `uv sync` to update `uv.lock` file 4. Commit all files together with message: `chore: bump version to NEW_VERSION` 3. Fetch merged PRs since the last release tag using: diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 2a7a2ae2..69b4c6d3 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -42,27 +42,27 @@ jobs: with: name: python-package-distributions path: dist/ - - name: Check that wheel and tarball contains ably/sync/ + - name: Check that wheel and tarball contains ably/pubsub/sync/ run: | # Check wheel WHEEL=$(ls dist/*.whl | head -n 1) echo "Checking wheel: $WHEEL" - if unzip -l "$WHEEL" | grep -q "ably/sync/"; then - echo "✅ Found ably/sync/ in wheel" + if unzip -l "$WHEEL" | grep -q "ably/pubsub/sync/"; then + echo "✅ Found ably/pubsub/sync/ in wheel" else unzip -l "$WHEEL" - echo "❌ ably/sync/ not found in wheel" + echo "❌ ably/pubsub/sync/ not found in wheel" exit 1 fi # Check tarball TARBALL=$(ls dist/*.tar.gz | head -n 1) echo "Checking tarball: $TARBALL" - if tar -tzf "$TARBALL" | grep -q "ably/sync/"; then - echo "✅ Found ably/sync/ in tarball" + if tar -tzf "$TARBALL" | grep -q "ably/pubsub/sync/"; then + echo "✅ Found ably/pubsub/sync/ in tarball" else tar -tzf "$TARBALL" - echo "❌ ably/sync/ not found in tarball" + echo "❌ ably/pubsub/sync/ not found in tarball" exit 1 fi diff --git a/.gitignore b/.gitignore index 75ec0f34..426ac840 100644 --- a/.gitignore +++ b/.gitignore @@ -55,5 +55,5 @@ ably/types/options.py.orig test/ably/restsetup.py.orig .idea/**/* -ably/sync/** +ably/pubsub/sync/** test/ably/sync/** diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index ed650adb..b75ffbaa 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -40,7 +40,7 @@ The release process must include the following steps: 1. Ensure that all work intended for this release has landed to `main` 2. Create a release branch named like `release/2.0.1` -3. Add a commit to bump the version number, updating [`pyproject.toml`](./pyproject.toml) and [`ably/__init__.py`](./ably/__init__.py) +3. Add a commit to bump the version number, updating [`pyproject.toml`](./pyproject.toml) and [`ably/pubsub/version.py`](./ably/pubsub/version.py) 4. Run [`github_changelog_generator`](https://github.com/github-changelog-generator/github-changelog-generator) to automate the update of the [CHANGELOG](./CHANGELOG.md). This may require some manual intervention, both in terms of how the command is run and how the change log file is modified. Your mileage may vary: - The command you will need to run will look something like this: `github_changelog_generator -u ably -p ably-pubsub-python --since-tag v2.0.0 --output delta.md --token $GITHUB_TOKEN_WITH_REPO_ACCESS`. Generate token [here](https://github.com/settings/tokens/new?description=GitHub%20Changelog%20Generator%20token). - Using the command above, `--output delta.md` writes changes made after `--since-tag` to a new file diff --git a/LONG_DESCRIPTION.rst b/LONG_DESCRIPTION.rst index 518f2f86..ec4622a3 100644 --- a/LONG_DESCRIPTION.rst +++ b/LONG_DESCRIPTION.rst @@ -1,7 +1,8 @@ -Official Ably Bindings for Python -================================== +Official Ably Pub/Sub Bindings for Python Servers +================================================= -A Python client library for Ably Realtime messaging. +A Python client library for Ably Pub/Sub realtime messaging, for use from +servers. Setup @@ -9,7 +10,13 @@ Setup You can install this package by using the pip tool and installing: - pip install ably + pip install ably-pubsub-server + +The package installs into ``ably.pubsub``. Both ``ably`` and ``ably.pubsub`` +are namespace packages shared with the other ``ably-*`` distributions, so the +whole public API is reached through ``ably.pubsub.server``:: + + from ably.pubsub.server import AblyRealtime, AblyRest Using Ably for Python diff --git a/README.md b/README.md index 5bfb89e0..c0e66cf8 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,5 @@ ![Ably Pub/Sub Python Header](images/pythonSDK-github.png) -[![PyPI version](https://badge.fury.io/py/ably.svg)](https://pypi.org/project/ably/) +[![PyPI version](https://badge.fury.io/py/ably-pubsub-server.svg)](https://pypi.org/project/ably-pubsub-server/) [![License](https://img.shields.io/github/license/ably/ably-pubsub-python)](https://github.com/ably/ably-pubsub-python/blob/main/LICENSE) @@ -48,9 +48,26 @@ The following platforms are supported: To get started with your project, install the package: ```sh -pip install ably +pip install ably-pubsub-server ``` +The package installs into `ably.pubsub`, and the whole public API is reached +through `ably.pubsub.server`: + +```python +from ably.pubsub.server import AblyRealtime, AblyRest +``` + +The synchronous, HTTP-only flavour lives alongside it: + +```python +from ably.pubsub.server.sync import AblyRestSync +``` + +Both `ably` and `ably.pubsub` are [namespace packages](https://peps.python.org/pep-0420/) +shared with the other `ably-*` distributions, so neither exports anything +itself — always import from `ably.pubsub.server`. + > [!NOTE] Install [Python](https://www.python.org/downloads/) version 3.8 or greater. @@ -59,6 +76,8 @@ Install [Python](https://www.python.org/downloads/) version 3.8 or greater. The following code connects to Ably's realtime messaging service, subscribes to a channel to receive messages, and publishes a test message to that same channel. ```python +from ably.pubsub.server import AblyRealtime + # Initialize Ably Realtime client async with AblyRealtime('your-ably-api-key', client_id='me') as realtime_client: # Wait for connection to be established diff --git a/UPDATING.md b/UPDATING.md index 4b4dd719..7ec94b63 100644 --- a/UPDATING.md +++ b/UPDATING.md @@ -1,5 +1,56 @@ # Upgrade / Migration Guide +## Version 3.x to 4.0.0 + +The package is now published to PyPI as `ably-pubsub-server` rather than `ably`, and +it installs into `ably.pubsub` rather than `ably`. Both `ably` and `ably.pubsub` are +[namespace packages](https://peps.python.org/pep-0420/) shared with the other +`ably-*` distributions, so neither exports anything of its own — the whole public +API is reached through `ably.pubsub.server`. + +### Installation + +Example 3.x: +```sh +pip install ably +``` + +Example 4.0.0: +```sh +pip install ably-pubsub-server +``` + +### Imports + +Example 3.x code: +```python +from ably import AblyRealtime, AblyRest +``` + +Example 4.0.0 code: +```python +from ably.pubsub.server import AblyRealtime, AblyRest +``` + +The synchronous, HTTP-only flavour moves the same way: + +Example 3.x code: +```python +from ably.sync import AblyRestSync +``` + +Example 4.0.0 code: +```python +from ably.pubsub.server.sync import AblyRestSync +``` + +Anything imported from a submodule gains the same `pubsub` segment — for +instance `ably.types.message` becomes `ably.pubsub.types.message`, and +`ably.sync.types.message` becomes `ably.pubsub.sync.types.message`. + +`api_version` and `lib_version` are still re-exported from +`ably.pubsub.server`, and now also live in `ably.pubsub.version`. + ## Version 2.x to 3.0.0 The 3.0.0 version of ably-python introduces several breaking changes to improve the realtime experience and align the API with the Ably specification. These include: diff --git a/ably/__init__.py b/ably/__init__.py deleted file mode 100644 index 95412484..00000000 --- a/ably/__init__.py +++ /dev/null @@ -1,24 +0,0 @@ -import logging - -from ably.realtime.realtime import AblyRealtime -from ably.rest.auth import Auth -from ably.rest.push import Push -from ably.rest.rest import AblyRest -from ably.types.annotation import Annotation, AnnotationAction -from ably.types.capability import Capability -from ably.types.channelmode import ChannelMode -from ably.types.channeloptions import ChannelOptions -from ably.types.channelsubscription import PushChannelSubscription -from ably.types.device import DeviceDetails -from ably.types.message import MessageAction, MessageVersion -from ably.types.operations import MessageOperation, PublishResult, UpdateDeleteResult -from ably.types.options import Options, VCDiffDecoder -from ably.util.crypto import CipherParams -from ably.util.exceptions import AblyAuthException, AblyException, IncompatibleClientIdException -from ably.vcdiff.defaultvcdiffdecoder import AblyVCDiffDecoder - -logger = logging.getLogger(__name__) -logger.addHandler(logging.NullHandler()) - -api_version = '5' -lib_version = '3.1.4' diff --git a/ably/http/__init__.py b/ably/pubsub/http/__init__.py similarity index 100% rename from ably/http/__init__.py rename to ably/pubsub/http/__init__.py diff --git a/ably/http/http.py b/ably/pubsub/http/http.py similarity index 97% rename from ably/http/http.py rename to ably/pubsub/http/http.py index 6ffd6d36..9677504e 100644 --- a/ably/http/http.py +++ b/ably/pubsub/http/http.py @@ -6,12 +6,12 @@ import httpx import msgpack -from ably.http.httputils import HttpUtils -from ably.rest.auth import Auth -from ably.transport.defaults import Defaults -from ably.util.clock import select_clock -from ably.util.exceptions import AblyException -from ably.util.helper import extract_url_params, is_token_error +from ably.pubsub.http.httputils import HttpUtils +from ably.pubsub.rest.auth import Auth +from ably.pubsub.transport.defaults import Defaults +from ably.pubsub.util.clock import select_clock +from ably.pubsub.util.exceptions import AblyException +from ably.pubsub.util.helper import extract_url_params, is_token_error log = logging.getLogger(__name__) diff --git a/ably/http/httputils.py b/ably/pubsub/http/httputils.py similarity index 90% rename from ably/http/httputils.py rename to ably/pubsub/http/httputils.py index 4bd3e78e..2410580f 100644 --- a/ably/http/httputils.py +++ b/ably/pubsub/http/httputils.py @@ -3,7 +3,7 @@ import os import platform -import ably +from ably.pubsub.version import api_version, lib_version class HttpUtils: @@ -53,10 +53,10 @@ def get_host_header(host): @staticmethod def default_headers(version=None): if version is None: - version = ably.api_version + version = api_version return { "X-Ably-Version": version, - "Ably-Agent": f'ably-python/{ably.lib_version} python/{platform.python_version()}' + "Ably-Agent": f'ably-python/{lib_version} python/{platform.python_version()}' } @staticmethod diff --git a/ably/http/paginatedresult.py b/ably/pubsub/http/paginatedresult.py similarity index 98% rename from ably/http/paginatedresult.py rename to ably/pubsub/http/paginatedresult.py index 5b8bc158..17df8b27 100644 --- a/ably/http/paginatedresult.py +++ b/ably/pubsub/http/paginatedresult.py @@ -2,8 +2,8 @@ import logging from urllib.parse import urlencode -from ably.http.http import Request -from ably.util import case +from ably.pubsub.http.http import Request +from ably.pubsub.util import case log = logging.getLogger(__name__) diff --git a/ably/realtime/__init__.py b/ably/pubsub/realtime/__init__.py similarity index 100% rename from ably/realtime/__init__.py rename to ably/pubsub/realtime/__init__.py diff --git a/ably/realtime/annotations.py b/ably/pubsub/realtime/annotations.py similarity index 94% rename from ably/realtime/annotations.py rename to ably/pubsub/realtime/annotations.py index 57400d45..01acfccc 100644 --- a/ably/realtime/annotations.py +++ b/ably/pubsub/realtime/annotations.py @@ -3,17 +3,17 @@ import logging from typing import TYPE_CHECKING -from ably.rest.annotations import RestAnnotations, construct_validate_annotation -from ably.transport.websockettransport import ProtocolMessageAction -from ably.types.annotation import Annotation, AnnotationAction -from ably.types.channelmode import ChannelMode -from ably.types.channelstate import ChannelState -from ably.util.eventemitter import EventEmitter -from ably.util.helper import is_callable_or_coroutine +from ably.pubsub.rest.annotations import RestAnnotations, construct_validate_annotation +from ably.pubsub.transport.websockettransport import ProtocolMessageAction +from ably.pubsub.types.annotation import Annotation, AnnotationAction +from ably.pubsub.types.channelmode import ChannelMode +from ably.pubsub.types.channelstate import ChannelState +from ably.pubsub.util.eventemitter import EventEmitter +from ably.pubsub.util.helper import is_callable_or_coroutine if TYPE_CHECKING: - from ably.realtime.channel import RealtimeChannel - from ably.realtime.connectionmanager import ConnectionManager + from ably.pubsub.realtime.channel import RealtimeChannel + from ably.pubsub.realtime.connectionmanager import ConnectionManager log = logging.getLogger(__name__) diff --git a/ably/realtime/channel.py b/ably/pubsub/realtime/channel.py similarity index 96% rename from ably/realtime/channel.py rename to ably/pubsub/realtime/channel.py index fde9bb06..cf57042e 100644 --- a/ably/realtime/channel.py +++ b/ably/pubsub/realtime/channel.py @@ -4,28 +4,28 @@ import logging from typing import TYPE_CHECKING -from ably.realtime.annotations import RealtimeAnnotations -from ably.realtime.connection import ConnectionState -from ably.realtime.presence import RealtimePresence -from ably.rest.channel import Channel -from ably.rest.channel import Channels as RestChannels -from ably.transport.websockettransport import ProtocolMessageAction -from ably.types.annotation import Annotation -from ably.types.channelmode import ChannelMode, decode_channel_mode, encode_channel_mode -from ably.types.channeloptions import ChannelOptions -from ably.types.channelstate import ChannelState, ChannelStateChange -from ably.types.flags import Flag, has_flag -from ably.types.message import Message, MessageAction, MessageVersion -from ably.types.mixins import DecodingContext -from ably.types.operations import MessageOperation, PublishResult, UpdateDeleteResult -from ably.types.presence import PresenceMessage -from ably.util.clock import select_clock -from ably.util.eventemitter import EventEmitter -from ably.util.exceptions import AblyException, IncompatibleClientIdException -from ably.util.helper import Timer, is_callable_or_coroutine, validate_message_size +from ably.pubsub.realtime.annotations import RealtimeAnnotations +from ably.pubsub.realtime.connection import ConnectionState +from ably.pubsub.realtime.presence import RealtimePresence +from ably.pubsub.rest.channel import Channel +from ably.pubsub.rest.channel import Channels as RestChannels +from ably.pubsub.transport.websockettransport import ProtocolMessageAction +from ably.pubsub.types.annotation import Annotation +from ably.pubsub.types.channelmode import ChannelMode, decode_channel_mode, encode_channel_mode +from ably.pubsub.types.channeloptions import ChannelOptions +from ably.pubsub.types.channelstate import ChannelState, ChannelStateChange +from ably.pubsub.types.flags import Flag, has_flag +from ably.pubsub.types.message import Message, MessageAction, MessageVersion +from ably.pubsub.types.mixins import DecodingContext +from ably.pubsub.types.operations import MessageOperation, PublishResult, UpdateDeleteResult +from ably.pubsub.types.presence import PresenceMessage +from ably.pubsub.util.clock import select_clock +from ably.pubsub.util.eventemitter import EventEmitter +from ably.pubsub.util.exceptions import AblyException, IncompatibleClientIdException +from ably.pubsub.util.helper import Timer, is_callable_or_coroutine, validate_message_size if TYPE_CHECKING: - from ably.realtime.realtime import AblyRealtime + from ably.pubsub.realtime.realtime import AblyRealtime log = logging.getLogger(__name__) diff --git a/ably/realtime/connection.py b/ably/pubsub/realtime/connection.py similarity index 90% rename from ably/realtime/connection.py rename to ably/pubsub/realtime/connection.py index 907f56a5..d13909a9 100644 --- a/ably/realtime/connection.py +++ b/ably/pubsub/realtime/connection.py @@ -5,14 +5,14 @@ import logging from typing import TYPE_CHECKING -from ably.realtime.connectionmanager import ConnectionManager -from ably.types.connectiondetails import ConnectionDetails -from ably.types.connectionstate import ConnectionEvent, ConnectionState, ConnectionStateChange -from ably.util.eventemitter import EventEmitter -from ably.util.exceptions import AblyException +from ably.pubsub.realtime.connectionmanager import ConnectionManager +from ably.pubsub.types.connectiondetails import ConnectionDetails +from ably.pubsub.types.connectionstate import ConnectionEvent, ConnectionState, ConnectionStateChange +from ably.pubsub.util.eventemitter import EventEmitter +from ably.pubsub.util.exceptions import AblyException if TYPE_CHECKING: - from ably.realtime.realtime import AblyRealtime + from ably.pubsub.realtime.realtime import AblyRealtime log = logging.getLogger(__name__) diff --git a/ably/realtime/connectionmanager.py b/ably/pubsub/realtime/connectionmanager.py similarity index 97% rename from ably/realtime/connectionmanager.py rename to ably/pubsub/realtime/connectionmanager.py index 9f920b07..b5523a47 100644 --- a/ably/realtime/connectionmanager.py +++ b/ably/pubsub/realtime/connectionmanager.py @@ -8,20 +8,20 @@ import httpx -from ably.transport.defaults import Defaults -from ably.transport.websockettransport import ProtocolMessageAction, WebSocketTransport -from ably.types.connectiondetails import ConnectionDetails -from ably.types.connectionerrors import ConnectionErrors -from ably.types.connectionstate import ConnectionEvent, ConnectionState, ConnectionStateChange -from ably.types.operations import PublishResult -from ably.types.tokendetails import TokenDetails -from ably.util.clock import Clock, select_clock -from ably.util.eventemitter import EventEmitter -from ably.util.exceptions import AblyException, IncompatibleClientIdException -from ably.util.helper import Timer, get_random_id, is_token_error +from ably.pubsub.transport.defaults import Defaults +from ably.pubsub.transport.websockettransport import ProtocolMessageAction, WebSocketTransport +from ably.pubsub.types.connectiondetails import ConnectionDetails +from ably.pubsub.types.connectionerrors import ConnectionErrors +from ably.pubsub.types.connectionstate import ConnectionEvent, ConnectionState, ConnectionStateChange +from ably.pubsub.types.operations import PublishResult +from ably.pubsub.types.tokendetails import TokenDetails +from ably.pubsub.util.clock import Clock, select_clock +from ably.pubsub.util.eventemitter import EventEmitter +from ably.pubsub.util.exceptions import AblyException, IncompatibleClientIdException +from ably.pubsub.util.helper import Timer, get_random_id, is_token_error if TYPE_CHECKING: - from ably.realtime.realtime import AblyRealtime + from ably.pubsub.realtime.realtime import AblyRealtime log = logging.getLogger(__name__) diff --git a/ably/realtime/presence.py b/ably/pubsub/realtime/presence.py similarity index 98% rename from ably/realtime/presence.py rename to ably/pubsub/realtime/presence.py index 79d73070..60b3b211 100644 --- a/ably/realtime/presence.py +++ b/ably/pubsub/realtime/presence.py @@ -12,15 +12,15 @@ from datetime import datetime, timezone from typing import TYPE_CHECKING, Any -from ably.realtime.connection import ConnectionState -from ably.realtime.presencemap import PresenceMap -from ably.types.channelstate import ChannelState, ChannelStateChange -from ably.types.presence import PresenceAction, PresenceMessage -from ably.util.eventemitter import EventEmitter -from ably.util.exceptions import AblyException +from ably.pubsub.realtime.connection import ConnectionState +from ably.pubsub.realtime.presencemap import PresenceMap +from ably.pubsub.types.channelstate import ChannelState, ChannelStateChange +from ably.pubsub.types.presence import PresenceAction, PresenceMessage +from ably.pubsub.util.eventemitter import EventEmitter +from ably.pubsub.util.exceptions import AblyException if TYPE_CHECKING: - from ably.realtime.channel import RealtimeChannel + from ably.pubsub.realtime.channel import RealtimeChannel log = logging.getLogger(__name__) @@ -354,7 +354,7 @@ async def _send_presence(self, presence_messages: list[dict]) -> None: Args: presence_messages: List of encoded presence messages to send """ - from ably.transport.websockettransport import ProtocolMessageAction + from ably.pubsub.transport.websockettransport import ProtocolMessageAction protocol_msg = { 'action': ProtocolMessageAction.PRESENCE, diff --git a/ably/realtime/presencemap.py b/ably/pubsub/realtime/presencemap.py similarity index 99% rename from ably/realtime/presencemap.py rename to ably/pubsub/realtime/presencemap.py index 9c5adace..a5d2c427 100644 --- a/ably/realtime/presencemap.py +++ b/ably/pubsub/realtime/presencemap.py @@ -7,7 +7,7 @@ import logging from typing import Callable, Dict, List, Optional, Tuple -from ably.types.presence import PresenceAction, PresenceMessage +from ably.pubsub.types.presence import PresenceAction, PresenceMessage logger = logging.getLogger(__name__) diff --git a/ably/realtime/realtime.py b/ably/pubsub/realtime/realtime.py similarity index 97% rename from ably/realtime/realtime.py rename to ably/pubsub/realtime/realtime.py index ab435304..4fa90934 100644 --- a/ably/realtime/realtime.py +++ b/ably/pubsub/realtime/realtime.py @@ -2,9 +2,9 @@ import logging from typing import Optional -from ably.realtime.channel import Channels -from ably.realtime.connection import Connection, ConnectionState -from ably.rest.rest import AblyRest +from ably.pubsub.realtime.channel import Channels +from ably.pubsub.realtime.connection import Connection, ConnectionState +from ably.pubsub.rest.rest import AblyRest log = logging.getLogger(__name__) diff --git a/ably/rest/__init__.py b/ably/pubsub/rest/__init__.py similarity index 100% rename from ably/rest/__init__.py rename to ably/pubsub/rest/__init__.py diff --git a/ably/rest/annotations.py b/ably/pubsub/rest/annotations.py similarity index 96% rename from ably/rest/annotations.py rename to ably/pubsub/rest/annotations.py index 39c1664d..b84d2bd4 100644 --- a/ably/rest/annotations.py +++ b/ably/pubsub/rest/annotations.py @@ -8,15 +8,15 @@ import msgpack -from ably.http.paginatedresult import PaginatedResult, format_params -from ably.types.annotation import ( +from ably.pubsub.http.paginatedresult import PaginatedResult, format_params +from ably.pubsub.types.annotation import ( Annotation, AnnotationAction, make_annotation_response_handler, ) -from ably.types.message import Message -from ably.types.options import Options -from ably.util.exceptions import AblyException +from ably.pubsub.types.message import Message +from ably.pubsub.types.options import Options +from ably.pubsub.util.exceptions import AblyException log = logging.getLogger(__name__) diff --git a/ably/rest/auth.py b/ably/pubsub/rest/auth.py similarity index 96% rename from ably/rest/auth.py rename to ably/pubsub/rest/auth.py index fe2aeae3..4360b744 100644 --- a/ably/rest/auth.py +++ b/ably/pubsub/rest/auth.py @@ -6,18 +6,18 @@ from datetime import timedelta from typing import TYPE_CHECKING -from ably.types.options import Options +from ably.pubsub.types.options import Options if TYPE_CHECKING: - from ably.realtime.realtime import AblyRealtime - from ably.rest.rest import AblyRest + from ably.pubsub.realtime.realtime import AblyRealtime + from ably.pubsub.rest.rest import AblyRest -from ably.types.capability import Capability -from ably.types.tokendetails import TokenDetails -from ably.types.tokenrequest import TokenRequest -from ably.util.clock import select_clock -from ably.util.exceptions import AblyAuthException, AblyException, IncompatibleClientIdException -from ably.util.helper import extract_url_params +from ably.pubsub.types.capability import Capability +from ably.pubsub.types.tokendetails import TokenDetails +from ably.pubsub.types.tokenrequest import TokenRequest +from ably.pubsub.util.clock import select_clock +from ably.pubsub.util.exceptions import AblyAuthException, AblyException, IncompatibleClientIdException +from ably.pubsub.util.helper import extract_url_params __all__ = ["Auth"] @@ -409,7 +409,7 @@ async def token_request_from_auth_url(self, method: str, url: str, token_params, # Use clean URL for the request url = clean_url - from ably.http.http import Response + from ably.pubsub.http.http import Response resp = await self.ably.http.request_external( method=method, url=url, headers=headers, params=params, body=body) response = Response(resp) diff --git a/ably/rest/channel.py b/ably/pubsub/rest/channel.py similarity index 97% rename from ably/rest/channel.py rename to ably/pubsub/rest/channel.py index 7603a224..109c0ec7 100644 --- a/ably/rest/channel.py +++ b/ably/pubsub/rest/channel.py @@ -8,20 +8,20 @@ import msgpack -from ably.http.paginatedresult import PaginatedResult, format_params -from ably.rest.annotations import RestAnnotations -from ably.types.channeldetails import ChannelDetails -from ably.types.message import ( +from ably.pubsub.http.paginatedresult import PaginatedResult, format_params +from ably.pubsub.rest.annotations import RestAnnotations +from ably.pubsub.types.channeldetails import ChannelDetails +from ably.pubsub.types.message import ( Message, MessageAction, MessageVersion, make_message_response_handler, make_single_message_response_handler, ) -from ably.types.operations import MessageOperation, PublishResult, UpdateDeleteResult -from ably.types.presence import Presence -from ably.util.crypto import get_cipher -from ably.util.exceptions import ( +from ably.pubsub.types.operations import MessageOperation, PublishResult, UpdateDeleteResult +from ably.pubsub.types.presence import Presence +from ably.pubsub.util.crypto import get_cipher +from ably.pubsub.util.exceptions import ( AblyException, IncompatibleClientIdException, catch_all, diff --git a/ably/rest/push.py b/ably/pubsub/rest/push.py similarity index 96% rename from ably/rest/push.py rename to ably/pubsub/rest/push.py index f99b2b1d..264fea64 100644 --- a/ably/rest/push.py +++ b/ably/pubsub/rest/push.py @@ -1,12 +1,12 @@ from typing import Optional -from ably.http.paginatedresult import PaginatedResult, format_params -from ably.types.channelsubscription import ( +from ably.pubsub.http.paginatedresult import PaginatedResult, format_params +from ably.pubsub.types.channelsubscription import ( PushChannelSubscription, channel_subscriptions_response_processor, channels_response_processor, ) -from ably.types.device import DeviceDetails, device_details_response_processor +from ably.pubsub.types.device import DeviceDetails, device_details_response_processor class Push: diff --git a/ably/rest/rest.py b/ably/pubsub/rest/rest.py similarity index 91% rename from ably/rest/rest.py rename to ably/pubsub/rest/rest.py index bc84e638..c89ce3ed 100644 --- a/ably/rest/rest.py +++ b/ably/pubsub/rest/rest.py @@ -2,15 +2,15 @@ from typing import Optional from urllib.parse import urlencode -from ably.http.http import Http -from ably.http.paginatedresult import HttpPaginatedResponse, PaginatedResult, format_params -from ably.rest.auth import Auth -from ably.rest.channel import Channels -from ably.rest.push import Push -from ably.types.options import Options -from ably.types.stats import stats_response_processor -from ably.types.tokendetails import TokenDetails -from ably.util.exceptions import AblyException, catch_all +from ably.pubsub.http.http import Http +from ably.pubsub.http.paginatedresult import HttpPaginatedResponse, PaginatedResult, format_params +from ably.pubsub.rest.auth import Auth +from ably.pubsub.rest.channel import Channels +from ably.pubsub.rest.push import Push +from ably.pubsub.types.options import Options +from ably.pubsub.types.stats import stats_response_processor +from ably.pubsub.types.tokendetails import TokenDetails +from ably.pubsub.util.exceptions import AblyException, catch_all log = logging.getLogger(__name__) diff --git a/ably/scripts/__init__.py b/ably/pubsub/scripts/__init__.py similarity index 100% rename from ably/scripts/__init__.py rename to ably/pubsub/scripts/__init__.py diff --git a/ably/scripts/unasync.py b/ably/pubsub/scripts/unasync.py similarity index 87% rename from ably/scripts/unasync.py rename to ably/pubsub/scripts/unasync.py index d13e20f2..b253567f 100644 --- a/ably/scripts/unasync.py +++ b/ably/pubsub/scripts/unasync.py @@ -241,7 +241,11 @@ def run(): _TOKEN_REPLACE["aclose"] = "close" _TOKEN_REPLACE["assert_waiter"] = "assert_waiter_sync" - _IMPORTS_REPLACE["ably"] = "ably.sync" + # Ordering matters: _replace_import takes the first key that is a substring + # of the imported module, and the sync flavour of the public surface lives + # at ably.pubsub.server.sync rather than under ably.pubsub.sync. + _IMPORTS_REPLACE["ably.pubsub.server"] = "ably.pubsub.server.sync" + _IMPORTS_REPLACE["ably.pubsub"] = "ably.pubsub.sync" # here... for class_name in rename_classes: @@ -249,11 +253,15 @@ def run(): _STRING_REPLACE["Auth"] = "AuthSync" - src_dir_path = os.path.join(os.getcwd(), "ably") - dest_dir_path = os.path.join(os.getcwd(), "ably", "sync") + src_dir_path = os.path.join(os.getcwd(), "ably", "pubsub") + dest_dir_path = os.path.join(os.getcwd(), "ably", "pubsub", "sync") + # ably/pubsub/server is the hand-written public re-export surface; its + # synchronous flavour lives in ably/pubsub/server/sync.py, not under sync/. + server_dir_path = os.path.join(os.getcwd(), "ably", "pubsub", "server") relevant_src_files = (set(find_files(src_dir_path, "*.py")) - - set(find_files(dest_dir_path, "*.py"))) + set(find_files(dest_dir_path, "*.py")) - + set(find_files(server_dir_path, "*.py"))) unasync_files(list(relevant_src_files), [Rule(fromdir=src_dir_path, todir=dest_dir_path)]) @@ -269,14 +277,15 @@ def run(): _IMPORTS_REPLACE["test.ably"] = "test.ably.sync" _STRING_REPLACE['/../assets/testAppSpec.json'] = '/../../assets/testAppSpec.json' - _STRING_REPLACE['ably.rest.auth.Auth.request_token'] = 'ably.sync.rest.auth.AuthSync.request_token' - _STRING_REPLACE['ably.rest.auth.TokenRequest'] = 'ably.sync.rest.auth.TokenRequest' - _STRING_REPLACE['ably.rest.rest.Http.post'] = 'ably.sync.rest.rest.HttpSync.post' + _STRING_REPLACE['ably.pubsub.rest.auth.Auth.request_token'] = \ + 'ably.pubsub.sync.rest.auth.AuthSync.request_token' + _STRING_REPLACE['ably.pubsub.rest.auth.TokenRequest'] = 'ably.pubsub.sync.rest.auth.TokenRequest' + _STRING_REPLACE['ably.pubsub.rest.rest.Http.post'] = 'ably.pubsub.sync.rest.rest.HttpSync.post' _STRING_REPLACE['httpx.AsyncClient.send'] = 'httpx.Client.send' - _STRING_REPLACE['ably.util.exceptions.AblyException.raise_for_response'] = \ - 'ably.sync.util.exceptions.AblyException.raise_for_response' - _STRING_REPLACE['ably.rest.rest.AblyRest.time'] = 'ably.sync.rest.rest.AblyRestSync.time' - _STRING_REPLACE['ably.rest.auth.Auth._timestamp'] = 'ably.sync.rest.auth.AuthSync._timestamp' + _STRING_REPLACE['ably.pubsub.util.exceptions.AblyException.raise_for_response'] = \ + 'ably.pubsub.sync.util.exceptions.AblyException.raise_for_response' + _STRING_REPLACE['ably.pubsub.rest.rest.AblyRest.time'] = 'ably.pubsub.sync.rest.rest.AblyRestSync.time' + _STRING_REPLACE['ably.pubsub.rest.auth.Auth._timestamp'] = 'ably.pubsub.sync.rest.auth.AuthSync._timestamp' # round 1 src_dir_path = os.path.join(os.getcwd(), "test", "ably") diff --git a/ably/pubsub/server/__init__.py b/ably/pubsub/server/__init__.py new file mode 100644 index 00000000..f7907532 --- /dev/null +++ b/ably/pubsub/server/__init__.py @@ -0,0 +1,59 @@ +"""The Ably Pub/Sub client for servers. + +Servers are trusted environments which typically authenticate with an API key, +and whose connections are exempt from monthly-active-user counting. This +package names that side, so the client an application reaches for is the one +whose package matches where it runs. + +This is the public API of the ``ably-pubsub-server`` distribution. Both +``ably`` and ``ably.pubsub`` are PEP 420 namespace packages shared with the +other ``ably-*`` distributions, so neither carries an ``__init__.py`` of its +own and neither exports anything; everything the library offers is reachable +from here. +""" + +from ably.pubsub.realtime.realtime import AblyRealtime +from ably.pubsub.rest.auth import Auth +from ably.pubsub.rest.push import Push +from ably.pubsub.rest.rest import AblyRest +from ably.pubsub.types.annotation import Annotation, AnnotationAction +from ably.pubsub.types.capability import Capability +from ably.pubsub.types.channelmode import ChannelMode +from ably.pubsub.types.channeloptions import ChannelOptions +from ably.pubsub.types.channelsubscription import PushChannelSubscription +from ably.pubsub.types.device import DeviceDetails +from ably.pubsub.types.message import MessageAction, MessageVersion +from ably.pubsub.types.operations import MessageOperation, PublishResult, UpdateDeleteResult +from ably.pubsub.types.options import Options, VCDiffDecoder +from ably.pubsub.util.crypto import CipherParams +from ably.pubsub.util.exceptions import AblyAuthException, AblyException, IncompatibleClientIdException +from ably.pubsub.vcdiff.defaultvcdiffdecoder import AblyVCDiffDecoder +from ably.pubsub.version import api_version, lib_version + +__all__ = [ + 'AblyAuthException', + 'AblyException', + 'AblyRealtime', + 'AblyRest', + 'AblyVCDiffDecoder', + 'Annotation', + 'AnnotationAction', + 'Auth', + 'Capability', + 'ChannelMode', + 'ChannelOptions', + 'CipherParams', + 'DeviceDetails', + 'IncompatibleClientIdException', + 'MessageAction', + 'MessageOperation', + 'MessageVersion', + 'Options', + 'PublishResult', + 'Push', + 'PushChannelSubscription', + 'UpdateDeleteResult', + 'VCDiffDecoder', + 'api_version', + 'lib_version', +] diff --git a/ably/pubsub/server/sync.py b/ably/pubsub/server/sync.py new file mode 100644 index 00000000..1a18fa67 --- /dev/null +++ b/ably/pubsub/server/sync.py @@ -0,0 +1,54 @@ +"""The synchronous flavour of the Ably Pub/Sub client for servers. + +Exposes the same surface as :mod:`ably.pubsub.server`, generated without +``async``/``await`` so it can be driven from a thread rather than an event +loop. ``AblyRestSync`` is the client to reach for; a server that needs to +subscribe to channels should use the asynchronous +:class:`ably.pubsub.server.AblyRealtime` instead. +""" + +from ably.pubsub.sync.realtime.realtime import AblyRealtime +from ably.pubsub.sync.rest.auth import AuthSync +from ably.pubsub.sync.rest.push import PushSync +from ably.pubsub.sync.rest.rest import AblyRestSync +from ably.pubsub.sync.types.annotation import Annotation, AnnotationAction +from ably.pubsub.sync.types.capability import Capability +from ably.pubsub.sync.types.channelmode import ChannelMode +from ably.pubsub.sync.types.channeloptions import ChannelOptions +from ably.pubsub.sync.types.channelsubscription import PushChannelSubscription +from ably.pubsub.sync.types.device import DeviceDetails +from ably.pubsub.sync.types.message import MessageAction, MessageVersion +from ably.pubsub.sync.types.operations import MessageOperation, PublishResult, UpdateDeleteResult +from ably.pubsub.sync.types.options import Options, VCDiffDecoder +from ably.pubsub.sync.util.crypto import CipherParams +from ably.pubsub.sync.util.exceptions import AblyAuthException, AblyException, IncompatibleClientIdException +from ably.pubsub.sync.vcdiff.defaultvcdiffdecoder import AblyVCDiffDecoder +from ably.pubsub.version import api_version, lib_version + +__all__ = [ + 'AblyAuthException', + 'AblyException', + 'AblyRealtime', + 'AblyRestSync', + 'AblyVCDiffDecoder', + 'Annotation', + 'AnnotationAction', + 'AuthSync', + 'Capability', + 'ChannelMode', + 'ChannelOptions', + 'CipherParams', + 'DeviceDetails', + 'IncompatibleClientIdException', + 'MessageAction', + 'MessageOperation', + 'MessageVersion', + 'Options', + 'PublishResult', + 'PushChannelSubscription', + 'PushSync', + 'UpdateDeleteResult', + 'VCDiffDecoder', + 'api_version', + 'lib_version', +] diff --git a/ably/transport/__init__.py b/ably/pubsub/transport/__init__.py similarity index 100% rename from ably/transport/__init__.py rename to ably/pubsub/transport/__init__.py diff --git a/ably/transport/defaults.py b/ably/pubsub/transport/defaults.py similarity index 100% rename from ably/transport/defaults.py rename to ably/pubsub/transport/defaults.py diff --git a/ably/transport/websockettransport.py b/ably/pubsub/transport/websockettransport.py similarity index 96% rename from ably/transport/websockettransport.py rename to ably/pubsub/transport/websockettransport.py index dbde5413..2cbe92a7 100644 --- a/ably/transport/websockettransport.py +++ b/ably/pubsub/transport/websockettransport.py @@ -10,13 +10,13 @@ import msgpack -from ably.http.httputils import HttpUtils -from ably.transport.defaults import Defaults -from ably.types.connectiondetails import ConnectionDetails -from ably.types.operations import PublishResult -from ably.util.clock import select_clock -from ably.util.eventemitter import EventEmitter -from ably.util.exceptions import AblyException +from ably.pubsub.http.httputils import HttpUtils +from ably.pubsub.transport.defaults import Defaults +from ably.pubsub.types.connectiondetails import ConnectionDetails +from ably.pubsub.types.operations import PublishResult +from ably.pubsub.util.clock import select_clock +from ably.pubsub.util.eventemitter import EventEmitter +from ably.pubsub.util.exceptions import AblyException try: # websockets 15+ preferred imports @@ -30,7 +30,7 @@ from websockets.exceptions import ConnectionClosedOK, WebSocketException if TYPE_CHECKING: - from ably.realtime.connection import ConnectionManager + from ably.pubsub.realtime.connection import ConnectionManager log = logging.getLogger(__name__) diff --git a/ably/types/__init__.py b/ably/pubsub/types/__init__.py similarity index 100% rename from ably/types/__init__.py rename to ably/pubsub/types/__init__.py diff --git a/ably/types/annotation.py b/ably/pubsub/types/annotation.py similarity index 98% rename from ably/types/annotation.py rename to ably/pubsub/types/annotation.py index 66ce5271..98e845a3 100644 --- a/ably/types/annotation.py +++ b/ably/pubsub/types/annotation.py @@ -1,11 +1,11 @@ import logging from enum import IntEnum -from ably.types.mixins import EncodeDataMixin -from ably.types.typedbuffer import TypedBuffer -from ably.util.crypto import CipherData -from ably.util.encoding import encode_data -from ably.util.helper import to_text +from ably.pubsub.types.mixins import EncodeDataMixin +from ably.pubsub.types.typedbuffer import TypedBuffer +from ably.pubsub.util.crypto import CipherData +from ably.pubsub.util.encoding import encode_data +from ably.pubsub.util.helper import to_text log = logging.getLogger(__name__) diff --git a/ably/types/authoptions.py b/ably/pubsub/types/authoptions.py similarity index 98% rename from ably/types/authoptions.py rename to ably/pubsub/types/authoptions.py index 7ee06af7..18f0fb30 100644 --- a/ably/types/authoptions.py +++ b/ably/pubsub/types/authoptions.py @@ -1,4 +1,4 @@ -from ably.util.exceptions import AblyException +from ably.pubsub.util.exceptions import AblyException class AuthOptions: diff --git a/ably/types/capability.py b/ably/pubsub/types/capability.py similarity index 100% rename from ably/types/capability.py rename to ably/pubsub/types/capability.py diff --git a/ably/types/channeldetails.py b/ably/pubsub/types/channeldetails.py similarity index 100% rename from ably/types/channeldetails.py rename to ably/pubsub/types/channeldetails.py diff --git a/ably/types/channelmode.py b/ably/pubsub/types/channelmode.py similarity index 96% rename from ably/types/channelmode.py rename to ably/pubsub/types/channelmode.py index 23ed735c..b16b355d 100644 --- a/ably/types/channelmode.py +++ b/ably/pubsub/types/channelmode.py @@ -2,7 +2,7 @@ from enum import Enum -from ably.types.flags import Flag +from ably.pubsub.types.flags import Flag class ChannelMode(int, Enum): diff --git a/ably/types/channeloptions.py b/ably/pubsub/types/channeloptions.py similarity index 94% rename from ably/types/channeloptions.py rename to ably/pubsub/types/channeloptions.py index 3e5052c6..9f7a99e1 100644 --- a/ably/types/channeloptions.py +++ b/ably/pubsub/types/channeloptions.py @@ -2,9 +2,9 @@ from typing import Any -from ably.types.channelmode import ChannelMode -from ably.util.crypto import CipherParams -from ably.util.exceptions import AblyException +from ably.pubsub.types.channelmode import ChannelMode +from ably.pubsub.util.crypto import CipherParams +from ably.pubsub.util.exceptions import AblyException class ChannelOptions: diff --git a/ably/types/channelstate.py b/ably/pubsub/types/channelstate.py similarity index 89% rename from ably/types/channelstate.py rename to ably/pubsub/types/channelstate.py index dcb68d67..fee06ce2 100644 --- a/ably/types/channelstate.py +++ b/ably/pubsub/types/channelstate.py @@ -2,7 +2,7 @@ from enum import Enum from typing import Optional -from ably.util.exceptions import AblyException +from ably.pubsub.util.exceptions import AblyException class ChannelState(str, Enum): diff --git a/ably/types/channelsubscription.py b/ably/pubsub/types/channelsubscription.py similarity index 98% rename from ably/types/channelsubscription.py rename to ably/pubsub/types/channelsubscription.py index b4c0dbf8..9cc386c9 100644 --- a/ably/types/channelsubscription.py +++ b/ably/pubsub/types/channelsubscription.py @@ -1,4 +1,4 @@ -from ably.util import case +from ably.pubsub.util import case class PushChannelSubscription: diff --git a/ably/types/connectiondetails.py b/ably/pubsub/types/connectiondetails.py similarity index 100% rename from ably/types/connectiondetails.py rename to ably/pubsub/types/connectiondetails.py diff --git a/ably/types/connectionerrors.py b/ably/pubsub/types/connectionerrors.py similarity index 84% rename from ably/types/connectionerrors.py rename to ably/pubsub/types/connectionerrors.py index bb2fa1f4..4b2157d8 100644 --- a/ably/types/connectionerrors.py +++ b/ably/pubsub/types/connectionerrors.py @@ -1,5 +1,5 @@ -from ably.types.connectionstate import ConnectionState -from ably.util.exceptions import AblyException +from ably.pubsub.types.connectionstate import ConnectionState +from ably.pubsub.util.exceptions import AblyException ConnectionErrors = { ConnectionState.DISCONNECTED: AblyException( diff --git a/ably/types/connectionstate.py b/ably/pubsub/types/connectionstate.py similarity index 93% rename from ably/types/connectionstate.py rename to ably/pubsub/types/connectionstate.py index ec958358..fde91caa 100644 --- a/ably/types/connectionstate.py +++ b/ably/pubsub/types/connectionstate.py @@ -2,7 +2,7 @@ from enum import Enum from typing import Optional -from ably.util.exceptions import AblyException +from ably.pubsub.util.exceptions import AblyException class ConnectionState(str, Enum): diff --git a/ably/types/device.py b/ably/pubsub/types/device.py similarity index 98% rename from ably/types/device.py rename to ably/pubsub/types/device.py index c2b84ee5..3948b751 100644 --- a/ably/types/device.py +++ b/ably/pubsub/types/device.py @@ -1,4 +1,4 @@ -from ably.util import case +from ably.pubsub.util import case DevicePushTransportType = {'fcm', 'gcm', 'apns', 'web'} DevicePlatform = {'android', 'ios', 'browser'} diff --git a/ably/types/flags.py b/ably/pubsub/types/flags.py similarity index 100% rename from ably/types/flags.py rename to ably/pubsub/types/flags.py diff --git a/ably/types/message.py b/ably/pubsub/types/message.py similarity index 97% rename from ably/types/message.py rename to ably/pubsub/types/message.py index 2d84350b..93a20f0c 100644 --- a/ably/types/message.py +++ b/ably/pubsub/types/message.py @@ -1,12 +1,12 @@ import logging from enum import IntEnum -from ably.types.mixins import DeltaExtras, EncodeDataMixin -from ably.types.typedbuffer import TypedBuffer -from ably.util.crypto import CipherData -from ably.util.encoding import encode_data -from ably.util.exceptions import AblyException -from ably.util.helper import to_text +from ably.pubsub.types.mixins import DeltaExtras, EncodeDataMixin +from ably.pubsub.types.typedbuffer import TypedBuffer +from ably.pubsub.util.crypto import CipherData +from ably.pubsub.util.encoding import encode_data +from ably.pubsub.util.exceptions import AblyException +from ably.pubsub.util.helper import to_text log = logging.getLogger(__name__) diff --git a/ably/types/mixins.py b/ably/pubsub/types/mixins.py similarity index 98% rename from ably/types/mixins.py rename to ably/pubsub/types/mixins.py index c5fad258..976da3b1 100644 --- a/ably/types/mixins.py +++ b/ably/pubsub/types/mixins.py @@ -3,8 +3,8 @@ import json import logging -from ably.util.crypto import CipherData -from ably.util.exceptions import AblyException +from ably.pubsub.util.crypto import CipherData +from ably.pubsub.util.exceptions import AblyException log = logging.getLogger(__name__) diff --git a/ably/types/operations.py b/ably/pubsub/types/operations.py similarity index 100% rename from ably/types/operations.py rename to ably/pubsub/types/operations.py diff --git a/ably/types/options.py b/ably/pubsub/types/options.py similarity index 98% rename from ably/types/options.py rename to ably/pubsub/types/options.py index 5030a7cd..51f6016e 100644 --- a/ably/types/options.py +++ b/ably/pubsub/types/options.py @@ -2,9 +2,9 @@ import random from abc import ABC, abstractmethod -from ably.transport.defaults import Defaults -from ably.types.authoptions import AuthOptions -from ably.util.exceptions import AblyException +from ably.pubsub.transport.defaults import Defaults +from ably.pubsub.types.authoptions import AuthOptions +from ably.pubsub.util.exceptions import AblyException log = logging.getLogger(__name__) @@ -81,7 +81,7 @@ def __init__(self, client_id=None, log_level=0, tls=True, rest_host=None, realti ) if idempotent_rest_publishing is None: - from ably import api_version + from ably.pubsub.version import api_version idempotent_rest_publishing = api_version >= '1.2' if environment is not None and endpoint is None: diff --git a/ably/types/presence.py b/ably/pubsub/types/presence.py similarity index 96% rename from ably/types/presence.py rename to ably/pubsub/types/presence.py index 669f1ede..12e965e1 100644 --- a/ably/types/presence.py +++ b/ably/pubsub/types/presence.py @@ -1,11 +1,11 @@ from datetime import datetime, timedelta from urllib import parse -from ably.http.paginatedresult import PaginatedResult -from ably.types.mixins import EncodeDataMixin -from ably.types.typedbuffer import TypedBuffer -from ably.util.crypto import CipherData -from ably.util.encoding import encode_data +from ably.pubsub.http.paginatedresult import PaginatedResult +from ably.pubsub.types.mixins import EncodeDataMixin +from ably.pubsub.types.typedbuffer import TypedBuffer +from ably.pubsub.util.crypto import CipherData +from ably.pubsub.util.encoding import encode_data def _ms_since_epoch(dt): diff --git a/ably/types/stats.py b/ably/pubsub/types/stats.py similarity index 100% rename from ably/types/stats.py rename to ably/pubsub/types/stats.py diff --git a/ably/types/testoptions.py b/ably/pubsub/types/testoptions.py similarity index 93% rename from ably/types/testoptions.py rename to ably/pubsub/types/testoptions.py index bc50f702..4c795411 100644 --- a/ably/types/testoptions.py +++ b/ably/pubsub/types/testoptions.py @@ -13,7 +13,7 @@ class TestOptions: context manager yielding an object supporting `__aiter__`, `send` and `close`. - `clock`: the source of time the client reads, in place of - `ably.util.clock.Clock`. It supplies `now_ms()`, `monotonic_ms()` and + `ably.pubsub.util.clock.Clock`. It supplies `now_ms()`, `monotonic_ms()` and `timer(timeout_ms, callback)`, where `callback` is either a coroutine function or a plain callable and the return value has a `cancel()` method. diff --git a/ably/types/tokendetails.py b/ably/pubsub/types/tokendetails.py similarity index 97% rename from ably/types/tokendetails.py rename to ably/pubsub/types/tokendetails.py index 1980308d..29c7dd24 100644 --- a/ably/types/tokendetails.py +++ b/ably/pubsub/types/tokendetails.py @@ -1,7 +1,7 @@ import json import time -from ably.types.capability import Capability +from ably.pubsub.types.capability import Capability class TokenDetails: diff --git a/ably/types/tokenrequest.py b/ably/pubsub/types/tokenrequest.py similarity index 100% rename from ably/types/tokenrequest.py rename to ably/pubsub/types/tokenrequest.py diff --git a/ably/types/typedbuffer.py b/ably/pubsub/types/typedbuffer.py similarity index 100% rename from ably/types/typedbuffer.py rename to ably/pubsub/types/typedbuffer.py diff --git a/ably/pubsub/util/__init__.py b/ably/pubsub/util/__init__.py new file mode 100644 index 00000000..ac235ae9 --- /dev/null +++ b/ably/pubsub/util/__init__.py @@ -0,0 +1,10 @@ +import logging + +# Silence "no handler" output for every logger in the library. This would +# normally go in the top-level package's __init__.py, but `ably.pubsub` is a +# PEP 420 namespace package and so has none; `ably.pubsub.util` is the one +# module every entry point, async and sync alike, imports on its way in. The +# unasync'd copy of this module runs the same line, hence the guard. +_root_logger = logging.getLogger('ably.pubsub') +if not any(isinstance(handler, logging.NullHandler) for handler in _root_logger.handlers): + _root_logger.addHandler(logging.NullHandler()) diff --git a/ably/util/case.py b/ably/pubsub/util/case.py similarity index 100% rename from ably/util/case.py rename to ably/pubsub/util/case.py diff --git a/ably/util/clock.py b/ably/pubsub/util/clock.py similarity index 97% rename from ably/util/clock.py rename to ably/pubsub/util/clock.py index 00aef3fd..0d4dd0ed 100644 --- a/ably/util/clock.py +++ b/ably/pubsub/util/clock.py @@ -1,7 +1,7 @@ import time from typing import Callable -from ably.util.helper import Timer +from ably.pubsub.util.helper import Timer class Clock: diff --git a/ably/util/crypto.py b/ably/pubsub/util/crypto.py similarity index 98% rename from ably/util/crypto.py rename to ably/pubsub/util/crypto.py index 8d8ddfd9..88a62f92 100644 --- a/ably/util/crypto.py +++ b/ably/pubsub/util/crypto.py @@ -7,8 +7,8 @@ except ImportError: from .nocrypto import AES, Random -from ably.types.typedbuffer import TypedBuffer -from ably.util.exceptions import AblyException +from ably.pubsub.types.typedbuffer import TypedBuffer +from ably.pubsub.util.exceptions import AblyException log = logging.getLogger(__name__) diff --git a/ably/util/encoding.py b/ably/pubsub/util/encoding.py similarity index 91% rename from ably/util/encoding.py rename to ably/pubsub/util/encoding.py index 5187aec2..7f4b89da 100644 --- a/ably/util/encoding.py +++ b/ably/pubsub/util/encoding.py @@ -2,8 +2,8 @@ import json from typing import Any -from ably.util.crypto import CipherData -from ably.util.exceptions import AblyException +from ably.pubsub.util.crypto import CipherData +from ably.pubsub.util.exceptions import AblyException def encode_data(data: Any, encoding_array: list, binary: bool = False): diff --git a/ably/util/eventemitter.py b/ably/pubsub/util/eventemitter.py similarity index 99% rename from ably/util/eventemitter.py rename to ably/pubsub/util/eventemitter.py index 74f0beb6..491b0a00 100644 --- a/ably/util/eventemitter.py +++ b/ably/pubsub/util/eventemitter.py @@ -3,7 +3,7 @@ from pyee.asyncio import AsyncIOEventEmitter -from ably.util.helper import is_callable_or_coroutine +from ably.pubsub.util.helper import is_callable_or_coroutine # pyee's event emitter doesn't support attaching a listener to all events # so to patch it, we create a wrapper which uses two event emitters, one diff --git a/ably/util/exceptions.py b/ably/pubsub/util/exceptions.py similarity index 100% rename from ably/util/exceptions.py rename to ably/pubsub/util/exceptions.py diff --git a/ably/util/helper.py b/ably/pubsub/util/helper.py similarity index 98% rename from ably/util/helper.py rename to ably/pubsub/util/helper.py index 4d37d4f4..934a1af1 100644 --- a/ably/util/helper.py +++ b/ably/pubsub/util/helper.py @@ -9,7 +9,7 @@ import msgpack -from ably.util.exceptions import AblyException +from ably.pubsub.util.exceptions import AblyException def get_random_id(): diff --git a/ably/util/nocrypto.py b/ably/pubsub/util/nocrypto.py similarity index 100% rename from ably/util/nocrypto.py rename to ably/pubsub/util/nocrypto.py diff --git a/ably/util/__init__.py b/ably/pubsub/vcdiff/__init__.py similarity index 100% rename from ably/util/__init__.py rename to ably/pubsub/vcdiff/__init__.py diff --git a/ably/vcdiff/defaultvcdiffdecoder.py b/ably/pubsub/vcdiff/defaultvcdiffdecoder.py similarity index 92% rename from ably/vcdiff/defaultvcdiffdecoder.py rename to ably/pubsub/vcdiff/defaultvcdiffdecoder.py index ae2d3263..ee86abd1 100644 --- a/ably/vcdiff/defaultvcdiffdecoder.py +++ b/ably/pubsub/vcdiff/defaultvcdiffdecoder.py @@ -5,7 +5,7 @@ It implements the VCDiffDecoder interface. Usage: - from ably.vcdiff import AblyVCDiffDecoder, AblyRealtime + from ably.pubsub.vcdiff import AblyVCDiffDecoder, AblyRealtime # Create VCDiff decoder vcdiff_decoder = AblyVCDiffDecoder() @@ -19,8 +19,8 @@ import logging -from ably.types.options import VCDiffDecoder -from ably.util.exceptions import AblyException +from ably.pubsub.types.options import VCDiffDecoder +from ably.pubsub.util.exceptions import AblyException log = logging.getLogger(__name__) diff --git a/ably/pubsub/version.py b/ably/pubsub/version.py new file mode 100644 index 00000000..8389a068 --- /dev/null +++ b/ably/pubsub/version.py @@ -0,0 +1,9 @@ +"""Version constants for the Ably Pub/Sub library. + +These live in a module of their own rather than in a package ``__init__.py`` +because ``ably.pubsub`` is a PEP 420 namespace package, shared with the other +``ably-pubsub-*`` distributions, and so cannot carry one. +""" + +api_version = '5' +lib_version = '3.1.4' diff --git a/ably/vcdiff/__init__.py b/ably/vcdiff/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/pyproject.toml b/pyproject.toml index 342f92c2..42eaee5d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,7 +1,7 @@ [project] -name = "ably" +name = "ably-pubsub-server" version = "3.1.4" -description = "Python REST and Realtime client library SDK for Ably realtime messaging service" +description = "Python REST and Realtime server client library SDK for Ably Pub/Sub" readme = "LONG_DESCRIPTION.rst" requires-python = ">=3.7" license = { text = "Apache-2.0" } @@ -59,7 +59,7 @@ dev = [ ] [project.scripts] -unasync = "ably.scripts.unasync:run" +unasync = "ably.pubsub.scripts.unasync:run" [project.urls] Homepage = "https://ably.com" @@ -88,7 +88,9 @@ exclude = [ [tool.hatch.build.targets.wheel] ignore-vcs = true -packages = ["ably"] +# `ably` is a PEP 420 namespace package shared with the other ably-* distributions, +# so it must not be shipped with an __init__.py of its own. +only-include = ["ably/pubsub"] [tool.pytest.ini_options] timeout = 30 @@ -102,7 +104,7 @@ explicit = true [tool.ruff] line-length = 115 extend-exclude = [ - "ably/sync", + "ably/pubsub/sync", "test/ably/sync", ] diff --git a/test/ably/realtime/deltadecoding_test.py b/test/ably/realtime/deltadecoding_test.py index f2930dfb..1c3d250e 100644 --- a/test/ably/realtime/deltadecoding_test.py +++ b/test/ably/realtime/deltadecoding_test.py @@ -9,12 +9,12 @@ import pytest -from ably import AblyRealtime -from ably.types.channelstate import ChannelState -from ably.types.message import Message -from ably.types.mixins import DecodingContext -from ably.types.options import VCDiffDecoder -from ably.util.exceptions import AblyException +from ably.pubsub.server import AblyRealtime +from ably.pubsub.types.channelstate import ChannelState +from ably.pubsub.types.message import Message +from ably.pubsub.types.mixins import DecodingContext +from ably.pubsub.types.options import VCDiffDecoder +from ably.pubsub.util.exceptions import AblyException from test.ably.utils import BaseAsyncTestCase diff --git a/test/ably/realtime/eventemitter_test.py b/test/ably/realtime/eventemitter_test.py index 71db2c74..f9913248 100644 --- a/test/ably/realtime/eventemitter_test.py +++ b/test/ably/realtime/eventemitter_test.py @@ -2,7 +2,7 @@ import pytest -from ably.realtime.connection import ConnectionState +from ably.pubsub.realtime.connection import ConnectionState from test.ably.testapp import TestApp from test.ably.utils import BaseAsyncTestCase diff --git a/test/ably/realtime/presencemap_test.py b/test/ably/realtime/presencemap_test.py index 043baeb0..de6002bf 100644 --- a/test/ably/realtime/presencemap_test.py +++ b/test/ably/realtime/presencemap_test.py @@ -8,8 +8,8 @@ import pytest -from ably.realtime.presencemap import PresenceMap, _is_newer -from ably.types.presence import PresenceAction, PresenceMessage +from ably.pubsub.realtime.presencemap import PresenceMap, _is_newer +from ably.pubsub.types.presence import PresenceAction, PresenceMessage from test.ably.utils import BaseAsyncTestCase diff --git a/test/ably/realtime/realtimeannotations_test.py b/test/ably/realtime/realtimeannotations_test.py index 2c5e0658..79521766 100644 --- a/test/ably/realtime/realtimeannotations_test.py +++ b/test/ably/realtime/realtimeannotations_test.py @@ -5,10 +5,10 @@ import pytest -from ably.types.annotation import Annotation, AnnotationAction -from ably.types.channelmode import ChannelMode -from ably.types.channeloptions import ChannelOptions -from ably.types.message import MessageAction +from ably.pubsub.types.annotation import Annotation, AnnotationAction +from ably.pubsub.types.channelmode import ChannelMode +from ably.pubsub.types.channeloptions import ChannelOptions +from ably.pubsub.types.message import MessageAction from test.ably.testapp import TestApp from test.ably.utils import BaseAsyncTestCase, ReusableFuture, assert_waiter @@ -334,7 +334,7 @@ async def on_annotation(annotation): pass # RTAN4e: Should log a warning (not raise), and still register the listener - with caplog.at_level(logging.WARNING, logger='ably.realtime.annotations'): + with caplog.at_level(logging.WARNING, logger='ably.pubsub.realtime.annotations'): await channel.annotations.subscribe(on_annotation) # Verify warning was logged mentioning the missing mode diff --git a/test/ably/realtime/realtimeauth_test.py b/test/ably/realtime/realtimeauth_test.py index a2620489..da073117 100644 --- a/test/ably/realtime/realtimeauth_test.py +++ b/test/ably/realtime/realtimeauth_test.py @@ -5,11 +5,11 @@ import httpx import pytest -from ably.realtime.connection import ConnectionState -from ably.transport.websockettransport import ProtocolMessageAction -from ably.types.channelstate import ChannelState -from ably.types.connectionstate import ConnectionEvent -from ably.types.tokendetails import TokenDetails +from ably.pubsub.realtime.connection import ConnectionState +from ably.pubsub.transport.websockettransport import ProtocolMessageAction +from ably.pubsub.types.channelstate import ChannelState +from ably.pubsub.types.connectionstate import ConnectionEvent +from ably.pubsub.types.tokendetails import TokenDetails from test.ably.testapp import TestApp from test.ably.utils import BaseAsyncTestCase, random_string diff --git a/test/ably/realtime/realtimechannel_publish_test.py b/test/ably/realtime/realtimechannel_publish_test.py index 9ecf10f9..a5af09f5 100644 --- a/test/ably/realtime/realtimechannel_publish_test.py +++ b/test/ably/realtime/realtimechannel_publish_test.py @@ -2,13 +2,13 @@ import pytest -from ably.realtime.channel import ChannelState -from ably.realtime.connection import ConnectionState -from ably.transport.websockettransport import ProtocolMessageAction -from ably.types.channeloptions import ChannelOptions -from ably.types.message import Message -from ably.util.crypto import CipherParams -from ably.util.exceptions import AblyException, IncompatibleClientIdException +from ably.pubsub.realtime.channel import ChannelState +from ably.pubsub.realtime.connection import ConnectionState +from ably.pubsub.transport.websockettransport import ProtocolMessageAction +from ably.pubsub.types.channeloptions import ChannelOptions +from ably.pubsub.types.message import Message +from ably.pubsub.util.crypto import CipherParams +from ably.pubsub.util.exceptions import AblyException, IncompatibleClientIdException from test.ably.testapp import TestApp from test.ably.utils import BaseAsyncTestCase, WaitableEvent, assert_waiter @@ -328,7 +328,7 @@ async def test_msgserial_resets_on_new_connection_id(self): new_connection_id = 'new_connection_id_12345' # Simulate server sending CONNECTED with new connectionId - from ably.types.connectiondetails import ConnectionDetails + from ably.pubsub.types.connectiondetails import ConnectionDetails new_connection_details = ConnectionDetails( connection_state_ttl=120000, max_idle_interval=15000, @@ -365,7 +365,7 @@ async def test_msgserial_not_reset_on_same_connection_id(self): # Simulate reconnection with SAME connectionId (transport change, not new connection) same_connection_id = connection_manager.connection_id - from ably.types.connectiondetails import ConnectionDetails + from ably.pubsub.types.connectiondetails import ConnectionDetails connection_details = ConnectionDetails( connection_state_ttl=120000, max_idle_interval=15000, @@ -491,7 +491,7 @@ async def check_pending(): assert connection_manager.msg_serial == 1 # Simulate resume with SAME connectionId - from ably.types.connectiondetails import ConnectionDetails + from ably.pubsub.types.connectiondetails import ConnectionDetails connection_details = ConnectionDetails( connection_state_ttl=120000, max_idle_interval=15000, @@ -541,7 +541,7 @@ async def check_pending(): assert connection_manager.msg_serial == 1 # Simulate NEW connection (different connectionId = failed resume) - from ably.types.connectiondetails import ConnectionDetails + from ably.pubsub.types.connectiondetails import ConnectionDetails new_connection_details = ConnectionDetails( connection_state_ttl=120000, max_idle_interval=15000, diff --git a/test/ably/realtime/realtimechannel_test.py b/test/ably/realtime/realtimechannel_test.py index 6d2865f2..477f412a 100644 --- a/test/ably/realtime/realtimechannel_test.py +++ b/test/ably/realtime/realtimechannel_test.py @@ -2,12 +2,12 @@ import pytest -from ably.realtime.channel import ChannelState, RealtimeChannel -from ably.realtime.connection import ConnectionState -from ably.transport.websockettransport import ProtocolMessageAction -from ably.types.channeloptions import ChannelOptions -from ably.types.message import Message -from ably.util.exceptions import AblyException +from ably.pubsub.realtime.channel import ChannelState, RealtimeChannel +from ably.pubsub.realtime.connection import ConnectionState +from ably.pubsub.transport.websockettransport import ProtocolMessageAction +from ably.pubsub.types.channeloptions import ChannelOptions +from ably.pubsub.types.message import Message +from ably.pubsub.util.exceptions import AblyException from test.ably.testapp import TestApp from test.ably.utils import BaseAsyncTestCase, random_string diff --git a/test/ably/realtime/realtimechannel_vcdiff_test.py b/test/ably/realtime/realtimechannel_vcdiff_test.py index 48a484a9..ebc6c7c8 100644 --- a/test/ably/realtime/realtimechannel_vcdiff_test.py +++ b/test/ably/realtime/realtimechannel_vcdiff_test.py @@ -3,10 +3,10 @@ import pytest -from ably import AblyVCDiffDecoder -from ably.realtime.connection import ConnectionState -from ably.types.channeloptions import ChannelOptions -from ably.types.options import VCDiffDecoder +from ably.pubsub.realtime.connection import ConnectionState +from ably.pubsub.server import AblyVCDiffDecoder +from ably.pubsub.types.channeloptions import ChannelOptions +from ably.pubsub.types.options import VCDiffDecoder from test.ably.testapp import TestApp from test.ably.utils import BaseAsyncTestCase, WaitableEvent diff --git a/test/ably/realtime/realtimechannelmutablemessages_test.py b/test/ably/realtime/realtimechannelmutablemessages_test.py index afe8a60f..fa4d8db7 100644 --- a/test/ably/realtime/realtimechannelmutablemessages_test.py +++ b/test/ably/realtime/realtimechannelmutablemessages_test.py @@ -3,9 +3,9 @@ import pytest -from ably import AblyException, CipherParams, MessageAction -from ably.types.message import Message -from ably.types.operations import MessageOperation +from ably.pubsub.server import AblyException, CipherParams, MessageAction +from ably.pubsub.types.message import Message +from ably.pubsub.types.operations import MessageOperation from test.ably.testapp import TestApp from test.ably.utils import BaseAsyncTestCase, WaitableEvent, assert_waiter diff --git a/test/ably/realtime/realtimeconnection_test.py b/test/ably/realtime/realtimeconnection_test.py index d187a378..464cf460 100644 --- a/test/ably/realtime/realtimeconnection_test.py +++ b/test/ably/realtime/realtimeconnection_test.py @@ -11,10 +11,10 @@ # websockets 14 and earlier fallback from websockets.server import serve as ws_serve -from ably.realtime.connection import ConnectionEvent, ConnectionState -from ably.transport.defaults import Defaults -from ably.transport.websockettransport import ProtocolMessageAction -from ably.util.exceptions import AblyException +from ably.pubsub.realtime.connection import ConnectionEvent, ConnectionState +from ably.pubsub.transport.defaults import Defaults +from ably.pubsub.transport.websockettransport import ProtocolMessageAction +from ably.pubsub.util.exceptions import AblyException from test.ably.testapp import TestApp from test.ably.utils import BaseAsyncTestCase diff --git a/test/ably/realtime/realtimeinit_test.py b/test/ably/realtime/realtimeinit_test.py index 4009d046..1bc5b7ba 100644 --- a/test/ably/realtime/realtimeinit_test.py +++ b/test/ably/realtime/realtimeinit_test.py @@ -2,9 +2,9 @@ import pytest -from ably import Auth -from ably.realtime.connection import ConnectionState -from ably.util.exceptions import AblyAuthException +from ably.pubsub.realtime.connection import ConnectionState +from ably.pubsub.server import Auth +from ably.pubsub.util.exceptions import AblyAuthException from test.ably.testapp import TestApp from test.ably.utils import BaseAsyncTestCase diff --git a/test/ably/realtime/realtimepresence_test.py b/test/ably/realtime/realtimepresence_test.py index 06067f14..63da265d 100644 --- a/test/ably/realtime/realtimepresence_test.py +++ b/test/ably/realtime/realtimepresence_test.py @@ -9,10 +9,10 @@ import pytest -from ably.realtime.connection import ConnectionState -from ably.types.channelstate import ChannelState -from ably.types.presence import PresenceAction -from ably.util.exceptions import AblyException +from ably.pubsub.realtime.connection import ConnectionState +from ably.pubsub.types.channelstate import ChannelState +from ably.pubsub.types.presence import PresenceAction +from ably.pubsub.util.exceptions import AblyException from test.ably.testapp import TestApp from test.ably.utils import BaseAsyncTestCase diff --git a/test/ably/realtime/realtimeresume_test.py b/test/ably/realtime/realtimeresume_test.py index bfe77efa..4f3a89e2 100644 --- a/test/ably/realtime/realtimeresume_test.py +++ b/test/ably/realtime/realtimeresume_test.py @@ -2,9 +2,9 @@ import pytest -from ably.realtime.channel import ChannelState -from ably.realtime.connection import ConnectionState -from ably.transport.websockettransport import ProtocolMessageAction +from ably.pubsub.realtime.channel import ChannelState +from ably.pubsub.realtime.connection import ConnectionState +from ably.pubsub.transport.websockettransport import ProtocolMessageAction from test.ably.testapp import TestApp from test.ably.utils import BaseAsyncTestCase, random_string diff --git a/test/ably/rest/encoders_test.py b/test/ably/rest/encoders_test.py index f8023c5d..79e4e3d0 100644 --- a/test/ably/rest/encoders_test.py +++ b/test/ably/rest/encoders_test.py @@ -7,9 +7,9 @@ import msgpack import pytest -from ably import CipherParams -from ably.types.message import Message -from ably.util.crypto import get_cipher +from ably.pubsub.server import CipherParams +from ably.pubsub.types.message import Message +from ably.pubsub.util.crypto import get_cipher from test.ably.testapp import TestApp from test.ably.utils import BaseAsyncTestCase @@ -31,7 +31,7 @@ async def setup(self): async def test_text_utf8(self): channel = self.ably.channels["persisted:publish"] - with mock.patch('ably.rest.rest.Http.post', new_callable=AsyncMock) as post_mock: + with mock.patch('ably.pubsub.rest.rest.Http.post', new_callable=AsyncMock) as post_mock: await channel.publish('event', 'foó') _, kwargs = post_mock.call_args assert json.loads(kwargs['body'])['data'] == 'foó' @@ -41,7 +41,7 @@ async def test_str(self): # This test only makes sense for py2 channel = self.ably.channels["persisted:publish"] - with mock.patch('ably.rest.rest.Http.post', new_callable=AsyncMock) as post_mock: + with mock.patch('ably.pubsub.rest.rest.Http.post', new_callable=AsyncMock) as post_mock: await channel.publish('event', 'foo') _, kwargs = post_mock.call_args assert json.loads(kwargs['body'])['data'] == 'foo' @@ -50,7 +50,7 @@ async def test_str(self): async def test_with_binary_type(self): channel = self.ably.channels["persisted:publish"] - with mock.patch('ably.rest.rest.Http.post', new_callable=AsyncMock) as post_mock: + with mock.patch('ably.pubsub.rest.rest.Http.post', new_callable=AsyncMock) as post_mock: await channel.publish('event', bytearray(b'foo')) _, kwargs = post_mock.call_args raw_data = json.loads(kwargs['body'])['data'] @@ -60,7 +60,7 @@ async def test_with_binary_type(self): async def test_with_bytes_type(self): channel = self.ably.channels["persisted:publish"] - with mock.patch('ably.rest.rest.Http.post', new_callable=AsyncMock) as post_mock: + with mock.patch('ably.pubsub.rest.rest.Http.post', new_callable=AsyncMock) as post_mock: await channel.publish('event', b'foo') _, kwargs = post_mock.call_args raw_data = json.loads(kwargs['body'])['data'] @@ -70,7 +70,7 @@ async def test_with_bytes_type(self): async def test_with_json_dict_data(self): channel = self.ably.channels["persisted:publish"] data = {'foó': 'bár'} - with mock.patch('ably.rest.rest.Http.post', new_callable=AsyncMock) as post_mock: + with mock.patch('ably.pubsub.rest.rest.Http.post', new_callable=AsyncMock) as post_mock: await channel.publish('event', data) _, kwargs = post_mock.call_args raw_data = json.loads(json.loads(kwargs['body'])['data']) @@ -80,7 +80,7 @@ async def test_with_json_dict_data(self): async def test_with_json_list_data(self): channel = self.ably.channels["persisted:publish"] data = ['foó', 'bár'] - with mock.patch('ably.rest.rest.Http.post', new_callable=AsyncMock) as post_mock: + with mock.patch('ably.pubsub.rest.rest.Http.post', new_callable=AsyncMock) as post_mock: await channel.publish('event', data) _, kwargs = post_mock.call_args raw_data = json.loads(json.loads(kwargs['body'])['data']) @@ -161,7 +161,7 @@ def decrypt(self, payload, options=None): async def test_text_utf8(self): channel = self.ably.channels.get("persisted:publish_enc", cipher=self.cipher_params) - with mock.patch('ably.rest.rest.Http.post', new_callable=AsyncMock) as post_mock: + with mock.patch('ably.pubsub.rest.rest.Http.post', new_callable=AsyncMock) as post_mock: await channel.publish('event', 'fóo') _, kwargs = post_mock.call_args assert json.loads(kwargs['body'])['encoding'].strip('/') == 'utf-8/cipher+aes-128-cbc/base64' @@ -172,7 +172,7 @@ async def test_str(self): # This test only makes sense for py2 channel = self.ably.channels["persisted:publish"] - with mock.patch('ably.rest.rest.Http.post', new_callable=AsyncMock) as post_mock: + with mock.patch('ably.pubsub.rest.rest.Http.post', new_callable=AsyncMock) as post_mock: await channel.publish('event', 'foo') _, kwargs = post_mock.call_args assert json.loads(kwargs['body'])['data'] == 'foo' @@ -182,7 +182,7 @@ async def test_with_binary_type(self): channel = self.ably.channels.get("persisted:publish_enc", cipher=self.cipher_params) - with mock.patch('ably.rest.rest.Http.post', new_callable=AsyncMock) as post_mock: + with mock.patch('ably.pubsub.rest.rest.Http.post', new_callable=AsyncMock) as post_mock: await channel.publish('event', bytearray(b'foo')) _, kwargs = post_mock.call_args @@ -195,7 +195,7 @@ async def test_with_json_dict_data(self): channel = self.ably.channels.get("persisted:publish_enc", cipher=self.cipher_params) data = {'foó': 'bár'} - with mock.patch('ably.rest.rest.Http.post', new_callable=AsyncMock) as post_mock: + with mock.patch('ably.pubsub.rest.rest.Http.post', new_callable=AsyncMock) as post_mock: await channel.publish('event', data) _, kwargs = post_mock.call_args assert json.loads(kwargs['body'])['encoding'].strip('/') == 'json/utf-8/cipher+aes-128-cbc/base64' @@ -206,7 +206,7 @@ async def test_with_json_list_data(self): channel = self.ably.channels.get("persisted:publish_enc", cipher=self.cipher_params) data = ['foó', 'bár'] - with mock.patch('ably.rest.rest.Http.post', new_callable=AsyncMock) as post_mock: + with mock.patch('ably.pubsub.rest.rest.Http.post', new_callable=AsyncMock) as post_mock: await channel.publish('event', data) _, kwargs = post_mock.call_args assert json.loads(kwargs['body'])['encoding'].strip('/') == 'json/utf-8/cipher+aes-128-cbc/base64' @@ -269,7 +269,7 @@ def decode(self, data): async def test_text_utf8(self): channel = self.ably.channels["persisted:publish"] - with mock.patch('ably.rest.rest.Http.post', + with mock.patch('ably.pubsub.rest.rest.Http.post', wraps=channel.ably.http.post) as post_mock: await channel.publish('event', 'foó') _, kwargs = post_mock.call_args @@ -279,7 +279,7 @@ async def test_text_utf8(self): async def test_with_binary_type(self): channel = self.ably.channels["persisted:publish"] - with mock.patch('ably.rest.rest.Http.post', + with mock.patch('ably.pubsub.rest.rest.Http.post', wraps=channel.ably.http.post) as post_mock: await channel.publish('event', bytearray(b'foo')) _, kwargs = post_mock.call_args @@ -289,7 +289,7 @@ async def test_with_binary_type(self): async def test_with_json_dict_data(self): channel = self.ably.channels["persisted:publish"] data = {'foó': 'bár'} - with mock.patch('ably.rest.rest.Http.post', + with mock.patch('ably.pubsub.rest.rest.Http.post', wraps=channel.ably.http.post) as post_mock: await channel.publish('event', data) _, kwargs = post_mock.call_args @@ -300,7 +300,7 @@ async def test_with_json_dict_data(self): async def test_with_json_list_data(self): channel = self.ably.channels["persisted:publish"] data = ['foó', 'bár'] - with mock.patch('ably.rest.rest.Http.post', + with mock.patch('ably.pubsub.rest.rest.Http.post', wraps=channel.ably.http.post) as post_mock: await channel.publish('event', data) _, kwargs = post_mock.call_args @@ -367,7 +367,7 @@ def decode(self, data): async def test_text_utf8(self): channel = self.ably.channels.get("persisted:publish_enc", cipher=self.cipher_params) - with mock.patch('ably.rest.rest.Http.post', + with mock.patch('ably.pubsub.rest.rest.Http.post', wraps=channel.ably.http.post) as post_mock: await channel.publish('event', 'fóo') _, kwargs = post_mock.call_args @@ -379,7 +379,7 @@ async def test_with_binary_type(self): channel = self.ably.channels.get("persisted:publish_enc", cipher=self.cipher_params) - with mock.patch('ably.rest.rest.Http.post', + with mock.patch('ably.pubsub.rest.rest.Http.post', wraps=channel.ably.http.post) as post_mock: await channel.publish('event', bytearray(b'foo')) _, kwargs = post_mock.call_args @@ -393,7 +393,7 @@ async def test_with_json_dict_data(self): channel = self.ably.channels.get("persisted:publish_enc", cipher=self.cipher_params) data = {'foó': 'bár'} - with mock.patch('ably.rest.rest.Http.post', + with mock.patch('ably.pubsub.rest.rest.Http.post', wraps=channel.ably.http.post) as post_mock: await channel.publish('event', data) _, kwargs = post_mock.call_args @@ -405,7 +405,7 @@ async def test_with_json_list_data(self): channel = self.ably.channels.get("persisted:publish_enc", cipher=self.cipher_params) data = ['foó', 'bár'] - with mock.patch('ably.rest.rest.Http.post', + with mock.patch('ably.pubsub.rest.rest.Http.post', wraps=channel.ably.http.post) as post_mock: await channel.publish('event', data) _, kwargs = post_mock.call_args diff --git a/test/ably/rest/restannotations_test.py b/test/ably/rest/restannotations_test.py index fcf2c696..7599f776 100644 --- a/test/ably/rest/restannotations_test.py +++ b/test/ably/rest/restannotations_test.py @@ -4,9 +4,9 @@ import pytest -from ably import AblyException -from ably.types.annotation import Annotation, AnnotationAction -from ably.types.message import Message +from ably.pubsub.server import AblyException +from ably.pubsub.types.annotation import Annotation, AnnotationAction +from ably.pubsub.types.message import Message from test.ably.testapp import TestApp from test.ably.utils import BaseAsyncTestCase, assert_waiter diff --git a/test/ably/rest/restauth_test.py b/test/ably/rest/restauth_test.py index 185021e1..70aac5e7 100644 --- a/test/ably/rest/restauth_test.py +++ b/test/ably/rest/restauth_test.py @@ -10,9 +10,9 @@ import respx from httpx import AsyncClient, Response -import ably -from ably import AblyAuthException, AblyRest, Auth -from ably.types.tokendetails import TokenDetails +import ably.pubsub.types.tokenrequest +from ably.pubsub.server import AblyAuthException, AblyRest, Auth +from ably.pubsub.types.tokendetails import TokenDetails from test.ably.testapp import TestApp from test.ably.utils import BaseAsyncTestCase, VaryByProtocolTestsMetaclass, dont_vary_protocol @@ -198,7 +198,7 @@ async def test_authorize_always_creates_new_token(self): async def test_authorize_create_new_token_if_expired(self): token = await self.ably.auth.authorize() - with mock.patch('ably.rest.auth.Auth.token_details_has_expired', + with mock.patch('ably.pubsub.rest.auth.Auth.token_details_has_expired', return_value=True): new_token = await self.ably.auth.authorize() @@ -212,7 +212,7 @@ async def test_authorize_returns_a_token_details(self): async def test_authorize_adheres_to_request_token(self): token_params = {'ttl': 10, 'client_id': 'client_id'} auth_params = {'auth_url': 'somewhere.com', 'query_time': True} - with mock.patch('ably.rest.auth.Auth.request_token', new_callable=AsyncMock) as request_mock: + with mock.patch('ably.pubsub.rest.auth.Auth.request_token', new_callable=AsyncMock) as request_mock: await self.ably.auth.authorize(token_params, auth_params) token_called, auth_called = request_mock.call_args @@ -251,7 +251,7 @@ async def test_if_parameters_are_stored_and_used_as_defaults(self): auth_options = dict(self.ably.auth.auth_options.auth_options) auth_options['auth_headers'] = {'a_headers': 'a_value'} await self.ably.auth.authorize({'ttl': 555}, auth_options) - with mock.patch('ably.rest.auth.Auth.request_token', + with mock.patch('ably.pubsub.rest.auth.Auth.request_token', wraps=self.ably.auth.request_token) as request_mock: await self.ably.auth.authorize() @@ -263,7 +263,7 @@ async def test_if_parameters_are_stored_and_used_as_defaults(self): auth_options = dict(self.ably.auth.auth_options.auth_options) auth_options['auth_headers'] = None await self.ably.auth.authorize({}, auth_options) - with mock.patch('ably.rest.auth.Auth.request_token', + with mock.patch('ably.pubsub.rest.auth.Auth.request_token', wraps=self.ably.auth.request_token) as request_mock: await self.ably.auth.authorize() @@ -283,8 +283,8 @@ async def test_timestamp_is_not_stored(self): # call authorize again with timestamp set timestamp = await self.ably.time() - with mock.patch('ably.rest.auth.TokenRequest', - wraps=ably.types.tokenrequest.TokenRequest) as tr_mock: + with mock.patch('ably.pubsub.rest.auth.TokenRequest', + wraps=ably.pubsub.types.tokenrequest.TokenRequest) as tr_mock: auth_options = dict(self.ably.auth.auth_options.auth_options) auth_options['auth_headers'] = {'a_headers': 'a_value'} token_2 = await self.ably.auth.authorize( @@ -295,8 +295,8 @@ async def test_timestamp_is_not_stored(self): assert tr_mock.call_args[1]['timestamp'] == timestamp # call authorize again with no params - with mock.patch('ably.rest.auth.TokenRequest', - wraps=ably.types.tokenrequest.TokenRequest) as tr_mock: + with mock.patch('ably.pubsub.rest.auth.TokenRequest', + wraps=ably.pubsub.types.tokenrequest.TokenRequest) as tr_mock: token_4 = await self.ably.auth.authorize() assert isinstance(token_4, TokenDetails) assert token_2 != token_4 diff --git a/test/ably/rest/restcapability_test.py b/test/ably/rest/restcapability_test.py index c95c651d..f83d1641 100644 --- a/test/ably/rest/restcapability_test.py +++ b/test/ably/rest/restcapability_test.py @@ -1,7 +1,7 @@ import pytest -from ably.types.capability import Capability -from ably.util.exceptions import AblyException +from ably.pubsub.types.capability import Capability +from ably.pubsub.util.exceptions import AblyException from test.ably.testapp import TestApp from test.ably.utils import BaseAsyncTestCase, VaryByProtocolTestsMetaclass, dont_vary_protocol diff --git a/test/ably/rest/restchannelhistory_test.py b/test/ably/rest/restchannelhistory_test.py index 2b15ae0d..c2fe5008 100644 --- a/test/ably/rest/restchannelhistory_test.py +++ b/test/ably/rest/restchannelhistory_test.py @@ -3,8 +3,8 @@ import pytest import respx -from ably import AblyException -from ably.http.paginatedresult import PaginatedResult +from ably.pubsub.http.paginatedresult import PaginatedResult +from ably.pubsub.server import AblyException from test.ably.testapp import TestApp from test.ably.utils import BaseAsyncTestCase, VaryByProtocolTestsMetaclass, dont_vary_protocol diff --git a/test/ably/rest/restchannelmutablemessages_test.py b/test/ably/rest/restchannelmutablemessages_test.py index b4f32ef4..cfcd8dae 100644 --- a/test/ably/rest/restchannelmutablemessages_test.py +++ b/test/ably/rest/restchannelmutablemessages_test.py @@ -3,9 +3,9 @@ import pytest -from ably import AblyException, CipherParams, MessageAction -from ably.types.message import Message -from ably.types.operations import MessageOperation +from ably.pubsub.server import AblyException, CipherParams, MessageAction +from ably.pubsub.types.message import Message +from ably.pubsub.types.operations import MessageOperation from test.ably.testapp import TestApp from test.ably.utils import BaseAsyncTestCase, assert_waiter diff --git a/test/ably/rest/restchannelpublish_test.py b/test/ably/rest/restchannelpublish_test.py index 5165a00e..2eacbf86 100644 --- a/test/ably/rest/restchannelpublish_test.py +++ b/test/ably/rest/restchannelpublish_test.py @@ -10,11 +10,11 @@ import msgpack import pytest -from ably import AblyException, IncompatibleClientIdException, api_version -from ably.rest.auth import Auth -from ably.types.message import Message -from ably.types.tokendetails import TokenDetails -from ably.util import case +from ably.pubsub.rest.auth import Auth +from ably.pubsub.server import AblyException, IncompatibleClientIdException, api_version +from ably.pubsub.types.message import Message +from ably.pubsub.types.tokendetails import TokenDetails +from ably.pubsub.util import case from test.ably import utils from test.ably.testapp import TestApp from test.ably.utils import BaseAsyncTestCase, VaryByProtocolTestsMetaclass, assert_waiter, dont_vary_protocol @@ -103,7 +103,7 @@ async def test_message_list_generate_one_request(self): expected_messages = [Message(f"name-{i}", str(i)) for i in range(3)] - with mock.patch('ably.rest.rest.Http.post', + with mock.patch('ably.pubsub.rest.rest.Http.post', wraps=channel.ably.http.post) as post_mock: await channel.publish(messages=expected_messages) assert post_mock.call_count == 1 @@ -184,7 +184,7 @@ async def test_publish_message_null_name_and_data_keys_arent_sent(self): channel = self.ably.channels[ self.get_channel_name('persisted:null_name_and_data_keys_arent_sent_channel')] - with mock.patch('ably.rest.rest.Http.post', + with mock.patch('ably.pubsub.rest.rest.Http.post', wraps=channel.ably.http.post) as post_mock: await channel.publish(name=None, data=None) @@ -244,7 +244,7 @@ async def test_publish_message_without_client_id_on_identified_client(self): channel = self.ably_with_client_id.channels[ self.get_channel_name('persisted:no_client_id_identified_client')] - with mock.patch('ably.rest.rest.Http.post', + with mock.patch('ably.pubsub.rest.rest.Http.post', wraps=channel.ably.http.post) as post_mock: await channel.publish(name='publish', data='test') diff --git a/test/ably/rest/restchannels_test.py b/test/ably/rest/restchannels_test.py index b5e59957..8172ec21 100644 --- a/test/ably/rest/restchannels_test.py +++ b/test/ably/rest/restchannels_test.py @@ -2,9 +2,9 @@ import pytest -from ably import AblyException -from ably.rest.channel import Channel, Channels, Presence -from ably.util.crypto import generate_random_key +from ably.pubsub.rest.channel import Channel, Channels, Presence +from ably.pubsub.server import AblyException +from ably.pubsub.util.crypto import generate_random_key from test.ably.testapp import TestApp from test.ably.utils import BaseAsyncTestCase diff --git a/test/ably/rest/restcrypto_test.py b/test/ably/rest/restcrypto_test.py index 94812b29..1ef65efc 100644 --- a/test/ably/rest/restcrypto_test.py +++ b/test/ably/rest/restcrypto_test.py @@ -6,9 +6,9 @@ import pytest from Crypto import Random -from ably import AblyException -from ably.types.message import Message -from ably.util.crypto import CipherParams, generate_random_key, get_cipher, get_default_params +from ably.pubsub.server import AblyException +from ably.pubsub.types.message import Message +from ably.pubsub.util.crypto import CipherParams, generate_random_key, get_cipher, get_default_params from test.ably import utils from test.ably.testapp import TestApp from test.ably.utils import BaseAsyncTestCase, BaseTestCase, VaryByProtocolTestsMetaclass, dont_vary_protocol diff --git a/test/ably/rest/resthttp_test.py b/test/ably/rest/resthttp_test.py index 385d63ec..1f5ffc7d 100644 --- a/test/ably/rest/resthttp_test.py +++ b/test/ably/rest/resthttp_test.py @@ -9,11 +9,11 @@ import respx from httpx import Response -from ably import AblyRest -from ably.http.httputils import HttpUtils -from ably.transport.defaults import Defaults -from ably.types.options import Options -from ably.util.exceptions import AblyException +from ably.pubsub.http.httputils import HttpUtils +from ably.pubsub.server import AblyRest +from ably.pubsub.transport.defaults import Defaults +from ably.pubsub.types.options import Options +from ably.pubsub.util.exceptions import AblyException from test.ably.testapp import TestApp from test.ably.utils import BaseAsyncTestCase diff --git a/test/ably/rest/restinit_test.py b/test/ably/rest/restinit_test.py index 25e7c5af..cbc2fea8 100644 --- a/test/ably/rest/restinit_test.py +++ b/test/ably/rest/restinit_test.py @@ -3,9 +3,9 @@ import pytest from httpx import AsyncClient -from ably import AblyException, AblyRest -from ably.transport.defaults import Defaults -from ably.types.tokendetails import TokenDetails +from ably.pubsub.server import AblyException, AblyRest +from ably.pubsub.transport.defaults import Defaults +from ably.pubsub.types.tokendetails import TokenDetails from test.ably.testapp import TestApp from test.ably.utils import BaseAsyncTestCase, VaryByProtocolTestsMetaclass, dont_vary_protocol @@ -168,8 +168,8 @@ async def test_query_time_param(self): use_binary_protocol=self.use_binary_protocol) timestamp = ably.auth._timestamp - with patch('ably.rest.rest.AblyRest.time', wraps=ably.time) as server_time,\ - patch('ably.rest.auth.Auth._timestamp', wraps=timestamp) as local_time: + with patch('ably.pubsub.rest.rest.AblyRest.time', wraps=ably.time) as server_time,\ + patch('ably.pubsub.rest.auth.Auth._timestamp', wraps=timestamp) as local_time: await ably.auth.request_token() assert local_time.call_count == 1 assert server_time.call_count == 1 diff --git a/test/ably/rest/restpaginatedresult_test.py b/test/ably/rest/restpaginatedresult_test.py index 9aa85689..fd7f9a81 100644 --- a/test/ably/rest/restpaginatedresult_test.py +++ b/test/ably/rest/restpaginatedresult_test.py @@ -2,7 +2,7 @@ import respx from httpx import Response -from ably.http.paginatedresult import PaginatedResult +from ably.pubsub.http.paginatedresult import PaginatedResult from test.ably.testapp import TestApp from test.ably.utils import BaseAsyncTestCase diff --git a/test/ably/rest/restpresence_test.py b/test/ably/rest/restpresence_test.py index 8767b0c6..4efbf35c 100644 --- a/test/ably/rest/restpresence_test.py +++ b/test/ably/rest/restpresence_test.py @@ -3,8 +3,8 @@ import pytest import respx -from ably.http.paginatedresult import PaginatedResult -from ably.types.presence import PresenceMessage +from ably.pubsub.http.paginatedresult import PaginatedResult +from ably.pubsub.types.presence import PresenceMessage from test.ably.testapp import TestApp from test.ably.utils import BaseAsyncTestCase, VaryByProtocolTestsMetaclass, dont_vary_protocol diff --git a/test/ably/rest/restpush_test.py b/test/ably/rest/restpush_test.py index 30c4798d..39a3dccd 100644 --- a/test/ably/rest/restpush_test.py +++ b/test/ably/rest/restpush_test.py @@ -5,8 +5,8 @@ import pytest -from ably import AblyAuthException, AblyException, DeviceDetails, PushChannelSubscription -from ably.http.paginatedresult import PaginatedResult +from ably.pubsub.http.paginatedresult import PaginatedResult +from ably.pubsub.server import AblyAuthException, AblyException, DeviceDetails, PushChannelSubscription from test.ably.testapp import TestApp from test.ably.utils import ( BaseAsyncTestCase, diff --git a/test/ably/rest/restrequest_test.py b/test/ably/rest/restrequest_test.py index 63c20ccf..7398a9d4 100644 --- a/test/ably/rest/restrequest_test.py +++ b/test/ably/rest/restrequest_test.py @@ -2,9 +2,9 @@ import pytest import respx -from ably import AblyRest -from ably.http.paginatedresult import HttpPaginatedResponse -from ably.transport.defaults import Defaults +from ably.pubsub.http.paginatedresult import HttpPaginatedResponse +from ably.pubsub.server import AblyRest +from ably.pubsub.transport.defaults import Defaults from test.ably.testapp import TestApp from test.ably.utils import BaseAsyncTestCase, VaryByProtocolTestsMetaclass, dont_vary_protocol diff --git a/test/ably/rest/reststats_test.py b/test/ably/rest/reststats_test.py index cef28817..577f5644 100644 --- a/test/ably/rest/reststats_test.py +++ b/test/ably/rest/reststats_test.py @@ -3,9 +3,9 @@ import pytest -from ably.http.paginatedresult import PaginatedResult -from ably.types.stats import Stats -from ably.util.exceptions import AblyException +from ably.pubsub.http.paginatedresult import PaginatedResult +from ably.pubsub.types.stats import Stats +from ably.pubsub.util.exceptions import AblyException from test.ably.testapp import TestApp from test.ably.utils import BaseAsyncTestCase, VaryByProtocolTestsMetaclass, dont_vary_protocol diff --git a/test/ably/rest/resttime_test.py b/test/ably/rest/resttime_test.py index 4b78620a..757d5743 100644 --- a/test/ably/rest/resttime_test.py +++ b/test/ably/rest/resttime_test.py @@ -2,7 +2,7 @@ import pytest -from ably import AblyException +from ably.pubsub.server import AblyException from test.ably.testapp import TestApp from test.ably.utils import BaseAsyncTestCase, VaryByProtocolTestsMetaclass, dont_vary_protocol diff --git a/test/ably/rest/resttoken_test.py b/test/ably/rest/resttoken_test.py index 5052f1be..c0957bba 100644 --- a/test/ably/rest/resttoken_test.py +++ b/test/ably/rest/resttoken_test.py @@ -5,9 +5,9 @@ import pytest -from ably import AblyException, AblyRest, Capability -from ably.types.tokendetails import TokenDetails -from ably.types.tokenrequest import TokenRequest +from ably.pubsub.server import AblyException, AblyRest, Capability +from ably.pubsub.types.tokendetails import TokenDetails +from ably.pubsub.types.tokenrequest import TokenRequest from test.ably.testapp import TestApp from test.ably.utils import BaseAsyncTestCase, VaryByProtocolTestsMetaclass, dont_vary_protocol @@ -120,8 +120,8 @@ async def test_token_generation_with_invalid_ttl(self): async def test_token_generation_with_local_time(self): timestamp = self.ably.auth._timestamp - with patch('ably.rest.rest.AblyRest.time', wraps=self.ably.time) as server_time,\ - patch('ably.rest.auth.Auth._timestamp', wraps=timestamp) as local_time: + with patch('ably.pubsub.rest.rest.AblyRest.time', wraps=self.ably.time) as server_time,\ + patch('ably.pubsub.rest.auth.Auth._timestamp', wraps=timestamp) as local_time: await self.ably.auth.request_token() assert local_time.called assert not server_time.called @@ -129,8 +129,8 @@ async def test_token_generation_with_local_time(self): # RSA10k async def test_token_generation_with_server_time(self): timestamp = self.ably.auth._timestamp - with patch('ably.rest.rest.AblyRest.time', wraps=self.ably.time) as server_time,\ - patch('ably.rest.auth.Auth._timestamp', wraps=timestamp) as local_time: + with patch('ably.pubsub.rest.rest.AblyRest.time', wraps=self.ably.time) as server_time,\ + patch('ably.pubsub.rest.auth.Auth._timestamp', wraps=timestamp) as local_time: await self.ably.auth.request_token(query_time=True) assert local_time.call_count == 1 assert server_time.call_count == 1 @@ -182,8 +182,8 @@ async def test_key_name_and_secret_are_required(self): @dont_vary_protocol async def test_with_local_time(self): timestamp = self.ably.auth._timestamp - with patch('ably.rest.rest.AblyRest.time', wraps=self.ably.time) as server_time,\ - patch('ably.rest.auth.Auth._timestamp', wraps=timestamp) as local_time: + with patch('ably.pubsub.rest.rest.AblyRest.time', wraps=self.ably.time) as server_time,\ + patch('ably.pubsub.rest.auth.Auth._timestamp', wraps=timestamp) as local_time: await self.ably.auth.create_token_request( key_name=self.key_name, key_secret=self.key_secret, query_time=False) assert local_time.called @@ -193,8 +193,8 @@ async def test_with_local_time(self): @dont_vary_protocol async def test_with_server_time(self): timestamp = self.ably.auth._timestamp - with patch('ably.rest.rest.AblyRest.time', wraps=self.ably.time) as server_time,\ - patch('ably.rest.auth.Auth._timestamp', wraps=timestamp) as local_time: + with patch('ably.pubsub.rest.rest.AblyRest.time', wraps=self.ably.time) as server_time,\ + patch('ably.pubsub.rest.auth.Auth._timestamp', wraps=timestamp) as local_time: await self.ably.auth.create_token_request( key_name=self.key_name, key_secret=self.key_secret, query_time=True) assert local_time.call_count == 1 @@ -329,7 +329,7 @@ async def test_hmac(self): # AO2g @dont_vary_protocol async def test_query_server_time(self): - with patch('ably.rest.rest.AblyRest.time', wraps=self.ably.time) as server_time: + with patch('ably.pubsub.rest.rest.AblyRest.time', wraps=self.ably.time) as server_time: await self.ably.auth.create_token_request( key_name=self.key_name, key_secret=self.key_secret, query_time=True) assert server_time.call_count == 1 diff --git a/test/ably/testapp.py b/test/ably/testapp.py index 744fd2c7..8b086e10 100644 --- a/test/ably/testapp.py +++ b/test/ably/testapp.py @@ -2,12 +2,12 @@ import logging import os -from ably.realtime.realtime import AblyRealtime -from ably.rest.rest import AblyRest -from ably.transport.defaults import Defaults -from ably.types.capability import Capability -from ably.types.options import Options -from ably.util.exceptions import AblyException +from ably.pubsub.realtime.realtime import AblyRealtime +from ably.pubsub.rest.rest import AblyRest +from ably.pubsub.transport.defaults import Defaults +from ably.pubsub.types.capability import Capability +from ably.pubsub.types.options import Options +from ably.pubsub.util.exceptions import AblyException log = logging.getLogger(__name__) diff --git a/test/ably/utils.py b/test/ably/utils.py index a72fdd5c..9b768c0f 100644 --- a/test/ably/utils.py +++ b/test/ably/utils.py @@ -11,7 +11,7 @@ import respx from httpx import Response -from ably.http.http import Http +from ably.pubsub.http.http import Http class BaseTestCase: diff --git a/test/unit/annotation_test.py b/test/unit/annotation_test.py index 2cfb0bef..ff16cf56 100644 --- a/test/unit/annotation_test.py +++ b/test/unit/annotation_test.py @@ -15,11 +15,11 @@ import pytest -from ably.rest.annotations import construct_validate_annotation, serial_from_msg_or_serial -from ably.types.annotation import Annotation, AnnotationAction -from ably.types.message import Message -from ably.util.crypto import generate_random_key, get_cipher -from ably.util.exceptions import AblyException +from ably.pubsub.rest.annotations import construct_validate_annotation, serial_from_msg_or_serial +from ably.pubsub.types.annotation import Annotation, AnnotationAction +from ably.pubsub.types.message import Message +from ably.pubsub.util.crypto import generate_random_key, get_cipher +from ably.pubsub.util.exceptions import AblyException # --- RSAN1a3: type validation --- diff --git a/test/unit/clock_test.py b/test/unit/clock_test.py index f8814e3b..a4d2e9bd 100644 --- a/test/unit/clock_test.py +++ b/test/unit/clock_test.py @@ -1,10 +1,10 @@ import httpx import pytest -from ably import AblyRest -from ably.types.testoptions import TestOptions -from ably.util.clock import Clock -from ably.util.exceptions import AblyException +from ably.pubsub.server import AblyRest +from ably.pubsub.types.testoptions import TestOptions +from ably.pubsub.util.clock import Clock +from ably.pubsub.util.exceptions import AblyException # A plausible wall-clock reading for a fake to start from, so that a time taken # against it looks like a real one. diff --git a/test/unit/http_test.py b/test/unit/http_test.py index 6418dc00..a3a3cf6d 100644 --- a/test/unit/http_test.py +++ b/test/unit/http_test.py @@ -1,9 +1,9 @@ import httpx import pytest -from ably import AblyRest -from ably.types.testoptions import TestOptions -from ably.util.exceptions import AblyException +from ably.pubsub.server import AblyRest +from ably.pubsub.types.testoptions import TestOptions +from ably.pubsub.util.exceptions import AblyException def test_http_get_rest_hosts_works_when_fallback_realtime_host_is_set(): diff --git a/test/unit/message_test.py b/test/unit/message_test.py index 917af30c..f49e9bf9 100644 --- a/test/unit/message_test.py +++ b/test/unit/message_test.py @@ -1,4 +1,4 @@ -import ably.types.message +import ably.pubsub.types.message # TM2a, TM2c, TM2f @@ -14,7 +14,7 @@ def test_update_inner_message_fields_tm2(): } ] } - ably.types.message.Message.update_inner_message_fields(proto_msg) + ably.pubsub.types.message.Message.update_inner_message_fields(proto_msg) messages: list[dict] = proto_msg.get('messages') msg_index = 0 for msg in messages: @@ -37,7 +37,7 @@ def test_update_inner_message_fields_for_presence_msg_tm2(): } ] } - ably.types.message.Message.update_inner_message_fields(proto_msg) + ably.pubsub.types.message.Message.update_inner_message_fields(proto_msg) presence_messages: list[dict] = proto_msg.get('presence') msg_index = 0 for presence_msg in presence_messages: @@ -49,7 +49,7 @@ def test_update_inner_message_fields_for_presence_msg_tm2(): # RSL6b def test_invalid_base64_data_is_delivered_with_remaining_encoding(): - message = ably.types.message.Message.from_encoded({ + message = ably.pubsub.types.message.Message.from_encoded({ 'name': 'event', 'data': 'encrypted-data-here', 'encoding': 'custom-encryption/base64', diff --git a/test/unit/mutable_message_test.py b/test/unit/mutable_message_test.py index 8ce603b9..2ea569c0 100644 --- a/test/unit/mutable_message_test.py +++ b/test/unit/mutable_message_test.py @@ -1,5 +1,5 @@ -from ably import MessageAction, MessageOperation, MessageVersion, UpdateDeleteResult -from ably.types.message import Message +from ably.pubsub.server import MessageAction, MessageOperation, MessageVersion, UpdateDeleteResult +from ably.pubsub.types.message import Message def test_message_version_none_values_filtered(): @@ -137,7 +137,7 @@ def test_message_extras_none_excluded_from_as_dict(): # RSL15b, RTL32b, TM2u def test_message_annotations_preserved_in_as_dict(): """Test that annotations are included when a Message with annotations is serialized.""" - from ably.types.message import MessageAnnotations + from ably.pubsub.types.message import MessageAnnotations annotations = MessageAnnotations(summary={'reaction:distinct.v1': {'thumbsup': 5}}) message = Message( name='test', diff --git a/test/unit/options_test.py b/test/unit/options_test.py index d3ba6129..84cc963b 100644 --- a/test/unit/options_test.py +++ b/test/unit/options_test.py @@ -1,7 +1,7 @@ import pytest -from ably.types.options import Options -from ably.util.exceptions import AblyException +from ably.pubsub.types.options import Options +from ably.pubsub.util.exceptions import AblyException # REC1b1: endpoint is incompatible with deprecated options diff --git a/test/unit/websockettransport_test.py b/test/unit/websockettransport_test.py index d45c3ca8..3cf1fffb 100644 --- a/test/unit/websockettransport_test.py +++ b/test/unit/websockettransport_test.py @@ -1,7 +1,7 @@ from unittest.mock import MagicMock, patch -from ably.transport.websockettransport import WebSocketTransport -from ably.types.options import Options +from ably.pubsub.transport.websockettransport import WebSocketTransport +from ably.pubsub.types.options import Options def _connect_url(host='example.com', **option_kwargs): @@ -9,7 +9,7 @@ def _connect_url(host='example.com', **option_kwargs): connection_manager.options = Options(auth_token='foo', **option_kwargs) transport = WebSocketTransport(connection_manager, host, {'format': 'json'}) with patch.object(transport, 'ws_connect', MagicMock()) as mock_ws_connect: - with patch('ably.transport.websockettransport.asyncio.create_task') as mock_create_task: + with patch('ably.pubsub.transport.websockettransport.asyncio.create_task') as mock_create_task: mock_create_task.return_value = MagicMock() transport.connect() return mock_ws_connect.call_args[0][0] diff --git a/test/uts/helpers/client.py b/test/uts/helpers/client.py index 57db6e6a..eaf1da3a 100644 --- a/test/uts/helpers/client.py +++ b/test/uts/helpers/client.py @@ -4,8 +4,8 @@ import inspect import logging -from ably import AblyRealtime, AblyRest -from ably.types.testoptions import TestOptions +from ably.pubsub.server import AblyRealtime, AblyRest +from ably.pubsub.types.testoptions import TestOptions from test.uts.helpers.clock import settle from test.uts.helpers.sandbox import SANDBOX_ENDPOINT @@ -274,7 +274,7 @@ async def connected_client(mock_websocket, **kwargs): Most channel specifications open this way, since a channel cannot attach until the connection carrying it is up. """ - from ably.realtime.connection import ConnectionState + from ably.pubsub.realtime.connection import ConnectionState from test.uts.helpers.mock_websocket import CONNECTED_MESSAGE if mock_websocket.on_connection_attempt is None: @@ -293,7 +293,7 @@ async def drop_transport(client, mock_websocket): specification expects to find it rather than reconnecting behind the assertions. Returns the connection states recorded along the way. """ - from ably.realtime.connection import ConnectionState + from ably.pubsub.realtime.connection import ConnectionState states = [] @@ -316,7 +316,7 @@ async def reconnect_transport(client, mock_websocket, connected_message=None): Waiting on the connection state alone would be satisfied by the CONNECTED the client already holds, so this counts a fresh arrival. """ - from ably.realtime.connection import ConnectionState + from ably.pubsub.realtime.connection import ConnectionState from test.uts.helpers.mock_websocket import CONNECTED_MESSAGE message = CONNECTED_MESSAGE if connected_message is None else connected_message diff --git a/test/uts/helpers/clock.py b/test/uts/helpers/clock.py index 6d818715..c2a07720 100644 --- a/test/uts/helpers/clock.py +++ b/test/uts/helpers/clock.py @@ -61,7 +61,7 @@ def __repr__(self): class FakeClock: """Records timers against a notional time which only `advance` moves. - `FakeClock` has the shape of `ably.util.clock.Clock`, so it can be passed + `FakeClock` has the shape of `ably.pubsub.util.clock.Clock`, so it can be passed straight to `TestOptions(clock=...)`. Nothing fires until a test asks for it, and every reading the client takes moves only when `advance` does. """ diff --git a/test/uts/helpers/mock_http_test.py b/test/uts/helpers/mock_http_test.py index d860a7d0..6d4c1f59 100644 --- a/test/uts/helpers/mock_http_test.py +++ b/test/uts/helpers/mock_http_test.py @@ -6,9 +6,9 @@ import msgpack import pytest -from ably import AblyRest -from ably.types.testoptions import TestOptions -from ably.util.exceptions import AblyException +from ably.pubsub.server import AblyRest +from ably.pubsub.types.testoptions import TestOptions +from ably.pubsub.util.exceptions import AblyException from test.uts.helpers.mock_http import MockHttpClient KEY = 'a.b:c' diff --git a/test/uts/helpers/mock_websocket.py b/test/uts/helpers/mock_websocket.py index bbf0075a..fc89bb93 100644 --- a/test/uts/helpers/mock_websocket.py +++ b/test/uts/helpers/mock_websocket.py @@ -27,7 +27,7 @@ import msgpack from websockets.exceptions import WebSocketException -from ably.transport.websockettransport import ProtocolMessageAction +from ably.pubsub.transport.websockettransport import ProtocolMessageAction from test.uts.helpers.mock_http import RecordedUrl MSGPACK_PROTOCOL = 'application/x-msgpack' diff --git a/test/uts/helpers/mock_websocket_test.py b/test/uts/helpers/mock_websocket_test.py index dc2866f2..39e7e098 100644 --- a/test/uts/helpers/mock_websocket_test.py +++ b/test/uts/helpers/mock_websocket_test.py @@ -12,8 +12,8 @@ import msgpack import pytest -from ably.realtime.connection import ConnectionState -from ably.transport.websockettransport import ProtocolMessageAction +from ably.pubsub.realtime.connection import ConnectionState +from ably.pubsub.transport.websockettransport import ProtocolMessageAction from test.uts.helpers.client import await_connection_state, realtime_client from test.uts.helpers.clock import FakeClock from test.uts.helpers.mock_http import MockHttpClient diff --git a/test/uts/helpers/presence.py b/test/uts/helpers/presence.py index 4d493bdc..cec057a9 100644 --- a/test/uts/helpers/presence.py +++ b/test/uts/helpers/presence.py @@ -8,10 +8,10 @@ from types import SimpleNamespace -from ably.realtime.presence import RealtimePresence -from ably.realtime.presencemap import PresenceMap -from ably.transport.websockettransport import ProtocolMessageAction -from ably.types.presence import PresenceAction, PresenceMessage +from ably.pubsub.realtime.presence import RealtimePresence +from ably.pubsub.realtime.presencemap import PresenceMap +from ably.pubsub.transport.websockettransport import ProtocolMessageAction +from ably.pubsub.types.presence import PresenceAction, PresenceMessage #: The lowercase action names `RealtimePresence` emits its events under, which come #: from `PresenceAction._action_name` and exist only once `presence.py` is imported. diff --git a/test/uts/helpers/sandbox.py b/test/uts/helpers/sandbox.py index 0fb89fec..203e7f41 100644 --- a/test/uts/helpers/sandbox.py +++ b/test/uts/helpers/sandbox.py @@ -80,7 +80,7 @@ def fixture_cipher_params(): key and the IV base64-encoded, which is not what `CipherParams` wants: it takes them as raw bytes and derives the key length from the key. """ - from ably.util.crypto import CipherParams + from ably.pubsub.util.crypto import CipherParams return CipherParams( algorithm=CIPHER_FIXTURE['algorithm'], diff --git a/test/uts/realtime/integration/auth/token_renewal_test.py b/test/uts/realtime/integration/auth/token_renewal_test.py index a1c4b25f..e8e4407c 100644 --- a/test/uts/realtime/integration/auth/token_renewal_test.py +++ b/test/uts/realtime/integration/auth/token_renewal_test.py @@ -13,7 +13,7 @@ introduce a value no assertion consumes. """ -from ably.realtime.connection import ConnectionState +from ably.pubsub.realtime.connection import ConnectionState from test.uts.helpers.client import await_connection_state, sandbox_realtime_client, wall_clock_poll_until from test.uts.helpers.sandbox import extract_key_name, extract_key_secret, generate_jwt diff --git a/test/uts/realtime/integration/auth/token_request_test.py b/test/uts/realtime/integration/auth/token_request_test.py index 285c8e26..9729b8c0 100644 --- a/test/uts/realtime/integration/auth/token_request_test.py +++ b/test/uts/realtime/integration/auth/token_request_test.py @@ -15,7 +15,7 @@ case, so `TokenParams(clientId: x)` is `{'client_id': x}`. """ -from ably.realtime.connection import ConnectionState +from ably.pubsub.realtime.connection import ConnectionState from test.uts.helpers.client import await_connection_state, sandbox_realtime_client, sandbox_rest_client from test.uts.helpers.sandbox import random_id diff --git a/test/uts/realtime/integration/auth_test.py b/test/uts/realtime/integration/auth_test.py index 7f82d260..3bb60ae2 100644 --- a/test/uts/realtime/integration/auth_test.py +++ b/test/uts/realtime/integration/auth_test.py @@ -15,7 +15,7 @@ registered beforehand by the time the call completes. The reauth test needs no settle. """ -from ably.realtime.connection import ConnectionState +from ably.pubsub.realtime.connection import ConnectionState from test.uts.helpers.client import await_connection_state, sandbox_realtime_client from test.uts.helpers.deviations import deviation from test.uts.helpers.sandbox import extract_key_name, extract_key_secret, generate_jwt, random_id diff --git a/test/uts/realtime/integration/channel_history_test.py b/test/uts/realtime/integration/channel_history_test.py index d49eb95a..ae16404e 100644 --- a/test/uts/realtime/integration/channel_history_test.py +++ b/test/uts/realtime/integration/channel_history_test.py @@ -19,7 +19,7 @@ specification writes raises `ValueError`. """ -from ably.realtime.connection import ConnectionState +from ably.pubsub.realtime.connection import ConnectionState from test.uts.helpers.client import await_connection_state, sandbox_realtime_client, wall_clock_poll_until from test.uts.helpers.sandbox import random_id diff --git a/test/uts/realtime/integration/channels/channel_attach_test.py b/test/uts/realtime/integration/channels/channel_attach_test.py index 68f31243..1dee7f99 100644 --- a/test/uts/realtime/integration/channels/channel_attach_test.py +++ b/test/uts/realtime/integration/channels/channel_attach_test.py @@ -18,9 +18,9 @@ import pytest -from ably.realtime.channel import ChannelState -from ably.realtime.connection import ConnectionState -from ably.util.exceptions import AblyException +from ably.pubsub.realtime.channel import ChannelState +from ably.pubsub.realtime.connection import ConnectionState +from ably.pubsub.util.exceptions import AblyException from test.uts.helpers.client import await_connection_state, sandbox_realtime_client from test.uts.helpers.sandbox import random_id diff --git a/test/uts/realtime/integration/channels/channel_publish_test.py b/test/uts/realtime/integration/channels/channel_publish_test.py index 53b9800a..13b0a29c 100644 --- a/test/uts/realtime/integration/channels/channel_publish_test.py +++ b/test/uts/realtime/integration/channels/channel_publish_test.py @@ -22,8 +22,8 @@ spells out for the same wait. """ -from ably.realtime.connection import ConnectionState -from ably.types.message import Message +from ably.pubsub.realtime.connection import ConnectionState +from ably.pubsub.types.message import Message from test.uts.helpers.client import await_connection_state, sandbox_realtime_client, wall_clock_poll_until from test.uts.helpers.sandbox import random_id diff --git a/test/uts/realtime/integration/channels/channel_subscribe_test.py b/test/uts/realtime/integration/channels/channel_subscribe_test.py index c8850930..99a1022e 100644 --- a/test/uts/realtime/integration/channels/channel_subscribe_test.py +++ b/test/uts/realtime/integration/channels/channel_subscribe_test.py @@ -20,7 +20,7 @@ spells out for the same wait. """ -from ably.realtime.connection import ConnectionState +from ably.pubsub.realtime.connection import ConnectionState from test.uts.helpers.client import await_connection_state, sandbox_realtime_client, wall_clock_poll_until from test.uts.helpers.sandbox import random_id diff --git a/test/uts/realtime/integration/connection/connection_failures_test.py b/test/uts/realtime/integration/connection/connection_failures_test.py index 3e31a24e..fa4cf3b9 100644 --- a/test/uts/realtime/integration/connection/connection_failures_test.py +++ b/test/uts/realtime/integration/connection/connection_failures_test.py @@ -18,7 +18,7 @@ 1008. It is noise from teardown, not a failure of either test. """ -from ably.realtime.connection import ConnectionState +from ably.pubsub.realtime.connection import ConnectionState from test.uts.helpers.client import await_connection_state, sandbox_realtime_client # The wait the specification gives each connect attempt. diff --git a/test/uts/realtime/integration/connection_lifecycle_test.py b/test/uts/realtime/integration/connection_lifecycle_test.py index ffa0c126..d58a66ba 100644 --- a/test/uts/realtime/integration/connection_lifecycle_test.py +++ b/test/uts/realtime/integration/connection_lifecycle_test.py @@ -16,7 +16,7 @@ import re -from ably.realtime.connection import ConnectionState +from ably.pubsub.realtime.connection import ConnectionState from test.uts.helpers.client import await_connection_state, sandbox_realtime_client # The waits the specification's `Integration Test Notes` give each leg: auth plus transport diff --git a/test/uts/realtime/integration/delta_decoding_test.py b/test/uts/realtime/integration/delta_decoding_test.py index 6538d84b..f6c8ef48 100644 --- a/test/uts/realtime/integration/delta_decoding_test.py +++ b/test/uts/realtime/integration/delta_decoding_test.py @@ -7,7 +7,7 @@ vcdiff delta, and the SDK decodes it against the payload it stored for the previous one. The specification's `plugins: { vcdiff: decoder }` is the `vcdiff_decoder` client option -here, and its `VCDiffDecoder` interface is `ably.types.options.VCDiffDecoder` — +here, and its `VCDiffDecoder` interface is `ably.pubsub.types.options.VCDiffDecoder` — `decode(delta, base) -> bytes`, the VD2a argument order. `AblyVCDiffDecoder` is the real implementation, backed by the `vcdiff-decoder` library; `CountingDecoder` wraps it so a test can assert how many deltas the server actually sent. @@ -20,11 +20,11 @@ import os -from ably import AblyVCDiffDecoder -from ably.realtime.connection import ConnectionState -from ably.types.channeloptions import ChannelOptions -from ably.types.channelstate import ChannelState -from ably.types.options import VCDiffDecoder +from ably.pubsub.realtime.connection import ConnectionState +from ably.pubsub.server import AblyVCDiffDecoder +from ably.pubsub.types.channeloptions import ChannelOptions +from ably.pubsub.types.channelstate import ChannelState +from ably.pubsub.types.options import VCDiffDecoder from test.uts.helpers.client import ( await_channel_state, await_connection_state, diff --git a/test/uts/realtime/integration/mutable_messages_test.py b/test/uts/realtime/integration/mutable_messages_test.py index 8d19691a..327bfcf6 100644 --- a/test/uts/realtime/integration/mutable_messages_test.py +++ b/test/uts/realtime/integration/mutable_messages_test.py @@ -18,15 +18,15 @@ consistent; the two polls below are the specification's `poll_until_success` around them. """ -from ably.http.paginatedresult import PaginatedResult -from ably.realtime.connection import ConnectionState -from ably.types.annotation import Annotation, AnnotationAction -from ably.types.channelmode import ChannelMode -from ably.types.channeloptions import ChannelOptions -from ably.types.channelstate import ChannelState -from ably.types.message import Message, MessageAction -from ably.types.operations import MessageOperation, UpdateDeleteResult -from ably.util.exceptions import AblyException +from ably.pubsub.http.paginatedresult import PaginatedResult +from ably.pubsub.realtime.connection import ConnectionState +from ably.pubsub.types.annotation import Annotation, AnnotationAction +from ably.pubsub.types.channelmode import ChannelMode +from ably.pubsub.types.channeloptions import ChannelOptions +from ably.pubsub.types.channelstate import ChannelState +from ably.pubsub.types.message import Message, MessageAction +from ably.pubsub.types.operations import MessageOperation, UpdateDeleteResult +from ably.pubsub.util.exceptions import AblyException from test.uts.helpers.client import ( await_channel_state, await_connection_state, diff --git a/test/uts/realtime/integration/presence/presence_sync_test.py b/test/uts/realtime/integration/presence/presence_sync_test.py index 6a78481d..20a46ed6 100644 --- a/test/uts/realtime/integration/presence/presence_sync_test.py +++ b/test/uts/realtime/integration/presence/presence_sync_test.py @@ -17,8 +17,8 @@ behalf of others is built with `client_id='*'`. """ -from ably.realtime.connection import ConnectionState -from ably.types.presence import PresenceAction +from ably.pubsub.realtime.connection import ConnectionState +from ably.pubsub.types.presence import PresenceAction from test.uts.helpers.client import await_connection_state, sandbox_realtime_client from test.uts.helpers.sandbox import random_id diff --git a/test/uts/realtime/integration/presence_lifecycle_test.py b/test/uts/realtime/integration/presence_lifecycle_test.py index a6a74603..0a8a7c70 100644 --- a/test/uts/realtime/integration/presence_lifecycle_test.py +++ b/test/uts/realtime/integration/presence_lifecycle_test.py @@ -22,8 +22,8 @@ import asyncio -from ably.realtime.connection import ConnectionState -from ably.types.presence import PresenceAction +from ably.pubsub.realtime.connection import ConnectionState +from ably.pubsub.types.presence import PresenceAction from test.uts.helpers.client import ( await_connection_state, sandbox_realtime_client, diff --git a/test/uts/realtime/integration/proxy/auth_reauth_test.py b/test/uts/realtime/integration/proxy/auth_reauth_test.py index 847f66f1..8e47f75e 100644 --- a/test/uts/realtime/integration/proxy/auth_reauth_test.py +++ b/test/uts/realtime/integration/proxy/auth_reauth_test.py @@ -22,7 +22,7 @@ an `auth` attribute left the client, and that the connection was never disturbed. """ -from ably.realtime.connection import ConnectionState +from ably.pubsub.realtime.connection import ConnectionState from test.uts.helpers.client import await_connection_state, sandbox_realtime_client, wall_clock_poll_until from test.uts.helpers.sandbox import extract_key_name, extract_key_secret, generate_jwt diff --git a/test/uts/realtime/integration/proxy/channel_faults_test.py b/test/uts/realtime/integration/proxy/channel_faults_test.py index dbcdd378..09ae9907 100644 --- a/test/uts/realtime/integration/proxy/channel_faults_test.py +++ b/test/uts/realtime/integration/proxy/channel_faults_test.py @@ -28,9 +28,9 @@ import pytest -from ably.realtime.connection import ConnectionState -from ably.types.channelstate import ChannelState -from ably.util.exceptions import AblyException +from ably.pubsub.realtime.connection import ConnectionState +from ably.pubsub.types.channelstate import ChannelState +from ably.pubsub.util.exceptions import AblyException from test.uts.helpers.client import ( await_channel_state, await_connection_state, diff --git a/test/uts/realtime/integration/proxy/connection_open_failures_test.py b/test/uts/realtime/integration/proxy/connection_open_failures_test.py index 58206871..5f0d009d 100644 --- a/test/uts/realtime/integration/proxy/connection_open_failures_test.py +++ b/test/uts/realtime/integration/proxy/connection_open_failures_test.py @@ -26,8 +26,8 @@ callback, because renewing a token is what it is about. """ -from ably import AblyRest -from ably.realtime.connection import ConnectionState +from ably.pubsub.realtime.connection import ConnectionState +from ably.pubsub.server import AblyRest from test.uts.helpers.client import await_connection_state, sandbox_realtime_client from test.uts.helpers.deviations import deviation from test.uts.helpers.mock_websocket import contains_in_order diff --git a/test/uts/realtime/integration/proxy/connection_resume_test.py b/test/uts/realtime/integration/proxy/connection_resume_test.py index 22e042b9..6f93dff8 100644 --- a/test/uts/realtime/integration/proxy/connection_resume_test.py +++ b/test/uts/realtime/integration/proxy/connection_resume_test.py @@ -39,8 +39,8 @@ import asyncio import json -from ably.realtime.channel import ChannelState -from ably.realtime.connection import ConnectionState +from ably.pubsub.realtime.channel import ChannelState +from ably.pubsub.realtime.connection import ConnectionState from test.uts.helpers.client import sandbox_realtime_client, sandbox_rest_client, wall_clock_poll_until from test.uts.helpers.deviations import deviation from test.uts.helpers.sandbox import extract_key_name, extract_key_secret, generate_jwt, random_id diff --git a/test/uts/realtime/integration/proxy/heartbeat_test.py b/test/uts/realtime/integration/proxy/heartbeat_test.py index b9602be6..c90a1b90 100644 --- a/test/uts/realtime/integration/proxy/heartbeat_test.py +++ b/test/uts/realtime/integration/proxy/heartbeat_test.py @@ -27,7 +27,7 @@ waiting for it, so the reconnection settles instead of cycling. """ -from ably.realtime.connection import ConnectionState +from ably.pubsub.realtime.connection import ConnectionState from test.uts.helpers.client import await_connection_state, sandbox_realtime_client, wall_clock_poll_until from test.uts.helpers.mock_websocket import contains_in_order from test.uts.helpers.sandbox import extract_key_name, extract_key_secret, generate_jwt diff --git a/test/uts/realtime/integration/proxy/presence_reentry_test.py b/test/uts/realtime/integration/proxy/presence_reentry_test.py index 9e20a4c0..674ae6e9 100644 --- a/test/uts/realtime/integration/proxy/presence_reentry_test.py +++ b/test/uts/realtime/integration/proxy/presence_reentry_test.py @@ -26,8 +26,8 @@ from datetime import datetime -from ably.realtime.connection import ConnectionState -from ably.types.channelstate import ChannelState +from ably.pubsub.realtime.connection import ConnectionState +from ably.pubsub.types.channelstate import ChannelState from test.uts.helpers.client import ( await_channel_state, await_connection_state, diff --git a/test/uts/realtime/integration/proxy/rest_faults_test.py b/test/uts/realtime/integration/proxy/rest_faults_test.py index fa8d2c1a..19deaeea 100644 --- a/test/uts/realtime/integration/proxy/rest_faults_test.py +++ b/test/uts/realtime/integration/proxy/rest_faults_test.py @@ -25,10 +25,10 @@ import pytest -from ably import AblyRest -from ably.realtime.connection import ConnectionState -from ably.types.channelstate import ChannelState -from ably.util.exceptions import AblyException +from ably.pubsub.realtime.connection import ConnectionState +from ably.pubsub.server import AblyRest +from ably.pubsub.types.channelstate import ChannelState +from ably.pubsub.util.exceptions import AblyException from test.uts.helpers.client import ( await_channel_state, await_connection_state, diff --git a/test/uts/realtime/unit/auth/auth_callback_errors_test.py b/test/uts/realtime/unit/auth/auth_callback_errors_test.py index 6b941e92..7f056457 100644 --- a/test/uts/realtime/unit/auth/auth_callback_errors_test.py +++ b/test/uts/realtime/unit/auth/auth_callback_errors_test.py @@ -8,10 +8,10 @@ import pytest -from ably.realtime.connection import ConnectionState -from ably.transport.websockettransport import ProtocolMessageAction -from ably.types.tokendetails import TokenDetails -from ably.util.exceptions import AblyException +from ably.pubsub.realtime.connection import ConnectionState +from ably.pubsub.transport.websockettransport import ProtocolMessageAction +from ably.pubsub.types.tokendetails import TokenDetails +from ably.pubsub.util.exceptions import AblyException from test.uts.helpers.client import await_connection_state, poll_until, realtime_client, rest_client from test.uts.helpers.clock import FakeClock, settle from test.uts.helpers.deviations import deviation diff --git a/test/uts/realtime/unit/auth/connection_auth_test.py b/test/uts/realtime/unit/auth/connection_auth_test.py index 8beaa42f..a1c32b77 100644 --- a/test/uts/realtime/unit/auth/connection_auth_test.py +++ b/test/uts/realtime/unit/auth/connection_auth_test.py @@ -5,10 +5,10 @@ import time -from ably.realtime.connection import ConnectionState -from ably.transport.websockettransport import ProtocolMessageAction -from ably.types.tokendetails import TokenDetails -from ably.util.exceptions import AblyException +from ably.pubsub.realtime.connection import ConnectionState +from ably.pubsub.transport.websockettransport import ProtocolMessageAction +from ably.pubsub.types.tokendetails import TokenDetails +from ably.pubsub.util.exceptions import AblyException from test.uts.helpers.client import await_connection_state, poll_until, realtime_client from test.uts.helpers.deviations import deviation from test.uts.helpers.mock_websocket import MockWebSocket, connected_message diff --git a/test/uts/realtime/unit/auth/realtime_authorize_test.py b/test/uts/realtime/unit/auth/realtime_authorize_test.py index 568611c0..aa9589bf 100644 --- a/test/uts/realtime/unit/auth/realtime_authorize_test.py +++ b/test/uts/realtime/unit/auth/realtime_authorize_test.py @@ -8,11 +8,11 @@ import pytest -from ably.realtime.channel import ChannelState -from ably.realtime.connection import ConnectionEvent, ConnectionState -from ably.transport.websockettransport import ProtocolMessageAction -from ably.types.tokendetails import TokenDetails -from ably.util.exceptions import AblyException +from ably.pubsub.realtime.channel import ChannelState +from ably.pubsub.realtime.connection import ConnectionEvent, ConnectionState +from ably.pubsub.transport.websockettransport import ProtocolMessageAction +from ably.pubsub.types.tokendetails import TokenDetails +from ably.pubsub.util.exceptions import AblyException from test.uts.helpers.client import ( await_channel_state, await_connection_state, diff --git a/test/uts/realtime/unit/auth/token_expiry_non_renewable_test.py b/test/uts/realtime/unit/auth/token_expiry_non_renewable_test.py index 2c36c07e..c8dbaab6 100644 --- a/test/uts/realtime/unit/auth/token_expiry_non_renewable_test.py +++ b/test/uts/realtime/unit/auth/token_expiry_non_renewable_test.py @@ -5,7 +5,7 @@ import logging -from ably.realtime.connection import ConnectionState +from ably.pubsub.realtime.connection import ConnectionState from test.uts.helpers.client import await_connection_state, realtime_client from test.uts.helpers.deviations import deviation from test.uts.helpers.mock_websocket import ERROR_MESSAGE, MockWebSocket, connected_message diff --git a/test/uts/realtime/unit/channels/channel_additional_attached_test.py b/test/uts/realtime/unit/channels/channel_additional_attached_test.py index eb9b64e9..3cd559f7 100644 --- a/test/uts/realtime/unit/channels/channel_additional_attached_test.py +++ b/test/uts/realtime/unit/channels/channel_additional_attached_test.py @@ -3,10 +3,10 @@ Spec points: RTL12 """ -from ably.realtime.connection import ConnectionState -from ably.transport.websockettransport import ProtocolMessageAction -from ably.types.channelstate import ChannelState -from ably.types.flags import Flag +from ably.pubsub.realtime.connection import ConnectionState +from ably.pubsub.transport.websockettransport import ProtocolMessageAction +from ably.pubsub.types.channelstate import ChannelState +from ably.pubsub.types.flags import Flag from test.uts.helpers.client import await_connection_state, realtime_client from test.uts.helpers.clock import settle from test.uts.helpers.mock_websocket import MockWebSocket, connected_message diff --git a/test/uts/realtime/unit/channels/channel_annotations_test.py b/test/uts/realtime/unit/channels/channel_annotations_test.py index 61fa13f5..b70a1e4a 100644 --- a/test/uts/realtime/unit/channels/channel_annotations_test.py +++ b/test/uts/realtime/unit/channels/channel_annotations_test.py @@ -21,13 +21,13 @@ import pytest -from ably.realtime.annotations import RealtimeAnnotations -from ably.realtime.connection import ConnectionState -from ably.transport.websockettransport import ProtocolMessageAction -from ably.types.annotation import Annotation, AnnotationAction -from ably.types.channelstate import ChannelState -from ably.types.flags import Flag -from ably.util.exceptions import AblyException +from ably.pubsub.realtime.annotations import RealtimeAnnotations +from ably.pubsub.realtime.connection import ConnectionState +from ably.pubsub.transport.websockettransport import ProtocolMessageAction +from ably.pubsub.types.annotation import Annotation, AnnotationAction +from ably.pubsub.types.channelstate import ChannelState +from ably.pubsub.types.flags import Flag +from ably.pubsub.util.exceptions import AblyException from test.uts.helpers.client import await_connection_state, poll_until, realtime_client from test.uts.helpers.clock import settle from test.uts.helpers.mock_websocket import MockWebSocket, attached_message, connected_message @@ -378,7 +378,7 @@ async def test_rtan4e_subscribe_warns_no_mode(caplog): mock_ws = annotating_mock(channel_name, flags=Flag.PUBLISH) channel = await attached_channel(mock_ws, channel_name) - with caplog.at_level('WARNING', logger='ably.realtime.annotations'): + with caplog.at_level('WARNING', logger='ably.pubsub.realtime.annotations'): await channel.annotations.subscribe(lambda annotation: None) warnings = [record.getMessage() for record in caplog.records if record.levelname == 'WARNING'] @@ -402,7 +402,7 @@ async def test_rtan4e1_no_warn_unattached(caplog): assert channel.state == ChannelState.INITIALIZED - with caplog.at_level('WARNING', logger='ably.realtime.annotations'): + with caplog.at_level('WARNING', logger='ably.pubsub.realtime.annotations'): subscribing = asyncio.ensure_future(channel.annotations.subscribe(lambda annotation: None)) await poll_until(lambda: channel.state == ChannelState.ATTACHING, description='the implicit attach is in flight') diff --git a/test/uts/realtime/unit/channels/channel_attach_test.py b/test/uts/realtime/unit/channels/channel_attach_test.py index 1be3771c..8f6b05d7 100644 --- a/test/uts/realtime/unit/channels/channel_attach_test.py +++ b/test/uts/realtime/unit/channels/channel_attach_test.py @@ -8,13 +8,13 @@ import pytest -from ably.realtime.connection import ConnectionState -from ably.transport.websockettransport import ProtocolMessageAction -from ably.types.channelmode import ChannelMode -from ably.types.channeloptions import ChannelOptions -from ably.types.channelstate import ChannelState -from ably.types.flags import Flag -from ably.util.exceptions import AblyException +from ably.pubsub.realtime.connection import ConnectionState +from ably.pubsub.transport.websockettransport import ProtocolMessageAction +from ably.pubsub.types.channelmode import ChannelMode +from ably.pubsub.types.channeloptions import ChannelOptions +from ably.pubsub.types.channelstate import ChannelState +from ably.pubsub.types.flags import Flag +from ably.pubsub.util.exceptions import AblyException from test.uts.helpers.client import ( await_channel_state, await_connection_state, diff --git a/test/uts/realtime/unit/channels/channel_attributes_test.py b/test/uts/realtime/unit/channels/channel_attributes_test.py index 78d09e00..aeeeaff2 100644 --- a/test/uts/realtime/unit/channels/channel_attributes_test.py +++ b/test/uts/realtime/unit/channels/channel_attributes_test.py @@ -7,10 +7,10 @@ import pytest -from ably.realtime.connection import ConnectionState -from ably.transport.websockettransport import ProtocolMessageAction -from ably.types.channelstate import ChannelState -from ably.util.exceptions import AblyException +from ably.pubsub.realtime.connection import ConnectionState +from ably.pubsub.transport.websockettransport import ProtocolMessageAction +from ably.pubsub.types.channelstate import ChannelState +from ably.pubsub.util.exceptions import AblyException from test.uts.helpers.client import ( await_channel_state, await_connection_state, diff --git a/test/uts/realtime/unit/channels/channel_connection_state_test.py b/test/uts/realtime/unit/channels/channel_connection_state_test.py index b2b65f50..d1b2a947 100644 --- a/test/uts/realtime/unit/channels/channel_connection_state_test.py +++ b/test/uts/realtime/unit/channels/channel_connection_state_test.py @@ -7,10 +7,10 @@ import pytest -from ably.realtime.connection import ConnectionState -from ably.transport.websockettransport import ProtocolMessageAction -from ably.types.channelstate import ChannelState -from ably.util.exceptions import AblyException +from ably.pubsub.realtime.connection import ConnectionState +from ably.pubsub.transport.websockettransport import ProtocolMessageAction +from ably.pubsub.types.channelstate import ChannelState +from ably.pubsub.util.exceptions import AblyException from test.uts.helpers.client import ( await_channel_state, await_connection_state, diff --git a/test/uts/realtime/unit/channels/channel_delta_decoding_test.py b/test/uts/realtime/unit/channels/channel_delta_decoding_test.py index fa71c466..c3a3262e 100644 --- a/test/uts/realtime/unit/channels/channel_delta_decoding_test.py +++ b/test/uts/realtime/unit/channels/channel_delta_decoding_test.py @@ -22,10 +22,10 @@ import base64 import uuid -from ably.realtime.connection import ConnectionState -from ably.transport.websockettransport import ProtocolMessageAction -from ably.types.channelstate import ChannelState -from ably.types.options import VCDiffDecoder +from ably.pubsub.realtime.connection import ConnectionState +from ably.pubsub.transport.websockettransport import ProtocolMessageAction +from ably.pubsub.types.channelstate import ChannelState +from ably.pubsub.types.options import VCDiffDecoder from test.uts.helpers.client import await_connection_state, poll_until, realtime_client from test.uts.helpers.clock import settle from test.uts.helpers.deviations import deviation diff --git a/test/uts/realtime/unit/channels/channel_detach_test.py b/test/uts/realtime/unit/channels/channel_detach_test.py index 90c8b397..e5a2a17d 100644 --- a/test/uts/realtime/unit/channels/channel_detach_test.py +++ b/test/uts/realtime/unit/channels/channel_detach_test.py @@ -7,10 +7,10 @@ import pytest -from ably.realtime.connection import ConnectionState -from ably.transport.websockettransport import ProtocolMessageAction -from ably.types.channelstate import ChannelState -from ably.util.exceptions import AblyException +from ably.pubsub.realtime.connection import ConnectionState +from ably.pubsub.transport.websockettransport import ProtocolMessageAction +from ably.pubsub.types.channelstate import ChannelState +from ably.pubsub.util.exceptions import AblyException from test.uts.helpers.client import ( await_channel_state, await_connection_state, diff --git a/test/uts/realtime/unit/channels/channel_error_test.py b/test/uts/realtime/unit/channels/channel_error_test.py index b135966b..268eca60 100644 --- a/test/uts/realtime/unit/channels/channel_error_test.py +++ b/test/uts/realtime/unit/channels/channel_error_test.py @@ -7,10 +7,10 @@ import pytest -from ably.realtime.connection import ConnectionState -from ably.transport.websockettransport import ProtocolMessageAction -from ably.types.channelstate import ChannelState -from ably.util.exceptions import AblyException +from ably.pubsub.realtime.connection import ConnectionState +from ably.pubsub.transport.websockettransport import ProtocolMessageAction +from ably.pubsub.types.channelstate import ChannelState +from ably.pubsub.util.exceptions import AblyException from test.uts.helpers.client import ( await_channel_state, await_connection_state, diff --git a/test/uts/realtime/unit/channels/channel_get_message_test.py b/test/uts/realtime/unit/channels/channel_get_message_test.py index 7a2cb2ab..4a03fbed 100644 --- a/test/uts/realtime/unit/channels/channel_get_message_test.py +++ b/test/uts/realtime/unit/channels/channel_get_message_test.py @@ -10,7 +10,7 @@ import uuid -from ably.types.message import Message +from ably.pubsub.types.message import Message from test.uts.helpers.client import realtime_client from test.uts.helpers.mock_http import MockHttpClient diff --git a/test/uts/realtime/unit/channels/channel_history_test.py b/test/uts/realtime/unit/channels/channel_history_test.py index 8f5647b2..cee3ad13 100644 --- a/test/uts/realtime/unit/channels/channel_history_test.py +++ b/test/uts/realtime/unit/channels/channel_history_test.py @@ -14,11 +14,11 @@ import pytest -from ably.http.paginatedresult import PaginatedResult -from ably.realtime.connection import ConnectionState -from ably.transport.websockettransport import ProtocolMessageAction -from ably.types.channelstate import ChannelState -from ably.util.exceptions import AblyException +from ably.pubsub.http.paginatedresult import PaginatedResult +from ably.pubsub.realtime.connection import ConnectionState +from ably.pubsub.transport.websockettransport import ProtocolMessageAction +from ably.pubsub.types.channelstate import ChannelState +from ably.pubsub.util.exceptions import AblyException from test.uts.helpers.client import await_connection_state, realtime_client from test.uts.helpers.deviations import deviation from test.uts.helpers.mock_http import MockHttpClient diff --git a/test/uts/realtime/unit/channels/channel_message_versions_test.py b/test/uts/realtime/unit/channels/channel_message_versions_test.py index ffac77fd..6a72b12a 100644 --- a/test/uts/realtime/unit/channels/channel_message_versions_test.py +++ b/test/uts/realtime/unit/channels/channel_message_versions_test.py @@ -10,8 +10,8 @@ import uuid -from ably.http.paginatedresult import PaginatedResult -from ably.types.message import Message, MessageAction +from ably.pubsub.http.paginatedresult import PaginatedResult +from ably.pubsub.types.message import Message, MessageAction from test.uts.helpers.client import realtime_client from test.uts.helpers.mock_http import MockHttpClient diff --git a/test/uts/realtime/unit/channels/channel_options_test.py b/test/uts/realtime/unit/channels/channel_options_test.py index 43284aa8..3da5dd78 100644 --- a/test/uts/realtime/unit/channels/channel_options_test.py +++ b/test/uts/realtime/unit/channels/channel_options_test.py @@ -15,13 +15,13 @@ import pytest -from ably.realtime.connection import ConnectionState -from ably.transport.websockettransport import ProtocolMessageAction -from ably.types.channelmode import ChannelMode -from ably.types.channeloptions import ChannelOptions -from ably.types.channelstate import ChannelState -from ably.util.crypto import get_default_params -from ably.util.exceptions import AblyException +from ably.pubsub.realtime.connection import ConnectionState +from ably.pubsub.transport.websockettransport import ProtocolMessageAction +from ably.pubsub.types.channelmode import ChannelMode +from ably.pubsub.types.channeloptions import ChannelOptions +from ably.pubsub.types.channelstate import ChannelState +from ably.pubsub.util.crypto import get_default_params +from ably.pubsub.util.exceptions import AblyException from test.uts.helpers.client import ( await_channel_state, await_connection_state, @@ -106,7 +106,7 @@ async def test_tb2d_options_with_modes(): async def test_tb3_with_cipher_key(): # DEVIATION: TB3's `withCipherKey` constructor is absent from `ChannelOptions` # (`ably/types/channeloptions.py`), which takes a `CipherParams` and offers no factory - # that builds one from a key. `ably.util.crypto.get_default_params({'key': key})` is the + # that builds one from a key. `ably.pubsub.util.crypto.get_default_params({'key': key})` is the # nearest equivalent, and it is not on ChannelOptions. options = ChannelOptions.with_cipher_key(CIPHER_KEY) @@ -261,7 +261,7 @@ async def test_rts5a_creates_derived_channel(): # `DeriveOptions`. `Channels.__getattr__` (`ably/rest/channel.py:408`) answers any # unknown attribute with a channel of that name, so the call raises # `TypeError: 'RealtimeChannel' object is not callable` rather than AttributeError. - from ably import DeriveOptions + from ably.pubsub.server import DeriveOptions base_channel_name = 'test-RTS5a' client = realtime_client() @@ -278,7 +278,7 @@ async def test_rts5a_creates_derived_channel(): @deviation async def test_rts5a1_filter_base64_encoded(): # DEVIATION: derived channels are absent; see the RTS5a test. - from ably import DeriveOptions + from ably.pubsub.server import DeriveOptions base_channel_name = 'test-RTS5a1' client = realtime_client() @@ -296,7 +296,7 @@ async def test_rts5a1_filter_base64_encoded(): @deviation async def test_rts5a2_derived_with_params(): # DEVIATION: derived channels are absent; see the RTS5a test. - from ably import DeriveOptions + from ably.pubsub.server import DeriveOptions base_channel_name = 'test-RTS5a2' client = realtime_client() @@ -323,7 +323,7 @@ async def test_rts5a2_derived_with_params(): async def test_rts5_get_derived_with_options(): # DEVIATION: derived channels are absent; see the RTS5a test. `attachOnSubscribe` is # absent too; see the TB4 test. - from ably import DeriveOptions + from ably.pubsub.server import DeriveOptions base_channel_name = 'test-RTS5' client = realtime_client() @@ -341,7 +341,7 @@ async def test_rts5_get_derived_with_options(): @deviation async def test_do2a_filter_attribute(): # DEVIATION: `DeriveOptions` is absent from the library, so the import fails. - from ably import DeriveOptions + from ably.pubsub.server import DeriveOptions derive_options = DeriveOptions(filter="name == 'event' && data.count > 10") diff --git a/test/uts/realtime/unit/channels/channel_properties_test.py b/test/uts/realtime/unit/channels/channel_properties_test.py index dfa0bcff..212f6f79 100644 --- a/test/uts/realtime/unit/channels/channel_properties_test.py +++ b/test/uts/realtime/unit/channels/channel_properties_test.py @@ -10,10 +10,10 @@ import asyncio -from ably.realtime.connection import ConnectionState -from ably.transport.websockettransport import ProtocolMessageAction -from ably.types.channelstate import ChannelState -from ably.types.flags import Flag +from ably.pubsub.realtime.connection import ConnectionState +from ably.pubsub.transport.websockettransport import ProtocolMessageAction +from ably.pubsub.types.channelstate import ChannelState +from ably.pubsub.types.flags import Flag from test.uts.helpers.client import ( await_channel_state, await_connection_state, diff --git a/test/uts/realtime/unit/channels/channel_publish_pending_test.py b/test/uts/realtime/unit/channels/channel_publish_pending_test.py index aa5e6e20..337edfd2 100644 --- a/test/uts/realtime/unit/channels/channel_publish_pending_test.py +++ b/test/uts/realtime/unit/channels/channel_publish_pending_test.py @@ -15,11 +15,11 @@ import pytest -from ably.realtime.channel import ChannelState -from ably.realtime.connection import ConnectionState -from ably.transport.websockettransport import ProtocolMessageAction -from ably.types.operations import PublishResult -from ably.util.exceptions import AblyException +from ably.pubsub.realtime.channel import ChannelState +from ably.pubsub.realtime.connection import ConnectionState +from ably.pubsub.transport.websockettransport import ProtocolMessageAction +from ably.pubsub.types.operations import PublishResult +from ably.pubsub.util.exceptions import AblyException from test.uts.helpers.client import ( STATE_TIMEOUT, await_channel_state, diff --git a/test/uts/realtime/unit/channels/channel_publish_test.py b/test/uts/realtime/unit/channels/channel_publish_test.py index 73d4ac41..a68df593 100644 --- a/test/uts/realtime/unit/channels/channel_publish_test.py +++ b/test/uts/realtime/unit/channels/channel_publish_test.py @@ -25,12 +25,12 @@ import msgpack import pytest -from ably.realtime.channel import ChannelState -from ably.realtime.connection import ConnectionState -from ably.transport.websockettransport import ProtocolMessageAction -from ably.types.message import Message -from ably.types.operations import PublishResult -from ably.util.exceptions import AblyException +from ably.pubsub.realtime.channel import ChannelState +from ably.pubsub.realtime.connection import ConnectionState +from ably.pubsub.transport.websockettransport import ProtocolMessageAction +from ably.pubsub.types.message import Message +from ably.pubsub.types.operations import PublishResult +from ably.pubsub.util.exceptions import AblyException from test.uts.helpers.client import ( await_channel_state, await_connection_state, diff --git a/test/uts/realtime/unit/channels/channel_server_initiated_detach_test.py b/test/uts/realtime/unit/channels/channel_server_initiated_detach_test.py index f531f587..fc681546 100644 --- a/test/uts/realtime/unit/channels/channel_server_initiated_detach_test.py +++ b/test/uts/realtime/unit/channels/channel_server_initiated_detach_test.py @@ -7,10 +7,10 @@ import pytest -from ably.realtime.connection import ConnectionState -from ably.transport.websockettransport import ProtocolMessageAction -from ably.types.channelstate import ChannelState -from ably.util.exceptions import AblyException +from ably.pubsub.realtime.connection import ConnectionState +from ably.pubsub.transport.websockettransport import ProtocolMessageAction +from ably.pubsub.types.channelstate import ChannelState +from ably.pubsub.util.exceptions import AblyException from test.uts.helpers.client import await_connection_state, poll_until, realtime_client from test.uts.helpers.clock import FakeClock, settle from test.uts.helpers.deviations import deviation diff --git a/test/uts/realtime/unit/channels/channel_state_events_test.py b/test/uts/realtime/unit/channels/channel_state_events_test.py index 8d800808..eb5ad4ac 100644 --- a/test/uts/realtime/unit/channels/channel_state_events_test.py +++ b/test/uts/realtime/unit/channels/channel_state_events_test.py @@ -7,11 +7,11 @@ import pytest -from ably.realtime.connection import ConnectionState -from ably.transport.websockettransport import ProtocolMessageAction -from ably.types.channelstate import ChannelState, ChannelStateChange -from ably.types.flags import Flag -from ably.util.exceptions import AblyException +from ably.pubsub.realtime.connection import ConnectionState +from ably.pubsub.transport.websockettransport import ProtocolMessageAction +from ably.pubsub.types.channelstate import ChannelState, ChannelStateChange +from ably.pubsub.types.flags import Flag +from ably.pubsub.util.exceptions import AblyException from test.uts.helpers.client import await_connection_state, realtime_client from test.uts.helpers.clock import settle from test.uts.helpers.deviations import deviation diff --git a/test/uts/realtime/unit/channels/channel_subscribe_test.py b/test/uts/realtime/unit/channels/channel_subscribe_test.py index 354ff5d3..4361a72e 100644 --- a/test/uts/realtime/unit/channels/channel_subscribe_test.py +++ b/test/uts/realtime/unit/channels/channel_subscribe_test.py @@ -8,10 +8,10 @@ import pytest -from ably.realtime.connection import ConnectionState -from ably.transport.websockettransport import ProtocolMessageAction -from ably.types.channelstate import ChannelState -from ably.util.exceptions import AblyException +from ably.pubsub.realtime.connection import ConnectionState +from ably.pubsub.transport.websockettransport import ProtocolMessageAction +from ably.pubsub.types.channelstate import ChannelState +from ably.pubsub.util.exceptions import AblyException from test.uts.helpers.client import ( await_channel_state, await_connection_state, @@ -46,7 +46,7 @@ def message_protocol_message(channel_name, messages, **fields): def message_filter(**criteria): """The specification's `MessageFilter` (MFI1), built from `criteria`.""" - from ably.types.messagefilter import MessageFilter + from ably.pubsub.types.messagefilter import MessageFilter return MessageFilter(**criteria) @@ -285,7 +285,7 @@ def accept_attach(msg): # UTS: realtime/unit/RTL7h/no-attach-on-subscribe-0 @deviation async def test_rtl7h_no_attach_on_subscribe(): - from ably.types.channeloptions import ChannelOptions + from ably.pubsub.types.channeloptions import ChannelOptions channel_name = 'test-RTL7h' attach_messages = [] diff --git a/test/uts/realtime/unit/channels/channel_update_delete_message_test.py b/test/uts/realtime/unit/channels/channel_update_delete_message_test.py index 6af97755..80867ef0 100644 --- a/test/uts/realtime/unit/channels/channel_update_delete_message_test.py +++ b/test/uts/realtime/unit/channels/channel_update_delete_message_test.py @@ -12,11 +12,11 @@ import pytest -from ably.realtime.connection import ConnectionState -from ably.transport.websockettransport import ProtocolMessageAction -from ably.types.message import Message, MessageAction -from ably.types.operations import MessageOperation, UpdateDeleteResult -from ably.util.exceptions import AblyException +from ably.pubsub.realtime.connection import ConnectionState +from ably.pubsub.transport.websockettransport import ProtocolMessageAction +from ably.pubsub.types.message import Message, MessageAction +from ably.pubsub.types.operations import MessageOperation, UpdateDeleteResult +from ably.pubsub.util.exceptions import AblyException from test.uts.helpers.client import await_connection_state, realtime_client from test.uts.helpers.mock_websocket import MockWebSocket, attached_message, connected_message diff --git a/test/uts/realtime/unit/channels/channel_when_state_test.py b/test/uts/realtime/unit/channels/channel_when_state_test.py index 76bd651e..252901f2 100644 --- a/test/uts/realtime/unit/channels/channel_when_state_test.py +++ b/test/uts/realtime/unit/channels/channel_when_state_test.py @@ -5,9 +5,9 @@ import asyncio -from ably.realtime.connection import ConnectionState -from ably.transport.websockettransport import ProtocolMessageAction -from ably.types.channelstate import ChannelState +from ably.pubsub.realtime.connection import ConnectionState +from ably.pubsub.transport.websockettransport import ProtocolMessageAction +from ably.pubsub.types.channelstate import ChannelState from test.uts.helpers.client import await_connection_state, realtime_client from test.uts.helpers.clock import settle from test.uts.helpers.deviations import deviation diff --git a/test/uts/realtime/unit/channels/channels_collection_test.py b/test/uts/realtime/unit/channels/channels_collection_test.py index 3498ee12..e455a584 100644 --- a/test/uts/realtime/unit/channels/channels_collection_test.py +++ b/test/uts/realtime/unit/channels/channels_collection_test.py @@ -11,11 +11,11 @@ import asyncio -from ably.realtime.channel import Channels as RealtimeChannels -from ably.realtime.channel import RealtimeChannel -from ably.realtime.connection import ConnectionState -from ably.transport.websockettransport import ProtocolMessageAction -from ably.types.channelstate import ChannelState +from ably.pubsub.realtime.channel import Channels as RealtimeChannels +from ably.pubsub.realtime.channel import RealtimeChannel +from ably.pubsub.realtime.connection import ConnectionState +from ably.pubsub.transport.websockettransport import ProtocolMessageAction +from ably.pubsub.types.channelstate import ChannelState from test.uts.helpers.client import await_connection_state, poll_until, realtime_client from test.uts.helpers.deviations import deviation from test.uts.helpers.mock_websocket import ( diff --git a/test/uts/realtime/unit/channels/message_field_population_test.py b/test/uts/realtime/unit/channels/message_field_population_test.py index 8efcd7df..b358442e 100644 --- a/test/uts/realtime/unit/channels/message_field_population_test.py +++ b/test/uts/realtime/unit/channels/message_field_population_test.py @@ -3,8 +3,8 @@ Spec points: TM2a, TM2c, TM2f """ -from ably.realtime.connection import ConnectionState -from ably.transport.websockettransport import ProtocolMessageAction +from ably.pubsub.realtime.connection import ConnectionState +from ably.pubsub.transport.websockettransport import ProtocolMessageAction from test.uts.helpers.client import ( await_connection_state, poll_until, diff --git a/test/uts/realtime/unit/client/realtime_client_test.py b/test/uts/realtime/unit/client/realtime_client_test.py index 2e765a4c..088a1607 100644 --- a/test/uts/realtime/unit/client/realtime_client_test.py +++ b/test/uts/realtime/unit/client/realtime_client_test.py @@ -8,13 +8,13 @@ import pytest -from ably.realtime.channel import Channels, RealtimeChannel -from ably.realtime.connection import Connection, ConnectionState -from ably.rest.auth import Auth -from ably.rest.push import Push, PushAdmin -from ably.transport.websockettransport import ProtocolMessageAction -from ably.util.exceptions import AblyException -from ably.util.helper import get_random_id +from ably.pubsub.realtime.channel import Channels, RealtimeChannel +from ably.pubsub.realtime.connection import Connection, ConnectionState +from ably.pubsub.rest.auth import Auth +from ably.pubsub.rest.push import Push, PushAdmin +from ably.pubsub.transport.websockettransport import ProtocolMessageAction +from ably.pubsub.util.exceptions import AblyException +from ably.pubsub.util.helper import get_random_id from test.uts.helpers.client import await_connection_state, realtime_client from test.uts.helpers.clock import settle from test.uts.helpers.deviations import deviation diff --git a/test/uts/realtime/unit/client/realtime_stats_test.py b/test/uts/realtime/unit/client/realtime_stats_test.py index ad9d8a44..77f8d073 100644 --- a/test/uts/realtime/unit/client/realtime_stats_test.py +++ b/test/uts/realtime/unit/client/realtime_stats_test.py @@ -3,7 +3,7 @@ Spec points: RTC5, RTC5a, RTC5b """ -from ably.http.paginatedresult import PaginatedResult +from ably.pubsub.http.paginatedresult import PaginatedResult from test.uts.helpers.client import realtime_client from test.uts.helpers.mock_http import MockHttpClient diff --git a/test/uts/realtime/unit/client/realtime_timeouts_test.py b/test/uts/realtime/unit/client/realtime_timeouts_test.py index 6b0d3eba..3a9c21d9 100644 --- a/test/uts/realtime/unit/client/realtime_timeouts_test.py +++ b/test/uts/realtime/unit/client/realtime_timeouts_test.py @@ -7,11 +7,11 @@ import pytest -from ably.realtime.channel import ChannelState -from ably.realtime.connection import ConnectionState -from ably.transport.websockettransport import ProtocolMessageAction -from ably.util.exceptions import AblyException -from ably.util.helper import get_random_id +from ably.pubsub.realtime.channel import ChannelState +from ably.pubsub.realtime.connection import ConnectionState +from ably.pubsub.transport.websockettransport import ProtocolMessageAction +from ably.pubsub.util.exceptions import AblyException +from ably.pubsub.util.helper import get_random_id from test.uts.helpers.client import await_connection_state, realtime_client from test.uts.helpers.clock import FakeClock, settle from test.uts.helpers.mock_http import MockHttpClient diff --git a/test/uts/realtime/unit/connection/auto_connect_test.py b/test/uts/realtime/unit/connection/auto_connect_test.py index 29e8e12f..f77c0c39 100644 --- a/test/uts/realtime/unit/connection/auto_connect_test.py +++ b/test/uts/realtime/unit/connection/auto_connect_test.py @@ -3,7 +3,7 @@ Spec points: RTN3 """ -from ably.realtime.connection import ConnectionState +from ably.pubsub.realtime.connection import ConnectionState from test.uts.helpers.client import await_connection_state, realtime_client from test.uts.helpers.clock import settle from test.uts.helpers.mock_websocket import MockWebSocket, connected_message diff --git a/test/uts/realtime/unit/connection/backoff_jitter_test.py b/test/uts/realtime/unit/connection/backoff_jitter_test.py index 35b865b5..eee5a880 100644 --- a/test/uts/realtime/unit/connection/backoff_jitter_test.py +++ b/test/uts/realtime/unit/connection/backoff_jitter_test.py @@ -12,9 +12,9 @@ exact rather than sampled. """ -from ably.realtime.channel import ChannelState -from ably.realtime.connection import ConnectionState -from ably.transport.websockettransport import ProtocolMessageAction +from ably.pubsub.realtime.channel import ChannelState +from ably.pubsub.realtime.connection import ConnectionState +from ably.pubsub.transport.websockettransport import ProtocolMessageAction from test.uts.helpers.client import await_connection_state, realtime_client from test.uts.helpers.clock import FakeClock, settle from test.uts.helpers.deviations import deviation diff --git a/test/uts/realtime/unit/connection/connection_failures_test.py b/test/uts/realtime/unit/connection/connection_failures_test.py index b2ea02c7..ffbcd2c8 100644 --- a/test/uts/realtime/unit/connection/connection_failures_test.py +++ b/test/uts/realtime/unit/connection/connection_failures_test.py @@ -5,8 +5,8 @@ import time -from ably.realtime.connection import ConnectionState -from ably.types.tokendetails import TokenDetails +from ably.pubsub.realtime.connection import ConnectionState +from ably.pubsub.types.tokendetails import TokenDetails from test.uts.helpers.client import await_connection_state, realtime_client from test.uts.helpers.clock import FakeClock, settle from test.uts.helpers.deviations import deviation diff --git a/test/uts/realtime/unit/connection/connection_id_key_test.py b/test/uts/realtime/unit/connection/connection_id_key_test.py index adfcd9aa..32f1ecbc 100644 --- a/test/uts/realtime/unit/connection/connection_id_key_test.py +++ b/test/uts/realtime/unit/connection/connection_id_key_test.py @@ -3,8 +3,8 @@ Spec points: RTN8, RTN8a, RTN8b, RTN8d, RTN9, RTN9a, RTN9b, RTN9d """ -from ably.realtime.connection import ConnectionState -from ably.transport.websockettransport import ProtocolMessageAction +from ably.pubsub.realtime.connection import ConnectionState +from ably.pubsub.transport.websockettransport import ProtocolMessageAction from test.uts.helpers.client import await_connection_state, realtime_client from test.uts.helpers.clock import FakeClock, settle from test.uts.helpers.deviations import deviation diff --git a/test/uts/realtime/unit/connection/connection_open_failures_test.py b/test/uts/realtime/unit/connection/connection_open_failures_test.py index af1e7393..0224e5b1 100644 --- a/test/uts/realtime/unit/connection/connection_open_failures_test.py +++ b/test/uts/realtime/unit/connection/connection_open_failures_test.py @@ -5,8 +5,8 @@ import time -from ably.realtime.connection import ConnectionState -from ably.types.tokendetails import TokenDetails +from ably.pubsub.realtime.connection import ConnectionState +from ably.pubsub.types.tokendetails import TokenDetails from test.uts.helpers.client import await_connection_state, realtime_client from test.uts.helpers.clock import FakeClock, settle from test.uts.helpers.mock_http import MockHttpClient diff --git a/test/uts/realtime/unit/connection/connection_ping_test.py b/test/uts/realtime/unit/connection/connection_ping_test.py index 1c3aaad7..dc3abd62 100644 --- a/test/uts/realtime/unit/connection/connection_ping_test.py +++ b/test/uts/realtime/unit/connection/connection_ping_test.py @@ -12,8 +12,8 @@ import asyncio -from ably.realtime.connection import ConnectionState -from ably.transport.websockettransport import ProtocolMessageAction +from ably.pubsub.realtime.connection import ConnectionState +from ably.pubsub.transport.websockettransport import ProtocolMessageAction from test.uts.helpers.client import ( await_connection_state, poll_until, diff --git a/test/uts/realtime/unit/connection/connection_recovery_test.py b/test/uts/realtime/unit/connection/connection_recovery_test.py index abc92660..9684513c 100644 --- a/test/uts/realtime/unit/connection/connection_recovery_test.py +++ b/test/uts/realtime/unit/connection/connection_recovery_test.py @@ -13,9 +13,9 @@ import asyncio import json -from ably.realtime.channel import ChannelState -from ably.realtime.connection import ConnectionState -from ably.transport.websockettransport import ProtocolMessageAction +from ably.pubsub.realtime.channel import ChannelState +from ably.pubsub.realtime.connection import ConnectionState +from ably.pubsub.transport.websockettransport import ProtocolMessageAction from test.uts.helpers.client import await_channel_state, await_connection_state, realtime_client from test.uts.helpers.clock import FakeClock from test.uts.helpers.deviations import deviation diff --git a/test/uts/realtime/unit/connection/error_reason_test.py b/test/uts/realtime/unit/connection/error_reason_test.py index 2ad7bca8..a0a3622e 100644 --- a/test/uts/realtime/unit/connection/error_reason_test.py +++ b/test/uts/realtime/unit/connection/error_reason_test.py @@ -3,8 +3,8 @@ Spec points: RTN25 """ -from ably.realtime.connection import ConnectionState -from ably.transport.websockettransport import ProtocolMessageAction +from ably.pubsub.realtime.connection import ConnectionState +from ably.pubsub.transport.websockettransport import ProtocolMessageAction from test.uts.helpers.client import await_connection_state, realtime_client from test.uts.helpers.clock import FakeClock, settle from test.uts.helpers.mock_websocket import MockWebSocket, connected_message diff --git a/test/uts/realtime/unit/connection/fallback_hosts_test.py b/test/uts/realtime/unit/connection/fallback_hosts_test.py index bcad6f43..69294ba8 100644 --- a/test/uts/realtime/unit/connection/fallback_hosts_test.py +++ b/test/uts/realtime/unit/connection/fallback_hosts_test.py @@ -24,9 +24,9 @@ import httpx -from ably.realtime import connectionmanager -from ably.realtime.connection import ConnectionState -from ably.transport.defaults import Defaults +from ably.pubsub.realtime import connectionmanager +from ably.pubsub.realtime.connection import ConnectionState +from ably.pubsub.transport.defaults import Defaults from test.uts.helpers.client import await_connection_state, poll_until, realtime_client from test.uts.helpers.mock_http import MockHttpClient from test.uts.helpers.mock_websocket import MockWebSocket, connected_message diff --git a/test/uts/realtime/unit/connection/forwards_compatibility_test.py b/test/uts/realtime/unit/connection/forwards_compatibility_test.py index 78f287b1..fcf0a8db 100644 --- a/test/uts/realtime/unit/connection/forwards_compatibility_test.py +++ b/test/uts/realtime/unit/connection/forwards_compatibility_test.py @@ -8,9 +8,9 @@ fields as written and no separate method is needed. """ -from ably.realtime.channel import ChannelState -from ably.realtime.connection import ConnectionState -from ably.transport.websockettransport import ProtocolMessageAction +from ably.pubsub.realtime.channel import ChannelState +from ably.pubsub.realtime.connection import ConnectionState +from ably.pubsub.transport.websockettransport import ProtocolMessageAction from test.uts.helpers.client import await_connection_state, realtime_client from test.uts.helpers.clock import settle from test.uts.helpers.mock_websocket import HEARTBEAT_MESSAGE, MockWebSocket, connected_message diff --git a/test/uts/realtime/unit/connection/heartbeat_test.py b/test/uts/realtime/unit/connection/heartbeat_test.py index 681da166..42b0efd5 100644 --- a/test/uts/realtime/unit/connection/heartbeat_test.py +++ b/test/uts/realtime/unit/connection/heartbeat_test.py @@ -11,7 +11,7 @@ import pytest -from ably.realtime.connection import ConnectionState +from ably.pubsub.realtime.connection import ConnectionState from test.uts.helpers.client import ( await_connection_state, poll_until, diff --git a/test/uts/realtime/unit/connection/server_initiated_reauth_test.py b/test/uts/realtime/unit/connection/server_initiated_reauth_test.py index 3b0fa5c4..7c24e0f2 100644 --- a/test/uts/realtime/unit/connection/server_initiated_reauth_test.py +++ b/test/uts/realtime/unit/connection/server_initiated_reauth_test.py @@ -5,9 +5,9 @@ import time -from ably.realtime.connection import ConnectionEvent, ConnectionState -from ably.transport.websockettransport import ProtocolMessageAction -from ably.types.tokendetails import TokenDetails +from ably.pubsub.realtime.connection import ConnectionEvent, ConnectionState +from ably.pubsub.transport.websockettransport import ProtocolMessageAction +from ably.pubsub.types.tokendetails import TokenDetails from test.uts.helpers.client import await_connection_state, realtime_client from test.uts.helpers.clock import settle from test.uts.helpers.mock_websocket import MockWebSocket, connected_message diff --git a/test/uts/realtime/unit/connection/update_events_test.py b/test/uts/realtime/unit/connection/update_events_test.py index 34f51b09..cfe5790f 100644 --- a/test/uts/realtime/unit/connection/update_events_test.py +++ b/test/uts/realtime/unit/connection/update_events_test.py @@ -3,8 +3,8 @@ Spec points: RTN24 """ -from ably.realtime.connection import ConnectionState -from ably.types.connectionstate import ConnectionEvent +from ably.pubsub.realtime.connection import ConnectionState +from ably.pubsub.types.connectionstate import ConnectionEvent from test.uts.helpers.client import await_connection_state, realtime_client from test.uts.helpers.clock import settle from test.uts.helpers.deviations import deviation diff --git a/test/uts/realtime/unit/connection/when_state_test.py b/test/uts/realtime/unit/connection/when_state_test.py index 63f2f9d0..d41c4fc1 100644 --- a/test/uts/realtime/unit/connection/when_state_test.py +++ b/test/uts/realtime/unit/connection/when_state_test.py @@ -5,7 +5,7 @@ import asyncio -from ably.realtime.connection import ConnectionState +from ably.pubsub.realtime.connection import ConnectionState from test.uts.helpers.client import STATE_TIMEOUT, await_connection_state, realtime_client from test.uts.helpers.clock import settle from test.uts.helpers.mock_websocket import MockWebSocket, connected_message diff --git a/test/uts/realtime/unit/presence/local_presence_map_test.py b/test/uts/realtime/unit/presence/local_presence_map_test.py index 9e2e5ce2..1048143a 100644 --- a/test/uts/realtime/unit/presence/local_presence_map_test.py +++ b/test/uts/realtime/unit/presence/local_presence_map_test.py @@ -10,8 +10,8 @@ test/uts/deviations.md. """ -from ably.realtime.presencemap import PresenceMap -from ably.types.presence import PresenceAction +from ably.pubsub.realtime.presencemap import PresenceMap +from ably.pubsub.types.presence import PresenceAction from test.uts.helpers.deviations import deviation from test.uts.helpers.presence import presence_message, subscribed_presence diff --git a/test/uts/realtime/unit/presence/presence_map_test.py b/test/uts/realtime/unit/presence/presence_map_test.py index 7277096e..8a09145c 100644 --- a/test/uts/realtime/unit/presence/presence_map_test.py +++ b/test/uts/realtime/unit/presence/presence_map_test.py @@ -11,7 +11,7 @@ test/uts/deviations.md. """ -from ably.types.presence import PresenceAction +from ably.pubsub.types.presence import PresenceAction from test.uts.helpers.presence import presence_map, presence_message, subscribed_presence diff --git a/test/uts/realtime/unit/presence/presence_sync_test.py b/test/uts/realtime/unit/presence/presence_sync_test.py index 2e291c7d..0d228862 100644 --- a/test/uts/realtime/unit/presence/presence_sync_test.py +++ b/test/uts/realtime/unit/presence/presence_sync_test.py @@ -12,7 +12,7 @@ from datetime import datetime, timezone -from ably.types.presence import PresenceAction +from ably.pubsub.types.presence import PresenceAction from test.uts.helpers.deviations import deviation from test.uts.helpers.presence import presence_map, presence_message, subscribed_presence diff --git a/test/uts/realtime/unit/presence/realtime_presence_channel_state_test.py b/test/uts/realtime/unit/presence/realtime_presence_channel_state_test.py index 39341669..15d50b36 100644 --- a/test/uts/realtime/unit/presence/realtime_presence_channel_state_test.py +++ b/test/uts/realtime/unit/presence/realtime_presence_channel_state_test.py @@ -19,13 +19,13 @@ import pytest -from ably.realtime.connection import ConnectionState -from ably.realtime.presence import RealtimePresence -from ably.transport.websockettransport import ProtocolMessageAction -from ably.types.channelstate import ChannelState -from ably.types.flags import Flag -from ably.types.presence import PresenceAction -from ably.util.exceptions import AblyException +from ably.pubsub.realtime.connection import ConnectionState +from ably.pubsub.realtime.presence import RealtimePresence +from ably.pubsub.transport.websockettransport import ProtocolMessageAction +from ably.pubsub.types.channelstate import ChannelState +from ably.pubsub.types.flags import Flag +from ably.pubsub.types.presence import PresenceAction +from ably.pubsub.util.exceptions import AblyException from test.uts.helpers.client import ( await_channel_state, connected_client, diff --git a/test/uts/realtime/unit/presence/realtime_presence_enter_test.py b/test/uts/realtime/unit/presence/realtime_presence_enter_test.py index 01ad53e5..9168d22a 100644 --- a/test/uts/realtime/unit/presence/realtime_presence_enter_test.py +++ b/test/uts/realtime/unit/presence/realtime_presence_enter_test.py @@ -19,12 +19,12 @@ import pytest -from ably.realtime.connection import ConnectionState -from ably.transport.websockettransport import ProtocolMessageAction -from ably.types.channelstate import ChannelState -from ably.types.flags import Flag -from ably.types.presence import PresenceAction -from ably.util.exceptions import AblyException +from ably.pubsub.realtime.connection import ConnectionState +from ably.pubsub.transport.websockettransport import ProtocolMessageAction +from ably.pubsub.types.channelstate import ChannelState +from ably.pubsub.types.flags import Flag +from ably.pubsub.types.presence import PresenceAction +from ably.pubsub.util.exceptions import AblyException from test.uts.helpers.client import ( await_channel_state, await_connection_state, diff --git a/test/uts/realtime/unit/presence/realtime_presence_get_test.py b/test/uts/realtime/unit/presence/realtime_presence_get_test.py index 78fc6712..d0afbf7c 100644 --- a/test/uts/realtime/unit/presence/realtime_presence_get_test.py +++ b/test/uts/realtime/unit/presence/realtime_presence_get_test.py @@ -19,11 +19,11 @@ import pytest -from ably.realtime.connection import ConnectionState -from ably.transport.websockettransport import ProtocolMessageAction -from ably.types.channelstate import ChannelState -from ably.types.flags import Flag -from ably.util.exceptions import AblyException +from ably.pubsub.realtime.connection import ConnectionState +from ably.pubsub.transport.websockettransport import ProtocolMessageAction +from ably.pubsub.types.channelstate import ChannelState +from ably.pubsub.types.flags import Flag +from ably.pubsub.util.exceptions import AblyException from test.uts.helpers.client import ( await_channel_state, await_connection_state, diff --git a/test/uts/realtime/unit/presence/realtime_presence_history_test.py b/test/uts/realtime/unit/presence/realtime_presence_history_test.py index c9943c74..9f022378 100644 --- a/test/uts/realtime/unit/presence/realtime_presence_history_test.py +++ b/test/uts/realtime/unit/presence/realtime_presence_history_test.py @@ -14,10 +14,10 @@ import uuid -from ably.http.paginatedresult import PaginatedResult -from ably.realtime.connection import ConnectionState -from ably.transport.websockettransport import ProtocolMessageAction -from ably.types.presence import PresenceAction, PresenceMessage +from ably.pubsub.http.paginatedresult import PaginatedResult +from ably.pubsub.realtime.connection import ConnectionState +from ably.pubsub.transport.websockettransport import ProtocolMessageAction +from ably.pubsub.types.presence import PresenceAction, PresenceMessage from test.uts.helpers.client import await_connection_state, realtime_client from test.uts.helpers.deviations import deviation from test.uts.helpers.mock_http import MockHttpClient diff --git a/test/uts/realtime/unit/presence/realtime_presence_reentry_test.py b/test/uts/realtime/unit/presence/realtime_presence_reentry_test.py index 702dd77d..0de3607e 100644 --- a/test/uts/realtime/unit/presence/realtime_presence_reentry_test.py +++ b/test/uts/realtime/unit/presence/realtime_presence_reentry_test.py @@ -15,10 +15,10 @@ import asyncio import uuid -from ably.transport.websockettransport import ProtocolMessageAction -from ably.types.channelstate import ChannelState -from ably.types.flags import Flag -from ably.types.presence import PresenceAction +from ably.pubsub.transport.websockettransport import ProtocolMessageAction +from ably.pubsub.types.channelstate import ChannelState +from ably.pubsub.types.flags import Flag +from ably.pubsub.types.presence import PresenceAction from test.uts.helpers.client import await_channel_state, connected_client, poll_until from test.uts.helpers.clock import settle from test.uts.helpers.deviations import deviation diff --git a/test/uts/realtime/unit/presence/realtime_presence_subscribe_test.py b/test/uts/realtime/unit/presence/realtime_presence_subscribe_test.py index cec42e4f..66b562d5 100644 --- a/test/uts/realtime/unit/presence/realtime_presence_subscribe_test.py +++ b/test/uts/realtime/unit/presence/realtime_presence_subscribe_test.py @@ -10,12 +10,12 @@ import uuid -from ably.realtime.connection import ConnectionState -from ably.transport.websockettransport import ProtocolMessageAction -from ably.types.channeloptions import ChannelOptions -from ably.types.channelstate import ChannelState -from ably.types.flags import Flag -from ably.types.presence import PresenceAction +from ably.pubsub.realtime.connection import ConnectionState +from ably.pubsub.transport.websockettransport import ProtocolMessageAction +from ably.pubsub.types.channeloptions import ChannelOptions +from ably.pubsub.types.channelstate import ChannelState +from ably.pubsub.types.flags import Flag +from ably.pubsub.types.presence import PresenceAction from test.uts.helpers.client import ( await_channel_state, await_connection_state, diff --git a/test/uts/rest/integration/auth_test.py b/test/uts/rest/integration/auth_test.py index eac29ee2..9d7c253d 100644 --- a/test/uts/rest/integration/auth_test.py +++ b/test/uts/rest/integration/auth_test.py @@ -17,8 +17,8 @@ import pytest -from ably.transport.defaults import Defaults -from ably.util.exceptions import AblyException +from ably.pubsub.transport.defaults import Defaults +from ably.pubsub.util.exceptions import AblyException from test.uts.helpers.client import sandbox_rest_client from test.uts.helpers.deviations import deviation from test.uts.helpers.sandbox import extract_key_name, extract_key_secret, generate_jwt, random_id diff --git a/test/uts/rest/integration/batch_presence_test.py b/test/uts/rest/integration/batch_presence_test.py index 2ccb356b..4e506570 100644 --- a/test/uts/rest/integration/batch_presence_test.py +++ b/test/uts/rest/integration/batch_presence_test.py @@ -26,7 +26,7 @@ *Auth*. """ -from ably.realtime.connection import ConnectionState +from ably.pubsub.realtime.connection import ConnectionState from test.uts.helpers.client import ( await_connection_state, sandbox_realtime_client, diff --git a/test/uts/rest/integration/mutable_messages_test.py b/test/uts/rest/integration/mutable_messages_test.py index b632b628..77fd8b20 100644 --- a/test/uts/rest/integration/mutable_messages_test.py +++ b/test/uts/rest/integration/mutable_messages_test.py @@ -3,11 +3,11 @@ Spec points: RSL1n, RSL11, RSL14, RSL15, RSAN1, RSAN2, RSAN3 """ -from ably.http.paginatedresult import PaginatedResult -from ably.types.annotation import Annotation, AnnotationAction -from ably.types.message import Message, MessageAction -from ably.types.operations import MessageOperation, PublishResult, UpdateDeleteResult -from ably.util.exceptions import AblyException +from ably.pubsub.http.paginatedresult import PaginatedResult +from ably.pubsub.types.annotation import Annotation, AnnotationAction +from ably.pubsub.types.message import Message, MessageAction +from ably.pubsub.types.operations import MessageOperation, PublishResult, UpdateDeleteResult +from ably.pubsub.util.exceptions import AblyException from test.uts.helpers.client import sandbox_rest_client, wall_clock_poll_until from test.uts.helpers.sandbox import random_id diff --git a/test/uts/rest/integration/pagination_test.py b/test/uts/rest/integration/pagination_test.py index d7cacd6d..291d0c9d 100644 --- a/test/uts/rest/integration/pagination_test.py +++ b/test/uts/rest/integration/pagination_test.py @@ -3,8 +3,8 @@ Spec points: TG1, TG2, TG3, TG4, TG5 """ -from ably.http.paginatedresult import PaginatedResult -from ably.types.message import Message +from ably.pubsub.http.paginatedresult import PaginatedResult +from ably.pubsub.types.message import Message from test.uts.helpers.client import sandbox_rest_client, wall_clock_poll_until from test.uts.helpers.sandbox import random_id diff --git a/test/uts/rest/integration/presence_test.py b/test/uts/rest/integration/presence_test.py index 7f2a013e..9287fc4e 100644 --- a/test/uts/rest/integration/presence_test.py +++ b/test/uts/rest/integration/presence_test.py @@ -5,10 +5,10 @@ import pytest -from ably.http.paginatedresult import PaginatedResult -from ably.realtime.connection import ConnectionState -from ably.types.presence import Presence, PresenceAction, PresenceMessage -from ably.util.exceptions import AblyException +from ably.pubsub.http.paginatedresult import PaginatedResult +from ably.pubsub.realtime.connection import ConnectionState +from ably.pubsub.types.presence import Presence, PresenceAction, PresenceMessage +from ably.pubsub.util.exceptions import AblyException from test.uts.helpers.client import ( await_connection_state, sandbox_realtime_client, @@ -65,7 +65,7 @@ async def test_rsp1_access_presence_from_channel(sandbox, use_binary_protocol): assert presence is not None # NOTE: the spec asserts `presence IS RestPresence`. The class behind - # `channel.presence` in ably-python is `ably.types.presence.Presence`. + # `channel.presence` in ably-python is `ably.pubsub.types.presence.Presence`. assert isinstance(presence, Presence) diff --git a/test/uts/rest/integration/proxy/rest_fallback_test.py b/test/uts/rest/integration/proxy/rest_fallback_test.py index 27eb1727..fc8455b5 100644 --- a/test/uts/rest/integration/proxy/rest_fallback_test.py +++ b/test/uts/rest/integration/proxy/rest_fallback_test.py @@ -19,8 +19,8 @@ import pytest -from ably import AblyRest -from ably.util.exceptions import AblyException +from ably.pubsub.server import AblyRest +from ably.pubsub.util.exceptions import AblyException from test.uts.helpers.client import sandbox_rest_client, wall_clock_poll_until from test.uts.helpers.deviations import deviation from test.uts.helpers.sandbox import SANDBOX_ENDPOINT, random_id diff --git a/test/uts/rest/integration/publish_test.py b/test/uts/rest/integration/publish_test.py index b81ca9dc..30b817bc 100644 --- a/test/uts/rest/integration/publish_test.py +++ b/test/uts/rest/integration/publish_test.py @@ -5,8 +5,8 @@ import pytest -from ably.types.message import Message -from ably.util.exceptions import AblyException +from ably.pubsub.types.message import Message +from ably.pubsub.util.exceptions import AblyException from test.uts.helpers.client import sandbox_rest_client, wall_clock_poll_until from test.uts.helpers.sandbox import random_id diff --git a/test/uts/rest/integration/push_admin_test.py b/test/uts/rest/integration/push_admin_test.py index 5b4a5f45..81a8301b 100644 --- a/test/uts/rest/integration/push_admin_test.py +++ b/test/uts/rest/integration/push_admin_test.py @@ -6,8 +6,8 @@ import pytest -from ably import AblyException, DeviceDetails, PushChannelSubscription -from ably.http.paginatedresult import PaginatedResult +from ably.pubsub.http.paginatedresult import PaginatedResult +from ably.pubsub.server import AblyException, DeviceDetails, PushChannelSubscription from test.uts.helpers.client import sandbox_rest_client, wall_clock_poll_until from test.uts.helpers.sandbox import random_id diff --git a/test/uts/rest/integration/push_channels_test.py b/test/uts/rest/integration/push_channels_test.py index 36f0d551..aeef48f8 100644 --- a/test/uts/rest/integration/push_channels_test.py +++ b/test/uts/rest/integration/push_channels_test.py @@ -8,7 +8,7 @@ Both tests here therefore depart from the specification and are gated behind RUN_DEVIATIONS, against the same spelling [test/uts/rest/unit/push/push_channels_test.py](../unit/push/push_channels_test.py) -gates on — `ably.types.device.LocalDevice`, `client.device` and +gates on — `ably.pubsub.types.device.LocalDevice`, `client.device` and `channel.push.subscribe_device()` and friends — so that dropping the marker is the only change either tier needs when RSH7 lands. @@ -31,7 +31,7 @@ Errors*. """ -from ably import AblyException, DeviceDetails +from ably.pubsub.server import AblyException, DeviceDetails from test.uts.helpers.client import sandbox_rest_client, wall_clock_poll_until from test.uts.helpers.deviations import deviation from test.uts.helpers.sandbox import random_id @@ -55,7 +55,7 @@ def set_local_device(client, device_id, device_identity_token=PLACEHOLDER_DEVICE The import is deliberately inside the call, so that a file-level import of a name that does not exist does not take the collection of the whole package down with it. """ - from ably.types.device import LocalDevice + from ably.pubsub.types.device import LocalDevice client.device = LocalDevice( id=device_id, diff --git a/test/uts/rest/integration/revoke_tokens_test.py b/test/uts/rest/integration/revoke_tokens_test.py index 7b2adacd..c83a5da1 100644 --- a/test/uts/rest/integration/revoke_tokens_test.py +++ b/test/uts/rest/integration/revoke_tokens_test.py @@ -27,8 +27,8 @@ import pytest -from ably.realtime.connection import ConnectionState -from ably.util.exceptions import AblyException +from ably.pubsub.realtime.connection import ConnectionState +from ably.pubsub.util.exceptions import AblyException from test.uts.helpers.client import ( await_connection_state, next_connection_state, diff --git a/test/uts/rest/integration/time_stats_test.py b/test/uts/rest/integration/time_stats_test.py index 39d6e465..d38f0c84 100644 --- a/test/uts/rest/integration/time_stats_test.py +++ b/test/uts/rest/integration/time_stats_test.py @@ -8,9 +8,9 @@ import pytest_asyncio -from ably import AblyRest -from ably.http.paginatedresult import PaginatedResult -from ably.types.stats import Stats +from ably.pubsub.http.paginatedresult import PaginatedResult +from ably.pubsub.server import AblyRest +from ably.pubsub.types.stats import Stats from test.uts.helpers.client import sandbox_rest_client from test.uts.helpers.sandbox import SANDBOX_ENDPOINT diff --git a/test/uts/rest/unit/auth/auth_callback_test.py b/test/uts/rest/unit/auth/auth_callback_test.py index 235abbb0..4511aefe 100644 --- a/test/uts/rest/unit/auth/auth_callback_test.py +++ b/test/uts/rest/unit/auth/auth_callback_test.py @@ -9,10 +9,10 @@ import pytest -from ably import api_version -from ably.types.tokendetails import TokenDetails -from ably.types.tokenrequest import TokenRequest -from ably.util.exceptions import AblyException +from ably.pubsub.server import api_version +from ably.pubsub.types.tokendetails import TokenDetails +from ably.pubsub.types.tokenrequest import TokenRequest +from ably.pubsub.util.exceptions import AblyException from test.uts.helpers.client import rest_client from test.uts.helpers.mock_http import MockHttpClient diff --git a/test/uts/rest/unit/auth/auth_scheme_test.py b/test/uts/rest/unit/auth/auth_scheme_test.py index 30ed44cc..e69b732a 100644 --- a/test/uts/rest/unit/auth/auth_scheme_test.py +++ b/test/uts/rest/unit/auth/auth_scheme_test.py @@ -8,9 +8,9 @@ import pytest -from ably import api_version -from ably.types.tokendetails import TokenDetails -from ably.util.exceptions import AblyException +from ably.pubsub.server import api_version +from ably.pubsub.types.tokendetails import TokenDetails +from ably.pubsub.util.exceptions import AblyException from test.uts.helpers.client import rest_client from test.uts.helpers.deviations import spec_error from test.uts.helpers.mock_http import MockHttpClient diff --git a/test/uts/rest/unit/auth/authorize_test.py b/test/uts/rest/unit/auth/authorize_test.py index a280fc2a..5d1f50d3 100644 --- a/test/uts/rest/unit/auth/authorize_test.py +++ b/test/uts/rest/unit/auth/authorize_test.py @@ -9,9 +9,9 @@ import pytest -from ably.types.authoptions import AuthOptions -from ably.types.tokendetails import TokenDetails -from ably.util.exceptions import AblyException +from ably.pubsub.types.authoptions import AuthOptions +from ably.pubsub.types.tokendetails import TokenDetails +from ably.pubsub.util.exceptions import AblyException from test.uts.helpers.client import rest_client from test.uts.helpers.deviations import deviation, spec_error from test.uts.helpers.mock_http import MockHttpClient diff --git a/test/uts/rest/unit/auth/client_id_test.py b/test/uts/rest/unit/auth/client_id_test.py index 5dcc536b..58a2990e 100644 --- a/test/uts/rest/unit/auth/client_id_test.py +++ b/test/uts/rest/unit/auth/client_id_test.py @@ -8,8 +8,8 @@ import pytest -from ably.types.tokendetails import TokenDetails -from ably.util.exceptions import AblyException, IncompatibleClientIdException +from ably.pubsub.types.tokendetails import TokenDetails +from ably.pubsub.util.exceptions import AblyException, IncompatibleClientIdException from test.uts.helpers.client import rest_client from test.uts.helpers.deviations import deviation from test.uts.helpers.mock_http import MockHttpClient diff --git a/test/uts/rest/unit/auth/revoke_tokens_test.py b/test/uts/rest/unit/auth/revoke_tokens_test.py index f8a02085..0adb05f9 100644 --- a/test/uts/rest/unit/auth/revoke_tokens_test.py +++ b/test/uts/rest/unit/auth/revoke_tokens_test.py @@ -17,7 +17,7 @@ import msgpack import pytest -from ably.util.exceptions import AblyException +from ably.pubsub.util.exceptions import AblyException from test.uts.helpers.client import rest_client from test.uts.helpers.deviations import deviation from test.uts.helpers.mock_http import MockHttpClient diff --git a/test/uts/rest/unit/auth/token_details_test.py b/test/uts/rest/unit/auth/token_details_test.py index e7e1bc88..d4031743 100644 --- a/test/uts/rest/unit/auth/token_details_test.py +++ b/test/uts/rest/unit/auth/token_details_test.py @@ -7,10 +7,10 @@ import pytest -from ably.types.authoptions import AuthOptions -from ably.types.capability import Capability -from ably.types.tokendetails import TokenDetails -from ably.util.exceptions import AblyException +from ably.pubsub.types.authoptions import AuthOptions +from ably.pubsub.types.capability import Capability +from ably.pubsub.types.tokendetails import TokenDetails +from ably.pubsub.util.exceptions import AblyException from test.uts.helpers.client import rest_client from test.uts.helpers.deviations import deviation from test.uts.helpers.mock_http import MockHttpClient diff --git a/test/uts/rest/unit/auth/token_renewal_test.py b/test/uts/rest/unit/auth/token_renewal_test.py index 6f4f6243..2918d1a1 100644 --- a/test/uts/rest/unit/auth/token_renewal_test.py +++ b/test/uts/rest/unit/auth/token_renewal_test.py @@ -9,8 +9,8 @@ import msgpack import pytest -from ably.types.tokendetails import TokenDetails -from ably.util.exceptions import AblyException +from ably.pubsub.types.tokendetails import TokenDetails +from ably.pubsub.util.exceptions import AblyException from test.uts.helpers.client import rest_client from test.uts.helpers.deviations import spec_error from test.uts.helpers.mock_http import MockHttpClient diff --git a/test/uts/rest/unit/batch_presence_test.py b/test/uts/rest/unit/batch_presence_test.py index 187d61ad..f6089fe0 100644 --- a/test/uts/rest/unit/batch_presence_test.py +++ b/test/uts/rest/unit/batch_presence_test.py @@ -20,8 +20,8 @@ import pytest -from ably.types.presence import PresenceAction -from ably.util.exceptions import AblyException +from ably.pubsub.types.presence import PresenceAction +from ably.pubsub.util.exceptions import AblyException from test.uts.helpers.client import rest_client from test.uts.helpers.deviations import deviation from test.uts.helpers.mock_http import MockHttpClient diff --git a/test/uts/rest/unit/batch_publish_test.py b/test/uts/rest/unit/batch_publish_test.py index 338de3ac..e3dacb44 100644 --- a/test/uts/rest/unit/batch_publish_test.py +++ b/test/uts/rest/unit/batch_publish_test.py @@ -23,8 +23,8 @@ import msgpack import pytest -from ably.types.message import Message -from ably.util.exceptions import AblyException +from ably.pubsub.types.message import Message +from ably.pubsub.util.exceptions import AblyException from test.uts.helpers.client import rest_client from test.uts.helpers.deviations import deviation from test.uts.helpers.mock_http import MockHttpClient diff --git a/test/uts/rest/unit/channel/annotations_test.py b/test/uts/rest/unit/channel/annotations_test.py index 1dd1ab57..61ddc14e 100644 --- a/test/uts/rest/unit/channel/annotations_test.py +++ b/test/uts/rest/unit/channel/annotations_test.py @@ -15,10 +15,10 @@ import msgpack import pytest -from ably.http.paginatedresult import PaginatedResult -from ably.rest.annotations import RestAnnotations -from ably.types.annotation import Annotation, AnnotationAction -from ably.util.exceptions import AblyException +from ably.pubsub.http.paginatedresult import PaginatedResult +from ably.pubsub.rest.annotations import RestAnnotations +from ably.pubsub.types.annotation import Annotation, AnnotationAction +from ably.pubsub.util.exceptions import AblyException from test.uts.helpers.client import rest_client from test.uts.helpers.mock_http import MockHttpClient diff --git a/test/uts/rest/unit/channel/get_message_test.py b/test/uts/rest/unit/channel/get_message_test.py index 44838b3e..8f3f1571 100644 --- a/test/uts/rest/unit/channel/get_message_test.py +++ b/test/uts/rest/unit/channel/get_message_test.py @@ -7,8 +7,8 @@ import pytest -from ably.types.message import Message -from ably.util.exceptions import AblyException +from ably.pubsub.types.message import Message +from ably.pubsub.util.exceptions import AblyException from test.uts.helpers.client import rest_client from test.uts.helpers.mock_http import MockHttpClient diff --git a/test/uts/rest/unit/channel/history_test.py b/test/uts/rest/unit/channel/history_test.py index c2fe1047..0eb04c95 100644 --- a/test/uts/rest/unit/channel/history_test.py +++ b/test/uts/rest/unit/channel/history_test.py @@ -5,8 +5,8 @@ import uuid -from ably.http.paginatedresult import PaginatedResult -from ably.types.message import Message +from ably.pubsub.http.paginatedresult import PaginatedResult +from ably.pubsub.types.message import Message from test.uts.helpers.client import rest_client from test.uts.helpers.mock_http import MockHttpClient diff --git a/test/uts/rest/unit/channel/idempotency_test.py b/test/uts/rest/unit/channel/idempotency_test.py index cb747579..35d8c5c5 100644 --- a/test/uts/rest/unit/channel/idempotency_test.py +++ b/test/uts/rest/unit/channel/idempotency_test.py @@ -8,7 +8,7 @@ import msgpack -from ably.types.message import Message +from ably.pubsub.types.message import Message from test.uts.helpers.client import rest_client from test.uts.helpers.deviations import spec_error from test.uts.helpers.mock_http import MockHttpClient diff --git a/test/uts/rest/unit/channel/message_versions_test.py b/test/uts/rest/unit/channel/message_versions_test.py index 2a5c3001..8ad2c1ff 100644 --- a/test/uts/rest/unit/channel/message_versions_test.py +++ b/test/uts/rest/unit/channel/message_versions_test.py @@ -5,8 +5,8 @@ import uuid -from ably.http.paginatedresult import PaginatedResult -from ably.types.message import Message, MessageAction +from ably.pubsub.http.paginatedresult import PaginatedResult +from ably.pubsub.types.message import Message, MessageAction from test.uts.helpers.client import rest_client from test.uts.helpers.mock_http import MockHttpClient diff --git a/test/uts/rest/unit/channel/publish_result_test.py b/test/uts/rest/unit/channel/publish_result_test.py index f7120027..afc5f997 100644 --- a/test/uts/rest/unit/channel/publish_result_test.py +++ b/test/uts/rest/unit/channel/publish_result_test.py @@ -5,8 +5,8 @@ import uuid -from ably.types.message import Message -from ably.types.operations import PublishResult +from ably.pubsub.types.message import Message +from ably.pubsub.types.operations import PublishResult from test.uts.helpers.client import rest_client from test.uts.helpers.mock_http import MockHttpClient diff --git a/test/uts/rest/unit/channel/publish_test.py b/test/uts/rest/unit/channel/publish_test.py index f03fb397..7ebef9d4 100644 --- a/test/uts/rest/unit/channel/publish_test.py +++ b/test/uts/rest/unit/channel/publish_test.py @@ -8,8 +8,8 @@ import msgpack import pytest -from ably.types.message import Message -from ably.util.exceptions import AblyException +from ably.pubsub.types.message import Message +from ably.pubsub.util.exceptions import AblyException from test.uts.helpers.client import rest_client from test.uts.helpers.deviations import deviation, spec_error from test.uts.helpers.mock_http import MockHttpClient diff --git a/test/uts/rest/unit/channel/rest_channel_attributes_test.py b/test/uts/rest/unit/channel/rest_channel_attributes_test.py index 7b2b2371..479ebd90 100644 --- a/test/uts/rest/unit/channel/rest_channel_attributes_test.py +++ b/test/uts/rest/unit/channel/rest_channel_attributes_test.py @@ -4,8 +4,8 @@ CHO2, CHO2a, CHM2, CHM2a, CHM2b, CHM2c, CHM2d, CHM2e, CHM2f, CHM2g, CHM2h """ -from ably.types.channeldetails import ChannelDetails -from ably.types.channeloptions import ChannelOptions +from ably.pubsub.types.channeldetails import ChannelDetails +from ably.pubsub.types.channeloptions import ChannelOptions from test.uts.helpers.client import rest_client from test.uts.helpers.deviations import deviation from test.uts.helpers.mock_http import MockHttpClient diff --git a/test/uts/rest/unit/channel/update_delete_message_test.py b/test/uts/rest/unit/channel/update_delete_message_test.py index 153cc566..cb4de9c7 100644 --- a/test/uts/rest/unit/channel/update_delete_message_test.py +++ b/test/uts/rest/unit/channel/update_delete_message_test.py @@ -10,9 +10,9 @@ import msgpack import pytest -from ably.types.message import Message -from ably.types.operations import MessageOperation, UpdateDeleteResult -from ably.util.exceptions import AblyException +from ably.pubsub.types.message import Message +from ably.pubsub.types.operations import MessageOperation, UpdateDeleteResult +from ably.pubsub.util.exceptions import AblyException from test.uts.helpers.client import rest_client from test.uts.helpers.mock_http import MockHttpClient diff --git a/test/uts/rest/unit/channels_collection_test.py b/test/uts/rest/unit/channels_collection_test.py index b7517fca..830225ef 100644 --- a/test/uts/rest/unit/channels_collection_test.py +++ b/test/uts/rest/unit/channels_collection_test.py @@ -8,7 +8,7 @@ import uuid -from ably.rest.channel import Channel, Channels +from ably.pubsub.rest.channel import Channel, Channels from test.uts.helpers.client import rest_client from test.uts.helpers.mock_http import MockHttpClient diff --git a/test/uts/rest/unit/encoding/message_encoding_test.py b/test/uts/rest/unit/encoding/message_encoding_test.py index 8a86ffc7..6d5d1957 100644 --- a/test/uts/rest/unit/encoding/message_encoding_test.py +++ b/test/uts/rest/unit/encoding/message_encoding_test.py @@ -10,7 +10,7 @@ import msgpack import pytest -from ably.util.exceptions import AblyException +from ably.pubsub.util.exceptions import AblyException from test.uts.helpers.client import rest_client from test.uts.helpers.mock_http import MockHttpClient diff --git a/test/uts/rest/unit/encoding/msgpack_interop_test.py b/test/uts/rest/unit/encoding/msgpack_interop_test.py index ee681dec..f92d802e 100644 --- a/test/uts/rest/unit/encoding/msgpack_interop_test.py +++ b/test/uts/rest/unit/encoding/msgpack_interop_test.py @@ -10,7 +10,7 @@ import msgpack import pytest -from ably.types.message import Message +from ably.pubsub.types.message import Message FIXTURES_PATH = os.path.join( os.path.dirname(__file__), '..', '..', '..', '..', '..', diff --git a/test/uts/rest/unit/fallback_test.py b/test/uts/rest/unit/fallback_test.py index 31446900..3cc8eafd 100644 --- a/test/uts/rest/unit/fallback_test.py +++ b/test/uts/rest/unit/fallback_test.py @@ -9,7 +9,7 @@ import pytest -from ably.util.exceptions import AblyException +from ably.pubsub.util.exceptions import AblyException from test.uts.helpers.client import rest_client from test.uts.helpers.mock_http import MockHttpClient diff --git a/test/uts/rest/unit/presence/rest_presence_test.py b/test/uts/rest/unit/presence/rest_presence_test.py index 09a98a3f..5c10f034 100644 --- a/test/uts/rest/unit/presence/rest_presence_test.py +++ b/test/uts/rest/unit/presence/rest_presence_test.py @@ -11,10 +11,10 @@ import msgpack import pytest -from ably.http.paginatedresult import PaginatedResult -from ably.types.presence import Presence, PresenceAction, PresenceMessage -from ably.util.crypto import CipherParams -from ably.util.exceptions import AblyException +from ably.pubsub.http.paginatedresult import PaginatedResult +from ably.pubsub.types.presence import Presence, PresenceAction, PresenceMessage +from ably.pubsub.util.crypto import CipherParams +from ably.pubsub.util.exceptions import AblyException from test.uts.helpers.client import rest_client from test.uts.helpers.deviations import deviation, spec_error from test.uts.helpers.mock_http import MockHttpClient diff --git a/test/uts/rest/unit/push/push_admin_publish_test.py b/test/uts/rest/unit/push/push_admin_publish_test.py index aaaec0c1..223f57bf 100644 --- a/test/uts/rest/unit/push/push_admin_publish_test.py +++ b/test/uts/rest/unit/push/push_admin_publish_test.py @@ -6,8 +6,8 @@ import msgpack import pytest -from ably.rest.push import Push, PushAdmin, PushChannelSubscriptions, PushDeviceRegistrations -from ably.util.exceptions import AblyException +from ably.pubsub.rest.push import Push, PushAdmin, PushChannelSubscriptions, PushDeviceRegistrations +from ably.pubsub.util.exceptions import AblyException from test.uts.helpers.client import rest_client from test.uts.helpers.mock_http import MockHttpClient diff --git a/test/uts/rest/unit/push/push_channel_subscriptions_test.py b/test/uts/rest/unit/push/push_channel_subscriptions_test.py index 25ab045c..0eeade5c 100644 --- a/test/uts/rest/unit/push/push_channel_subscriptions_test.py +++ b/test/uts/rest/unit/push/push_channel_subscriptions_test.py @@ -6,9 +6,9 @@ import msgpack import pytest -from ably.http.paginatedresult import PaginatedResult -from ably.types.channelsubscription import PushChannelSubscription -from ably.util.exceptions import AblyException +from ably.pubsub.http.paginatedresult import PaginatedResult +from ably.pubsub.types.channelsubscription import PushChannelSubscription +from ably.pubsub.util.exceptions import AblyException from test.uts.helpers.client import rest_client from test.uts.helpers.mock_http import MockHttpClient diff --git a/test/uts/rest/unit/push/push_channels_test.py b/test/uts/rest/unit/push/push_channels_test.py index b213f534..edf92753 100644 --- a/test/uts/rest/unit/push/push_channels_test.py +++ b/test/uts/rest/unit/push/push_channels_test.py @@ -7,16 +7,16 @@ ``push`` field on a channel) nor LocalDevice (RSH8), so every test in this file is gated behind RUN_DEVIATIONS. The assertions are written against the spec, using the names ably-python would use for these APIs: ``client.device``, -``ably.types.device.LocalDevice`` and ``channel.push.subscribe_device()`` and +``ably.pubsub.types.device.LocalDevice`` and ``channel.push.subscribe_device()`` and friends. Running them raises ImportError/AttributeError until the APIs land. """ import msgpack import pytest -from ably.http.paginatedresult import PaginatedResult -from ably.types.channelsubscription import PushChannelSubscription -from ably.util.exceptions import AblyException +from ably.pubsub.http.paginatedresult import PaginatedResult +from ably.pubsub.types.channelsubscription import PushChannelSubscription +from ably.pubsub.util.exceptions import AblyException from test.uts.helpers.client import rest_client from test.uts.helpers.deviations import deviation from test.uts.helpers.mock_http import MockHttpClient @@ -30,7 +30,7 @@ def set_local_device(client, device_id=DEVICE_ID, device_identity_token=DEVICE_IDENTITY_TOKEN, client_id=CLIENT_ID): """Configure the client's local device, standing in for the spec's ``client.device = ...``.""" - from ably.types.device import LocalDevice + from ably.pubsub.types.device import LocalDevice client.device = LocalDevice( id=device_id, diff --git a/test/uts/rest/unit/push/push_device_registrations_test.py b/test/uts/rest/unit/push/push_device_registrations_test.py index ccf7e4a1..8ab284c1 100644 --- a/test/uts/rest/unit/push/push_device_registrations_test.py +++ b/test/uts/rest/unit/push/push_device_registrations_test.py @@ -8,9 +8,9 @@ import msgpack import pytest -from ably.http.paginatedresult import PaginatedResult -from ably.types.device import DeviceDetails -from ably.util.exceptions import AblyException +from ably.pubsub.http.paginatedresult import PaginatedResult +from ably.pubsub.types.device import DeviceDetails +from ably.pubsub.util.exceptions import AblyException from test.uts.helpers.client import rest_client from test.uts.helpers.deviations import deviation from test.uts.helpers.mock_http import MockHttpClient diff --git a/test/uts/rest/unit/rest_client_test.py b/test/uts/rest/unit/rest_client_test.py index 91731161..c23e37c9 100644 --- a/test/uts/rest/unit/rest_client_test.py +++ b/test/uts/rest/unit/rest_client_test.py @@ -8,8 +8,8 @@ import msgpack import pytest -from ably.rest.auth import Auth -from ably.util.exceptions import AblyException +from ably.pubsub.rest.auth import Auth +from ably.pubsub.util.exceptions import AblyException from test.uts.helpers.client import rest_client from test.uts.helpers.mock_http import MockHttpClient diff --git a/test/uts/rest/unit/stats_test.py b/test/uts/rest/unit/stats_test.py index 1e4bd328..72dca8bc 100644 --- a/test/uts/rest/unit/stats_test.py +++ b/test/uts/rest/unit/stats_test.py @@ -7,8 +7,8 @@ import pytest -from ably.http.paginatedresult import PaginatedResult -from ably.util.exceptions import AblyException +from ably.pubsub.http.paginatedresult import PaginatedResult +from ably.pubsub.util.exceptions import AblyException from test.uts.helpers.client import rest_client from test.uts.helpers.mock_http import MockHttpClient diff --git a/test/uts/rest/unit/time_test.py b/test/uts/rest/unit/time_test.py index ad2d4b6e..b7504395 100644 --- a/test/uts/rest/unit/time_test.py +++ b/test/uts/rest/unit/time_test.py @@ -5,7 +5,7 @@ import pytest -from ably.util.exceptions import AblyException +from ably.pubsub.util.exceptions import AblyException from test.uts.helpers.client import rest_client from test.uts.helpers.mock_http import MockHttpClient diff --git a/test/uts/rest/unit/types/error_types_test.py b/test/uts/rest/unit/types/error_types_test.py index ae6f0ce6..ef46afec 100644 --- a/test/uts/rest/unit/types/error_types_test.py +++ b/test/uts/rest/unit/types/error_types_test.py @@ -9,7 +9,7 @@ import pytest -from ably.util.exceptions import AblyException +from ably.pubsub.util.exceptions import AblyException from test.uts.helpers.deviations import deviation diff --git a/test/uts/rest/unit/types/message_types_test.py b/test/uts/rest/unit/types/message_types_test.py index df2ae6ff..b0db98ac 100644 --- a/test/uts/rest/unit/types/message_types_test.py +++ b/test/uts/rest/unit/types/message_types_test.py @@ -5,7 +5,7 @@ import pytest -from ably.types.message import Message +from ably.pubsub.types.message import Message # `fromEncoded` is spelled `from_encoded` here, and "no encoding" is rendered as the empty # string rather than null, since EncodeDataMixin joins an empty list of transforms. diff --git a/test/uts/rest/unit/types/mutable_message_types_test.py b/test/uts/rest/unit/types/mutable_message_types_test.py index 180e117f..72ee6efb 100644 --- a/test/uts/rest/unit/types/mutable_message_types_test.py +++ b/test/uts/rest/unit/types/mutable_message_types_test.py @@ -4,9 +4,9 @@ MOP2a, MOP2b, MOP2c, UDR1, UDR2, UDR2a, TAN1, TAN2, TAN2a-TAN2l """ -from ably.types.annotation import Annotation, AnnotationAction -from ably.types.message import Message, MessageAction, MessageAnnotations, MessageVersion -from ably.types.operations import MessageOperation, UpdateDeleteResult +from ably.pubsub.types.annotation import Annotation, AnnotationAction +from ably.pubsub.types.message import Message, MessageAction, MessageAnnotations, MessageVersion +from ably.pubsub.types.operations import MessageOperation, UpdateDeleteResult # `fromJson` is spelled `from_encoded` for wire payloads and `from_dict` for plain records, # and `toJson` is spelled `as_dict`. diff --git a/test/uts/rest/unit/types/options_types_test.py b/test/uts/rest/unit/types/options_types_test.py index fb443069..a835d57d 100644 --- a/test/uts/rest/unit/types/options_types_test.py +++ b/test/uts/rest/unit/types/options_types_test.py @@ -5,11 +5,11 @@ import pytest -from ably import AblyRest -from ably.types.authoptions import AuthOptions -from ably.types.options import Options -from ably.types.tokendetails import TokenDetails -from ably.util.exceptions import AblyException +from ably.pubsub.server import AblyRest +from ably.pubsub.types.authoptions import AuthOptions +from ably.pubsub.types.options import Options +from ably.pubsub.types.tokendetails import TokenDetails +from ably.pubsub.util.exceptions import AblyException from test.uts.helpers.client import rest_client from test.uts.helpers.deviations import deviation from test.uts.helpers.mock_http import MockHttpClient diff --git a/test/uts/rest/unit/types/paginated_result_test.py b/test/uts/rest/unit/types/paginated_result_test.py index a18525e4..22f0f1bb 100644 --- a/test/uts/rest/unit/types/paginated_result_test.py +++ b/test/uts/rest/unit/types/paginated_result_test.py @@ -5,9 +5,9 @@ import pytest -from ably.types.message import Message -from ably.types.presence import PresenceMessage -from ably.util.exceptions import AblyException +from ably.pubsub.types.message import Message +from ably.pubsub.types.presence import PresenceMessage +from ably.pubsub.util.exceptions import AblyException from test.uts.helpers.client import rest_client from test.uts.helpers.mock_http import MockHttpClient diff --git a/test/uts/rest/unit/types/presence_message_types_test.py b/test/uts/rest/unit/types/presence_message_types_test.py index be0f044f..7c6b2b7c 100644 --- a/test/uts/rest/unit/types/presence_message_types_test.py +++ b/test/uts/rest/unit/types/presence_message_types_test.py @@ -5,7 +5,7 @@ from datetime import datetime, timedelta -from ably.types.presence import PresenceAction, PresenceMessage +from ably.pubsub.types.presence import PresenceAction, PresenceMessage from test.uts.helpers.deviations import deviation, spec_error diff --git a/test/uts/rest/unit/types/token_types_test.py b/test/uts/rest/unit/types/token_types_test.py index bc8764af..243291b3 100644 --- a/test/uts/rest/unit/types/token_types_test.py +++ b/test/uts/rest/unit/types/token_types_test.py @@ -10,8 +10,8 @@ import json -from ably.types.tokendetails import TokenDetails -from ably.types.tokenrequest import TokenRequest +from ably.pubsub.types.tokendetails import TokenDetails +from ably.pubsub.types.tokenrequest import TokenRequest from test.uts.helpers.client import rest_client from test.uts.helpers.mock_http import MockHttpClient diff --git a/uv.lock b/uv.lock index 29ccca45..f0af7384 100644 --- a/uv.lock +++ b/uv.lock @@ -9,7 +9,7 @@ resolution-markers = [ ] [[package]] -name = "ably" +name = "ably-pubsub-server" version = "3.1.4" source = { editable = "." } dependencies = [ From 85f579d318f3e140194ded355e28f9c17ff49e44 Mon Sep 17 00:00:00 2001 From: evgeny Date: Tue, 15 Sep 2026 13:53:49 +0100 Subject: [PATCH 02/71] server: build clients through factories, not constructors MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Expose create_http_client() and create_realtime_client() as the only way to build a client, so that the package a client comes from names the side the application runs on. AblyRest and AblyRealtime are no longer exported from ably.pubsub.server and raise TypeError when constructed directly, naming the factory to reach for instead; a ContextVar opened by the factories is what separates a permitted construction from a direct one. Callers who annotated against the client classes should use the prototypes the factories return. ably/pubsub/prototypes.py declares RestClient and RealtimeClient as typing.Protocol definitions of the client surface, which the concrete classes satisfy structurally. They are called prototypes because "protocol" already means the Ably wire protocol here. - typing.Protocol makes 3.8 the minimum supported Python, which the CI matrix and README already assumed. requires-python and the classifiers now say so, which in turn retires two dead sys.version_info blocks in the tests. - Expand the factory docstrings to cover every client option, grouped by concern and derived from Options and AuthOptions rather than copied from the constructors. - Drop `keep_alive` from the AblyRest docstring. It has been documented since the constructor was written, but no option of that name exists and passing it raises TypeError. test/unit/pubsub_server_test.py covers the factories, the guard — including that permission does not leak past a factory call — the classes being unexported, and the concrete clients satisfying their prototypes. Co-Authored-By: Claude Opus 5 (1M context) --- .claude/skills/uts-to-python/SKILL.md | 2 +- LONG_DESCRIPTION.rst | 2 +- README.md | 12 +- UPDATING.md | 28 +- ably/pubsub/prototypes.py | 120 ++++ ably/pubsub/realtime/realtime.py | 2 + ably/pubsub/rest/rest.py | 4 +- ably/pubsub/scripts/unasync.py | 5 + ably/pubsub/server/__init__.py | 218 ++++++- ably/pubsub/server/sync.py | 44 +- ably/pubsub/util/construction.py | 41 ++ ably/pubsub/vcdiff/defaultvcdiffdecoder.py | 4 +- pyproject.toml | 3 +- test/ably/realtime/deltadecoding_test.py | 4 +- test/ably/rest/encoders_test.py | 7 +- test/ably/rest/restauth_test.py | 44 +- test/ably/rest/resthttp_test.py | 16 +- test/ably/rest/restinit_test.py | 66 +- test/ably/rest/restrequest_test.py | 6 +- test/ably/rest/resttoken_test.py | 4 +- test/ably/testapp.py | 19 +- test/unit/clock_test.py | 16 +- test/unit/http_test.py | 16 +- test/unit/pubsub_server_test.py | 80 +++ test/uts/README.md | 8 +- test/uts/helpers/client.py | 10 +- test/uts/helpers/mock_http_test.py | 7 +- .../proxy/connection_open_failures_test.py | 4 +- .../integration/proxy/rest_faults_test.py | 4 +- .../integration/proxy/rest_fallback_test.py | 4 +- test/uts/rest/integration/time_stats_test.py | 6 +- .../uts/rest/unit/types/options_types_test.py | 4 +- uv.lock | 604 ++---------------- 33 files changed, 703 insertions(+), 711 deletions(-) create mode 100644 ably/pubsub/prototypes.py create mode 100644 ably/pubsub/util/construction.py create mode 100644 test/unit/pubsub_server_test.py diff --git a/.claude/skills/uts-to-python/SKILL.md b/.claude/skills/uts-to-python/SKILL.md index a413d6bb..5a93bf7a 100644 --- a/.claude/skills/uts-to-python/SKILL.md +++ b/.claude/skills/uts-to-python/SKILL.md @@ -728,7 +728,7 @@ Established against the real proxy and the real sandbox while deriving asks for a token through a client pointed at the session puts a request in front of the waiting rule and an extra `http_request` in the log, which breaks every assertion that counts requests exactly (`== 1` for RSC15l's 4xx test, `>= 2` for the - fallback ones). Build an inner `AblyRest(key=api_key, endpoint=SANDBOX_ENDPOINT)`, + fallback ones). Build an inner `create_http_client(key=api_key, endpoint=SANDBOX_ENDPOINT)`, request the token through that, and close it. - **`http_response` events carry no `path`.** They have `status` and `ruleMatched` only, so "the injected response fired" is read off the response events **in order** diff --git a/LONG_DESCRIPTION.rst b/LONG_DESCRIPTION.rst index ec4622a3..54f7289c 100644 --- a/LONG_DESCRIPTION.rst +++ b/LONG_DESCRIPTION.rst @@ -16,7 +16,7 @@ The package installs into ``ably.pubsub``. Both ``ably`` and ``ably.pubsub`` are namespace packages shared with the other ``ably-*`` distributions, so the whole public API is reached through ``ably.pubsub.server``:: - from ably.pubsub.server import AblyRealtime, AblyRest + from ably.pubsub.server import create_http_client, create_realtime_client Using Ably for Python diff --git a/README.md b/README.md index c0e66cf8..5b95efe8 100644 --- a/README.md +++ b/README.md @@ -55,13 +55,17 @@ The package installs into `ably.pubsub`, and the whole public API is reached through `ably.pubsub.server`: ```python -from ably.pubsub.server import AblyRealtime, AblyRest +from ably.pubsub.server import create_http_client, create_realtime_client ``` +Clients are built by these factories rather than by constructing a class, so +that the package a client comes from names the side your application runs on. +Annotate against the prototypes they return, `RestClient` and `RealtimeClient`. + The synchronous, HTTP-only flavour lives alongside it: ```python -from ably.pubsub.server.sync import AblyRestSync +from ably.pubsub.server.sync import create_http_client ``` Both `ably` and `ably.pubsub` are [namespace packages](https://peps.python.org/pep-0420/) @@ -76,10 +80,10 @@ Install [Python](https://www.python.org/downloads/) version 3.8 or greater. The following code connects to Ably's realtime messaging service, subscribes to a channel to receive messages, and publishes a test message to that same channel. ```python -from ably.pubsub.server import AblyRealtime +from ably.pubsub.server import create_realtime_client # Initialize Ably Realtime client -async with AblyRealtime('your-ably-api-key', client_id='me') as realtime_client: +async with create_realtime_client('your-ably-api-key', client_id='me') as realtime_client: # Wait for connection to be established await realtime_client.connection.once_async('connected') print('Connected to Ably') diff --git a/UPDATING.md b/UPDATING.md index 7ec94b63..6f3b7ee5 100644 --- a/UPDATING.md +++ b/UPDATING.md @@ -29,7 +29,7 @@ from ably import AblyRealtime, AblyRest Example 4.0.0 code: ```python -from ably.pubsub.server import AblyRealtime, AblyRest +from ably.pubsub.server import create_http_client, create_realtime_client ``` The synchronous, HTTP-only flavour moves the same way: @@ -41,7 +41,7 @@ from ably.sync import AblyRestSync Example 4.0.0 code: ```python -from ably.pubsub.server.sync import AblyRestSync +from ably.pubsub.server.sync import create_http_client ``` Anything imported from a submodule gains the same `pubsub` segment — for @@ -51,6 +51,30 @@ instance `ably.types.message` becomes `ably.pubsub.types.message`, and `api_version` and `lib_version` are still re-exported from `ably.pubsub.server`, and now also live in `ably.pubsub.version`. +### Clients are built by factories + +`AblyRest` and `AblyRealtime` are internal in 4.0.0 and raise `TypeError` if +constructed directly. Build clients through the factories instead, which take +the same arguments: + +Example 3.x code: +```python +rest = AblyRest(key='xxx') +realtime = AblyRealtime(key='xxx') +``` + +Example 4.0.0 code: +```python +rest = create_http_client(key='xxx') +realtime = create_realtime_client(key='xxx') +``` + +Where you previously annotated against `AblyRest` or `AblyRealtime`, use the +prototypes the factories return — `ably.pubsub.server.RestClient` and +`ably.pubsub.server.RealtimeClient`. These are `typing.Protocol` definitions +describing the client surface, so `Protocol` support raises the minimum +supported Python to 3.8 (which the CI matrix and README already assumed). + ## Version 2.x to 3.0.0 The 3.0.0 version of ably-python introduces several breaking changes to improve the realtime experience and align the API with the Ably specification. These include: diff --git a/ably/pubsub/prototypes.py b/ably/pubsub/prototypes.py new file mode 100644 index 00000000..2ad63960 --- /dev/null +++ b/ably/pubsub/prototypes.py @@ -0,0 +1,120 @@ +"""The shapes of the Pub/Sub clients, as callers see them. + +The client classes themselves are internal: they are reachable only through the +factory functions in :mod:`ably.pubsub.server` and :mod:`ably.pubsub.server.sync`, +so that the package a client comes from names the side the application runs on. +These protocols are what those factories are declared to return, and what +application code should annotate against. + +They are named "prototypes" rather than "protocols" because this codebase +already uses "protocol" for the Ably wire protocol. +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING, Protocol + +if TYPE_CHECKING: + from ably.pubsub.http.http import Http + from ably.pubsub.http.paginatedresult import HttpPaginatedResponse, PaginatedResult + from ably.pubsub.realtime.channel import Channels as RealtimeChannels + from ably.pubsub.realtime.connection import Connection + from ably.pubsub.rest.auth import Auth + from ably.pubsub.rest.channel import Channels as RestChannels + from ably.pubsub.rest.push import Push + from ably.pubsub.types.options import Options + + +class RestClient(Protocol): + """A Pub/Sub client that operates entirely over HTTP. + + Build one with ``create_http_client``, from :mod:`ably.pubsub.server` for the + asynchronous flavour or :mod:`ably.pubsub.server.sync` for the synchronous one. + """ + + @property + def auth(self) -> Auth: + """The authentication object for this client.""" + ... + + @property + def channels(self) -> RestChannels: + """The channels container object.""" + ... + + @property + def client_id(self) -> str | None: + """The client ID this client authenticates as, if any.""" + ... + + @property + def http(self) -> Http: + """The HTTP transport this client sends requests over.""" + ... + + @property + def options(self) -> Options: + """The client options this client was built with.""" + ... + + @property + def push(self) -> Push: + """The push notification admin object.""" + ... + + async def stats(self, direction: str | None = None, start=None, end=None, params: dict | None = None, + limit: int | None = None, paginated=None, unit=None, timeout=None) -> PaginatedResult: + """Return the stats for this application.""" + ... + + async def time(self, timeout: float | None = None) -> float: + """Return the current server time in ms since the unix epoch.""" + ... + + async def request(self, method: str, path: str, version: str, params: dict | None = None, + body=None, headers=None) -> HttpPaginatedResponse: + """Make an arbitrary request against the Ably REST API.""" + ... + + async def close(self) -> None: + """Release the resources this client holds.""" + ... + + async def __aenter__(self) -> RestClient: + ... + + async def __aexit__(self, *excinfo) -> None: + ... + + +class RealtimeClient(RestClient, Protocol): + """A Pub/Sub client with a persistent realtime connection. + + Everything :class:`RestClient` does, plus subscribing to channels and + entering presence. Build one with + :func:`ably.pubsub.server.create_realtime_client`. + """ + + @property + def channels(self) -> RealtimeChannels: + """The realtime channels container object.""" + ... + + @property + def connection(self) -> Connection: + """The realtime connection object.""" + ... + + def connect(self) -> None: + """Establish the realtime connection. + + Unnecessary unless the client was built with ``auto_connect=False``. + """ + ... + + async def close(self) -> None: + """Close the realtime connection and release the client's resources.""" + ... + + async def __aenter__(self) -> RealtimeClient: + ... diff --git a/ably/pubsub/realtime/realtime.py b/ably/pubsub/realtime/realtime.py index 4fa90934..24c11d91 100644 --- a/ably/pubsub/realtime/realtime.py +++ b/ably/pubsub/realtime/realtime.py @@ -5,6 +5,7 @@ from ably.pubsub.realtime.channel import Channels from ably.pubsub.realtime.connection import Connection, ConnectionState from ably.pubsub.rest.rest import AblyRest +from ably.pubsub.util.construction import reject_direct_construction log = logging.getLogger(__name__) @@ -88,6 +89,7 @@ def __init__(self, key: Optional[str] = None, loop: Optional[asyncio.AbstractEve ValueError If no authentication key is not provided """ + reject_direct_construction(type(self), 'ably.pubsub.server.create_realtime_client') if loop is None: try: diff --git a/ably/pubsub/rest/rest.py b/ably/pubsub/rest/rest.py index c89ce3ed..d746828a 100644 --- a/ably/pubsub/rest/rest.py +++ b/ably/pubsub/rest/rest.py @@ -10,6 +10,7 @@ from ably.pubsub.types.options import Options from ably.pubsub.types.stats import stats_response_processor from ably.pubsub.types.tokendetails import TokenDetails +from ably.pubsub.util.construction import reject_direct_construction from ably.pubsub.util.exceptions import AblyException, catch_all log = logging.getLogger(__name__) @@ -47,8 +48,9 @@ def __init__(self, key: Optional[str] = None, token: Optional[str] = None, - `auth_token`: Undocumented - `auth_callback`: Undocumented - `auth_url`: Undocumented - - `keep_alive`: use persistent connections. Defaults to True """ + reject_direct_construction(type(self), 'ably.pubsub.server.create_http_client') + if key is not None and ('key_name' in kwargs or 'key_secret' in kwargs): raise ValueError("key and key_name or key_secret are mutually exclusive. " "Provider either a key or key_name & key_secret") diff --git a/ably/pubsub/scripts/unasync.py b/ably/pubsub/scripts/unasync.py index b253567f..40e03267 100644 --- a/ably/pubsub/scripts/unasync.py +++ b/ably/pubsub/scripts/unasync.py @@ -252,6 +252,11 @@ def run(): _CLASS_RENAME[class_name] = f"{class_name}Sync" _STRING_REPLACE["Auth"] = "AuthSync" + # The sync flavour's factory lives in ably.pubsub.server.sync. There is no + # sync realtime factory, so create_realtime_client is left pointing at the + # asynchronous one, which is the right advice for that case. + _STRING_REPLACE['ably.pubsub.server.create_http_client'] = \ + 'ably.pubsub.server.sync.create_http_client' src_dir_path = os.path.join(os.getcwd(), "ably", "pubsub") dest_dir_path = os.path.join(os.getcwd(), "ably", "pubsub", "sync") diff --git a/ably/pubsub/server/__init__.py b/ably/pubsub/server/__init__.py index f7907532..8b4c0c99 100644 --- a/ably/pubsub/server/__init__.py +++ b/ably/pubsub/server/__init__.py @@ -5,6 +5,12 @@ package names that side, so the client an application reaches for is the one whose package matches where it runs. +Build a client with :func:`create_http_client` or +:func:`create_realtime_client`; the classes behind them are internal and +reject direct construction. Annotate against the prototypes they return, +:class:`~ably.pubsub.prototypes.RestClient` and +:class:`~ably.pubsub.prototypes.RealtimeClient`. + This is the public API of the ``ably-pubsub-server`` distribution. Both ``ably`` and ``ably.pubsub`` are PEP 420 namespace packages shared with the other ``ably-*`` distributions, so neither carries an ``__init__.py`` of its @@ -12,10 +18,11 @@ from here. """ -from ably.pubsub.realtime.realtime import AblyRealtime +from ably.pubsub.prototypes import RealtimeClient, RestClient +from ably.pubsub.realtime.realtime import AblyRealtime as _AblyRealtime from ably.pubsub.rest.auth import Auth from ably.pubsub.rest.push import Push -from ably.pubsub.rest.rest import AblyRest +from ably.pubsub.rest.rest import AblyRest as _AblyRest from ably.pubsub.types.annotation import Annotation, AnnotationAction from ably.pubsub.types.capability import Capability from ably.pubsub.types.channelmode import ChannelMode @@ -25,16 +32,214 @@ from ably.pubsub.types.message import MessageAction, MessageVersion from ably.pubsub.types.operations import MessageOperation, PublishResult, UpdateDeleteResult from ably.pubsub.types.options import Options, VCDiffDecoder +from ably.pubsub.types.tokendetails import TokenDetails +from ably.pubsub.util.construction import factory_construction from ably.pubsub.util.crypto import CipherParams from ably.pubsub.util.exceptions import AblyAuthException, AblyException, IncompatibleClientIdException from ably.pubsub.vcdiff.defaultvcdiffdecoder import AblyVCDiffDecoder from ably.pubsub.version import api_version, lib_version + +def create_http_client(**kwargs) -> RestClient: + """Create a server Pub/Sub client that operates entirely over HTTP. + + Handles publish, history, presence reads, stats and token issuing. Give the client one + of `key`, `token` or `token_details` to authenticate, or configure token auth through + `auth_callback` or `auth_url`. + + Parameters + ---------- + **kwargs: client options + **Credentials** + + key: str + A valid Ably API key, in `name:secret` form. Mutually exclusive with `key_name` + and `key_secret`. + token: str + A valid token string to authenticate with in place of an API key. + token_details: TokenDetails + A TokenDetails instance to authenticate with in place of an API key. + key_name: str + The first half of an API key, as an alternative to passing `key`. + key_secret: str + The second half of an API key, as an alternative to passing `key`. + + **Token authentication** + + auth_callback: callable + Called when a new token is needed, and returns a token string, a TokenDetails, + a TokenRequest or a signed token request dict. Prefer this to `auth_url` when + the token source is in-process. + auth_url: str + A URL to request a token from when a new token is needed. The client issues a + request to this URL and uses the response as the token. + auth_method: str + The HTTP method used for the `auth_url` request. Defaults to `GET`. + auth_headers: dict + Headers to include in the `auth_url` request. + auth_params: dict + Query parameters to include in the `auth_url` request. + use_token_auth: bool + Force token authentication even when an API key is available. By default the + client uses basic auth where a key is present. + query_time: bool + Query the Ably service for the current time when signing token requests, rather + than trusting the local clock. Defaults to False. + default_token_params: dict + Parameters applied to every token request this client makes. + + **Connection** + + client_id: str + The ID this client identifies as. Messages it publishes carry this ID, and it + cannot be changed once the client has authenticated. + endpoint: str + Either a routing policy name or a fully qualified domain name to connect to. + Defaults to `main`. Incompatible with `environment`, `rest_host` and + `realtime_host`. + tls: bool + Whether the client should use TLS. Defaults to True. + port: int + The non-TLS port to connect to. Defaults to 80. + tls_port: int + The TLS port to connect to. Defaults to 443. + fallback_hosts: list[str] + Hosts to fall back to when the primary endpoint is unreachable. Set this only + if Ably has provided you with a custom set. + fallback_retry_timeout: float + How long (in milliseconds) a fallback host stays preferred once the client has + failed over to it. The default is 10 minutes (600000 milliseconds). + + **HTTP behaviour** + + http_open_timeout: float + Timeout (in seconds) for establishing an HTTP connection. The default is 4 + seconds. + http_request_timeout: float + Timeout (in seconds) for a complete HTTP request. The default is 10 seconds. + http_max_retry_duration: float + How long (in seconds) the client keeps retrying a request across hosts before + giving up. The default is 15 seconds. + http_max_retry_count: int + The most hosts the client will try for a single request. + + **Protocol** + + use_binary_protocol: bool + Encode messages with MessagePack rather than JSON. Defaults to True. + idempotent_rest_publishing: bool + Assign an ID to published messages so that a retried publish is not duplicated. + Defaults to True. + add_request_ids: bool + Add a `request_id` query parameter to every request, which makes a request + traceable in Ably's logs. Defaults to False. + log_level: int + Verbosity of the library's own logging. Defaults to 0. + + **Deprecated** + + rest_host: str + Deprecated: this option will be removed in a future version. Use `endpoint`. + environment: str + Deprecated: this option will be removed in a future version. Use `endpoint`. + + Returns + ------- + RestClient + A client ready to use. It opens no connection until its first request. + + Raises + ------ + ValueError + If `key` is combined with `key_name` or `key_secret`. + AblyException + If no means of authenticating is given, or if `endpoint` is combined with + `environment`, `rest_host` or `realtime_host`. + """ + with factory_construction(): + return _AblyRest(**kwargs) + + +def create_realtime_client(**kwargs) -> RealtimeClient: + """Create a server Pub/Sub client with a persistent realtime connection. + + Everything the HTTP client does, plus subscribing to channels and entering presence. + Accepts every option :func:`create_http_client` does — see there for credentials, + token authentication, connection and HTTP behaviour — and the following in addition. + + Parameters + ---------- + **kwargs: client options + **Connection lifecycle** + + loop: asyncio.AbstractEventLoop + The event loop the client runs on. Defaults to the running loop; the client + logs a warning if it is created outside one. + auto_connect: bool + Connect as soon as the client is created. Set this to False to connect later + with `connect()`. Defaults to True. + recover: str + A recovery key from a previous connection, used to resume that connection's + state and message continuity rather than starting fresh. + queue_messages: bool + Hold messages published while the connection is not yet established and send + them once it is, rather than failing them. Defaults to True. + transport_params: dict + Additional query parameters to send when opening the realtime connection. + + **Timeouts and retries** + + realtime_request_timeout: float + How long (in milliseconds) to wait for acknowledgement of an operation + performed over the realtime connection, such as establishing a connection or + sending a HEARTBEAT, CONNECT, ATTACH, DETACH or CLOSE request. The default is + 10 seconds (10000 milliseconds). + disconnected_retry_timeout: float + How long (in milliseconds) to stay DISCONNECTED before attempting to reconnect + automatically. The default is 15 seconds (15000 milliseconds). + suspended_retry_timeout: float + How long (in milliseconds) to stay SUSPENDED before attempting to reconnect + automatically. The default is 30 seconds (30000 milliseconds). + channel_retry_timeout: float + How long (in milliseconds) a channel stays SUSPENDED, following a server + initiated DETACHED, before the client re-attaches it — provided the connection + is CONNECTED. The default is 15 seconds (15000 milliseconds). + connectivity_check_url: str + The URL the client requests to decide whether the internet is reachable. On + failing to reach the primary endpoint, a success response from this URL tells + the client to try a fallback host instead of giving up. + + **Deltas** + + vcdiff_decoder: VCDiffDecoder + The decoder used to apply delta-encoded messages on channels configured with + `params={"delta": "vcdiff"}`. Pass an :class:`AblyVCDiffDecoder`. + + **Deprecated** + + realtime_host: str + Deprecated: this option will be removed in a future version. Use `endpoint`. + + Returns + ------- + RealtimeClient + A client which, unless `auto_connect=False`, is already connecting. + + Raises + ------ + ValueError + If `key` is combined with `key_name` or `key_secret`. + AblyException + If no means of authenticating is given, or if `endpoint` is combined with + `environment`, `rest_host` or `realtime_host`. + """ + with factory_construction(): + return _AblyRealtime(**kwargs) + + __all__ = [ 'AblyAuthException', 'AblyException', - 'AblyRealtime', - 'AblyRest', 'AblyVCDiffDecoder', 'Annotation', 'AnnotationAction', @@ -52,8 +257,13 @@ 'PublishResult', 'Push', 'PushChannelSubscription', + 'RealtimeClient', + 'RestClient', + 'TokenDetails', 'UpdateDeleteResult', 'VCDiffDecoder', 'api_version', + 'create_http_client', + 'create_realtime_client', 'lib_version', ] diff --git a/ably/pubsub/server/sync.py b/ably/pubsub/server/sync.py index 1a18fa67..131b5603 100644 --- a/ably/pubsub/server/sync.py +++ b/ably/pubsub/server/sync.py @@ -1,16 +1,16 @@ """The synchronous flavour of the Ably Pub/Sub client for servers. -Exposes the same surface as :mod:`ably.pubsub.server`, generated without +Exposes the same HTTP surface as :mod:`ably.pubsub.server`, generated without ``async``/``await`` so it can be driven from a thread rather than an event -loop. ``AblyRestSync`` is the client to reach for; a server that needs to -subscribe to channels should use the asynchronous -:class:`ably.pubsub.server.AblyRealtime` instead. +loop. There is no synchronous realtime client, so a server that needs to +subscribe to channels should use :func:`ably.pubsub.server.create_realtime_client` +instead. """ -from ably.pubsub.sync.realtime.realtime import AblyRealtime +from ably.pubsub.sync.prototypes import RestClient from ably.pubsub.sync.rest.auth import AuthSync from ably.pubsub.sync.rest.push import PushSync -from ably.pubsub.sync.rest.rest import AblyRestSync +from ably.pubsub.sync.rest.rest import AblyRestSync as _AblyRestSync from ably.pubsub.sync.types.annotation import Annotation, AnnotationAction from ably.pubsub.sync.types.capability import Capability from ably.pubsub.sync.types.channelmode import ChannelMode @@ -20,16 +20,41 @@ from ably.pubsub.sync.types.message import MessageAction, MessageVersion from ably.pubsub.sync.types.operations import MessageOperation, PublishResult, UpdateDeleteResult from ably.pubsub.sync.types.options import Options, VCDiffDecoder +from ably.pubsub.sync.types.tokendetails import TokenDetails +from ably.pubsub.sync.util.construction import factory_construction from ably.pubsub.sync.util.crypto import CipherParams from ably.pubsub.sync.util.exceptions import AblyAuthException, AblyException, IncompatibleClientIdException from ably.pubsub.sync.vcdiff.defaultvcdiffdecoder import AblyVCDiffDecoder from ably.pubsub.version import api_version, lib_version + +def create_http_client(**kwargs) -> RestClient: + """Create a synchronous server Pub/Sub client that operates entirely over HTTP. + + Accepts every option :func:`ably.pubsub.server.create_http_client` does — see there + for credentials, token authentication, connection and HTTP behaviour — and behaves + identically, without the event loop. The realtime-only options have no effect here. + + Returns + ------- + RestClient + A client ready to use. It opens no connection until its first request. + + Raises + ------ + ValueError + If `key` is combined with `key_name` or `key_secret`. + AblyException + If no means of authenticating is given, or if `endpoint` is combined with + `environment`, `rest_host` or `realtime_host`. + """ + with factory_construction(): + return _AblyRestSync(**kwargs) + + __all__ = [ 'AblyAuthException', 'AblyException', - 'AblyRealtime', - 'AblyRestSync', 'AblyVCDiffDecoder', 'Annotation', 'AnnotationAction', @@ -47,8 +72,11 @@ 'PublishResult', 'PushChannelSubscription', 'PushSync', + 'RestClient', + 'TokenDetails', 'UpdateDeleteResult', 'VCDiffDecoder', 'api_version', + 'create_http_client', 'lib_version', ] diff --git a/ably/pubsub/util/construction.py b/ably/pubsub/util/construction.py new file mode 100644 index 00000000..f1a0cd94 --- /dev/null +++ b/ably/pubsub/util/construction.py @@ -0,0 +1,41 @@ +"""Enforcement of the factories as the only way to build a client. + +:mod:`ably.pubsub.server` exposes ``create_http_client`` and +``create_realtime_client`` rather than the client classes themselves, so that +the package a client comes from names the side the application runs on. The +classes stay importable for the library's own use and reject construction from +anywhere else. +""" + +from contextlib import contextmanager +from contextvars import ContextVar + +# True for the duration of a factory call. A ContextVar rather than a module +# global so that concurrent tasks cannot observe each other's state. +_in_factory: ContextVar = ContextVar('ably_pubsub_in_factory', default=False) + + +@contextmanager +def factory_construction(): + """Permit client construction within this block. + + For use by Ably's own factory functions only. + """ + token = _in_factory.set(True) + try: + yield + finally: + _in_factory.reset(token) + + +def reject_direct_construction(cls: type, factory: str) -> None: + """Raise unless we are inside a factory call. + + `factory` names the entry point the caller should have reached for, so + that the error says what to do rather than only what went wrong. + """ + if _in_factory.get(): + return + raise TypeError( + f'{cls.__name__} cannot be constructed directly. Use {factory} instead.' + ) diff --git a/ably/pubsub/vcdiff/defaultvcdiffdecoder.py b/ably/pubsub/vcdiff/defaultvcdiffdecoder.py index ee86abd1..07af3b9e 100644 --- a/ably/pubsub/vcdiff/defaultvcdiffdecoder.py +++ b/ably/pubsub/vcdiff/defaultvcdiffdecoder.py @@ -5,13 +5,13 @@ It implements the VCDiffDecoder interface. Usage: - from ably.pubsub.vcdiff import AblyVCDiffDecoder, AblyRealtime + from ably.pubsub.server import AblyVCDiffDecoder, ChannelOptions, create_realtime_client # Create VCDiff decoder vcdiff_decoder = AblyVCDiffDecoder() # Create client with decoder - client = AblyRealtime(key="your-key", vcdiff_decoder=vcdiff_decoder) + client = create_realtime_client(key="your-key", vcdiff_decoder=vcdiff_decoder) # Get channel with delta enabled channel = client.channels.get("test", ChannelOptions(params={"delta": "vcdiff"})) diff --git a/pyproject.toml b/pyproject.toml index 42eaee5d..b91f41f5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,7 +3,7 @@ name = "ably-pubsub-server" version = "3.1.4" description = "Python REST and Realtime server client library SDK for Ably Pub/Sub" readme = "LONG_DESCRIPTION.rst" -requires-python = ">=3.7" +requires-python = ">=3.8" license = { text = "Apache-2.0" } authors = [ { name = "Ably", email = "support@ably.com" } @@ -15,7 +15,6 @@ classifiers = [ "Operating System :: OS Independent", "Programming Language :: Python", "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.7", "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", diff --git a/test/ably/realtime/deltadecoding_test.py b/test/ably/realtime/deltadecoding_test.py index 1c3d250e..b067023e 100644 --- a/test/ably/realtime/deltadecoding_test.py +++ b/test/ably/realtime/deltadecoding_test.py @@ -9,7 +9,7 @@ import pytest -from ably.pubsub.server import AblyRealtime +from ably.pubsub.server import create_realtime_client from ably.pubsub.types.channelstate import ChannelState from ably.pubsub.types.message import Message from ably.pubsub.types.mixins import DecodingContext @@ -131,7 +131,7 @@ class TestChannelDeltaBatch(BaseAsyncTestCase): """RealtimeChannel handling of a protocol message carrying several deltas""" def setup_channel(self, decoder): - ably = AblyRealtime(key='not_a.real:key', auto_connect=False, vcdiff_decoder=decoder) + ably = create_realtime_client(key='not_a.real:key', auto_connect=False, vcdiff_decoder=decoder) channel = ably.channels.get('delta') received = [] # subscribe() would also wait for the channel to attach, which this client cannot do diff --git a/test/ably/rest/encoders_test.py b/test/ably/rest/encoders_test.py index 79e4e3d0..f2964849 100644 --- a/test/ably/rest/encoders_test.py +++ b/test/ably/rest/encoders_test.py @@ -1,8 +1,8 @@ import base64 import json import logging -import sys from unittest import mock +from unittest.mock import AsyncMock import msgpack import pytest @@ -13,11 +13,6 @@ from test.ably.testapp import TestApp from test.ably.utils import BaseAsyncTestCase -if sys.version_info >= (3, 8): - from unittest.mock import AsyncMock -else: - from mock import AsyncMock - log = logging.getLogger(__name__) diff --git a/test/ably/rest/restauth_test.py b/test/ably/rest/restauth_test.py index 70aac5e7..5f2eacc6 100644 --- a/test/ably/rest/restauth_test.py +++ b/test/ably/rest/restauth_test.py @@ -1,9 +1,9 @@ import base64 import logging -import sys import time import uuid from unittest import mock +from unittest.mock import AsyncMock from urllib.parse import parse_qs import pytest @@ -11,16 +11,11 @@ from httpx import AsyncClient, Response import ably.pubsub.types.tokenrequest -from ably.pubsub.server import AblyAuthException, AblyRest, Auth +from ably.pubsub.server import AblyAuthException, Auth, create_http_client from ably.pubsub.types.tokendetails import TokenDetails from test.ably.testapp import TestApp from test.ably.utils import BaseAsyncTestCase, VaryByProtocolTestsMetaclass, dont_vary_protocol -if sys.version_info >= (3, 8): - from unittest.mock import AsyncMock -else: - from mock import AsyncMock - log = logging.getLogger(__name__) @@ -31,19 +26,19 @@ async def setup(self): self.test_vars = await TestApp.get_test_vars() def test_auth_init_key_only(self): - ably = AblyRest(key=self.test_vars["keys"][0]["key_str"]) + ably = create_http_client(key=self.test_vars["keys"][0]["key_str"]) assert Auth.Method.BASIC == ably.auth.auth_mechanism, "Unexpected Auth method mismatch" assert ably.auth.auth_options.key_name == self.test_vars["keys"][0]['key_name'] assert ably.auth.auth_options.key_secret == self.test_vars["keys"][0]['key_secret'] def test_auth_init_token_only(self): - ably = AblyRest(token="this_is_not_really_a_token") + ably = create_http_client(token="this_is_not_really_a_token") assert Auth.Method.TOKEN == ably.auth.auth_mechanism, "Unexpected Auth method mismatch" def test_auth_token_details(self): td = TokenDetails() - ably = AblyRest(token_details=td) + ably = create_http_client(token_details=td) assert Auth.Method.TOKEN == ably.auth.auth_mechanism assert ably.auth.token_details is td @@ -69,7 +64,7 @@ def token_callback(token_params): assert Auth.Method.TOKEN == ably.auth.auth_mechanism, "Unexpected Auth method mismatch" def test_auth_init_with_key_and_client_id(self): - ably = AblyRest(key=self.test_vars["keys"][0]["key_str"], client_id='testClientId') + ably = create_http_client(key=self.test_vars["keys"][0]["key_str"], client_id='testClientId') assert Auth.Method.BASIC == ably.auth.auth_mechanism, "Unexpected Auth method mismatch" assert ably.auth.client_id == 'testClientId' @@ -80,7 +75,7 @@ async def test_auth_init_with_token(self): # RSA11 async def test_request_basic_auth_header(self): - ably = AblyRest(key_secret='foo', key_name='bar') + ably = create_http_client(key_secret='foo', key_name='bar') with mock.patch.object(AsyncClient, 'send') as get_mock: get_mock.return_value = {"status": 200, "headers": {}} @@ -94,7 +89,7 @@ async def test_request_basic_auth_header(self): # RSA7e2 async def test_request_basic_auth_header_with_client_id(self): - ably = AblyRest(key_secret='foo', key_name='bar', client_id='client_id') + ably = create_http_client(key_secret='foo', key_name='bar', client_id='client_id') with mock.patch.object(AsyncClient, 'send') as get_mock: get_mock.return_value = {"status": 200, "headers": {}} @@ -107,7 +102,7 @@ async def test_request_basic_auth_header_with_client_id(self): assert client_id == base64.b64encode('client_id'.encode('ascii')).decode('utf-8') async def test_request_token_auth_header(self): - ably = AblyRest(token='not_a_real_token') + ably = create_http_client(token='not_a_real_token') with mock.patch.object(AsyncClient, 'send') as get_mock: get_mock.return_value = {"status": 200, "headers": {}} @@ -122,26 +117,27 @@ async def test_request_token_auth_header(self): def test_if_cant_authenticate_via_token(self): with pytest.raises(ValueError): - AblyRest(use_token_auth=True) + create_http_client(use_token_auth=True) def test_use_auth_token(self): - ably = AblyRest(use_token_auth=True, key=self.test_vars["keys"][0]["key_str"]) + ably = create_http_client(use_token_auth=True, key=self.test_vars["keys"][0]["key_str"]) assert ably.auth.auth_mechanism == Auth.Method.TOKEN def test_with_client_id(self): - ably = AblyRest(use_token_auth=True, client_id='client_id', key=self.test_vars["keys"][0]["key_str"]) + ably = create_http_client(use_token_auth=True, client_id='client_id', + key=self.test_vars["keys"][0]["key_str"]) assert ably.auth.auth_mechanism == Auth.Method.TOKEN def test_with_auth_url(self): - ably = AblyRest(auth_url='auth_url') + ably = create_http_client(auth_url='auth_url') assert ably.auth.auth_mechanism == Auth.Method.TOKEN def test_with_auth_callback(self): - ably = AblyRest(auth_callback=lambda x: x) + ably = create_http_client(auth_callback=lambda x: x) assert ably.auth.auth_mechanism == Auth.Method.TOKEN def test_with_token(self): - ably = AblyRest(token='a token') + ably = create_http_client(token='a token') assert ably.auth.auth_mechanism == Auth.Method.TOKEN def test_default_ttl_is_1hour(self): @@ -149,19 +145,19 @@ def test_default_ttl_is_1hour(self): assert TokenDetails.DEFAULTS['ttl'] == one_hour_in_ms def test_with_auth_method(self): - ably = AblyRest(token='a token', auth_method='POST') + ably = create_http_client(token='a token', auth_method='POST') assert ably.auth.auth_options.auth_method == 'POST' def test_with_auth_headers(self): - ably = AblyRest(token='a token', auth_headers={'h1': 'v1'}) + ably = create_http_client(token='a token', auth_headers={'h1': 'v1'}) assert ably.auth.auth_options.auth_headers == {'h1': 'v1'} def test_with_auth_params(self): - ably = AblyRest(token='a token', auth_params={'p': 'v'}) + ably = create_http_client(token='a token', auth_params={'p': 'v'}) assert ably.auth.auth_options.auth_params == {'p': 'v'} def test_with_default_token_params(self): - ably = AblyRest(key=self.test_vars["keys"][0]["key_str"], + ably = create_http_client(key=self.test_vars["keys"][0]["key_str"], default_token_params={'ttl': 12345}) assert ably.auth.auth_options.default_token_params == {'ttl': 12345} diff --git a/test/ably/rest/resthttp_test.py b/test/ably/rest/resthttp_test.py index 1f5ffc7d..3a295e43 100644 --- a/test/ably/rest/resthttp_test.py +++ b/test/ably/rest/resthttp_test.py @@ -10,7 +10,7 @@ from httpx import Response from ably.pubsub.http.httputils import HttpUtils -from ably.pubsub.server import AblyRest +from ably.pubsub.server import create_http_client from ably.pubsub.transport.defaults import Defaults from ably.pubsub.types.options import Options from ably.pubsub.util.exceptions import AblyException @@ -29,7 +29,7 @@ async def test_host_for_url_brackets_only_ipv6_literals(self): assert HttpUtils.host_for_url('[::1]') == '[::1]' async def test_max_retry_attempts_and_timeouts_defaults(self): - ably = AblyRest(token="foo") + ably = create_http_client(token="foo") assert 'http_open_timeout' in ably.http.CONNECTION_RETRY_DEFAULTS assert 'http_request_timeout' in ably.http.CONNECTION_RETRY_DEFAULTS @@ -42,7 +42,7 @@ async def test_max_retry_attempts_and_timeouts_defaults(self): await ably.close() async def test_cumulative_timeout(self): - ably = AblyRest(token="foo") + ably = create_http_client(token="foo") assert 'http_max_retry_duration' in ably.http.CONNECTION_RETRY_DEFAULTS ably.options.http_max_retry_duration = 0.5 @@ -59,7 +59,7 @@ def sleep_and_raise(*args, **kwargs): await ably.close() async def test_host_fallback(self): - ably = AblyRest(token="foo") + ably = create_http_client(token="foo") def make_url(host): base_url = f"{ably.http.preferred_scheme}://{host}:{ably.http.preferred_port}" @@ -89,7 +89,7 @@ def make_url(host): @respx.mock async def test_no_host_fallback_nor_retries_if_custom_host(self): custom_host = 'example.org' - ably = AblyRest(token="foo", endpoint=custom_host) + ably = create_http_client(token="foo", endpoint=custom_host) mock_route = respx.get("https://example.org").mock(side_effect=httpx.RequestError('')) @@ -139,7 +139,7 @@ async def side_effect(*args, **kwargs): @respx.mock async def test_no_retry_if_not_500_to_599_http_code(self): default_host = Options().get_host() - ably = AblyRest(token="foo") + ably = create_http_client(token="foo") default_url = f"{ably.http.preferred_scheme}://{default_host}:{ably.http.preferred_port}/" @@ -162,7 +162,7 @@ async def test_500_errors(self): https://github.com/ably/ably-python/issues/160 """ - ably = AblyRest(token="foo") + ably = create_http_client(token="foo") mock_request = respx.route().mock(return_value=httpx.Response(500, text="Internal Server Error")) @@ -174,7 +174,7 @@ async def test_500_errors(self): await ably.close() def test_custom_http_timeouts(self): - ably = AblyRest( + ably = create_http_client( token="foo", http_request_timeout=30, http_open_timeout=8, http_max_retry_count=6, http_max_retry_duration=20) diff --git a/test/ably/rest/restinit_test.py b/test/ably/rest/restinit_test.py index cbc2fea8..929df98c 100644 --- a/test/ably/rest/restinit_test.py +++ b/test/ably/rest/restinit_test.py @@ -3,7 +3,7 @@ import pytest from httpx import AsyncClient -from ably.pubsub.server import AblyException, AblyRest +from ably.pubsub.server import AblyException, create_http_client from ably.pubsub.transport.defaults import Defaults from ably.pubsub.types.tokendetails import TokenDetails from test.ably.testapp import TestApp @@ -18,7 +18,7 @@ async def setup(self): @dont_vary_protocol def test_key_only(self): - ably = AblyRest(key=self.test_vars["keys"][0]["key_str"]) + ably = create_http_client(key=self.test_vars["keys"][0]["key_str"]) assert ably.options.key_name == self.test_vars["keys"][0]["key_name"], "Key name does not match" assert ably.options.key_secret == self.test_vars["keys"][0]["key_secret"], "Key secret does not match" @@ -27,65 +27,65 @@ def per_protocol_setup(self, use_binary_protocol): @dont_vary_protocol def test_with_token(self): - ably = AblyRest(token="foo") + ably = create_http_client(token="foo") assert ably.options.auth_token == "foo", "Token not set at options" @dont_vary_protocol def test_with_token_details(self): td = TokenDetails() - ably = AblyRest(token_details=td) + ably = create_http_client(token_details=td) assert ably.options.token_details is td @dont_vary_protocol def test_with_options_token_callback(self): def token_callback(**params): return "this_is_not_really_a_token_request" - AblyRest(auth_callback=token_callback) + create_http_client(auth_callback=token_callback) @dont_vary_protocol def test_ambiguous_key_raises_value_error(self): with pytest.raises(ValueError, match="mutually exclusive"): - AblyRest(key=self.test_vars["keys"][0]["key_str"], key_name='x') + create_http_client(key=self.test_vars["keys"][0]["key_str"], key_name='x') with pytest.raises(ValueError, match="mutually exclusive"): - AblyRest(key=self.test_vars["keys"][0]["key_str"], key_secret='x') + create_http_client(key=self.test_vars["keys"][0]["key_str"], key_secret='x') @dont_vary_protocol def test_with_key_name_or_secret_only(self): with pytest.raises(ValueError, match="key is missing"): - AblyRest(key_name='x') + create_http_client(key_name='x') with pytest.raises(ValueError, match="key is missing"): - AblyRest(key_secret='x') + create_http_client(key_secret='x') @dont_vary_protocol def test_with_key_name_and_secret(self): - ably = AblyRest(key_name="foo", key_secret="bar") + ably = create_http_client(key_name="foo", key_secret="bar") assert ably.options.key_name == "foo", "Key name does not match" assert ably.options.key_secret == "bar", "Key secret does not match" @dont_vary_protocol def test_with_options_auth_url(self): - AblyRest(auth_url='not_really_an_url') + create_http_client(auth_url='not_really_an_url') # RSC11 @dont_vary_protocol def test_rest_host_and_environment(self): # endpoint host - ably = AblyRest(token='foo', endpoint="some.other.host") + ably = create_http_client(token='foo', endpoint="some.other.host") assert "some.other.host" == ably.options.get_host(), "Unexpected host mismatch" # endpoint: main - ably = AblyRest(token='foo', endpoint="main") + ably = create_http_client(token='foo', endpoint="main") host = ably.options.get_host() assert "main.realtime.ably.net" == host, f"Unexpected host mismatch {host}" # endpoint: other - ably = AblyRest(token='foo', endpoint="nonprod:sandbox") + ably = create_http_client(token='foo', endpoint="nonprod:sandbox") host = ably.options.get_host() assert "sandbox.realtime.ably-nonprod.net" == host, f"Unexpected host mismatch {host}" # both, as per #TO3k2 with pytest.raises(AblyException): - ably = AblyRest(token='foo', rest_host="some.other.host", + ably = create_http_client(token='foo', rest_host="some.other.host", endpoint="some.other.environment") # RSC15 @@ -99,68 +99,68 @@ def test_fallback_hosts(self): # Fallback hosts specified (RSC15g1) for aux in fallback_hosts: - ably = AblyRest(token='foo', fallback_hosts=aux) + ably = create_http_client(token='foo', fallback_hosts=aux) assert sorted(aux) == sorted(ably.options.get_fallback_hosts()) # Specify endpoint (RSC15g2) - ably = AblyRest(token='foo', endpoint='nonprod:sandbox', http_max_retry_count=10) + ably = create_http_client(token='foo', endpoint='nonprod:sandbox', http_max_retry_count=10) assert sorted(Defaults.get_fallback_hosts('nonprod:sandbox')) == sorted( ably.options.get_fallback_hosts()) # Fallback hosts and endpoint not specified (RSC15g3) - ably = AblyRest(token='foo', http_max_retry_count=10) + ably = create_http_client(token='foo', http_max_retry_count=10) assert sorted(Defaults.get_fallback_hosts()) == sorted(ably.options.get_fallback_hosts()) # RSC15f - ably = AblyRest(token='foo') + ably = create_http_client(token='foo') assert 600000 == ably.options.fallback_retry_timeout - ably = AblyRest(token='foo', fallback_retry_timeout=1000) + ably = create_http_client(token='foo', fallback_retry_timeout=1000) assert 1000 == ably.options.fallback_retry_timeout @dont_vary_protocol def test_specified_host(self): - ably = AblyRest(token='foo', endpoint="some.other.host") + ably = create_http_client(token='foo', endpoint="some.other.host") assert "some.other.host" == ably.options.get_host(), "Unexpected host mismatch" @dont_vary_protocol def test_specified_port(self): - ably = AblyRest(token='foo', port=9998, tls_port=9999) + ably = create_http_client(token='foo', port=9998, tls_port=9999) assert 9999 == Defaults.get_port(ably.options),\ f"Unexpected port mismatch. Expected: 9999. Actual: {ably.options.tls_port}" @dont_vary_protocol def test_specified_non_tls_port(self): - ably = AblyRest(token='foo', port=9998, tls=False) + ably = create_http_client(token='foo', port=9998, tls=False) assert 9998 == Defaults.get_port(ably.options),\ f"Unexpected port mismatch. Expected: 9999. Actual: {ably.options.tls_port}" @dont_vary_protocol def test_specified_tls_port(self): - ably = AblyRest(token='foo', tls_port=9999, tls=True) + ably = create_http_client(token='foo', tls_port=9999, tls=True) assert 9999 == Defaults.get_port(ably.options),\ f"Unexpected port mismatch. Expected: 9999. Actual: {ably.options.tls_port}" @dont_vary_protocol def test_tls_defaults_to_true(self): - ably = AblyRest(token='foo') + ably = create_http_client(token='foo') assert ably.options.tls, "Expected encryption to default to true" assert Defaults.tls_port == Defaults.get_port(ably.options), "Unexpected port mismatch" @dont_vary_protocol def test_tls_can_be_disabled(self): - ably = AblyRest(token='foo', tls=False) + ably = create_http_client(token='foo', tls=False) assert not ably.options.tls, "Expected encryption to be False" assert Defaults.port == Defaults.get_port(ably.options), "Unexpected port mismatch" @dont_vary_protocol def test_with_no_params(self): with pytest.raises(ValueError): - AblyRest() + create_http_client() @dont_vary_protocol def test_with_no_auth_params(self): with pytest.raises(ValueError): - AblyRest(port=111) + create_http_client(port=111) # RSA10k async def test_query_time_param(self): @@ -181,19 +181,19 @@ async def test_query_time_param(self): @dont_vary_protocol def test_requests_over_https_production(self): - ably = AblyRest(token='token') + ably = create_http_client(token='token') assert 'https://main.realtime.ably.net' == f'{ably.http.preferred_scheme}://{ably.http.preferred_host}' assert ably.http.preferred_port == 443 @dont_vary_protocol def test_requests_over_http_production(self): - ably = AblyRest(token='token', tls=False) + ably = create_http_client(token='token', tls=False) assert 'http://main.realtime.ably.net' == f'{ably.http.preferred_scheme}://{ ably.http.preferred_host}' assert ably.http.preferred_port == 80 @dont_vary_protocol async def test_request_basic_auth_over_http_fails(self): - ably = AblyRest(key_secret='foo', key_name='bar', tls=False) + ably = create_http_client(key_secret='foo', key_name='bar', tls=False) with pytest.raises(AblyException) as excinfo: await ably.http.get('/time', skip_auth=False) @@ -204,7 +204,7 @@ async def test_request_basic_auth_over_http_fails(self): @dont_vary_protocol async def test_environment(self): - ably = AblyRest(token='token', endpoint='custom') + ably = create_http_client(token='token', endpoint='custom') with patch.object(AsyncClient, 'send', wraps=ably.http._Http__client.send) as get_mock: try: await ably.time() @@ -217,7 +217,7 @@ async def test_environment(self): @dont_vary_protocol def test_accepts_custom_http_timeouts(self): - ably = AblyRest( + ably = create_http_client( token="foo", http_request_timeout=30, http_open_timeout=8, http_max_retry_count=6, http_max_retry_duration=20) diff --git a/test/ably/rest/restrequest_test.py b/test/ably/rest/restrequest_test.py index 7398a9d4..bdeb725f 100644 --- a/test/ably/rest/restrequest_test.py +++ b/test/ably/rest/restrequest_test.py @@ -3,7 +3,7 @@ import respx from ably.pubsub.http.paginatedresult import HttpPaginatedResponse -from ably.pubsub.server import AblyRest +from ably.pubsub.server import create_http_client from ably.pubsub.transport.defaults import Defaults from test.ably.testapp import TestApp from test.ably.utils import BaseAsyncTestCase, VaryByProtocolTestsMetaclass, dont_vary_protocol @@ -98,7 +98,7 @@ async def test_headers(self): async def test_timeout(self): # Timeout timeout = 0.000001 - ably = AblyRest(token="foo", http_request_timeout=timeout) + ably = create_http_client(token="foo", http_request_timeout=timeout) assert ably.http.http_request_timeout == timeout # `http_open_timeout` keeps its four-second default, so the request carries a # connect budget as well as a read budget and either can be the one to expire @@ -122,7 +122,7 @@ async def test_timeout(self): await ably.close() # Bad host, no Fallback - ably = AblyRest(key=self.test_vars["keys"][0]["key_str"], + ably = create_http_client(key=self.test_vars["keys"][0]["key_str"], endpoint='some.other.host', port=self.test_vars["port"], tls_port=self.test_vars["tls_port"], diff --git a/test/ably/rest/resttoken_test.py b/test/ably/rest/resttoken_test.py index c0957bba..160e8eca 100644 --- a/test/ably/rest/resttoken_test.py +++ b/test/ably/rest/resttoken_test.py @@ -5,7 +5,7 @@ import pytest -from ably.pubsub.server import AblyException, AblyRest, Capability +from ably.pubsub.server import AblyException, Capability, create_http_client from ably.pubsub.types.tokendetails import TokenDetails from ably.pubsub.types.tokenrequest import TokenRequest from test.ably.testapp import TestApp @@ -314,7 +314,7 @@ async def auth_callback(token_params): @dont_vary_protocol async def test_hmac(self): - ably = AblyRest(key_name='a_key_name', key_secret='a_secret') + ably = create_http_client(key_name='a_key_name', key_secret='a_secret') token_params = { 'ttl': 1000, 'nonce': 'abcde100', diff --git a/test/ably/testapp.py b/test/ably/testapp.py index 8b086e10..2e1f9b96 100644 --- a/test/ably/testapp.py +++ b/test/ably/testapp.py @@ -2,8 +2,11 @@ import logging import os -from ably.pubsub.realtime.realtime import AblyRealtime -from ably.pubsub.rest.rest import AblyRest +# unasync rewrites `from ably.pubsub.server import ...` to the sync package, which +# is HTTP-only. Reaching the realtime factory through the module keeps the +# generated sync copy of this helper importable; it has no realtime tests. +import ably.pubsub.server +from ably.pubsub.server import create_http_client from ably.pubsub.transport.defaults import Defaults from ably.pubsub.types.capability import Capability from ably.pubsub.types.options import Options @@ -20,10 +23,10 @@ port = 80 tls_port = 443 -ably = AblyRest(token='not_a_real_token', - port=port, tls_port=tls_port, tls=tls, - endpoint=endpoint, - use_binary_protocol=False) +ably = create_http_client(token='not_a_real_token', + port=port, tls_port=tls_port, tls=tls, + endpoint=endpoint, + use_binary_protocol=False) class TestApp: @@ -65,7 +68,7 @@ async def get_ably_rest(**kw): test_vars = await TestApp.get_test_vars() options = TestApp.get_options(test_vars, **kw) options.update(kw) - return AblyRest(**options) + return create_http_client(**options) @staticmethod async def get_ably_realtime(**kw): @@ -76,7 +79,7 @@ async def get_ably_realtime(**kw): # for reaching CONNECTED, so a short one keeps a single failed attempt from using # up the whole budget. Tests that assert on retry timing set their own value. options.setdefault('disconnected_retry_timeout', 1000) - return AblyRealtime(**options) + return ably.pubsub.server.create_realtime_client(**options) @staticmethod def get_options(test_vars, **kwargs): diff --git a/test/unit/clock_test.py b/test/unit/clock_test.py index a4d2e9bd..b02a5fb6 100644 --- a/test/unit/clock_test.py +++ b/test/unit/clock_test.py @@ -1,7 +1,7 @@ import httpx import pytest -from ably.pubsub.server import AblyRest +from ably.pubsub.server import create_http_client from ably.pubsub.types.testoptions import TestOptions from ably.pubsub.util.clock import Clock from ably.pubsub.util.exceptions import AblyException @@ -56,12 +56,12 @@ async def handle_async_request(self, request): def test_auth_timestamps_from_the_injected_clock(): - ably = AblyRest(key='name:secret', _test_options=TestOptions(clock=FakeClock(1_500_000_000_000))) + ably = create_http_client(key='name:secret', _test_options=TestOptions(clock=FakeClock(1_500_000_000_000))) assert ably.auth._timestamp() == 1_500_000_000_000 def test_a_client_without_test_options_reads_the_real_clock(): - ably = AblyRest(token='foo') + ably = create_http_client(token='foo') assert isinstance(ably.auth._Auth__clock, Clock) assert isinstance(ably.http._Http__clock, Clock) @@ -70,8 +70,8 @@ async def test_the_cached_fallback_host_expires_on_the_clock(): # RSC15f: the cache lasts fallback_retry_timeout milliseconds clock = FakeClock() transport = RecordingTransport() - ably = AblyRest(token='foo', fallback_retry_timeout=2000, - _test_options=TestOptions(http_transport=transport, clock=clock)) + ably = create_http_client(token='foo', fallback_retry_timeout=2000, + _test_options=TestOptions(http_transport=transport, clock=clock)) primary = ably.options.get_host() transport.refusing = (primary,) @@ -96,7 +96,7 @@ async def test_every_host_is_tried_while_the_retry_budget_lasts(): # A second of clock time per reading, against the default 15 second budget clock = FakeClock(step_ms=1000) transport = RecordingTransport() - ably = AblyRest(token='foo', _test_options=TestOptions(http_transport=transport, clock=clock)) + ably = create_http_client(token='foo', _test_options=TestOptions(http_transport=transport, clock=clock)) hosts = ably.http.get_hosts() transport.refusing = tuple(hosts) @@ -112,8 +112,8 @@ async def test_retrying_stops_once_the_retry_budget_is_spent(): # RSC15l: http_max_retry_duration is seconds where the clock is milliseconds clock = FakeClock(step_ms=1000) transport = RecordingTransport() - ably = AblyRest(token='foo', http_max_retry_duration=0.5, - _test_options=TestOptions(http_transport=transport, clock=clock)) + ably = create_http_client(token='foo', http_max_retry_duration=0.5, + _test_options=TestOptions(http_transport=transport, clock=clock)) transport.refusing = tuple(ably.http.get_hosts()) with pytest.raises(AblyException): diff --git a/test/unit/http_test.py b/test/unit/http_test.py index a3a3cf6d..ebae07b2 100644 --- a/test/unit/http_test.py +++ b/test/unit/http_test.py @@ -1,13 +1,13 @@ import httpx import pytest -from ably.pubsub.server import AblyRest +from ably.pubsub.server import create_http_client from ably.pubsub.types.testoptions import TestOptions from ably.pubsub.util.exceptions import AblyException def test_http_get_rest_hosts_works_when_fallback_realtime_host_is_set(): - ably = AblyRest(token="foo") + ably = create_http_client(token="foo") ably.options.fallback_host = ably.options.get_hosts()[0] # Should not raise TypeError hosts = ably.http.get_hosts() @@ -16,7 +16,7 @@ def test_http_get_rest_hosts_works_when_fallback_realtime_host_is_set(): def test_http_get_rest_hosts_works_when_fallback_realtime_host_is_not_set(): - ably = AblyRest(token="foo") + ably = create_http_client(token="foo") ably.options.fallback_host = None # Should not raise TypeError hosts = ably.http.get_hosts() @@ -36,7 +36,7 @@ async def handle_async_request(self, request): async def test_http_sends_requests_through_an_injected_transport(): transport = RecordingTransport(lambda request: httpx.Response(200, json=[1500000000000])) - ably = AblyRest(token="foo", _test_options=TestOptions(http_transport=transport)) + ably = create_http_client(token="foo", _test_options=TestOptions(http_transport=transport)) server_time = await ably.time() @@ -52,7 +52,7 @@ def refuse(request): raise httpx.ConnectError("connection refused", request=request) transport = RecordingTransport(refuse) - ably = AblyRest(token="foo", _test_options=TestOptions(http_transport=transport)) + ably = create_http_client(token="foo", _test_options=TestOptions(http_transport=transport)) with pytest.raises(AblyException): await ably.time() @@ -63,7 +63,7 @@ def refuse(request): def test_http_uses_a_network_transport_without_test_options(): - ably = AblyRest(token="foo") + ably = create_http_client(token="foo") assert isinstance(ably.http._Http__client._transport, httpx.AsyncHTTPTransport) @@ -74,8 +74,8 @@ def respond(request): return httpx.Response(200, json=[1500000000000]) transport = RecordingTransport(respond) - ably = AblyRest(auth_url='https://auth.example.com/token', - _test_options=TestOptions(http_transport=transport)) + ably = create_http_client(auth_url='https://auth.example.com/token', + _test_options=TestOptions(http_transport=transport)) await ably.auth.authorize() diff --git a/test/unit/pubsub_server_test.py b/test/unit/pubsub_server_test.py new file mode 100644 index 00000000..8f1aa009 --- /dev/null +++ b/test/unit/pubsub_server_test.py @@ -0,0 +1,80 @@ +import pytest + +import ably.pubsub.server as server +import ably.pubsub.server.sync as server_sync +from ably.pubsub.prototypes import RealtimeClient, RestClient +from ably.pubsub.realtime.realtime import AblyRealtime +from ably.pubsub.rest.rest import AblyRest +from ably.pubsub.server import create_http_client, create_realtime_client +from ably.pubsub.sync.rest.rest import AblyRestSync + + +def protocol_members(protocol): + # Protocol records its declared members here; fall back to the class body + # on interpreters that do not expose it. + return getattr(protocol, '__protocol_attrs__', None) or { + name for name in vars(protocol) if not name.startswith('_abc_') and name != '_is_protocol' + } + + +class TestFactories: + def test_http_factory_returns_a_client(self): + client = create_http_client(token='foo') + assert isinstance(client, AblyRest) + + async def test_realtime_factory_returns_a_client(self): + client = create_realtime_client(key='foo:bar', auto_connect=False) + assert isinstance(client, AblyRealtime) + await client.close() + + def test_sync_http_factory_returns_a_client(self): + assert isinstance(server_sync.create_http_client(token='foo'), AblyRestSync) + + def test_factory_passes_options_through(self): + client = create_http_client(token='foo', client_id='me') + assert client.client_id == 'me' + + +class TestConstructorsAreInternal: + def test_rest_constructor_is_rejected(self): + with pytest.raises(TypeError) as excinfo: + AblyRest(token='foo') + assert 'ably.pubsub.server.create_http_client' in str(excinfo.value) + + def test_realtime_constructor_is_rejected(self): + with pytest.raises(TypeError) as excinfo: + AblyRealtime(key='foo:bar', auto_connect=False) + assert 'ably.pubsub.server.create_realtime_client' in str(excinfo.value) + + def test_sync_rest_constructor_is_rejected(self): + with pytest.raises(TypeError) as excinfo: + AblyRestSync(token='foo') + assert 'ably.pubsub.server.sync.create_http_client' in str(excinfo.value) + + def test_permission_does_not_leak_past_the_factory(self): + create_http_client(token='foo') + with pytest.raises(TypeError): + AblyRest(token='foo') + + @pytest.mark.parametrize('name', ['AblyRest', 'AblyRealtime']) + def test_client_classes_are_not_exported(self, name): + assert not hasattr(server, name) + assert name not in server.__all__ + + def test_sync_client_class_is_not_exported(self): + assert not hasattr(server_sync, 'AblyRestSync') + assert 'AblyRestSync' not in server_sync.__all__ + + +class TestPrototypes: + @pytest.mark.parametrize('protocol,cls', [(RestClient, AblyRest), (RealtimeClient, AblyRealtime)]) + def test_client_satisfies_its_prototype(self, protocol, cls): + missing = sorted(member for member in protocol_members(protocol) if not hasattr(cls, member)) + assert missing == [] + + @pytest.mark.parametrize('name', ['RestClient', 'RealtimeClient']) + def test_prototypes_are_exported(self, name): + assert name in server.__all__ + + def test_sync_prototype_is_exported(self): + assert 'RestClient' in server_sync.__all__ diff --git a/test/uts/README.md b/test/uts/README.md index acab71e5..8690d290 100644 --- a/test/uts/README.md +++ b/test/uts/README.md @@ -40,7 +40,7 @@ mock_http = MockHttpClient( on_connection_attempt=lambda conn: conn.respond_with_success(), on_request=lambda req: req.respond_with(200, {'result': 'ok'}), ) -ably = AblyRest(key=key, _test_options=TestOptions(http_transport=mock_http.as_transport())) +ably = create_http_client(key=key, _test_options=TestOptions(http_transport=mock_http.as_transport())) ``` A realtime client takes its websocket mock the same way, through @@ -51,9 +51,9 @@ A realtime client takes its websocket mock the same way, through mock_ws = MockWebSocket( on_connection_attempt=lambda conn: conn.respond_with_success(CONNECTED_MESSAGE), ) -ably = AblyRealtime(key=key, auto_connect=False, - _test_options=TestOptions(websocket_connect=mock_ws.as_connect(), - clock=FakeClock())) +ably = create_realtime_client(key=key, auto_connect=False, + _test_options=TestOptions(websocket_connect=mock_ws.as_connect(), + clock=FakeClock())) ``` `rest_client(mock_http, ...)` and `realtime_client(mock_ws, ...)` in diff --git a/test/uts/helpers/client.py b/test/uts/helpers/client.py index eaf1da3a..061a6f6e 100644 --- a/test/uts/helpers/client.py +++ b/test/uts/helpers/client.py @@ -4,7 +4,7 @@ import inspect import logging -from ably.pubsub.server import AblyRealtime, AblyRest +from ably.pubsub.server import create_http_client, create_realtime_client from ably.pubsub.types.testoptions import TestOptions from test.uts.helpers.clock import settle from test.uts.helpers.sandbox import SANDBOX_ENDPOINT @@ -41,7 +41,7 @@ def rest_client(mock_http, clock=None, **kwargs): if not any(option in kwargs for option in CREDENTIAL_OPTIONS): kwargs['key'] = DEFAULT_KEY test_options = TestOptions(http_transport=mock_http.as_transport(), clock=clock) - client = AblyRest(_test_options=test_options, **kwargs) + client = create_http_client(_test_options=test_options, **kwargs) __open_clients.append(client) return client @@ -63,7 +63,7 @@ def realtime_client(mock_websocket=None, mock_http=None, clock=None, **kwargs): kwargs['key'] = DEFAULT_KEY kwargs.setdefault('auto_connect', False) kwargs.setdefault('fallback_hosts', []) - client = AblyRealtime(_test_options=TestOptions( + client = create_realtime_client(_test_options=TestOptions( http_transport=mock_http.as_transport() if mock_http is not None else None, websocket_connect=mock_websocket.as_connect() if mock_websocket is not None else None, clock=clock, @@ -92,7 +92,7 @@ def sandbox_rest_client(key=None, **kwargs): kwargs['key'] = key kwargs.setdefault('endpoint', SANDBOX_ENDPOINT) kwargs.setdefault('use_binary_protocol', False) - client = AblyRest(**kwargs) + client = create_http_client(**kwargs) __open_clients.append(client) return client @@ -112,7 +112,7 @@ def sandbox_realtime_client(key=None, **kwargs): kwargs['key'] = key kwargs.setdefault('endpoint', SANDBOX_ENDPOINT) kwargs.setdefault('use_binary_protocol', False) - client = AblyRealtime(**kwargs) + client = create_realtime_client(**kwargs) __open_clients.append(client) return client diff --git a/test/uts/helpers/mock_http_test.py b/test/uts/helpers/mock_http_test.py index 6d4c1f59..e48d65a8 100644 --- a/test/uts/helpers/mock_http_test.py +++ b/test/uts/helpers/mock_http_test.py @@ -6,7 +6,7 @@ import msgpack import pytest -from ably.pubsub.server import AblyRest +from ably.pubsub.server import create_http_client from ably.pubsub.types.testoptions import TestOptions from ably.pubsub.util.exceptions import AblyException from test.uts.helpers.mock_http import MockHttpClient @@ -20,7 +20,7 @@ def rest_client(mock, **kwargs): """A REST client whose HTTP calls are served by `mock`.""" - return AblyRest(key=KEY, _test_options=TestOptions(http_transport=mock.as_transport()), **kwargs) + return create_http_client(key=KEY, _test_options=TestOptions(http_transport=mock.as_transport()), **kwargs) async def raw_get(ably, path='/time'): @@ -175,7 +175,8 @@ def connect(connection): mock = MockHttpClient(on_connection_attempt=connect, on_request=lambda request: request.respond_with(200, [SERVER_TIME])) - ably = AblyRest(token='foo', tls=False, _test_options=TestOptions(http_transport=mock.as_transport())) + ably = create_http_client(token='foo', tls=False, + _test_options=TestOptions(http_transport=mock.as_transport())) await ably.time() diff --git a/test/uts/realtime/integration/proxy/connection_open_failures_test.py b/test/uts/realtime/integration/proxy/connection_open_failures_test.py index 5f0d009d..9b7de332 100644 --- a/test/uts/realtime/integration/proxy/connection_open_failures_test.py +++ b/test/uts/realtime/integration/proxy/connection_open_failures_test.py @@ -27,7 +27,7 @@ """ from ably.pubsub.realtime.connection import ConnectionState -from ably.pubsub.server import AblyRest +from ably.pubsub.server import create_http_client from test.uts.helpers.client import await_connection_state, sandbox_realtime_client from test.uts.helpers.deviations import deviation from test.uts.helpers.mock_websocket import contains_in_order @@ -99,7 +99,7 @@ def token_auth_callback(api_key): again; the token arrives over a connection the proxy never sees. """ async def auth_callback(params): - inner_rest = AblyRest(key=api_key, endpoint=SANDBOX_ENDPOINT) + inner_rest = create_http_client(key=api_key, endpoint=SANDBOX_ENDPOINT) try: return await inner_rest.auth.request_token() finally: diff --git a/test/uts/realtime/integration/proxy/rest_faults_test.py b/test/uts/realtime/integration/proxy/rest_faults_test.py index 19deaeea..dcbf0f38 100644 --- a/test/uts/realtime/integration/proxy/rest_faults_test.py +++ b/test/uts/realtime/integration/proxy/rest_faults_test.py @@ -26,7 +26,7 @@ import pytest from ably.pubsub.realtime.connection import ConnectionState -from ably.pubsub.server import AblyRest +from ably.pubsub.server import create_http_client from ably.pubsub.types.channelstate import ChannelState from ably.pubsub.util.exceptions import AblyException from test.uts.helpers.client import ( @@ -67,7 +67,7 @@ def token_auth_callback(api_key, invocations=None): async def auth_callback(params): if invocations is not None: invocations.append(params) - inner_rest = AblyRest(key=api_key, endpoint=SANDBOX_ENDPOINT) + inner_rest = create_http_client(key=api_key, endpoint=SANDBOX_ENDPOINT) try: return await inner_rest.auth.request_token() finally: diff --git a/test/uts/rest/integration/proxy/rest_fallback_test.py b/test/uts/rest/integration/proxy/rest_fallback_test.py index fc8455b5..aed99207 100644 --- a/test/uts/rest/integration/proxy/rest_fallback_test.py +++ b/test/uts/rest/integration/proxy/rest_fallback_test.py @@ -19,7 +19,7 @@ import pytest -from ably.pubsub.server import AblyRest +from ably.pubsub.server import create_http_client from ably.pubsub.util.exceptions import AblyException from test.uts.helpers.client import sandbox_rest_client, wall_clock_poll_until from test.uts.helpers.deviations import deviation @@ -44,7 +44,7 @@ def token_auth_callback(api_key): it: the token arrives over a connection the proxy never sees. """ async def auth_callback(params): - inner_rest = AblyRest(key=api_key, endpoint=SANDBOX_ENDPOINT) + inner_rest = create_http_client(key=api_key, endpoint=SANDBOX_ENDPOINT) try: return await inner_rest.auth.request_token() finally: diff --git a/test/uts/rest/integration/time_stats_test.py b/test/uts/rest/integration/time_stats_test.py index d38f0c84..f402e800 100644 --- a/test/uts/rest/integration/time_stats_test.py +++ b/test/uts/rest/integration/time_stats_test.py @@ -9,7 +9,7 @@ import pytest_asyncio from ably.pubsub.http.paginatedresult import PaginatedResult -from ably.pubsub.server import AblyRest +from ably.pubsub.server import create_http_client from ably.pubsub.types.stats import Stats from test.uts.helpers.client import sandbox_rest_client from test.uts.helpers.sandbox import SANDBOX_ENDPOINT @@ -48,8 +48,8 @@ async def app_with_stats(sandbox): """ interval = (datetime.now(timezone.utc).replace(tzinfo=None) - STATS_INTERVAL_AGE).replace( second=0, microsecond=0) - client = AblyRest(key=sandbox.key(0).key_str, endpoint=SANDBOX_ENDPOINT, - use_binary_protocol=False) + client = create_http_client(key=sandbox.key(0).key_str, endpoint=SANDBOX_ENDPOINT, + use_binary_protocol=False) try: await client.http.post('/stats', body=[{ 'intervalId': Stats.to_interval_id(interval, 'minute'), diff --git a/test/uts/rest/unit/types/options_types_test.py b/test/uts/rest/unit/types/options_types_test.py index a835d57d..bc5848cd 100644 --- a/test/uts/rest/unit/types/options_types_test.py +++ b/test/uts/rest/unit/types/options_types_test.py @@ -5,7 +5,7 @@ import pytest -from ably.pubsub.server import AblyRest +from ably.pubsub.server import create_http_client from ably.pubsub.types.authoptions import AuthOptions from ably.pubsub.types.options import Options from ably.pubsub.types.tokendetails import TokenDetails @@ -191,5 +191,5 @@ async def auth_callback(params): # The missing-credentials check is a constructor argument check, so it raises ValueError # rather than an AblyException with pytest.raises(ValueError) as value_error: - AblyRest() + create_http_client() assert 'key' in str(value_error.value) diff --git a/uv.lock b/uv.lock index f0af7384..b8f70971 100644 --- a/uv.lock +++ b/uv.lock @@ -1,11 +1,10 @@ version = 1 revision = 3 -requires-python = ">=3.7" +requires-python = ">=3.8" resolution-markers = [ "python_full_version >= '3.10'", "python_full_version == '3.9.*'", - "python_full_version == '3.8.*'", - "python_full_version < '3.8'", + "python_full_version < '3.9'", ] [[package]] @@ -15,15 +14,11 @@ source = { editable = "." } dependencies = [ { name = "h2", version = "4.1.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.9'" }, { name = "h2", version = "4.3.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.9'" }, - { name = "httpx", version = "0.24.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.8'" }, - { name = "httpx", version = "0.28.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.8'" }, - { name = "msgpack", version = "1.0.5", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.8'" }, - { name = "msgpack", version = "1.1.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.8.*'" }, + { name = "httpx" }, + { name = "msgpack", version = "1.1.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.9'" }, { name = "msgpack", version = "1.1.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.9'" }, - { name = "pyee", version = "9.1.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.8'" }, - { name = "pyee", version = "13.0.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.8'" }, - { name = "websockets", version = "11.0.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.8'" }, - { name = "websockets", version = "13.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.8.*'" }, + { name = "pyee" }, + { name = "websockets", version = "13.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.9'" }, { name = "websockets", version = "15.0.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.9'" }, ] @@ -32,20 +27,16 @@ crypto = [ { name = "pycryptodome" }, ] dev = [ - { name = "async-case", marker = "python_full_version < '3.8'" }, { name = "importlib-metadata" }, { name = "mock" }, { name = "pytest" }, - { name = "pytest-asyncio", version = "0.21.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.8'" }, - { name = "pytest-asyncio", version = "0.23.8", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.8'" }, + { name = "pytest-asyncio" }, { name = "pytest-cov" }, { name = "pytest-timeout" }, { name = "pytest-xdist" }, - { name = "respx", version = "0.20.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.8'" }, - { name = "respx", version = "0.22.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.8'" }, + { name = "respx" }, { name = "ruff" }, - { name = "tokenize-rt", version = "5.0.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.8'" }, - { name = "tokenize-rt", version = "6.0.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.8.*'" }, + { name = "tokenize-rt", version = "6.0.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.9'" }, { name = "tokenize-rt", version = "6.2.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.9'" }, { name = "vcdiff-decoder" }, ] @@ -87,36 +78,18 @@ requires-dist = [ ] provides-extras = ["oldcrypto", "crypto", "vcdiff", "dev"] -[[package]] -name = "anyio" -version = "3.7.1" -source = { registry = "https://pypi.org/simple" } -resolution-markers = [ - "python_full_version < '3.8'", -] -dependencies = [ - { name = "exceptiongroup", marker = "python_full_version < '3.8'" }, - { name = "idna", version = "3.10", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.8'" }, - { name = "sniffio", marker = "python_full_version < '3.8'" }, - { name = "typing-extensions", version = "4.7.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.8'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/28/99/2dfd53fd55ce9838e6ff2d4dac20ce58263798bd1a0dbe18b3a9af3fcfce/anyio-3.7.1.tar.gz", hash = "sha256:44a3c9aba0f5defa43261a8b3efb97891f2bd7d804e0e1f56419befa1adfc780", size = 142927, upload-time = "2023-07-05T16:45:02.294Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/19/24/44299477fe7dcc9cb58d0a57d5a7588d6af2ff403fdd2d47a246c91a3246/anyio-3.7.1-py3-none-any.whl", hash = "sha256:91dee416e570e92c64041bd18b900d1d6fa78dff7048769ce5ac5ddad004fbb5", size = 80896, upload-time = "2023-07-05T16:44:59.805Z" }, -] - [[package]] name = "anyio" version = "4.5.2" source = { registry = "https://pypi.org/simple" } resolution-markers = [ - "python_full_version == '3.8.*'", + "python_full_version < '3.9'", ] dependencies = [ - { name = "exceptiongroup", marker = "python_full_version == '3.8.*'" }, - { name = "idna", version = "3.11", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.8.*'" }, - { name = "sniffio", marker = "python_full_version == '3.8.*'" }, - { name = "typing-extensions", version = "4.13.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.8.*'" }, + { name = "exceptiongroup", marker = "python_full_version < '3.9'" }, + { name = "idna", marker = "python_full_version < '3.9'" }, + { name = "sniffio", marker = "python_full_version < '3.9'" }, + { name = "typing-extensions", version = "4.13.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.9'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/4d/f9/9a7ce600ebe7804daf90d4d48b1c0510a4561ddce43a596be46676f82343/anyio-4.5.2.tar.gz", hash = "sha256:23009af4ed04ce05991845451e11ef02fc7c5ed29179ac9a420e5ad0ac7ddc5b", size = 171293, upload-time = "2024-10-13T22:18:03.307Z" } wheels = [ @@ -133,7 +106,7 @@ resolution-markers = [ ] dependencies = [ { name = "exceptiongroup", marker = "python_full_version >= '3.9' and python_full_version < '3.11'" }, - { name = "idna", version = "3.11", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.9'" }, + { name = "idna", marker = "python_full_version >= '3.9'" }, { name = "sniffio", marker = "python_full_version >= '3.9'" }, { name = "typing-extensions", version = "4.15.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.9' and python_full_version < '3.13'" }, ] @@ -142,12 +115,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/15/b3/9b1a8074496371342ec1e796a96f99c82c945a339cd81a8e73de28b4cf9e/anyio-4.11.0-py3-none-any.whl", hash = "sha256:0287e96f4d26d4149305414d4e3bc32f0dcd0862365a4bddea19d7a1ec38c4fc", size = 109097, upload-time = "2025-09-23T09:19:10.601Z" }, ] -[[package]] -name = "async-case" -version = "10.1.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/c8/09/87f2a23f5696ac6deb2fff92421f8af46226ea2410d101b453d5aa63e53a/async_case-10.1.0.tar.gz", hash = "sha256:b819f68c78f6c640ab1101ecf69fac189402b490901fa2abc314c48edab5d3da", size = 3668, upload-time = "2022-03-15T21:56:16.795Z" } - [[package]] name = "certifi" version = "2026.1.4" @@ -166,82 +133,12 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/d1/d6/3965ed04c63042e047cb6a3e6ed1a63a35087b6a609aa3a15ed8ac56c221/colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6", size = 25335, upload-time = "2022-10-25T02:36:20.889Z" }, ] -[[package]] -name = "coverage" -version = "7.2.7" -source = { registry = "https://pypi.org/simple" } -resolution-markers = [ - "python_full_version < '3.8'", -] -sdist = { url = "https://files.pythonhosted.org/packages/45/8b/421f30467e69ac0e414214856798d4bc32da1336df745e49e49ae5c1e2a8/coverage-7.2.7.tar.gz", hash = "sha256:924d94291ca674905fe9481f12294eb11f2d3d3fd1adb20314ba89e94f44ed59", size = 762575, upload-time = "2023-05-29T20:08:50.273Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/01/24/be01e62a7bce89bcffe04729c540382caa5a06bee45ae42136c93e2499f5/coverage-7.2.7-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:d39b5b4f2a66ccae8b7263ac3c8170994b65266797fb96cbbfd3fb5b23921db8", size = 200724, upload-time = "2023-05-29T20:07:03.422Z" }, - { url = "https://files.pythonhosted.org/packages/3d/80/7060a445e1d2c9744b683dc935248613355657809d6c6b2716cdf4ca4766/coverage-7.2.7-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:6d040ef7c9859bb11dfeb056ff5b3872436e3b5e401817d87a31e1750b9ae2fb", size = 201024, upload-time = "2023-05-29T20:07:05.694Z" }, - { url = "https://files.pythonhosted.org/packages/b8/9d/926fce7e03dbfc653104c2d981c0fa71f0572a9ebd344d24c573bd6f7c4f/coverage-7.2.7-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ba90a9563ba44a72fda2e85302c3abc71c5589cea608ca16c22b9804262aaeb6", size = 229528, upload-time = "2023-05-29T20:07:07.307Z" }, - { url = "https://files.pythonhosted.org/packages/d1/3a/67f5d18f911abf96857f6f7e4df37ca840e38179e2cc9ab6c0b9c3380f19/coverage-7.2.7-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e7d9405291c6928619403db1d10bd07888888ec1abcbd9748fdaa971d7d661b2", size = 227842, upload-time = "2023-05-29T20:07:09.331Z" }, - { url = "https://files.pythonhosted.org/packages/b4/bd/1b2331e3a04f4cc9b7b332b1dd0f3a1261dfc4114f8479bebfcc2afee9e8/coverage-7.2.7-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:31563e97dae5598556600466ad9beea39fb04e0229e61c12eaa206e0aa202063", size = 228717, upload-time = "2023-05-29T20:07:11.38Z" }, - { url = "https://files.pythonhosted.org/packages/2b/86/3dbf9be43f8bf6a5ca28790a713e18902b2d884bc5fa9512823a81dff601/coverage-7.2.7-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:ebba1cd308ef115925421d3e6a586e655ca5a77b5bf41e02eb0e4562a111f2d1", size = 234632, upload-time = "2023-05-29T20:07:13.376Z" }, - { url = "https://files.pythonhosted.org/packages/91/e8/469ed808a782b9e8305a08bad8c6fa5f8e73e093bda6546c5aec68275bff/coverage-7.2.7-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:cb017fd1b2603ef59e374ba2063f593abe0fc45f2ad9abdde5b4d83bd922a353", size = 232875, upload-time = "2023-05-29T20:07:15.093Z" }, - { url = "https://files.pythonhosted.org/packages/29/8f/4fad1c2ba98104425009efd7eaa19af9a7c797e92d40cd2ec026fa1f58cb/coverage-7.2.7-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:d62a5c7dad11015c66fbb9d881bc4caa5b12f16292f857842d9d1871595f4495", size = 234094, upload-time = "2023-05-29T20:07:17.013Z" }, - { url = "https://files.pythonhosted.org/packages/94/4e/d4e46a214ae857be3d7dc5de248ba43765f60daeb1ab077cb6c1536c7fba/coverage-7.2.7-cp310-cp310-win32.whl", hash = "sha256:ee57190f24fba796e36bb6d3aa8a8783c643d8fa9760c89f7a98ab5455fbf818", size = 203184, upload-time = "2023-05-29T20:07:18.69Z" }, - { url = "https://files.pythonhosted.org/packages/1f/e9/d6730247d8dec2a3dddc520ebe11e2e860f0f98cee3639e23de6cf920255/coverage-7.2.7-cp310-cp310-win_amd64.whl", hash = "sha256:f75f7168ab25dd93110c8a8117a22450c19976afbc44234cbf71481094c1b850", size = 204096, upload-time = "2023-05-29T20:07:20.153Z" }, - { url = "https://files.pythonhosted.org/packages/c6/fa/529f55c9a1029c840bcc9109d5a15ff00478b7ff550a1ae361f8745f8ad5/coverage-7.2.7-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:06a9a2be0b5b576c3f18f1a241f0473575c4a26021b52b2a85263a00f034d51f", size = 200895, upload-time = "2023-05-29T20:07:21.963Z" }, - { url = "https://files.pythonhosted.org/packages/67/d7/cd8fe689b5743fffac516597a1222834c42b80686b99f5b44ef43ccc2a43/coverage-7.2.7-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:5baa06420f837184130752b7c5ea0808762083bf3487b5038d68b012e5937dbe", size = 201120, upload-time = "2023-05-29T20:07:23.765Z" }, - { url = "https://files.pythonhosted.org/packages/8c/95/16eed713202406ca0a37f8ac259bbf144c9d24f9b8097a8e6ead61da2dbb/coverage-7.2.7-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fdec9e8cbf13a5bf63290fc6013d216a4c7232efb51548594ca3631a7f13c3a3", size = 233178, upload-time = "2023-05-29T20:07:25.281Z" }, - { url = "https://files.pythonhosted.org/packages/c1/49/4d487e2ad5d54ed82ac1101e467e8994c09d6123c91b2a962145f3d262c2/coverage-7.2.7-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:52edc1a60c0d34afa421c9c37078817b2e67a392cab17d97283b64c5833f427f", size = 230754, upload-time = "2023-05-29T20:07:27.044Z" }, - { url = "https://files.pythonhosted.org/packages/a7/cd/3ce94ad9d407a052dc2a74fbeb1c7947f442155b28264eb467ee78dea812/coverage-7.2.7-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:63426706118b7f5cf6bb6c895dc215d8a418d5952544042c8a2d9fe87fcf09cb", size = 232558, upload-time = "2023-05-29T20:07:28.743Z" }, - { url = "https://files.pythonhosted.org/packages/8f/a8/12cc7b261f3082cc299ab61f677f7e48d93e35ca5c3c2f7241ed5525ccea/coverage-7.2.7-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:afb17f84d56068a7c29f5fa37bfd38d5aba69e3304af08ee94da8ed5b0865833", size = 241509, upload-time = "2023-05-29T20:07:30.434Z" }, - { url = "https://files.pythonhosted.org/packages/04/fa/43b55101f75a5e9115259e8be70ff9279921cb6b17f04c34a5702ff9b1f7/coverage-7.2.7-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:48c19d2159d433ccc99e729ceae7d5293fbffa0bdb94952d3579983d1c8c9d97", size = 239924, upload-time = "2023-05-29T20:07:32.065Z" }, - { url = "https://files.pythonhosted.org/packages/68/5f/d2bd0f02aa3c3e0311986e625ccf97fdc511b52f4f1a063e4f37b624772f/coverage-7.2.7-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:0e1f928eaf5469c11e886fe0885ad2bf1ec606434e79842a879277895a50942a", size = 240977, upload-time = "2023-05-29T20:07:34.184Z" }, - { url = "https://files.pythonhosted.org/packages/ba/92/69c0722882643df4257ecc5437b83f4c17ba9e67f15dc6b77bad89b6982e/coverage-7.2.7-cp311-cp311-win32.whl", hash = "sha256:33d6d3ea29d5b3a1a632b3c4e4f4ecae24ef170b0b9ee493883f2df10039959a", size = 203168, upload-time = "2023-05-29T20:07:35.869Z" }, - { url = "https://files.pythonhosted.org/packages/b1/96/c12ed0dfd4ec587f3739f53eb677b9007853fd486ccb0e7d5512a27bab2e/coverage-7.2.7-cp311-cp311-win_amd64.whl", hash = "sha256:5b7540161790b2f28143191f5f8ec02fb132660ff175b7747b95dcb77ac26562", size = 204185, upload-time = "2023-05-29T20:07:37.39Z" }, - { url = "https://files.pythonhosted.org/packages/ff/d5/52fa1891d1802ab2e1b346d37d349cb41cdd4fd03f724ebbf94e80577687/coverage-7.2.7-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:f2f67fe12b22cd130d34d0ef79206061bfb5eda52feb6ce0dba0644e20a03cf4", size = 201020, upload-time = "2023-05-29T20:07:38.724Z" }, - { url = "https://files.pythonhosted.org/packages/24/df/6765898d54ea20e3197a26d26bb65b084deefadd77ce7de946b9c96dfdc5/coverage-7.2.7-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a342242fe22407f3c17f4b499276a02b01e80f861f1682ad1d95b04018e0c0d4", size = 233994, upload-time = "2023-05-29T20:07:40.274Z" }, - { url = "https://files.pythonhosted.org/packages/15/81/b108a60bc758b448c151e5abceed027ed77a9523ecbc6b8a390938301841/coverage-7.2.7-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:171717c7cb6b453aebac9a2ef603699da237f341b38eebfee9be75d27dc38e01", size = 231358, upload-time = "2023-05-29T20:07:41.998Z" }, - { url = "https://files.pythonhosted.org/packages/61/90/c76b9462f39897ebd8714faf21bc985b65c4e1ea6dff428ea9dc711ed0dd/coverage-7.2.7-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:49969a9f7ffa086d973d91cec8d2e31080436ef0fb4a359cae927e742abfaaa6", size = 233316, upload-time = "2023-05-29T20:07:43.539Z" }, - { url = "https://files.pythonhosted.org/packages/04/d6/8cba3bf346e8b1a4fb3f084df7d8cea25a6b6c56aaca1f2e53829be17e9e/coverage-7.2.7-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:b46517c02ccd08092f4fa99f24c3b83d8f92f739b4657b0f146246a0ca6a831d", size = 240159, upload-time = "2023-05-29T20:07:44.982Z" }, - { url = "https://files.pythonhosted.org/packages/6e/ea/4a252dc77ca0605b23d477729d139915e753ee89e4c9507630e12ad64a80/coverage-7.2.7-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:a3d33a6b3eae87ceaefa91ffdc130b5e8536182cd6dfdbfc1aa56b46ff8c86de", size = 238127, upload-time = "2023-05-29T20:07:46.522Z" }, - { url = "https://files.pythonhosted.org/packages/9f/5c/d9760ac497c41f9c4841f5972d0edf05d50cad7814e86ee7d133ec4a0ac8/coverage-7.2.7-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:976b9c42fb2a43ebf304fa7d4a310e5f16cc99992f33eced91ef6f908bd8f33d", size = 239833, upload-time = "2023-05-29T20:07:47.992Z" }, - { url = "https://files.pythonhosted.org/packages/69/8c/26a95b08059db1cbb01e4b0e6d40f2e9debb628c6ca86b78f625ceaf9bab/coverage-7.2.7-cp312-cp312-win32.whl", hash = "sha256:8de8bb0e5ad103888d65abef8bca41ab93721647590a3f740100cd65c3b00511", size = 203463, upload-time = "2023-05-29T20:07:49.939Z" }, - { url = "https://files.pythonhosted.org/packages/b7/00/14b00a0748e9eda26e97be07a63cc911108844004687321ddcc213be956c/coverage-7.2.7-cp312-cp312-win_amd64.whl", hash = "sha256:9e31cb64d7de6b6f09702bb27c02d1904b3aebfca610c12772452c4e6c21a0d3", size = 204347, upload-time = "2023-05-29T20:07:51.909Z" }, - { url = "https://files.pythonhosted.org/packages/80/d7/67937c80b8fd4c909fdac29292bc8b35d9505312cff6bcab41c53c5b1df6/coverage-7.2.7-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:58c2ccc2f00ecb51253cbe5d8d7122a34590fac9646a960d1430d5b15321d95f", size = 200580, upload-time = "2023-05-29T20:07:54.076Z" }, - { url = "https://files.pythonhosted.org/packages/7a/05/084864fa4bbf8106f44fb72a56e67e0cd372d3bf9d893be818338c81af5d/coverage-7.2.7-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d22656368f0e6189e24722214ed8d66b8022db19d182927b9a248a2a8a2f67eb", size = 226237, upload-time = "2023-05-29T20:07:56.28Z" }, - { url = "https://files.pythonhosted.org/packages/67/a2/6fa66a50e6e894286d79a3564f42bd54a9bd27049dc0a63b26d9924f0aa3/coverage-7.2.7-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a895fcc7b15c3fc72beb43cdcbdf0ddb7d2ebc959edac9cef390b0d14f39f8a9", size = 224256, upload-time = "2023-05-29T20:07:58.189Z" }, - { url = "https://files.pythonhosted.org/packages/e2/c0/73f139794c742840b9ab88e2e17fe14a3d4668a166ff95d812ac66c0829d/coverage-7.2.7-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e84606b74eb7de6ff581a7915e2dab7a28a0517fbe1c9239eb227e1354064dcd", size = 225550, upload-time = "2023-05-29T20:08:00.383Z" }, - { url = "https://files.pythonhosted.org/packages/03/ec/6f30b4e0c96ce03b0e64aec46b4af2a8c49b70d1b5d0d69577add757b946/coverage-7.2.7-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:0a5f9e1dbd7fbe30196578ca36f3fba75376fb99888c395c5880b355e2875f8a", size = 232440, upload-time = "2023-05-29T20:08:02.495Z" }, - { url = "https://files.pythonhosted.org/packages/22/c1/2f6c1b6f01a0996c9e067a9c780e1824351dbe17faae54388a4477e6d86f/coverage-7.2.7-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:419bfd2caae268623dd469eff96d510a920c90928b60f2073d79f8fe2bbc5959", size = 230897, upload-time = "2023-05-29T20:08:04.382Z" }, - { url = "https://files.pythonhosted.org/packages/8d/d6/53e999ec1bf7498ca4bc5f3b8227eb61db39068d2de5dcc359dec5601b5a/coverage-7.2.7-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:2aee274c46590717f38ae5e4650988d1af340fe06167546cc32fe2f58ed05b02", size = 232024, upload-time = "2023-05-29T20:08:06.031Z" }, - { url = "https://files.pythonhosted.org/packages/e9/40/383305500d24122dbed73e505a4d6828f8f3356d1f68ab6d32c781754b81/coverage-7.2.7-cp37-cp37m-win32.whl", hash = "sha256:61b9a528fb348373c433e8966535074b802c7a5d7f23c4f421e6c6e2f1697a6f", size = 203293, upload-time = "2023-05-29T20:08:07.598Z" }, - { url = "https://files.pythonhosted.org/packages/0e/bc/7e3a31534fabb043269f14fb64e2bb2733f85d4cf39e5bbc71357c57553a/coverage-7.2.7-cp37-cp37m-win_amd64.whl", hash = "sha256:b1c546aca0ca4d028901d825015dc8e4d56aac4b541877690eb76490f1dc8ed0", size = 204040, upload-time = "2023-05-29T20:08:09.919Z" }, - { url = "https://files.pythonhosted.org/packages/c6/fc/be19131010930a6cf271da48202c8cc1d3f971f68c02fb2d3a78247f43dc/coverage-7.2.7-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:54b896376ab563bd38453cecb813c295cf347cf5906e8b41d340b0321a5433e5", size = 200689, upload-time = "2023-05-29T20:08:11.594Z" }, - { url = "https://files.pythonhosted.org/packages/28/d7/9a8de57d87f4bbc6f9a6a5ded1eaac88a89bf71369bb935dac3c0cf2893e/coverage-7.2.7-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:3d376df58cc111dc8e21e3b6e24606b5bb5dee6024f46a5abca99124b2229ef5", size = 200986, upload-time = "2023-05-29T20:08:13.228Z" }, - { url = "https://files.pythonhosted.org/packages/c8/e4/e6182e4697665fb594a7f4e4f27cb3a4dd00c2e3d35c5c706765de8c7866/coverage-7.2.7-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5e330fc79bd7207e46c7d7fd2bb4af2963f5f635703925543a70b99574b0fea9", size = 230648, upload-time = "2023-05-29T20:08:15.11Z" }, - { url = "https://files.pythonhosted.org/packages/7b/e3/f552d5871943f747165b92a924055c5d6daa164ae659a13f9018e22f3990/coverage-7.2.7-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1e9d683426464e4a252bf70c3498756055016f99ddaec3774bf368e76bbe02b6", size = 228511, upload-time = "2023-05-29T20:08:16.877Z" }, - { url = "https://files.pythonhosted.org/packages/44/55/49f65ccdd4dfd6d5528e966b28c37caec64170c725af32ab312889d2f857/coverage-7.2.7-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8d13c64ee2d33eccf7437961b6ea7ad8673e2be040b4f7fd4fd4d4d28d9ccb1e", size = 229852, upload-time = "2023-05-29T20:08:18.47Z" }, - { url = "https://files.pythonhosted.org/packages/0d/31/340428c238eb506feb96d4fb5c9ea614db1149517f22cc7ab8c6035ef6d9/coverage-7.2.7-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:b7aa5f8a41217360e600da646004f878250a0d6738bcdc11a0a39928d7dc2050", size = 235578, upload-time = "2023-05-29T20:08:20.298Z" }, - { url = "https://files.pythonhosted.org/packages/dd/ce/97c1dd6592c908425622fe7f31c017d11cf0421729b09101d4de75bcadc8/coverage-7.2.7-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:8fa03bce9bfbeeef9f3b160a8bed39a221d82308b4152b27d82d8daa7041fee5", size = 234079, upload-time = "2023-05-29T20:08:22.365Z" }, - { url = "https://files.pythonhosted.org/packages/de/a3/5a98dc9e239d0dc5f243ef5053d5b1bdcaa1dee27a691dfc12befeccf878/coverage-7.2.7-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:245167dd26180ab4c91d5e1496a30be4cd721a5cf2abf52974f965f10f11419f", size = 234991, upload-time = "2023-05-29T20:08:24.974Z" }, - { url = "https://files.pythonhosted.org/packages/4a/fb/78986d3022e5ccf2d4370bc43a5fef8374f092b3c21d32499dee8e30b7b6/coverage-7.2.7-cp38-cp38-win32.whl", hash = "sha256:d2c2db7fd82e9b72937969bceac4d6ca89660db0a0967614ce2481e81a0b771e", size = 203160, upload-time = "2023-05-29T20:08:26.701Z" }, - { url = "https://files.pythonhosted.org/packages/c3/1c/6b3c9c363fb1433c79128e0d692863deb761b1b78162494abb9e5c328bc0/coverage-7.2.7-cp38-cp38-win_amd64.whl", hash = "sha256:2e07b54284e381531c87f785f613b833569c14ecacdcb85d56b25c4622c16c3c", size = 204085, upload-time = "2023-05-29T20:08:28.146Z" }, - { url = "https://files.pythonhosted.org/packages/88/da/495944ebf0ad246235a6bd523810d9f81981f9b81c6059ba1f56e943abe0/coverage-7.2.7-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:537891ae8ce59ef63d0123f7ac9e2ae0fc8b72c7ccbe5296fec45fd68967b6c9", size = 200725, upload-time = "2023-05-29T20:08:29.851Z" }, - { url = "https://files.pythonhosted.org/packages/ca/0c/3dfeeb1006c44b911ee0ed915350db30325d01808525ae7cc8d57643a2ce/coverage-7.2.7-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:06fb182e69f33f6cd1d39a6c597294cff3143554b64b9825d1dc69d18cc2fff2", size = 201022, upload-time = "2023-05-29T20:08:31.429Z" }, - { url = "https://files.pythonhosted.org/packages/61/af/5964b8d7d9a5c767785644d9a5a63cacba9a9c45cc42ba06d25895ec87be/coverage-7.2.7-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:201e7389591af40950a6480bd9edfa8ed04346ff80002cec1a66cac4549c1ad7", size = 229102, upload-time = "2023-05-29T20:08:32.982Z" }, - { url = "https://files.pythonhosted.org/packages/d9/1d/cd467fceb62c371f9adb1d739c92a05d4e550246daa90412e711226bd320/coverage-7.2.7-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f6951407391b639504e3b3be51b7ba5f3528adbf1a8ac3302b687ecababf929e", size = 227441, upload-time = "2023-05-29T20:08:35.044Z" }, - { url = "https://files.pythonhosted.org/packages/fe/57/e4f8ad64d84ca9e759d783a052795f62a9f9111585e46068845b1cb52c2b/coverage-7.2.7-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6f48351d66575f535669306aa7d6d6f71bc43372473b54a832222803eb956fd1", size = 228265, upload-time = "2023-05-29T20:08:36.861Z" }, - { url = "https://files.pythonhosted.org/packages/88/8b/b0d9fe727acae907fa7f1c8194ccb6fe9d02e1c3e9001ecf74c741f86110/coverage-7.2.7-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:b29019c76039dc3c0fd815c41392a044ce555d9bcdd38b0fb60fb4cd8e475ba9", size = 234217, upload-time = "2023-05-29T20:08:38.837Z" }, - { url = "https://files.pythonhosted.org/packages/66/2e/c99fe1f6396d93551aa352c75410686e726cd4ea104479b9af1af22367ce/coverage-7.2.7-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:81c13a1fc7468c40f13420732805a4c38a105d89848b7c10af65a90beff25250", size = 232466, upload-time = "2023-05-29T20:08:40.768Z" }, - { url = "https://files.pythonhosted.org/packages/bb/e9/88747b40c8fb4a783b40222510ce6d66170217eb05d7f46462c36b4fa8cc/coverage-7.2.7-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:975d70ab7e3c80a3fe86001d8751f6778905ec723f5b110aed1e450da9d4b7f2", size = 233669, upload-time = "2023-05-29T20:08:42.944Z" }, - { url = "https://files.pythonhosted.org/packages/b1/d5/a8e276bc005e42114468d4fe03e0a9555786bc51cbfe0d20827a46c1565a/coverage-7.2.7-cp39-cp39-win32.whl", hash = "sha256:7ee7d9d4822c8acc74a5e26c50604dff824710bc8de424904c0982e25c39c6cb", size = 203199, upload-time = "2023-05-29T20:08:44.734Z" }, - { url = "https://files.pythonhosted.org/packages/a9/0c/4a848ae663b47f1195abcb09a951751dd61f80b503303b9b9d768e0fd321/coverage-7.2.7-cp39-cp39-win_amd64.whl", hash = "sha256:eb393e5ebc85245347950143969b241d08b52b88a3dc39479822e073a1a8eb27", size = 204109, upload-time = "2023-05-29T20:08:46.417Z" }, - { url = "https://files.pythonhosted.org/packages/67/fb/b3b1d7887e1ea25a9608b0776e480e4bbc303ca95a31fd585555ec4fff5a/coverage-7.2.7-pp37.pp38.pp39-none-any.whl", hash = "sha256:b7b4c971f05e6ae490fef852c218b0e79d4e52f79ef0c8475566584a8fb3e01d", size = 193207, upload-time = "2023-05-29T20:08:48.153Z" }, -] - [[package]] name = "coverage" version = "7.6.1" source = { registry = "https://pypi.org/simple" } resolution-markers = [ - "python_full_version == '3.8.*'", + "python_full_version < '3.9'", ] sdist = { url = "https://files.pythonhosted.org/packages/f7/08/7e37f82e4d1aead42a7443ff06a1e406aabf7302c4f00a546e4b320b994c/coverage-7.6.1.tar.gz", hash = "sha256:953510dfb7b12ab69d20135a0662397f077c59b1e6379a768e97c59d852ee51d", size = 798791, upload-time = "2024-08-04T19:45:30.9Z" } wheels = [ @@ -539,8 +436,7 @@ name = "exceptiongroup" version = "1.3.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "typing-extensions", version = "4.7.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.8'" }, - { name = "typing-extensions", version = "4.13.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.8.*'" }, + { name = "typing-extensions", version = "4.13.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.9'" }, { name = "typing-extensions", version = "4.15.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.9' and python_full_version < '3.13'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/0b/9f/a65090624ecf468cdca03533906e7c69ed7588582240cfe7cc9e770b50eb/exceptiongroup-1.3.0.tar.gz", hash = "sha256:b241f5885f560bc56a59ee63ca4c6a8bfa46ae4ad651af316d4e81817bb9fd88", size = 29749, upload-time = "2025-05-10T17:42:51.123Z" } @@ -548,56 +444,19 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/36/f4/c6e662dade71f56cd2f3735141b265c3c79293c109549c1e6933b0651ffc/exceptiongroup-1.3.0-py3-none-any.whl", hash = "sha256:4d111e6e0c13d0644cad6ddaa7ed0261a0b36971f6d23e7ec9b4b9097da78a10", size = 16674, upload-time = "2025-05-10T17:42:49.33Z" }, ] -[[package]] -name = "execnet" -version = "2.0.2" -source = { registry = "https://pypi.org/simple" } -resolution-markers = [ - "python_full_version < '3.8'", -] -sdist = { url = "https://files.pythonhosted.org/packages/e4/c8/d382dc7a1e68a165f4a4ab612a08b20d8534a7d20cc590630b734ca0c54b/execnet-2.0.2.tar.gz", hash = "sha256:cc59bc4423742fd71ad227122eb0dd44db51efb3dc4095b45ac9a08c770096af", size = 161098, upload-time = "2023-07-09T17:14:03.468Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/e8/9c/a079946da30fac4924d92dbc617e5367d454954494cf1e71567bcc4e00ee/execnet-2.0.2-py3-none-any.whl", hash = "sha256:88256416ae766bc9e8895c76a87928c0012183da3cc4fc18016e6f050e025f41", size = 37097, upload-time = "2023-07-09T17:14:01.888Z" }, -] - [[package]] name = "execnet" version = "2.1.2" source = { registry = "https://pypi.org/simple" } -resolution-markers = [ - "python_full_version >= '3.10'", - "python_full_version == '3.9.*'", - "python_full_version == '3.8.*'", -] sdist = { url = "https://files.pythonhosted.org/packages/bf/89/780e11f9588d9e7128a3f87788354c7946a9cbb1401ad38a48c4db9a4f07/execnet-2.1.2.tar.gz", hash = "sha256:63d83bfdd9a23e35b9c6a3261412324f964c2ec8dcd8d3c6916ee9373e0befcd", size = 166622, upload-time = "2025-11-12T09:56:37.75Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/ab/84/02fc1827e8cdded4aa65baef11296a9bbe595c474f0d6d758af082d849fd/execnet-2.1.2-py3-none-any.whl", hash = "sha256:67fba928dd5a544b783f6056f449e5e3931a5c378b128bc18501f7ea79e296ec", size = 40708, upload-time = "2025-11-12T09:56:36.333Z" }, ] -[[package]] -name = "h11" -version = "0.14.0" -source = { registry = "https://pypi.org/simple" } -resolution-markers = [ - "python_full_version < '3.8'", -] -dependencies = [ - { name = "typing-extensions", version = "4.7.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.8'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/f5/38/3af3d3633a34a3316095b39c8e8fb4853a28a536e55d347bd8d8e9a14b03/h11-0.14.0.tar.gz", hash = "sha256:8f19fbbe99e72420ff35c00b27a34cb9937e902a8b810e2c88300c6f0a3b699d", size = 100418, upload-time = "2022-09-25T15:40:01.519Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/95/04/ff642e65ad6b90db43e668d70ffb6736436c7ce41fcc549f4e9472234127/h11-0.14.0-py3-none-any.whl", hash = "sha256:e3fe4ac4b851c468cc8363d500db52c2ead036020723024a109d37346efaa761", size = 58259, upload-time = "2022-09-25T15:39:59.68Z" }, -] - [[package]] name = "h11" version = "0.16.0" source = { registry = "https://pypi.org/simple" } -resolution-markers = [ - "python_full_version >= '3.10'", - "python_full_version == '3.9.*'", - "python_full_version == '3.8.*'", -] sdist = { url = "https://files.pythonhosted.org/packages/01/ee/02a2c011bdab74c6fb3c75474d40b3052059d95df7e73351460c8588d963/h11-0.16.0.tar.gz", hash = "sha256:4e35b956cf45792e4caa5885e69fba00bdbc6ffafbfa020300e549b208ee5ff1", size = 101250, upload-time = "2025-04-24T03:35:25.427Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/04/4b/29cac41a4d98d144bf5f6d33995617b185d14b22401f75ca86f384e87ff1/h11-0.16.0-py3-none-any.whl", hash = "sha256:63cf8bbe7522de3bf65932fda1d9c2772064ffb3dae62d55932da54b31cb6c86", size = 37515, upload-time = "2025-04-24T03:35:24.344Z" }, @@ -608,8 +467,7 @@ name = "h2" version = "4.1.0" source = { registry = "https://pypi.org/simple" } resolution-markers = [ - "python_full_version == '3.8.*'", - "python_full_version < '3.8'", + "python_full_version < '3.9'", ] dependencies = [ { name = "hpack", version = "4.0.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.9'" }, @@ -642,8 +500,7 @@ name = "hpack" version = "4.0.0" source = { registry = "https://pypi.org/simple" } resolution-markers = [ - "python_full_version == '3.8.*'", - "python_full_version < '3.8'", + "python_full_version < '3.9'", ] sdist = { url = "https://files.pythonhosted.org/packages/3e/9b/fda93fb4d957db19b0f6b370e79d586b3e8528b20252c729c476a2c02954/hpack-4.0.0.tar.gz", hash = "sha256:fc41de0c63e687ebffde81187a948221294896f6bdc0ae2312708df339430095", size = 49117, upload-time = "2020-08-30T10:35:57.868Z" } wheels = [ @@ -663,75 +520,29 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/07/c6/80c95b1b2b94682a72cbdbfb85b81ae2daffa4291fbfa1b1464502ede10d/hpack-4.1.0-py3-none-any.whl", hash = "sha256:157ac792668d995c657d93111f46b4535ed114f0c9c8d672271bbec7eae1b496", size = 34357, upload-time = "2025-01-22T21:44:56.92Z" }, ] -[[package]] -name = "httpcore" -version = "0.17.3" -source = { registry = "https://pypi.org/simple" } -resolution-markers = [ - "python_full_version < '3.8'", -] -dependencies = [ - { name = "anyio", version = "3.7.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.8'" }, - { name = "certifi", marker = "python_full_version < '3.8'" }, - { name = "h11", version = "0.14.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.8'" }, - { name = "sniffio", marker = "python_full_version < '3.8'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/63/ad/c98ecdbfe04417e71e143bf2f2fb29128e4787d78d1cedba21bd250c7e7a/httpcore-0.17.3.tar.gz", hash = "sha256:a6f30213335e34c1ade7be6ec7c47f19f50c56db36abef1a9dfa3815b1cb3888", size = 62676, upload-time = "2023-07-05T12:09:31.29Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/94/2c/2bde7ff8dd2064395555220cbf7cba79991172bf5315a07eb3ac7688d9f1/httpcore-0.17.3-py3-none-any.whl", hash = "sha256:c2789b767ddddfa2a5782e3199b2b7f6894540b17b16ec26b2c4d8e103510b87", size = 74513, upload-time = "2023-07-05T12:09:29.425Z" }, -] - [[package]] name = "httpcore" version = "1.0.9" source = { registry = "https://pypi.org/simple" } -resolution-markers = [ - "python_full_version >= '3.10'", - "python_full_version == '3.9.*'", - "python_full_version == '3.8.*'", -] dependencies = [ - { name = "certifi", marker = "python_full_version >= '3.8'" }, - { name = "h11", version = "0.16.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.8'" }, + { name = "certifi" }, + { name = "h11" }, ] sdist = { url = "https://files.pythonhosted.org/packages/06/94/82699a10bca87a5556c9c59b5963f2d039dbd239f25bc2a63907a05a14cb/httpcore-1.0.9.tar.gz", hash = "sha256:6e34463af53fd2ab5d807f399a9b45ea31c3dfa2276f15a2c3f00afff6e176e8", size = 85484, upload-time = "2025-04-24T22:06:22.219Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/7e/f5/f66802a942d491edb555dd61e3a9961140fd64c90bce1eafd741609d334d/httpcore-1.0.9-py3-none-any.whl", hash = "sha256:2d400746a40668fc9dec9810239072b40b4484b640a8c38fd654a024c7a1bf55", size = 78784, upload-time = "2025-04-24T22:06:20.566Z" }, ] -[[package]] -name = "httpx" -version = "0.24.1" -source = { registry = "https://pypi.org/simple" } -resolution-markers = [ - "python_full_version < '3.8'", -] -dependencies = [ - { name = "certifi", marker = "python_full_version < '3.8'" }, - { name = "httpcore", version = "0.17.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.8'" }, - { name = "idna", version = "3.10", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.8'" }, - { name = "sniffio", marker = "python_full_version < '3.8'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/f8/2a/114d454cb77657dbf6a293e69390b96318930ace9cd96b51b99682493276/httpx-0.24.1.tar.gz", hash = "sha256:5853a43053df830c20f8110c5e69fe44d035d850b2dfe795e196f00fdb774bdd", size = 81858, upload-time = "2023-05-19T00:50:56.678Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/ec/91/e41f64f03d2a13aee7e8c819d82ee3aa7cdc484d18c0ae859742597d5aa0/httpx-0.24.1-py3-none-any.whl", hash = "sha256:06781eb9ac53cde990577af654bd990a4949de37a28bdb4a230d434f3a30b9bd", size = 75377, upload-time = "2023-05-19T00:50:54.91Z" }, -] - [[package]] name = "httpx" version = "0.28.1" source = { registry = "https://pypi.org/simple" } -resolution-markers = [ - "python_full_version >= '3.10'", - "python_full_version == '3.9.*'", - "python_full_version == '3.8.*'", -] dependencies = [ - { name = "anyio", version = "4.5.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.8.*'" }, + { name = "anyio", version = "4.5.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.9'" }, { name = "anyio", version = "4.11.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.9'" }, - { name = "certifi", marker = "python_full_version >= '3.8'" }, - { name = "httpcore", version = "1.0.9", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.8'" }, - { name = "idna", version = "3.11", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.8'" }, + { name = "certifi" }, + { name = "httpcore" }, + { name = "idna" }, ] sdist = { url = "https://files.pythonhosted.org/packages/b1/df/48c586a5fe32a0f01324ee087459e112ebb7224f646c0b5023f5e79e9956/httpx-0.28.1.tar.gz", hash = "sha256:75e98c5f16b0f35b567856f597f06ff2270a374470a5c2392242528e3e3e42fc", size = 141406, upload-time = "2024-12-06T15:37:23.222Z" } wheels = [ @@ -743,8 +554,7 @@ name = "hyperframe" version = "6.0.1" source = { registry = "https://pypi.org/simple" } resolution-markers = [ - "python_full_version == '3.8.*'", - "python_full_version < '3.8'", + "python_full_version < '3.9'", ] sdist = { url = "https://files.pythonhosted.org/packages/5a/2a/4747bff0a17f7281abe73e955d60d80aae537a5d203f417fa1c2e7578ebb/hyperframe-6.0.1.tar.gz", hash = "sha256:ae510046231dc8e9ecb1a6586f63d2347bf4c8905914aa84ba585ae85f28a914", size = 25008, upload-time = "2021-04-17T12:11:22.757Z" } wheels = [ @@ -764,27 +574,10 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/48/30/47d0bf6072f7252e6521f3447ccfa40b421b6824517f82854703d0f5a98b/hyperframe-6.1.0-py3-none-any.whl", hash = "sha256:b03380493a519fce58ea5af42e4a42317bf9bd425596f7a0835ffce80f1a42e5", size = 13007, upload-time = "2025-01-22T21:41:47.295Z" }, ] -[[package]] -name = "idna" -version = "3.10" -source = { registry = "https://pypi.org/simple" } -resolution-markers = [ - "python_full_version < '3.8'", -] -sdist = { url = "https://files.pythonhosted.org/packages/f1/70/7703c29685631f5a7590aa73f1f1d3fa9a380e654b86af429e0934a32f7d/idna-3.10.tar.gz", hash = "sha256:12f65c9b470abda6dc35cf8e63cc574b1c52b11df2c86030af0ac09b01b13ea9", size = 190490, upload-time = "2024-09-15T18:07:39.745Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/76/c6/c88e154df9c4e1a2a66ccf0005a88dfb2650c1dffb6f5ce603dfbd452ce3/idna-3.10-py3-none-any.whl", hash = "sha256:946d195a0d259cbba61165e88e65941f16e9b36ea6ddb97f00452bae8b1287d3", size = 70442, upload-time = "2024-09-15T18:07:37.964Z" }, -] - [[package]] name = "idna" version = "3.11" source = { registry = "https://pypi.org/simple" } -resolution-markers = [ - "python_full_version >= '3.10'", - "python_full_version == '3.9.*'", - "python_full_version == '3.8.*'", -] sdist = { url = "https://files.pythonhosted.org/packages/6f/6d/0703ccc57f3a7233505399edb88de3cbd678da106337b9fcde432b65ed60/idna-3.11.tar.gz", hash = "sha256:795dafcc9c04ed0c1fb032c2aa73654d8e8c5023a7df64a53f39190ada629902", size = 194582, upload-time = "2025-10-12T14:55:20.501Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/0e/61/66938bbb5fc52dbdf84594873d5b51fb1f7c7794e9c0f5bd885f30bc507b/idna-3.11-py3-none-any.whl", hash = "sha256:771a87f49d9defaf64091e6e6fe9c18d4833f140bd19464795bc32d966ca37ea", size = 71008, upload-time = "2025-10-12T14:55:18.883Z" }, @@ -795,9 +588,7 @@ name = "importlib-metadata" version = "4.13.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "typing-extensions", version = "4.7.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.8'" }, - { name = "zipp", version = "3.15.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.8'" }, - { name = "zipp", version = "3.20.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.8.*'" }, + { name = "zipp", version = "3.20.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.9'" }, { name = "zipp", version = "3.23.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.9'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/55/12/ab288357b884ebc807e3f4eff63ce5ba6b941ba61499071bf19f1bbc7f7f/importlib_metadata-4.13.0.tar.gz", hash = "sha256:dd0173e8f150d6815e098fd354f6414b0f079af4644ddfe90c71e2fc6174346d", size = 50445, upload-time = "2022-10-01T17:09:15.698Z" } @@ -805,25 +596,13 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/d0/98/c277899f5aa21f6e6946e1c83f2af650cbfee982763ffb91db07ff7d3a13/importlib_metadata-4.13.0-py3-none-any.whl", hash = "sha256:8a8a81bcf996e74fee46f0d16bd3eaa382a7eb20fd82445c3ad11f4090334116", size = 23010, upload-time = "2022-10-01T17:09:13.903Z" }, ] -[[package]] -name = "iniconfig" -version = "2.0.0" -source = { registry = "https://pypi.org/simple" } -resolution-markers = [ - "python_full_version < '3.8'", -] -sdist = { url = "https://files.pythonhosted.org/packages/d7/4b/cbd8e699e64a6f16ca3a8220661b5f83792b3017d0f79807cb8708d33913/iniconfig-2.0.0.tar.gz", hash = "sha256:2d91e135bf72d31a410b17c16da610a82cb55f6b0477d1a902134b24a455b8b3", size = 4646, upload-time = "2023-01-07T11:08:11.254Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/ef/a6/62565a6e1cf69e10f5727360368e451d4b7f58beeac6173dc9db836a5b46/iniconfig-2.0.0-py3-none-any.whl", hash = "sha256:b6a85871a79d2e3b22d2d1b94ac2824226a63c6b741c88f7ae975f18b6778374", size = 5892, upload-time = "2023-01-07T11:08:09.864Z" }, -] - [[package]] name = "iniconfig" version = "2.1.0" source = { registry = "https://pypi.org/simple" } resolution-markers = [ "python_full_version == '3.9.*'", - "python_full_version == '3.8.*'", + "python_full_version < '3.9'", ] sdist = { url = "https://files.pythonhosted.org/packages/f2/97/ebf4da567aa6827c909642694d71c9fcf53e5b504f2d96afea02718862f3/iniconfig-2.1.0.tar.gz", hash = "sha256:3abbd2e30b36733fee78f9c7f7308f2d0050e88f0087fd25c2645f63c773e1c7", size = 4793, upload-time = "2025-03-19T20:09:59.721Z" } wheels = [ @@ -851,76 +630,12 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/5c/03/b7e605db4a57c0f6fba744b11ef3ddf4ddebcada35022927a2b5fc623fdf/mock-4.0.3-py3-none-any.whl", hash = "sha256:122fcb64ee37cfad5b3f48d7a7d51875d7031aaf3d8be7c42e2bee25044eee62", size = 28536, upload-time = "2020-12-10T07:33:11.564Z" }, ] -[[package]] -name = "msgpack" -version = "1.0.5" -source = { registry = "https://pypi.org/simple" } -resolution-markers = [ - "python_full_version < '3.8'", -] -sdist = { url = "https://files.pythonhosted.org/packages/dc/a1/eba11a0d4b764bc62966a565b470f8c6f38242723ba3057e9b5098678c30/msgpack-1.0.5.tar.gz", hash = "sha256:c075544284eadc5cddc70f4757331d99dcbc16b2bbd4849d15f8aae4cf36d31c", size = 127834, upload-time = "2023-03-08T17:50:48.564Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/9f/4a/36d936e54cf71e23ad276564465f6a54fb129e3d61520b76e13e0bb29167/msgpack-1.0.5-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:525228efd79bb831cf6830a732e2e80bc1b05436b086d4264814b4b2955b2fa9", size = 129738, upload-time = "2023-03-08T17:49:18.464Z" }, - { url = "https://files.pythonhosted.org/packages/f2/da/770118f8d48e11cc9a2c7cb60d7d3c8016266526bd42c6ff5bd21013d099/msgpack-1.0.5-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:4f8d8b3bf1ff2672567d6b5c725a1b347fe838b912772aa8ae2bf70338d5a198", size = 74671, upload-time = "2023-03-08T17:49:20.311Z" }, - { url = "https://files.pythonhosted.org/packages/73/99/f338ce8b69e934c04e5d9187f85de1ae395882cd56e7deb48e78a1749af8/msgpack-1.0.5-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:cdc793c50be3f01106245a61b739328f7dccc2c648b501e237f0699fe1395b81", size = 70230, upload-time = "2023-03-08T17:49:21.958Z" }, - { url = "https://files.pythonhosted.org/packages/3e/80/bc7fdb75a35bf32c7c529c247dcadfd0502aac2309e207a89b0be6fe42ea/msgpack-1.0.5-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5cb47c21a8a65b165ce29f2bec852790cbc04936f502966768e4aae9fa763cb7", size = 309410, upload-time = "2023-03-08T17:49:23.337Z" }, - { url = "https://files.pythonhosted.org/packages/59/67/f992ada3b42889f1b984e5651d63ea21ca3a92049cff6d75fe0a4a63e422/msgpack-1.0.5-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e42b9594cc3bf4d838d67d6ed62b9e59e201862a25e9a157019e171fbe672dd3", size = 316846, upload-time = "2023-03-08T17:49:24.786Z" }, - { url = "https://files.pythonhosted.org/packages/10/fe/9e004c4deb457f1ef1ad88c1188da5691ff1855e0d03a5ac3635ae1f6530/msgpack-1.0.5-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:55b56a24893105dc52c1253649b60f475f36b3aa0fc66115bffafb624d7cb30b", size = 311396, upload-time = "2023-03-08T17:49:26.075Z" }, - { url = "https://files.pythonhosted.org/packages/95/c9/560c3203c4327881c9f2de26c42dacdd9567bfe7fa43458e2a680c4bdcaf/msgpack-1.0.5-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:1967f6129fc50a43bfe0951c35acbb729be89a55d849fab7686004da85103f1c", size = 311165, upload-time = "2023-03-08T17:49:27.494Z" }, - { url = "https://files.pythonhosted.org/packages/10/ca/50c3a5e92d459a942169747315afd8c226d05427eccff903ddf33135c574/msgpack-1.0.5-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:20a97bf595a232c3ee6d57ddaadd5453d174a52594bf9c21d10407e2a2d9b3bd", size = 348664, upload-time = "2023-03-08T17:49:28.736Z" }, - { url = "https://files.pythonhosted.org/packages/29/56/1fb6b96aab759ab3bc05b03ba6d936b350db72aac203cde56ea6bd001237/msgpack-1.0.5-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:d25dd59bbbbb996eacf7be6b4ad082ed7eacc4e8f3d2df1ba43822da9bfa122a", size = 316731, upload-time = "2023-03-08T17:49:30.041Z" }, - { url = "https://files.pythonhosted.org/packages/7b/e9/b47f9e93fc381885624c40cbbbd0480b18ae11ca588162fe724d43495372/msgpack-1.0.5-cp310-cp310-win32.whl", hash = "sha256:382b2c77589331f2cb80b67cc058c00f225e19827dbc818d700f61513ab47bea", size = 57134, upload-time = "2023-03-08T17:49:31.365Z" }, - { url = "https://files.pythonhosted.org/packages/3c/e5/3d436bed11849ba05d777ed3fd1a0440170bad460335ea541dd6946047ed/msgpack-1.0.5-cp310-cp310-win_amd64.whl", hash = "sha256:4867aa2df9e2a5fa5f76d7d5565d25ec76e84c106b55509e78c1ede0f152659a", size = 61631, upload-time = "2023-03-08T17:49:32.482Z" }, - { url = "https://files.pythonhosted.org/packages/27/ad/4edfe383ec3185611441179ffee8cbc8155d7575fbad73f6d31015e35451/msgpack-1.0.5-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:9f5ae84c5c8a857ec44dc180a8b0cc08238e021f57abdf51a8182e915e6299f0", size = 127502, upload-time = "2023-03-08T17:49:33.974Z" }, - { url = "https://files.pythonhosted.org/packages/c1/57/01f2d8805160f559ec21d095fc7576a26fbaed2475af24ce4a135c380c14/msgpack-1.0.5-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:9e6ca5d5699bcd89ae605c150aee83b5321f2115695e741b99618f4856c50898", size = 73747, upload-time = "2023-03-08T17:49:35.158Z" }, - { url = "https://files.pythonhosted.org/packages/6c/fe/8a7747ca57074307a2e8f1de58441952a9dbdf9e8a8e5873d53a5ce0835c/msgpack-1.0.5-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:5494ea30d517a3576749cad32fa27f7585c65f5f38309c88c6d137877fa28a5a", size = 69041, upload-time = "2023-03-08T17:49:36.44Z" }, - { url = "https://files.pythonhosted.org/packages/33/0a/aa7b53ae17cf1dc1c352d705ab3162fc572c55048cc3177c1a88009c47fd/msgpack-1.0.5-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1ab2f3331cb1b54165976a9d976cb251a83183631c88076613c6c780f0d6e45a", size = 316114, upload-time = "2023-03-08T17:49:38.252Z" }, - { url = "https://files.pythonhosted.org/packages/6b/6d/de239d77d347f1990c41b4800075a15e06f748186dd120166270dd071734/msgpack-1.0.5-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:28592e20bbb1620848256ebc105fc420436af59515793ed27d5c77a217477705", size = 325080, upload-time = "2023-03-08T17:49:39.528Z" }, - { url = "https://files.pythonhosted.org/packages/7e/1c/9d0fd241a4e88e1cd2f5babea4a27ac25b1b86dbbc05fa10741e82079a93/msgpack-1.0.5-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:fe5c63197c55bce6385d9aee16c4d0641684628f63ace85f73571e65ad1c1e8d", size = 319393, upload-time = "2023-03-08T17:49:40.755Z" }, - { url = "https://files.pythonhosted.org/packages/b8/bc/1d5fe4732dc78ff86aaf677596da08f0ae736e60ca8ab49c1f1c7366cb1a/msgpack-1.0.5-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:ed40e926fa2f297e8a653c954b732f125ef97bdd4c889f243182299de27e2aa9", size = 316118, upload-time = "2023-03-08T17:49:41.964Z" }, - { url = "https://files.pythonhosted.org/packages/2c/e9/c79ecc36cfa34d850a01773565e0fccafd69efff07172028c3a5f758b83f/msgpack-1.0.5-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:b2de4c1c0538dcb7010902a2b97f4e00fc4ddf2c8cda9749af0e594d3b7fa3d7", size = 354984, upload-time = "2023-03-08T17:49:43.251Z" }, - { url = "https://files.pythonhosted.org/packages/45/85/6b55b0cabad846d3e730226a897f878f8f63ee505668bb6c55a697b0bfb0/msgpack-1.0.5-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:bf22a83f973b50f9d38e55c6aade04c41ddda19b00c4ebc558930d78eecc64ed", size = 323580, upload-time = "2023-03-08T17:49:44.71Z" }, - { url = "https://files.pythonhosted.org/packages/0e/69/3d10e741dd2bbb806af5cdc76551735baab5f5f9773701eb05502c913a6e/msgpack-1.0.5-cp311-cp311-win32.whl", hash = "sha256:c396e2cc213d12ce017b686e0f53497f94f8ba2b24799c25d913d46c08ec422c", size = 56419, upload-time = "2023-03-08T17:49:46.603Z" }, - { url = "https://files.pythonhosted.org/packages/6b/79/0dec8f035160464ca88b221cc79691a71cf88dc25207c17f1d918b2c7bb0/msgpack-1.0.5-cp311-cp311-win_amd64.whl", hash = "sha256:6c4c68d87497f66f96d50142a2b73b97972130d93677ce930718f68828b382e2", size = 60781, upload-time = "2023-03-08T17:49:47.912Z" }, - { url = "https://files.pythonhosted.org/packages/c5/c1/1b591574ba71481fbf38359a8fca5108e4ad130a6dbb9b2acb3e9277d0fe/msgpack-1.0.5-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:137850656634abddfb88236008339fdaba3178f4751b28f270d2ebe77a563b6c", size = 72520, upload-time = "2023-03-08T17:50:02.199Z" }, - { url = "https://files.pythonhosted.org/packages/62/57/170af6c6fccd2d950ea01e1faa58cae9643226fa8705baded11eca3aa8b5/msgpack-1.0.5-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0c05a4a96585525916b109bb85f8cb6511db1c6f5b9d9cbcbc940dc6b4be944b", size = 289288, upload-time = "2023-03-08T17:50:04.213Z" }, - { url = "https://files.pythonhosted.org/packages/2b/c4/f2c8695ae69d1425eddc5e2f849c525b562dc8409bc2979e525f3dd4fecd/msgpack-1.0.5-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:56a62ec00b636583e5cb6ad313bbed36bb7ead5fa3a3e38938503142c72cba4f", size = 299695, upload-time = "2023-03-08T17:50:05.622Z" }, - { url = "https://files.pythonhosted.org/packages/62/5c/9c7fed4ca0235a2d7b8d15b4047c328976b97d2b227719e54cad1e47c244/msgpack-1.0.5-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ef8108f8dedf204bb7b42994abf93882da1159728a2d4c5e82012edd92c9da9f", size = 293149, upload-time = "2023-03-08T17:50:06.915Z" }, - { url = "https://files.pythonhosted.org/packages/ef/13/c110d89d5079169354394dc226e6f84d818722939bc1fe3f9c25f982e903/msgpack-1.0.5-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:1835c84d65f46900920b3708f5ba829fb19b1096c1800ad60bae8418652a951d", size = 292899, upload-time = "2023-03-08T17:50:08.215Z" }, - { url = "https://files.pythonhosted.org/packages/72/ac/2eda5af7cd1450c52d031e48c76b280eac5bb2e588678876612f95be34ab/msgpack-1.0.5-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:e57916ef1bd0fee4f21c4600e9d1da352d8816b52a599c46460e93a6e9f17086", size = 334408, upload-time = "2023-03-08T17:50:10Z" }, - { url = "https://files.pythonhosted.org/packages/e8/60/78906f564804aae23eb1102eca8b8830f1e08a649c179774c05fa7dc0aad/msgpack-1.0.5-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:17358523b85973e5f242ad74aa4712b7ee560715562554aa2134d96e7aa4cbbf", size = 302791, upload-time = "2023-03-08T17:50:11.414Z" }, - { url = "https://files.pythonhosted.org/packages/ce/b8/89cb1809b076a4651169851aa1f98128b75cbfe14034b914c9040b13c4cf/msgpack-1.0.5-cp37-cp37m-win32.whl", hash = "sha256:cb5aaa8c17760909ec6cb15e744c3ebc2ca8918e727216e79607b7bbce9c8f77", size = 57095, upload-time = "2023-03-08T17:50:12.741Z" }, - { url = "https://files.pythonhosted.org/packages/e8/1f/be19c9c9cfdcc2ae8ee8c65dbe5f281cc1f3331f9b9523735f39b090b448/msgpack-1.0.5-cp37-cp37m-win_amd64.whl", hash = "sha256:ab31e908d8424d55601ad7075e471b7d0140d4d3dd3272daf39c5c19d936bd82", size = 62112, upload-time = "2023-03-08T17:50:13.947Z" }, - { url = "https://files.pythonhosted.org/packages/1a/f7/df5814697c25bdebb14ea97d27ddca04f5d4c6e249f096d086fea521c139/msgpack-1.0.5-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:b72d0698f86e8d9ddf9442bdedec15b71df3598199ba33322d9711a19f08145c", size = 126923, upload-time = "2023-03-08T17:50:15.214Z" }, - { url = "https://files.pythonhosted.org/packages/33/52/099f0dde1283bac7bf267ab941dfa3b7c89ee701e4252973f8d3c10e68d6/msgpack-1.0.5-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:379026812e49258016dd84ad79ac8446922234d498058ae1d415f04b522d5b2d", size = 73246, upload-time = "2023-03-08T17:50:16.487Z" }, - { url = "https://files.pythonhosted.org/packages/f1/1f/cc3e8274934c8323f6106dae22cba8bad413166f4efb3819573de58c215c/msgpack-1.0.5-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:332360ff25469c346a1c5e47cbe2a725517919892eda5cfaffe6046656f0b7bb", size = 68947, upload-time = "2023-03-08T17:50:17.767Z" }, - { url = "https://files.pythonhosted.org/packages/19/0c/2c3b443df88f5d400f2e19a3d867564d004b26e137f18c2f2663913987bc/msgpack-1.0.5-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:476a8fe8fae289fdf273d6d2a6cb6e35b5a58541693e8f9f019bfe990a51e4ba", size = 313568, upload-time = "2023-03-08T17:50:19.016Z" }, - { url = "https://files.pythonhosted.org/packages/0a/04/bc319ba061f6dc9077745988be288705b3f9f18c5a209772a3e8fcd419fd/msgpack-1.0.5-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a9985b214f33311df47e274eb788a5893a761d025e2b92c723ba4c63936b69b1", size = 322443, upload-time = "2023-03-08T17:50:20.341Z" }, - { url = "https://files.pythonhosted.org/packages/2f/21/e488871f8e498efe14821b0c870eb95af52cfafb9b8dd41d83fad85b383b/msgpack-1.0.5-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:48296af57cdb1d885843afd73c4656be5c76c0c6328db3440c9601a98f303d87", size = 315490, upload-time = "2023-03-08T17:50:22.301Z" }, - { url = "https://files.pythonhosted.org/packages/28/8f/c58c53c884217cc572c19349c7e1129b5a6eae36df0a017aae3a8f3d7aa8/msgpack-1.0.5-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:addab7e2e1fcc04bd08e4eb631c2a90960c340e40dfc4a5e24d2ff0d5a3b3edb", size = 324288, upload-time = "2023-03-08T17:50:23.739Z" }, - { url = "https://files.pythonhosted.org/packages/0d/90/44edef4a8c6f035b054c4b017c5adcb22a35ec377e17e50dd5dced279a6b/msgpack-1.0.5-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:916723458c25dfb77ff07f4c66aed34e47503b2eb3188b3adbec8d8aa6e00f48", size = 361405, upload-time = "2023-03-08T17:50:24.992Z" }, - { url = "https://files.pythonhosted.org/packages/56/50/bfcc0fad07067b6f1b09d940272ec749d5fe82570d938c2348c3ad0babf7/msgpack-1.0.5-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:821c7e677cc6acf0fd3f7ac664c98803827ae6de594a9f99563e48c5a2f27eb0", size = 329585, upload-time = "2023-03-08T17:50:26.387Z" }, - { url = "https://files.pythonhosted.org/packages/80/f0/c1fadb4e4a38fda19e35b1b6f887d72cc9c57778af43b53f64a8cd62e922/msgpack-1.0.5-cp38-cp38-win32.whl", hash = "sha256:1c0f7c47f0087ffda62961d425e4407961a7ffd2aa004c81b9c07d9269512f6e", size = 57668, upload-time = "2023-03-08T17:50:28.037Z" }, - { url = "https://files.pythonhosted.org/packages/da/46/855bdcbf004fd87b6a4451e8dcd61329439dcd9039887f71ca5085769216/msgpack-1.0.5-cp38-cp38-win_amd64.whl", hash = "sha256:bae7de2026cbfe3782c8b78b0db9cbfc5455e079f1937cb0ab8d133496ac55e1", size = 62509, upload-time = "2023-03-08T17:50:29.85Z" }, - { url = "https://files.pythonhosted.org/packages/12/6e/0cfd1dc07f61a6ac606587a393f489c3ca463469d285a73c8e5e2f61b021/msgpack-1.0.5-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:20c784e66b613c7f16f632e7b5e8a1651aa5702463d61394671ba07b2fc9e025", size = 130498, upload-time = "2023-03-08T17:50:31.551Z" }, - { url = "https://files.pythonhosted.org/packages/4b/3d/cc5eb6d69e0ecde80a78cc42f48579971ec333e509d56a4a6de1a2c40ba2/msgpack-1.0.5-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:266fa4202c0eb94d26822d9bfd7af25d1e2c088927fe8de9033d929dd5ba24c5", size = 75178, upload-time = "2023-03-08T17:50:32.78Z" }, - { url = "https://files.pythonhosted.org/packages/bf/68/032e62ad44f92ba6a4ae7c45054843cdec7f0c405ecdfd166f25123b0c47/msgpack-1.0.5-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:18334484eafc2b1aa47a6d42427da7fa8f2ab3d60b674120bce7a895a0a85bdd", size = 70460, upload-time = "2023-03-08T17:50:34.57Z" }, - { url = "https://files.pythonhosted.org/packages/49/57/a28120d82f8e77622a1e1efc652389c71145f6b89b47b39814a7c6038373/msgpack-1.0.5-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:57e1f3528bd95cc44684beda696f74d3aaa8a5e58c816214b9046512240ef437", size = 313499, upload-time = "2023-03-08T17:50:36.329Z" }, - { url = "https://files.pythonhosted.org/packages/ab/ff/ca74e519c47139b6c08fb21db5ead2bd2eed6cb1225f9be69390cdb48182/msgpack-1.0.5-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:586d0d636f9a628ddc6a17bfd45aa5b5efaf1606d2b60fa5d87b8986326e933f", size = 322301, upload-time = "2023-03-08T17:50:38.097Z" }, - { url = "https://files.pythonhosted.org/packages/43/87/6507d56f62b958d822ae4ffe1c4507ed7d3cf37ad61114665816adcf4adc/msgpack-1.0.5-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a740fa0e4087a734455f0fc3abf5e746004c9da72fbd541e9b113013c8dc3282", size = 316630, upload-time = "2023-03-08T17:50:39.397Z" }, - { url = "https://files.pythonhosted.org/packages/67/f8/e3ab674f4a945308362e9342297fe6b35a89dd0f648aa325aabffa5dc210/msgpack-1.0.5-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:3055b0455e45810820db1f29d900bf39466df96ddca11dfa6d074fa47054376d", size = 316251, upload-time = "2023-03-08T17:50:41.153Z" }, - { url = "https://files.pythonhosted.org/packages/e9/f1/45b73a9e97f702bcb5f51569b93990e456bc969363e55122374c22ed7d24/msgpack-1.0.5-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:a61215eac016f391129a013c9e46f3ab308db5f5ec9f25811e811f96962599a8", size = 352781, upload-time = "2023-03-08T17:50:42.435Z" }, - { url = "https://files.pythonhosted.org/packages/17/10/be97811782473d709d07b65a3955a5a76d47686aff3d62bb41d48aea7c92/msgpack-1.0.5-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:362d9655cd369b08fda06b6657a303eb7172d5279997abe094512e919cf74b11", size = 321996, upload-time = "2023-03-08T17:50:43.726Z" }, - { url = "https://files.pythonhosted.org/packages/18/3f/3860151fbdf50e369bbe4ffd307a588417669c725025e383f3ce5893690f/msgpack-1.0.5-cp39-cp39-win32.whl", hash = "sha256:ac9dd47af78cae935901a9a500104e2dea2e253207c924cc95de149606dc43cc", size = 57827, upload-time = "2023-03-08T17:50:45.517Z" }, - { url = "https://files.pythonhosted.org/packages/6c/fa/3ca00fb1e53bcacf8c186fa6aff2d2086862b12e289bcf38227d9d40bd86/msgpack-1.0.5-cp39-cp39-win_amd64.whl", hash = "sha256:06f5174b5f8ed0ed919da0e62cbd4ffde676a374aba4020034da05fab67b9164", size = 62775, upload-time = "2023-03-08T17:50:47.305Z" }, -] - [[package]] name = "msgpack" version = "1.1.1" source = { registry = "https://pypi.org/simple" } resolution-markers = [ - "python_full_version == '3.8.*'", + "python_full_version < '3.9'", ] sdist = { url = "https://files.pythonhosted.org/packages/45/b1/ea4f68038a18c77c9467400d166d74c4ffa536f34761f7983a104357e614/msgpack-1.1.1.tar.gz", hash = "sha256:77b79ce34a2bdab2594f490c8e80dd62a02d650b91a75159a63ec413b8d104cd", size = 173555, upload-time = "2025-06-13T06:52:51.324Z" } wheels = [ @@ -1057,53 +772,21 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/14/a7/b1992b4fb3da3b413f5fb78a63bad42f256c3be2352eb69273c3789c2c96/msgpack-1.1.2-cp39-cp39-win_amd64.whl", hash = "sha256:67016ae8c8965124fdede9d3769528ad8284f14d635337ffa6a713a580f6c030", size = 71540, upload-time = "2025-10-08T09:15:55.573Z" }, ] -[[package]] -name = "packaging" -version = "24.0" -source = { registry = "https://pypi.org/simple" } -resolution-markers = [ - "python_full_version < '3.8'", -] -sdist = { url = "https://files.pythonhosted.org/packages/ee/b5/b43a27ac7472e1818c4bafd44430e69605baefe1f34440593e0332ec8b4d/packaging-24.0.tar.gz", hash = "sha256:eb82c5e3e56209074766e6885bb04b8c38a0c015d0a30036ebe7ece34c9989e9", size = 147882, upload-time = "2024-03-10T09:39:28.33Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/49/df/1fceb2f8900f8639e278b056416d49134fb8d84c5942ffaa01ad34782422/packaging-24.0-py3-none-any.whl", hash = "sha256:2ddfb553fdf02fb784c234c7ba6ccc288296ceabec964ad2eae3777778130bc5", size = 53488, upload-time = "2024-03-10T09:39:25.947Z" }, -] - [[package]] name = "packaging" version = "25.0" source = { registry = "https://pypi.org/simple" } -resolution-markers = [ - "python_full_version >= '3.10'", - "python_full_version == '3.9.*'", - "python_full_version == '3.8.*'", -] sdist = { url = "https://files.pythonhosted.org/packages/a1/d4/1fc4078c65507b51b96ca8f8c3ba19e6a61c8253c72794544580a7b6c24d/packaging-25.0.tar.gz", hash = "sha256:d443872c98d677bf60f6a1f2f8c1cb748e8fe762d2bf9d3148b5599295b0fc4f", size = 165727, upload-time = "2025-04-19T11:48:59.673Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/20/12/38679034af332785aac8774540895e234f4d07f7545804097de4b666afd8/packaging-25.0-py3-none-any.whl", hash = "sha256:29572ef2b1f17581046b3a2227d5c611fb25ec70ca1ba8554b24b0e69331a484", size = 66469, upload-time = "2025-04-19T11:48:57.875Z" }, ] -[[package]] -name = "pluggy" -version = "1.2.0" -source = { registry = "https://pypi.org/simple" } -resolution-markers = [ - "python_full_version < '3.8'", -] -dependencies = [ - { name = "importlib-metadata", marker = "python_full_version < '3.8'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/8a/42/8f2833655a29c4e9cb52ee8a2be04ceac61bcff4a680fb338cbd3d1e322d/pluggy-1.2.0.tar.gz", hash = "sha256:d12f0c4b579b15f5e054301bb226ee85eeeba08ffec228092f8defbaa3a4c4b3", size = 61613, upload-time = "2023-06-21T09:12:28.745Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/51/32/4a79112b8b87b21450b066e102d6608907f4c885ed7b04c3fdb085d4d6ae/pluggy-1.2.0-py3-none-any.whl", hash = "sha256:c2fd55a7d7a3863cba1a013e4e2414658b1d07b6bc57b3919e0c63c9abb99849", size = 17695, upload-time = "2023-06-21T09:12:27.397Z" }, -] - [[package]] name = "pluggy" version = "1.5.0" source = { registry = "https://pypi.org/simple" } resolution-markers = [ - "python_full_version == '3.8.*'", + "python_full_version < '3.9'", ] sdist = { url = "https://files.pythonhosted.org/packages/96/2d/02d4312c973c6050a18b314a5ad0b3210edb65a906f868e31c111dede4a6/pluggy-1.5.0.tar.gz", hash = "sha256:2cffa88e94fdc978c4c574f15f9e59b7f4201d439195c3715ca9e2486f1d0cf1", size = 67955, upload-time = "2024-04-20T21:34:42.531Z" } wheels = [ @@ -1178,32 +861,12 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/2b/62/2392b7879f4d2c1bfa20815720b89d464687877851716936b9609959c201/pycryptodome-3.23.0-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:ce64e84a962b63a47a592690bdc16a7eaf709d2c2697ababf24a0def566899a6", size = 1802461, upload-time = "2025-05-17T17:21:41.722Z" }, ] -[[package]] -name = "pyee" -version = "9.1.1" -source = { registry = "https://pypi.org/simple" } -resolution-markers = [ - "python_full_version < '3.8'", -] -dependencies = [ - { name = "typing-extensions", version = "4.7.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.8'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/eb/2c/ebe4fd8213b3d720b193a62f07169607e945dd02a08edc45b28ca52fbe07/pyee-9.1.1.tar.gz", hash = "sha256:a1ebcd38d92e7d780635ab3442c0f6ce49840d9bfe0b0549921a6188860af0db", size = 22634, upload-time = "2023-06-09T06:13:29.141Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/f2/53/39b67ce3841a5bb2d444f64ed969fb79ebd5bfed6867c3f88f3916407270/pyee-9.1.1-py2.py3-none-any.whl", hash = "sha256:f4a9853503d2f5a69d4350b54ba70841ebc535c53ebfaaa40c0fb47e63e78b3e", size = 15073, upload-time = "2023-06-09T06:13:27.255Z" }, -] - [[package]] name = "pyee" version = "13.0.0" source = { registry = "https://pypi.org/simple" } -resolution-markers = [ - "python_full_version >= '3.10'", - "python_full_version == '3.9.*'", - "python_full_version == '3.8.*'", -] dependencies = [ - { name = "typing-extensions", version = "4.13.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.8.*'" }, + { name = "typing-extensions", version = "4.13.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.9'" }, { name = "typing-extensions", version = "4.15.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.9'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/95/03/1fd98d5841cd7964a27d729ccf2199602fe05eb7a405c1462eb7277945ed/pyee-13.0.0.tar.gz", hash = "sha256:b391e3c5a434d1f5118a25615001dbc8f669cf410ab67d04c4d4e07c55481c37", size = 31250, upload-time = "2025-03-17T18:53:15.955Z" } @@ -1218,50 +881,24 @@ source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "colorama", marker = "sys_platform == 'win32'" }, { name = "exceptiongroup", marker = "python_full_version < '3.11'" }, - { name = "importlib-metadata", marker = "python_full_version < '3.8'" }, - { name = "iniconfig", version = "2.0.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.8'" }, - { name = "iniconfig", version = "2.1.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.8' and python_full_version < '3.10'" }, + { name = "iniconfig", version = "2.1.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" }, { name = "iniconfig", version = "2.3.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" }, - { name = "packaging", version = "24.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.8'" }, - { name = "packaging", version = "25.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.8'" }, - { name = "pluggy", version = "1.2.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.8'" }, - { name = "pluggy", version = "1.5.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.8.*'" }, + { name = "packaging" }, + { name = "pluggy", version = "1.5.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.9'" }, { name = "pluggy", version = "1.6.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.9'" }, - { name = "tomli", version = "2.0.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.8'" }, - { name = "tomli", version = "2.3.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.8' and python_full_version < '3.11'" }, + { name = "tomli", marker = "python_full_version < '3.11'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/80/1f/9d8e98e4133ffb16c90f3b405c43e38d3abb715bb5d7a63a5a684f7e46a3/pytest-7.4.4.tar.gz", hash = "sha256:2cf0005922c6ace4a3e2ec8b4080eb0d9753fdc93107415332f50ce9e7994280", size = 1357116, upload-time = "2023-12-31T12:00:18.035Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/51/ff/f6e8b8f39e08547faece4bd80f89d5a8de68a38b2d179cc1c4490ffa3286/pytest-7.4.4-py3-none-any.whl", hash = "sha256:b090cdf5ed60bf4c45261be03239c2c1c22df034fbffe691abe93cd80cea01d8", size = 325287, upload-time = "2023-12-31T12:00:13.963Z" }, ] -[[package]] -name = "pytest-asyncio" -version = "0.21.2" -source = { registry = "https://pypi.org/simple" } -resolution-markers = [ - "python_full_version < '3.8'", -] -dependencies = [ - { name = "pytest", marker = "python_full_version < '3.8'" }, - { name = "typing-extensions", version = "4.7.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.8'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/ae/53/57663d99acaac2fcdafdc697e52a9b1b7d6fcf36616281ff9768a44e7ff3/pytest_asyncio-0.21.2.tar.gz", hash = "sha256:d67738fc232b94b326b9d060750beb16e0074210b98dd8b58a5239fa2a154f45", size = 30656, upload-time = "2024-04-29T13:23:24.738Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/9c/ce/1e4b53c213dce25d6e8b163697fbce2d43799d76fa08eea6ad270451c370/pytest_asyncio-0.21.2-py3-none-any.whl", hash = "sha256:ab664c88bb7998f711d8039cacd4884da6430886ae8bbd4eded552ed2004f16b", size = 13368, upload-time = "2024-04-29T13:23:23.126Z" }, -] - [[package]] name = "pytest-asyncio" version = "0.23.8" source = { registry = "https://pypi.org/simple" } -resolution-markers = [ - "python_full_version >= '3.10'", - "python_full_version == '3.9.*'", - "python_full_version == '3.8.*'", -] dependencies = [ - { name = "pytest", marker = "python_full_version >= '3.8'" }, + { name = "pytest" }, ] sdist = { url = "https://files.pythonhosted.org/packages/de/b4/0b378b7bf26a8ae161c3890c0b48a91a04106c5713ce81b4b080ea2f4f18/pytest_asyncio-0.23.8.tar.gz", hash = "sha256:759b10b33a6dc61cce40a8bd5205e302978bbbcc00e279a8b61d9a6a3c82e4d3", size = 46920, upload-time = "2024-07-17T17:39:34.617Z" } wheels = [ @@ -1273,8 +910,7 @@ name = "pytest-cov" version = "2.12.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "coverage", version = "7.2.7", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.8'" }, - { name = "coverage", version = "7.6.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.8.*'" }, + { name = "coverage", version = "7.6.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.9'" }, { name = "coverage", version = "7.10.7", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.9.*'" }, { name = "coverage", version = "7.12.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" }, { name = "pytest" }, @@ -1315,8 +951,7 @@ name = "pytest-xdist" version = "1.34.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "execnet", version = "2.0.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.8'" }, - { name = "execnet", version = "2.1.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.8'" }, + { name = "execnet" }, { name = "pytest" }, { name = "pytest-forked" }, { name = "six" }, @@ -1326,32 +961,12 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/6a/fc/30821e7799bddd56989523ee003cde488c6e6053dfd29ba07db2ba934a04/pytest_xdist-1.34.0-py2.py3-none-any.whl", hash = "sha256:ba5d10729372d65df3ac150872f9df5d2ed004a3b0d499cc0164aafedd8c7b66", size = 36841, upload-time = "2020-07-27T23:05:23.851Z" }, ] -[[package]] -name = "respx" -version = "0.20.2" -source = { registry = "https://pypi.org/simple" } -resolution-markers = [ - "python_full_version < '3.8'", -] -dependencies = [ - { name = "httpx", version = "0.24.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.8'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/e6/0b/e0df26ea5c7145d95f1ab8ecb20f0778dd8af718e56747977dca9d28362a/respx-0.20.2.tar.gz", hash = "sha256:07cf4108b1c88b82010f67d3c831dae33a375c7b436e54d87737c7f9f99be643", size = 26080, upload-time = "2023-07-20T23:01:23.618Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/47/47/8c5a8b02c2144770fe353585b6db21e392c4318b8cff897738159feff562/respx-0.20.2-py2.py3-none-any.whl", hash = "sha256:ab8e1cf6da28a5b2dd883ea617f8130f77f676736e6e9e4a25817ad116a172c9", size = 22849, upload-time = "2023-07-20T23:01:21.994Z" }, -] - [[package]] name = "respx" version = "0.22.0" source = { registry = "https://pypi.org/simple" } -resolution-markers = [ - "python_full_version >= '3.10'", - "python_full_version == '3.9.*'", - "python_full_version == '3.8.*'", -] dependencies = [ - { name = "httpx", version = "0.28.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.8'" }, + { name = "httpx" }, ] sdist = { url = "https://files.pythonhosted.org/packages/f4/7c/96bd0bc759cf009675ad1ee1f96535edcb11e9666b985717eb8c87192a95/respx-0.22.0.tar.gz", hash = "sha256:3c8924caa2a50bd71aefc07aa812f2466ff489f1848c96e954a5362d17095d91", size = 28439, upload-time = "2024-12-19T22:33:59.374Z" } wheels = [ @@ -1402,24 +1017,12 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/e9/44/75a9c9421471a6c4805dbf2356f7c181a29c1879239abab1ea2cc8f38b40/sniffio-1.3.1-py3-none-any.whl", hash = "sha256:2f6da418d1f1e0fddd844478f41680e794e6051915791a034ff65e5f100525a2", size = 10235, upload-time = "2024-02-25T23:20:01.196Z" }, ] -[[package]] -name = "tokenize-rt" -version = "5.0.0" -source = { registry = "https://pypi.org/simple" } -resolution-markers = [ - "python_full_version < '3.8'", -] -sdist = { url = "https://files.pythonhosted.org/packages/40/01/fb40ea8c465f680bf7aa3f5bee39c62ba8b7f52c38048c27aa95aff4f779/tokenize_rt-5.0.0.tar.gz", hash = "sha256:3160bc0c3e8491312d0485171dea861fc160a240f5f5766b72a1165408d10740", size = 5329, upload-time = "2022-10-03T23:28:00.861Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/8d/12/4c7495f25b4c9131706f3aaffb185d4de32c02a6ee49d875e929c5b7c919/tokenize_rt-5.0.0-py2.py3-none-any.whl", hash = "sha256:c67772c662c6b3dc65edf66808577968fb10badfc2042e3027196bed4daf9e5a", size = 5848, upload-time = "2022-10-03T23:27:59.459Z" }, -] - [[package]] name = "tokenize-rt" version = "6.0.0" source = { registry = "https://pypi.org/simple" } resolution-markers = [ - "python_full_version == '3.8.*'", + "python_full_version < '3.9'", ] sdist = { url = "https://files.pythonhosted.org/packages/7d/09/6257dabdeab5097d72c5d874f29b33cd667ec411af6667922d84f85b79b5/tokenize_rt-6.0.0.tar.gz", hash = "sha256:b9711bdfc51210211137499b5e355d3de5ec88a85d2025c520cbb921b5194367", size = 5360, upload-time = "2024-08-04T21:01:19.405Z" } wheels = [ @@ -1448,27 +1051,10 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/44/6f/7120676b6d73228c96e17f1f794d8ab046fc910d781c8d151120c3f1569e/toml-0.10.2-py2.py3-none-any.whl", hash = "sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b", size = 16588, upload-time = "2020-11-01T01:40:20.672Z" }, ] -[[package]] -name = "tomli" -version = "2.0.1" -source = { registry = "https://pypi.org/simple" } -resolution-markers = [ - "python_full_version < '3.8'", -] -sdist = { url = "https://files.pythonhosted.org/packages/c0/3f/d7af728f075fb08564c5949a9c95e44352e23dee646869fa104a3b2060a3/tomli-2.0.1.tar.gz", hash = "sha256:de526c12914f0c550d15924c62d72abc48d6fe7364aa87328337a31007fe8a4f", size = 15164, upload-time = "2022-02-08T10:54:04.006Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/97/75/10a9ebee3fd790d20926a90a2547f0bf78f371b2f13aa822c759680ca7b9/tomli-2.0.1-py3-none-any.whl", hash = "sha256:939de3e7a6161af0c887ef91b7d41a53e7c5a1ca976325f429cb46ea9bc30ecc", size = 12757, upload-time = "2022-02-08T10:54:02.017Z" }, -] - [[package]] name = "tomli" version = "2.3.0" source = { registry = "https://pypi.org/simple" } -resolution-markers = [ - "python_full_version >= '3.10'", - "python_full_version == '3.9.*'", - "python_full_version == '3.8.*'", -] sdist = { url = "https://files.pythonhosted.org/packages/52/ed/3f73f72945444548f33eba9a87fc7a6e969915e7b1acc8260b30e1f76a2f/tomli-2.3.0.tar.gz", hash = "sha256:64be704a875d2a59753d80ee8a533c3fe183e3f06807ff7dc2232938ccb01549", size = 17392, upload-time = "2025-10-08T22:01:47.119Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/b3/2e/299f62b401438d5fe1624119c723f5d877acc86a4c2492da405626665f12/tomli-2.3.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:88bd15eb972f3664f5ed4b57c1634a97153b4bac4479dcb6a495f41921eb7f45", size = 153236, upload-time = "2025-10-08T22:01:00.137Z" }, @@ -1514,24 +1100,12 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/77/b8/0135fadc89e73be292b473cb820b4f5a08197779206b33191e801feeae40/tomli-2.3.0-py3-none-any.whl", hash = "sha256:e95b1af3c5b07d9e643909b5abbec77cd9f1217e6d0bca72b0234736b9fb1f1b", size = 14408, upload-time = "2025-10-08T22:01:46.04Z" }, ] -[[package]] -name = "typing-extensions" -version = "4.7.1" -source = { registry = "https://pypi.org/simple" } -resolution-markers = [ - "python_full_version < '3.8'", -] -sdist = { url = "https://files.pythonhosted.org/packages/3c/8b/0111dd7d6c1478bf83baa1cab85c686426c7a6274119aceb2bd9d35395ad/typing_extensions-4.7.1.tar.gz", hash = "sha256:b75ddc264f0ba5615db7ba217daeb99701ad295353c45f9e95963337ceeeffb2", size = 72876, upload-time = "2023-07-02T14:20:55.045Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/ec/6b/63cc3df74987c36fe26157ee12e09e8f9db4de771e0f3404263117e75b95/typing_extensions-4.7.1-py3-none-any.whl", hash = "sha256:440d5dd3af93b060174bf433bccd69b0babc3b15b1a8dca43789fd7f61514b36", size = 33232, upload-time = "2023-07-02T14:20:53.275Z" }, -] - [[package]] name = "typing-extensions" version = "4.13.2" source = { registry = "https://pypi.org/simple" } resolution-markers = [ - "python_full_version == '3.8.*'", + "python_full_version < '3.9'", ] sdist = { url = "https://files.pythonhosted.org/packages/f6/37/23083fcd6e35492953e8d2aaaa68b860eb422b34627b13f2ce3eb6106061/typing_extensions-4.13.2.tar.gz", hash = "sha256:e6c81219bd689f51865d9e372991c540bda33a0379d5573cddb9a3a23f7caaef", size = 106967, upload-time = "2025-04-10T14:19:05.416Z" } wheels = [ @@ -1560,92 +1134,12 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/6f/81/5aace343f579809a50e152eac66458ad159c4fae9e06bef438ddb0ebaf92/vcdiff_decoder-0.2.0-py3-none-any.whl", hash = "sha256:0be7163871d0c79fb57b0c9f0a994ee151c4e348a442d3d508da6975983b91aa", size = 19277, upload-time = "2026-09-23T13:05:57.346Z" }, ] -[[package]] -name = "websockets" -version = "11.0.3" -source = { registry = "https://pypi.org/simple" } -resolution-markers = [ - "python_full_version < '3.8'", -] -sdist = { url = "https://files.pythonhosted.org/packages/d8/3b/2ed38e52eed4cf277f9df5f0463a99199a04d9e29c9e227cfafa57bd3993/websockets-11.0.3.tar.gz", hash = "sha256:88fc51d9a26b10fc331be344f1781224a375b78488fc343620184e95a4b27016", size = 104235, upload-time = "2023-05-07T14:25:20.083Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/e1/76/88640f8aeac7eb0d058b913e7bb72682f8d569db44c7d30e576ec4777ce1/websockets-11.0.3-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:3ccc8a0c387629aec40f2fc9fdcb4b9d5431954f934da3eaf16cdc94f67dbfac", size = 123714, upload-time = "2023-05-07T14:23:15.309Z" }, - { url = "https://files.pythonhosted.org/packages/b9/6b/26b28115b46e23e74ede76d95792eedfe8c58b21f4daabfff1e9f159c8fe/websockets-11.0.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:d67ac60a307f760c6e65dad586f556dde58e683fab03323221a4e530ead6f74d", size = 120949, upload-time = "2023-05-07T14:23:17.656Z" }, - { url = "https://files.pythonhosted.org/packages/f3/82/2d1f3395d47fab65fa8b801e2251b324300ed8db54753b6fb7919cef0c11/websockets-11.0.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:84d27a4832cc1a0ee07cdcf2b0629a8a72db73f4cf6de6f0904f6661227f256f", size = 121032, upload-time = "2023-05-07T14:23:20.096Z" }, - { url = "https://files.pythonhosted.org/packages/b2/ec/56bdd12d847e4fc2d0a7ba2d7f1476f79cda50599d11ffb6080b86f21ef1/websockets-11.0.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ffd7dcaf744f25f82190856bc26ed81721508fc5cbf2a330751e135ff1283564", size = 130620, upload-time = "2023-05-07T14:23:21.545Z" }, - { url = "https://files.pythonhosted.org/packages/c9/fb/ae5ed4be3514287cf8f6c348c87e1392a6e3f4d6eadae75c18847a2f84b6/websockets-11.0.3-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7622a89d696fc87af8e8d280d9b421db5133ef5b29d3f7a1ce9f1a7bf7fcfa11", size = 129628, upload-time = "2023-05-07T14:23:23.105Z" }, - { url = "https://files.pythonhosted.org/packages/58/0a/7570e15661a0a546c3a1152d95fe8c05480459bab36247f0acbf41f01a41/websockets-11.0.3-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bceab846bac555aff6427d060f2fcfff71042dba6f5fca7dc4f75cac815e57ca", size = 129938, upload-time = "2023-05-07T14:23:24.959Z" }, - { url = "https://files.pythonhosted.org/packages/ba/6c/5c0322b2875e8395e6bf0eff11f43f3e25da7ef5b12f4d908cd3a19ea841/websockets-11.0.3-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:54c6e5b3d3a8936a4ab6870d46bdd6ec500ad62bde9e44462c32d18f1e9a8e54", size = 134663, upload-time = "2023-05-07T14:23:26.382Z" }, - { url = "https://files.pythonhosted.org/packages/de/0e/d7274e4d41d7b34f204744c27a23707be2ecefaf6f7df2145655f086ecd7/websockets-11.0.3-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:41f696ba95cd92dc047e46b41b26dd24518384749ed0d99bea0a941ca87404c4", size = 133900, upload-time = "2023-05-07T14:23:28.307Z" }, - { url = "https://files.pythonhosted.org/packages/82/3c/00f051abcf88aec5e952a8840076749b0b26a30c219dcae8ba70200998aa/websockets-11.0.3-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:86d2a77fd490ae3ff6fae1c6ceaecad063d3cc2320b44377efdde79880e11526", size = 134520, upload-time = "2023-05-07T14:23:30.734Z" }, - { url = "https://files.pythonhosted.org/packages/8f/7b/4d4ecd29be7d08486e38f987a6603c491296d1e33fe55127d79aebb0333e/websockets-11.0.3-cp310-cp310-win32.whl", hash = "sha256:2d903ad4419f5b472de90cd2d40384573b25da71e33519a67797de17ef849b69", size = 124152, upload-time = "2023-05-07T14:23:33.183Z" }, - { url = "https://files.pythonhosted.org/packages/98/a7/0ed69892981351e5acf88fac0ff4c801fabca2c3bdef9fca4c7d3fde8c53/websockets-11.0.3-cp310-cp310-win_amd64.whl", hash = "sha256:1d2256283fa4b7f4c7d7d3e84dc2ece74d341bce57d5b9bf385df109c2a1a82f", size = 124674, upload-time = "2023-05-07T14:23:35.331Z" }, - { url = "https://files.pythonhosted.org/packages/16/49/ae616bd221efba84a3d78737b417f704af1ffa36f40dcaba5eb954dd4753/websockets-11.0.3-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:e848f46a58b9fcf3d06061d17be388caf70ea5b8cc3466251963c8345e13f7eb", size = 123748, upload-time = "2023-05-07T14:23:37.977Z" }, - { url = "https://files.pythonhosted.org/packages/0a/84/68b848a373493b58615d6c10e9e8ccbaadfd540f84905421739a807704f8/websockets-11.0.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:aa5003845cdd21ac0dc6c9bf661c5beddd01116f6eb9eb3c8e272353d45b3288", size = 120975, upload-time = "2023-05-07T14:23:40.339Z" }, - { url = "https://files.pythonhosted.org/packages/8c/a8/e81533499f84ef6cdd95d11d5b05fa827c0f097925afd86f16e6a2631d8e/websockets-11.0.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:b58cbf0697721120866820b89f93659abc31c1e876bf20d0b3d03cef14faf84d", size = 121017, upload-time = "2023-05-07T14:23:41.874Z" }, - { url = "https://files.pythonhosted.org/packages/6b/ca/65d6986665888494eca4d5435a9741c822022996f0f4200c57ce4b9242f7/websockets-11.0.3-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:660e2d9068d2bedc0912af508f30bbeb505bbbf9774d98def45f68278cea20d3", size = 131200, upload-time = "2023-05-07T14:23:43.309Z" }, - { url = "https://files.pythonhosted.org/packages/c0/a8/a8a582ebeeecc8b5f332997d44c57e241748f8a9856e06a38a5a13b30796/websockets-11.0.3-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c1f0524f203e3bd35149f12157438f406eff2e4fb30f71221c8a5eceb3617b6b", size = 130195, upload-time = "2023-05-07T14:23:45.337Z" }, - { url = "https://files.pythonhosted.org/packages/a9/5e/b25c60067d700e811dccb4e3c318eeadd3a19d8b3620de9f97434af777a7/websockets-11.0.3-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:def07915168ac8f7853812cc593c71185a16216e9e4fa886358a17ed0fd9fcf6", size = 130569, upload-time = "2023-05-07T14:23:46.926Z" }, - { url = "https://files.pythonhosted.org/packages/14/fc/5cbbf439c925e1e184a0392ec477a30cee2fabc0e63807c1d4b6d570fb52/websockets-11.0.3-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:b30c6590146e53149f04e85a6e4fcae068df4289e31e4aee1fdf56a0dead8f97", size = 136015, upload-time = "2023-05-07T14:23:48.43Z" }, - { url = "https://files.pythonhosted.org/packages/0f/d8/a997d3546aef9cc995a1126f7d7ade96c0e16c1a0efb9d2d430aee57c925/websockets-11.0.3-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:619d9f06372b3a42bc29d0cd0354c9bb9fb39c2cbc1a9c5025b4538738dbffaf", size = 135292, upload-time = "2023-05-07T14:23:50.744Z" }, - { url = "https://files.pythonhosted.org/packages/89/8f/707a05d5725f956c78d252a5fd73b89fa3ac57dd3959381c2d1acb41cb13/websockets-11.0.3-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:01f5567d9cf6f502d655151645d4e8b72b453413d3819d2b6f1185abc23e82dd", size = 135890, upload-time = "2023-05-07T14:23:52.707Z" }, - { url = "https://files.pythonhosted.org/packages/b5/94/ac47552208583d5dbcce468430c1eb2ae18962f6b3a694a2b7727cc60d4a/websockets-11.0.3-cp311-cp311-win32.whl", hash = "sha256:e1459677e5d12be8bbc7584c35b992eea142911a6236a3278b9b5ce3326f282c", size = 124149, upload-time = "2023-05-07T14:23:53.848Z" }, - { url = "https://files.pythonhosted.org/packages/e1/7c/0ad6e7ef0a054d73092f616d20d3d9bd3e1b837554cb20a52d8dd9f5b049/websockets-11.0.3-cp311-cp311-win_amd64.whl", hash = "sha256:e7837cb169eca3b3ae94cc5787c4fed99eef74c0ab9506756eea335e0d6f3ed8", size = 124670, upload-time = "2023-05-07T14:23:55.812Z" }, - { url = "https://files.pythonhosted.org/packages/b5/a8/8900184ab0b06b6e620ba7e92cf2faa5caa9ba86e148541b8fff1c7b6646/websockets-11.0.3-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:9f59a3c656fef341a99e3d63189852be7084c0e54b75734cde571182c087b152", size = 120868, upload-time = "2023-05-07T14:23:57.24Z" }, - { url = "https://files.pythonhosted.org/packages/44/a8/66c3a66b70b01a6c55fde486298766177fa11dd0d3a2c1cfc6820f25b4dc/websockets-11.0.3-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2529338a6ff0eb0b50c7be33dc3d0e456381157a31eefc561771ee431134a97f", size = 130557, upload-time = "2023-05-07T14:23:59.274Z" }, - { url = "https://files.pythonhosted.org/packages/70/fc/71377f36ef3049f3bc7db7c0f3a7696929d5f836d7a18777131d994192a9/websockets-11.0.3-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:34fd59a4ac42dff6d4681d8843217137f6bc85ed29722f2f7222bd619d15e95b", size = 129640, upload-time = "2023-05-07T14:24:01.412Z" }, - { url = "https://files.pythonhosted.org/packages/36/19/0da435afb26a6c47c0c045a82e414912aa2ac10de5721276a342bd9fdfee/websockets-11.0.3-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:332d126167ddddec94597c2365537baf9ff62dfcc9db4266f263d455f2f031cb", size = 129903, upload-time = "2023-05-07T14:24:02.872Z" }, - { url = "https://files.pythonhosted.org/packages/38/ed/b8b133416536b6816e480594864e5950051db522714623eefc9e5275ec04/websockets-11.0.3-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:6505c1b31274723ccaf5f515c1824a4ad2f0d191cec942666b3d0f3aa4cb4007", size = 135302, upload-time = "2023-05-07T14:24:04.326Z" }, - { url = "https://files.pythonhosted.org/packages/e9/26/1dfaa81788f61c485b4d65f1b28a19615e39f9c45100dce5e2cbf5ad1352/websockets-11.0.3-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:f467ba0050b7de85016b43f5a22b46383ef004c4f672148a8abf32bc999a87f0", size = 134562, upload-time = "2023-05-07T14:24:05.829Z" }, - { url = "https://files.pythonhosted.org/packages/a7/f7/1e852351e8073c32885172a6bef64c95d14c13ff3634b01d4a1086321491/websockets-11.0.3-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:9d9acd80072abcc98bd2c86c3c9cd4ac2347b5a5a0cae7ed5c0ee5675f86d9af", size = 135191, upload-time = "2023-05-07T14:24:07.659Z" }, - { url = "https://files.pythonhosted.org/packages/19/d3/2ea3f95d83033675144b0848a0ae2e4998b3f763da09ec3df6bce97ea4e6/websockets-11.0.3-cp37-cp37m-win32.whl", hash = "sha256:e590228200fcfc7e9109509e4d9125eace2042fd52b595dd22bbc34bb282307f", size = 124138, upload-time = "2023-05-07T14:24:09.697Z" }, - { url = "https://files.pythonhosted.org/packages/94/8c/266155c14b7a26deca6fa4c4d5fd15b0ab32725d78a2acfcf6b24943585d/websockets-11.0.3-cp37-cp37m-win_amd64.whl", hash = "sha256:b16fff62b45eccb9c7abb18e60e7e446998093cdcb50fed33134b9b6878836de", size = 124672, upload-time = "2023-05-07T14:24:12.595Z" }, - { url = "https://files.pythonhosted.org/packages/c4/5b/60eccd7e9703bbe93fc4167d1e7ada7e8e8e51544122198d63fd8e3460b7/websockets-11.0.3-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:fb06eea71a00a7af0ae6aefbb932fb8a7df3cb390cc217d51a9ad7343de1b8d0", size = 123706, upload-time = "2023-05-07T14:24:14.633Z" }, - { url = "https://files.pythonhosted.org/packages/d1/ec/7e2b9bebc2e9b4a48404144106bbc6a7ace781feeb0e6a3829551e725fa5/websockets-11.0.3-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:8a34e13a62a59c871064dfd8ffb150867e54291e46d4a7cf11d02c94a5275bae", size = 120944, upload-time = "2023-05-07T14:24:16.144Z" }, - { url = "https://files.pythonhosted.org/packages/8a/bd/a5e5973899d78d44a540f50a9e30b01c6771e8bf7883204ee762060cf95a/websockets-11.0.3-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:4841ed00f1026dfbced6fca7d963c4e7043aa832648671b5138008dc5a8f6d99", size = 121030, upload-time = "2023-05-07T14:24:17.905Z" }, - { url = "https://files.pythonhosted.org/packages/ec/3f/0c5cae14e9e86401105833383405787ae4caddd476a8fc5561259253dab7/websockets-11.0.3-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1a073fc9ab1c8aff37c99f11f1641e16da517770e31a37265d2755282a5d28aa", size = 130811, upload-time = "2023-05-07T14:24:21.175Z" }, - { url = "https://files.pythonhosted.org/packages/a0/39/acc3d4b15c5207ef7cca823c37eca8c74e3e1a1a63a397798986be3bdef7/websockets-11.0.3-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:68b977f21ce443d6d378dbd5ca38621755f2063d6fdb3335bda981d552cfff86", size = 129876, upload-time = "2023-05-07T14:24:22.498Z" }, - { url = "https://files.pythonhosted.org/packages/58/05/2efb520317340ece74bfc4d88e8f011dd71a4e6c263000bfffb71a343685/websockets-11.0.3-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e1a99a7a71631f0efe727c10edfba09ea6bee4166a6f9c19aafb6c0b5917d09c", size = 130158, upload-time = "2023-05-07T14:24:25.193Z" }, - { url = "https://files.pythonhosted.org/packages/30/a5/d641f2a9a4b4079cfddbb0726fc1b914be76a610aaedb45e4760899a4ce1/websockets-11.0.3-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:bee9fcb41db2a23bed96c6b6ead6489702c12334ea20a297aa095ce6d31370d0", size = 134494, upload-time = "2023-05-07T14:24:26.463Z" }, - { url = "https://files.pythonhosted.org/packages/ca/20/25211be61d50189650fb0ec6084b6d6339f5c7c6436a6c217608dcb553e4/websockets-11.0.3-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:4b253869ea05a5a073ebfdcb5cb3b0266a57c3764cf6fe114e4cd90f4bfa5f5e", size = 133735, upload-time = "2023-05-07T14:24:29.598Z" }, - { url = "https://files.pythonhosted.org/packages/c6/91/f36454b87edf10a95be9c7212d2dcb8c606ddbf7a183afdc498933acdd19/websockets-11.0.3-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:1553cb82942b2a74dd9b15a018dce645d4e68674de2ca31ff13ebc2d9f283788", size = 134368, upload-time = "2023-05-07T14:24:30.923Z" }, - { url = "https://files.pythonhosted.org/packages/58/68/9403771de1b1c21a2e878e4841815af8c9f8893b094654934e2a5ee4dbc8/websockets-11.0.3-cp38-cp38-win32.whl", hash = "sha256:f61bdb1df43dc9c131791fbc2355535f9024b9a04398d3bd0684fc16ab07df74", size = 124148, upload-time = "2023-05-07T14:24:32.299Z" }, - { url = "https://files.pythonhosted.org/packages/25/25/48540419005d07ed2d368a7eafb44ed4f33a2691ae4c210850bf31123c4a/websockets-11.0.3-cp38-cp38-win_amd64.whl", hash = "sha256:03aae4edc0b1c68498f41a6772d80ac7c1e33c06c6ffa2ac1c27a07653e79d6f", size = 124665, upload-time = "2023-05-07T14:24:34.482Z" }, - { url = "https://files.pythonhosted.org/packages/c0/21/cb9dfbbea8dc0ad89ced52630e7e61edb425fb9fdc6002f8d0c5dd26b94b/websockets-11.0.3-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:777354ee16f02f643a4c7f2b3eff8027a33c9861edc691a2003531f5da4f6bc8", size = 123707, upload-time = "2023-05-07T14:24:36.007Z" }, - { url = "https://files.pythonhosted.org/packages/8f/f2/8a3eb016be19743c7eb9e67c855df0fdfa5912534ffaf83a05b62667d761/websockets-11.0.3-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:8c82f11964f010053e13daafdc7154ce7385ecc538989a354ccc7067fd7028fd", size = 120963, upload-time = "2023-05-07T14:24:37.478Z" }, - { url = "https://files.pythonhosted.org/packages/a0/1a/3da73e69ebc00649d11ed836541c92c1a2df0b8a8aa641a2c8746e7c2b9c/websockets-11.0.3-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:3580dd9c1ad0701169e4d6fc41e878ffe05e6bdcaf3c412f9d559389d0c9e016", size = 121014, upload-time = "2023-05-07T14:24:39.009Z" }, - { url = "https://files.pythonhosted.org/packages/d9/36/5741e62ccf629c8e38cc20f930491f8a33ce7dba972cae93dba3d6f02552/websockets-11.0.3-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6f1a3f10f836fab6ca6efa97bb952300b20ae56b409414ca85bff2ad241d2a61", size = 130408, upload-time = "2023-05-07T14:24:40.825Z" }, - { url = "https://files.pythonhosted.org/packages/66/89/799f595c67b97a8a17e13d2764e088f631616bd95668aaa4c04b7cada136/websockets-11.0.3-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:df41b9bc27c2c25b486bae7cf42fccdc52ff181c8c387bfd026624a491c2671b", size = 129407, upload-time = "2023-05-07T14:24:42.479Z" }, - { url = "https://files.pythonhosted.org/packages/a6/9c/2356ecb952fd3992b73f7a897d65e57d784a69b94bb8d8fd5f97531e5c02/websockets-11.0.3-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:279e5de4671e79a9ac877427f4ac4ce93751b8823f276b681d04b2156713b9dd", size = 129712, upload-time = "2023-05-07T14:24:44.186Z" }, - { url = "https://files.pythonhosted.org/packages/c4/f5/15998b164c183af0513bba744b51ecb08d396ff86c0db3b55d62624d1f15/websockets-11.0.3-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:1fdf26fa8a6a592f8f9235285b8affa72748dc12e964a5518c6c5e8f916716f7", size = 134386, upload-time = "2023-05-07T14:24:45.702Z" }, - { url = "https://files.pythonhosted.org/packages/8a/77/a04d2911f6e2b9e781ce7ffc1e8516b54b85f985369eec8c853fd619d8e8/websockets-11.0.3-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:69269f3a0b472e91125b503d3c0b3566bda26da0a3261c49f0027eb6075086d1", size = 133639, upload-time = "2023-05-07T14:24:46.966Z" }, - { url = "https://files.pythonhosted.org/packages/72/89/0d150939f2e592ed78c071d69237ac1c872462cc62a750c5f592f3d4ab18/websockets-11.0.3-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:97b52894d948d2f6ea480171a27122d77af14ced35f62e5c892ca2fae9344311", size = 134260, upload-time = "2023-05-07T14:24:48.633Z" }, - { url = "https://files.pythonhosted.org/packages/9a/6e/0fd7274042f46acb589161407f4b505b44c68d369437ce919bae1fa9b8c4/websockets-11.0.3-cp39-cp39-win32.whl", hash = "sha256:c7f3cb904cce8e1be667c7e6fef4516b98d1a6a0635a58a57528d577ac18a128", size = 124146, upload-time = "2023-05-07T14:24:50.566Z" }, - { url = "https://files.pythonhosted.org/packages/f4/3f/65dfa50084a06ab0a05f3ca74195c2c17a1c075b8361327d831ccce0a483/websockets-11.0.3-cp39-cp39-win_amd64.whl", hash = "sha256:c792ea4eabc0159535608fc5658a74d1a81020eb35195dd63214dcf07556f67e", size = 124665, upload-time = "2023-05-07T14:24:52.111Z" }, - { url = "https://files.pythonhosted.org/packages/e2/2f/3ad8ac4a9dc9d685e098e534180a36ed68fe2e85e82e225e00daec86bb94/websockets-11.0.3-pp37-pypy37_pp73-macosx_10_9_x86_64.whl", hash = "sha256:f2e58f2c36cc52d41f2659e4c0cbf7353e28c8c9e63e30d8c6d3494dc9fdedcf", size = 120795, upload-time = "2023-05-07T14:24:53.421Z" }, - { url = "https://files.pythonhosted.org/packages/a7/8c/7100e9cf310fe1d83d1ae1322203f4eb2b767a7c2b301c1e70db6270306f/websockets-11.0.3-pp37-pypy37_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:de36fe9c02995c7e6ae6efe2e205816f5f00c22fd1fbf343d4d18c3d5ceac2f5", size = 122910, upload-time = "2023-05-07T14:24:54.851Z" }, - { url = "https://files.pythonhosted.org/packages/78/b2/df5452031b02b857851139806308f2af7c749069e25bfe15f2d559ade6e7/websockets-11.0.3-pp37-pypy37_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0ac56b661e60edd453585f4bd68eb6a29ae25b5184fd5ba51e97652580458998", size = 122516, upload-time = "2023-05-07T14:24:56.477Z" }, - { url = "https://files.pythonhosted.org/packages/03/28/3a51ffcf51ac45746639f83128908bbb1cd212aa631e42d15a7acebce5cb/websockets-11.0.3-pp37-pypy37_pp73-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e052b8467dd07d4943936009f46ae5ce7b908ddcac3fda581656b1b19c083d9b", size = 122462, upload-time = "2023-05-07T14:24:57.77Z" }, - { url = "https://files.pythonhosted.org/packages/eb/fb/2af7fc3ce2c3f1378d48a15802b4ff2caf6c0dfac13291e73c557caf04f7/websockets-11.0.3-pp37-pypy37_pp73-win_amd64.whl", hash = "sha256:42cc5452a54a8e46a032521d7365da775823e21bfba2895fb7b77633cce031bb", size = 124704, upload-time = "2023-05-07T14:24:59.322Z" }, - { url = "https://files.pythonhosted.org/packages/20/62/5c6039c4069912adb27889ddd000403a2de9e0fe6aebe439b4e6b128a6b8/websockets-11.0.3-pp38-pypy38_pp73-macosx_10_9_x86_64.whl", hash = "sha256:e6316827e3e79b7b8e7d8e3b08f4e331af91a48e794d5d8b099928b6f0b85f20", size = 120795, upload-time = "2023-05-07T14:25:01.047Z" }, - { url = "https://files.pythonhosted.org/packages/38/30/01a10fbf4cc1e7ffa07be9b0401501918fc9433d71fb7da4cfcef3bd26ca/websockets-11.0.3-pp38-pypy38_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8531fdcad636d82c517b26a448dcfe62f720e1922b33c81ce695d0edb91eb931", size = 122908, upload-time = "2023-05-07T14:25:02.734Z" }, - { url = "https://files.pythonhosted.org/packages/99/23/43071c989c0f87f612e7bccee98d00b04bddd3aca0cdc1ffaf31f6f8a4b4/websockets-11.0.3-pp38-pypy38_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c114e8da9b475739dde229fd3bc6b05a6537a88a578358bc8eb29b4030fac9c9", size = 122515, upload-time = "2023-05-07T14:25:04.803Z" }, - { url = "https://files.pythonhosted.org/packages/b6/96/0d586c25d043aeab9457dad8e407251e3baf314d871215f91847e7b995c4/websockets-11.0.3-pp38-pypy38_pp73-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e063b1865974611313a3849d43f2c3f5368093691349cf3c7c8f8f75ad7cb280", size = 122465, upload-time = "2023-05-07T14:25:06.352Z" }, - { url = "https://files.pythonhosted.org/packages/27/e9/605b0618d0864e9be7c2a78f22bff57aba9cf56b9fccde3205db9023ae22/websockets-11.0.3-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:92b2065d642bf8c0a82d59e59053dd2fdde64d4ed44efe4870fa816c1232647b", size = 124707, upload-time = "2023-05-07T14:25:07.782Z" }, - { url = "https://files.pythonhosted.org/packages/1b/3d/3dc77699fa4d003f2e810c321592f80f62b81d7b78483509de72ffe581fd/websockets-11.0.3-pp39-pypy39_pp73-macosx_10_9_x86_64.whl", hash = "sha256:0ee68fe502f9031f19d495dae2c268830df2760c0524cbac5d759921ba8c8e82", size = 120795, upload-time = "2023-05-07T14:25:09.785Z" }, - { url = "https://files.pythonhosted.org/packages/a6/1b/5c83c40f8d3efaf0bb2fdf05af94fb920f74842b7aaf31d7598e3ee44d58/websockets-11.0.3-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:dcacf2c7a6c3a84e720d1bb2b543c675bf6c40e460300b628bab1b1efc7c034c", size = 122909, upload-time = "2023-05-07T14:25:11.243Z" }, - { url = "https://files.pythonhosted.org/packages/32/2c/ab8ea64e9a7d8bf62a7ea7a037fb8d328d8bd46dbfe083787a9d452a148e/websockets-11.0.3-pp39-pypy39_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b67c6f5e5a401fc56394f191f00f9b3811fe843ee93f4a70df3c389d1adf857d", size = 122517, upload-time = "2023-05-07T14:25:12.881Z" }, - { url = "https://files.pythonhosted.org/packages/8b/97/34178f5f7c29e679372d597cebfeff2aa45991d741d938117d4616e81a74/websockets-11.0.3-pp39-pypy39_pp73-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1d5023a4b6a5b183dc838808087033ec5df77580485fc533e7dab2567851b0a4", size = 122463, upload-time = "2023-05-07T14:25:15.154Z" }, - { url = "https://files.pythonhosted.org/packages/ed/45/466944e00b324ae3a1fddb305b4abf641f582e131548f07bcd970971b154/websockets-11.0.3-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:ed058398f55163a79bb9f06a90ef9ccc063b204bb346c4de78efc5d15abfe602", size = 124707, upload-time = "2023-05-07T14:25:17.112Z" }, - { url = "https://files.pythonhosted.org/packages/47/96/9d5749106ff57629b54360664ae7eb9afd8302fad1680ead385383e33746/websockets-11.0.3-py3-none-any.whl", hash = "sha256:6681ba9e7f8f3b19440921e99efbb40fc89f26cd71bf539e45d8c8a25c976dc6", size = 118056, upload-time = "2023-05-07T14:25:18.508Z" }, -] - [[package]] name = "websockets" version = "13.1" source = { registry = "https://pypi.org/simple" } resolution-markers = [ - "python_full_version == '3.8.*'", + "python_full_version < '3.9'", ] sdist = { url = "https://files.pythonhosted.org/packages/e2/73/9223dbc7be3dcaf2a7bbf756c351ec8da04b1fa573edaf545b95f6b0c7fd/websockets-13.1.tar.gz", hash = "sha256:a3b3366087c1bc0a2795111edcadddb8b3b59509d5db5d7ea3fdd69f954a8878", size = 158549, upload-time = "2024-09-21T17:34:21.54Z" } wheels = [ @@ -1816,24 +1310,12 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/fa/a8/5b41e0da817d64113292ab1f8247140aac61cbf6cfd085d6a0fa77f4984f/websockets-15.0.1-py3-none-any.whl", hash = "sha256:f7a866fbc1e97b5c617ee4116daaa09b722101d4a3c170c787450ba409f9736f", size = 169743, upload-time = "2025-03-05T20:03:39.41Z" }, ] -[[package]] -name = "zipp" -version = "3.15.0" -source = { registry = "https://pypi.org/simple" } -resolution-markers = [ - "python_full_version < '3.8'", -] -sdist = { url = "https://files.pythonhosted.org/packages/00/27/f0ac6b846684cecce1ee93d32450c45ab607f65c2e0255f0092032d91f07/zipp-3.15.0.tar.gz", hash = "sha256:112929ad649da941c23de50f356a2b5570c954b65150642bccdd66bf194d224b", size = 18454, upload-time = "2023-02-25T02:17:22.503Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/5b/fa/c9e82bbe1af6266adf08afb563905eb87cab83fde00a0a08963510621047/zipp-3.15.0-py3-none-any.whl", hash = "sha256:48904fc76a60e542af151aded95726c1a5c34ed43ab4134b597665c86d7ad556", size = 6758, upload-time = "2023-02-25T02:17:20.807Z" }, -] - [[package]] name = "zipp" version = "3.20.2" source = { registry = "https://pypi.org/simple" } resolution-markers = [ - "python_full_version == '3.8.*'", + "python_full_version < '3.9'", ] sdist = { url = "https://files.pythonhosted.org/packages/54/bf/5c0000c44ebc80123ecbdddba1f5dcd94a5ada602a9c225d84b5aaa55e86/zipp-3.20.2.tar.gz", hash = "sha256:bc9eb26f4506fda01b81bcde0ca78103b6e62f991b381fec825435c836edbc29", size = 24199, upload-time = "2024-09-13T13:44:16.101Z" } wheels = [ From f9417d988802bc5514e786092975e9375681567e Mon Sep 17 00:00:00 2001 From: evgeny Date: Tue, 15 Sep 2026 13:58:51 +0100 Subject: [PATCH 03/71] pubsub: report ably-pubsub-python and ably-pubsub-server in Ably-Agent The distribution was renamed to ably-pubsub-server, so the agent header should no longer identify the library as ably-python. Report the SDK as ably-pubsub-python and add a versionless ably-pubsub-server agent entry so requests are attributable to the server package. Update the Ably-Agent assertion in the REST HTTP tests to match the new header format. Co-Authored-By: Claude Opus 5 (1M context) --- ably/pubsub/http/httputils.py | 3 ++- test/ably/rest/resthttp_test.py | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/ably/pubsub/http/httputils.py b/ably/pubsub/http/httputils.py index 2410580f..cf7dec93 100644 --- a/ably/pubsub/http/httputils.py +++ b/ably/pubsub/http/httputils.py @@ -56,7 +56,8 @@ def default_headers(version=None): version = api_version return { "X-Ably-Version": version, - "Ably-Agent": f'ably-python/{lib_version} python/{platform.python_version()}' + "Ably-Agent": f'ably-pubsub-python/{lib_version} ably-pubsub-server ' + f'python/{platform.python_version()}' } @staticmethod diff --git a/test/ably/rest/resthttp_test.py b/test/ably/rest/resthttp_test.py index 3a295e43..4530621b 100644 --- a/test/ably/rest/resthttp_test.py +++ b/test/ably/rest/resthttp_test.py @@ -194,7 +194,7 @@ async def test_request_headers(self): # Agent assert 'Ably-Agent' in r.request.headers - expr = r"^ably-python\/\d+.\d+.\d+(-beta\.\d+)? python\/\d.\d+.\d+$" + expr = r"^ably-pubsub-python\/\d+.\d+.\d+(-beta\.\d+)? ably-pubsub-server python\/\d.\d+.\d+$" assert re.search(expr, r.request.headers['Ably-Agent']) await ably.close() From 1e93728871d44e96f5241f3c46255fcadb9627e1 Mon Sep 17 00:00:00 2001 From: evgeny Date: Tue, 15 Sep 2026 19:43:45 +0100 Subject: [PATCH 04/71] tests: improve protocol member detection and resolve client naming conflicts - Updated `protocol_members` to handle protocol attribute detection more accurately, relying on `typing._get_protocol_attrs` where available, avoiding unintended attribute inclusion. - Renamed `ably` test client to `app_setup_client` to prevent shadowing the imported `ably` package and ensure correct resolution during test execution. - Adjusted `pyproject.toml` to configure `ruff` for consistent lint behavior in both local and CI environments. --- ably/pubsub/http/httputils.py | 3 +-- pyproject.toml | 7 +++++++ test/ably/rest/resthttp_test.py | 2 +- test/ably/testapp.py | 13 ++++++++----- test/unit/pubsub_server_test.py | 19 ++++++++++++++----- 5 files changed, 31 insertions(+), 13 deletions(-) diff --git a/ably/pubsub/http/httputils.py b/ably/pubsub/http/httputils.py index cf7dec93..f24a2f00 100644 --- a/ably/pubsub/http/httputils.py +++ b/ably/pubsub/http/httputils.py @@ -56,8 +56,7 @@ def default_headers(version=None): version = api_version return { "X-Ably-Version": version, - "Ably-Agent": f'ably-pubsub-python/{lib_version} ably-pubsub-server ' - f'python/{platform.python_version()}' + "Ably-Agent": f'ably-pubsub-python/{lib_version} python/{platform.python_version()}' } @staticmethod diff --git a/pyproject.toml b/pyproject.toml index b91f41f5..2e0d5eed 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -117,3 +117,10 @@ ignore = [ [tool.ruff.lint.per-file-ignores] "__init__.py" = ["F401"] # imported but unused + +[tool.ruff.lint.isort] +# `ably.pubsub.sync` and `test.ably.sync` only exist once unasync has generated +# them, and the lint job does not generate them. Left to resolve imports itself, +# ruff files those modules as third-party there and first-party here, so the same +# commit lints clean locally and fails in CI. State it instead of inferring it. +known-first-party = ["ably", "test"] diff --git a/test/ably/rest/resthttp_test.py b/test/ably/rest/resthttp_test.py index 4530621b..ca2122bb 100644 --- a/test/ably/rest/resthttp_test.py +++ b/test/ably/rest/resthttp_test.py @@ -194,7 +194,7 @@ async def test_request_headers(self): # Agent assert 'Ably-Agent' in r.request.headers - expr = r"^ably-pubsub-python\/\d+.\d+.\d+(-beta\.\d+)? ably-pubsub-server python\/\d.\d+.\d+$" + expr = r"^ably-pubsub-python\/\d+.\d+.\d+(-beta\.\d+)? python\/\d.\d+.\d+$" assert re.search(expr, r.request.headers['Ably-Agent']) await ably.close() diff --git a/test/ably/testapp.py b/test/ably/testapp.py index 2e1f9b96..30a0b6d9 100644 --- a/test/ably/testapp.py +++ b/test/ably/testapp.py @@ -23,10 +23,13 @@ port = 80 tls_port = 443 -ably = create_http_client(token='not_a_real_token', - port=port, tls_port=tls_port, tls=tls, - endpoint=endpoint, - use_binary_protocol=False) +# Not named `ably`: that would shadow the `ably` package imported above, and +# `ably.pubsub.server.create_realtime_client` below would resolve against this +# client instead of the module. +app_setup_client = create_http_client(token='not_a_real_token', + port=port, tls_port=tls_port, tls=tls, + endpoint=endpoint, + use_binary_protocol=False) class TestApp: @@ -35,7 +38,7 @@ class TestApp: @staticmethod async def get_test_vars(): if not TestApp.__test_vars: - r = await ably.http.post("/apps", body=app_spec_local, skip_auth=True) + r = await app_setup_client.http.post("/apps", body=app_spec_local, skip_auth=True) AblyException.raise_for_response(r) app_spec = r.json() diff --git a/test/unit/pubsub_server_test.py b/test/unit/pubsub_server_test.py index 8f1aa009..08b823fc 100644 --- a/test/unit/pubsub_server_test.py +++ b/test/unit/pubsub_server_test.py @@ -1,3 +1,5 @@ +import typing + import pytest import ably.pubsub.server as server @@ -10,11 +12,18 @@ def protocol_members(protocol): - # Protocol records its declared members here; fall back to the class body - # on interpreters that do not expose it. - return getattr(protocol, '__protocol_attrs__', None) or { - name for name in vars(protocol) if not name.startswith('_abc_') and name != '_is_protocol' - } + """The members `protocol` declares. + + Python 3.12 records these on the class. Earlier versions only have typing's + own helper, which is the function that builds that attribute, so the two + agree. Reading `vars(protocol)` instead would sweep in `__init__`, + `__abstractmethods__`, `__parameters__` and the rest of the Protocol + machinery, and report them as members the client fails to implement. + """ + members = getattr(protocol, '__protocol_attrs__', None) + if members is None: + members = typing._get_protocol_attrs(protocol) + return set(members) class TestFactories: From 1ff159254a34d842a6b07397fec061db1d7696ac Mon Sep 17 00:00:00 2001 From: evgeny Date: Fri, 18 Sep 2026 14:40:12 +0100 Subject: [PATCH 05/71] pubsub: name the HTTP client surface after its factory MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The public entry point is create_http_client, but everything behind it was named "rest": the ably.pubsub.rest package, AblyRest, and the RestClient prototype. Rename that surface to "http" so the names a caller reads match the name they call. The transport package already holding ably.pubsub.http — Http, HttpUtils and PaginatedResult — moves to ably.pubsub.request to free the name, which also describes it better: it models individual requests, not the client. - ably/pubsub/rest/ -> ably/pubsub/http/, with rest.py -> http.py - AblyRest -> AblyHttp, RestAnnotations -> HttpAnnotations, RestChannels -> HttpChannels - the exported prototypes RestClient and RealtimeClient become PubSubHttpClient and PubSubRealtimeClient - test/ably/rest/ -> test/ably/http/, rest*_test.py -> http*_test.py, TestRest* -> TestHttp* unasync's rename list, mocked-path replacements and test directories follow the move, so the generated synchronous flavour lands at ably.pubsub.sync.http. The spec-defined options rest_host and idempotent_rest_publishing keep their names, as does "REST" where it refers to the Ably API rather than to a symbol. Co-Authored-By: Claude Opus 5 (1M context) --- README.md | 3 +- UPDATING.md | 9 +- ably/pubsub/{rest => http}/annotations.py | 6 +- ably/pubsub/{rest => http}/auth.py | 6 +- ably/pubsub/{rest => http}/channel.py | 10 +- ably/pubsub/http/http.py | 415 +++++------------- ably/pubsub/{rest => http}/push.py | 2 +- ably/pubsub/prototypes.py | 22 +- ably/pubsub/realtime/annotations.py | 6 +- ably/pubsub/realtime/channel.py | 8 +- ably/pubsub/realtime/realtime.py | 6 +- ably/pubsub/{rest => request}/__init__.py | 0 ably/pubsub/request/http.py | 328 ++++++++++++++ ably/pubsub/{http => request}/httputils.py | 0 .../{http => request}/paginatedresult.py | 2 +- ably/pubsub/rest/rest.py | 153 ------- ably/pubsub/scripts/unasync.py | 18 +- ably/pubsub/server/__init__.py | 26 +- ably/pubsub/server/sync.py | 16 +- ably/pubsub/transport/websockettransport.py | 2 +- ably/pubsub/types/presence.py | 2 +- test/ably/{rest => http}/authheaders_test.py | 4 +- test/ably/{rest => http}/encoders_test.py | 38 +- .../httpannotations_test.py} | 2 +- .../httpauth_test.py} | 12 +- .../httpcapability_test.py} | 2 +- .../httpchannelhistory_test.py} | 4 +- .../httpchannelmutablemessages_test.py} | 2 +- .../httpchannelpublish_test.py} | 12 +- .../httpchannels_test.py} | 2 +- .../httpchannelstatus_test.py} | 2 +- .../httpcrypto_test.py} | 2 +- .../httphttp_test.py} | 4 +- .../httpinit_test.py} | 6 +- .../httppaginatedresult_test.py} | 2 +- .../httppresence_test.py} | 2 +- .../httppush_test.py} | 2 +- .../httprequest_test.py} | 4 +- .../httpstats_test.py} | 18 +- .../httptime_test.py} | 2 +- .../httptoken_test.py} | 20 +- test/ably/utils.py | 2 +- test/unit/annotation_test.py | 2 +- test/unit/mutable_message_test.py | 2 +- test/unit/pubsub_server_test.py | 28 +- test/uts/helpers/sandbox.py | 2 +- .../channels/channel_attach_test.py | 2 +- .../channels/channel_publish_test.py | 2 +- .../integration/mutable_messages_test.py | 2 +- .../unit/channels/channel_history_test.py | 6 +- .../channels/channel_message_versions_test.py | 2 +- .../unit/channels/channel_publish_test.py | 2 +- .../unit/client/realtime_client_test.py | 4 +- .../unit/client/realtime_request_test.py | 2 +- .../unit/client/realtime_stats_test.py | 4 +- .../unit/client/realtime_time_test.py | 2 +- .../realtime_presence_history_test.py | 2 +- .../rest/integration/batch_presence_test.py | 2 +- .../rest/integration/mutable_messages_test.py | 2 +- test/uts/rest/integration/pagination_test.py | 2 +- test/uts/rest/integration/presence_test.py | 2 +- test/uts/rest/integration/push_admin_test.py | 2 +- .../rest/integration/push_channels_test.py | 2 +- test/uts/rest/integration/time_stats_test.py | 2 +- test/uts/rest/unit/batch_presence_test.py | 2 +- test/uts/rest/unit/batch_publish_test.py | 2 +- .../uts/rest/unit/channel/annotations_test.py | 6 +- test/uts/rest/unit/channel/history_test.py | 2 +- .../unit/channel/message_versions_test.py | 2 +- .../uts/rest/unit/channels_collection_test.py | 2 +- .../rest/unit/presence/rest_presence_test.py | 2 +- .../rest/unit/push/push_admin_publish_test.py | 2 +- .../push/push_channel_subscriptions_test.py | 2 +- test/uts/rest/unit/push/push_channels_test.py | 2 +- .../push/push_device_registrations_test.py | 2 +- test/uts/rest/unit/rest_client_test.py | 2 +- test/uts/rest/unit/stats_test.py | 2 +- 77 files changed, 647 insertions(+), 645 deletions(-) rename ably/pubsub/{rest => http}/annotations.py (98%) rename ably/pubsub/{rest => http}/auth.py (99%) rename ably/pubsub/{rest => http}/channel.py (98%) rename ably/pubsub/{rest => http}/push.py (98%) rename ably/pubsub/{rest => request}/__init__.py (100%) create mode 100644 ably/pubsub/request/http.py rename ably/pubsub/{http => request}/httputils.py (100%) rename ably/pubsub/{http => request}/paginatedresult.py (98%) delete mode 100644 ably/pubsub/rest/rest.py rename test/ably/{rest => http}/authheaders_test.py (95%) rename test/ably/{rest => http}/encoders_test.py (93%) rename test/ably/{rest/restannotations_test.py => http/httpannotations_test.py} (99%) rename test/ably/{rest/restauth_test.py => http/httpauth_test.py} (98%) rename test/ably/{rest/restcapability_test.py => http/httpcapability_test.py} (99%) rename test/ably/{rest/restchannelhistory_test.py => http/httpchannelhistory_test.py} (99%) rename test/ably/{rest/restchannelmutablemessages_test.py => http/httpchannelmutablemessages_test.py} (99%) rename test/ably/{rest/restchannelpublish_test.py => http/httpchannelpublish_test.py} (98%) rename test/ably/{rest/restchannels_test.py => http/httpchannels_test.py} (98%) rename test/ably/{rest/restchannelstatus_test.py => http/httpchannelstatus_test.py} (97%) rename test/ably/{rest/restcrypto_test.py => http/httpcrypto_test.py} (99%) rename test/ably/{rest/resthttp_test.py => http/httphttp_test.py} (98%) rename test/ably/{rest/restinit_test.py => http/httpinit_test.py} (97%) rename test/ably/{rest/restpaginatedresult_test.py => http/httppaginatedresult_test.py} (98%) rename test/ably/{rest/restpresence_test.py => http/httppresence_test.py} (99%) rename test/ably/{rest/restpush_test.py => http/httppush_test.py} (99%) rename test/ably/{rest/restrequest_test.py => http/httprequest_test.py} (98%) rename test/ably/{rest/reststats_test.py => http/httpstats_test.py} (95%) rename test/ably/{rest/resttime_test.py => http/httptime_test.py} (95%) rename test/ably/{rest/resttoken_test.py => http/httptoken_test.py} (95%) diff --git a/README.md b/README.md index 5b95efe8..78c0058e 100644 --- a/README.md +++ b/README.md @@ -60,7 +60,8 @@ from ably.pubsub.server import create_http_client, create_realtime_client Clients are built by these factories rather than by constructing a class, so that the package a client comes from names the side your application runs on. -Annotate against the prototypes they return, `RestClient` and `RealtimeClient`. +Annotate against the prototypes they return, `PubSubHttpClient` and +`PubSubRealtimeClient`. The synchronous, HTTP-only flavour lives alongside it: diff --git a/UPDATING.md b/UPDATING.md index 6f3b7ee5..1077f181 100644 --- a/UPDATING.md +++ b/UPDATING.md @@ -54,8 +54,9 @@ instance `ably.types.message` becomes `ably.pubsub.types.message`, and ### Clients are built by factories `AblyRest` and `AblyRealtime` are internal in 4.0.0 and raise `TypeError` if -constructed directly. Build clients through the factories instead, which take -the same arguments: +constructed directly (`AblyRest` is also renamed to `AblyHttp`, to match the +factory that builds it). Build clients through the factories instead, which +take the same arguments: Example 3.x code: ```python @@ -70,8 +71,8 @@ realtime = create_realtime_client(key='xxx') ``` Where you previously annotated against `AblyRest` or `AblyRealtime`, use the -prototypes the factories return — `ably.pubsub.server.RestClient` and -`ably.pubsub.server.RealtimeClient`. These are `typing.Protocol` definitions +prototypes the factories return — `ably.pubsub.server.PubSubHttpClient` and +`ably.pubsub.server.PubSubRealtimeClient`. These are `typing.Protocol` definitions describing the client surface, so `Protocol` support raises the minimum supported Python to 3.8 (which the CI matrix and README already assumed). diff --git a/ably/pubsub/rest/annotations.py b/ably/pubsub/http/annotations.py similarity index 98% rename from ably/pubsub/rest/annotations.py rename to ably/pubsub/http/annotations.py index b84d2bd4..88332352 100644 --- a/ably/pubsub/rest/annotations.py +++ b/ably/pubsub/http/annotations.py @@ -8,7 +8,7 @@ import msgpack -from ably.pubsub.http.paginatedresult import PaginatedResult, format_params +from ably.pubsub.request.paginatedresult import PaginatedResult, format_params from ably.pubsub.types.annotation import ( Annotation, AnnotationAction, @@ -89,7 +89,7 @@ def construct_validate_annotation(msg_or_serial, annotation: Annotation) -> Anno ) -class RestAnnotations: +class HttpAnnotations: """ Provides REST API methods for managing annotations on messages. """ @@ -98,7 +98,7 @@ class RestAnnotations: def __init__(self, channel): """ - Initialize RestAnnotations. + Initialize HttpAnnotations. Args: channel: The REST Channel this annotations instance belongs to diff --git a/ably/pubsub/rest/auth.py b/ably/pubsub/http/auth.py similarity index 99% rename from ably/pubsub/rest/auth.py rename to ably/pubsub/http/auth.py index 4360b744..11e4de82 100644 --- a/ably/pubsub/rest/auth.py +++ b/ably/pubsub/http/auth.py @@ -9,8 +9,8 @@ from ably.pubsub.types.options import Options if TYPE_CHECKING: + from ably.pubsub.http.http import AblyHttp from ably.pubsub.realtime.realtime import AblyRealtime - from ably.pubsub.rest.rest import AblyRest from ably.pubsub.types.capability import Capability from ably.pubsub.types.tokendetails import TokenDetails @@ -29,7 +29,7 @@ class Method: BASIC = "BASIC" TOKEN = "TOKEN" - def __init__(self, ably: AblyRest | AblyRealtime, options: Options): + def __init__(self, ably: AblyHttp | AblyRealtime, options: Options): self.__ably = ably self.__auth_options = options self.__clock = select_clock(options) @@ -409,7 +409,7 @@ async def token_request_from_auth_url(self, method: str, url: str, token_params, # Use clean URL for the request url = clean_url - from ably.pubsub.http.http import Response + from ably.pubsub.request.http import Response resp = await self.ably.http.request_external( method=method, url=url, headers=headers, params=params, body=body) response = Response(resp) diff --git a/ably/pubsub/rest/channel.py b/ably/pubsub/http/channel.py similarity index 98% rename from ably/pubsub/rest/channel.py rename to ably/pubsub/http/channel.py index 109c0ec7..531a17cd 100644 --- a/ably/pubsub/rest/channel.py +++ b/ably/pubsub/http/channel.py @@ -8,8 +8,8 @@ import msgpack -from ably.pubsub.http.paginatedresult import PaginatedResult, format_params -from ably.pubsub.rest.annotations import RestAnnotations +from ably.pubsub.http.annotations import HttpAnnotations +from ably.pubsub.request.paginatedresult import PaginatedResult, format_params from ably.pubsub.types.channeldetails import ChannelDetails from ably.pubsub.types.message import ( Message, @@ -31,7 +31,7 @@ class Channel: - __annotations: RestAnnotations + __annotations: HttpAnnotations def __init__(self, ably, name, options): self.__ably = ably @@ -40,7 +40,7 @@ def __init__(self, ably, name, options): self.__cipher = None self.options = options self.__presence = Presence(self) - self.__annotations = RestAnnotations(self) + self.__annotations = HttpAnnotations(self) @catch_all async def history(self, direction=None, limit: int = None, start=None, end=None): @@ -370,7 +370,7 @@ def presence(self): return self.__presence @property - def annotations(self) -> RestAnnotations: + def annotations(self) -> HttpAnnotations: return self.__annotations @options.setter diff --git a/ably/pubsub/http/http.py b/ably/pubsub/http/http.py index 9677504e..3ba41fa7 100644 --- a/ably/pubsub/http/http.py +++ b/ably/pubsub/http/http.py @@ -1,328 +1,153 @@ -import functools -import json import logging -from urllib.parse import urljoin - -import httpx -import msgpack - -from ably.pubsub.http.httputils import HttpUtils -from ably.pubsub.rest.auth import Auth -from ably.pubsub.transport.defaults import Defaults -from ably.pubsub.util.clock import select_clock -from ably.pubsub.util.exceptions import AblyException -from ably.pubsub.util.helper import extract_url_params, is_token_error +from typing import Optional +from urllib.parse import urlencode + +from ably.pubsub.http.auth import Auth +from ably.pubsub.http.channel import Channels +from ably.pubsub.http.push import Push +from ably.pubsub.request.http import Http +from ably.pubsub.request.paginatedresult import HttpPaginatedResponse, PaginatedResult, format_params +from ably.pubsub.types.options import Options +from ably.pubsub.types.stats import stats_response_processor +from ably.pubsub.types.tokendetails import TokenDetails +from ably.pubsub.util.construction import reject_direct_construction +from ably.pubsub.util.exceptions import AblyException, catch_all log = logging.getLogger(__name__) -def reauth_if_expired(func): - @functools.wraps(func) - async def wrapper(rest, *args, **kwargs): - if kwargs.get("skip_auth"): - return await func(rest, *args, **kwargs) - - # RSA4b1 Detect expired token to avoid round-trip request - auth = rest.auth - token_details = auth.token_details - if token_details and auth.time_offset is not None and auth.token_details_has_expired(): - await auth.authorize() - retried = True +class AblyHttp: + """Ably HTTP Client""" + + def __init__(self, key: Optional[str] = None, token: Optional[str] = None, + token_details: Optional[TokenDetails] = None, **kwargs): + """Create an AblyHttp instance. + + :Parameters: + **Credentials** + - `key`: a valid key string + + **Or** + - `token`: a valid token string + - `token_details`: an instance of TokenDetails class + + **Optional Parameters** + - `client_id`: Undocumented + - `endpoint`: Endpoint specifies either a routing policy name or + fully qualified domain name to connect to Ably. + - `rest_host`: Deprecated: this property is deprecated and will + be removed in a future version. The host to connect to. + Defaults to rest.ably.io + - `environment`: Deprecated: this property is deprecated and + will be removed in a future version. The environment to use. + Defaults to 'production' + - `port`: The port to connect to. Defaults to 80 + - `tls_port`: The tls_port to connect to. Defaults to 443 + - `tls`: Specifies whether the client should use TLS. Defaults + to True + - `auth_token`: Undocumented + - `auth_callback`: Undocumented + - `auth_url`: Undocumented + """ + reject_direct_construction(type(self), 'ably.pubsub.server.create_http_client') + + if key is not None and ('key_name' in kwargs or 'key_secret' in kwargs): + raise ValueError("key and key_name or key_secret are mutually exclusive. " + "Provider either a key or key_name & key_secret") + if key is not None: + options = Options(key=key, **kwargs) + elif token is not None: + options = Options(auth_token=token, **kwargs) + elif token_details is not None: + if not isinstance(token_details, TokenDetails): + raise ValueError("token_details must be an instance of TokenDetails") + options = Options(token_details=token_details, **kwargs) + elif not ('auth_callback' in kwargs or 'auth_url' in kwargs or + # and don't have both key_name and key_secret + ('key_name' in kwargs and 'key_secret' in kwargs)): + raise ValueError("key is missing. Either an API key, token, or token auth method must be provided") else: - retried = False - - try: - return await func(rest, *args, **kwargs) - except AblyException as e: - if is_token_error(e) and not retried: - await auth.authorize() - return await func(rest, *args, **kwargs) + options = Options(**kwargs) - raise e + if not hasattr(self, '_is_realtime'): + self._is_realtime = False - return wrapper + self.__http = Http(self, options) + self.__auth = Auth(self, options) + self.__http.auth = self.__auth + self.__channels = Channels(self) + self.__options = options + self.__push = Push(self) -class Request: - def __init__(self, method='GET', url='/', version=None, headers=None, body=None, - skip_auth=False, raise_on_error=True): - self.__method = method - self.__headers = headers or {} - self.__body = body - self.__skip_auth = skip_auth - self.__url = url - self.__version = version - self.raise_on_error = raise_on_error - - def with_relative_url(self, relative_url): - url = urljoin(self.url, relative_url) - return Request(self.method, url, self.version, self.headers, self.body, - self.skip_auth, self.raise_on_error) + async def __aenter__(self): + return self - @property - def method(self): - return self.__method + @catch_all + async def stats(self, direction: Optional[str] = None, start=None, end=None, params: Optional[dict] = None, + limit: Optional[int] = None, paginated=None, unit=None, timeout=None): + """Returns the stats for this application""" + formatted_params = format_params(params, direction=direction, start=start, end=end, limit=limit, unit=unit) + url = '/stats' + formatted_params + return await PaginatedResult.paginated_query( + self.http, url=url, response_processor=stats_response_processor) - @property - def url(self): - return self.__url + @catch_all + async def time(self, timeout: Optional[float] = None) -> float: + """Returns the current server time in ms since the unix epoch""" + r = await self.http.get('/time', skip_auth=True, timeout=timeout) + AblyException.raise_for_response(r) + return r.to_native()[0] @property - def headers(self): - return self.__headers + def client_id(self) -> Optional[str]: + return self.options.client_id @property - def body(self): - return self.__body - - @property - def skip_auth(self): - return self.__skip_auth - - @property - def version(self): - return self.__version - - -class Response: - """ - Composition for httpx.Response with delegation - """ - - def __init__(self, response): - self.__response = response - - def to_native(self): - content = self.__response.content - if not content: - return None - - content_type = self.__response.headers.get('content-type') - if isinstance(content_type, str): - if content_type.startswith('application/x-msgpack'): - return msgpack.unpackb(content) - elif content_type.startswith('application/json'): - return self.__response.json() - - raise AblyException("Unsupported content type", 400, 40013) - - @property - def response(self): - return self.__response - - def __getattr__(self, attr): - return getattr(self.__response, attr) - - -class Http: - CONNECTION_RETRY_DEFAULTS = { - 'http_open_timeout': 4, - 'http_request_timeout': 10, - 'http_max_retry_count': 3, - 'http_max_retry_duration': 15, - } - - def __init__(self, ably, options): - options = options or {} - self.__ably = ably - self.__options = options - self.__auth = None - # Cached fallback host (RSC15f) - self.__host = None - self.__host_expires = None - self.__clock = select_clock(options) - self.__client = self.__create_client(options) - - @staticmethod - def __create_client(options): - test_options = getattr(options, '_test_options', None) - if test_options is not None and test_options.http_transport is not None: - return httpx.AsyncClient(transport=test_options.http_transport) - return httpx.AsyncClient(http2=True) - - async def close(self): - await self.__client.aclose() - - def dump_body(self, body): - if self.options.use_binary_protocol: - return msgpack.packb(body, use_bin_type=False) - else: - return json.dumps(body, separators=(',', ':')) - - def get_hosts(self): - hosts = self.options.get_hosts() - host = self.__host or self.options.fallback_host - if host is None: - return hosts - - # unstore saved fallback host after fallbackRetryTimeout (RSC15f) - if self.__host_expires is not None and self.__clock.now_ms() > self.__host_expires: - self.__host = None - self.__host_expires = None - return hosts - - hosts = list(hosts) - hosts.remove(host) - hosts.insert(0, host) - return hosts - - @reauth_if_expired - async def make_request(self, method, path, version=None, headers=None, body=None, - skip_auth=False, timeout=None, raise_on_error=True): - - if body is not None and type(body) not in (bytes, str): - body = self.dump_body(body) - - if body: - all_headers = HttpUtils.default_post_headers(self.options.use_binary_protocol, version=version) - else: - all_headers = HttpUtils.default_get_headers(self.options.use_binary_protocol, version=version) - - params = HttpUtils.get_query_params(self.options) - - if not skip_auth: - if self.auth.auth_mechanism == Auth.Method.BASIC and self.preferred_scheme.lower() == 'http': - raise AblyException( - "Cannot use Basic Auth over non-TLS connections", - 401, - 40103) - auth_headers = await self.auth._get_auth_headers() - all_headers.update(auth_headers) - if headers: - all_headers.update(headers) - - timeout = (self.http_open_timeout, self.http_request_timeout) - http_max_retry_duration = self.http_max_retry_duration - requested_at = self.__clock.now_ms() - - hosts = self.get_hosts() - for retry_count, host in enumerate(hosts): - def should_stop_retrying(retry_count=retry_count): - time_passed = (self.__clock.now_ms() - requested_at) / 1000 - # if it's the last try or cumulative timeout is done, we stop retrying - return retry_count == len(hosts) - 1 or time_passed > http_max_retry_duration - - base_url = f"{self.preferred_scheme}://{HttpUtils.host_for_url(host)}:{self.preferred_port}" - url = urljoin(base_url, path) - - (clean_url, url_params) = extract_url_params(url) - - request = self.__client.build_request( - method=method, - url=clean_url, - content=body, - params=dict(url_params, **params), - headers=all_headers, - timeout=timeout, - ) - try: - response = await self.__client.send(request) - except Exception as e: - if should_stop_retrying(): - raise e - else: - # RSC15l4 - cloud_front_error = (response.headers.get('Server', '').lower() == 'cloudfront' - and response.status_code >= 400) - # RSC15l3 - retryable_server_error = response.status_code >= 500 and response.status_code <= 504 - # Resending requests that have failed for other failure conditions will not fix the problem - # and will simply increase the load on other datacenters unnecessarily - should_fallback = cloud_front_error or retryable_server_error - - try: - if raise_on_error: - AblyException.raise_for_response(response) - - if should_fallback and not should_stop_retrying(): - continue - - # Keep fallback host for later (RSC15f) - if retry_count > 0 and host != self.options.get_host(): - self.__host = host - self.__host_expires = self.__clock.now_ms() + self.options.fallback_retry_timeout - - return Response(response) - except AblyException as e: - if should_stop_retrying() or not should_fallback: - raise e - - async def request_external(self, method, url, headers=None, params=None, body=None): - """Performs a request to a URL outside the Ably endpoint. - - Host fallback, authentication and the default headers do not apply. - """ - return await self.__client.request(method=method, url=url, headers=headers, params=params, data=body) - - async def delete(self, url, headers=None, skip_auth=False, timeout=None): - result = await self.make_request('DELETE', url, headers=headers, - skip_auth=skip_auth, timeout=timeout) - return result - - async def get(self, url, headers=None, skip_auth=False, timeout=None): - result = await self.make_request('GET', url, headers=headers, - skip_auth=skip_auth, timeout=timeout) - return result - - async def patch(self, url, headers=None, body=None, skip_auth=False, timeout=None): - result = await self.make_request('PATCH', url, headers=headers, body=body, - skip_auth=skip_auth, timeout=timeout) - return result - - async def post(self, url, headers=None, body=None, skip_auth=False, timeout=None): - result = await self.make_request('POST', url, headers=headers, body=body, - skip_auth=skip_auth, timeout=timeout) - return result - - async def put(self, url, headers=None, body=None, skip_auth=False, timeout=None): - result = await self.make_request('PUT', url, headers=headers, body=body, - skip_auth=skip_auth, timeout=timeout) - return result + def channels(self): + """Returns the channels container object""" + return self.__channels @property def auth(self): return self.__auth - @auth.setter - def auth(self, value): - self.__auth = value + @property + def http(self): + return self.__http @property def options(self): return self.__options @property - def preferred_host(self): - return self.options.get_host() + def push(self): + return self.__push - @property - def preferred_port(self): - return Defaults.get_port(self.options) + async def request(self, method: str, path: str, version: str, params: + Optional[dict] = None, body=None, headers=None): + if version is None: + raise AblyException("No version parameter", 400, 40000) - @property - def preferred_scheme(self): - return Defaults.get_scheme(self.options) + url = path + if params: + url += '?' + urlencode(params) - @property - def http_open_timeout(self): - if self.options.http_open_timeout is not None: - return self.options.http_open_timeout - return self.CONNECTION_RETRY_DEFAULTS['http_open_timeout'] + def response_processor(response): + items = response.to_native() + if not items: + return [] + if type(items) is not list: + items = [items] + return items - @property - def http_request_timeout(self): - if self.options.http_request_timeout is not None: - return self.options.http_request_timeout - return self.CONNECTION_RETRY_DEFAULTS['http_request_timeout'] + return await HttpPaginatedResponse.paginated_query( + self.http, method, url, version=version, body=body, headers=headers, + response_processor=response_processor, + raise_on_error=False) - @property - def http_max_retry_count(self): - if self.options.http_max_retry_count is not None: - return self.options.http_max_retry_count - return self.CONNECTION_RETRY_DEFAULTS['http_max_retry_count'] + async def __aexit__(self, *excinfo): + await self.close() - @property - def http_max_retry_duration(self): - if self.options.http_max_retry_duration is not None: - return self.options.http_max_retry_duration - return self.CONNECTION_RETRY_DEFAULTS['http_max_retry_duration'] + async def close(self): + await self.http.close() diff --git a/ably/pubsub/rest/push.py b/ably/pubsub/http/push.py similarity index 98% rename from ably/pubsub/rest/push.py rename to ably/pubsub/http/push.py index 264fea64..afcea029 100644 --- a/ably/pubsub/rest/push.py +++ b/ably/pubsub/http/push.py @@ -1,6 +1,6 @@ from typing import Optional -from ably.pubsub.http.paginatedresult import PaginatedResult, format_params +from ably.pubsub.request.paginatedresult import PaginatedResult, format_params from ably.pubsub.types.channelsubscription import ( PushChannelSubscription, channel_subscriptions_response_processor, diff --git a/ably/pubsub/prototypes.py b/ably/pubsub/prototypes.py index 2ad63960..c3a3d3e6 100644 --- a/ably/pubsub/prototypes.py +++ b/ably/pubsub/prototypes.py @@ -15,17 +15,17 @@ from typing import TYPE_CHECKING, Protocol if TYPE_CHECKING: - from ably.pubsub.http.http import Http - from ably.pubsub.http.paginatedresult import HttpPaginatedResponse, PaginatedResult + from ably.pubsub.http.auth import Auth + from ably.pubsub.http.channel import Channels as HttpChannels + from ably.pubsub.http.push import Push from ably.pubsub.realtime.channel import Channels as RealtimeChannels from ably.pubsub.realtime.connection import Connection - from ably.pubsub.rest.auth import Auth - from ably.pubsub.rest.channel import Channels as RestChannels - from ably.pubsub.rest.push import Push + from ably.pubsub.request.http import Http + from ably.pubsub.request.paginatedresult import HttpPaginatedResponse, PaginatedResult from ably.pubsub.types.options import Options -class RestClient(Protocol): +class PubSubHttpClient(Protocol): """A Pub/Sub client that operates entirely over HTTP. Build one with ``create_http_client``, from :mod:`ably.pubsub.server` for the @@ -38,7 +38,7 @@ def auth(self) -> Auth: ... @property - def channels(self) -> RestChannels: + def channels(self) -> HttpChannels: """The channels container object.""" ... @@ -80,17 +80,17 @@ async def close(self) -> None: """Release the resources this client holds.""" ... - async def __aenter__(self) -> RestClient: + async def __aenter__(self) -> PubSubHttpClient: ... async def __aexit__(self, *excinfo) -> None: ... -class RealtimeClient(RestClient, Protocol): +class PubSubRealtimeClient(PubSubHttpClient, Protocol): """A Pub/Sub client with a persistent realtime connection. - Everything :class:`RestClient` does, plus subscribing to channels and + Everything :class:`PubSubHttpClient` does, plus subscribing to channels and entering presence. Build one with :func:`ably.pubsub.server.create_realtime_client`. """ @@ -116,5 +116,5 @@ async def close(self) -> None: """Close the realtime connection and release the client's resources.""" ... - async def __aenter__(self) -> RealtimeClient: + async def __aenter__(self) -> PubSubRealtimeClient: ... diff --git a/ably/pubsub/realtime/annotations.py b/ably/pubsub/realtime/annotations.py index 01acfccc..cf83c609 100644 --- a/ably/pubsub/realtime/annotations.py +++ b/ably/pubsub/realtime/annotations.py @@ -3,7 +3,7 @@ import logging from typing import TYPE_CHECKING -from ably.pubsub.rest.annotations import RestAnnotations, construct_validate_annotation +from ably.pubsub.http.annotations import HttpAnnotations, construct_validate_annotation from ably.pubsub.transport.websockettransport import ProtocolMessageAction from ably.pubsub.types.annotation import Annotation, AnnotationAction from ably.pubsub.types.channelmode import ChannelMode @@ -37,7 +37,7 @@ def __init__(self, channel: RealtimeChannel, connection_manager: ConnectionManag self.__channel = channel self.__connection_manager = connection_manager self.__subscriptions = EventEmitter() - self.__rest_annotations = RestAnnotations(channel) + self.__http_annotations = HttpAnnotations(channel) async def __send_annotation(self, annotation: Annotation, params: dict | None = None): """ @@ -264,4 +264,4 @@ async def get(self, msg_or_serial, params: dict | None = None): AblyException: If the request fails or serial is invalid """ # Delegate to REST implementation - return await self.__rest_annotations.get(msg_or_serial, params) + return await self.__http_annotations.get(msg_or_serial, params) diff --git a/ably/pubsub/realtime/channel.py b/ably/pubsub/realtime/channel.py index cf57042e..796f9d01 100644 --- a/ably/pubsub/realtime/channel.py +++ b/ably/pubsub/realtime/channel.py @@ -4,11 +4,11 @@ import logging from typing import TYPE_CHECKING +from ably.pubsub.http.channel import Channel +from ably.pubsub.http.channel import Channels as HttpChannels from ably.pubsub.realtime.annotations import RealtimeAnnotations from ably.pubsub.realtime.connection import ConnectionState from ably.pubsub.realtime.presence import RealtimePresence -from ably.pubsub.rest.channel import Channel -from ably.pubsub.rest.channel import Channels as RestChannels from ably.pubsub.transport.websockettransport import ProtocolMessageAction from ably.pubsub.types.annotation import Annotation from ably.pubsub.types.channelmode import ChannelMode, decode_channel_mode, encode_channel_mode @@ -409,7 +409,7 @@ async def publish(self, *args, **kwargs) -> PublishResult: 400, 40012) - # Encode messages (RTL6a: same encoding as RestChannel#publish) + # Encode messages (RTL6a: same encoding as HttpChannel#publish) encoded_messages = [] for m in messages: # Encode the message with encryption if needed @@ -962,7 +962,7 @@ def _encode_flags(self) -> int | None: return flags -class Channels(RestChannels): +class Channels(HttpChannels): """Creates and destroys RealtimeChannel objects. Methods diff --git a/ably/pubsub/realtime/realtime.py b/ably/pubsub/realtime/realtime.py index 24c11d91..e2c12c5a 100644 --- a/ably/pubsub/realtime/realtime.py +++ b/ably/pubsub/realtime/realtime.py @@ -2,15 +2,15 @@ import logging from typing import Optional +from ably.pubsub.http.http import AblyHttp from ably.pubsub.realtime.channel import Channels from ably.pubsub.realtime.connection import Connection, ConnectionState -from ably.pubsub.rest.rest import AblyRest from ably.pubsub.util.construction import reject_direct_construction log = logging.getLogger(__name__) -class AblyRealtime(AblyRest): +class AblyRealtime(AblyHttp): """ Ably Realtime Client @@ -36,7 +36,7 @@ class AblyRealtime(AblyRest): """ def __init__(self, key: Optional[str] = None, loop: Optional[asyncio.AbstractEventLoop] = None, **kwargs): - """Constructs a RealtimeClient object using an Ably API key. + """Constructs a PubSubRealtimeClient object using an Ably API key. Parameters ---------- diff --git a/ably/pubsub/rest/__init__.py b/ably/pubsub/request/__init__.py similarity index 100% rename from ably/pubsub/rest/__init__.py rename to ably/pubsub/request/__init__.py diff --git a/ably/pubsub/request/http.py b/ably/pubsub/request/http.py new file mode 100644 index 00000000..c664f1ab --- /dev/null +++ b/ably/pubsub/request/http.py @@ -0,0 +1,328 @@ +import functools +import json +import logging +from urllib.parse import urljoin + +import httpx +import msgpack + +from ably.pubsub.http.auth import Auth +from ably.pubsub.request.httputils import HttpUtils +from ably.pubsub.transport.defaults import Defaults +from ably.pubsub.util.clock import select_clock +from ably.pubsub.util.exceptions import AblyException +from ably.pubsub.util.helper import extract_url_params, is_token_error + +log = logging.getLogger(__name__) + + +def reauth_if_expired(func): + @functools.wraps(func) + async def wrapper(rest, *args, **kwargs): + if kwargs.get("skip_auth"): + return await func(rest, *args, **kwargs) + + # RSA4b1 Detect expired token to avoid round-trip request + auth = rest.auth + token_details = auth.token_details + if token_details and auth.time_offset is not None and auth.token_details_has_expired(): + await auth.authorize() + retried = True + else: + retried = False + + try: + return await func(rest, *args, **kwargs) + except AblyException as e: + if is_token_error(e) and not retried: + await auth.authorize() + return await func(rest, *args, **kwargs) + + raise e + + return wrapper + + +class Request: + def __init__(self, method='GET', url='/', version=None, headers=None, body=None, + skip_auth=False, raise_on_error=True): + self.__method = method + self.__headers = headers or {} + self.__body = body + self.__skip_auth = skip_auth + self.__url = url + self.__version = version + self.raise_on_error = raise_on_error + + def with_relative_url(self, relative_url): + url = urljoin(self.url, relative_url) + return Request(self.method, url, self.version, self.headers, self.body, + self.skip_auth, self.raise_on_error) + + @property + def method(self): + return self.__method + + @property + def url(self): + return self.__url + + @property + def headers(self): + return self.__headers + + @property + def body(self): + return self.__body + + @property + def skip_auth(self): + return self.__skip_auth + + @property + def version(self): + return self.__version + + +class Response: + """ + Composition for httpx.Response with delegation + """ + + def __init__(self, response): + self.__response = response + + def to_native(self): + content = self.__response.content + if not content: + return None + + content_type = self.__response.headers.get('content-type') + if isinstance(content_type, str): + if content_type.startswith('application/x-msgpack'): + return msgpack.unpackb(content) + elif content_type.startswith('application/json'): + return self.__response.json() + + raise AblyException("Unsupported content type", 400, 40013) + + @property + def response(self): + return self.__response + + def __getattr__(self, attr): + return getattr(self.__response, attr) + + +class Http: + CONNECTION_RETRY_DEFAULTS = { + 'http_open_timeout': 4, + 'http_request_timeout': 10, + 'http_max_retry_count': 3, + 'http_max_retry_duration': 15, + } + + def __init__(self, ably, options): + options = options or {} + self.__ably = ably + self.__options = options + self.__auth = None + # Cached fallback host (RSC15f) + self.__host = None + self.__host_expires = None + self.__clock = select_clock(options) + self.__client = self.__create_client(options) + + @staticmethod + def __create_client(options): + test_options = getattr(options, '_test_options', None) + if test_options is not None and test_options.http_transport is not None: + return httpx.AsyncClient(transport=test_options.http_transport) + return httpx.AsyncClient(http2=True) + + async def close(self): + await self.__client.aclose() + + def dump_body(self, body): + if self.options.use_binary_protocol: + return msgpack.packb(body, use_bin_type=False) + else: + return json.dumps(body, separators=(',', ':')) + + def get_hosts(self): + hosts = self.options.get_hosts() + host = self.__host or self.options.fallback_host + if host is None: + return hosts + + # unstore saved fallback host after fallbackRetryTimeout (RSC15f) + if self.__host_expires is not None and self.__clock.now_ms() > self.__host_expires: + self.__host = None + self.__host_expires = None + return hosts + + hosts = list(hosts) + hosts.remove(host) + hosts.insert(0, host) + return hosts + + @reauth_if_expired + async def make_request(self, method, path, version=None, headers=None, body=None, + skip_auth=False, timeout=None, raise_on_error=True): + + if body is not None and type(body) not in (bytes, str): + body = self.dump_body(body) + + if body: + all_headers = HttpUtils.default_post_headers(self.options.use_binary_protocol, version=version) + else: + all_headers = HttpUtils.default_get_headers(self.options.use_binary_protocol, version=version) + + params = HttpUtils.get_query_params(self.options) + + if not skip_auth: + if self.auth.auth_mechanism == Auth.Method.BASIC and self.preferred_scheme.lower() == 'http': + raise AblyException( + "Cannot use Basic Auth over non-TLS connections", + 401, + 40103) + auth_headers = await self.auth._get_auth_headers() + all_headers.update(auth_headers) + if headers: + all_headers.update(headers) + + timeout = (self.http_open_timeout, self.http_request_timeout) + http_max_retry_duration = self.http_max_retry_duration + requested_at = self.__clock.now_ms() + + hosts = self.get_hosts() + for retry_count, host in enumerate(hosts): + def should_stop_retrying(retry_count=retry_count): + time_passed = (self.__clock.now_ms() - requested_at) / 1000 + # if it's the last try or cumulative timeout is done, we stop retrying + return retry_count == len(hosts) - 1 or time_passed > http_max_retry_duration + + base_url = f"{self.preferred_scheme}://{HttpUtils.host_for_url(host)}:{self.preferred_port}" + url = urljoin(base_url, path) + + (clean_url, url_params) = extract_url_params(url) + + request = self.__client.build_request( + method=method, + url=clean_url, + content=body, + params=dict(url_params, **params), + headers=all_headers, + timeout=timeout, + ) + try: + response = await self.__client.send(request) + except Exception as e: + if should_stop_retrying(): + raise e + else: + # RSC15l4 + cloud_front_error = (response.headers.get('Server', '').lower() == 'cloudfront' + and response.status_code >= 400) + # RSC15l3 + retryable_server_error = response.status_code >= 500 and response.status_code <= 504 + # Resending requests that have failed for other failure conditions will not fix the problem + # and will simply increase the load on other datacenters unnecessarily + should_fallback = cloud_front_error or retryable_server_error + + try: + if raise_on_error: + AblyException.raise_for_response(response) + + if should_fallback and not should_stop_retrying(): + continue + + # Keep fallback host for later (RSC15f) + if retry_count > 0 and host != self.options.get_host(): + self.__host = host + self.__host_expires = self.__clock.now_ms() + self.options.fallback_retry_timeout + + return Response(response) + except AblyException as e: + if should_stop_retrying() or not should_fallback: + raise e + + async def request_external(self, method, url, headers=None, params=None, body=None): + """Performs a request to a URL outside the Ably endpoint. + + Host fallback, authentication and the default headers do not apply. + """ + return await self.__client.request(method=method, url=url, headers=headers, params=params, data=body) + + async def delete(self, url, headers=None, skip_auth=False, timeout=None): + result = await self.make_request('DELETE', url, headers=headers, + skip_auth=skip_auth, timeout=timeout) + return result + + async def get(self, url, headers=None, skip_auth=False, timeout=None): + result = await self.make_request('GET', url, headers=headers, + skip_auth=skip_auth, timeout=timeout) + return result + + async def patch(self, url, headers=None, body=None, skip_auth=False, timeout=None): + result = await self.make_request('PATCH', url, headers=headers, body=body, + skip_auth=skip_auth, timeout=timeout) + return result + + async def post(self, url, headers=None, body=None, skip_auth=False, timeout=None): + result = await self.make_request('POST', url, headers=headers, body=body, + skip_auth=skip_auth, timeout=timeout) + return result + + async def put(self, url, headers=None, body=None, skip_auth=False, timeout=None): + result = await self.make_request('PUT', url, headers=headers, body=body, + skip_auth=skip_auth, timeout=timeout) + return result + + @property + def auth(self): + return self.__auth + + @auth.setter + def auth(self, value): + self.__auth = value + + @property + def options(self): + return self.__options + + @property + def preferred_host(self): + return self.options.get_host() + + @property + def preferred_port(self): + return Defaults.get_port(self.options) + + @property + def preferred_scheme(self): + return Defaults.get_scheme(self.options) + + @property + def http_open_timeout(self): + if self.options.http_open_timeout is not None: + return self.options.http_open_timeout + return self.CONNECTION_RETRY_DEFAULTS['http_open_timeout'] + + @property + def http_request_timeout(self): + if self.options.http_request_timeout is not None: + return self.options.http_request_timeout + return self.CONNECTION_RETRY_DEFAULTS['http_request_timeout'] + + @property + def http_max_retry_count(self): + if self.options.http_max_retry_count is not None: + return self.options.http_max_retry_count + return self.CONNECTION_RETRY_DEFAULTS['http_max_retry_count'] + + @property + def http_max_retry_duration(self): + if self.options.http_max_retry_duration is not None: + return self.options.http_max_retry_duration + return self.CONNECTION_RETRY_DEFAULTS['http_max_retry_duration'] diff --git a/ably/pubsub/http/httputils.py b/ably/pubsub/request/httputils.py similarity index 100% rename from ably/pubsub/http/httputils.py rename to ably/pubsub/request/httputils.py diff --git a/ably/pubsub/http/paginatedresult.py b/ably/pubsub/request/paginatedresult.py similarity index 98% rename from ably/pubsub/http/paginatedresult.py rename to ably/pubsub/request/paginatedresult.py index 17df8b27..d3361ee1 100644 --- a/ably/pubsub/http/paginatedresult.py +++ b/ably/pubsub/request/paginatedresult.py @@ -2,7 +2,7 @@ import logging from urllib.parse import urlencode -from ably.pubsub.http.http import Request +from ably.pubsub.request.http import Request from ably.pubsub.util import case log = logging.getLogger(__name__) diff --git a/ably/pubsub/rest/rest.py b/ably/pubsub/rest/rest.py deleted file mode 100644 index d746828a..00000000 --- a/ably/pubsub/rest/rest.py +++ /dev/null @@ -1,153 +0,0 @@ -import logging -from typing import Optional -from urllib.parse import urlencode - -from ably.pubsub.http.http import Http -from ably.pubsub.http.paginatedresult import HttpPaginatedResponse, PaginatedResult, format_params -from ably.pubsub.rest.auth import Auth -from ably.pubsub.rest.channel import Channels -from ably.pubsub.rest.push import Push -from ably.pubsub.types.options import Options -from ably.pubsub.types.stats import stats_response_processor -from ably.pubsub.types.tokendetails import TokenDetails -from ably.pubsub.util.construction import reject_direct_construction -from ably.pubsub.util.exceptions import AblyException, catch_all - -log = logging.getLogger(__name__) - - -class AblyRest: - """Ably Rest Client""" - - def __init__(self, key: Optional[str] = None, token: Optional[str] = None, - token_details: Optional[TokenDetails] = None, **kwargs): - """Create an AblyRest instance. - - :Parameters: - **Credentials** - - `key`: a valid key string - - **Or** - - `token`: a valid token string - - `token_details`: an instance of TokenDetails class - - **Optional Parameters** - - `client_id`: Undocumented - - `endpoint`: Endpoint specifies either a routing policy name or - fully qualified domain name to connect to Ably. - - `rest_host`: Deprecated: this property is deprecated and will - be removed in a future version. The host to connect to. - Defaults to rest.ably.io - - `environment`: Deprecated: this property is deprecated and - will be removed in a future version. The environment to use. - Defaults to 'production' - - `port`: The port to connect to. Defaults to 80 - - `tls_port`: The tls_port to connect to. Defaults to 443 - - `tls`: Specifies whether the client should use TLS. Defaults - to True - - `auth_token`: Undocumented - - `auth_callback`: Undocumented - - `auth_url`: Undocumented - """ - reject_direct_construction(type(self), 'ably.pubsub.server.create_http_client') - - if key is not None and ('key_name' in kwargs or 'key_secret' in kwargs): - raise ValueError("key and key_name or key_secret are mutually exclusive. " - "Provider either a key or key_name & key_secret") - if key is not None: - options = Options(key=key, **kwargs) - elif token is not None: - options = Options(auth_token=token, **kwargs) - elif token_details is not None: - if not isinstance(token_details, TokenDetails): - raise ValueError("token_details must be an instance of TokenDetails") - options = Options(token_details=token_details, **kwargs) - elif not ('auth_callback' in kwargs or 'auth_url' in kwargs or - # and don't have both key_name and key_secret - ('key_name' in kwargs and 'key_secret' in kwargs)): - raise ValueError("key is missing. Either an API key, token, or token auth method must be provided") - else: - options = Options(**kwargs) - - if not hasattr(self, '_is_realtime'): - self._is_realtime = False - - self.__http = Http(self, options) - self.__auth = Auth(self, options) - self.__http.auth = self.__auth - - self.__channels = Channels(self) - self.__options = options - self.__push = Push(self) - - async def __aenter__(self): - return self - - @catch_all - async def stats(self, direction: Optional[str] = None, start=None, end=None, params: Optional[dict] = None, - limit: Optional[int] = None, paginated=None, unit=None, timeout=None): - """Returns the stats for this application""" - formatted_params = format_params(params, direction=direction, start=start, end=end, limit=limit, unit=unit) - url = '/stats' + formatted_params - return await PaginatedResult.paginated_query( - self.http, url=url, response_processor=stats_response_processor) - - @catch_all - async def time(self, timeout: Optional[float] = None) -> float: - """Returns the current server time in ms since the unix epoch""" - r = await self.http.get('/time', skip_auth=True, timeout=timeout) - AblyException.raise_for_response(r) - return r.to_native()[0] - - @property - def client_id(self) -> Optional[str]: - return self.options.client_id - - @property - def channels(self): - """Returns the channels container object""" - return self.__channels - - @property - def auth(self): - return self.__auth - - @property - def http(self): - return self.__http - - @property - def options(self): - return self.__options - - @property - def push(self): - return self.__push - - async def request(self, method: str, path: str, version: str, params: - Optional[dict] = None, body=None, headers=None): - if version is None: - raise AblyException("No version parameter", 400, 40000) - - url = path - if params: - url += '?' + urlencode(params) - - def response_processor(response): - items = response.to_native() - if not items: - return [] - if type(items) is not list: - items = [items] - return items - - return await HttpPaginatedResponse.paginated_query( - self.http, method, url, version=version, body=body, headers=headers, - response_processor=response_processor, - raise_on_error=False) - - async def __aexit__(self, *excinfo): - await self.close() - - async def close(self): - await self.http.close() diff --git a/ably/pubsub/scripts/unasync.py b/ably/pubsub/scripts/unasync.py index 40e03267..877dedd6 100644 --- a/ably/pubsub/scripts/unasync.py +++ b/ably/pubsub/scripts/unasync.py @@ -5,7 +5,7 @@ import tokenize_rt rename_classes = [ - "AblyRest", + "AblyHttp", "Push", "PushAdmin", "Channel", @@ -282,15 +282,15 @@ def run(): _IMPORTS_REPLACE["test.ably"] = "test.ably.sync" _STRING_REPLACE['/../assets/testAppSpec.json'] = '/../../assets/testAppSpec.json' - _STRING_REPLACE['ably.pubsub.rest.auth.Auth.request_token'] = \ - 'ably.pubsub.sync.rest.auth.AuthSync.request_token' - _STRING_REPLACE['ably.pubsub.rest.auth.TokenRequest'] = 'ably.pubsub.sync.rest.auth.TokenRequest' - _STRING_REPLACE['ably.pubsub.rest.rest.Http.post'] = 'ably.pubsub.sync.rest.rest.HttpSync.post' + _STRING_REPLACE['ably.pubsub.http.auth.Auth.request_token'] = \ + 'ably.pubsub.sync.http.auth.AuthSync.request_token' + _STRING_REPLACE['ably.pubsub.http.auth.TokenRequest'] = 'ably.pubsub.sync.http.auth.TokenRequest' + _STRING_REPLACE['ably.pubsub.http.http.Http.post'] = 'ably.pubsub.sync.http.http.HttpSync.post' _STRING_REPLACE['httpx.AsyncClient.send'] = 'httpx.Client.send' _STRING_REPLACE['ably.pubsub.util.exceptions.AblyException.raise_for_response'] = \ 'ably.pubsub.sync.util.exceptions.AblyException.raise_for_response' - _STRING_REPLACE['ably.pubsub.rest.rest.AblyRest.time'] = 'ably.pubsub.sync.rest.rest.AblyRestSync.time' - _STRING_REPLACE['ably.pubsub.rest.auth.Auth._timestamp'] = 'ably.pubsub.sync.rest.auth.AuthSync._timestamp' + _STRING_REPLACE['ably.pubsub.http.http.AblyHttp.time'] = 'ably.pubsub.sync.http.http.AblyHttpSync.time' + _STRING_REPLACE['ably.pubsub.http.auth.Auth._timestamp'] = 'ably.pubsub.sync.http.auth.AuthSync._timestamp' # round 1 src_dir_path = os.path.join(os.getcwd(), "test", "ably") @@ -301,8 +301,8 @@ def run(): unasync_files(src_files, [Rule(fromdir=src_dir_path, todir=dest_dir_path)]) # round 2 - src_dir_path = os.path.join(os.getcwd(), "test", "ably", "rest") - dest_dir_path = os.path.join(os.getcwd(), "test", "ably", "sync", "rest") + src_dir_path = os.path.join(os.getcwd(), "test", "ably", "http") + dest_dir_path = os.path.join(os.getcwd(), "test", "ably", "sync", "http") src_files = find_files(src_dir_path, "*.py") unasync_files(src_files, [Rule(fromdir=src_dir_path, todir=dest_dir_path, output_file_prefix="sync_")]) diff --git a/ably/pubsub/server/__init__.py b/ably/pubsub/server/__init__.py index 8b4c0c99..8e7153d2 100644 --- a/ably/pubsub/server/__init__.py +++ b/ably/pubsub/server/__init__.py @@ -8,8 +8,8 @@ Build a client with :func:`create_http_client` or :func:`create_realtime_client`; the classes behind them are internal and reject direct construction. Annotate against the prototypes they return, -:class:`~ably.pubsub.prototypes.RestClient` and -:class:`~ably.pubsub.prototypes.RealtimeClient`. +:class:`~ably.pubsub.prototypes.PubSubHttpClient` and +:class:`~ably.pubsub.prototypes.PubSubRealtimeClient`. This is the public API of the ``ably-pubsub-server`` distribution. Both ``ably`` and ``ably.pubsub`` are PEP 420 namespace packages shared with the @@ -18,11 +18,11 @@ from here. """ -from ably.pubsub.prototypes import RealtimeClient, RestClient +from ably.pubsub.http.auth import Auth +from ably.pubsub.http.http import AblyHttp as _AblyHttp +from ably.pubsub.http.push import Push +from ably.pubsub.prototypes import PubSubHttpClient, PubSubRealtimeClient from ably.pubsub.realtime.realtime import AblyRealtime as _AblyRealtime -from ably.pubsub.rest.auth import Auth -from ably.pubsub.rest.push import Push -from ably.pubsub.rest.rest import AblyRest as _AblyRest from ably.pubsub.types.annotation import Annotation, AnnotationAction from ably.pubsub.types.capability import Capability from ably.pubsub.types.channelmode import ChannelMode @@ -40,7 +40,7 @@ from ably.pubsub.version import api_version, lib_version -def create_http_client(**kwargs) -> RestClient: +def create_http_client(**kwargs) -> PubSubHttpClient: """Create a server Pub/Sub client that operates entirely over HTTP. Handles publish, history, presence reads, stats and token issuing. Give the client one @@ -145,7 +145,7 @@ def create_http_client(**kwargs) -> RestClient: Returns ------- - RestClient + PubSubHttpClient A client ready to use. It opens no connection until its first request. Raises @@ -157,10 +157,10 @@ def create_http_client(**kwargs) -> RestClient: `environment`, `rest_host` or `realtime_host`. """ with factory_construction(): - return _AblyRest(**kwargs) + return _AblyHttp(**kwargs) -def create_realtime_client(**kwargs) -> RealtimeClient: +def create_realtime_client(**kwargs) -> PubSubRealtimeClient: """Create a server Pub/Sub client with a persistent realtime connection. Everything the HTTP client does, plus subscribing to channels and entering presence. @@ -222,7 +222,7 @@ def create_realtime_client(**kwargs) -> RealtimeClient: Returns ------- - RealtimeClient + PubSubRealtimeClient A client which, unless `auto_connect=False`, is already connecting. Raises @@ -254,11 +254,11 @@ def create_realtime_client(**kwargs) -> RealtimeClient: 'MessageOperation', 'MessageVersion', 'Options', + 'PubSubHttpClient', + 'PubSubRealtimeClient', 'PublishResult', 'Push', 'PushChannelSubscription', - 'RealtimeClient', - 'RestClient', 'TokenDetails', 'UpdateDeleteResult', 'VCDiffDecoder', diff --git a/ably/pubsub/server/sync.py b/ably/pubsub/server/sync.py index 131b5603..19e3637f 100644 --- a/ably/pubsub/server/sync.py +++ b/ably/pubsub/server/sync.py @@ -7,10 +7,10 @@ instead. """ -from ably.pubsub.sync.prototypes import RestClient -from ably.pubsub.sync.rest.auth import AuthSync -from ably.pubsub.sync.rest.push import PushSync -from ably.pubsub.sync.rest.rest import AblyRestSync as _AblyRestSync +from ably.pubsub.sync.http.auth import AuthSync +from ably.pubsub.sync.http.http import AblyHttpSync as _AblyHttpSync +from ably.pubsub.sync.http.push import PushSync +from ably.pubsub.sync.prototypes import PubSubHttpClient from ably.pubsub.sync.types.annotation import Annotation, AnnotationAction from ably.pubsub.sync.types.capability import Capability from ably.pubsub.sync.types.channelmode import ChannelMode @@ -28,7 +28,7 @@ from ably.pubsub.version import api_version, lib_version -def create_http_client(**kwargs) -> RestClient: +def create_http_client(**kwargs) -> PubSubHttpClient: """Create a synchronous server Pub/Sub client that operates entirely over HTTP. Accepts every option :func:`ably.pubsub.server.create_http_client` does — see there @@ -37,7 +37,7 @@ def create_http_client(**kwargs) -> RestClient: Returns ------- - RestClient + PubSubHttpClient A client ready to use. It opens no connection until its first request. Raises @@ -49,7 +49,7 @@ def create_http_client(**kwargs) -> RestClient: `environment`, `rest_host` or `realtime_host`. """ with factory_construction(): - return _AblyRestSync(**kwargs) + return _AblyHttpSync(**kwargs) __all__ = [ @@ -69,10 +69,10 @@ def create_http_client(**kwargs) -> RestClient: 'MessageOperation', 'MessageVersion', 'Options', + 'PubSubHttpClient', 'PublishResult', 'PushChannelSubscription', 'PushSync', - 'RestClient', 'TokenDetails', 'UpdateDeleteResult', 'VCDiffDecoder', diff --git a/ably/pubsub/transport/websockettransport.py b/ably/pubsub/transport/websockettransport.py index 2cbe92a7..7177732c 100644 --- a/ably/pubsub/transport/websockettransport.py +++ b/ably/pubsub/transport/websockettransport.py @@ -10,7 +10,7 @@ import msgpack -from ably.pubsub.http.httputils import HttpUtils +from ably.pubsub.request.httputils import HttpUtils from ably.pubsub.transport.defaults import Defaults from ably.pubsub.types.connectiondetails import ConnectionDetails from ably.pubsub.types.operations import PublishResult diff --git a/ably/pubsub/types/presence.py b/ably/pubsub/types/presence.py index 12e965e1..66cbdea7 100644 --- a/ably/pubsub/types/presence.py +++ b/ably/pubsub/types/presence.py @@ -1,7 +1,7 @@ from datetime import datetime, timedelta from urllib import parse -from ably.pubsub.http.paginatedresult import PaginatedResult +from ably.pubsub.request.paginatedresult import PaginatedResult from ably.pubsub.types.mixins import EncodeDataMixin from ably.pubsub.types.typedbuffer import TypedBuffer from ably.pubsub.util.crypto import CipherData diff --git a/test/ably/rest/authheaders_test.py b/test/ably/http/authheaders_test.py similarity index 95% rename from test/ably/rest/authheaders_test.py rename to test/ably/http/authheaders_test.py index 69f0ad1b..ad4ffaa9 100644 --- a/test/ably/rest/authheaders_test.py +++ b/test/ably/http/authheaders_test.py @@ -2,7 +2,7 @@ import respx from httpx import Response -from ably import AblyRest +from ably.pubsub.server import create_http_client @pytest.fixture(scope='session', autouse=True) @@ -22,7 +22,7 @@ async def test_auth_headers_are_only_sent_to_auth_url(method, binary, override): options = {'auth_url': auth_url, 'auth_method': method, 'use_binary_protocol': binary} if not override: options['auth_headers'] = headers - client = AblyRest(**options) + client = create_http_client(**options) auth_route = respx.request(method, auth_url).mock(return_value=Response(200, json={ 'keyName': 'app.key', 'nonce': 'nonce', 'timestamp': 123456789, 'mac': 'signed-request', 'capability': '{"*":["*"]}', diff --git a/test/ably/rest/encoders_test.py b/test/ably/http/encoders_test.py similarity index 93% rename from test/ably/rest/encoders_test.py rename to test/ably/http/encoders_test.py index f2964849..24862161 100644 --- a/test/ably/rest/encoders_test.py +++ b/test/ably/http/encoders_test.py @@ -26,7 +26,7 @@ async def setup(self): async def test_text_utf8(self): channel = self.ably.channels["persisted:publish"] - with mock.patch('ably.pubsub.rest.rest.Http.post', new_callable=AsyncMock) as post_mock: + with mock.patch('ably.pubsub.http.http.Http.post', new_callable=AsyncMock) as post_mock: await channel.publish('event', 'foó') _, kwargs = post_mock.call_args assert json.loads(kwargs['body'])['data'] == 'foó' @@ -36,7 +36,7 @@ async def test_str(self): # This test only makes sense for py2 channel = self.ably.channels["persisted:publish"] - with mock.patch('ably.pubsub.rest.rest.Http.post', new_callable=AsyncMock) as post_mock: + with mock.patch('ably.pubsub.http.http.Http.post', new_callable=AsyncMock) as post_mock: await channel.publish('event', 'foo') _, kwargs = post_mock.call_args assert json.loads(kwargs['body'])['data'] == 'foo' @@ -45,7 +45,7 @@ async def test_str(self): async def test_with_binary_type(self): channel = self.ably.channels["persisted:publish"] - with mock.patch('ably.pubsub.rest.rest.Http.post', new_callable=AsyncMock) as post_mock: + with mock.patch('ably.pubsub.http.http.Http.post', new_callable=AsyncMock) as post_mock: await channel.publish('event', bytearray(b'foo')) _, kwargs = post_mock.call_args raw_data = json.loads(kwargs['body'])['data'] @@ -55,7 +55,7 @@ async def test_with_binary_type(self): async def test_with_bytes_type(self): channel = self.ably.channels["persisted:publish"] - with mock.patch('ably.pubsub.rest.rest.Http.post', new_callable=AsyncMock) as post_mock: + with mock.patch('ably.pubsub.http.http.Http.post', new_callable=AsyncMock) as post_mock: await channel.publish('event', b'foo') _, kwargs = post_mock.call_args raw_data = json.loads(kwargs['body'])['data'] @@ -65,7 +65,7 @@ async def test_with_bytes_type(self): async def test_with_json_dict_data(self): channel = self.ably.channels["persisted:publish"] data = {'foó': 'bár'} - with mock.patch('ably.pubsub.rest.rest.Http.post', new_callable=AsyncMock) as post_mock: + with mock.patch('ably.pubsub.http.http.Http.post', new_callable=AsyncMock) as post_mock: await channel.publish('event', data) _, kwargs = post_mock.call_args raw_data = json.loads(json.loads(kwargs['body'])['data']) @@ -75,7 +75,7 @@ async def test_with_json_dict_data(self): async def test_with_json_list_data(self): channel = self.ably.channels["persisted:publish"] data = ['foó', 'bár'] - with mock.patch('ably.pubsub.rest.rest.Http.post', new_callable=AsyncMock) as post_mock: + with mock.patch('ably.pubsub.http.http.Http.post', new_callable=AsyncMock) as post_mock: await channel.publish('event', data) _, kwargs = post_mock.call_args raw_data = json.loads(json.loads(kwargs['body'])['data']) @@ -156,7 +156,7 @@ def decrypt(self, payload, options=None): async def test_text_utf8(self): channel = self.ably.channels.get("persisted:publish_enc", cipher=self.cipher_params) - with mock.patch('ably.pubsub.rest.rest.Http.post', new_callable=AsyncMock) as post_mock: + with mock.patch('ably.pubsub.http.http.Http.post', new_callable=AsyncMock) as post_mock: await channel.publish('event', 'fóo') _, kwargs = post_mock.call_args assert json.loads(kwargs['body'])['encoding'].strip('/') == 'utf-8/cipher+aes-128-cbc/base64' @@ -167,7 +167,7 @@ async def test_str(self): # This test only makes sense for py2 channel = self.ably.channels["persisted:publish"] - with mock.patch('ably.pubsub.rest.rest.Http.post', new_callable=AsyncMock) as post_mock: + with mock.patch('ably.pubsub.http.http.Http.post', new_callable=AsyncMock) as post_mock: await channel.publish('event', 'foo') _, kwargs = post_mock.call_args assert json.loads(kwargs['body'])['data'] == 'foo' @@ -177,7 +177,7 @@ async def test_with_binary_type(self): channel = self.ably.channels.get("persisted:publish_enc", cipher=self.cipher_params) - with mock.patch('ably.pubsub.rest.rest.Http.post', new_callable=AsyncMock) as post_mock: + with mock.patch('ably.pubsub.http.http.Http.post', new_callable=AsyncMock) as post_mock: await channel.publish('event', bytearray(b'foo')) _, kwargs = post_mock.call_args @@ -190,7 +190,7 @@ async def test_with_json_dict_data(self): channel = self.ably.channels.get("persisted:publish_enc", cipher=self.cipher_params) data = {'foó': 'bár'} - with mock.patch('ably.pubsub.rest.rest.Http.post', new_callable=AsyncMock) as post_mock: + with mock.patch('ably.pubsub.http.http.Http.post', new_callable=AsyncMock) as post_mock: await channel.publish('event', data) _, kwargs = post_mock.call_args assert json.loads(kwargs['body'])['encoding'].strip('/') == 'json/utf-8/cipher+aes-128-cbc/base64' @@ -201,7 +201,7 @@ async def test_with_json_list_data(self): channel = self.ably.channels.get("persisted:publish_enc", cipher=self.cipher_params) data = ['foó', 'bár'] - with mock.patch('ably.pubsub.rest.rest.Http.post', new_callable=AsyncMock) as post_mock: + with mock.patch('ably.pubsub.http.http.Http.post', new_callable=AsyncMock) as post_mock: await channel.publish('event', data) _, kwargs = post_mock.call_args assert json.loads(kwargs['body'])['encoding'].strip('/') == 'json/utf-8/cipher+aes-128-cbc/base64' @@ -264,7 +264,7 @@ def decode(self, data): async def test_text_utf8(self): channel = self.ably.channels["persisted:publish"] - with mock.patch('ably.pubsub.rest.rest.Http.post', + with mock.patch('ably.pubsub.http.http.Http.post', wraps=channel.ably.http.post) as post_mock: await channel.publish('event', 'foó') _, kwargs = post_mock.call_args @@ -274,7 +274,7 @@ async def test_text_utf8(self): async def test_with_binary_type(self): channel = self.ably.channels["persisted:publish"] - with mock.patch('ably.pubsub.rest.rest.Http.post', + with mock.patch('ably.pubsub.http.http.Http.post', wraps=channel.ably.http.post) as post_mock: await channel.publish('event', bytearray(b'foo')) _, kwargs = post_mock.call_args @@ -284,7 +284,7 @@ async def test_with_binary_type(self): async def test_with_json_dict_data(self): channel = self.ably.channels["persisted:publish"] data = {'foó': 'bár'} - with mock.patch('ably.pubsub.rest.rest.Http.post', + with mock.patch('ably.pubsub.http.http.Http.post', wraps=channel.ably.http.post) as post_mock: await channel.publish('event', data) _, kwargs = post_mock.call_args @@ -295,7 +295,7 @@ async def test_with_json_dict_data(self): async def test_with_json_list_data(self): channel = self.ably.channels["persisted:publish"] data = ['foó', 'bár'] - with mock.patch('ably.pubsub.rest.rest.Http.post', + with mock.patch('ably.pubsub.http.http.Http.post', wraps=channel.ably.http.post) as post_mock: await channel.publish('event', data) _, kwargs = post_mock.call_args @@ -362,7 +362,7 @@ def decode(self, data): async def test_text_utf8(self): channel = self.ably.channels.get("persisted:publish_enc", cipher=self.cipher_params) - with mock.patch('ably.pubsub.rest.rest.Http.post', + with mock.patch('ably.pubsub.http.http.Http.post', wraps=channel.ably.http.post) as post_mock: await channel.publish('event', 'fóo') _, kwargs = post_mock.call_args @@ -374,7 +374,7 @@ async def test_with_binary_type(self): channel = self.ably.channels.get("persisted:publish_enc", cipher=self.cipher_params) - with mock.patch('ably.pubsub.rest.rest.Http.post', + with mock.patch('ably.pubsub.http.http.Http.post', wraps=channel.ably.http.post) as post_mock: await channel.publish('event', bytearray(b'foo')) _, kwargs = post_mock.call_args @@ -388,7 +388,7 @@ async def test_with_json_dict_data(self): channel = self.ably.channels.get("persisted:publish_enc", cipher=self.cipher_params) data = {'foó': 'bár'} - with mock.patch('ably.pubsub.rest.rest.Http.post', + with mock.patch('ably.pubsub.http.http.Http.post', wraps=channel.ably.http.post) as post_mock: await channel.publish('event', data) _, kwargs = post_mock.call_args @@ -400,7 +400,7 @@ async def test_with_json_list_data(self): channel = self.ably.channels.get("persisted:publish_enc", cipher=self.cipher_params) data = ['foó', 'bár'] - with mock.patch('ably.pubsub.rest.rest.Http.post', + with mock.patch('ably.pubsub.http.http.Http.post', wraps=channel.ably.http.post) as post_mock: await channel.publish('event', data) _, kwargs = post_mock.call_args diff --git a/test/ably/rest/restannotations_test.py b/test/ably/http/httpannotations_test.py similarity index 99% rename from test/ably/rest/restannotations_test.py rename to test/ably/http/httpannotations_test.py index 7599f776..3a8ca54b 100644 --- a/test/ably/rest/restannotations_test.py +++ b/test/ably/http/httpannotations_test.py @@ -14,7 +14,7 @@ @pytest.mark.parametrize("transport", ["json", "msgpack"], ids=["JSON", "MsgPack"]) -class TestRestAnnotations(BaseAsyncTestCase): +class TestHttpAnnotations(BaseAsyncTestCase): @pytest.fixture(autouse=True) async def setup(self, transport): diff --git a/test/ably/rest/restauth_test.py b/test/ably/http/httpauth_test.py similarity index 98% rename from test/ably/rest/restauth_test.py rename to test/ably/http/httpauth_test.py index 5f2eacc6..3f793565 100644 --- a/test/ably/rest/restauth_test.py +++ b/test/ably/http/httpauth_test.py @@ -194,7 +194,7 @@ async def test_authorize_always_creates_new_token(self): async def test_authorize_create_new_token_if_expired(self): token = await self.ably.auth.authorize() - with mock.patch('ably.pubsub.rest.auth.Auth.token_details_has_expired', + with mock.patch('ably.pubsub.http.auth.Auth.token_details_has_expired', return_value=True): new_token = await self.ably.auth.authorize() @@ -208,7 +208,7 @@ async def test_authorize_returns_a_token_details(self): async def test_authorize_adheres_to_request_token(self): token_params = {'ttl': 10, 'client_id': 'client_id'} auth_params = {'auth_url': 'somewhere.com', 'query_time': True} - with mock.patch('ably.pubsub.rest.auth.Auth.request_token', new_callable=AsyncMock) as request_mock: + with mock.patch('ably.pubsub.http.auth.Auth.request_token', new_callable=AsyncMock) as request_mock: await self.ably.auth.authorize(token_params, auth_params) token_called, auth_called = request_mock.call_args @@ -247,7 +247,7 @@ async def test_if_parameters_are_stored_and_used_as_defaults(self): auth_options = dict(self.ably.auth.auth_options.auth_options) auth_options['auth_headers'] = {'a_headers': 'a_value'} await self.ably.auth.authorize({'ttl': 555}, auth_options) - with mock.patch('ably.pubsub.rest.auth.Auth.request_token', + with mock.patch('ably.pubsub.http.auth.Auth.request_token', wraps=self.ably.auth.request_token) as request_mock: await self.ably.auth.authorize() @@ -259,7 +259,7 @@ async def test_if_parameters_are_stored_and_used_as_defaults(self): auth_options = dict(self.ably.auth.auth_options.auth_options) auth_options['auth_headers'] = None await self.ably.auth.authorize({}, auth_options) - with mock.patch('ably.pubsub.rest.auth.Auth.request_token', + with mock.patch('ably.pubsub.http.auth.Auth.request_token', wraps=self.ably.auth.request_token) as request_mock: await self.ably.auth.authorize() @@ -279,7 +279,7 @@ async def test_timestamp_is_not_stored(self): # call authorize again with timestamp set timestamp = await self.ably.time() - with mock.patch('ably.pubsub.rest.auth.TokenRequest', + with mock.patch('ably.pubsub.http.auth.TokenRequest', wraps=ably.pubsub.types.tokenrequest.TokenRequest) as tr_mock: auth_options = dict(self.ably.auth.auth_options.auth_options) auth_options['auth_headers'] = {'a_headers': 'a_value'} @@ -291,7 +291,7 @@ async def test_timestamp_is_not_stored(self): assert tr_mock.call_args[1]['timestamp'] == timestamp # call authorize again with no params - with mock.patch('ably.pubsub.rest.auth.TokenRequest', + with mock.patch('ably.pubsub.http.auth.TokenRequest', wraps=ably.pubsub.types.tokenrequest.TokenRequest) as tr_mock: token_4 = await self.ably.auth.authorize() assert isinstance(token_4, TokenDetails) diff --git a/test/ably/rest/restcapability_test.py b/test/ably/http/httpcapability_test.py similarity index 99% rename from test/ably/rest/restcapability_test.py rename to test/ably/http/httpcapability_test.py index f83d1641..7e96a57d 100644 --- a/test/ably/rest/restcapability_test.py +++ b/test/ably/http/httpcapability_test.py @@ -6,7 +6,7 @@ from test.ably.utils import BaseAsyncTestCase, VaryByProtocolTestsMetaclass, dont_vary_protocol -class TestRestCapability(BaseAsyncTestCase, metaclass=VaryByProtocolTestsMetaclass): +class TestHttpCapability(BaseAsyncTestCase, metaclass=VaryByProtocolTestsMetaclass): @pytest.fixture(autouse=True) async def setup(self): diff --git a/test/ably/rest/restchannelhistory_test.py b/test/ably/http/httpchannelhistory_test.py similarity index 99% rename from test/ably/rest/restchannelhistory_test.py rename to test/ably/http/httpchannelhistory_test.py index c2fe5008..e5773172 100644 --- a/test/ably/rest/restchannelhistory_test.py +++ b/test/ably/http/httpchannelhistory_test.py @@ -3,7 +3,7 @@ import pytest import respx -from ably.pubsub.http.paginatedresult import PaginatedResult +from ably.pubsub.request.paginatedresult import PaginatedResult from ably.pubsub.server import AblyException from test.ably.testapp import TestApp from test.ably.utils import BaseAsyncTestCase, VaryByProtocolTestsMetaclass, dont_vary_protocol @@ -11,7 +11,7 @@ log = logging.getLogger(__name__) -class TestRestChannelHistory(BaseAsyncTestCase, metaclass=VaryByProtocolTestsMetaclass): +class TestHttpChannelHistory(BaseAsyncTestCase, metaclass=VaryByProtocolTestsMetaclass): @pytest.fixture(autouse=True) async def setup(self): diff --git a/test/ably/rest/restchannelmutablemessages_test.py b/test/ably/http/httpchannelmutablemessages_test.py similarity index 99% rename from test/ably/rest/restchannelmutablemessages_test.py rename to test/ably/http/httpchannelmutablemessages_test.py index cfcd8dae..e965011c 100644 --- a/test/ably/rest/restchannelmutablemessages_test.py +++ b/test/ably/http/httpchannelmutablemessages_test.py @@ -13,7 +13,7 @@ @pytest.mark.parametrize("transport", ["json", "msgpack"], ids=["JSON", "MsgPack"]) -class TestRestChannelMutableMessages(BaseAsyncTestCase): +class TestHttpChannelMutableMessages(BaseAsyncTestCase): @pytest.fixture(autouse=True) async def setup(self, transport): diff --git a/test/ably/rest/restchannelpublish_test.py b/test/ably/http/httpchannelpublish_test.py similarity index 98% rename from test/ably/rest/restchannelpublish_test.py rename to test/ably/http/httpchannelpublish_test.py index 2eacbf86..1edff7be 100644 --- a/test/ably/rest/restchannelpublish_test.py +++ b/test/ably/http/httpchannelpublish_test.py @@ -10,7 +10,7 @@ import msgpack import pytest -from ably.pubsub.rest.auth import Auth +from ably.pubsub.http.auth import Auth from ably.pubsub.server import AblyException, IncompatibleClientIdException, api_version from ably.pubsub.types.message import Message from ably.pubsub.types.tokendetails import TokenDetails @@ -24,7 +24,7 @@ # Ignore library warning regarding client_id @pytest.mark.filterwarnings('ignore::DeprecationWarning') -class TestRestChannelPublish(BaseAsyncTestCase, metaclass=VaryByProtocolTestsMetaclass): +class TestHttpChannelPublish(BaseAsyncTestCase, metaclass=VaryByProtocolTestsMetaclass): @pytest.fixture(autouse=True) async def setup(self): @@ -103,7 +103,7 @@ async def test_message_list_generate_one_request(self): expected_messages = [Message(f"name-{i}", str(i)) for i in range(3)] - with mock.patch('ably.pubsub.rest.rest.Http.post', + with mock.patch('ably.pubsub.http.http.Http.post', wraps=channel.ably.http.post) as post_mock: await channel.publish(messages=expected_messages) assert post_mock.call_count == 1 @@ -184,7 +184,7 @@ async def test_publish_message_null_name_and_data_keys_arent_sent(self): channel = self.ably.channels[ self.get_channel_name('persisted:null_name_and_data_keys_arent_sent_channel')] - with mock.patch('ably.pubsub.rest.rest.Http.post', + with mock.patch('ably.pubsub.http.http.Http.post', wraps=channel.ably.http.post) as post_mock: await channel.publish(name=None, data=None) @@ -244,7 +244,7 @@ async def test_publish_message_without_client_id_on_identified_client(self): channel = self.ably_with_client_id.channels[ self.get_channel_name('persisted:no_client_id_identified_client')] - with mock.patch('ably.pubsub.rest.rest.Http.post', + with mock.patch('ably.pubsub.http.http.Http.post', wraps=channel.ably.http.post) as post_mock: await channel.publish(name='publish', data='test') @@ -446,7 +446,7 @@ async def test_publish_params(self): assert 40099 == excinfo.value.code -class TestRestChannelPublishIdempotent(BaseAsyncTestCase, metaclass=VaryByProtocolTestsMetaclass): +class TestHttpChannelPublishIdempotent(BaseAsyncTestCase, metaclass=VaryByProtocolTestsMetaclass): @pytest.fixture(autouse=True) async def setup(self): diff --git a/test/ably/rest/restchannels_test.py b/test/ably/http/httpchannels_test.py similarity index 98% rename from test/ably/rest/restchannels_test.py rename to test/ably/http/httpchannels_test.py index 8172ec21..255b23e9 100644 --- a/test/ably/rest/restchannels_test.py +++ b/test/ably/http/httpchannels_test.py @@ -2,7 +2,7 @@ import pytest -from ably.pubsub.rest.channel import Channel, Channels, Presence +from ably.pubsub.http.channel import Channel, Channels, Presence from ably.pubsub.server import AblyException from ably.pubsub.util.crypto import generate_random_key from test.ably.testapp import TestApp diff --git a/test/ably/rest/restchannelstatus_test.py b/test/ably/http/httpchannelstatus_test.py similarity index 97% rename from test/ably/rest/restchannelstatus_test.py rename to test/ably/http/httpchannelstatus_test.py index cb455362..cd027037 100644 --- a/test/ably/rest/restchannelstatus_test.py +++ b/test/ably/http/httpchannelstatus_test.py @@ -8,7 +8,7 @@ log = logging.getLogger(__name__) -class TestRestChannelStatus(BaseAsyncTestCase, metaclass=VaryByProtocolTestsMetaclass): +class TestHttpChannelStatus(BaseAsyncTestCase, metaclass=VaryByProtocolTestsMetaclass): @pytest.fixture(autouse=True) async def setup(self): diff --git a/test/ably/rest/restcrypto_test.py b/test/ably/http/httpcrypto_test.py similarity index 99% rename from test/ably/rest/restcrypto_test.py rename to test/ably/http/httpcrypto_test.py index 1ef65efc..77b63250 100644 --- a/test/ably/rest/restcrypto_test.py +++ b/test/ably/http/httpcrypto_test.py @@ -16,7 +16,7 @@ log = logging.getLogger(__name__) -class TestRestCrypto(BaseAsyncTestCase, metaclass=VaryByProtocolTestsMetaclass): +class TestHttpCrypto(BaseAsyncTestCase, metaclass=VaryByProtocolTestsMetaclass): @pytest.fixture(autouse=True) async def setup(self): diff --git a/test/ably/rest/resthttp_test.py b/test/ably/http/httphttp_test.py similarity index 98% rename from test/ably/rest/resthttp_test.py rename to test/ably/http/httphttp_test.py index ca2122bb..e89fb1c8 100644 --- a/test/ably/rest/resthttp_test.py +++ b/test/ably/http/httphttp_test.py @@ -9,7 +9,7 @@ import respx from httpx import Response -from ably.pubsub.http.httputils import HttpUtils +from ably.pubsub.request.httputils import HttpUtils from ably.pubsub.server import create_http_client from ably.pubsub.transport.defaults import Defaults from ably.pubsub.types.options import Options @@ -18,7 +18,7 @@ from test.ably.utils import BaseAsyncTestCase -class TestRestHttp(BaseAsyncTestCase): +class TestHttpHttp(BaseAsyncTestCase): async def test_host_for_url_brackets_only_ipv6_literals(self): assert HttpUtils.host_for_url('::1') == '[::1]' assert HttpUtils.host_for_url('2001:db8::1') == '[2001:db8::1]' diff --git a/test/ably/rest/restinit_test.py b/test/ably/http/httpinit_test.py similarity index 97% rename from test/ably/rest/restinit_test.py rename to test/ably/http/httpinit_test.py index 929df98c..e7361cb8 100644 --- a/test/ably/rest/restinit_test.py +++ b/test/ably/http/httpinit_test.py @@ -10,7 +10,7 @@ from test.ably.utils import BaseAsyncTestCase, VaryByProtocolTestsMetaclass, dont_vary_protocol -class TestRestInit(BaseAsyncTestCase, metaclass=VaryByProtocolTestsMetaclass): +class TestHttpInit(BaseAsyncTestCase, metaclass=VaryByProtocolTestsMetaclass): @pytest.fixture(autouse=True) async def setup(self): @@ -168,8 +168,8 @@ async def test_query_time_param(self): use_binary_protocol=self.use_binary_protocol) timestamp = ably.auth._timestamp - with patch('ably.pubsub.rest.rest.AblyRest.time', wraps=ably.time) as server_time,\ - patch('ably.pubsub.rest.auth.Auth._timestamp', wraps=timestamp) as local_time: + with patch('ably.pubsub.http.http.AblyHttp.time', wraps=ably.time) as server_time,\ + patch('ably.pubsub.http.auth.Auth._timestamp', wraps=timestamp) as local_time: await ably.auth.request_token() assert local_time.call_count == 1 assert server_time.call_count == 1 diff --git a/test/ably/rest/restpaginatedresult_test.py b/test/ably/http/httppaginatedresult_test.py similarity index 98% rename from test/ably/rest/restpaginatedresult_test.py rename to test/ably/http/httppaginatedresult_test.py index fd7f9a81..dd33a071 100644 --- a/test/ably/rest/restpaginatedresult_test.py +++ b/test/ably/http/httppaginatedresult_test.py @@ -2,7 +2,7 @@ import respx from httpx import Response -from ably.pubsub.http.paginatedresult import PaginatedResult +from ably.pubsub.request.paginatedresult import PaginatedResult from test.ably.testapp import TestApp from test.ably.utils import BaseAsyncTestCase diff --git a/test/ably/rest/restpresence_test.py b/test/ably/http/httppresence_test.py similarity index 99% rename from test/ably/rest/restpresence_test.py rename to test/ably/http/httppresence_test.py index 4efbf35c..1fa52d42 100644 --- a/test/ably/rest/restpresence_test.py +++ b/test/ably/http/httppresence_test.py @@ -3,7 +3,7 @@ import pytest import respx -from ably.pubsub.http.paginatedresult import PaginatedResult +from ably.pubsub.request.paginatedresult import PaginatedResult from ably.pubsub.types.presence import PresenceMessage from test.ably.testapp import TestApp from test.ably.utils import BaseAsyncTestCase, VaryByProtocolTestsMetaclass, dont_vary_protocol diff --git a/test/ably/rest/restpush_test.py b/test/ably/http/httppush_test.py similarity index 99% rename from test/ably/rest/restpush_test.py rename to test/ably/http/httppush_test.py index 39a3dccd..9c430cf7 100644 --- a/test/ably/rest/restpush_test.py +++ b/test/ably/http/httppush_test.py @@ -5,7 +5,7 @@ import pytest -from ably.pubsub.http.paginatedresult import PaginatedResult +from ably.pubsub.request.paginatedresult import PaginatedResult from ably.pubsub.server import AblyAuthException, AblyException, DeviceDetails, PushChannelSubscription from test.ably.testapp import TestApp from test.ably.utils import ( diff --git a/test/ably/rest/restrequest_test.py b/test/ably/http/httprequest_test.py similarity index 98% rename from test/ably/rest/restrequest_test.py rename to test/ably/http/httprequest_test.py index bdeb725f..ec41c5ff 100644 --- a/test/ably/rest/restrequest_test.py +++ b/test/ably/http/httprequest_test.py @@ -2,7 +2,7 @@ import pytest import respx -from ably.pubsub.http.paginatedresult import HttpPaginatedResponse +from ably.pubsub.request.paginatedresult import HttpPaginatedResponse from ably.pubsub.server import create_http_client from ably.pubsub.transport.defaults import Defaults from test.ably.testapp import TestApp @@ -10,7 +10,7 @@ # RSC19 -class TestRestRequest(BaseAsyncTestCase, metaclass=VaryByProtocolTestsMetaclass): +class TestHttpRequest(BaseAsyncTestCase, metaclass=VaryByProtocolTestsMetaclass): @pytest.fixture(autouse=True) async def setup(self): diff --git a/test/ably/rest/reststats_test.py b/test/ably/http/httpstats_test.py similarity index 95% rename from test/ably/rest/reststats_test.py rename to test/ably/http/httpstats_test.py index 577f5644..5f039058 100644 --- a/test/ably/rest/reststats_test.py +++ b/test/ably/http/httpstats_test.py @@ -3,7 +3,7 @@ import pytest -from ably.pubsub.http.paginatedresult import PaginatedResult +from ably.pubsub.request.paginatedresult import PaginatedResult from ably.pubsub.types.stats import Stats from ably.pubsub.util.exceptions import AblyException from test.ably.testapp import TestApp @@ -12,7 +12,7 @@ log = logging.getLogger(__name__) -class TestRestAppStatsSetup: +class TestHttpAppStatsSetup: __stats_added = False def get_params(self): @@ -68,9 +68,9 @@ async def setup(self): } ) # asynctest does not support setUpClass method - if not TestRestAppStatsSetup.__stats_added: + if not TestHttpAppStatsSetup.__stats_added: await self.ably.http.post('/stats', body=stats + previous_stats) - TestRestAppStatsSetup.__stats_added = True + TestHttpAppStatsSetup.__stats_added = True yield await self.ably.close() await self.ably_text.close() @@ -79,7 +79,7 @@ def per_protocol_setup(self, use_binary_protocol): self.ably.options.use_binary_protocol = use_binary_protocol -class TestDirectionForwards(TestRestAppStatsSetup, BaseAsyncTestCase, +class TestDirectionForwards(TestHttpAppStatsSetup, BaseAsyncTestCase, metaclass=VaryByProtocolTestsMetaclass): def get_params(self): @@ -105,7 +105,7 @@ async def test_three_pages(self): assert page3.items[0].entries["messages.inbound.realtime.all.count"] == 70 -class TestDirectionBackwards(TestRestAppStatsSetup, BaseAsyncTestCase, +class TestDirectionBackwards(TestHttpAppStatsSetup, BaseAsyncTestCase, metaclass=VaryByProtocolTestsMetaclass): def get_params(self): @@ -131,7 +131,7 @@ async def test_three_pages(self): assert page3.items[0].entries["messages.inbound.realtime.all.count"] == 50 -class TestOnlyLastYear(TestRestAppStatsSetup, BaseAsyncTestCase, +class TestOnlyLastYear(TestHttpAppStatsSetup, BaseAsyncTestCase, metaclass=VaryByProtocolTestsMetaclass): def get_params(self): @@ -148,7 +148,7 @@ async def test_default_is_backwards(self): assert stats[-1].entries["messages.inbound.realtime.messages.count"] == 50 -class TestPreviousYear(TestRestAppStatsSetup, BaseAsyncTestCase, +class TestPreviousYear(TestHttpAppStatsSetup, BaseAsyncTestCase, metaclass=VaryByProtocolTestsMetaclass): def get_params(self): @@ -165,7 +165,7 @@ async def test_default_100_pagination(self): assert len(next_page.items) == 20 -class TestRestAppStats(TestRestAppStatsSetup, BaseAsyncTestCase, +class TestHttpAppStats(TestHttpAppStatsSetup, BaseAsyncTestCase, metaclass=VaryByProtocolTestsMetaclass): @dont_vary_protocol diff --git a/test/ably/rest/resttime_test.py b/test/ably/http/httptime_test.py similarity index 95% rename from test/ably/rest/resttime_test.py rename to test/ably/http/httptime_test.py index 757d5743..5dbeaee2 100644 --- a/test/ably/rest/resttime_test.py +++ b/test/ably/http/httptime_test.py @@ -7,7 +7,7 @@ from test.ably.utils import BaseAsyncTestCase, VaryByProtocolTestsMetaclass, dont_vary_protocol -class TestRestTime(BaseAsyncTestCase, metaclass=VaryByProtocolTestsMetaclass): +class TestHttpTime(BaseAsyncTestCase, metaclass=VaryByProtocolTestsMetaclass): def per_protocol_setup(self, use_binary_protocol): self.ably.options.use_binary_protocol = use_binary_protocol diff --git a/test/ably/rest/resttoken_test.py b/test/ably/http/httptoken_test.py similarity index 95% rename from test/ably/rest/resttoken_test.py rename to test/ably/http/httptoken_test.py index 160e8eca..34ea56f7 100644 --- a/test/ably/rest/resttoken_test.py +++ b/test/ably/http/httptoken_test.py @@ -14,7 +14,7 @@ log = logging.getLogger(__name__) -class TestRestToken(BaseAsyncTestCase, metaclass=VaryByProtocolTestsMetaclass): +class TestHttpToken(BaseAsyncTestCase, metaclass=VaryByProtocolTestsMetaclass): async def server_time(self): return await self.ably.time() @@ -120,8 +120,8 @@ async def test_token_generation_with_invalid_ttl(self): async def test_token_generation_with_local_time(self): timestamp = self.ably.auth._timestamp - with patch('ably.pubsub.rest.rest.AblyRest.time', wraps=self.ably.time) as server_time,\ - patch('ably.pubsub.rest.auth.Auth._timestamp', wraps=timestamp) as local_time: + with patch('ably.pubsub.http.http.AblyHttp.time', wraps=self.ably.time) as server_time,\ + patch('ably.pubsub.http.auth.Auth._timestamp', wraps=timestamp) as local_time: await self.ably.auth.request_token() assert local_time.called assert not server_time.called @@ -129,8 +129,8 @@ async def test_token_generation_with_local_time(self): # RSA10k async def test_token_generation_with_server_time(self): timestamp = self.ably.auth._timestamp - with patch('ably.pubsub.rest.rest.AblyRest.time', wraps=self.ably.time) as server_time,\ - patch('ably.pubsub.rest.auth.Auth._timestamp', wraps=timestamp) as local_time: + with patch('ably.pubsub.http.http.AblyHttp.time', wraps=self.ably.time) as server_time,\ + patch('ably.pubsub.http.auth.Auth._timestamp', wraps=timestamp) as local_time: await self.ably.auth.request_token(query_time=True) assert local_time.call_count == 1 assert server_time.call_count == 1 @@ -182,8 +182,8 @@ async def test_key_name_and_secret_are_required(self): @dont_vary_protocol async def test_with_local_time(self): timestamp = self.ably.auth._timestamp - with patch('ably.pubsub.rest.rest.AblyRest.time', wraps=self.ably.time) as server_time,\ - patch('ably.pubsub.rest.auth.Auth._timestamp', wraps=timestamp) as local_time: + with patch('ably.pubsub.http.http.AblyHttp.time', wraps=self.ably.time) as server_time,\ + patch('ably.pubsub.http.auth.Auth._timestamp', wraps=timestamp) as local_time: await self.ably.auth.create_token_request( key_name=self.key_name, key_secret=self.key_secret, query_time=False) assert local_time.called @@ -193,8 +193,8 @@ async def test_with_local_time(self): @dont_vary_protocol async def test_with_server_time(self): timestamp = self.ably.auth._timestamp - with patch('ably.pubsub.rest.rest.AblyRest.time', wraps=self.ably.time) as server_time,\ - patch('ably.pubsub.rest.auth.Auth._timestamp', wraps=timestamp) as local_time: + with patch('ably.pubsub.http.http.AblyHttp.time', wraps=self.ably.time) as server_time,\ + patch('ably.pubsub.http.auth.Auth._timestamp', wraps=timestamp) as local_time: await self.ably.auth.create_token_request( key_name=self.key_name, key_secret=self.key_secret, query_time=True) assert local_time.call_count == 1 @@ -329,7 +329,7 @@ async def test_hmac(self): # AO2g @dont_vary_protocol async def test_query_server_time(self): - with patch('ably.pubsub.rest.rest.AblyRest.time', wraps=self.ably.time) as server_time: + with patch('ably.pubsub.http.http.AblyHttp.time', wraps=self.ably.time) as server_time: await self.ably.auth.create_token_request( key_name=self.key_name, key_secret=self.key_secret, query_time=True) assert server_time.call_count == 1 diff --git a/test/ably/utils.py b/test/ably/utils.py index 9b768c0f..6715ced7 100644 --- a/test/ably/utils.py +++ b/test/ably/utils.py @@ -11,7 +11,7 @@ import respx from httpx import Response -from ably.pubsub.http.http import Http +from ably.pubsub.request.http import Http class BaseTestCase: diff --git a/test/unit/annotation_test.py b/test/unit/annotation_test.py index ff16cf56..674f14c1 100644 --- a/test/unit/annotation_test.py +++ b/test/unit/annotation_test.py @@ -15,7 +15,7 @@ import pytest -from ably.pubsub.rest.annotations import construct_validate_annotation, serial_from_msg_or_serial +from ably.pubsub.http.annotations import construct_validate_annotation, serial_from_msg_or_serial from ably.pubsub.types.annotation import Annotation, AnnotationAction from ably.pubsub.types.message import Message from ably.pubsub.util.crypto import generate_random_key, get_cipher diff --git a/test/unit/mutable_message_test.py b/test/unit/mutable_message_test.py index 2ea569c0..e30ba443 100644 --- a/test/unit/mutable_message_test.py +++ b/test/unit/mutable_message_test.py @@ -100,7 +100,7 @@ def test_message_version_serialization(): def test_message_extras_preserved_in_as_dict(): """Test that extras are included when a Message with extras is serialized. - Regression test: _send_update() in both RestChannel and RealtimeChannel + Regression test: _send_update() in both HttpChannel and RealtimeChannel constructed a new Message without copying extras or annotations from the user-supplied message, violating RSL15b/RTL32b which require "whatever fields were in the user-supplied Message" to be sent. diff --git a/test/unit/pubsub_server_test.py b/test/unit/pubsub_server_test.py index 08b823fc..4f3f6f05 100644 --- a/test/unit/pubsub_server_test.py +++ b/test/unit/pubsub_server_test.py @@ -4,11 +4,11 @@ import ably.pubsub.server as server import ably.pubsub.server.sync as server_sync -from ably.pubsub.prototypes import RealtimeClient, RestClient +from ably.pubsub.http.http import AblyHttp +from ably.pubsub.prototypes import PubSubHttpClient, PubSubRealtimeClient from ably.pubsub.realtime.realtime import AblyRealtime -from ably.pubsub.rest.rest import AblyRest from ably.pubsub.server import create_http_client, create_realtime_client -from ably.pubsub.sync.rest.rest import AblyRestSync +from ably.pubsub.sync.http.http import AblyHttpSync def protocol_members(protocol): @@ -29,7 +29,7 @@ def protocol_members(protocol): class TestFactories: def test_http_factory_returns_a_client(self): client = create_http_client(token='foo') - assert isinstance(client, AblyRest) + assert isinstance(client, AblyHttp) async def test_realtime_factory_returns_a_client(self): client = create_realtime_client(key='foo:bar', auto_connect=False) @@ -37,7 +37,7 @@ async def test_realtime_factory_returns_a_client(self): await client.close() def test_sync_http_factory_returns_a_client(self): - assert isinstance(server_sync.create_http_client(token='foo'), AblyRestSync) + assert isinstance(server_sync.create_http_client(token='foo'), AblyHttpSync) def test_factory_passes_options_through(self): client = create_http_client(token='foo', client_id='me') @@ -47,7 +47,7 @@ def test_factory_passes_options_through(self): class TestConstructorsAreInternal: def test_rest_constructor_is_rejected(self): with pytest.raises(TypeError) as excinfo: - AblyRest(token='foo') + AblyHttp(token='foo') assert 'ably.pubsub.server.create_http_client' in str(excinfo.value) def test_realtime_constructor_is_rejected(self): @@ -57,33 +57,33 @@ def test_realtime_constructor_is_rejected(self): def test_sync_rest_constructor_is_rejected(self): with pytest.raises(TypeError) as excinfo: - AblyRestSync(token='foo') + AblyHttpSync(token='foo') assert 'ably.pubsub.server.sync.create_http_client' in str(excinfo.value) def test_permission_does_not_leak_past_the_factory(self): create_http_client(token='foo') with pytest.raises(TypeError): - AblyRest(token='foo') + AblyHttp(token='foo') - @pytest.mark.parametrize('name', ['AblyRest', 'AblyRealtime']) + @pytest.mark.parametrize('name', ['AblyHttp', 'AblyRealtime']) def test_client_classes_are_not_exported(self, name): assert not hasattr(server, name) assert name not in server.__all__ def test_sync_client_class_is_not_exported(self): - assert not hasattr(server_sync, 'AblyRestSync') - assert 'AblyRestSync' not in server_sync.__all__ + assert not hasattr(server_sync, 'AblyHttpSync') + assert 'AblyHttpSync' not in server_sync.__all__ class TestPrototypes: - @pytest.mark.parametrize('protocol,cls', [(RestClient, AblyRest), (RealtimeClient, AblyRealtime)]) + @pytest.mark.parametrize('protocol,cls', [(PubSubHttpClient, AblyHttp), (PubSubRealtimeClient, AblyRealtime)]) def test_client_satisfies_its_prototype(self, protocol, cls): missing = sorted(member for member in protocol_members(protocol) if not hasattr(cls, member)) assert missing == [] - @pytest.mark.parametrize('name', ['RestClient', 'RealtimeClient']) + @pytest.mark.parametrize('name', ['PubSubHttpClient', 'PubSubRealtimeClient']) def test_prototypes_are_exported(self, name): assert name in server.__all__ def test_sync_prototype_is_exported(self): - assert 'RestClient' in server_sync.__all__ + assert 'PubSubHttpClient' in server_sync.__all__ diff --git a/test/uts/helpers/sandbox.py b/test/uts/helpers/sandbox.py index 203e7f41..65ea66d3 100644 --- a/test/uts/helpers/sandbox.py +++ b/test/uts/helpers/sandbox.py @@ -4,7 +4,7 @@ POST the canonical app setup body to the sandbox, take the keys out of the response, and DELETE the app when the tests are done. -Provisioning goes over plain `httpx` rather than through `AblyRest`. It is +Provisioning goes over plain `httpx` rather than through `AblyHttp`. It is infrastructure, and a client that cannot form a request would otherwise look like a broken fixture rather than a failing test. diff --git a/test/uts/realtime/integration/channels/channel_attach_test.py b/test/uts/realtime/integration/channels/channel_attach_test.py index 1dee7f99..707ffb5d 100644 --- a/test/uts/realtime/integration/channels/channel_attach_test.py +++ b/test/uts/realtime/integration/channels/channel_attach_test.py @@ -6,7 +6,7 @@ `use_binary_protocol`; `sandbox_realtime_client` already defaults to it. `RealtimeChannel.publish()` takes its arguments positionally. The keyword form the -specification writes, which `RestChannel.publish()` does accept, raises +specification writes, which `HttpChannel.publish()` does accept, raises `ValueError: publish() expects either (name, data) or a message object or array of messages` before anything reaches the server. diff --git a/test/uts/realtime/integration/channels/channel_publish_test.py b/test/uts/realtime/integration/channels/channel_publish_test.py index 13b0a29c..d106a243 100644 --- a/test/uts/realtime/integration/channels/channel_publish_test.py +++ b/test/uts/realtime/integration/channels/channel_publish_test.py @@ -6,7 +6,7 @@ per protocol and passes `use_binary_protocol` to both clients it builds. `RealtimeChannel.publish()` takes its arguments positionally; the keyword form the -specification writes, which `RestChannel.publish()` does accept, raises `ValueError`. +specification writes, which `HttpChannel.publish()` does accept, raises `ValueError`. A binary payload arrives as a `bytearray` rather than `bytes` under either protocol, so the type assertion reads both. `bytearray` compares equal to the `bytes` that was diff --git a/test/uts/realtime/integration/mutable_messages_test.py b/test/uts/realtime/integration/mutable_messages_test.py index 327bfcf6..481f50da 100644 --- a/test/uts/realtime/integration/mutable_messages_test.py +++ b/test/uts/realtime/integration/mutable_messages_test.py @@ -18,8 +18,8 @@ consistent; the two polls below are the specification's `poll_until_success` around them. """ -from ably.pubsub.http.paginatedresult import PaginatedResult from ably.pubsub.realtime.connection import ConnectionState +from ably.pubsub.request.paginatedresult import PaginatedResult from ably.pubsub.types.annotation import Annotation, AnnotationAction from ably.pubsub.types.channelmode import ChannelMode from ably.pubsub.types.channeloptions import ChannelOptions diff --git a/test/uts/realtime/unit/channels/channel_history_test.py b/test/uts/realtime/unit/channels/channel_history_test.py index cee3ad13..509e043a 100644 --- a/test/uts/realtime/unit/channels/channel_history_test.py +++ b/test/uts/realtime/unit/channels/channel_history_test.py @@ -14,8 +14,8 @@ import pytest -from ably.pubsub.http.paginatedresult import PaginatedResult from ably.pubsub.realtime.connection import ConnectionState +from ably.pubsub.request.paginatedresult import PaginatedResult from ably.pubsub.transport.websockettransport import ProtocolMessageAction from ably.pubsub.types.channelstate import ChannelState from ably.pubsub.util.exceptions import AblyException @@ -59,7 +59,7 @@ def on_message_from_client(msg): # UTS: realtime/unit/RTL10a/supports-rest-params-0 async def test_rtl10a_supports_rest_params(): # The specification directs uts/rest/unit/channel/history.md (RSL2) at a realtime - # channel in place of a REST one. `AblyRealtime` subclasses `AblyRest`, so the same + # channel in place of a REST one. `AblyRealtime` subclasses `AblyHttp`, so the same # HTTP mock serves it, and this mirrors that suite's `RSL2a/returns-paginated-result-0` # and `RSL2b/query-parameters-0`. channel_name = f'test-RTL10a-{random_id()}' @@ -86,7 +86,7 @@ async def test_rtl10a_supports_rest_params(): assert captured_requests[0].url.path == f'/channels/{channel_name}/messages' - # RTL10a: the parameters `RestChannel#history` takes + # RTL10a: the parameters `HttpChannel#history` takes await channel.history(direction='forwards', limit=50, start=1000, end=2000) query_params = captured_requests[1].url.query_params diff --git a/test/uts/realtime/unit/channels/channel_message_versions_test.py b/test/uts/realtime/unit/channels/channel_message_versions_test.py index 6a72b12a..00198be0 100644 --- a/test/uts/realtime/unit/channels/channel_message_versions_test.py +++ b/test/uts/realtime/unit/channels/channel_message_versions_test.py @@ -10,7 +10,7 @@ import uuid -from ably.pubsub.http.paginatedresult import PaginatedResult +from ably.pubsub.request.paginatedresult import PaginatedResult from ably.pubsub.types.message import Message, MessageAction from test.uts.helpers.client import realtime_client from test.uts.helpers.mock_http import MockHttpClient diff --git a/test/uts/realtime/unit/channels/channel_publish_test.py b/test/uts/realtime/unit/channels/channel_publish_test.py index a68df593..bd531b82 100644 --- a/test/uts/realtime/unit/channels/channel_publish_test.py +++ b/test/uts/realtime/unit/channels/channel_publish_test.py @@ -15,7 +15,7 @@ as a task. `RealtimeChannel.publish()` takes its arguments positionally; the keyword form the -specifications write, which `RestChannel.publish()` does accept, raises `ValueError` here. +specifications write, which `HttpChannel.publish()` does accept, raises `ValueError` here. """ import asyncio diff --git a/test/uts/realtime/unit/client/realtime_client_test.py b/test/uts/realtime/unit/client/realtime_client_test.py index 088a1607..21e6aa37 100644 --- a/test/uts/realtime/unit/client/realtime_client_test.py +++ b/test/uts/realtime/unit/client/realtime_client_test.py @@ -8,10 +8,10 @@ import pytest +from ably.pubsub.http.auth import Auth +from ably.pubsub.http.push import Push, PushAdmin from ably.pubsub.realtime.channel import Channels, RealtimeChannel from ably.pubsub.realtime.connection import Connection, ConnectionState -from ably.pubsub.rest.auth import Auth -from ably.pubsub.rest.push import Push, PushAdmin from ably.pubsub.transport.websockettransport import ProtocolMessageAction from ably.pubsub.util.exceptions import AblyException from ably.pubsub.util.helper import get_random_id diff --git a/test/uts/realtime/unit/client/realtime_request_test.py b/test/uts/realtime/unit/client/realtime_request_test.py index 3a0fe289..9765194f 100644 --- a/test/uts/realtime/unit/client/realtime_request_test.py +++ b/test/uts/realtime/unit/client/realtime_request_test.py @@ -17,7 +17,7 @@ # UTS: realtime/unit/RTC9/request-proxies-rest-0 async def test_rtc9_request_proxies_rest(): # The specification directs uts/rest/unit/request.md (RSC19) at a realtime client - # in place of a REST one. `AblyRealtime` subclasses `AblyRest`, so the same HTTP + # in place of a REST one. `AblyRealtime` subclasses `AblyHttp`, so the same HTTP # mock serves it, and this mirrors that suite's `RSC19f/supports-http-methods-0` # and `RSC19d/response-items-decoded-5`. captured_requests = [] diff --git a/test/uts/realtime/unit/client/realtime_stats_test.py b/test/uts/realtime/unit/client/realtime_stats_test.py index 77f8d073..1138c8d2 100644 --- a/test/uts/realtime/unit/client/realtime_stats_test.py +++ b/test/uts/realtime/unit/client/realtime_stats_test.py @@ -3,7 +3,7 @@ Spec points: RTC5, RTC5a, RTC5b """ -from ably.pubsub.http.paginatedresult import PaginatedResult +from ably.pubsub.request.paginatedresult import PaginatedResult from test.uts.helpers.client import realtime_client from test.uts.helpers.mock_http import MockHttpClient @@ -30,7 +30,7 @@ # UTS: realtime/unit/RTC5/stats-proxies-rest-0 async def test_rtc5_stats_proxies_rest(): # The specification directs uts/rest/unit/stats.md (RSC6) at a realtime client - # in place of a REST one. `AblyRealtime` subclasses `AblyRest`, so the same HTTP + # in place of a REST one. `AblyRealtime` subclasses `AblyHttp`, so the same HTTP # mock serves it, and this mirrors that suite's `RSC6a/returns-paginated-stats-0`. captured_requests = [] diff --git a/test/uts/realtime/unit/client/realtime_time_test.py b/test/uts/realtime/unit/client/realtime_time_test.py index 64c1fa8e..435de0a4 100644 --- a/test/uts/realtime/unit/client/realtime_time_test.py +++ b/test/uts/realtime/unit/client/realtime_time_test.py @@ -12,7 +12,7 @@ # UTS: realtime/unit/RTC6/time-proxies-rest-0 async def test_rtc6_time_proxies_rest(): # The specification directs uts/rest/unit/time.md (RSC16) at a realtime client - # in place of a REST one. `AblyRealtime` subclasses `AblyRest`, so the same + # in place of a REST one. `AblyRealtime` subclasses `AblyHttp`, so the same # HTTP mock serves it, and this mirrors that suite's `RSC16/returns-server-time-0`. captured_requests = [] diff --git a/test/uts/realtime/unit/presence/realtime_presence_history_test.py b/test/uts/realtime/unit/presence/realtime_presence_history_test.py index 9f022378..59b60675 100644 --- a/test/uts/realtime/unit/presence/realtime_presence_history_test.py +++ b/test/uts/realtime/unit/presence/realtime_presence_history_test.py @@ -14,8 +14,8 @@ import uuid -from ably.pubsub.http.paginatedresult import PaginatedResult from ably.pubsub.realtime.connection import ConnectionState +from ably.pubsub.request.paginatedresult import PaginatedResult from ably.pubsub.transport.websockettransport import ProtocolMessageAction from ably.pubsub.types.presence import PresenceAction, PresenceMessage from test.uts.helpers.client import await_connection_state, realtime_client diff --git a/test/uts/rest/integration/batch_presence_test.py b/test/uts/rest/integration/batch_presence_test.py index 4e506570..e846e276 100644 --- a/test/uts/rest/integration/batch_presence_test.py +++ b/test/uts/rest/integration/batch_presence_test.py @@ -2,7 +2,7 @@ Spec points: RSC24, BGR2, BGF2 -DEVIATION: ably-python has no batch API. `AblyRest` exposes no `batch_presence`, and +DEVIATION: ably-python has no batch API. `AblyHttp` exposes no `batch_presence`, and the package defines neither `BatchResult` nor `BatchPresenceSuccessResult` / `BatchPresenceFailureResult`; the word "batch" appears nowhere under `ably/`. Every test here therefore departs from the specification and is gated behind diff --git a/test/uts/rest/integration/mutable_messages_test.py b/test/uts/rest/integration/mutable_messages_test.py index 77fd8b20..225302af 100644 --- a/test/uts/rest/integration/mutable_messages_test.py +++ b/test/uts/rest/integration/mutable_messages_test.py @@ -3,7 +3,7 @@ Spec points: RSL1n, RSL11, RSL14, RSL15, RSAN1, RSAN2, RSAN3 """ -from ably.pubsub.http.paginatedresult import PaginatedResult +from ably.pubsub.request.paginatedresult import PaginatedResult from ably.pubsub.types.annotation import Annotation, AnnotationAction from ably.pubsub.types.message import Message, MessageAction from ably.pubsub.types.operations import MessageOperation, PublishResult, UpdateDeleteResult diff --git a/test/uts/rest/integration/pagination_test.py b/test/uts/rest/integration/pagination_test.py index 291d0c9d..984d3044 100644 --- a/test/uts/rest/integration/pagination_test.py +++ b/test/uts/rest/integration/pagination_test.py @@ -3,7 +3,7 @@ Spec points: TG1, TG2, TG3, TG4, TG5 """ -from ably.pubsub.http.paginatedresult import PaginatedResult +from ably.pubsub.request.paginatedresult import PaginatedResult from ably.pubsub.types.message import Message from test.uts.helpers.client import sandbox_rest_client, wall_clock_poll_until from test.uts.helpers.sandbox import random_id diff --git a/test/uts/rest/integration/presence_test.py b/test/uts/rest/integration/presence_test.py index 9287fc4e..6a3a5ab5 100644 --- a/test/uts/rest/integration/presence_test.py +++ b/test/uts/rest/integration/presence_test.py @@ -5,8 +5,8 @@ import pytest -from ably.pubsub.http.paginatedresult import PaginatedResult from ably.pubsub.realtime.connection import ConnectionState +from ably.pubsub.request.paginatedresult import PaginatedResult from ably.pubsub.types.presence import Presence, PresenceAction, PresenceMessage from ably.pubsub.util.exceptions import AblyException from test.uts.helpers.client import ( diff --git a/test/uts/rest/integration/push_admin_test.py b/test/uts/rest/integration/push_admin_test.py index 81a8301b..16720dd3 100644 --- a/test/uts/rest/integration/push_admin_test.py +++ b/test/uts/rest/integration/push_admin_test.py @@ -6,7 +6,7 @@ import pytest -from ably.pubsub.http.paginatedresult import PaginatedResult +from ably.pubsub.request.paginatedresult import PaginatedResult from ably.pubsub.server import AblyException, DeviceDetails, PushChannelSubscription from test.uts.helpers.client import sandbox_rest_client, wall_clock_poll_until from test.uts.helpers.sandbox import random_id diff --git a/test/uts/rest/integration/push_channels_test.py b/test/uts/rest/integration/push_channels_test.py index aeef48f8..109daa4e 100644 --- a/test/uts/rest/integration/push_channels_test.py +++ b/test/uts/rest/integration/push_channels_test.py @@ -4,7 +4,7 @@ DEVIATION: ably-python implements neither the PushChannel interface (RSH7, the `push` field on a channel) nor LocalDevice (RSH8). `ably/rest/channel.py` gives a channel no -`push`, `AblyRest` no `device`, and `ably/types/device.py` defines only `DeviceDetails`. +`push`, `AblyHttp` no `device`, and `ably/types/device.py` defines only `DeviceDetails`. Both tests here therefore depart from the specification and are gated behind RUN_DEVIATIONS, against the same spelling [test/uts/rest/unit/push/push_channels_test.py](../unit/push/push_channels_test.py) diff --git a/test/uts/rest/integration/time_stats_test.py b/test/uts/rest/integration/time_stats_test.py index f402e800..b2d3abb7 100644 --- a/test/uts/rest/integration/time_stats_test.py +++ b/test/uts/rest/integration/time_stats_test.py @@ -8,7 +8,7 @@ import pytest_asyncio -from ably.pubsub.http.paginatedresult import PaginatedResult +from ably.pubsub.request.paginatedresult import PaginatedResult from ably.pubsub.server import create_http_client from ably.pubsub.types.stats import Stats from test.uts.helpers.client import sandbox_rest_client diff --git a/test/uts/rest/unit/batch_presence_test.py b/test/uts/rest/unit/batch_presence_test.py index f6089fe0..b085f453 100644 --- a/test/uts/rest/unit/batch_presence_test.py +++ b/test/uts/rest/unit/batch_presence_test.py @@ -2,7 +2,7 @@ Spec points: RSC24, BAR2, BGR2, BGF2 -NOTE: ably-python has no batch API. `AblyRest` exposes no `batch_presence`, and the package +NOTE: ably-python has no batch API. `AblyHttp` exposes no `batch_presence`, and the package defines neither `BatchResult`/`BatchPresenceResponse` nor `BatchPresenceSuccessResult`/ `BatchPresenceFailureResult`; the word "batch" appears nowhere under `ably/`. Every test in this file therefore departs from the specification and is gated behind `RUN_DEVIATIONS`. diff --git a/test/uts/rest/unit/batch_publish_test.py b/test/uts/rest/unit/batch_publish_test.py index e3dacb44..96b89163 100644 --- a/test/uts/rest/unit/batch_publish_test.py +++ b/test/uts/rest/unit/batch_publish_test.py @@ -2,7 +2,7 @@ Spec points: RSC22c, RSC22d, BSP2a, BSP2b, BPR2a, BPR2b, BPR2c, BPF2a, BPF2b -NOTE: ably-python has no batch API. `AblyRest` exposes no `batch_publish`, and the package +NOTE: ably-python has no batch API. `AblyHttp` exposes no `batch_publish`, and the package defines neither `BatchPublishSpec` nor `BatchResult`/`BatchPublishSuccessResult`/ `BatchPublishFailureResult`; the word "batch" appears nowhere under `ably/`. Every test in this file therefore departs from the specification and is gated behind `RUN_DEVIATIONS`. diff --git a/test/uts/rest/unit/channel/annotations_test.py b/test/uts/rest/unit/channel/annotations_test.py index 61ddc14e..c3d39264 100644 --- a/test/uts/rest/unit/channel/annotations_test.py +++ b/test/uts/rest/unit/channel/annotations_test.py @@ -15,8 +15,8 @@ import msgpack import pytest -from ably.pubsub.http.paginatedresult import PaginatedResult -from ably.pubsub.rest.annotations import RestAnnotations +from ably.pubsub.http.annotations import HttpAnnotations +from ably.pubsub.request.paginatedresult import PaginatedResult from ably.pubsub.types.annotation import Annotation, AnnotationAction from ably.pubsub.util.exceptions import AblyException from test.uts.helpers.client import rest_client @@ -53,7 +53,7 @@ async def test_rsl10_annotations_attribute_type(): client = rest_client(mock_http) channel = client.channels.get('test-RSL10') - assert isinstance(channel.annotations, RestAnnotations) + assert isinstance(channel.annotations, HttpAnnotations) # UTS: rest/unit/RSAN1c6/publish-post-annotation-create-0 diff --git a/test/uts/rest/unit/channel/history_test.py b/test/uts/rest/unit/channel/history_test.py index 0eb04c95..cfae1d4b 100644 --- a/test/uts/rest/unit/channel/history_test.py +++ b/test/uts/rest/unit/channel/history_test.py @@ -5,7 +5,7 @@ import uuid -from ably.pubsub.http.paginatedresult import PaginatedResult +from ably.pubsub.request.paginatedresult import PaginatedResult from ably.pubsub.types.message import Message from test.uts.helpers.client import rest_client from test.uts.helpers.mock_http import MockHttpClient diff --git a/test/uts/rest/unit/channel/message_versions_test.py b/test/uts/rest/unit/channel/message_versions_test.py index 8ad2c1ff..b533c8da 100644 --- a/test/uts/rest/unit/channel/message_versions_test.py +++ b/test/uts/rest/unit/channel/message_versions_test.py @@ -5,7 +5,7 @@ import uuid -from ably.pubsub.http.paginatedresult import PaginatedResult +from ably.pubsub.request.paginatedresult import PaginatedResult from ably.pubsub.types.message import Message, MessageAction from test.uts.helpers.client import rest_client from test.uts.helpers.mock_http import MockHttpClient diff --git a/test/uts/rest/unit/channels_collection_test.py b/test/uts/rest/unit/channels_collection_test.py index 830225ef..3a43f167 100644 --- a/test/uts/rest/unit/channels_collection_test.py +++ b/test/uts/rest/unit/channels_collection_test.py @@ -8,7 +8,7 @@ import uuid -from ably.pubsub.rest.channel import Channel, Channels +from ably.pubsub.http.channel import Channel, Channels from test.uts.helpers.client import rest_client from test.uts.helpers.mock_http import MockHttpClient diff --git a/test/uts/rest/unit/presence/rest_presence_test.py b/test/uts/rest/unit/presence/rest_presence_test.py index 5c10f034..584949c8 100644 --- a/test/uts/rest/unit/presence/rest_presence_test.py +++ b/test/uts/rest/unit/presence/rest_presence_test.py @@ -11,7 +11,7 @@ import msgpack import pytest -from ably.pubsub.http.paginatedresult import PaginatedResult +from ably.pubsub.request.paginatedresult import PaginatedResult from ably.pubsub.types.presence import Presence, PresenceAction, PresenceMessage from ably.pubsub.util.crypto import CipherParams from ably.pubsub.util.exceptions import AblyException diff --git a/test/uts/rest/unit/push/push_admin_publish_test.py b/test/uts/rest/unit/push/push_admin_publish_test.py index 223f57bf..846baa70 100644 --- a/test/uts/rest/unit/push/push_admin_publish_test.py +++ b/test/uts/rest/unit/push/push_admin_publish_test.py @@ -6,7 +6,7 @@ import msgpack import pytest -from ably.pubsub.rest.push import Push, PushAdmin, PushChannelSubscriptions, PushDeviceRegistrations +from ably.pubsub.http.push import Push, PushAdmin, PushChannelSubscriptions, PushDeviceRegistrations from ably.pubsub.util.exceptions import AblyException from test.uts.helpers.client import rest_client from test.uts.helpers.mock_http import MockHttpClient diff --git a/test/uts/rest/unit/push/push_channel_subscriptions_test.py b/test/uts/rest/unit/push/push_channel_subscriptions_test.py index 0eeade5c..7826276f 100644 --- a/test/uts/rest/unit/push/push_channel_subscriptions_test.py +++ b/test/uts/rest/unit/push/push_channel_subscriptions_test.py @@ -6,7 +6,7 @@ import msgpack import pytest -from ably.pubsub.http.paginatedresult import PaginatedResult +from ably.pubsub.request.paginatedresult import PaginatedResult from ably.pubsub.types.channelsubscription import PushChannelSubscription from ably.pubsub.util.exceptions import AblyException from test.uts.helpers.client import rest_client diff --git a/test/uts/rest/unit/push/push_channels_test.py b/test/uts/rest/unit/push/push_channels_test.py index edf92753..699dec11 100644 --- a/test/uts/rest/unit/push/push_channels_test.py +++ b/test/uts/rest/unit/push/push_channels_test.py @@ -14,7 +14,7 @@ import msgpack import pytest -from ably.pubsub.http.paginatedresult import PaginatedResult +from ably.pubsub.request.paginatedresult import PaginatedResult from ably.pubsub.types.channelsubscription import PushChannelSubscription from ably.pubsub.util.exceptions import AblyException from test.uts.helpers.client import rest_client diff --git a/test/uts/rest/unit/push/push_device_registrations_test.py b/test/uts/rest/unit/push/push_device_registrations_test.py index 8ab284c1..495ecc64 100644 --- a/test/uts/rest/unit/push/push_device_registrations_test.py +++ b/test/uts/rest/unit/push/push_device_registrations_test.py @@ -8,7 +8,7 @@ import msgpack import pytest -from ably.pubsub.http.paginatedresult import PaginatedResult +from ably.pubsub.request.paginatedresult import PaginatedResult from ably.pubsub.types.device import DeviceDetails from ably.pubsub.util.exceptions import AblyException from test.uts.helpers.client import rest_client diff --git a/test/uts/rest/unit/rest_client_test.py b/test/uts/rest/unit/rest_client_test.py index c23e37c9..1a19adb9 100644 --- a/test/uts/rest/unit/rest_client_test.py +++ b/test/uts/rest/unit/rest_client_test.py @@ -8,7 +8,7 @@ import msgpack import pytest -from ably.pubsub.rest.auth import Auth +from ably.pubsub.http.auth import Auth from ably.pubsub.util.exceptions import AblyException from test.uts.helpers.client import rest_client from test.uts.helpers.mock_http import MockHttpClient diff --git a/test/uts/rest/unit/stats_test.py b/test/uts/rest/unit/stats_test.py index 72dca8bc..0cfdaaa9 100644 --- a/test/uts/rest/unit/stats_test.py +++ b/test/uts/rest/unit/stats_test.py @@ -7,7 +7,7 @@ import pytest -from ably.pubsub.http.paginatedresult import PaginatedResult +from ably.pubsub.request.paginatedresult import PaginatedResult from ably.pubsub.util.exceptions import AblyException from test.uts.helpers.client import rest_client from test.uts.helpers.mock_http import MockHttpClient From 71fdbd037f880e2c014e33338fa49dfd559bb768 Mon Sep 17 00:00:00 2001 From: evgeny Date: Fri, 18 Sep 2026 14:53:30 +0100 Subject: [PATCH 06/71] pubsub: tie the client classes to the prototypes they implement Nothing connected AblyHttp and AblyRealtime to the prototypes the factories return. A hasattr loop in the tests was the only check, and it was weak enough to call a class with no implemented members conformant. Have each client name its prototype as a base class, so the relationship is stated where the client is defined and a type checker verifies it. That alone would weaken the runtime check, because an unimplemented member then resolves to the prototype's `...` stub and returns None rather than raising, so the conformance tests are strengthened to match: - the prototype must actually appear in the client's MRO - every member must be defined by the client itself, found by dropping the prototype's own MRO so an inherited stub cannot satisfy it - parameter names, kinds and requiredness must match the prototype, as must whether the member is awaitable The synchronous client is covered too, since unasync generates both sides of that pair. Annotations are excluded from the signature comparison: the prototype spells its types as strings under `from __future__ import annotations` while the clients evaluate theirs, and return types are a type checker's concern. Co-Authored-By: Claude Opus 5 (1M context) --- ably/pubsub/http/http.py | 3 +- ably/pubsub/prototypes.py | 7 +++ ably/pubsub/realtime/realtime.py | 3 +- test/unit/pubsub_server_test.py | 73 ++++++++++++++++++++++++++++++-- 4 files changed, 81 insertions(+), 5 deletions(-) diff --git a/ably/pubsub/http/http.py b/ably/pubsub/http/http.py index 3ba41fa7..928f0679 100644 --- a/ably/pubsub/http/http.py +++ b/ably/pubsub/http/http.py @@ -5,6 +5,7 @@ from ably.pubsub.http.auth import Auth from ably.pubsub.http.channel import Channels from ably.pubsub.http.push import Push +from ably.pubsub.prototypes import PubSubHttpClient from ably.pubsub.request.http import Http from ably.pubsub.request.paginatedresult import HttpPaginatedResponse, PaginatedResult, format_params from ably.pubsub.types.options import Options @@ -16,7 +17,7 @@ log = logging.getLogger(__name__) -class AblyHttp: +class AblyHttp(PubSubHttpClient): """Ably HTTP Client""" def __init__(self, key: Optional[str] = None, token: Optional[str] = None, diff --git a/ably/pubsub/prototypes.py b/ably/pubsub/prototypes.py index c3a3d3e6..c3596e21 100644 --- a/ably/pubsub/prototypes.py +++ b/ably/pubsub/prototypes.py @@ -6,6 +6,13 @@ These protocols are what those factories are declared to return, and what application code should annotate against. +The clients name their prototype as a base class rather than satisfying it +structurally, so the relationship is stated where the client is defined and a +type checker verifies it. One consequence to know about: an unimplemented +member resolves to the stub below and returns None instead of raising, so +``test/unit/pubsub_server_test.py`` checks that each client defines every +member itself, and with the signature declared here. + They are named "prototypes" rather than "protocols" because this codebase already uses "protocol" for the Ably wire protocol. """ diff --git a/ably/pubsub/realtime/realtime.py b/ably/pubsub/realtime/realtime.py index e2c12c5a..7948a47a 100644 --- a/ably/pubsub/realtime/realtime.py +++ b/ably/pubsub/realtime/realtime.py @@ -3,6 +3,7 @@ from typing import Optional from ably.pubsub.http.http import AblyHttp +from ably.pubsub.prototypes import PubSubRealtimeClient from ably.pubsub.realtime.channel import Channels from ably.pubsub.realtime.connection import Connection, ConnectionState from ably.pubsub.util.construction import reject_direct_construction @@ -10,7 +11,7 @@ log = logging.getLogger(__name__) -class AblyRealtime(AblyHttp): +class AblyRealtime(AblyHttp, PubSubRealtimeClient): """ Ably Realtime Client diff --git a/test/unit/pubsub_server_test.py b/test/unit/pubsub_server_test.py index 4f3f6f05..b5e902f3 100644 --- a/test/unit/pubsub_server_test.py +++ b/test/unit/pubsub_server_test.py @@ -1,3 +1,4 @@ +import inspect import typing import pytest @@ -9,6 +10,7 @@ from ably.pubsub.realtime.realtime import AblyRealtime from ably.pubsub.server import create_http_client, create_realtime_client from ably.pubsub.sync.http.http import AblyHttpSync +from ably.pubsub.sync.prototypes import PubSubHttpClient as PubSubHttpClientSync def protocol_members(protocol): @@ -26,6 +28,42 @@ def protocol_members(protocol): return set(members) +def implementation_classes(protocol, cls): + """The classes in `cls`'s MRO that implement, rather than declare, the prototype. + + The clients subclass their prototype, so every prototype member is reachable + on them whether or not they implement it — an unimplemented one resolves to + the prototype's `...` stub and silently returns None. Dropping the + prototype's own MRO leaves only the classes that can genuinely define a + member. + """ + return [k for k in cls.__mro__ if k not in protocol.__mro__] + + +def declared_member(classes, name): + """The attribute `name` as the first class in `classes` defines it, if any.""" + for k in classes: + if name in k.__dict__: + return k.__dict__[name] + return None + + +def parameter_shape(member): + """A member's callable shape: whether it awaits, and its parameters. + + Annotations are left out deliberately — the prototype spells its types as + strings under `from __future__ import annotations` while the clients + evaluate theirs, and return types are a type checker's business. Parameter + names, kinds and which ones are required are what callers depend on. + """ + fn = member.fget if isinstance(member, property) else member + parameters = [ + (p.name, p.kind, p.default is inspect.Parameter.empty) + for p in inspect.signature(fn).parameters.values() + ] + return inspect.iscoroutinefunction(fn), parameters + + class TestFactories: def test_http_factory_returns_a_client(self): client = create_http_client(token='foo') @@ -75,12 +113,41 @@ def test_sync_client_class_is_not_exported(self): assert 'AblyHttpSync' not in server_sync.__all__ +# Every prototype paired with the client declared to implement it, the +# synchronous flavour included — unasync generates both sides of that pair, +# so the pairing is worth checking rather than assuming. +PROTOTYPE_IMPLEMENTATIONS = [ + (PubSubHttpClient, AblyHttp), + (PubSubRealtimeClient, AblyRealtime), + (PubSubHttpClientSync, AblyHttpSync), +] + + class TestPrototypes: - @pytest.mark.parametrize('protocol,cls', [(PubSubHttpClient, AblyHttp), (PubSubRealtimeClient, AblyRealtime)]) - def test_client_satisfies_its_prototype(self, protocol, cls): - missing = sorted(member for member in protocol_members(protocol) if not hasattr(cls, member)) + @pytest.mark.parametrize('protocol,cls', PROTOTYPE_IMPLEMENTATIONS) + def test_client_is_declared_to_satisfy_its_prototype(self, protocol, cls): + assert protocol in cls.__mro__ + + @pytest.mark.parametrize('protocol,cls', PROTOTYPE_IMPLEMENTATIONS) + def test_client_implements_every_prototype_member(self, protocol, cls): + owners = implementation_classes(protocol, cls) + missing = sorted( + member for member in protocol_members(protocol) + if declared_member(owners, member) is None + ) assert missing == [] + @pytest.mark.parametrize('protocol,cls', PROTOTYPE_IMPLEMENTATIONS) + def test_client_members_match_the_prototype_signature(self, protocol, cls): + owners = implementation_classes(protocol, cls) + mismatched = {} + for member in sorted(protocol_members(protocol)): + expected = parameter_shape(declared_member(protocol.__mro__, member)) + actual = parameter_shape(declared_member(owners, member)) + if expected != actual: + mismatched[member] = {'prototype': expected, 'client': actual} + assert mismatched == {} + @pytest.mark.parametrize('name', ['PubSubHttpClient', 'PubSubRealtimeClient']) def test_prototypes_are_exported(self, name): assert name in server.__all__ From 9b9ca93311e7427ba78939f0a02656ab64d1634b Mon Sep 17 00:00:00 2001 From: evgeny Date: Fri, 18 Sep 2026 14:54:53 +0100 Subject: [PATCH 07/71] pubsub: name the client classes after the prototypes they implement The clients now declare their prototype as a base class, so calling them AblyHttp and AblyRealtime left the two halves of each pair reading as unrelated types. Name each class after what it is: the default implementation of its prototype. - AblyHttp -> DefaultPubSubHttpClient - AblyRealtime -> DefaultPubSubRealtimeClient - AblyHttpSync -> DefaultPubSubHttpClientSync, through unasync's rename list Neither name is exported, and both are reachable only through create_http_client and create_realtime_client; each class docstring now says as much. Tidy the 4.0.0 migration notes in passing. They no longer spell out the internal rename, which was never part of the public API. Co-Authored-By: Claude Opus 5 (1M context) --- UPDATING.md | 50 ++++++++++++------- ably/pubsub/http/auth.py | 6 +-- ably/pubsub/http/http.py | 10 ++-- ably/pubsub/prototypes.py | 6 --- ably/pubsub/realtime/channel.py | 5 +- ably/pubsub/realtime/connection.py | 4 +- ably/pubsub/realtime/connectionmanager.py | 4 +- ably/pubsub/realtime/realtime.py | 9 ++-- ably/pubsub/scripts/unasync.py | 5 +- ably/pubsub/server/__init__.py | 17 +++++-- ably/pubsub/server/sync.py | 9 +++- test/ably/http/httpinit_test.py | 2 +- test/ably/http/httptoken_test.py | 10 ++-- test/ably/realtime/realtimeconnection_test.py | 2 +- test/unit/pubsub_server_test.py | 32 ++++++------ test/uts/helpers/sandbox.py | 2 +- .../unit/channels/channel_get_message_test.py | 2 +- .../unit/channels/channel_history_test.py | 8 +-- .../channels/channel_message_versions_test.py | 2 +- .../unit/client/realtime_client_test.py | 2 +- .../unit/client/realtime_request_test.py | 2 +- .../unit/client/realtime_stats_test.py | 2 +- .../unit/client/realtime_time_test.py | 2 +- .../rest/integration/batch_presence_test.py | 2 +- .../rest/integration/push_channels_test.py | 2 +- test/uts/rest/unit/batch_presence_test.py | 2 +- test/uts/rest/unit/batch_publish_test.py | 2 +- 27 files changed, 117 insertions(+), 84 deletions(-) diff --git a/UPDATING.md b/UPDATING.md index 1077f181..8aa645c2 100644 --- a/UPDATING.md +++ b/UPDATING.md @@ -5,7 +5,7 @@ The package is now published to PyPI as `ably-pubsub-server` rather than `ably`, and it installs into `ably.pubsub` rather than `ably`. Both `ably` and `ably.pubsub` are [namespace packages](https://peps.python.org/pep-0420/) shared with the other -`ably-*` distributions, so neither exports anything of its own — the whole public +`ably-*` distributions, so neither exports anything of its own. The whole public API is reached through `ably.pubsub.server`. ### Installation @@ -44,19 +44,15 @@ Example 4.0.0 code: from ably.pubsub.server.sync import create_http_client ``` -Anything imported from a submodule gains the same `pubsub` segment — for -instance `ably.types.message` becomes `ably.pubsub.types.message`, and -`ably.sync.types.message` becomes `ably.pubsub.sync.types.message`. - -`api_version` and `lib_version` are still re-exported from -`ably.pubsub.server`, and now also live in `ably.pubsub.version`. +Where 3.x left you reaching into submodules such as `ably.types.message` or +`ably.http.paginatedresult`, import from `ably.pubsub.server` instead. It is +the whole public API, and `ably.pubsub.server.sync` is its synchronous +counterpart. The packages beneath them are internal and free to move. ### Clients are built by factories `AblyRest` and `AblyRealtime` are internal in 4.0.0 and raise `TypeError` if -constructed directly (`AblyRest` is also renamed to `AblyHttp`, to match the -factory that builds it). Build clients through the factories instead, which -take the same arguments: +constructed directly. Build clients through the factories instead, which take the same arguments: Example 3.x code: ```python @@ -66,15 +62,35 @@ realtime = AblyRealtime(key='xxx') Example 4.0.0 code: ```python -rest = create_http_client(key='xxx') -realtime = create_realtime_client(key='xxx') +pubsub_http_client = create_http_client(key='xxx') +pubsub_realtime_client = create_realtime_client(key='xxx') +``` + +### Type annotations + +The client classes are internal, so annotate against the prototypes the +factories are declared to return. Each is exported from the same module as the +factory that returns it. + +Example 3.x code: +```python +from ably import AblyRealtime, AblyRest +from ably.sync import AblyRestSync + +async def publish(client: AblyRest, name: str) -> None: ... +async def subscribe(client: AblyRealtime, name: str) -> None: ... +def publish_blocking(client: AblyRestSync, name: str) -> None: ... ``` -Where you previously annotated against `AblyRest` or `AblyRealtime`, use the -prototypes the factories return — `ably.pubsub.server.PubSubHttpClient` and -`ably.pubsub.server.PubSubRealtimeClient`. These are `typing.Protocol` definitions -describing the client surface, so `Protocol` support raises the minimum -supported Python to 3.8 (which the CI matrix and README already assumed). +Example 4.0.0 code: +```python +from ably.pubsub.server import PubSubHttpClient, PubSubRealtimeClient +from ably.pubsub.server.sync import PubSubHttpClient as PubSubHttpClientSync + +async def publish(client: PubSubHttpClient, name: str) -> None: ... +async def subscribe(client: PubSubRealtimeClient, name: str) -> None: ... +def publish_blocking(client: PubSubHttpClientSync, name: str) -> None: ... +``` ## Version 2.x to 3.0.0 diff --git a/ably/pubsub/http/auth.py b/ably/pubsub/http/auth.py index 11e4de82..aa71d517 100644 --- a/ably/pubsub/http/auth.py +++ b/ably/pubsub/http/auth.py @@ -9,8 +9,8 @@ from ably.pubsub.types.options import Options if TYPE_CHECKING: - from ably.pubsub.http.http import AblyHttp - from ably.pubsub.realtime.realtime import AblyRealtime + from ably.pubsub.http.http import DefaultPubSubHttpClient + from ably.pubsub.realtime.realtime import DefaultPubSubRealtimeClient from ably.pubsub.types.capability import Capability from ably.pubsub.types.tokendetails import TokenDetails @@ -29,7 +29,7 @@ class Method: BASIC = "BASIC" TOKEN = "TOKEN" - def __init__(self, ably: AblyHttp | AblyRealtime, options: Options): + def __init__(self, ably: DefaultPubSubHttpClient | DefaultPubSubRealtimeClient, options: Options): self.__ably = ably self.__auth_options = options self.__clock = select_clock(options) diff --git a/ably/pubsub/http/http.py b/ably/pubsub/http/http.py index 928f0679..457c0635 100644 --- a/ably/pubsub/http/http.py +++ b/ably/pubsub/http/http.py @@ -17,12 +17,16 @@ log = logging.getLogger(__name__) -class AblyHttp(PubSubHttpClient): - """Ably HTTP Client""" +class DefaultPubSubHttpClient(PubSubHttpClient): + """The default :class:`~ably.pubsub.prototypes.PubSubHttpClient` implementation. + + Built by :func:`ably.pubsub.server.create_http_client`, which is the only + way to construct one. + """ def __init__(self, key: Optional[str] = None, token: Optional[str] = None, token_details: Optional[TokenDetails] = None, **kwargs): - """Create an AblyHttp instance. + """Create an DefaultPubSubHttpClient instance. :Parameters: **Credentials** diff --git a/ably/pubsub/prototypes.py b/ably/pubsub/prototypes.py index c3596e21..f269ab7f 100644 --- a/ably/pubsub/prototypes.py +++ b/ably/pubsub/prototypes.py @@ -27,7 +27,6 @@ from ably.pubsub.http.push import Push from ably.pubsub.realtime.channel import Channels as RealtimeChannels from ably.pubsub.realtime.connection import Connection - from ably.pubsub.request.http import Http from ably.pubsub.request.paginatedresult import HttpPaginatedResponse, PaginatedResult from ably.pubsub.types.options import Options @@ -54,11 +53,6 @@ def client_id(self) -> str | None: """The client ID this client authenticates as, if any.""" ... - @property - def http(self) -> Http: - """The HTTP transport this client sends requests over.""" - ... - @property def options(self) -> Options: """The client options this client was built with.""" diff --git a/ably/pubsub/realtime/channel.py b/ably/pubsub/realtime/channel.py index 796f9d01..0dfd720b 100644 --- a/ably/pubsub/realtime/channel.py +++ b/ably/pubsub/realtime/channel.py @@ -25,7 +25,7 @@ from ably.pubsub.util.helper import Timer, is_callable_or_coroutine, validate_message_size if TYPE_CHECKING: - from ably.pubsub.realtime.realtime import AblyRealtime + from ably.pubsub.realtime.realtime import DefaultPubSubRealtimeClient log = logging.getLogger(__name__) @@ -55,7 +55,8 @@ class RealtimeChannel(EventEmitter, Channel): Unsubscribe to messages from a channel """ - def __init__(self, realtime: AblyRealtime, name: str, channel_options: ChannelOptions | None = None): + def __init__(self, realtime: DefaultPubSubRealtimeClient, name: str, + channel_options: ChannelOptions | None = None): EventEmitter.__init__(self) self.__name = name self.__realtime = realtime diff --git a/ably/pubsub/realtime/connection.py b/ably/pubsub/realtime/connection.py index d13909a9..fb7854df 100644 --- a/ably/pubsub/realtime/connection.py +++ b/ably/pubsub/realtime/connection.py @@ -12,7 +12,7 @@ from ably.pubsub.util.exceptions import AblyException if TYPE_CHECKING: - from ably.pubsub.realtime.realtime import AblyRealtime + from ably.pubsub.realtime.realtime import DefaultPubSubRealtimeClient log = logging.getLogger(__name__) @@ -40,7 +40,7 @@ class Connection(EventEmitter): # RTN4 Pings a realtime connection """ - def __init__(self, realtime: AblyRealtime): + def __init__(self, realtime: DefaultPubSubRealtimeClient): self.__realtime = realtime self.__error_reason: AblyException | None = None self.__state = ConnectionState.CONNECTING if realtime.options.auto_connect else ConnectionState.INITIALIZED diff --git a/ably/pubsub/realtime/connectionmanager.py b/ably/pubsub/realtime/connectionmanager.py index b5523a47..09ce5eab 100644 --- a/ably/pubsub/realtime/connectionmanager.py +++ b/ably/pubsub/realtime/connectionmanager.py @@ -21,7 +21,7 @@ from ably.pubsub.util.helper import Timer, get_random_id, is_token_error if TYPE_CHECKING: - from ably.pubsub.realtime.realtime import AblyRealtime + from ably.pubsub.realtime.realtime import DefaultPubSubRealtimeClient log = logging.getLogger(__name__) @@ -139,7 +139,7 @@ def response_time_ms(self) -> float: class ConnectionManager(EventEmitter): - def __init__(self, realtime: AblyRealtime, initial_state): + def __init__(self, realtime: DefaultPubSubRealtimeClient, initial_state): self.options = realtime.options self.clock = select_clock(self.options) self.__ably = realtime diff --git a/ably/pubsub/realtime/realtime.py b/ably/pubsub/realtime/realtime.py index 7948a47a..36b1118d 100644 --- a/ably/pubsub/realtime/realtime.py +++ b/ably/pubsub/realtime/realtime.py @@ -2,7 +2,7 @@ import logging from typing import Optional -from ably.pubsub.http.http import AblyHttp +from ably.pubsub.http.http import DefaultPubSubHttpClient from ably.pubsub.prototypes import PubSubRealtimeClient from ably.pubsub.realtime.channel import Channels from ably.pubsub.realtime.connection import Connection, ConnectionState @@ -11,9 +11,12 @@ log = logging.getLogger(__name__) -class AblyRealtime(AblyHttp, PubSubRealtimeClient): +class DefaultPubSubRealtimeClient(DefaultPubSubHttpClient, PubSubRealtimeClient): """ - Ably Realtime Client + The default :class:`~ably.pubsub.prototypes.PubSubRealtimeClient` implementation. + + Built by :func:`ably.pubsub.server.create_realtime_client`, which is the + only way to construct one. Attributes ---------- diff --git a/ably/pubsub/scripts/unasync.py b/ably/pubsub/scripts/unasync.py index 877dedd6..4f6c8f16 100644 --- a/ably/pubsub/scripts/unasync.py +++ b/ably/pubsub/scripts/unasync.py @@ -5,7 +5,7 @@ import tokenize_rt rename_classes = [ - "AblyHttp", + "DefaultPubSubHttpClient", "Push", "PushAdmin", "Channel", @@ -289,7 +289,8 @@ def run(): _STRING_REPLACE['httpx.AsyncClient.send'] = 'httpx.Client.send' _STRING_REPLACE['ably.pubsub.util.exceptions.AblyException.raise_for_response'] = \ 'ably.pubsub.sync.util.exceptions.AblyException.raise_for_response' - _STRING_REPLACE['ably.pubsub.http.http.AblyHttp.time'] = 'ably.pubsub.sync.http.http.AblyHttpSync.time' + _STRING_REPLACE['ably.pubsub.http.http.DefaultPubSubHttpClient.time'] = \ + 'ably.pubsub.sync.http.http.DefaultPubSubHttpClientSync.time' _STRING_REPLACE['ably.pubsub.http.auth.Auth._timestamp'] = 'ably.pubsub.sync.http.auth.AuthSync._timestamp' # round 1 diff --git a/ably/pubsub/server/__init__.py b/ably/pubsub/server/__init__.py index 8e7153d2..203b86d1 100644 --- a/ably/pubsub/server/__init__.py +++ b/ably/pubsub/server/__init__.py @@ -19,10 +19,14 @@ """ from ably.pubsub.http.auth import Auth -from ably.pubsub.http.http import AblyHttp as _AblyHttp +from ably.pubsub.http.channel import Channels as HttpChannels +from ably.pubsub.http.http import DefaultPubSubHttpClient as _DefaultPubSubHttpClient from ably.pubsub.http.push import Push from ably.pubsub.prototypes import PubSubHttpClient, PubSubRealtimeClient -from ably.pubsub.realtime.realtime import AblyRealtime as _AblyRealtime +from ably.pubsub.realtime.channel import Channels as RealtimeChannels +from ably.pubsub.realtime.connection import Connection +from ably.pubsub.realtime.realtime import DefaultPubSubRealtimeClient as _DefaultPubSubRealtimeClient +from ably.pubsub.request.paginatedresult import HttpPaginatedResponse, PaginatedResult from ably.pubsub.types.annotation import Annotation, AnnotationAction from ably.pubsub.types.capability import Capability from ably.pubsub.types.channelmode import ChannelMode @@ -157,7 +161,7 @@ def create_http_client(**kwargs) -> PubSubHttpClient: `environment`, `rest_host` or `realtime_host`. """ with factory_construction(): - return _AblyHttp(**kwargs) + return _DefaultPubSubHttpClient(**kwargs) def create_realtime_client(**kwargs) -> PubSubRealtimeClient: @@ -234,7 +238,7 @@ def create_realtime_client(**kwargs) -> PubSubRealtimeClient: `environment`, `rest_host` or `realtime_host`. """ with factory_construction(): - return _AblyRealtime(**kwargs) + return _DefaultPubSubRealtimeClient(**kwargs) __all__ = [ @@ -248,17 +252,22 @@ def create_realtime_client(**kwargs) -> PubSubRealtimeClient: 'ChannelMode', 'ChannelOptions', 'CipherParams', + 'Connection', 'DeviceDetails', + 'HttpChannels', + 'HttpPaginatedResponse', 'IncompatibleClientIdException', 'MessageAction', 'MessageOperation', 'MessageVersion', 'Options', + 'PaginatedResult', 'PubSubHttpClient', 'PubSubRealtimeClient', 'PublishResult', 'Push', 'PushChannelSubscription', + 'RealtimeChannels', 'TokenDetails', 'UpdateDeleteResult', 'VCDiffDecoder', diff --git a/ably/pubsub/server/sync.py b/ably/pubsub/server/sync.py index 19e3637f..80344076 100644 --- a/ably/pubsub/server/sync.py +++ b/ably/pubsub/server/sync.py @@ -8,9 +8,11 @@ """ from ably.pubsub.sync.http.auth import AuthSync -from ably.pubsub.sync.http.http import AblyHttpSync as _AblyHttpSync +from ably.pubsub.sync.http.channel import ChannelsSync as HttpChannelsSync +from ably.pubsub.sync.http.http import DefaultPubSubHttpClientSync as _DefaultPubSubHttpClientSync from ably.pubsub.sync.http.push import PushSync from ably.pubsub.sync.prototypes import PubSubHttpClient +from ably.pubsub.sync.request.paginatedresult import HttpPaginatedResponseSync, PaginatedResultSync from ably.pubsub.sync.types.annotation import Annotation, AnnotationAction from ably.pubsub.sync.types.capability import Capability from ably.pubsub.sync.types.channelmode import ChannelMode @@ -49,7 +51,7 @@ def create_http_client(**kwargs) -> PubSubHttpClient: `environment`, `rest_host` or `realtime_host`. """ with factory_construction(): - return _AblyHttpSync(**kwargs) + return _DefaultPubSubHttpClientSync(**kwargs) __all__ = [ @@ -64,11 +66,14 @@ def create_http_client(**kwargs) -> PubSubHttpClient: 'ChannelOptions', 'CipherParams', 'DeviceDetails', + 'HttpChannelsSync', + 'HttpPaginatedResponseSync', 'IncompatibleClientIdException', 'MessageAction', 'MessageOperation', 'MessageVersion', 'Options', + 'PaginatedResultSync', 'PubSubHttpClient', 'PublishResult', 'PushChannelSubscription', diff --git a/test/ably/http/httpinit_test.py b/test/ably/http/httpinit_test.py index e7361cb8..b8202a3b 100644 --- a/test/ably/http/httpinit_test.py +++ b/test/ably/http/httpinit_test.py @@ -168,7 +168,7 @@ async def test_query_time_param(self): use_binary_protocol=self.use_binary_protocol) timestamp = ably.auth._timestamp - with patch('ably.pubsub.http.http.AblyHttp.time', wraps=ably.time) as server_time,\ + with patch('ably.pubsub.http.http.DefaultPubSubHttpClient.time', wraps=ably.time) as server_time,\ patch('ably.pubsub.http.auth.Auth._timestamp', wraps=timestamp) as local_time: await ably.auth.request_token() assert local_time.call_count == 1 diff --git a/test/ably/http/httptoken_test.py b/test/ably/http/httptoken_test.py index 34ea56f7..2406f120 100644 --- a/test/ably/http/httptoken_test.py +++ b/test/ably/http/httptoken_test.py @@ -120,7 +120,7 @@ async def test_token_generation_with_invalid_ttl(self): async def test_token_generation_with_local_time(self): timestamp = self.ably.auth._timestamp - with patch('ably.pubsub.http.http.AblyHttp.time', wraps=self.ably.time) as server_time,\ + with patch('ably.pubsub.http.http.DefaultPubSubHttpClient.time', wraps=self.ably.time) as server_time,\ patch('ably.pubsub.http.auth.Auth._timestamp', wraps=timestamp) as local_time: await self.ably.auth.request_token() assert local_time.called @@ -129,7 +129,7 @@ async def test_token_generation_with_local_time(self): # RSA10k async def test_token_generation_with_server_time(self): timestamp = self.ably.auth._timestamp - with patch('ably.pubsub.http.http.AblyHttp.time', wraps=self.ably.time) as server_time,\ + with patch('ably.pubsub.http.http.DefaultPubSubHttpClient.time', wraps=self.ably.time) as server_time,\ patch('ably.pubsub.http.auth.Auth._timestamp', wraps=timestamp) as local_time: await self.ably.auth.request_token(query_time=True) assert local_time.call_count == 1 @@ -182,7 +182,7 @@ async def test_key_name_and_secret_are_required(self): @dont_vary_protocol async def test_with_local_time(self): timestamp = self.ably.auth._timestamp - with patch('ably.pubsub.http.http.AblyHttp.time', wraps=self.ably.time) as server_time,\ + with patch('ably.pubsub.http.http.DefaultPubSubHttpClient.time', wraps=self.ably.time) as server_time,\ patch('ably.pubsub.http.auth.Auth._timestamp', wraps=timestamp) as local_time: await self.ably.auth.create_token_request( key_name=self.key_name, key_secret=self.key_secret, query_time=False) @@ -193,7 +193,7 @@ async def test_with_local_time(self): @dont_vary_protocol async def test_with_server_time(self): timestamp = self.ably.auth._timestamp - with patch('ably.pubsub.http.http.AblyHttp.time', wraps=self.ably.time) as server_time,\ + with patch('ably.pubsub.http.http.DefaultPubSubHttpClient.time', wraps=self.ably.time) as server_time,\ patch('ably.pubsub.http.auth.Auth._timestamp', wraps=timestamp) as local_time: await self.ably.auth.create_token_request( key_name=self.key_name, key_secret=self.key_secret, query_time=True) @@ -329,7 +329,7 @@ async def test_hmac(self): # AO2g @dont_vary_protocol async def test_query_server_time(self): - with patch('ably.pubsub.http.http.AblyHttp.time', wraps=self.ably.time) as server_time: + with patch('ably.pubsub.http.http.DefaultPubSubHttpClient.time', wraps=self.ably.time) as server_time: await self.ably.auth.create_token_request( key_name=self.key_name, key_secret=self.key_secret, query_time=True) assert server_time.call_count == 1 diff --git a/test/ably/realtime/realtimeconnection_test.py b/test/ably/realtime/realtimeconnection_test.py index 464cf460..e6507e6a 100644 --- a/test/ably/realtime/realtimeconnection_test.py +++ b/test/ably/realtime/realtimeconnection_test.py @@ -67,7 +67,7 @@ async def close_active_connection(self): @property def endpoint(self) -> str: - """Host to pass to AblyRealtime (combine with tls=False and port=self.port).""" + """Host to pass to DefaultPubSubRealtimeClient (combine with tls=False and port=self.port).""" return "127.0.0.1" async def __aenter__(self): diff --git a/test/unit/pubsub_server_test.py b/test/unit/pubsub_server_test.py index b5e902f3..ef0ea243 100644 --- a/test/unit/pubsub_server_test.py +++ b/test/unit/pubsub_server_test.py @@ -5,11 +5,11 @@ import ably.pubsub.server as server import ably.pubsub.server.sync as server_sync -from ably.pubsub.http.http import AblyHttp +from ably.pubsub.http.http import DefaultPubSubHttpClient from ably.pubsub.prototypes import PubSubHttpClient, PubSubRealtimeClient -from ably.pubsub.realtime.realtime import AblyRealtime +from ably.pubsub.realtime.realtime import DefaultPubSubRealtimeClient from ably.pubsub.server import create_http_client, create_realtime_client -from ably.pubsub.sync.http.http import AblyHttpSync +from ably.pubsub.sync.http.http import DefaultPubSubHttpClientSync from ably.pubsub.sync.prototypes import PubSubHttpClient as PubSubHttpClientSync @@ -67,15 +67,15 @@ def parameter_shape(member): class TestFactories: def test_http_factory_returns_a_client(self): client = create_http_client(token='foo') - assert isinstance(client, AblyHttp) + assert isinstance(client, DefaultPubSubHttpClient) async def test_realtime_factory_returns_a_client(self): client = create_realtime_client(key='foo:bar', auto_connect=False) - assert isinstance(client, AblyRealtime) + assert isinstance(client, DefaultPubSubRealtimeClient) await client.close() def test_sync_http_factory_returns_a_client(self): - assert isinstance(server_sync.create_http_client(token='foo'), AblyHttpSync) + assert isinstance(server_sync.create_http_client(token='foo'), DefaultPubSubHttpClientSync) def test_factory_passes_options_through(self): client = create_http_client(token='foo', client_id='me') @@ -85,41 +85,41 @@ def test_factory_passes_options_through(self): class TestConstructorsAreInternal: def test_rest_constructor_is_rejected(self): with pytest.raises(TypeError) as excinfo: - AblyHttp(token='foo') + DefaultPubSubHttpClient(token='foo') assert 'ably.pubsub.server.create_http_client' in str(excinfo.value) def test_realtime_constructor_is_rejected(self): with pytest.raises(TypeError) as excinfo: - AblyRealtime(key='foo:bar', auto_connect=False) + DefaultPubSubRealtimeClient(key='foo:bar', auto_connect=False) assert 'ably.pubsub.server.create_realtime_client' in str(excinfo.value) def test_sync_rest_constructor_is_rejected(self): with pytest.raises(TypeError) as excinfo: - AblyHttpSync(token='foo') + DefaultPubSubHttpClientSync(token='foo') assert 'ably.pubsub.server.sync.create_http_client' in str(excinfo.value) def test_permission_does_not_leak_past_the_factory(self): create_http_client(token='foo') with pytest.raises(TypeError): - AblyHttp(token='foo') + DefaultPubSubHttpClient(token='foo') - @pytest.mark.parametrize('name', ['AblyHttp', 'AblyRealtime']) + @pytest.mark.parametrize('name', ['DefaultPubSubHttpClient', 'DefaultPubSubRealtimeClient']) def test_client_classes_are_not_exported(self, name): assert not hasattr(server, name) assert name not in server.__all__ def test_sync_client_class_is_not_exported(self): - assert not hasattr(server_sync, 'AblyHttpSync') - assert 'AblyHttpSync' not in server_sync.__all__ + assert not hasattr(server_sync, 'DefaultPubSubHttpClientSync') + assert 'DefaultPubSubHttpClientSync' not in server_sync.__all__ # Every prototype paired with the client declared to implement it, the # synchronous flavour included — unasync generates both sides of that pair, # so the pairing is worth checking rather than assuming. PROTOTYPE_IMPLEMENTATIONS = [ - (PubSubHttpClient, AblyHttp), - (PubSubRealtimeClient, AblyRealtime), - (PubSubHttpClientSync, AblyHttpSync), + (PubSubHttpClient, DefaultPubSubHttpClient), + (PubSubRealtimeClient, DefaultPubSubRealtimeClient), + (PubSubHttpClientSync, DefaultPubSubHttpClientSync), ] diff --git a/test/uts/helpers/sandbox.py b/test/uts/helpers/sandbox.py index 65ea66d3..a29236c3 100644 --- a/test/uts/helpers/sandbox.py +++ b/test/uts/helpers/sandbox.py @@ -4,7 +4,7 @@ POST the canonical app setup body to the sandbox, take the keys out of the response, and DELETE the app when the tests are done. -Provisioning goes over plain `httpx` rather than through `AblyHttp`. It is +Provisioning goes over plain `httpx` rather than through `DefaultPubSubHttpClient`. It is infrastructure, and a client that cannot form a request would otherwise look like a broken fixture rather than a failing test. diff --git a/test/uts/realtime/unit/channels/channel_get_message_test.py b/test/uts/realtime/unit/channels/channel_get_message_test.py index 4a03fbed..4019ea2a 100644 --- a/test/uts/realtime/unit/channels/channel_get_message_test.py +++ b/test/uts/realtime/unit/channels/channel_get_message_test.py @@ -4,7 +4,7 @@ The specification points at uts/rest/unit/channel/get_message.md (RSL11) rather than listing steps of its own, so its single Test ID becomes one test driving an -`AblyRealtime` over the HTTP mock and asserting the core observable of the derived +`DefaultPubSubRealtimeClient` over the HTTP mock and asserting the core observable of the derived REST suite: the endpoint the call reaches and the `Message` it returns. """ diff --git a/test/uts/realtime/unit/channels/channel_history_test.py b/test/uts/realtime/unit/channels/channel_history_test.py index 509e043a..3675e239 100644 --- a/test/uts/realtime/unit/channels/channel_history_test.py +++ b/test/uts/realtime/unit/channels/channel_history_test.py @@ -3,7 +3,7 @@ Spec points: RTL10, RTL10a, RTL10b, RTL10c RTL10a points at uts/rest/unit/channel/history.md (RSL2) rather than listing steps of -its own, so its Test ID becomes one test driving an `AblyRealtime` over the HTTP mock +its own, so its Test ID becomes one test driving an `DefaultPubSubRealtimeClient` over the HTTP mock and asserting the core observable of the derived REST suite. The two RTL10b tests cover `untilAttach`, which ably-python does not implement; see @@ -59,9 +59,9 @@ def on_message_from_client(msg): # UTS: realtime/unit/RTL10a/supports-rest-params-0 async def test_rtl10a_supports_rest_params(): # The specification directs uts/rest/unit/channel/history.md (RSL2) at a realtime - # channel in place of a REST one. `AblyRealtime` subclasses `AblyHttp`, so the same - # HTTP mock serves it, and this mirrors that suite's `RSL2a/returns-paginated-result-0` - # and `RSL2b/query-parameters-0`. + # channel in place of a REST one. `DefaultPubSubRealtimeClient` subclasses + # `DefaultPubSubHttpClient`, so the same HTTP mock serves it, and this mirrors that + # suite's `RSL2a/returns-paginated-result-0` and `RSL2b/query-parameters-0`. channel_name = f'test-RTL10a-{random_id()}' captured_requests = [] diff --git a/test/uts/realtime/unit/channels/channel_message_versions_test.py b/test/uts/realtime/unit/channels/channel_message_versions_test.py index 00198be0..c8ea4a5f 100644 --- a/test/uts/realtime/unit/channels/channel_message_versions_test.py +++ b/test/uts/realtime/unit/channels/channel_message_versions_test.py @@ -4,7 +4,7 @@ The specification points at uts/rest/unit/channel/message_versions.md (RSL14) rather than listing steps of its own, so its single Test ID becomes one test driving an -`AblyRealtime` over the HTTP mock and asserting the core observable of the derived +`DefaultPubSubRealtimeClient` over the HTTP mock and asserting the core observable of the derived REST suite: the endpoint the call reaches and the paginated versions it returns. """ diff --git a/test/uts/realtime/unit/client/realtime_client_test.py b/test/uts/realtime/unit/client/realtime_client_test.py index 21e6aa37..3c548191 100644 --- a/test/uts/realtime/unit/client/realtime_client_test.py +++ b/test/uts/realtime/unit/client/realtime_client_test.py @@ -144,7 +144,7 @@ async def test_rtc17_client_id_attribute(): assert client.client_id == 'explicit-client-id' # DEVIATION: the spec asserts `client.clientId == client.auth.clientId`. - # `AblyRealtime.client_id` reads the client options, while `Auth.client_id` is + # `DefaultPubSubRealtimeClient.client_id` reads the client options, while `Auth.client_id` is # held at None for a realtime client until the server confirms one in a CONNECTED # message, so the two disagree before the connection is established. assert client.auth.client_id is None diff --git a/test/uts/realtime/unit/client/realtime_request_test.py b/test/uts/realtime/unit/client/realtime_request_test.py index 9765194f..858e9df4 100644 --- a/test/uts/realtime/unit/client/realtime_request_test.py +++ b/test/uts/realtime/unit/client/realtime_request_test.py @@ -17,7 +17,7 @@ # UTS: realtime/unit/RTC9/request-proxies-rest-0 async def test_rtc9_request_proxies_rest(): # The specification directs uts/rest/unit/request.md (RSC19) at a realtime client - # in place of a REST one. `AblyRealtime` subclasses `AblyHttp`, so the same HTTP + # in place of a REST one. `DefaultPubSubRealtimeClient` subclasses `DefaultPubSubHttpClient`, so the same HTTP # mock serves it, and this mirrors that suite's `RSC19f/supports-http-methods-0` # and `RSC19d/response-items-decoded-5`. captured_requests = [] diff --git a/test/uts/realtime/unit/client/realtime_stats_test.py b/test/uts/realtime/unit/client/realtime_stats_test.py index 1138c8d2..a4fd40be 100644 --- a/test/uts/realtime/unit/client/realtime_stats_test.py +++ b/test/uts/realtime/unit/client/realtime_stats_test.py @@ -30,7 +30,7 @@ # UTS: realtime/unit/RTC5/stats-proxies-rest-0 async def test_rtc5_stats_proxies_rest(): # The specification directs uts/rest/unit/stats.md (RSC6) at a realtime client - # in place of a REST one. `AblyRealtime` subclasses `AblyHttp`, so the same HTTP + # in place of a REST one. `DefaultPubSubRealtimeClient` subclasses `DefaultPubSubHttpClient`, so the same HTTP # mock serves it, and this mirrors that suite's `RSC6a/returns-paginated-stats-0`. captured_requests = [] diff --git a/test/uts/realtime/unit/client/realtime_time_test.py b/test/uts/realtime/unit/client/realtime_time_test.py index 435de0a4..162336ed 100644 --- a/test/uts/realtime/unit/client/realtime_time_test.py +++ b/test/uts/realtime/unit/client/realtime_time_test.py @@ -12,7 +12,7 @@ # UTS: realtime/unit/RTC6/time-proxies-rest-0 async def test_rtc6_time_proxies_rest(): # The specification directs uts/rest/unit/time.md (RSC16) at a realtime client - # in place of a REST one. `AblyRealtime` subclasses `AblyHttp`, so the same + # in place of a REST one. `DefaultPubSubRealtimeClient` subclasses `DefaultPubSubHttpClient`, so the same # HTTP mock serves it, and this mirrors that suite's `RSC16/returns-server-time-0`. captured_requests = [] diff --git a/test/uts/rest/integration/batch_presence_test.py b/test/uts/rest/integration/batch_presence_test.py index e846e276..b9d4e7af 100644 --- a/test/uts/rest/integration/batch_presence_test.py +++ b/test/uts/rest/integration/batch_presence_test.py @@ -2,7 +2,7 @@ Spec points: RSC24, BGR2, BGF2 -DEVIATION: ably-python has no batch API. `AblyHttp` exposes no `batch_presence`, and +DEVIATION: ably-python has no batch API. `DefaultPubSubHttpClient` exposes no `batch_presence`, and the package defines neither `BatchResult` nor `BatchPresenceSuccessResult` / `BatchPresenceFailureResult`; the word "batch" appears nowhere under `ably/`. Every test here therefore departs from the specification and is gated behind diff --git a/test/uts/rest/integration/push_channels_test.py b/test/uts/rest/integration/push_channels_test.py index 109daa4e..79b60a65 100644 --- a/test/uts/rest/integration/push_channels_test.py +++ b/test/uts/rest/integration/push_channels_test.py @@ -4,7 +4,7 @@ DEVIATION: ably-python implements neither the PushChannel interface (RSH7, the `push` field on a channel) nor LocalDevice (RSH8). `ably/rest/channel.py` gives a channel no -`push`, `AblyHttp` no `device`, and `ably/types/device.py` defines only `DeviceDetails`. +`push`, `DefaultPubSubHttpClient` no `device`, and `ably/types/device.py` defines only `DeviceDetails`. Both tests here therefore depart from the specification and are gated behind RUN_DEVIATIONS, against the same spelling [test/uts/rest/unit/push/push_channels_test.py](../unit/push/push_channels_test.py) diff --git a/test/uts/rest/unit/batch_presence_test.py b/test/uts/rest/unit/batch_presence_test.py index b085f453..599a3892 100644 --- a/test/uts/rest/unit/batch_presence_test.py +++ b/test/uts/rest/unit/batch_presence_test.py @@ -2,7 +2,7 @@ Spec points: RSC24, BAR2, BGR2, BGF2 -NOTE: ably-python has no batch API. `AblyHttp` exposes no `batch_presence`, and the package +NOTE: ably-python has no batch API. `DefaultPubSubHttpClient` exposes no `batch_presence`, and the package defines neither `BatchResult`/`BatchPresenceResponse` nor `BatchPresenceSuccessResult`/ `BatchPresenceFailureResult`; the word "batch" appears nowhere under `ably/`. Every test in this file therefore departs from the specification and is gated behind `RUN_DEVIATIONS`. diff --git a/test/uts/rest/unit/batch_publish_test.py b/test/uts/rest/unit/batch_publish_test.py index 96b89163..a4765c2f 100644 --- a/test/uts/rest/unit/batch_publish_test.py +++ b/test/uts/rest/unit/batch_publish_test.py @@ -2,7 +2,7 @@ Spec points: RSC22c, RSC22d, BSP2a, BSP2b, BPR2a, BPR2b, BPR2c, BPF2a, BPF2b -NOTE: ably-python has no batch API. `AblyHttp` exposes no `batch_publish`, and the package +NOTE: ably-python has no batch API. `DefaultPubSubHttpClient` exposes no `batch_publish`, and the package defines neither `BatchPublishSpec` nor `BatchResult`/`BatchPublishSuccessResult`/ `BatchPublishFailureResult`; the word "batch" appears nowhere under `ably/`. Every test in this file therefore departs from the specification and is gated behind `RUN_DEVIATIONS`. From 4a869cf5f29d6a643bf8b7b7b9d9941eec3c64a7 Mon Sep 17 00:00:00 2001 From: evgeny Date: Tue, 6 Oct 2026 13:12:00 +0100 Subject: [PATCH 08/71] test: accept ably-pubsub-python as the Ably-Agent library name The library now reports itself as `ably-pubsub-python/`, which the two Ably-Agent tests that came in from main did not expect. - The mock websocket helper test looked for `ably-python` and now looks for the new name. - `RSC7d/ably-agent-header-format-0` asserts `ably-[a-z]+/`, which rejects a hyphenated library name. RSC7d1 puts no limit on the characters of a product key, so the specification is at fault rather than the SDK. The test is gated with `@spec_error` and the fault is recorded in deviations.md. `update_message()` takes a `Message`, and the migration guide tells users to import types like it from `ably.pubsub.server`, but neither server package exported it. Export it from both. The README passed the API key positionally to `create_realtime_client`, which accepts keyword arguments only and so raised TypeError. Pass it as `key=`, and say in the migration guide that the factories no longer accept a positional key. The release job still read the version from `dist/ably-*.tar.gz` and pointed both environments at `/p/ably`. The sdist is now `ably_pubsub_server-.tar.gz`, so the tag check would never have matched it. Co-Authored-By: Claude Opus 5.5 (1M context) --- .github/workflows/release.yml | 6 +++--- README.md | 2 +- UPDATING.md | 3 ++- ably/pubsub/server/__init__.py | 3 ++- ably/pubsub/server/sync.py | 3 ++- test/unit/pubsub_server_test.py | 7 +++++++ test/uts/deviations.md | 14 ++++++++++++-- test/uts/helpers/mock_websocket_test.py | 2 +- test/uts/rest/unit/rest_client_test.py | 5 +++++ 9 files changed, 35 insertions(+), 10 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 69b4c6d3..f40e27e9 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -74,7 +74,7 @@ jobs: runs-on: ubuntu-latest environment: name: pypi - url: https://pypi.org/p/ably + url: https://pypi.org/p/ably-pubsub-server permissions: id-token: write # IMPORTANT: mandatory for trusted publishing @@ -94,7 +94,7 @@ jobs: - name: Read VERSION_NAME from dist/ id: version run: | - VERSION_NAME=$(basename dist/ably-*.tar.gz | sed -E 's/^ably-([^-]+)\.tar\.gz$/\1/') + VERSION_NAME=$(basename dist/ably_pubsub_server-*.tar.gz | sed -E 's/^ably_pubsub_server-([^-]+)\.tar\.gz$/\1/') echo "version=$VERSION_NAME" >> $GITHUB_OUTPUT - name: Compare version with tag @@ -118,7 +118,7 @@ jobs: environment: name: testpypi - url: https://test.pypi.org/p/ably + url: https://test.pypi.org/p/ably-pubsub-server permissions: id-token: write # IMPORTANT: mandatory for trusted publishing diff --git a/README.md b/README.md index 78c0058e..e833df38 100644 --- a/README.md +++ b/README.md @@ -84,7 +84,7 @@ The following code connects to Ably's realtime messaging service, subscribes to from ably.pubsub.server import create_realtime_client # Initialize Ably Realtime client -async with create_realtime_client('your-ably-api-key', client_id='me') as realtime_client: +async with create_realtime_client(key='your-ably-api-key', client_id='me') as realtime_client: # Wait for connection to be established await realtime_client.connection.once_async('connected') print('Connected to Ably') diff --git a/UPDATING.md b/UPDATING.md index 8aa645c2..a2aa7db7 100644 --- a/UPDATING.md +++ b/UPDATING.md @@ -52,7 +52,8 @@ counterpart. The packages beneath them are internal and free to move. ### Clients are built by factories `AblyRest` and `AblyRealtime` are internal in 4.0.0 and raise `TypeError` if -constructed directly. Build clients through the factories instead, which take the same arguments: +constructed directly. Build clients through the factories instead, which take the same options +as keyword arguments. The API key is no longer accepted positionally, so pass it as `key=`: Example 3.x code: ```python diff --git a/ably/pubsub/server/__init__.py b/ably/pubsub/server/__init__.py index 203b86d1..41efc61e 100644 --- a/ably/pubsub/server/__init__.py +++ b/ably/pubsub/server/__init__.py @@ -33,7 +33,7 @@ from ably.pubsub.types.channeloptions import ChannelOptions from ably.pubsub.types.channelsubscription import PushChannelSubscription from ably.pubsub.types.device import DeviceDetails -from ably.pubsub.types.message import MessageAction, MessageVersion +from ably.pubsub.types.message import Message, MessageAction, MessageVersion from ably.pubsub.types.operations import MessageOperation, PublishResult, UpdateDeleteResult from ably.pubsub.types.options import Options, VCDiffDecoder from ably.pubsub.types.tokendetails import TokenDetails @@ -257,6 +257,7 @@ def create_realtime_client(**kwargs) -> PubSubRealtimeClient: 'HttpChannels', 'HttpPaginatedResponse', 'IncompatibleClientIdException', + 'Message', 'MessageAction', 'MessageOperation', 'MessageVersion', diff --git a/ably/pubsub/server/sync.py b/ably/pubsub/server/sync.py index 80344076..5ccab431 100644 --- a/ably/pubsub/server/sync.py +++ b/ably/pubsub/server/sync.py @@ -19,7 +19,7 @@ from ably.pubsub.sync.types.channeloptions import ChannelOptions from ably.pubsub.sync.types.channelsubscription import PushChannelSubscription from ably.pubsub.sync.types.device import DeviceDetails -from ably.pubsub.sync.types.message import MessageAction, MessageVersion +from ably.pubsub.sync.types.message import Message, MessageAction, MessageVersion from ably.pubsub.sync.types.operations import MessageOperation, PublishResult, UpdateDeleteResult from ably.pubsub.sync.types.options import Options, VCDiffDecoder from ably.pubsub.sync.types.tokendetails import TokenDetails @@ -69,6 +69,7 @@ def create_http_client(**kwargs) -> PubSubHttpClient: 'HttpChannelsSync', 'HttpPaginatedResponseSync', 'IncompatibleClientIdException', + 'Message', 'MessageAction', 'MessageOperation', 'MessageVersion', diff --git a/test/unit/pubsub_server_test.py b/test/unit/pubsub_server_test.py index ef0ea243..a06d2582 100644 --- a/test/unit/pubsub_server_test.py +++ b/test/unit/pubsub_server_test.py @@ -154,3 +154,10 @@ def test_prototypes_are_exported(self, name): def test_sync_prototype_is_exported(self): assert 'PubSubHttpClient' in server_sync.__all__ + + +class TestPublicTypes: + # update_message() takes a Message, so callers need a public way to build one + def test_message_is_exported(self): + assert 'Message' in server.__all__ + assert 'Message' in server_sync.__all__ diff --git a/test/uts/deviations.md b/test/uts/deviations.md index bf08ee74..498f0ad3 100644 --- a/test/uts/deviations.md +++ b/test/uts/deviations.md @@ -75,7 +75,7 @@ from the specification text, so that correcting the specification is all it take it pass, and it is marked `@spec_error` — a skip gated on `RUN_DEVIATIONS`, the same gate `@deviation` uses, with a reason naming the specification rather than the SDK. The suite stays green, a real regression still shows, and the failure is one environment variable -away. Each is filed upstream, in the issues named below. Ten tests are gated +away. Each is filed upstream, in the issues named below. Eleven tests are gated this way: | Test | Spec error | @@ -90,6 +90,7 @@ this way: | `test_rsp4_history_pagination` | RSP4 - wire action 4 asserted to be LEAVE | | `test_tp3_presence_to_json` | TP3 - an outgoing action asserted as the string `"enter"` | | `test_tp3_null_attributes_omitted` | TP3 - the same outgoing string assertion | +| `test_rsc7d_ably_agent_header_format` | RSC7d - a library-name pattern that rejects hyphens | Where instead only a specification's *fixture*, *setup* or *label* is at fault, the assertion it carries still stands. Those tests keep the corrected fixture (or the @@ -139,7 +140,7 @@ RSA4c3 contradiction, since that issue is what decides it. `#547` to `#550` are the `uts/rest/integration` faults and `#551` to `#554` the `uts/realtime/integration` ones. Each of them is of the second kind — a fixture, a setup step or a header label — so the derived test keeps the corrected fixture and passes, and -none of them is among the ten gated above. +none of them is among the eleven gated above. Not every entry has an issue of its own: the URL-safe base64 alphabet is recorded below and not filed, because ably-python's own encoding settles the tests either way. Line @@ -153,6 +154,15 @@ references in these entries are against `ably/specification@d9a04ca`. returns a one-element array, which is what `time.md` itself uses and what RSC16 describes. Any SDK that indexes the array raises. +### RSC7d's Agent pattern rejects a hyphenated library name + +`rest_client.md`'s `RSC7d/ably-agent-header-format-0` asserts that the `Ably-Agent` +header matches `ably-[a-z]+/[0-9]+\.[0-9]+\.[0-9]+`. RSC7d1 asks only for a series of +`key[/value]` entries that include the library's name and version, and puts no limit on +the characters a key may hold. ably-python reports itself as `ably-pubsub-python/`, +which RSC7d1 allows and the pattern rejects. The test is gated with `@spec_error`. The +fault is not filed upstream yet. + ### Spec points are mislabelled across four specs The assertions are sound; the points they are filed under are not. diff --git a/test/uts/helpers/mock_websocket_test.py b/test/uts/helpers/mock_websocket_test.py index 39e7e098..6ecb74a3 100644 --- a/test/uts/helpers/mock_websocket_test.py +++ b/test/uts/helpers/mock_websocket_test.py @@ -189,7 +189,7 @@ async def test_a_pending_connection_carries_the_headers_the_client_sent(): headers = mock.connection_attempts[0].headers assert 'ably-agent' in headers - assert 'ably-python' in headers['Ably-Agent'] + assert 'ably-pubsub-python' in headers['Ably-Agent'] async def test_the_protocol_follows_use_binary_protocol(): diff --git a/test/uts/rest/unit/rest_client_test.py b/test/uts/rest/unit/rest_client_test.py index 1a19adb9..fd3454c5 100644 --- a/test/uts/rest/unit/rest_client_test.py +++ b/test/uts/rest/unit/rest_client_test.py @@ -11,6 +11,7 @@ from ably.pubsub.http.auth import Auth from ably.pubsub.util.exceptions import AblyException from test.uts.helpers.client import rest_client +from test.uts.helpers.deviations import spec_error from test.uts.helpers.mock_http import MockHttpClient SERVER_TIME_MS = 1234567890000 @@ -56,6 +57,10 @@ def on_request(request): # UTS: rest/unit/RSC7d/ably-agent-header-format-0 +# SPEC ERROR RSC7d: the pattern `ably-[a-z]+/` rejects a hyphenated library name. +# RSC7d1 allows any product key, and ably-python reports itself as +# `ably-pubsub-python`. Recorded in deviations.md. Fix the specification first. +@spec_error async def test_rsc7d_ably_agent_header_format(): mock_http = MockHttpClient(on_connection_attempt=connect_successfully) mock_http.queue_response(200, TIME_RESPONSE) From 1059edcd5cfb6e134eacf740b6c433849e530c6c Mon Sep 17 00:00:00 2001 From: owenpearson Date: Thu, 8 Oct 2026 13:42:52 +0100 Subject: [PATCH 09/71] feat: add batch publish and batch presence to the HTTP client `batch_publish` (RSC22) posts one `BatchPublishSpec`, or a list of them, to `/messages`, encoding each message per RSL4 and applying RSL1k1 to each spec separately when idempotent REST publishing is on (RSC22d). The server answers with an array holding a `BatchResult` per spec, so a single spec gets the one element back and a list gets the list. A spec naming no channels or carrying no messages is refused locally with the 400/40000 the server would answer it with. `batch_presence` (RSC24) sends the channel names comma-joined in the `channels` parameter of a GET to `/presence`, and returns the server's `BatchResult`. The server leaves `presence` out for a channel with no members, which is read as an empty list. Both methods are declared on `PubSubHttpClient`, so the realtime and synchronous clients carry them too. `BatchResult`, `BatchPublishSpec` and the four per-channel result types (BAR2, BSP2, BPR2, BPF2, BGR2, BGF2) are exported from `ably.pubsub.server` and `ably.pubsub.server.sync`. A dict with `channels` and `messages` keys stands in for a `BatchPublishSpec`, as a dict does for the push admin types. The RSL1k1 id assignment moves out of `Channel` into `assign_idempotent_ids`, which both publish paths share. Co-Authored-By: Claude Opus 5.5 (1M context) --- ably/pubsub/http/channel.py | 9 +- ably/pubsub/http/http.py | 69 +++++++++- ably/pubsub/prototypes.py | 10 ++ ably/pubsub/server/__init__.py | 14 +++ ably/pubsub/server/sync.py | 14 +++ ably/pubsub/types/batch.py | 208 +++++++++++++++++++++++++++++++ ably/pubsub/types/message.py | 10 ++ test/ably/http/httpbatch_test.py | 112 +++++++++++++++++ test/unit/batch_test.py | 38 ++++++ 9 files changed, 476 insertions(+), 8 deletions(-) create mode 100644 ably/pubsub/types/batch.py create mode 100644 test/ably/http/httpbatch_test.py create mode 100644 test/unit/batch_test.py diff --git a/ably/pubsub/http/channel.py b/ably/pubsub/http/channel.py index 531a17cd..321ea275 100644 --- a/ably/pubsub/http/channel.py +++ b/ably/pubsub/http/channel.py @@ -1,7 +1,5 @@ -import base64 import json import logging -import os from collections import OrderedDict from typing import Iterator, Optional from urllib import parse @@ -15,6 +13,7 @@ Message, MessageAction, MessageVersion, + assign_idempotent_ids, make_message_response_handler, make_single_message_response_handler, ) @@ -58,11 +57,7 @@ def __publish_request_body(self, messages): """ # Idempotent publishing if self.ably.options.idempotent_rest_publishing: - # RSL1k1 - if all(message.id is None for message in messages): - base_id = base64.urlsafe_b64encode(os.urandom(12)).decode() - for serial, message in enumerate(messages): - message.id = f'{base_id}:{serial}' + assign_idempotent_ids(messages) request_body_list = [] for m in messages: diff --git a/ably/pubsub/http/http.py b/ably/pubsub/http/http.py index 457c0635..7d912d0b 100644 --- a/ably/pubsub/http/http.py +++ b/ably/pubsub/http/http.py @@ -1,13 +1,23 @@ +import json import logging -from typing import Optional +from typing import List, Optional, Union from urllib.parse import urlencode +import msgpack + from ably.pubsub.http.auth import Auth from ably.pubsub.http.channel import Channels from ably.pubsub.http.push import Push from ably.pubsub.prototypes import PubSubHttpClient from ably.pubsub.request.http import Http from ably.pubsub.request.paginatedresult import HttpPaginatedResponse, PaginatedResult, format_params +from ably.pubsub.types.batch import ( + BatchPublishSpec, + BatchResult, + batch_presence_result_from_dict, + batch_publish_result_from_dict, +) +from ably.pubsub.types.message import assign_idempotent_ids from ably.pubsub.types.options import Options from ably.pubsub.types.stats import stats_response_processor from ably.pubsub.types.tokendetails import TokenDetails @@ -104,6 +114,63 @@ async def time(self, timeout: Optional[float] = None) -> float: AblyException.raise_for_response(r) return r.to_native()[0] + async def batch_publish(self, specs) -> Union[BatchResult, List[BatchResult]]: + """Publishes messages to several channels in a single request (RSC22) + + :Parameters: + - `specs`: a `BatchPublishSpec`, or a list of them. A dict with + `channels` and `messages` keys may stand in for a `BatchPublishSpec`. + + Returns a `BatchResult` holding a `BatchPublishSuccessResult` or a + `BatchPublishFailureResult` for each channel the spec names, or, given + a list of specs, a list of `BatchResult`s in the same order. + """ + single = not isinstance(specs, (list, tuple)) + specs = [BatchPublishSpec.factory(spec) for spec in ([specs] if single else specs)] + + for spec in specs: + if not spec.channels: + raise AblyException('A batch publish spec must name at least one channel', 400, 40000) + if not spec.messages: + raise AblyException('A batch publish spec must include at least one message', 400, 40000) + + # RSC22d: RSL1k1 applies to each spec separately + if self.options.idempotent_rest_publishing: + for spec in specs: + assign_idempotent_ids(spec.messages) + + binary = self.options.use_binary_protocol + body = [spec.as_dict(binary=binary) for spec in specs] + if single: + body = body[0] + if binary: + body = msgpack.packb(body, use_bin_type=True) + else: + body = json.dumps(body, separators=(',', ':')) + + response = await self.http.post('/messages', body=body) + + # RSC22b: the response is an array with a result for each spec, even when a single spec was sent + results = [BatchResult.from_dict(result, batch_publish_result_from_dict) + for result in response.to_native()] + return results[0] if single else results + + async def batch_presence(self, channels: List[str]) -> BatchResult: + """Retrieves the members present on several channels in a single request (RSC24) + + :Parameters: + - `channels`: the names of the channels + + Returns a `BatchResult` holding a `BatchPresenceSuccessResult` or a + `BatchPresenceFailureResult` for each channel. + """ + if isinstance(channels, str): + raise TypeError('Unexpected str channels, expected a list of channel names') + + path = '/presence?' + urlencode({'channels': ','.join(channels)}) + response = await self.http.get(path) + return BatchResult.from_dict(response.to_native(), batch_presence_result_from_dict) + @property def client_id(self) -> Optional[str]: return self.options.client_id diff --git a/ably/pubsub/prototypes.py b/ably/pubsub/prototypes.py index f269ab7f..4fc2613e 100644 --- a/ably/pubsub/prototypes.py +++ b/ably/pubsub/prototypes.py @@ -28,6 +28,7 @@ from ably.pubsub.realtime.channel import Channels as RealtimeChannels from ably.pubsub.realtime.connection import Connection from ably.pubsub.request.paginatedresult import HttpPaginatedResponse, PaginatedResult + from ably.pubsub.types.batch import BatchPublishSpec, BatchResult from ably.pubsub.types.options import Options @@ -72,6 +73,15 @@ async def time(self, timeout: float | None = None) -> float: """Return the current server time in ms since the unix epoch.""" ... + async def batch_publish(self, specs: BatchPublishSpec | dict | list[BatchPublishSpec | dict] + ) -> BatchResult | list[BatchResult]: + """Publish messages to one or more channels in a single request.""" + ... + + async def batch_presence(self, channels: list[str]) -> BatchResult: + """Return the presence members of several channels in a single request.""" + ... + async def request(self, method: str, path: str, version: str, params: dict | None = None, body=None, headers=None) -> HttpPaginatedResponse: """Make an arbitrary request against the Ably REST API.""" diff --git a/ably/pubsub/server/__init__.py b/ably/pubsub/server/__init__.py index 41efc61e..dcc0f2bf 100644 --- a/ably/pubsub/server/__init__.py +++ b/ably/pubsub/server/__init__.py @@ -28,6 +28,14 @@ from ably.pubsub.realtime.realtime import DefaultPubSubRealtimeClient as _DefaultPubSubRealtimeClient from ably.pubsub.request.paginatedresult import HttpPaginatedResponse, PaginatedResult from ably.pubsub.types.annotation import Annotation, AnnotationAction +from ably.pubsub.types.batch import ( + BatchPresenceFailureResult, + BatchPresenceSuccessResult, + BatchPublishFailureResult, + BatchPublishSpec, + BatchPublishSuccessResult, + BatchResult, +) from ably.pubsub.types.capability import Capability from ably.pubsub.types.channelmode import ChannelMode from ably.pubsub.types.channeloptions import ChannelOptions @@ -248,6 +256,12 @@ def create_realtime_client(**kwargs) -> PubSubRealtimeClient: 'Annotation', 'AnnotationAction', 'Auth', + 'BatchPresenceFailureResult', + 'BatchPresenceSuccessResult', + 'BatchPublishFailureResult', + 'BatchPublishSpec', + 'BatchPublishSuccessResult', + 'BatchResult', 'Capability', 'ChannelMode', 'ChannelOptions', diff --git a/ably/pubsub/server/sync.py b/ably/pubsub/server/sync.py index 5ccab431..d57b5cd0 100644 --- a/ably/pubsub/server/sync.py +++ b/ably/pubsub/server/sync.py @@ -14,6 +14,14 @@ from ably.pubsub.sync.prototypes import PubSubHttpClient from ably.pubsub.sync.request.paginatedresult import HttpPaginatedResponseSync, PaginatedResultSync from ably.pubsub.sync.types.annotation import Annotation, AnnotationAction +from ably.pubsub.sync.types.batch import ( + BatchPresenceFailureResult, + BatchPresenceSuccessResult, + BatchPublishFailureResult, + BatchPublishSpec, + BatchPublishSuccessResult, + BatchResult, +) from ably.pubsub.sync.types.capability import Capability from ably.pubsub.sync.types.channelmode import ChannelMode from ably.pubsub.sync.types.channeloptions import ChannelOptions @@ -61,6 +69,12 @@ def create_http_client(**kwargs) -> PubSubHttpClient: 'Annotation', 'AnnotationAction', 'AuthSync', + 'BatchPresenceFailureResult', + 'BatchPresenceSuccessResult', + 'BatchPublishFailureResult', + 'BatchPublishSpec', + 'BatchPublishSuccessResult', + 'BatchResult', 'Capability', 'ChannelMode', 'ChannelOptions', diff --git a/ably/pubsub/types/batch.py b/ably/pubsub/types/batch.py new file mode 100644 index 00000000..d6d37735 --- /dev/null +++ b/ably/pubsub/types/batch.py @@ -0,0 +1,208 @@ +from typing import Generic, List, TypeVar + +from ably.pubsub.types.presence import PresenceMessage +from ably.pubsub.util.exceptions import AblyException + +T = TypeVar('T') + + +class BatchResult(Generic[T]): + """The results of a batch operation, one per channel (BAR2).""" + + def __init__(self, success_count: int, failure_count: int, results: List[T]): + self.__success_count = success_count + self.__failure_count = failure_count + self.__results = results + + @property + def success_count(self) -> int: + """The number of successful operations (BAR2a).""" + return self.__success_count + + @property + def failure_count(self) -> int: + """The number of unsuccessful operations (BAR2b).""" + return self.__failure_count + + @property + def results(self) -> List[T]: + """The result of each operation in the batch (BAR2c).""" + return self.__results + + @staticmethod + def from_dict(obj, result_from_dict): + """Create a BatchResult from a response envelope, building each result with `result_from_dict`.""" + return BatchResult( + success_count=obj.get('successCount'), + failure_count=obj.get('failureCount'), + results=[result_from_dict(result) for result in obj.get('results') or []], + ) + + +class BatchPublishSpec: + """The messages a batch publish sends, and the channels it sends every one of them to (BSP2).""" + + def __init__(self, channels, messages): + """ + Args: + channels: The names of the channels to publish the messages to (BSP2a). + messages: The `Message` objects to publish to each channel (BSP2b). + """ + self.__channels = channels + self.__messages = messages + + @property + def channels(self): + return self.__channels + + @property + def messages(self): + return self.__messages + + def as_dict(self, binary=False): + """Convert BatchPublishSpec to the wire format, encoding each message per RSL4.""" + return { + 'channels': list(self.channels), + 'messages': [message.as_dict(binary=binary) for message in self.messages], + } + + @classmethod + def factory(cls, spec): + """A BatchPublishSpec from either an instance or a dict with `channels` and `messages` keys.""" + if isinstance(spec, cls): + return spec + if isinstance(spec, dict): + return cls(channels=spec.get('channels'), messages=spec.get('messages')) + raise TypeError(f'Unexpected {type(spec)} batch publish spec, expected a BatchPublishSpec or a dict') + + +class BatchPublishSuccessResult: + """The result of publishing a spec's messages to one channel (BPR2).""" + + def __init__(self, channel, message_id, serials): + """ + Args: + channel: The name of the channel (BPR2a). + message_id: The id prefix shared by the published messages (BPR2b). + serials: The serial of each published message, or None where a conflation rule discarded it (BPR2c). + """ + self.__channel = channel + self.__message_id = message_id + self.__serials = serials + + @property + def channel(self): + return self.__channel + + @property + def message_id(self): + return self.__message_id + + @property + def serials(self): + return self.__serials + + @staticmethod + def from_dict(obj): + return BatchPublishSuccessResult( + channel=obj.get('channel'), + message_id=obj.get('messageId'), + serials=obj.get('serials') or [], + ) + + +class BatchPublishFailureResult: + """The reason a spec's messages could not be published to one channel (BPF2).""" + + def __init__(self, channel, error): + """ + Args: + channel: The name of the channel (BPF2a). + error: An `AblyException` describing why the publish failed (BPF2b). + """ + self.__channel = channel + self.__error = error + + @property + def channel(self): + return self.__channel + + @property + def error(self): + return self.__error + + @staticmethod + def from_dict(obj): + return BatchPublishFailureResult( + channel=obj.get('channel'), + error=AblyException.from_dict(obj['error']), + ) + + +class BatchPresenceSuccessResult: + """The members present on one channel of a batch presence request (BGR2).""" + + def __init__(self, channel, presence): + """ + Args: + channel: The name of the channel (BGR2a). + presence: A `PresenceMessage` for each member present on the channel (BGR2b). + """ + self.__channel = channel + self.__presence = presence + + @property + def channel(self): + return self.__channel + + @property + def presence(self): + return self.__presence + + @staticmethod + def from_dict(obj): + # The server leaves `presence` out for a channel with no members + return BatchPresenceSuccessResult( + channel=obj.get('channel'), + presence=PresenceMessage.from_encoded_array(obj.get('presence') or []), + ) + + +class BatchPresenceFailureResult: + """The reason the members of one channel of a batch presence request could not be retrieved (BGF2).""" + + def __init__(self, channel, error): + """ + Args: + channel: The name of the channel (BGF2a). + error: An `AblyException` describing why the request failed for the channel (BGF2b). + """ + self.__channel = channel + self.__error = error + + @property + def channel(self): + return self.__channel + + @property + def error(self): + return self.__error + + @staticmethod + def from_dict(obj): + return BatchPresenceFailureResult( + channel=obj.get('channel'), + error=AblyException.from_dict(obj['error']), + ) + + +def batch_publish_result_from_dict(obj): + if obj.get('error') is not None: + return BatchPublishFailureResult.from_dict(obj) + return BatchPublishSuccessResult.from_dict(obj) + + +def batch_presence_result_from_dict(obj): + if obj.get('error') is not None: + return BatchPresenceFailureResult.from_dict(obj) + return BatchPresenceSuccessResult.from_dict(obj) diff --git a/ably/pubsub/types/message.py b/ably/pubsub/types/message.py index 93a20f0c..ad574154 100644 --- a/ably/pubsub/types/message.py +++ b/ably/pubsub/types/message.py @@ -1,4 +1,6 @@ +import base64 import logging +import os from enum import IntEnum from ably.pubsub.types.mixins import DeltaExtras, EncodeDataMixin @@ -391,6 +393,14 @@ def update_inner_message_fields(proto_msg: dict): msg_index = msg_index + 1 +def assign_idempotent_ids(messages): + """Gives every message a library-generated id, unless one of them already has an id (RSL1k1).""" + if all(message.id is None for message in messages): + base_id = base64.urlsafe_b64encode(os.urandom(12)).decode() + for serial, message in enumerate(messages): + message.id = f'{base_id}:{serial}' + + def make_message_response_handler(cipher): def encrypted_message_response_handler(response): messages = response.to_native() diff --git a/test/ably/http/httpbatch_test.py b/test/ably/http/httpbatch_test.py new file mode 100644 index 00000000..f362d5cd --- /dev/null +++ b/test/ably/http/httpbatch_test.py @@ -0,0 +1,112 @@ +import logging + +import pytest + +from ably.pubsub.server import AblyException +from ably.pubsub.types.batch import ( + BatchPresenceSuccessResult, + BatchPublishFailureResult, + BatchPublishSpec, + BatchPublishSuccessResult, + BatchResult, +) +from ably.pubsub.types.message import Message +from test.ably.testapp import TestApp +from test.ably.utils import BaseAsyncTestCase, VaryByProtocolTestsMetaclass + +log = logging.getLogger(__name__) + + +class TestHttpBatch(BaseAsyncTestCase, metaclass=VaryByProtocolTestsMetaclass): + + @pytest.fixture(autouse=True) + async def setup(self): + self.test_vars = await TestApp.get_test_vars() + self.ably = await TestApp.get_ably_rest() + # keys[2] may publish to `canpublish:*` and to nothing else + self.ably_restricted = await TestApp.get_ably_rest(key=self.test_vars['keys'][2]['key_str']) + yield + await self.ably.close() + await self.ably_restricted.close() + + def per_protocol_setup(self, use_binary_protocol): + self.ably.options.use_binary_protocol = use_binary_protocol + self.ably_restricted.options.use_binary_protocol = use_binary_protocol + self.use_binary_protocol = use_binary_protocol + + async def test_batch_publish_single_spec(self): + channel_names = [self.get_channel_name('persisted:batch_publish_a'), + self.get_channel_name('persisted:batch_publish_b')] + + result = await self.ably.batch_publish(BatchPublishSpec(channels=channel_names, messages=[ + Message('string', 'This is a string message payload'), + Message('binary', b'This is a byte[] message payload'), + Message('json', {'test': 'This is a JSONObject message payload'}), + ])) + + assert isinstance(result, BatchResult) + assert result.success_count == 2 + assert result.failure_count == 0 + assert [entry.channel for entry in result.results] == channel_names + for entry in result.results: + assert isinstance(entry, BatchPublishSuccessResult) + assert len(entry.serials) == 3 + + for channel_name in channel_names: + history = await self.ably.channels[channel_name].history(direction='forwards') + messages = history.items + assert [message.name for message in messages] == ['string', 'binary', 'json'] + assert messages[0].data == 'This is a string message payload' + assert bytes(messages[1].data) == b'This is a byte[] message payload' + assert messages[2].data == {'test': 'This is a JSONObject message payload'} + # The library-generated ids share the prefix the result reports + assert [message.id for message in messages] == [f'{result.results[0].message_id}:{serial}' + for serial in range(3)] + + async def test_batch_publish_array_of_specs(self): + channel_a = self.get_channel_name('batch_publish_a') + channel_b = self.get_channel_name('batch_publish_b') + + results = await self.ably.batch_publish([ + BatchPublishSpec(channels=[channel_a], messages=[Message('a', 'data-a')]), + {'channels': [channel_a, channel_b], 'messages': [Message('b', 'data-b')]}, + ]) + + assert isinstance(results, list) + assert len(results) == 2 + assert [entry.channel for entry in results[0].results] == [channel_a] + assert [entry.channel for entry in results[1].results] == [channel_a, channel_b] + assert results[0].results[0].message_id != results[1].results[0].message_id + + async def test_batch_publish_partial_failure(self): + allowed_channel = self.get_channel_name('canpublish:batch_publish') + denied_channel = self.get_channel_name('batch_publish') + + result = await self.ably_restricted.batch_publish(BatchPublishSpec( + channels=[allowed_channel, denied_channel], + messages=[Message('event', 'data')], + )) + + assert result.success_count == 1 + assert result.failure_count == 1 + + success, failure = result.results + assert isinstance(success, BatchPublishSuccessResult) + assert success.channel == allowed_channel + assert isinstance(failure, BatchPublishFailureResult) + assert failure.channel == denied_channel + assert isinstance(failure.error, AblyException) + assert failure.error.code == 40160 + assert failure.error.status_code == 401 + + async def test_batch_presence_empty_channels(self): + channel_names = [self.get_channel_name('batch_presence_a'), self.get_channel_name('batch_presence_b')] + + result = await self.ably.batch_presence(channel_names) + + assert result.success_count == 2 + assert result.failure_count == 0 + assert [entry.channel for entry in result.results] == channel_names + for entry in result.results: + assert isinstance(entry, BatchPresenceSuccessResult) + assert entry.presence == [] diff --git a/test/unit/batch_test.py b/test/unit/batch_test.py new file mode 100644 index 00000000..8300c3bd --- /dev/null +++ b/test/unit/batch_test.py @@ -0,0 +1,38 @@ +"""Unit tests for the batch types and the inputs the batch methods accept. + +Tests cover: +- BSP2: a dict standing in for a BatchPublishSpec +- RSC22, RSC24: inputs refused before any request is made +""" + +import pytest + +from ably.pubsub.server import create_http_client +from ably.pubsub.types.batch import BatchPublishSpec +from ably.pubsub.types.message import Message + + +def test_batch_publish_spec_factory_accepts_spec(): + spec = BatchPublishSpec(channels=['a'], messages=[Message('event', 'data')]) + assert BatchPublishSpec.factory(spec) is spec + + +def test_batch_publish_spec_factory_accepts_dict(): + messages = [Message('event', 'data')] + spec = BatchPublishSpec.factory({'channels': ['a', 'b'], 'messages': messages}) + assert spec.channels == ['a', 'b'] + assert spec.messages is messages + + +def test_batch_publish_spec_factory_rejects_other_types(): + with pytest.raises(TypeError): + BatchPublishSpec.factory('a') + + +async def test_batch_presence_rejects_a_single_channel_name(): + client = create_http_client(key='fake.key:secret') + try: + with pytest.raises(TypeError): + await client.batch_presence('channel') + finally: + await client.close() From b5068cc42c5cae783681d5b5f59e048eaad27a50 Mon Sep 17 00:00:00 2001 From: owenpearson Date: Thu, 8 Oct 2026 13:55:16 +0100 Subject: [PATCH 10/71] test: run the batch specifications against the batch API The 44 batch Test IDs (47 cases) across `rest/unit` and `rest/integration` drop their `@deviation` gates and pass. They now build `BatchPublishSpec` and check result types with `isinstance`, as the specifications write them. Every `batch_publish.md` mock answers in the legacy format, a flat array of per-channel results, which the sandbox sends only to a client sending `X-Ably-Version: 2` or none. At version 5, which ably-python sends, `POST /messages` answers every batch, mixed and all-failure ones included, with 201 and an array holding one `BatchResult` envelope per spec, a single spec sent as a bare object included. The fixtures are corrected to that shape through `batch_result()`, and the assertions stand. Two RSL4c3 assertions compared a stringified JSON payload byte for byte; they now parse it, as the encoding tests do, since the specification does not fix the whitespace. deviations.md drops the batch row from Unimplemented features, and rewrites the envelope entry: it had concluded that `batch_presence.md` was the specification to revisit, and the server says it is `batch_publish.md`. The header and measured counts are re-measured: 172 gated cases, the RSC7d `Ably-Agent` test among them, 1047 passing derived cases, and 130 helper cases where 122 had been recorded. Three Smaller faults rows that had been superseded by their filed copies are removed. Co-Authored-By: Claude Opus 5.5 (1M context) --- test/uts/deviations.md | 76 ++-- .../rest/integration/batch_presence_test.py | 66 +--- test/uts/rest/unit/batch_presence_test.py | 42 +-- test/uts/rest/unit/batch_publish_test.py | 344 +++++++++--------- 4 files changed, 246 insertions(+), 282 deletions(-) diff --git a/test/uts/deviations.md b/test/uts/deviations.md index 498f0ad3..62a560e8 100644 --- a/test/uts/deviations.md +++ b/test/uts/deviations.md @@ -31,8 +31,8 @@ Variants` section and run every one of their tests twice, once per protocol, and `rest/unit` tests are parametrized over a table of fixtures the specification gives inline. That turns 1141 derived tests into 1234 pytest cases. -Of **1132 Test IDs, derived as 1141 tests and run as 1234 pytest cases**: 904 Test IDs -(913 tests, 1001 cases) pass, 213 (213 tests, 218 cases) are gated behind +Of **1132 Test IDs, derived as 1141 tests and run as 1234 pytest cases**: 947 Test IDs +(956 tests, 1047 cases) pass, 170 (170 tests, 172 cases) are gated behind `RUN_DEVIATIONS`, and 15 (15 tests, 15 cases) cannot be run at all. The three groups are disjoint: two Test IDs, and one parametrized test, have a gated part and a passing part, and are counted with the gated. Every gated test has been confirmed to fail when @@ -40,18 +40,18 @@ enabled, so none of them passes under both behaviours. 494 of the Test IDs come `uts/rest/unit` (503 tests, 536 cases), 481 from `uts/realtime/unit` (481, 481), 84 from `uts/rest/integration` (84, 122) and 73 from `uts/realtime/integration` (73, 95); 8 of the REST integration ids (8, 8) and 30 of the realtime ones (30, 30) come from the -`proxy` package within each. Of the gated Test IDs 122 are REST and 91 realtime, which is -126 REST cases and 92 realtime. -A further 122 pytest cases under `helpers/` cover the mock infrastructure itself and are +`proxy` package within each. Of the gated Test IDs 79 are REST and 91 realtime, which is +80 REST cases and 92 realtime. +A further 130 pytest cases under `helpers/` cover the mock infrastructure itself and are not derived from a specification. -The 203 gated Test IDs that record SDK non-compliance — 203 tests, 208 cases — reduce to -**71 distinct root causes**, 27 on the REST side and 44 on the realtime side. Three further +The 159 gated Test IDs that record SDK non-compliance — 159 tests, 161 cases — reduce to +**70 distinct root causes**, 26 on the REST side and 44 on the realtime side. Three further defects are recorded below with no test of their own, because the specification's test cannot discriminate (RTP18a), has nothing to assert against (the timezone split on synthesized LEAVE timestamps), or is worked around in the setup of every test that would otherwise trip over it (`enterClient` on an anonymous connection), so the file -carries **74 SDK root causes** in all. The remaining 10 gated Test IDs are +carries **73 SDK root causes** in all. The remaining 10 gated Test IDs are specification faults, and reduce to 7. Entries closed by a fix are removed rather than kept as history; `git log` holds that. @@ -271,16 +271,34 @@ file, and the protocol, which fix LEAVE at 3 and UPDATE at 4. The closing note o `batch_presence.md` states that with `X-Ably-Version >= 3` the server returns a `BatchResult` envelope "for all batch responses" and calls the plain array legacy. -Every mock in `batch_publish.md` uses the plain array. `features.md` RSC22b backs -`batch_publish.md` — "the response will still be an array" — so `batch_presence.md`'s -claim is the one to revisit. `revoke_tokens.md` has the same internal split: -`RSA17c_1` and `TRS2_1` stub a bare array while asserting envelope fields. +Every mock in `batch_publish.md` uses the plain array, and its RSC22_Headers1 pins +`X-Ably-Version: 2`: `batch_publish.md` is written against the legacy protocol, and +`batch_presence.md` is the one that matches the server ably-python talks to. Measured +against the sandbox, `POST /messages` answers: -`batch_publish.md` RSC22_Headers1 also pins `X-Ably-Version: 2` and -`Content-Type: application/json`; CSV2b templates the version, the sibling spec says -">= 3", and the binary protocol default makes the content type msgpack. - -The revocation half of that split is settled by the server. `POST +| `X-Ably-Version` | Every channel succeeds | Any channel fails | +|---|---|---| +| 5 | 201 with an array holding one `{successCount, failureCount, results}` envelope per spec | the same, 201 | +| 2, or none | 201 with one flat array of `{channel, messageId}` across every spec | 400 with error 40020, and the flat array, failures included, as `batchResponse` | + +A spec sent as a bare object is answered like an array of one: an array holding a single +envelope. That is the array RSC22b means when it says the response "will still be an array" +and the single-spec overload "will have to extract the element" — an array of +`BatchResult`s, one per spec, rather than of per-channel results. `GET /presence` follows the +same split by version, answering a mixed batch with 200 and the envelope at version 5 and +with 400/40020 and `batchResponse` at version 2. The legacy shape carries no `serials` +either; `batch_publish.md`'s mocks add them to it. + +ably-python sends version 5, so every `batch_publish.md` mock is corrected to the envelope, +through `batch_result()` in `rest/unit/batch_publish_test.py`, and the assertions stand as +written: they read `result.results[...]`, the layout BAR2c gives the envelope. RSC22_Headers1's +two pins are corrected beside them — CSV2b templates the version, which ably-python sends as +5, and the binary protocol default (TO3f) makes the content type msgpack, as +[#527](https://github.com/ably/specification/issues/527) records for the unit specs that +read a body without pinning the protocol. The envelope disagreement itself is not filed yet. + +`revoke_tokens.md` has the same internal split: `RSA17c_1` and `TRS2_1` stub a bare array +while asserting envelope fields. The revocation half is settled by the server too. `POST /keys/{keyName}/revokeTokens` with `X-Ably-Version: 5` answers 201 with the `{successCount, failureCount, results}` envelope `revoke_tokens.md`'s "Server Response Format" section describes, for a mixed success/failure batch as well as an all-success @@ -773,9 +791,6 @@ Filed as [#554](https://github.com/ably/specification/issues/554). | `message_encoding.md`, `msgpack_interop.md`, `annotations.md` | Six sections carry no Test ID; ids were inferred by sibling convention | | `publish.md`, `rest_presence.md`, `message_encoding.md`, `history.md`, `idempotency.md` | All point at `/Users/paddy/data/worknew/dev/dart-experiments/...` for the mock contract | | `revoke_tokens.md` | The all-success Setup blocks stub HTTP 200 with a plain array, the shape the file's own "Server Response Format" section calls legacy and says no current SDK sees. The `BatchResult` envelope and HTTP 201 that section prescribes appear only in the mixed and all-failure blocks | -| `publish.md` (integration) | The `Spec points:` header reads RSL1d, RSL1l1, RSL1m4, RSL1n, and the file carries a fifth section, `## RSL1k5 - Idempotent publish with client-supplied IDs`, with its own Test ID. The section is sound; only the header is short. Same housekeeping class as [#532](https://github.com/ably/specification/issues/532) | -| `auth.md` (integration) | RSC10's expired-JWT fixture is `generate_jwt(expires_at: now() - 5_seconds)`, naming `exp` and leaving `iat` open. Ably reads a JWT's lifetime as `exp - iat` and rejects a negative one with 400/40003 "Invalid value for ttl" before it considers expiry, so `iat` at now produces a token that fails the wrong way and never reaches the 40140–40149 renewal path the test is about. Backdating `iat` past `exp` gives the already-expired token the test wants, answered 401/40142. An SDK signing its own Ably JWT has to choose, so the fixture should say which | -| `batch_presence.md` | BGR2 says a channel with no members "returns a success result with an empty `presence` array", and the unit tier's mocks all send `'presence': []`. The server sends no `presence` key at all, so an implementation has to default the field for the assertion to hold. The derived test asserts the specification's `length == 0`, with the wire shape in a comment | | `publish.md` (integration) | The `Spec points:` header reads RSL1d, RSL1l1, RSL1m4, RSL1n, and the file carries a fifth section, `## RSL1k5 - Idempotent publish with client-supplied IDs`, with its own Test ID. The section is sound; only the header is short. `auth.md` (integration) has the same shape: its header reads RSA4, RSA8 and it carries `## RSC10` with its own Test ID. Same housekeeping class as [#532](https://github.com/ably/specification/issues/532); filed as [#550](https://github.com/ably/specification/issues/550) | | `auth.md` (integration) | RSC10's expired-JWT fixture is `generate_jwt(expires_at: now() - 5_seconds)`, naming `exp` and leaving `iat` open. Ably reads a JWT's lifetime as `exp - iat` and rejects a negative one with 400/40003 "Invalid value for ttl" before it considers expiry, so `iat` at now produces a token that fails the wrong way and never reaches the 40140–40149 renewal path the test is about. Backdating `iat` past `exp` gives the already-expired token the test wants, answered 401/40142. An SDK signing its own Ably JWT has to choose, so the fixture should say which. Filed as [#550](https://github.com/ably/specification/issues/550) | | `batch_presence.md` | The restricted-key setup's comment reads "only has access to \"batch-allowed\" channel" while the setup fixes `allowed_channel = "channel6"`; `batch-allowed` appears nowhere in the file. Filed with [#548](https://github.com/ably/specification/issues/548), whose fix replaces the same lines | @@ -792,8 +807,8 @@ Nothing to fix here, only something to build. Each row is one feature, and the c the number of gated Test IDs that fall with it, with the pytest case count beside it where the two differ. -Five of these rows are gated at both tiers. `batchPresence`, `Auth#revokeTokens`, the -`PushChannel` surface and the `clientId` filter on `RestPresence#get` each carry +Four of these rows are gated at both tiers. `Auth#revokeTokens`, the `PushChannel` surface +and the `clientId` filter on `RestPresence#get` each carry `uts/rest/integration` tests as well as unit ones, and connection recovery carries two `uts/realtime/integration` ones; all are written against the spelling the unit tier already gates on, so both tiers go green together when the API lands. Those @@ -801,17 +816,15 @@ integration tests do all their real work first — the sandbox app, the channels presence members entered over a realtime connection, the registered device and the issued token are all real, and each test reaches the missing call before it fails, so the assertions either side of it are known to hold against real server responses. The -`batch_presence` and `push_channels` files were additionally run against throwaway shims -— a `batch_presence` forwarding to `GET /presence`, and a `PushChannel` posting and -deleting `/push/channelSubscriptions` with `X-Ably-DeviceToken` — and pass in full -against them. The two recovery tests are a different shape: there is no missing call for +`push_channels` file was additionally run against a throwaway shim — a `PushChannel` +posting and deleting `/push/channelSubscriptions` with `X-Ably-DeviceToken` — and passes in +full against it. The two recovery tests are a different shape: there is no missing call for them to reach, so each runs end to end against the sandbox through `uts-proxy` and fails on the `recover` parameter the connection never sends. | Spec points | Missing | Test IDs | |---|---|---| -| RSC22, RSC24, BSP2, BPR2, BPF2, BAR2, BGR2, BGF2 | `batchPublish` and `batchPresence`, and all six result types. `grep -rn batch ably/` finds nothing | 44 (47 cases) | -| RSA17, RSA17b–g, BAR2, TRS2, TRF2 | `Auth#revokeTokens`, `TokenRevocationTargetSpecifier`, `BatchResult`. Gated against `auth.revoke_tokens(targets, issued_before=, allow_reauth_margin=)` returning `success_count` / `failure_count` / `results`, with `target` / `issued_before` / `applies_at` / `error` per result. RSA17d is the one case that needs no server at all — a token-authenticated client must refuse locally with 40162/401 — so it can be satisfied before any of the wire work | 21 | +| RSA17, RSA17b–g, BAR2, TRS2, TRF2 | `Auth#revokeTokens`, `TokenRevocationTargetSpecifier` and the two token revocation result types; the `BatchResult` they would arrive in is the batch API's. Gated against `auth.revoke_tokens(targets, issued_before=, allow_reauth_margin=)` returning `success_count` / `failure_count` / `results`, with `target` / `issued_before` / `applies_at` / `error` per result. RSA17d is the one case that needs no server at all — a token-authenticated client must refuse locally with 40162/401 — so it can be satisfied before any of the wire work | 21 | | RSH7, RSH7a–e, RSH6, RSH8 | `PushChannel`: `channel.push`, `client.device`, `LocalDevice`. The push *admin* surface (RSH1) does exist | 12 | | RTN16, RTN16f–k, RTC1c (TO3i) | Connection recovery, entire. `recover` is in the `Options` signature, stored, and given a property and a setter (`options.py:30,111,193,196`), and read nowhere. No `Connection#createRecoveryKey`, no `recover` connect parameter, no recovery-key decoding. Measured through the proxy: a client built with a valid `recover=` opened a `ws_connect` whose query parameters were `{'accessToken': …, 'v': '5'}` — no `recover` — and was given a fresh `connectionId`. RTN16l is otherwise fully compliant, taking the proxy's `recovery-failed-new-id`, `recovery-failed-new-key` and error 80008 and staying CONNECTED; only the absent parameter fails it | 8 | | RTL22, RTL22a–d, MFI1, MFI2a–e | `MessageFilter`. `RealtimeChannel.subscribe` (`channel.py:262-273`) accepts only a `str` or a callable, and there is no filter type of any shape to spell. Each test builds its filter through the module's `message_filter()` helper, which is the one place to repoint when the type lands | 5 | @@ -3217,9 +3230,10 @@ The header states how many derived tests there are, how many pass, how many are gated and how many cannot run. Those numbers are the check that the file is still true: in pytest cases, the gated count must equal the number of failures under `RUN_DEVIATIONS=1`, and gated plus unrunnable must equal the number of skips without -it. As of this writing that is 217 failures and 15 skips with the variable set, and -232 skips and 1124 passes without it, the 1124 being 1002 derived cases and 122 -`helpers/` ones. +it. As of this writing that is 172 failures and 15 skips with the variable set, and +187 skips and 1177 passes without it, the 1177 being 1047 derived cases and 130 +`helpers/` ones. Those figures need the `submodules` checkout CI makes: without it, three +`rest/unit/encoding` tests that read the ably-common fixtures skip as well. The other two counts are measured from the source rather than from a run. The number of **derived tests** is the number of `# UTS:` comments, 1141. The number of **Test IDs** is diff --git a/test/uts/rest/integration/batch_presence_test.py b/test/uts/rest/integration/batch_presence_test.py index b9d4e7af..ecf2d93c 100644 --- a/test/uts/rest/integration/batch_presence_test.py +++ b/test/uts/rest/integration/batch_presence_test.py @@ -2,38 +2,26 @@ Spec points: RSC24, BGR2, BGF2 -DEVIATION: ably-python has no batch API. `DefaultPubSubHttpClient` exposes no `batch_presence`, and -the package defines neither `BatchResult` nor `BatchPresenceSuccessResult` / -`BatchPresenceFailureResult`; the word "batch" appears nowhere under `ably/`. Every -test here therefore departs from the specification and is gated behind -RUN_DEVIATIONS, against the same spelling -[test/uts/rest/unit/batch_presence_test.py](../unit/batch_presence_test.py) gates on — -`client.batch_presence([...])` giving a result with `success_count`, `failure_count` -and `results` — so that dropping the marker is the only change either tier needs when -RSC24 lands. - -The setup halves are real, and the responses they assert against were confirmed by -hand through `client.request('GET', '/presence', params={'channels': ...})` against -the sandbox: the server does return the `successCount` / `failureCount` / `results` -envelope the specification describes, with `code` 40160 and `statusCode` 401 for a -channel the key has no capability for. Two details of that confirmation are recorded -beside the assertions they bear on — the `presence` key the server omits for an empty -channel, and the presence members a closed connection takes with it. - -See [deviations.md](../../deviations.md): the gating under *Failing Tests* -> -*Unimplemented features*, the omitted `presence` key and the closed connection under -*UTS Spec Errors*, and `enterClient` on an anonymous connection under *Failing Tests* -> -*Auth*. +The server returns the `successCount` / `failureCount` / `results` envelope the +specification describes, with `code` 40160 and `statusCode` 401 for a channel the key +has no capability for. Two details of the server's responses are recorded beside the +assertions they bear on — the `presence` key the server omits for an empty channel, and +the presence members a closed connection takes with it. + +See [deviations.md](../../deviations.md): the omitted `presence` key and the closed +connection under *UTS Spec Errors*, and `enterClient` on an anonymous connection under +*Failing Tests* -> *Auth*. """ from ably.pubsub.realtime.connection import ConnectionState +from ably.pubsub.types.batch import BatchPresenceFailureResult, BatchPresenceSuccessResult +from ably.pubsub.util.exceptions import AblyException from test.uts.helpers.client import ( await_connection_state, sandbox_realtime_client, sandbox_rest_client, wall_clock_poll_until, ) -from test.uts.helpers.deviations import deviation from test.uts.helpers.sandbox import random_id @@ -45,20 +33,6 @@ def result_for(result, channel_name): raise AssertionError(f'No batch result for channel {channel_name!r}') -def is_success_result(entry): - """The specifications' `entry IS BatchPresenceSuccessResult`. - - Neither result class exists to name, so a success is told apart from a failure by - which attribute it carries, as the unit tier does. - """ - return getattr(entry, 'presence', None) is not None - - -def is_failure_result(entry): - """The specifications' `entry IS BatchPresenceFailureResult`.""" - return getattr(entry, 'error', None) is not None - - def member_for(entry, client_id): """The specifications' `presence.find(m => m.clientId == client_id)`.""" for member in entry.presence: @@ -96,7 +70,6 @@ async def enter_members(realtime, channel_name, members): return channel -@deviation # UTS: rest/integration/RSC24/batch-presence-multiple-channels-0 async def test_rsc24_batch_presence_multiple_channels(sandbox, use_binary_protocol): channel_a_name = 'batch-presence-a-' + random_id() @@ -119,7 +92,7 @@ async def test_rsc24_batch_presence_multiple_channels(sandbox, use_binary_protoc result_a = result_for(result, channel_a_name) result_b = result_for(result, channel_b_name) - assert is_success_result(result_a) + assert isinstance(result_a, BatchPresenceSuccessResult) assert len(result_a.presence) == 2 client_ids_a = [member.client_id for member in result_a.presence] assert 'user-1' in client_ids_a @@ -127,13 +100,12 @@ async def test_rsc24_batch_presence_multiple_channels(sandbox, use_binary_protoc assert member_for(result_a, 'user-1').data == 'data-a1' - assert is_success_result(result_b) + assert isinstance(result_b, BatchPresenceSuccessResult) assert len(result_b.presence) == 1 assert result_b.presence[0].client_id == 'user-3' assert result_b.presence[0].data == 'data-b1' -@deviation # UTS: rest/integration/RSC24/restricted-key-channel-failure-1 async def test_rsc24_restricted_key_channel_failure(sandbox, use_binary_protocol): # The specification hard-codes "channel6" as the channel `keys[2]` is allowed. The @@ -165,7 +137,7 @@ async def test_rsc24_restricted_key_channel_failure(sandbox, use_binary_protocol async def one_member_on_the_allowed_channel(): result = await restricted_rest.batch_presence([allowed_channel, denied_channel]) success = result_for(result, allowed_channel) - return result if is_success_result(success) and len(success.presence) == 1 else None + return result if isinstance(success, BatchPresenceSuccessResult) and len(success.presence) == 1 else None result = await wall_clock_poll_until( one_member_on_the_allowed_channel, @@ -178,16 +150,16 @@ async def one_member_on_the_allowed_channel(): success = result_for(result, allowed_channel) failure = result_for(result, denied_channel) - assert is_success_result(success) + assert isinstance(success, BatchPresenceSuccessResult) assert len(success.presence) == 1 assert success.presence[0].client_id == 'member-1' - assert is_failure_result(failure) + assert isinstance(failure, BatchPresenceFailureResult) + assert isinstance(failure.error, AblyException) assert failure.error.code == 40160 assert failure.error.status_code == 401 -@deviation # UTS: rest/integration/RSC24/empty-channel-presence-2 async def test_rsc24_empty_channel_presence(sandbox, use_binary_protocol): empty_channel = 'batch-empty-' + random_id() @@ -210,9 +182,9 @@ async def test_rsc24_empty_channel_presence(sandbox, use_binary_protocol): # The server counts the empty channel a success and leaves `presence` out of its # result altogether rather than sending `[]`, so an implementation of BGR2 has to # default the field for the specification's assertion to hold. - assert is_success_result(empty_result) + assert isinstance(empty_result, BatchPresenceSuccessResult) assert len(empty_result.presence) == 0 - assert is_success_result(populated_result) + assert isinstance(populated_result, BatchPresenceSuccessResult) assert len(populated_result.presence) == 1 assert populated_result.presence[0].client_id == 'someone' diff --git a/test/uts/rest/unit/batch_presence_test.py b/test/uts/rest/unit/batch_presence_test.py index 599a3892..e1c67fc0 100644 --- a/test/uts/rest/unit/batch_presence_test.py +++ b/test/uts/rest/unit/batch_presence_test.py @@ -2,28 +2,16 @@ Spec points: RSC24, BAR2, BGR2, BGF2 -NOTE: ably-python has no batch API. `DefaultPubSubHttpClient` exposes no `batch_presence`, and the package -defines neither `BatchResult`/`BatchPresenceResponse` nor `BatchPresenceSuccessResult`/ -`BatchPresenceFailureResult`; the word "batch" appears nowhere under `ably/`. Every test in -this file therefore departs from the specification and is gated behind `RUN_DEVIATIONS`. -Each carries the assertion the spec calls for, written against the name ably-python would -use once RSC24 is implemented, so that dropping the `@deviation` marker is the only change -needed when it is. Today a batch presence query has to be hand-rolled by the caller through -`client.request('GET', '/presence', version=..., params={'channels': ...})`, which returns -an `HttpPaginatedResponse` of raw dicts rather than decoded `PresenceMessage`s, and which -does not raise on an error status. - -Counts are read as `success_count` / `failure_count`, the snake_case spelling of BAR2a and -BAR2b, and a success result is told apart from a failure result by which attributes it -carries rather than by `isinstance`, since neither class exists to name. +The specification's `BatchPresenceResponse` is ably-python's `BatchResult`, whose counts are +read as `success_count` / `failure_count`, the snake_case spelling of BAR2a and BAR2b. """ import pytest +from ably.pubsub.types.batch import BatchPresenceFailureResult, BatchPresenceSuccessResult from ably.pubsub.types.presence import PresenceAction from ably.pubsub.util.exceptions import AblyException from test.uts.helpers.client import rest_client -from test.uts.helpers.deviations import deviation from test.uts.helpers.mock_http import MockHttpClient @@ -40,7 +28,6 @@ def respond_with(status, body): # UTS: rest/unit/RSC24/get-presence-channels-param-0 -@deviation async def test_rsc24_batch_presence_get_presence_channels_param(): captured_requests = [] mock_http = MockHttpClient( @@ -65,7 +52,6 @@ async def test_rsc24_batch_presence_get_presence_channels_param(): # UTS: rest/unit/RSC24/single-channel-param-0 -@deviation async def test_rsc24_batch_presence_single_channel_param(): captured_requests = [] mock_http = MockHttpClient( @@ -86,7 +72,6 @@ async def test_rsc24_batch_presence_single_channel_param(): # UTS: rest/unit/RSC24/special-chars-comma-joined-0 -@deviation async def test_rsc24_batch_presence_special_chars_comma_joined(): captured_requests = [] mock_http = MockHttpClient( @@ -108,7 +93,6 @@ async def test_rsc24_batch_presence_special_chars_comma_joined(): # UTS: rest/unit/BAR2/mixed-success-failure-counts-0 -@deviation async def test_bar2_batch_presence_mixed_success_failure_counts(): mock_http = MockHttpClient( on_connection_attempt=lambda conn: conn.respond_with_success(), @@ -136,7 +120,6 @@ async def test_bar2_batch_presence_mixed_success_failure_counts(): # UTS: rest/unit/BAR2/all-success-counts-0 -@deviation async def test_bar2_batch_presence_all_success_counts(): mock_http = MockHttpClient( on_connection_attempt=lambda conn: conn.respond_with_success(), @@ -159,7 +142,6 @@ async def test_bar2_batch_presence_all_success_counts(): # UTS: rest/unit/BAR2/all-failure-counts-0 -@deviation async def test_bar2_batch_presence_all_failure_counts(): error = {'code': 40160, 'statusCode': 401, 'message': 'Not permitted'} mock_http = MockHttpClient( @@ -183,7 +165,6 @@ async def test_bar2_batch_presence_all_failure_counts(): # UTS: rest/unit/BGR2/success-with-members-0 -@deviation async def test_bgr2_batch_presence_success_with_members(): mock_http = MockHttpClient( on_connection_attempt=lambda conn: conn.respond_with_success(), @@ -222,7 +203,7 @@ async def test_bgr2_batch_presence_success_with_members(): assert len(result.results) == 1 success = result.results[0] - assert getattr(success, 'presence', None) is not None + assert isinstance(success, BatchPresenceSuccessResult) assert success.channel == 'my-channel' assert len(success.presence) == 2 @@ -237,7 +218,6 @@ async def test_bgr2_batch_presence_success_with_members(): # UTS: rest/unit/BGR2/success-empty-presence-0 -@deviation async def test_bgr2_batch_presence_success_empty_presence(): mock_http = MockHttpClient( on_connection_attempt=lambda conn: conn.respond_with_success(), @@ -254,13 +234,12 @@ async def test_bgr2_batch_presence_success_empty_presence(): result = await client.batch_presence(['empty-channel']) success = result.results[0] - assert getattr(success, 'presence', None) is not None + assert isinstance(success, BatchPresenceSuccessResult) assert success.channel == 'empty-channel' assert len(success.presence) == 0 # UTS: rest/unit/BGF2/failure-error-details-0 -@deviation async def test_bgf2_batch_presence_failure_error_details(): mock_http = MockHttpClient( on_connection_attempt=lambda conn: conn.respond_with_success(), @@ -286,7 +265,8 @@ async def test_bgf2_batch_presence_failure_error_details(): assert len(result.results) == 1 failure = result.results[0] - assert getattr(failure, 'error', None) is not None + assert isinstance(failure, BatchPresenceFailureResult) + assert isinstance(failure.error, AblyException) assert failure.channel == 'restricted-channel' assert failure.error.code == 40160 assert failure.error.status_code == 401 @@ -294,7 +274,6 @@ async def test_bgf2_batch_presence_failure_error_details(): # UTS: rest/unit/RSC24/mixed-success-failure-results-0 -@deviation async def test_rsc24_batch_presence_mixed_success_failure_results(): mock_http = MockHttpClient( on_connection_attempt=lambda conn: conn.respond_with_success(), @@ -329,18 +308,17 @@ async def test_rsc24_batch_presence_mixed_success_failure_results(): assert result.failure_count == 1 assert len(result.results) == 2 - assert getattr(result.results[0], 'presence', None) is not None + assert isinstance(result.results[0], BatchPresenceSuccessResult) assert result.results[0].channel == 'allowed-channel' assert len(result.results[0].presence) == 1 assert result.results[0].presence[0].client_id == 'user-1' - assert getattr(result.results[1], 'error', None) is not None + assert isinstance(result.results[1], BatchPresenceFailureResult) assert result.results[1].channel == 'restricted-channel' assert result.results[1].error.code == 40160 # UTS: rest/unit/RSC24/server-error-propagated-0 -@deviation async def test_rsc24_batch_presence_server_error_propagated(): mock_http = MockHttpClient( on_connection_attempt=lambda conn: conn.respond_with_success(), @@ -358,7 +336,6 @@ async def test_rsc24_batch_presence_server_error_propagated(): # UTS: rest/unit/RSC24/auth-error-propagated-0 -@deviation async def test_rsc24_batch_presence_auth_error_propagated(): mock_http = MockHttpClient( on_connection_attempt=lambda conn: conn.respond_with_success(), @@ -376,7 +353,6 @@ async def test_rsc24_batch_presence_auth_error_propagated(): # UTS: rest/unit/RSC24/uses-configured-auth-0 -@deviation async def test_rsc24_batch_presence_uses_configured_auth(): captured_requests = [] mock_http = MockHttpClient( diff --git a/test/uts/rest/unit/batch_publish_test.py b/test/uts/rest/unit/batch_publish_test.py index a4765c2f..bf40f99d 100644 --- a/test/uts/rest/unit/batch_publish_test.py +++ b/test/uts/rest/unit/batch_publish_test.py @@ -2,31 +2,30 @@ Spec points: RSC22c, RSC22d, BSP2a, BSP2b, BPR2a, BPR2b, BPR2c, BPF2a, BPF2b -NOTE: ably-python has no batch API. `DefaultPubSubHttpClient` exposes no `batch_publish`, and the package -defines neither `BatchPublishSpec` nor `BatchResult`/`BatchPublishSuccessResult`/ -`BatchPublishFailureResult`; the word "batch" appears nowhere under `ably/`. Every test in -this file therefore departs from the specification and is gated behind `RUN_DEVIATIONS`. -Each carries the assertion the spec calls for, written against the name ably-python would -use once RSC22 is implemented, so that dropping the `@deviation` marker is the only change -needed when it is. Today a batch publish has to be hand-rolled by the caller through -`client.request('POST', '/messages', version=..., body=...)`, which does no spec -construction, no RSL4 message encoding and no RSL1k1 idempotent ID generation. - -Because `BatchPublishSpec` does not exist to construct, a spec is written as a mapping of -the BSP2 attributes. Results are read by attribute, as the spec writes them, and a success -result is told apart from a failure result by which attributes it carries rather than by -`isinstance`, since neither class exists to name. +The specification's mocks answer in the legacy response format, a flat array of per-channel +results, which the server sends to a client sending `X-Ably-Version: 2` or no version at all. +ably-python sends version 5, and to that the server answers every batch publish with HTTP 201 +and an array holding a `BatchResult` envelope for each spec sent, a single spec sent as a bare +object included. Every fixture is corrected to that shape through `batch_result()`, and the +assertions stand as the specification writes them. See *Batch response envelopes disagree +between sibling specs* in [deviations.md](../../deviations.md). """ +import json import uuid import msgpack import pytest +from ably.pubsub.types.batch import ( + BatchPublishFailureResult, + BatchPublishSpec, + BatchPublishSuccessResult, + BatchResult, +) from ably.pubsub.types.message import Message from ably.pubsub.util.exceptions import AblyException from test.uts.helpers.client import rest_client -from test.uts.helpers.deviations import deviation from test.uts.helpers.mock_http import MockHttpClient @@ -37,7 +36,7 @@ def random_id(): def capture_and_respond(captured_requests, status=201, body=None): def on_request(request): captured_requests.append(request) - request.respond_with(status, body if body is not None else []) + request.respond_with(status, body if body is not None else success_response(request)) return on_request @@ -50,8 +49,29 @@ def success_result(channel, message_id='msg', serials=('s1',)): return {'channel': channel, 'messageId': message_id, 'serials': list(serials)} +def batch_result(*results): + """One spec's `BatchResult` envelope, holding the per-channel `results` given. + + UTS SPEC ERROR: batch_publish.md - the specification's mocks send the per-channel results + as a flat array, the legacy format, where the server answers ably-python's + `X-Ably-Version: 5` with an array of these envelopes, one per spec. + """ + failure_count = sum(1 for result in results if 'error' in result) + return { + 'successCount': len(results) - failure_count, + 'failureCount': failure_count, + 'results': list(results), + } + + +def success_response(request): + """What the server answers a batch publish with when every channel succeeds.""" + body = msgpack.unpackb(request.body) + specs = body if isinstance(body, list) else [body] + return [batch_result(*(success_result(channel) for channel in spec['channels'])) for spec in specs] + + # UTS: rest/unit/RSC22c/single-spec-post-messages-0 -@deviation async def test_rsc22c_batch_publish_single_spec_post_messages(): channel_name_1 = f'test-RSC22c1-a-{random_id()}' channel_name_2 = f'test-RSC22c1-b-{random_id()}' @@ -63,10 +83,10 @@ async def test_rsc22c_batch_publish_single_spec_post_messages(): ) client = rest_client(mock_http) - await client.batch_publish({ - 'channels': [channel_name_1, channel_name_2], - 'messages': [Message(name='event', data='hello')], - }) + await client.batch_publish(BatchPublishSpec( + channels=[channel_name_1, channel_name_2], + messages=[Message(name='event', data='hello')], + )) assert len(captured_requests) == 1 request = captured_requests[0] @@ -81,7 +101,6 @@ async def test_rsc22c_batch_publish_single_spec_post_messages(): # UTS: rest/unit/RSC22c/array-specs-post-messages-0 -@deviation async def test_rsc22c_batch_publish_array_specs_post_messages(): channel_name_1 = f'test-RSC22c2-a-{random_id()}' channel_name_2 = f'test-RSC22c2-b-{random_id()}' @@ -94,8 +113,8 @@ async def test_rsc22c_batch_publish_array_specs_post_messages(): client = rest_client(mock_http) await client.batch_publish([ - {'channels': [channel_name_1], 'messages': [Message(name='e1', data='d1')]}, - {'channels': [channel_name_2], 'messages': [Message(name='e2', data='d2')]}, + BatchPublishSpec(channels=[channel_name_1], messages=[Message(name='e1', data='d1')]), + BatchPublishSpec(channels=[channel_name_2], messages=[Message(name='e2', data='d2')]), ]) assert len(captured_requests) == 1 @@ -113,13 +132,13 @@ async def test_rsc22c_batch_publish_array_specs_post_messages(): # UTS: rest/unit/RSC22c/single-spec-single-result-0 -@deviation async def test_rsc22c_batch_publish_single_spec_single_result(): channel_name = f'test-RSC22c3-{random_id()}' # UTS SPEC ERROR: RSC22c3 - the mock body is a bare result object, but RSC22b says the REST # response "will still be an array", from which the single-spec overload extracts one element. - response_body = [{'channel': channel_name, 'messageId': 'msg123', 'serials': ['serial1']}] + # The server sends that element as the spec's BatchResult envelope. + response_body = [batch_result({'channel': channel_name, 'messageId': 'msg123', 'serials': ['serial1']})] mock_http = MockHttpClient( on_connection_attempt=lambda conn: conn.respond_with_success(), @@ -127,26 +146,25 @@ async def test_rsc22c_batch_publish_single_spec_single_result(): ) client = rest_client(mock_http) - result = await client.batch_publish({ - 'channels': [channel_name], - 'messages': [Message(name='event', data='hello')], - }) + result = await client.batch_publish(BatchPublishSpec( + channels=[channel_name], + messages=[Message(name='event', data='hello')], + )) - assert not isinstance(result, list) + assert isinstance(result, BatchResult) assert len(result.results) == 1 assert result.results[0].channel == channel_name assert result.results[0].message_id == 'msg123' # UTS: rest/unit/RSC22c/array-specs-array-results-0 -@deviation async def test_rsc22c_batch_publish_array_specs_array_results(): channel_name_1 = f'test-RSC22c4-a-{random_id()}' channel_name_2 = f'test-RSC22c4-b-{random_id()}' response_body = [ - success_result(channel_name_1, 'msg1', ['s1']), - success_result(channel_name_2, 'msg2', ['s2']), + batch_result(success_result(channel_name_1, 'msg1', ['s1'])), + batch_result(success_result(channel_name_2, 'msg2', ['s2'])), ] mock_http = MockHttpClient( @@ -156,28 +174,28 @@ async def test_rsc22c_batch_publish_array_specs_array_results(): client = rest_client(mock_http) results = await client.batch_publish([ - {'channels': [channel_name_1], 'messages': [Message(name='e1', data='d1')]}, - {'channels': [channel_name_2], 'messages': [Message(name='e2', data='d2')]}, + BatchPublishSpec(channels=[channel_name_1], messages=[Message(name='e1', data='d1')]), + BatchPublishSpec(channels=[channel_name_2], messages=[Message(name='e2', data='d2')]), ]) assert isinstance(results, list) assert len(results) == 2 + assert all(isinstance(result, BatchResult) for result in results) assert results[0].results[0].channel == channel_name_1 assert results[1].results[0].channel == channel_name_2 # UTS: rest/unit/RSC22c/multiple-channels-multiple-results-0 -@deviation async def test_rsc22c_batch_publish_multiple_channels_multiple_results(): channel_name_1 = f'test-RSC22c5-a-{random_id()}' channel_name_2 = f'test-RSC22c5-b-{random_id()}' channel_name_3 = f'test-RSC22c5-c-{random_id()}' - response_body = [ + response_body = [batch_result( success_result(channel_name_1, 'msg1', ['s1']), success_result(channel_name_2, 'msg2', ['s2']), success_result(channel_name_3, 'msg3', ['s3']), - ] + )] mock_http = MockHttpClient( on_connection_attempt=lambda conn: conn.respond_with_success(), @@ -185,17 +203,16 @@ async def test_rsc22c_batch_publish_multiple_channels_multiple_results(): ) client = rest_client(mock_http) - result = await client.batch_publish({ - 'channels': [channel_name_1, channel_name_2, channel_name_3], - 'messages': [Message(name='event', data='hello')], - }) + result = await client.batch_publish(BatchPublishSpec( + channels=[channel_name_1, channel_name_2, channel_name_3], + messages=[Message(name='event', data='hello')], + )) assert len(result.results) == 3 assert [entry.channel for entry in result.results] == [channel_name_1, channel_name_2, channel_name_3] # UTS: rest/unit/RSC22c/messages-encoded-per-rsl4-0 -@deviation async def test_rsc22c_batch_publish_messages_encoded_per_rsl4(): channel_name = f'test-RSC22c6-{random_id()}' @@ -206,14 +223,14 @@ async def test_rsc22c_batch_publish_messages_encoded_per_rsl4(): ) client = rest_client(mock_http) - await client.batch_publish({ - 'channels': [channel_name], - 'messages': [ + await client.batch_publish(BatchPublishSpec( + channels=[channel_name], + messages=[ Message(name='string', data='plain text'), Message(name='binary', data=b'\x01\x02\x03'), Message(name='json', data={'key': 'value'}), ], - }) + )) assert len(captured_requests) == 1 messages = msgpack.unpackb(captured_requests[0].body)['messages'] @@ -228,12 +245,11 @@ async def test_rsc22c_batch_publish_messages_encoded_per_rsl4(): assert 'encoding' not in messages[1] or messages[1]['encoding'] is None # RSL4c3: a JSON payload is stringified and the encoding attribute is set to "json" - assert messages[2]['data'] == '{"key":"value"}' + assert json.loads(messages[2]['data']) == {'key': 'value'} assert messages[2]['encoding'] == 'json' # UTS: rest/unit/RSC22c/uses-configured-auth-0 -@deviation async def test_rsc22c_batch_publish_uses_configured_auth(): channel_name = f'test-RSC22c7-{random_id()}' @@ -244,10 +260,10 @@ async def test_rsc22c_batch_publish_uses_configured_auth(): ) token_client = rest_client(mock_http, token='fake-token') - await token_client.batch_publish({ - 'channels': [channel_name], - 'messages': [Message(name='event', data='hello')], - }) + await token_client.batch_publish(BatchPublishSpec( + channels=[channel_name], + messages=[Message(name='event', data='hello')], + )) assert len(captured_requests) == 1 assert captured_requests[0].headers['Authorization'].startswith('Bearer ') @@ -258,17 +274,16 @@ async def test_rsc22c_batch_publish_uses_configured_auth(): mock_http.on_request = capture_and_respond(basic_requests) basic_client = rest_client(mock_http) - await basic_client.batch_publish({ - 'channels': [basic_channel_name], - 'messages': [Message(name='event', data='hello')], - }) + await basic_client.batch_publish(BatchPublishSpec( + channels=[basic_channel_name], + messages=[Message(name='event', data='hello')], + )) assert len(basic_requests) == 1 assert basic_requests[0].headers['Authorization'].startswith('Basic ') # UTS: rest/unit/RSC22d/idempotent-ids-generated-0 -@deviation async def test_rsc22d_batch_publish_idempotent_ids_generated(): captured_requests = [] mock_http = MockHttpClient( @@ -278,14 +293,14 @@ async def test_rsc22d_batch_publish_idempotent_ids_generated(): client = rest_client(mock_http, idempotent_rest_publishing=True) await client.batch_publish([ - { - 'channels': [f'test-RSC22d-a-{random_id()}'], - 'messages': [Message(name='e1', data='d1'), Message(name='e2', data='d2')], - }, - { - 'channels': [f'test-RSC22d-b-{random_id()}'], - 'messages': [Message(name='e3', data='d3'), Message(name='e4', data='d4')], - }, + BatchPublishSpec( + channels=[f'test-RSC22d-a-{random_id()}'], + messages=[Message(name='e1', data='d1'), Message(name='e2', data='d2')], + ), + BatchPublishSpec( + channels=[f'test-RSC22d-b-{random_id()}'], + messages=[Message(name='e3', data='d3'), Message(name='e4', data='d4')], + ), ]) assert len(captured_requests) == 1 @@ -307,7 +322,6 @@ async def test_rsc22d_batch_publish_idempotent_ids_generated(): # UTS: rest/unit/RSC22d/explicit-ids-preserved-0 -@deviation async def test_rsc22d_batch_publish_explicit_ids_preserved(): channel_name = f'test-RSC22d-explicit-{random_id()}' @@ -318,13 +332,13 @@ async def test_rsc22d_batch_publish_explicit_ids_preserved(): ) client = rest_client(mock_http, idempotent_rest_publishing=True) - await client.batch_publish({ - 'channels': [channel_name], - 'messages': [ + await client.batch_publish(BatchPublishSpec( + channels=[channel_name], + messages=[ Message(name='e1', data='d1', id='explicit-id-1'), Message(name='e2', data='d2', id='explicit-id-2'), ], - }) + )) assert len(captured_requests) == 1 messages = msgpack.unpackb(captured_requests[0].body)['messages'] @@ -334,7 +348,6 @@ async def test_rsc22d_batch_publish_explicit_ids_preserved(): # UTS: rest/unit/RSC22d/ids-not-generated-disabled-0 -@deviation async def test_rsc22d_batch_publish_ids_not_generated_disabled(): channel_name = f'test-RSC22d-disabled-{random_id()}' @@ -345,10 +358,10 @@ async def test_rsc22d_batch_publish_ids_not_generated_disabled(): ) client = rest_client(mock_http, idempotent_rest_publishing=False) - await client.batch_publish({ - 'channels': [channel_name], - 'messages': [Message(name='e1', data='d1'), Message(name='e2', data='d2')], - }) + await client.batch_publish(BatchPublishSpec( + channels=[channel_name], + messages=[Message(name='e1', data='d1'), Message(name='e2', data='d2')], + )) assert len(captured_requests) == 1 messages = msgpack.unpackb(captured_requests[0].body)['messages'] @@ -358,7 +371,6 @@ async def test_rsc22d_batch_publish_ids_not_generated_disabled(): # UTS: rest/unit/BSP2a/channels-array-strings-0 -@deviation async def test_bsp2a_batch_publish_spec_channels_array_strings(): channel_name_1 = f'test-BSP2a-a-{random_id()}' channel_name_2 = f'test-BSP2a-b-{random_id()}' @@ -371,10 +383,10 @@ async def test_bsp2a_batch_publish_spec_channels_array_strings(): ) client = rest_client(mock_http) - await client.batch_publish({ - 'channels': [channel_name_1, channel_name_2, channel_name_3], - 'messages': [Message(name='event', data='hello')], - }) + await client.batch_publish(BatchPublishSpec( + channels=[channel_name_1, channel_name_2, channel_name_3], + messages=[Message(name='event', data='hello')], + )) assert len(captured_requests) == 1 channels = msgpack.unpackb(captured_requests[0].body)['channels'] @@ -385,7 +397,6 @@ async def test_bsp2a_batch_publish_spec_channels_array_strings(): # UTS: rest/unit/BSP2b/messages-array-objects-0 -@deviation async def test_bsp2b_batch_publish_spec_messages_array_objects(): channel_name = f'test-BSP2b-{random_id()}' @@ -396,13 +407,13 @@ async def test_bsp2b_batch_publish_spec_messages_array_objects(): ) client = rest_client(mock_http) - await client.batch_publish({ - 'channels': [channel_name], - 'messages': [ + await client.batch_publish(BatchPublishSpec( + channels=[channel_name], + messages=[ Message(name='event1', data='data1'), Message(name='event2', data={'key': 'value'}), ], - }) + )) assert len(captured_requests) == 1 messages = msgpack.unpackb(captured_requests[0].body)['messages'] @@ -415,118 +426,117 @@ async def test_bsp2b_batch_publish_spec_messages_array_objects(): assert messages[0]['name'] == 'event1' assert messages[0]['data'] == 'data1' assert messages[1]['name'] == 'event2' - assert messages[1]['data'] == '{"key":"value"}' + assert isinstance(messages[1]['data'], str) + assert json.loads(messages[1]['data']) == {'key': 'value'} assert messages[1]['encoding'] == 'json' # UTS: rest/unit/BPR2a/success-channel-name-0 -@deviation async def test_bpr2a_batch_publish_success_channel_name(): channel_name = f'test-BPR2a-{random_id()}' mock_http = MockHttpClient( on_connection_attempt=lambda conn: conn.respond_with_success(), - on_request=respond_with(201, [success_result(channel_name, 'msg123', ['s1'])]), + on_request=respond_with(201, [batch_result(success_result(channel_name, 'msg123', ['s1']))]), ) client = rest_client(mock_http) - result = await client.batch_publish({ - 'channels': [channel_name], - 'messages': [Message(name='event', data='hello')], - }) + result = await client.batch_publish(BatchPublishSpec( + channels=[channel_name], + messages=[Message(name='event', data='hello')], + )) assert result.results[0].channel == channel_name # UTS: rest/unit/BPR2b/success-message-id-prefix-0 -@deviation async def test_bpr2b_batch_publish_success_message_id_prefix(): channel_name = f'test-BPR2b-{random_id()}' mock_http = MockHttpClient( on_connection_attempt=lambda conn: conn.respond_with_success(), - on_request=respond_with(201, [success_result(channel_name, 'unique-id-prefix', ['s1', 's2'])]), + on_request=respond_with(201, [batch_result( + success_result(channel_name, 'unique-id-prefix', ['s1', 's2']), + )]), ) client = rest_client(mock_http) - result = await client.batch_publish({ - 'channels': [channel_name], - 'messages': [Message(name='e1', data='d1'), Message(name='e2', data='d2')], - }) + result = await client.batch_publish(BatchPublishSpec( + channels=[channel_name], + messages=[Message(name='e1', data='d1'), Message(name='e2', data='d2')], + )) assert result.results[0].message_id == 'unique-id-prefix' # UTS: rest/unit/BPR2c/serials-array-0 -@deviation async def test_bpr2c_batch_publish_serials_array(): channel_name = f'test-BPR2c-{random_id()}' mock_http = MockHttpClient( on_connection_attempt=lambda conn: conn.respond_with_success(), - on_request=respond_with(201, [success_result(channel_name, 'msg', ['serial1', 'serial2', 'serial3'])]), + on_request=respond_with(201, [batch_result( + success_result(channel_name, 'msg', ['serial1', 'serial2', 'serial3']), + )]), ) client = rest_client(mock_http) messages = [Message(name=f'e{index}', data=f'd{index}') for index in range(3)] - result = await client.batch_publish({'channels': [channel_name], 'messages': messages}) + result = await client.batch_publish(BatchPublishSpec(channels=[channel_name], messages=messages)) assert result.results[0].serials == ['serial1', 'serial2', 'serial3'] assert len(result.results[0].serials) == len(messages) # UTS: rest/unit/BPR2c/serials-null-conflated-0 -@deviation async def test_bpr2c_batch_publish_serials_null_conflated(): channel_name = f'test-BPR2c1-{random_id()}' mock_http = MockHttpClient( on_connection_attempt=lambda conn: conn.respond_with_success(), - on_request=respond_with(201, [{ + on_request=respond_with(201, [batch_result({ 'channel': channel_name, 'messageId': 'msg', 'serials': ['serial1', None, 'serial3'], - }]), + })]), ) client = rest_client(mock_http) messages = [Message(name=f'e{index}', data=f'd{index}') for index in range(3)] - result = await client.batch_publish({'channels': [channel_name], 'messages': messages}) + result = await client.batch_publish(BatchPublishSpec(channels=[channel_name], messages=messages)) # BPR2c: a null serial marks a message discarded by a conflation rule assert result.results[0].serials == ['serial1', None, 'serial3'] # UTS: rest/unit/BPF2a/failure-channel-name-0 -@deviation async def test_bpf2a_batch_publish_failure_channel_name(): channel_name = f'test-BPF2a-{random_id()}' mock_http = MockHttpClient( on_connection_attempt=lambda conn: conn.respond_with_success(), - on_request=respond_with(201, [{ + on_request=respond_with(201, [batch_result({ 'channel': channel_name, 'error': {'code': 40160, 'statusCode': 401, 'message': 'Not permitted'}, - }]), + })]), ) client = rest_client(mock_http) - result = await client.batch_publish({ - 'channels': [channel_name], - 'messages': [Message(name='event', data='hello')], - }) + result = await client.batch_publish(BatchPublishSpec( + channels=[channel_name], + messages=[Message(name='event', data='hello')], + )) assert result.results[0].channel == channel_name # UTS: rest/unit/BPF2b/failure-error-info-0 -@deviation async def test_bpf2b_batch_publish_failure_error_info(): channel_name = f'test-BPF2b-{random_id()}' mock_http = MockHttpClient( on_connection_attempt=lambda conn: conn.respond_with_success(), - on_request=respond_with(201, [{ + on_request=respond_with(201, [batch_result({ 'channel': channel_name, 'error': { 'code': 40160, @@ -534,78 +544,78 @@ async def test_bpf2b_batch_publish_failure_error_info(): 'message': 'Channel operation not permitted', 'href': 'https://help.ably.io/error/40160', }, - }]), + })]), ) client = rest_client(mock_http) - result = await client.batch_publish({ - 'channels': [channel_name], - 'messages': [Message(name='event', data='hello')], - }) + result = await client.batch_publish(BatchPublishSpec( + channels=[channel_name], + messages=[Message(name='event', data='hello')], + )) error = result.results[0].error + assert isinstance(error, AblyException) assert error.code == 40160 assert error.status_code == 401 assert 'not permitted' in error.message # UTS: rest/unit/RSC22c/partial-success-mixed-results-0 -@deviation async def test_rsc22c_batch_publish_partial_success_mixed_results(): channel_name_allowed = f'test-BatchResult1-allowed-{random_id()}' channel_name_restricted = f'test-BatchResult1-restricted-{random_id()}' mock_http = MockHttpClient( on_connection_attempt=lambda conn: conn.respond_with_success(), - on_request=respond_with(201, [ + on_request=respond_with(201, [batch_result( success_result(channel_name_allowed, 'msg1', ['s1']), { 'channel': channel_name_restricted, 'error': {'code': 40160, 'statusCode': 401, 'message': 'Not permitted'}, }, - ]), + )]), ) client = rest_client(mock_http) - result = await client.batch_publish({ - 'channels': [channel_name_allowed, channel_name_restricted], - 'messages': [Message(name='event', data='hello')], - }) + result = await client.batch_publish(BatchPublishSpec( + channels=[channel_name_allowed, channel_name_restricted], + messages=[Message(name='event', data='hello')], + )) # NOTE: the spec writes result[0] / result[1]; BAR2c puts the per-channel results in # BatchResult.results, so they are read from there. assert len(result.results) == 2 + assert isinstance(result.results[0], BatchPublishSuccessResult) assert result.results[0].channel == channel_name_allowed assert result.results[0].message_id == 'msg1' - assert getattr(result.results[0], 'error', None) is None + assert isinstance(result.results[1], BatchPublishFailureResult) assert result.results[1].channel == channel_name_restricted assert result.results[1].error.code == 40160 # UTS: rest/unit/RSC22c/distinguish-success-failure-0 -@deviation async def test_rsc22c_batch_publish_distinguish_success_failure(): channel_name = f'test-BatchResult2-{random_id()}' failed_channel_name = f'test-BatchResult2-failed-{random_id()}' mock_http = MockHttpClient( on_connection_attempt=lambda conn: conn.respond_with_success(), - on_request=respond_with(201, [ + on_request=respond_with(201, [batch_result( success_result(channel_name, 'msg1', ['s1']), { 'channel': failed_channel_name, 'error': {'code': 40160, 'statusCode': 401, 'message': 'Not permitted'}, }, - ]), + )]), ) client = rest_client(mock_http) - result = await client.batch_publish({ - 'channels': [channel_name, failed_channel_name], - 'messages': [Message(name='event', data='hello')], - }) + result = await client.batch_publish(BatchPublishSpec( + channels=[channel_name, failed_channel_name], + messages=[Message(name='event', data='hello')], + )) for entry in result.results: has_success_fields = getattr(entry, 'message_id', None) is not None and \ @@ -615,7 +625,6 @@ async def test_rsc22c_batch_publish_distinguish_success_failure(): # UTS: rest/unit/RSC22/empty-channels-rejected-0 -@deviation async def test_rsc22_batch_publish_empty_channels_rejected(): mock_http = MockHttpClient( on_connection_attempt=lambda conn: conn.respond_with_success(), @@ -624,14 +633,13 @@ async def test_rsc22_batch_publish_empty_channels_rejected(): client = rest_client(mock_http) with pytest.raises(AblyException) as excinfo: - await client.batch_publish({'channels': [], 'messages': [Message(name='event', data='hello')]}) + await client.batch_publish(BatchPublishSpec(channels=[], messages=[Message(name='event', data='hello')])) # NOTE: the spec says only "the error indicates invalid request"; read as a 400. assert excinfo.value.status_code == 400 # UTS: rest/unit/RSC22/empty-messages-rejected-0 -@deviation async def test_rsc22_batch_publish_empty_messages_rejected(): channel_name = f'test-RSC22-Error2-{random_id()}' @@ -642,14 +650,13 @@ async def test_rsc22_batch_publish_empty_messages_rejected(): client = rest_client(mock_http) with pytest.raises(AblyException) as excinfo: - await client.batch_publish({'channels': [channel_name], 'messages': []}) + await client.batch_publish(BatchPublishSpec(channels=[channel_name], messages=[])) # NOTE: the spec says only "the error indicates invalid request"; read as a 400. assert excinfo.value.status_code == 400 # UTS: rest/unit/RSC22/server-error-propagated-0 -@deviation async def test_rsc22_batch_publish_server_error_propagated(): channel_name = f'test-RSC22-Error3-{random_id()}' @@ -662,17 +669,16 @@ async def test_rsc22_batch_publish_server_error_propagated(): client = rest_client(mock_http) with pytest.raises(AblyException) as excinfo: - await client.batch_publish({ - 'channels': [channel_name], - 'messages': [Message(name='event', data='hello')], - }) + await client.batch_publish(BatchPublishSpec( + channels=[channel_name], + messages=[Message(name='event', data='hello')], + )) assert excinfo.value.code == 50000 assert excinfo.value.status_code == 500 # UTS: rest/unit/RSC22/auth-error-propagated-0 -@deviation async def test_rsc22_batch_publish_auth_error_propagated(): channel_name = f'test-RSC22-Error4-{random_id()}' @@ -685,17 +691,16 @@ async def test_rsc22_batch_publish_auth_error_propagated(): client = rest_client(mock_http) with pytest.raises(AblyException) as excinfo: - await client.batch_publish({ - 'channels': [channel_name], - 'messages': [Message(name='event', data='hello')], - }) + await client.batch_publish(BatchPublishSpec( + channels=[channel_name], + messages=[Message(name='event', data='hello')], + )) assert excinfo.value.code == 40101 assert excinfo.value.status_code == 401 # UTS: rest/unit/RSC22/standard-headers-included-0 -@deviation async def test_rsc22_batch_publish_standard_headers_included(): channel_name = f'test-RSC22-Headers1-{random_id()}' @@ -706,10 +711,10 @@ async def test_rsc22_batch_publish_standard_headers_included(): ) client = rest_client(mock_http) - await client.batch_publish({ - 'channels': [channel_name], - 'messages': [Message(name='event', data='hello')], - }) + await client.batch_publish(BatchPublishSpec( + channels=[channel_name], + messages=[Message(name='event', data='hello')], + )) assert len(captured_requests) == 1 request = captured_requests[0] @@ -726,7 +731,6 @@ async def test_rsc22_batch_publish_standard_headers_included(): # UTS: rest/unit/RSC22/request-id-included-0 -@deviation async def test_rsc22_batch_publish_request_id_included(): channel_name = f'test-RSC22-Headers2-{random_id()}' @@ -737,10 +741,10 @@ async def test_rsc22_batch_publish_request_id_included(): ) client = rest_client(mock_http, add_request_ids=True) - await client.batch_publish({ - 'channels': [channel_name], - 'messages': [Message(name='event', data='hello')], - }) + await client.batch_publish(BatchPublishSpec( + channels=[channel_name], + messages=[Message(name='event', data='hello')], + )) assert len(captured_requests) == 1 request_id = captured_requests[0].url.query_params['request_id'] @@ -751,21 +755,20 @@ async def test_rsc22_batch_publish_request_id_included(): # UTS: rest/unit/RSC22/multiple-messages-per-channel-0 -@deviation async def test_rsc22_batch_publish_multiple_messages_per_channel(): channel_name = f'test-RSC22-Batch1-{random_id()}' captured_requests = [] mock_http = MockHttpClient( on_connection_attempt=lambda conn: conn.respond_with_success(), - on_request=capture_and_respond(captured_requests, body=[ + on_request=capture_and_respond(captured_requests, body=[batch_result( success_result(channel_name, 'msg', [f's{index}' for index in range(100)]), - ]), + )]), ) client = rest_client(mock_http) messages = [Message(name=f'event-{index}', data=f'data-{index}') for index in range(100)] - result = await client.batch_publish({'channels': [channel_name], 'messages': messages}) + result = await client.batch_publish(BatchPublishSpec(channels=[channel_name], messages=messages)) assert len(captured_requests) == 1 sent_messages = msgpack.unpackb(captured_requests[0].body)['messages'] @@ -777,7 +780,6 @@ async def test_rsc22_batch_publish_multiple_messages_per_channel(): # UTS: rest/unit/RSC22/multiple-channels-multiple-messages-0 -@deviation async def test_rsc22_batch_publish_multiple_channels_multiple_messages(): channel_name_1 = f'test-RSC22-Batch2-a-{random_id()}' channel_name_2 = f'test-RSC22-Batch2-b-{random_id()}' @@ -786,22 +788,22 @@ async def test_rsc22_batch_publish_multiple_channels_multiple_messages(): captured_requests = [] mock_http = MockHttpClient( on_connection_attempt=lambda conn: conn.respond_with_success(), - on_request=capture_and_respond(captured_requests, body=[ + on_request=capture_and_respond(captured_requests, body=[batch_result( success_result(channel_name_1, 'msg1', ['s1', 's2', 's3']), success_result(channel_name_2, 'msg2', ['s4', 's5', 's6']), success_result(channel_name_3, 'msg3', ['s7', 's8', 's9']), - ]), + )]), ) client = rest_client(mock_http) - result = await client.batch_publish({ - 'channels': [channel_name_1, channel_name_2, channel_name_3], - 'messages': [ + result = await client.batch_publish(BatchPublishSpec( + channels=[channel_name_1, channel_name_2, channel_name_3], + messages=[ Message(name='msg1', data='d1'), Message(name='msg2', data='d2'), Message(name='msg3', data='d3'), ], - }) + )) assert len(captured_requests) == 1 body = msgpack.unpackb(captured_requests[0].body) From 0bb255063143654ffa64c84c612847e7b184c711 Mon Sep 17 00:00:00 2001 From: owenpearson Date: Thu, 8 Oct 2026 17:22:29 +0100 Subject: [PATCH 11/71] docs: cite the batch response format issue in deviations.md The batch publish and token revocation fixtures written in the response format below protocol version 3 are filed as ably/specification#559. Co-Authored-By: Claude Opus 5.5 (1M context) --- test/uts/deviations.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/test/uts/deviations.md b/test/uts/deviations.md index 62a560e8..2eb9daa1 100644 --- a/test/uts/deviations.md +++ b/test/uts/deviations.md @@ -131,6 +131,7 @@ Raised upstream: | [#552](https://github.com/ably/specification/issues/552) | `proxy/connection_resume.md`: a status code neither SDK returns, a proxy substitution that does not exist, and event-log fields the proxy does not emit | | [#553](https://github.com/ably/specification/issues/553) | A heartbeat-starvation test that closes the socket thirteen seconds inside the idle window | | [#554](https://github.com/ably/specification/issues/554) | Two sections provoking one server response, leaving the revoked-key point uncovered | +| [#559](https://github.com/ably/specification/issues/559) | Batch publish and token revocation fixtures in the response format the server sends below protocol version 3 | `#527` also carries a comment on the realtime wire-format assertions, `#532` one on the same housekeeping categories in `realtime/unit`, and @@ -295,7 +296,9 @@ written: they read `result.results[...]`, the layout BAR2c gives the envelope. R two pins are corrected beside them — CSV2b templates the version, which ably-python sends as 5, and the binary protocol default (TO3f) makes the content type msgpack, as [#527](https://github.com/ably/specification/issues/527) records for the unit specs that -read a body without pinning the protocol. The envelope disagreement itself is not filed yet. +read a body without pinning the protocol. The envelope disagreement is filed as +[#559](https://github.com/ably/specification/issues/559), together with the two +`revoke_tokens.md` mocks below. `revoke_tokens.md` has the same internal split: `RSA17c_1` and `TRS2_1` stub a bare array while asserting envelope fields. The revocation half is settled by the server too. `POST From d23c98c68536d6568dbdede4193120a0b0ff66fa Mon Sep 17 00:00:00 2001 From: sacOO7 Date: Fri, 16 Oct 2026 13:08:40 +0530 Subject: [PATCH 12/71] pubsub: export the encryption helpers from ably.pubsub.server 4.0 makes ably.pubsub.server the whole public API, but its export list was carried over from 3.x's top-level ably package, which only re-exported CipherParams. 3.x users reached generate_random_key and get_default_params through ably.util.crypto, a submodule that 4.0 moves and declares internal, so 4.0 left no public way to generate a key (RSE2) or build default cipher params. Export both from ably.pubsub.server and ably.pubsub.server.sync, show the 3.x to 4.0 import in UPDATING.md, and make the missing-crypto error name the ably-pubsub-server[crypto] extra instead of the old ably[crypto]. --- UPDATING.md | 12 ++++++++++++ ably/pubsub/server/__init__.py | 4 +++- ably/pubsub/server/sync.py | 4 +++- ably/pubsub/util/nocrypto.py | 2 +- test/unit/pubsub_server_test.py | 6 ++++++ 5 files changed, 25 insertions(+), 3 deletions(-) diff --git a/UPDATING.md b/UPDATING.md index a2aa7db7..a63e8ea1 100644 --- a/UPDATING.md +++ b/UPDATING.md @@ -49,6 +49,18 @@ Where 3.x left you reaching into submodules such as `ably.types.message` or the whole public API, and `ably.pubsub.server.sync` is its synchronous counterpart. The packages beneath them are internal and free to move. +For example, the encryption helpers move from `ably.util.crypto`: + +Example 3.x code: +```python +from ably.util.crypto import generate_random_key, get_default_params +``` + +Example 4.0.0 code: +```python +from ably.pubsub.server import generate_random_key, get_default_params +``` + ### Clients are built by factories `AblyRest` and `AblyRealtime` are internal in 4.0.0 and raise `TypeError` if diff --git a/ably/pubsub/server/__init__.py b/ably/pubsub/server/__init__.py index 41efc61e..a95546c5 100644 --- a/ably/pubsub/server/__init__.py +++ b/ably/pubsub/server/__init__.py @@ -38,7 +38,7 @@ from ably.pubsub.types.options import Options, VCDiffDecoder from ably.pubsub.types.tokendetails import TokenDetails from ably.pubsub.util.construction import factory_construction -from ably.pubsub.util.crypto import CipherParams +from ably.pubsub.util.crypto import CipherParams, generate_random_key, get_default_params from ably.pubsub.util.exceptions import AblyAuthException, AblyException, IncompatibleClientIdException from ably.pubsub.vcdiff.defaultvcdiffdecoder import AblyVCDiffDecoder from ably.pubsub.version import api_version, lib_version @@ -275,5 +275,7 @@ def create_realtime_client(**kwargs) -> PubSubRealtimeClient: 'api_version', 'create_http_client', 'create_realtime_client', + 'generate_random_key', + 'get_default_params', 'lib_version', ] diff --git a/ably/pubsub/server/sync.py b/ably/pubsub/server/sync.py index 5ccab431..edcf8fb9 100644 --- a/ably/pubsub/server/sync.py +++ b/ably/pubsub/server/sync.py @@ -24,7 +24,7 @@ from ably.pubsub.sync.types.options import Options, VCDiffDecoder from ably.pubsub.sync.types.tokendetails import TokenDetails from ably.pubsub.sync.util.construction import factory_construction -from ably.pubsub.sync.util.crypto import CipherParams +from ably.pubsub.sync.util.crypto import CipherParams, generate_random_key, get_default_params from ably.pubsub.sync.util.exceptions import AblyAuthException, AblyException, IncompatibleClientIdException from ably.pubsub.sync.vcdiff.defaultvcdiffdecoder import AblyVCDiffDecoder from ably.pubsub.version import api_version, lib_version @@ -84,5 +84,7 @@ def create_http_client(**kwargs) -> PubSubHttpClient: 'VCDiffDecoder', 'api_version', 'create_http_client', + 'generate_random_key', + 'get_default_params', 'lib_version', ] diff --git a/ably/pubsub/util/nocrypto.py b/ably/pubsub/util/nocrypto.py index a66669b3..b6b71265 100644 --- a/ably/pubsub/util/nocrypto.py +++ b/ably/pubsub/util/nocrypto.py @@ -2,7 +2,7 @@ class InstallPycrypto: def __getattr__(self, name): raise ImportError( - "This requires to install ably with crypto support: pip install 'ably[crypto]'" + "This requires ably-pubsub-server with crypto support: pip install 'ably-pubsub-server[crypto]'" ) diff --git a/test/unit/pubsub_server_test.py b/test/unit/pubsub_server_test.py index a06d2582..a806f56e 100644 --- a/test/unit/pubsub_server_test.py +++ b/test/unit/pubsub_server_test.py @@ -161,3 +161,9 @@ class TestPublicTypes: def test_message_is_exported(self): assert 'Message' in server.__all__ assert 'Message' in server_sync.__all__ + + # RSE2: channel encryption needs a public way to generate a key and build cipher params + @pytest.mark.parametrize('name', ['generate_random_key', 'get_default_params']) + def test_crypto_helpers_are_exported(self, name): + assert name in server.__all__ + assert name in server_sync.__all__ From a920dc25976f18baa587666c6afc544a681b4f2a Mon Sep 17 00:00:00 2001 From: owenpearson Date: Fri, 9 Oct 2026 13:09:22 +0100 Subject: [PATCH 13/71] test: derive the LiveObjects unit specifications Derives the fifteen specifications under uts/objects/unit, 322 tests, and the shared helper uts/objects/helpers/standard_test_pool.md describes: the standard pool, its wire builders, the synced-channel setup and the REST provisioning the integration tier uses. The pure specifications drive the CRDT objects, the pool and the sync state machine directly; the rest run against a mock websocket through channel.object. The public API is reached through the typed views of LODR-061, so a specification's untyped value(), set() or entries() reads as as_live_counter().value(), as_live_map().set() and so on. Where a specification step cannot fail, or reads state before the frame that changes it has been processed, the test carries the corrected step under a UTS SPEC ERROR comment and keeps the specification's own assertions. Co-Authored-By: Claude Opus 5.5 (1M context) --- test/uts/objects/__init__.py | 0 test/uts/objects/helpers/__init__.py | 0 .../uts/objects/helpers/standard_test_pool.py | 514 ++++++++++ .../helpers/standard_test_pool_test.py | 292 ++++++ test/uts/objects/unit/__init__.py | 0 test/uts/objects/unit/instance_test.py | 353 +++++++ .../unit/internal_live_counter_api_test.py | 142 +++ .../unit/internal_live_counter_test.py | 471 +++++++++ .../unit/internal_live_map_api_test.py | 300 ++++++ .../objects/unit/internal_live_map_test.py | 928 +++++++++++++++++ .../unit/live_object_subscribe_test.py | 321 ++++++ test/uts/objects/unit/object_id_test.py | 99 ++ test/uts/objects/unit/objects_pool_test.py | 687 +++++++++++++ .../objects/unit/parent_references_test.py | 433 ++++++++ .../unit/path_object_mutations_test.py | 152 +++ .../unit/path_object_subscribe_test.py | 553 ++++++++++ test/uts/objects/unit/path_object_test.py | 363 +++++++ .../unit/public_object_message_test.py | 345 +++++++ test/uts/objects/unit/realtime_object_test.py | 951 ++++++++++++++++++ test/uts/objects/unit/value_types_test.py | 291 ++++++ 20 files changed, 7195 insertions(+) create mode 100644 test/uts/objects/__init__.py create mode 100644 test/uts/objects/helpers/__init__.py create mode 100644 test/uts/objects/helpers/standard_test_pool.py create mode 100644 test/uts/objects/helpers/standard_test_pool_test.py create mode 100644 test/uts/objects/unit/__init__.py create mode 100644 test/uts/objects/unit/instance_test.py create mode 100644 test/uts/objects/unit/internal_live_counter_api_test.py create mode 100644 test/uts/objects/unit/internal_live_counter_test.py create mode 100644 test/uts/objects/unit/internal_live_map_api_test.py create mode 100644 test/uts/objects/unit/internal_live_map_test.py create mode 100644 test/uts/objects/unit/live_object_subscribe_test.py create mode 100644 test/uts/objects/unit/object_id_test.py create mode 100644 test/uts/objects/unit/objects_pool_test.py create mode 100644 test/uts/objects/unit/parent_references_test.py create mode 100644 test/uts/objects/unit/path_object_mutations_test.py create mode 100644 test/uts/objects/unit/path_object_subscribe_test.py create mode 100644 test/uts/objects/unit/path_object_test.py create mode 100644 test/uts/objects/unit/public_object_message_test.py create mode 100644 test/uts/objects/unit/realtime_object_test.py create mode 100644 test/uts/objects/unit/value_types_test.py diff --git a/test/uts/objects/__init__.py b/test/uts/objects/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/test/uts/objects/helpers/__init__.py b/test/uts/objects/helpers/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/test/uts/objects/helpers/standard_test_pool.py b/test/uts/objects/helpers/standard_test_pool.py new file mode 100644 index 00000000..1507b1ea --- /dev/null +++ b/test/uts/objects/helpers/standard_test_pool.py @@ -0,0 +1,514 @@ +"""The fixtures every LiveObjects specification shares. + +Derived from uts/objects/helpers/standard_test_pool.md in ably/specification: the +standard pool of objects, the canonical serials, the ObjectMessage and ProtocolMessage +builders, the synced-channel setup the mock-backed specifications open with, the +negative-assertion quiescence pattern, and REST provisioning for the integration tier. + +Every builder returns the JSON-wire dictionary the specification describes, ready for +`mock_ws.send_to_client(...)`: camelCase keys, numeric operation actions and map +semantics, `json` values as JSON-encoded strings and `bytes` values as base64 strings. +The specifications write actions and semantics by name; here they are the numbers the +wire carries, so pass `LWW` where a specification writes `semantics: "LWW"`. Templates +shared at module level (`STANDARD_POOL_OBJECTS`) must not be mutated; build a variant +instead. A test that drives the msgpack wire converts a dictionary first with +`ObjectMessage.from_dict(d).to_dict('msgpack')`. + +The pure unit specifications construct internal objects rather than wire messages; +`object_message()` and `object_messages()` decode a builder's output into the internal +`ObjectMessage` they take, and `capture_updates()` records the updates an object emits, +since `apply_operation` returns whether it applied rather than the update itself. +""" + +import base64 +import copy +import json +from typing import NamedTuple +from urllib.parse import quote + +from ably.pubsub.objects import publicmessage +from ably.pubsub.objects.objectmessage import ( + WIRE_FORMAT_JSON, + ObjectMessage, + ObjectOperationAction, + ObjectsMapSemantics, +) +from ably.pubsub.transport.websockettransport import ProtocolMessageAction +from ably.pubsub.types.channelmode import ChannelMode +from ably.pubsub.types.channeloptions import ChannelOptions +from ably.pubsub.types.flags import Flag +from ably.pubsub.util.clock import Clock +from test.uts.helpers.client import poll_until, realtime_client, sandbox_rest_client +from test.uts.helpers.clock import settle +from test.uts.helpers.mock_http import MockHttpClient +from test.uts.helpers.mock_websocket import MockWebSocket + +# --- Canonical constants --------------------------------------------------------------- + +# The siteCode of the harness connection, under which the client applies its own +# operations on ACK +SITE_CODE = 'test-site' + +# The timeserial every standard-pool object and entry is seeded with. Serials compare +# lexicographically (RTLM9e), so every synthetic serial below is defined relative to it. +POOL_SERIAL = 't:0' + +CONNECTION_ID = 'conn-1' +CONNECTION_KEY = 'conn-key-1' + +# RTO10b3's default, which the harness CONNECTED message states explicitly +GC_GRACE_PERIOD = 86400000 + +# The wire values the specifications write by name +HAS_OBJECTS = int(Flag.HAS_OBJECTS) +OBJECT_SUBSCRIBE_FLAG = int(Flag.OBJECT_SUBSCRIBE) +OBJECT_PUBLISH_FLAG = int(Flag.OBJECT_PUBLISH) +LWW = int(ObjectsMapSemantics.LWW) + +ACK = int(ProtocolMessageAction.ACK) +ATTACH = int(ProtocolMessageAction.ATTACH) +ATTACHED = int(ProtocolMessageAction.ATTACHED) +CONNECTED = int(ProtocolMessageAction.CONNECTED) +DETACH = int(ProtocolMessageAction.DETACH) +DETACHED = int(ProtocolMessageAction.DETACHED) +ERROR = int(ProtocolMessageAction.ERROR) +OBJECT = int(ProtocolMessageAction.OBJECT) +OBJECT_SYNC = int(ProtocolMessageAction.OBJECT_SYNC) + +# The modes every standard channel requests +OBJECTS_MODES = (ChannelMode.OBJECT_SUBSCRIBE, ChannelMode.OBJECT_PUBLISH) + +# The X-Ably-Version the objects REST endpoint is called with: protocol v6, whose +# operation payloads (`mapSet`, `counterInc` and the rest) are what the specification's +# provisioning operations are written in +OBJECTS_REST_API_VERSION = '6' + + +# --- Serials ---------------------------------------------------------------------------- + +def ack_serial(msg_serial, index): + """The serial the harness ACKs operation `index` of the publish `msg_serial` with. + + The first publish's first operation is `ack_serial(0, 0) == 't:1:0'`, which sorts after + `POOL_SERIAL`. These serials are recorded in appliedOnAckSerials (RTO9a2a4) and an echo + carrying one is discarded (RTO9a3), so never reuse one as an inbound serial meant to apply. + """ + return f't:{msg_serial + 1}:{index}' + + +def remote_serial(index): + """A serial for a remote MAP_SET or MAP_REMOVE that wins LWW against a pool entry. + + 0-based: `remote_serial(0) == 't:1'`. A bare number such as `'99'` sorts before `'t:0'` + and would be rejected as stale. + """ + return f't:{index + 1}' + + +def below_ack_serial(index): + """A serial that is not an ACK serial, sorting after `POOL_SERIAL` but before `ack_serial(0, 0)`. + + 0-based: `below_ack_serial(9) == 't:0:9'`. + """ + return f't:0:{index}' + + +# --- Values ----------------------------------------------------------------------------- + +def json_value(value): + """An `ObjectData` wire dictionary carrying `value` as JSON (OD2g).""" + return {'json': json.dumps(value)} + + +def bytes_value(value): + """An `ObjectData` wire dictionary carrying `value` as binary (OD2d).""" + return {'bytes': base64.b64encode(value).decode('ascii')} + + +# --- ProtocolMessage builders ----------------------------------------------------------- + +def build_object_sync_message(channel, channel_serial, object_messages): + """An OBJECT_SYNC ProtocolMessage. A `channel_serial` of None leaves it out (RTO5a5).""" + message = {'action': OBJECT_SYNC, 'channel': channel, 'state': list(object_messages)} + if channel_serial is not None: + message['channelSerial'] = channel_serial + return message + + +def build_object_message(channel, object_messages): + """An OBJECT ProtocolMessage.""" + return {'action': OBJECT, 'channel': channel, 'state': list(object_messages)} + + +def build_ack_message(msg_serial, serials): + """An ACK for one ProtocolMessage, carrying a `PublishResult` with `serials` (TR4s).""" + return {'action': ACK, 'msgSerial': msg_serial, 'count': 1, 'res': [{'serials': list(serials)}]} + + +# --- ObjectMessage builders: operations ------------------------------------------------- + +def _operation_message(serial, site_code, operation, serial_timestamp=None): + message = {'serial': serial, 'siteCode': site_code, 'operation': operation} + if serial_timestamp is not None: + message['serialTimestamp'] = serial_timestamp + return message + + +def build_counter_inc(object_id, number, serial, site_code): + return _operation_message(serial, site_code, { + 'action': int(ObjectOperationAction.COUNTER_INC), + 'objectId': object_id, + 'counterInc': {'number': number}, + }) + + +def build_map_set(object_id, key, value, serial, site_code): + """A MAP_SET; `value` is an `ObjectData` wire dictionary, such as `{'string': 'Bob'}`.""" + return _operation_message(serial, site_code, { + 'action': int(ObjectOperationAction.MAP_SET), + 'objectId': object_id, + 'mapSet': {'key': key, 'value': value}, + }) + + +def build_map_remove(object_id, key, serial, site_code, serial_timestamp=None): + return _operation_message(serial, site_code, { + 'action': int(ObjectOperationAction.MAP_REMOVE), + 'objectId': object_id, + 'mapRemove': {'key': key}, + }, serial_timestamp) + + +def build_map_clear(object_id, serial, site_code): + return _operation_message(serial, site_code, { + 'action': int(ObjectOperationAction.MAP_CLEAR), + 'objectId': object_id, + }) + + +def build_object_delete(object_id, serial, site_code, serial_timestamp=None): + return _operation_message(serial, site_code, { + 'action': int(ObjectOperationAction.OBJECT_DELETE), + 'objectId': object_id, + }, serial_timestamp) + + +def build_counter_create(object_id, counter_create, serial, site_code): + """A COUNTER_CREATE; `counter_create` is the wire payload, such as `{'count': 42}`.""" + return _operation_message(serial, site_code, { + 'action': int(ObjectOperationAction.COUNTER_CREATE), + 'objectId': object_id, + 'counterCreate': counter_create, + }) + + +def build_map_create(object_id, map_create, serial, site_code): + """A MAP_CREATE; `map_create` is the wire payload, `{'semantics': LWW, 'entries': {...}}`.""" + return _operation_message(serial, site_code, { + 'action': int(ObjectOperationAction.MAP_CREATE), + 'objectId': object_id, + 'mapCreate': map_create, + }) + + +# --- ObjectMessage builders: state ------------------------------------------------------ + +def build_object_state(object_id, site_timeserials, map=None, counter=None, tombstone=None, create_op=None): + """An OBJECT_SYNC ObjectMessage wrapping an `ObjectState`. + + `map`, `counter`, `tombstone` and `create_op` are the specification's `opts`, as wire + dictionaries. A terse `create_op` such as `{'counterCreate': {...}}` has its mandatory + `objectId` and `action` filled in (OOP2), as the specification's builder does. + """ + state = {'objectId': object_id, 'siteTimeserials': dict(site_timeserials)} + if map is not None: + state['map'] = map + if counter is not None: + state['counter'] = counter + if tombstone is not None: + state['tombstone'] = tombstone + if create_op is not None: + create_op = dict(create_op) + create_op.setdefault('objectId', object_id) + if 'action' not in create_op: + create_op['action'] = int(ObjectOperationAction.COUNTER_CREATE if 'counterCreate' in create_op + else ObjectOperationAction.MAP_CREATE) + state['createOp'] = create_op + return {'object': state} + + +def build_object_message_with_state(object_state): + """An ObjectMessage wrapping an existing `ObjectState` wire dictionary.""" + return {'object': object_state} + + +# The standard tree. Counters carry their value in the create operation, with +# `counter.count` 0, so that applying the state yields 100 and 5 rather than double that. +STANDARD_POOL_OBJECTS = ( + build_object_state('root', {'aaa': POOL_SERIAL}, map={ + 'semantics': LWW, + 'entries': { + 'name': {'data': {'string': 'Alice'}, 'timeserial': POOL_SERIAL}, + 'age': {'data': {'number': 30}, 'timeserial': POOL_SERIAL}, + 'active': {'data': {'boolean': True}, 'timeserial': POOL_SERIAL}, + 'score': {'data': {'objectId': 'counter:score@1000'}, 'timeserial': POOL_SERIAL}, + 'profile': {'data': {'objectId': 'map:profile@1000'}, 'timeserial': POOL_SERIAL}, + 'data': {'data': json_value({'tags': ['a', 'b']}), 'timeserial': POOL_SERIAL}, + 'avatar': {'data': {'bytes': 'AQID'}, 'timeserial': POOL_SERIAL}, + }, + }, create_op={'mapCreate': {'semantics': LWW, 'entries': {}}}), + build_object_state('counter:score@1000', {'aaa': POOL_SERIAL}, counter={'count': 0}, + create_op={'counterCreate': {'count': 100}}), + build_object_state('map:profile@1000', {'aaa': POOL_SERIAL}, map={ + 'semantics': LWW, + 'entries': { + 'email': {'data': {'string': 'alice@example.com'}, 'timeserial': POOL_SERIAL}, + 'nested_counter': {'data': {'objectId': 'counter:nested@1000'}, 'timeserial': POOL_SERIAL}, + 'prefs': {'data': {'objectId': 'map:prefs@1000'}, 'timeserial': POOL_SERIAL}, + }, + }, create_op={'mapCreate': {'semantics': LWW, 'entries': {}}}), + build_object_state('counter:nested@1000', {'aaa': POOL_SERIAL}, counter={'count': 0}, + create_op={'counterCreate': {'count': 5}}), + build_object_state('map:prefs@1000', {'aaa': POOL_SERIAL}, map={ + 'semantics': LWW, + 'entries': { + 'theme': {'data': {'string': 'dark'}, 'timeserial': POOL_SERIAL}, + }, + }, create_op={'mapCreate': {'semantics': LWW, 'entries': {}}}), +) + + +# --- Internal objects for the pure unit specifications ---------------------------------- + +def object_message(wire, format=WIRE_FORMAT_JSON): + """The internal `ObjectMessage` a builder's wire dictionary decodes to.""" + return ObjectMessage.from_dict(wire, format) + + +def object_messages(protocol_message, format=WIRE_FORMAT_JSON): + """The internal ObjectMessages an OBJECT or OBJECT_SYNC ProtocolMessage's `state` decodes to.""" + return ObjectMessage.from_protocol_message(protocol_message, format) + + +def capture_updates(live_object): + """Records every update `live_object` emits, the no-ops included, and returns the list. + + `apply_operation` returns whether the operation applied (RTLC7g, RTLM15g) and emits the + resulting update through `notify_updated`, so a specification's + `update = obj.applyOperation(...)` is `updates[-1]` after the call. The object's own + `notify_updated` still runs. + """ + updates = [] + original = live_object.notify_updated + + def notify_updated(update): + updates.append(update) + original(update) + + live_object.notify_updated = notify_updated + return updates + + +def build_public_object_message(message, channel_name): + """The public `ObjectMessage` PAOM3 derives from `message` received on `channel_name`. + + `message` is an internal `ObjectMessage` or a builder's wire dictionary. This is the + expected value a subscription test compares against, built here independently of the + library's own derivation. + """ + if isinstance(message, dict): + message = object_message(message) + operation = message.operation + public_operation = publicmessage.ObjectOperation( + action=operation.action, + object_id=operation.object_id, + map_create=operation.resolved_map_create, + map_set=operation.map_set, + map_remove=operation.map_remove, + counter_create=operation.resolved_counter_create, + counter_inc=operation.counter_inc, + object_delete=operation.object_delete, + map_clear=operation.map_clear, + ) + return publicmessage.ObjectMessage( + channel=channel_name, + operation=public_operation, + id=message.id, + client_id=message.client_id, + connection_id=message.connection_id, + timestamp=message.timestamp, + serial=message.serial, + serial_timestamp=message.serial_timestamp, + site_code=message.site_code, + extras=message.extras, + ) + + +# --- The mock-backed client ------------------------------------------------------------- + +def objects_connected_message(site_code=SITE_CODE, objects_gc_grace_period=GC_GRACE_PERIOD, + connection_id=CONNECTION_ID, connection_key=CONNECTION_KEY): + """The harness CONNECTED message, carrying `siteCode` and `objectsGCGracePeriod`. + + Passing None for either leaves it out. `connectionId` travels on the ProtocolMessage + itself, which is where the library reads it, though the specification writes it among + the `connectionDetails`. `maxIdleInterval` is 0 so that the transport schedules no + idle timer, which a `FakeClock` test would otherwise fire. + """ + details = {'connectionKey': connection_key, 'connectionStateTtl': 120000, 'maxIdleInterval': 0} + if site_code is not None: + details['siteCode'] = site_code + if objects_gc_grace_period is not None: + details['objectsGCGracePeriod'] = objects_gc_grace_period + return {'action': CONNECTED, 'connectionId': connection_id, 'connectionDetails': details} + + +def objects_attached_message(channel, channel_serial='sync1:', flags=HAS_OBJECTS): + """An ATTACHED for `channel`. + + The specifications write the modes a server grants as `modes: [...]`; on the wire they + are bits of `flags`, so pass `HAS_OBJECTS | OBJECT_SUBSCRIBE_FLAG` for + `flags: HAS_OBJECTS, modes: ["OBJECT_SUBSCRIBE"]`. + """ + return {'action': ATTACHED, 'channel': channel, 'channelSerial': channel_serial, 'flags': flags} + + +def standard_mock_websocket(auto_ack=True, on_object=None, connected=None, attached_channel_serial='sync1:', + attached_flags=HAS_OBJECTS, sync_objects=STANDARD_POOL_OBJECTS, + sync_channel_serial='sync1:'): + """The specification's synced-channel `MockWebSocket`. + + It accepts every connection with `connected` (by default `objects_connected_message()`); + answers an ATTACH with an ATTACHED and then an OBJECT_SYNC of `sync_objects`, or no + OBJECT_SYNC if `sync_objects` is None; answers a DETACH with a DETACHED; and passes each + OBJECT the client sends to `on_object`, then, with `auto_ack`, ACKs it with + `ack_serial(msgSerial, i)` for each of its operations. + """ + connected_message = connected if connected is not None else objects_connected_message() + mock_ws = MockWebSocket(on_connection_attempt=lambda conn: conn.respond_with_success(connected_message)) + + def on_message_from_client(message): + action = message.get('action') + if action == ATTACH: + channel = message.get('channel') + mock_ws.send_to_client(objects_attached_message(channel, attached_channel_serial, attached_flags)) + if sync_objects is not None: + mock_ws.send_to_client(build_object_sync_message(channel, sync_channel_serial, sync_objects)) + elif action == OBJECT: + if on_object is not None: + on_object(message) + if auto_ack: + msg_serial = message['msgSerial'] + serials = [ack_serial(msg_serial, i) for i in range(len(message.get('state') or []))] + mock_ws.send_to_client(build_ack_message(msg_serial, serials)) + elif action == DETACH: + mock_ws.send_to_client({'action': DETACHED, 'channel': message.get('channel')}) + + mock_ws.on_message_from_client = on_message_from_client + return mock_ws + + +def time_mock_http(clock=None): + """A `MockHttpClient` answering `GET /time` with the time on `clock`. + + Creating an object reads the server time (RTLCV4e, RTLMV4h, RTO16), and a mock-backed + client must not reach the network for it. Any other request is answered 404. + """ + time_source = clock if clock is not None else Clock() + + def on_request(request): + if request.method == 'GET' and request.path == '/time': + request.respond_with(200, [time_source.now_ms()]) + else: + request.respond_with(404, {'error': {'code': 40400, 'statusCode': 404, 'message': 'Not found'}}) + + return MockHttpClient(on_connection_attempt=lambda conn: conn.respond_with_success(), on_request=on_request) + + +def objects_client(mock_ws, clock=None, mock_http=None, **kwargs): + """A realtime client on `mock_ws`, as the specifications' `Realtime(options: {key: ...})`. + + It connects on its own, speaks JSON so that the builders' dictionaries go over the mock + as they are, and has `GET /time` answered by `time_mock_http(clock)` unless `mock_http` + is given. `clock` is a `FakeClock` for a specification's `enable_fake_timers()`. + """ + kwargs.setdefault('auto_connect', True) + kwargs.setdefault('use_binary_protocol', False) + if mock_http is None: + mock_http = time_mock_http(clock) + return realtime_client(mock_ws, mock_http=mock_http, clock=clock, **kwargs) + + +def objects_channel_options(*modes): + """`ChannelOptions` requesting `modes`, by default OBJECT_SUBSCRIBE and OBJECT_PUBLISH.""" + return ChannelOptions(modes=list(modes or OBJECTS_MODES)) + + +class SyncedChannel(NamedTuple): + """What `setup_synced_channel` returns. It unpacks as `client, channel, root, mock_ws`.""" + + client: object + channel: object + root: object + mock_ws: MockWebSocket + + +async def setup_synced_channel(channel_name='test', mock_ws=None, clock=None, modes=OBJECTS_MODES, + **client_options): + """The specification's `setup_synced_channel(channel_name)`. + + Connects a client to `mock_ws` (by default `standard_mock_websocket()`), gets + `channel_name` with `modes`, and awaits `channel.object.get()`, which attaches the + channel and waits for the standard pool to sync. `clock` and `client_options` go to + `objects_client`. + """ + if mock_ws is None: + mock_ws = standard_mock_websocket() + client = objects_client(mock_ws, clock=clock, **client_options) + channel = client.channels.get(channel_name, objects_channel_options(*modes)) + root = await channel.object.get() + return SyncedChannel(client, channel, root, mock_ws) + + +async def setup_synced_channel_no_ack(channel_name='test', **kwargs): + """The specification's `setup_synced_channel_no_ack`: OBJECT messages are recorded but not ACKed.""" + return await setup_synced_channel(channel_name, mock_ws=standard_mock_websocket(auto_ack=False), **kwargs) + + +# --- Negative-assertion quiescence ------------------------------------------------------ + +async def assert_unchanged_after_quiescence(count_under_test, control_delivered, timeout=5.0, + description='the control to be delivered'): + """The specification's `assert_unchanged_after_quiescence(count_under_test, control)`. + + Reads `count_under_test()`, waits until `control_delivered()` holds, which a control + listener (or a follow-up message) on the same dispatch makes true, settles, and asserts + the count has not moved. Send the message under test before calling this, and the + control's stimulus after it. + """ + before = count_under_test() + await poll_until(control_delivered, timeout=timeout, description=description) + await settle() + after = count_under_test() + assert after == before, f'expected the count to stay at {before} once {description}, but it was {after}' + + +# --- REST provisioning for the integration tier ---------------------------------------- + +async def provision_objects_via_rest(api_key, channel_name, operations, client=None): + """POSTs `operations` to the channel's objects REST endpoint, before any realtime client connects. + + `operations` is one operation dictionary or a list of them, in the REST API's format: + `{'mapSet': {'key': ..., 'value': {...}}, 'objectId': 'root'}` and so on. Returns the + `objectIds` of every result, flattened in request order. The REST client goes straight + to the sandbox and is closed when the test ends; `client` replaces it. + """ + if client is None: + client = sandbox_rest_client(api_key) + path = f"/channels/{quote(channel_name, safe='')}/object" + response = await client.request('POST', path, OBJECTS_REST_API_VERSION, body=copy.deepcopy(operations)) + if not response.success: + raise AssertionError( + f'Provisioning objects on {channel_name!r} failed: {response.status_code} ' + f'{response.error_code} {response.error_message}') + return [object_id for result in response.items for object_id in (result.get('objectIds') or [])] diff --git a/test/uts/objects/helpers/standard_test_pool_test.py b/test/uts/objects/helpers/standard_test_pool_test.py new file mode 100644 index 00000000..0f07d103 --- /dev/null +++ b/test/uts/objects/helpers/standard_test_pool_test.py @@ -0,0 +1,292 @@ +"""Tests for the `standard_test_pool` helper and the object wire types it builds on. + +These are not derived from a specification. They pin the builders to the wire shapes +`standard_test_pool.md` describes, the canonical serials to the ordering the +specifications rely on, the wire types to a lossless round trip on both protocols, and +the mock to the synced-channel conversation, which it can hold before any LiveObjects +behaviour exists. +""" + +import json + +import msgpack + +from ably.pubsub.objects import publicmessage +from ably.pubsub.objects.objectmessage import ( + WIRE_FORMAT_MSGPACK, + CounterCreate, + MapCreate, + MapCreateWithObjectId, + ObjectData, + ObjectMessage, + ObjectOperation, + ObjectOperationAction, + ObjectsMapSemantics, +) +from ably.pubsub.objects.realtimeobject import RealtimeObject +from ably.pubsub.realtime.connection import ConnectionState +from ably.pubsub.types.channelmode import ChannelMode, decode_channel_mode +from ably.pubsub.types.channelstate import ChannelState +from test.uts.helpers.client import await_channel_state, await_connection_state, poll_until, rest_client +from test.uts.helpers.clock import settle +from test.uts.helpers.mock_http import MockHttpClient +from test.uts.helpers.mock_websocket import MockEventType +from test.uts.objects.helpers.standard_test_pool import ( + GC_GRACE_PERIOD, + HAS_OBJECTS, + LWW, + OBJECT_SUBSCRIBE_FLAG, + OBJECT_SYNC, + POOL_SERIAL, + SITE_CODE, + STANDARD_POOL_OBJECTS, + ack_serial, + assert_unchanged_after_quiescence, + below_ack_serial, + build_ack_message, + build_counter_inc, + build_map_remove, + build_map_set, + build_object_delete, + build_object_message, + build_object_state, + build_object_sync_message, + build_public_object_message, + bytes_value, + capture_updates, + json_value, + object_message, + object_messages, + objects_channel_options, + objects_client, + provision_objects_via_rest, + remote_serial, + standard_mock_websocket, +) + + +def msgpack_round_trip(message): + """`message` encoded for the msgpack wire, sent through msgpack and decoded again.""" + raw = msgpack.packb(message.to_dict(WIRE_FORMAT_MSGPACK), use_bin_type=True) + return ObjectMessage.from_dict(msgpack.unpackb(raw, raw=False), WIRE_FORMAT_MSGPACK) + + +def test_serials_sort_as_the_specification_requires(): + assert ack_serial(0, 0) == 't:1:0' + assert remote_serial(0) == 't:1' + assert below_ack_serial(9) == 't:0:9' + # A remote write and the first ACK both beat the pool, and the probe sits between them + assert POOL_SERIAL < below_ack_serial(9) < ack_serial(0, 0) + assert remote_serial(0) > POOL_SERIAL + + +def test_operation_builders_produce_the_json_wire_shape(): + assert build_counter_inc('counter:a@1', 5, '01', 'site1') == { + 'serial': '01', 'siteCode': 'site1', + 'operation': {'action': 4, 'objectId': 'counter:a@1', 'counterInc': {'number': 5}}, + } + assert build_map_set('root', 'k', {'string': 'v'}, '02', 'site1')['operation'] == { + 'action': 1, 'objectId': 'root', 'mapSet': {'key': 'k', 'value': {'string': 'v'}}, + } + assert 'serialTimestamp' not in build_map_remove('root', 'k', '03', 'site1') + assert build_object_delete('counter:a@1', '04', 'site1', 1700000000000)['serialTimestamp'] == 1700000000000 + + +def test_protocol_message_builders(): + sync = build_object_sync_message('test', 'sync1:', STANDARD_POOL_OBJECTS) + assert sync['action'] == OBJECT_SYNC + assert sync['channelSerial'] == 'sync1:' + assert len(sync['state']) == 5 + # RTO5a5: a sync with no channelSerial carries no key at all + assert 'channelSerial' not in build_object_sync_message('test', None, []) + assert build_object_message('test', [])['action'] == 19 + assert build_ack_message(3, ['a', None]) == {'action': 1, 'msgSerial': 3, 'count': 1, + 'res': [{'serials': ['a', None]}]} + + +def test_object_state_builder_fills_in_the_create_op(): + counter = build_object_state('counter:a@1', {'s': '1'}, counter={'count': 0}, + create_op={'counterCreate': {'count': 7}}) + assert counter['object']['createOp'] == {'counterCreate': {'count': 7}, 'objectId': 'counter:a@1', + 'action': int(ObjectOperationAction.COUNTER_CREATE)} + a_map = build_object_state('map:a@1', {}, map={'semantics': LWW, 'entries': {}}, + create_op={'mapCreate': {'semantics': LWW, 'entries': {}}}, tombstone=True) + assert a_map['object']['createOp']['action'] == int(ObjectOperationAction.MAP_CREATE) + assert a_map['object']['tombstone'] is True + assert 'tombstone' not in counter['object'] + + +def test_standard_pool_decodes_to_the_specified_tree(): + root, score, profile, nested, prefs = (object_message(message).object for message in STANDARD_POOL_OBJECTS) + + assert root.object_id == 'root' + assert root.site_timeserials == {'aaa': POOL_SERIAL} + assert root.map.semantics is ObjectsMapSemantics.LWW + entries = root.map.entries + assert entries['name'].data == ObjectData(string='Alice') + assert entries['age'].data == ObjectData(number=30.0) + assert entries['active'].data == ObjectData(boolean=True) + assert entries['score'].data == ObjectData(object_id='counter:score@1000') + assert entries['data'].data.json == {'tags': ['a', 'b']} + assert entries['avatar'].data.bytes == bytes([1, 2, 3]) + assert all(entry.timeserial == POOL_SERIAL and not entry.tombstone for entry in entries.values()) + assert root.create_op.action is ObjectOperationAction.MAP_CREATE + assert root.create_op.object_id == 'root' + + assert score.counter.count == 0 + assert score.create_op.counter_create == CounterCreate(count=100.0) + assert nested.create_op.counter_create.count == 5 + assert set(profile.map.entries) == {'email', 'nested_counter', 'prefs'} + assert prefs.map.entries['theme'].data.string == 'dark' + + +def test_values_carry_json_and_binary_as_the_json_wire_does(): + assert json_value({'a': [1]}) == {'json': '{"a": [1]}'} + assert bytes_value(bytes([1, 2, 3])) == {'bytes': 'AQID'} + data = ObjectData.from_dict({'json': '{"a": [1]}'}) + assert data.json == {'a': [1]} + assert data.to_dict() == {'json': '{"a":[1]}'} + + +def test_wire_types_round_trip_through_json(): + for wire in [*STANDARD_POOL_OBJECTS, build_counter_inc('counter:a@1', 5, '01', 'site1'), + build_map_set('root', 'k', bytes_value(b'\x00\xff'), '02', 'site1'), + build_map_remove('root', 'k', '03', 'site1', 1700000000000)]: + decoded = object_message(wire) + assert object_message(json.loads(json.dumps(decoded.to_dict()))) == decoded + + +def test_wire_types_round_trip_through_msgpack(): + for wire in [*STANDARD_POOL_OBJECTS, build_map_set('root', 'k', bytes_value(b'\x00\xff'), '02', 'site1')]: + decoded = object_message(wire) + assert msgpack_round_trip(decoded) == decoded + # Binary is raw on the msgpack wire and base64 on the JSON one (OD2d) + avatar = object_message(STANDARD_POOL_OBJECTS[0]).object.map.entries['avatar'].data + assert avatar.to_dict(WIRE_FORMAT_MSGPACK) == {'bytes': bytes([1, 2, 3])} + assert avatar.to_dict() == {'bytes': 'AQID'} + + +def test_local_only_fields_are_never_encoded(): + derived = MapCreate(semantics=ObjectsMapSemantics.LWW, entries={}) + operation = ObjectOperation( + action=ObjectOperationAction.MAP_CREATE, object_id='map:a@1', + map_create_with_object_id=MapCreateWithObjectId(initial_value='{}', nonce='n' * 16, derived_from=derived)) + assert operation.to_dict() == { + 'action': 0, 'objectId': 'map:a@1', 'mapCreateWithObjectId': {'initialValue': '{}', 'nonce': 'n' * 16}, + } + assert operation.resolved_map_create is derived + entry = object_message(STANDARD_POOL_OBJECTS[0]).object.map.entries['name'] + entry.tombstoned_at = 1700000000000 + assert 'tombstonedAt' not in json.dumps(entry.to_dict()) + + +def test_unknown_action_decodes_without_failing(): + message = object_message({'serial': '01', 'siteCode': 's', 'operation': {'action': 99, 'objectId': 'x'}}) + assert message.operation.action is ObjectOperationAction.UNKNOWN + + +def test_object_messages_take_identity_from_the_protocol_message(): + protocol_message = build_object_message('test', [build_counter_inc('counter:a@1', 1, '01', 's'), + build_counter_inc('counter:a@1', 2, '02', 's')]) + protocol_message.update(id='pm-id', connectionId='conn-x', timestamp=1234) + first, second = object_messages(protocol_message) + assert (first.id, second.id) == ('pm-id:0', 'pm-id:1') + assert first.connection_id == 'conn-x' + assert second.timestamp == 1234 + + +def test_public_object_message_follows_paom3(): + wire = build_counter_inc('counter:score@1000', 42, 'serial-1', 'site-a') + wire['clientId'] = 'client-1' + public = build_public_object_message(wire, 'test') + assert isinstance(public, publicmessage.ObjectMessage) + assert public.channel == 'test' + assert public.serial == 'serial-1' + assert public.site_code == 'site-a' + assert public.client_id == 'client-1' + assert public.operation.action is ObjectOperationAction.COUNTER_INC + assert public.operation.counter_inc.number == 42 + assert public.operation.map_create is None + + +def test_capture_updates_records_and_delegates(): + delivered = [] + + class Recorder: + def notify_updated(self, update): + delivered.append(update) + + recorder = Recorder() + updates = capture_updates(recorder) + recorder.notify_updated('first') + recorder.notify_updated('second') + assert updates == ['first', 'second'] + assert delivered == ['first', 'second'] + + +async def test_quiescence_passes_when_only_the_control_moves(): + under_test = [] + control = ['delivered'] + await assert_unchanged_after_quiescence(lambda: len(under_test), lambda: len(control) >= 1) + + +async def test_standard_mock_holds_the_synced_channel_conversation(): + mock_ws = standard_mock_websocket() + client = objects_client(mock_ws) + await await_connection_state(client, ConnectionState.CONNECTED) + + details = client.connection.connection_details + assert details.site_code == SITE_CODE + assert details.objects_gc_grace_period == GC_GRACE_PERIOD + + channel = client.channels.get('test', objects_channel_options()) + await channel.attach() + await poll_until(lambda: any(event.data.get('action') == OBJECT_SYNC + for event in mock_ws.events_of_type(MockEventType.MESSAGE_TO_CLIENT)), + description='the OBJECT_SYNC to follow the ATTACHED') + attach = next(m for m in mock_ws.messages_from_client if m.get('channel') == 'test') + assert decode_channel_mode(attach['flags']) == [ChannelMode.OBJECT_SUBSCRIBE, ChannelMode.OBJECT_PUBLISH] + + await channel.detach() + await await_channel_state(channel, ChannelState.DETACHED) + + +async def test_granted_modes_decode_from_the_attached_flags(): + mock_ws = standard_mock_websocket(attached_flags=HAS_OBJECTS | OBJECT_SUBSCRIBE_FLAG, sync_objects=None) + client = objects_client(mock_ws) + channel = client.channels.get('test', objects_channel_options()) + await channel.attach() + assert channel.modes == [ChannelMode.OBJECT_SUBSCRIBE] + await settle() + assert not any(event.data.get('action') == OBJECT_SYNC + for event in mock_ws.events_of_type(MockEventType.MESSAGE_TO_CLIENT)) + + +async def test_each_channel_has_one_realtime_object(): + client = objects_client(standard_mock_websocket(), auto_connect=False) + channel = client.channels.get('test') + assert isinstance(channel.object, RealtimeObject) + assert channel.object is channel.object + assert channel.object._channel is channel + assert 'root' in channel.object._objects_pool + + +async def test_provisioning_posts_operations_and_flattens_object_ids(): + captured = [] + + def on_request(request): + captured.append(request) + request.respond_with(201, [{'objectIds': ['root', 'counter:a@1']}, {'objectIds': ['map:b@2']}]) + + mock_http = MockHttpClient(on_connection_attempt=lambda conn: conn.respond_with_success(), + on_request=on_request) + client = rest_client(mock_http, use_binary_protocol=False) + operations = [{'mapSet': {'key': 'k', 'value': {'string': 'v'}}, 'objectId': 'root'}, + {'counterCreate': {'count': 1}}] + + object_ids = await provision_objects_via_rest(None, 'a channel:1', operations, client=client) + + assert object_ids == ['root', 'counter:a@1', 'map:b@2'] + assert captured[0].method == 'POST' + assert captured[0].url.raw_path == '/channels/a%20channel%3A1/object' + assert json.loads(captured[0].body) == operations diff --git a/test/uts/objects/unit/__init__.py b/test/uts/objects/unit/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/test/uts/objects/unit/instance_test.py b/test/uts/objects/unit/instance_test.py new file mode 100644 index 00000000..53eb9a48 --- /dev/null +++ b/test/uts/objects/unit/instance_test.py @@ -0,0 +1,353 @@ +"""Derived from uts/objects/unit/instance.md in ably/specification. + +Spec points: RTINS3, RTINS4, RTINS5, RTINS6, RTINS9, RTINS10, RTINS12, RTINS13, RTINS14, +RTINS15, RTINS16, RTTS8a, RTTS9d, RTTS10 + +The specification is written against the untyped `Instance`, one class carrying every +method. ably-python partitions it per RTTS7-RTTS10: `id` and `type` are properties, `get`, +`compact` and the three view helpers are on the base `Instance`, and a type-specific method +is reached through the checked view helper for the wrapped type, so `inst.value()` on a +counter is `inst.as_live_counter().value()` and `inst.set(k, v)` on a map is +`await inst.as_live_map().set(k, v)`. A view helper asked for a type the instance does not +wrap raises AblyException 92007 (RTTS9d). + +That makes three of the specification's reads unreachable: `value()` on a map instance +(RTINS4d), `size()` on a counter instance (RTINS9c) and `subscribe` on a primitive instance +(RTINS16c), since the typed instance for those types has no such method and the views +that do have it refuse the wrapped type. Those tests assert the wrapped `type`, the +absence of the method, and the 92007 the view helpers raise in its place (S-5). + +`setup_synced_channel` uses the standard mock, which ACKs every OBJECT the client sends, +so each write is awaited to completion and is applied locally by the time it returns +(RTO20). +""" + +import pytest + +from ably.pubsub.objects import publicmessage +from ably.pubsub.objects.enums import ValueType +from ably.pubsub.objects.instance import Instance, InstanceSubscriptionEvent, PrimitiveInstance +from ably.pubsub.objects.objectmessage import ObjectOperationAction +from ably.pubsub.objects.subscription import Subscription +from ably.pubsub.types.channelstate import ChannelState +from ably.pubsub.util.exceptions import AblyException +from test.uts.helpers.client import poll_until +from test.uts.helpers.clock import settle +from test.uts.objects.helpers.standard_test_pool import ( + assert_unchanged_after_quiescence, + build_counter_inc, + build_map_set, + build_object_message, + build_public_object_message, + remote_serial, + setup_synced_channel, +) + + +def assert_view_refused(view_helper): + """Asserts that calling an `Instance` view helper fails fast with 92007 (RTTS9d).""" + with pytest.raises(AblyException) as excinfo: + view_helper() + assert excinfo.value.code == 92007 + assert excinfo.value.status_code == 400 + + +# UTS: objects/unit/RTINS3/id-returns-objectid-0 +async def test_rtins3_id_returns_objectid(): + client, channel, root, mock_ws = await setup_synced_channel('test') + + counter_inst = root.get('score').instance() + assert counter_inst.id == 'counter:score@1000' + + map_inst = root.get('profile').instance() + assert map_inst.id == 'map:profile@1000' + + +# UTS: objects/unit/RTINS4/value-counter-0 +async def test_rtins4_value_counter(): + client, channel, root, mock_ws = await setup_synced_channel('test') + + counter_inst = root.get('score').instance() + assert counter_inst.as_live_counter().value() == 100 + + map_inst = root.instance() + # S-5: the specification asserts `map_inst.value() == null` (RTINS4d). A map instance has + # no `value()` (RTTS10a), and the two views that do have one refuse a map (RTTS9d) + assert map_inst.type is ValueType.LIVE_MAP + assert not hasattr(map_inst, 'value') + assert_view_refused(map_inst.as_primitive) + assert_view_refused(map_inst.as_live_counter) + + +# UTS: objects/unit/RTINS5/get-wraps-entry-0 +async def test_rtins5_get_wraps_entry(): + client, channel, root, mock_ws = await setup_synced_channel('test') + root_inst = root.instance() + + name_inst = root_inst.get('name') + assert isinstance(name_inst, Instance) + assert name_inst.as_primitive().value() == 'Alice' + + score_inst = root_inst.get('score') + assert score_inst.id == 'counter:score@1000' + + null_inst = root_inst.get('nonexistent') + assert null_inst is None + + +# UTS: objects/unit/RTINS6/entries-yields-instances-0 +async def test_rtins6_entries_yields_instances(): + client, channel, root, mock_ws = await setup_synced_channel('test') + root_inst = root.instance() + + entries = {} + for key, inst in root_inst.as_live_map().entries(): + entries[key] = inst + + assert len(entries) == 7 + assert isinstance(entries['name'], Instance) + assert entries['name'].as_primitive().value() == 'Alice' + + +# UTS: objects/unit/RTINS9/size-0 +async def test_rtins9_size(): + client, channel, root, mock_ws = await setup_synced_channel('test') + + root_inst = root.instance() + assert root_inst.as_live_map().size() == 7 + + counter_inst = root.get('score').instance() + # S-5: the specification asserts `counter_inst.size() == null` (RTINS9c). A counter + # instance has no `size()` (RTTS10b), and the map view that has one refuses a counter + # (RTTS9d) + assert counter_inst.type is ValueType.LIVE_COUNTER + assert not hasattr(counter_inst, 'size') + assert_view_refused(counter_inst.as_live_map) + + +# UTS: objects/unit/RTINS10/compact-0 +async def test_rtins10_compact(): + client, channel, root, mock_ws = await setup_synced_channel('test') + root_inst = root.instance() + + result = root_inst.compact() + + assert result['name'] == 'Alice' + assert result['score'] == 100 + assert result['profile']['email'] == 'alice@example.com' + + +# UTS: objects/unit/RTINS12/set-delegates-0 +async def test_rtins12_set_delegates(): + client, channel, root, mock_ws = await setup_synced_channel('test') + root_inst = root.instance() + + await root_inst.as_live_map().set('name', 'Bob') + + assert root.get('name').as_primitive().value() == 'Bob' + + +# UTS: objects/unit/RTINS12d/set-non-map-throws-0 +async def test_rtins12d_set_non_map_throws(): + client, channel, root, mock_ws = await setup_synced_channel('test') + counter_inst = root.get('score').instance() + + # `set` is reached through the map view, which refuses a counter with the 92007 and + # status 400 that RTINS12d requires of `set` (RTTS9d) + with pytest.raises(AblyException) as excinfo: + await counter_inst.as_live_map().set('key', 'value') + + assert excinfo.value.code == 92007 + assert excinfo.value.status_code == 400 + + +# UTS: objects/unit/RTINS13/remove-delegates-0 +async def test_rtins13_remove_delegates(): + client, channel, root, mock_ws = await setup_synced_channel('test') + root_inst = root.instance() + assert root.get('name').as_primitive().value() == 'Alice' + + await root_inst.as_live_map().remove('name') + + assert root.get('name').as_primitive().value() is None + + +# UTS: objects/unit/RTINS14/increment-delegates-0 +async def test_rtins14_increment_delegates(): + client, channel, root, mock_ws = await setup_synced_channel('test') + counter_inst = root.get('score').instance() + + await counter_inst.as_live_counter().increment(25) + + assert root.get('score').as_live_counter().value() == 125 + + +# UTS: objects/unit/RTINS14d/increment-non-counter-throws-0 +async def test_rtins14d_increment_non_counter_throws(): + client, channel, root, mock_ws = await setup_synced_channel('test') + map_inst = root.instance() + + # `increment` is reached through the counter view, which refuses a map with the 92007 + # and status 400 that RTINS14d requires of `increment` (RTTS9d) + with pytest.raises(AblyException) as excinfo: + await map_inst.as_live_counter().increment(5) + + assert excinfo.value.code == 92007 + assert excinfo.value.status_code == 400 + + +# UTS: objects/unit/RTINS15/decrement-delegates-0 +async def test_rtins15_decrement_delegates(): + client, channel, root, mock_ws = await setup_synced_channel('test') + counter_inst = root.get('score').instance() + + await counter_inst.as_live_counter().decrement(10) + + assert root.get('score').as_live_counter().value() == 90 + + +# UTS: objects/unit/RTINS14a/increment-default-0 +async def test_rtins14a_increment_default(): + client, channel, root, mock_ws = await setup_synced_channel('test') + counter_inst = root.get('score').instance() + + await counter_inst.as_live_counter().increment() + + assert root.get('score').as_live_counter().value() == 101 + + +# UTS: objects/unit/RTINS15a/decrement-default-0 +async def test_rtins15a_decrement_default(): + client, channel, root, mock_ws = await setup_synced_channel('test') + counter_inst = root.get('score').instance() + + await counter_inst.as_live_counter().decrement() + + assert root.get('score').as_live_counter().value() == 99 + + +# UTS: objects/unit/RTINS16/subscribe-receives-events-0 +async def test_rtins16_subscribe_receives_events(): + client, channel, root, mock_ws = await setup_synced_channel('test') + counter_inst = root.get('score').instance() + events = [] + sub = counter_inst.as_live_counter().subscribe(events.append) + + mock_ws.send_to_client(build_object_message('test', [ + build_counter_inc('counter:score@1000', 7, '99', 'remote'), + ])) + await poll_until(lambda: len(events) >= 1, timeout=5, description='the listener to receive the COUNTER_INC') + await settle() + + assert isinstance(sub, Subscription) + assert len(events) == 1 + assert isinstance(events[0], InstanceSubscriptionEvent) + assert isinstance(events[0].object, Instance) + assert events[0].object.id == 'counter:score@1000' + + +# UTS: objects/unit/RTINS16c/subscribe-primitive-throws-0 +async def test_rtins16c_subscribe_primitive_throws(): + client, channel, root, mock_ws = await setup_synced_channel('test') + + name_inst = root.instance().get('name') + + # S-5: the specification calls `name_inst.subscribe(...)` and expects 92007 (RTINS16c). + # A primitive instance has no `subscribe` (RTTS7b, RTTS10c), and the two views that do + # have one refuse a primitive with 92007 (RTTS9d) + assert isinstance(name_inst, PrimitiveInstance) + assert name_inst.type is ValueType.STRING + assert not hasattr(name_inst, 'subscribe') + assert_view_refused(name_inst.as_live_map) + assert_view_refused(name_inst.as_live_counter) + + +# UTS: objects/unit/RTINS16e2/subscription-event-message-0 +async def test_rtins16e2_subscription_event_message(): + client, channel, root, mock_ws = await setup_synced_channel('test') + root_inst = root.instance() + events = [] + root_inst.as_live_map().subscribe(events.append) + + map_set = build_map_set('root', 'name', {'string': 'Bob'}, remote_serial(0), 'remote') + mock_ws.send_to_client(build_object_message('test', [map_set])) + await poll_until(lambda: len(events) >= 1, timeout=5, description='the listener to receive the MAP_SET') + + assert isinstance(events[0].object, Instance) + assert events[0].object.id == 'root' + assert events[0].message is not None + assert isinstance(events[0].message, publicmessage.ObjectMessage) + assert events[0].message.channel == 'test' + assert events[0].message.operation.action == ObjectOperationAction.MAP_SET + assert events[0].message.operation.object_id == 'root' + assert events[0].message.operation.map_set.key == 'name' + # The whole message is the one PAOM3 derives from the MAP_SET as it arrived + assert events[0].message == build_public_object_message(map_set, 'test') + + +# UTS: objects/unit/RTINS16f/subscribe-returns-subscription-0 +async def test_rtins16f_subscribe_returns_subscription(): + client, channel, root, mock_ws = await setup_synced_channel('test') + counter_inst = root.get('score').instance() + events = [] + sub = counter_inst.as_live_counter().subscribe(events.append) + sub.unsubscribe() + + # Quiescence control: a listener still subscribed to the same counter, which the + # same dispatch reaches + control_events = [] + counter_inst.as_live_counter().subscribe(control_events.append) + + mock_ws.send_to_client(build_object_message('test', [ + build_counter_inc('counter:score@1000', 7, '99', 'remote'), + ])) + + await assert_unchanged_after_quiescence( + lambda: len(events), lambda: len(control_events) >= 1, + description='the control listener receives the COUNTER_INC') + assert len(events) == 0 + + +# UTS: objects/unit/RTINS16g/subscription-follows-identity-0 +async def test_rtins16g_subscription_follows_identity(): + client, channel, root, mock_ws = await setup_synced_channel('test') + counter_inst = root.get('score').instance() + events = [] + counter_inst.as_live_counter().subscribe(events.append) + + mock_ws.send_to_client(build_object_message('test', [ + build_map_set('root', 'score', {'objectId': 'counter:new@2000'}, remote_serial(0), 'remote'), + ])) + + mock_ws.send_to_client(build_object_message('test', [ + build_counter_inc('counter:score@1000', 10, '100', 'remote'), + ])) + await poll_until(lambda: len(events) >= 1, timeout=5, description='the listener to receive the COUNTER_INC') + + assert len(events) >= 1 + # RTINS16e1: the delivered event carries the Instance wrapping the object that fired, + # so the assertion reads the event rather than the `counter_inst` handle + assert isinstance(events[0].object, Instance) + assert events[0].object.id == 'counter:score@1000' + # The frames are processed in order, so `score` already pointed at the new counter + # when the original one was incremented, and the event is that increment + assert root.get('score').instance().id == 'counter:new@2000' + assert events[0].message.operation.counter_inc.number == 10 + assert events[0].object.as_live_counter().value() == 110 + + +# UTS: objects/unit/RTINS16h/subscribe-no-side-effects-0 +async def test_rtins16h_subscribe_no_side_effects(): + client, channel, root, mock_ws = await setup_synced_channel('test') + counter_inst = root.get('score').instance() + channel_state_before = channel.state + sent_before = len(mock_ws.messages_from_client) + assert channel_state_before == ChannelState.ATTACHED + + sub = counter_inst.as_live_counter().subscribe(lambda event: None) + await settle() + + assert isinstance(sub, Subscription) + assert channel.state == channel_state_before + # Nothing was sent to the server either, such as an attach or a detach + assert len(mock_ws.messages_from_client) == sent_before diff --git a/test/uts/objects/unit/internal_live_counter_api_test.py b/test/uts/objects/unit/internal_live_counter_api_test.py new file mode 100644 index 00000000..00ebb54a --- /dev/null +++ b/test/uts/objects/unit/internal_live_counter_api_test.py @@ -0,0 +1,142 @@ +"""Derived from uts/objects/unit/internal_live_counter_api.md in ably/specification. + +Spec points: RTLC5, RTLC5c, RTLC11, RTLC11b1, RTLC12, RTLC12e1, RTLC12e2, RTLC12e3, RTLC12e5, +RTLC12g, RTLC13, RTLC13b + +The specification reaches `InternalLiveCounter` through the untyped `PathObject` and +`Instance`; ably-python partitions those by type (RTTS), so a counter's `value`, +`increment`, `decrement` and `subscribe` are read through `as_live_counter()`. + +The tests that capture what the client sends use the standard synced-channel mock, which +is the specification's hand-written one: it records each OBJECT message and then ACKs it +with `ack_serial(msgSerial, i)`, so every awaited write resolves (RTO20). Captured messages +are the decoded JSON wire, and each captured operation is also compared with the whole +protocol v6 operation, so a legacy (v5) payload field fails the test. +""" + +import pytest + +from ably.pubsub.objects.objectmessage import ObjectOperationAction +from ably.pubsub.util.exceptions import AblyException +from test.uts.helpers.client import poll_until +from test.uts.objects.helpers.standard_test_pool import ( + build_counter_inc, + build_object_message, + objects_connected_message, + setup_synced_channel, + standard_mock_websocket, +) + + +def _capturing_mock_websocket(captured): + """The specification's capturing mock: the standard synced-channel conversation, with each + OBJECT message the client sends appended to `captured` before it is ACKed.""" + return standard_mock_websocket( + on_object=captured.append, + connected=objects_connected_message(connection_key='key-1'), + ) + + +# UTS: objects/unit/RTLC5/value-returns-data-0 +async def test_rtlc5_value_returns_data(): + ctx = await setup_synced_channel('test') + + counter = ctx.root.get('score') + assert counter.as_live_counter().value() == 100 + + +# UTS: objects/unit/RTLC12/increment-sends-counter-inc-0 +async def test_rtlc12_increment_sends_counter_inc(): + captured = [] + ctx = await setup_synced_channel('test', mock_ws=_capturing_mock_websocket(captured)) + + await ctx.root.get('score').as_live_counter().increment(25) + + assert len(captured) == 1 + obj_msg = captured[0]['state'][0] + assert obj_msg['operation']['action'] == ObjectOperationAction.COUNTER_INC + assert obj_msg['operation']['objectId'] == 'counter:score@1000' + assert obj_msg['operation']['counterInc']['number'] == 25 + # The whole operation is the v6 wire shape: a numeric action and a `counterInc` payload + assert obj_msg['operation'] == { + 'action': int(ObjectOperationAction.COUNTER_INC), + 'objectId': 'counter:score@1000', + 'counterInc': {'number': 25}, + } + + +# UTS: objects/unit/RTLC12/increment-applies-locally-0 +async def test_rtlc12_increment_applies_locally(): + ctx = await setup_synced_channel('test') + + await ctx.root.get('score').as_live_counter().increment(50) + + assert ctx.root.get('score').as_live_counter().value() == 150 + + +# UTS: objects/unit/RTLC12e1/increment-non-number-0 +async def test_rtlc12e1_increment_non_number(): + ctx = await setup_synced_channel('test') + + with pytest.raises(AblyException) as excinfo: + await ctx.root.get('score').as_live_counter().increment('not_a_number') + + assert excinfo.value.code == 40003 + + +# UTS: objects/unit/RTLC13/decrement-negates-0 +async def test_rtlc13_decrement_negates(): + captured = [] + ctx = await setup_synced_channel('test', mock_ws=_capturing_mock_websocket(captured)) + + await ctx.root.get('score').as_live_counter().decrement(15) + + assert captured[0]['state'][0]['operation']['counterInc']['number'] == -15 + assert ctx.root.get('score').as_live_counter().value() == 85 + assert captured[0]['state'][0]['operation'] == { + 'action': int(ObjectOperationAction.COUNTER_INC), + 'objectId': 'counter:score@1000', + 'counterInc': {'number': -15}, + } + + +# UTS: objects/unit/RTLC11/counter-update-on-inc-0 +async def test_rtlc11_counter_update_on_inc(): + ctx = await setup_synced_channel('test') + + updates = [] + instance = ctx.root.get('score').instance() + instance.as_live_counter().subscribe(lambda event: updates.append(event)) + + ctx.mock_ws.send_to_client(build_object_message('test', [ + build_counter_inc('counter:score@1000', 7, '99', 'remote-site'), + ])) + + await poll_until(lambda: len(updates) >= 1, timeout=5, description='the counter update to be delivered') + + assert updates[0].message.operation.counter_inc.number == 7 + + +# The `null` row applies: in Python `None` is distinguishable from an omitted amount, which +# defaults to 1. `True` is an `int` in Python and must still be rejected as a non-number. +INVALID_AMOUNTS = [ + pytest.param(None, id='null'), + pytest.param(float('nan'), id='NaN'), + pytest.param(float('inf'), id='Infinity'), + pytest.param(float('-inf'), id='-Infinity'), + pytest.param('10', id='string'), + pytest.param(True, id='boolean'), + pytest.param([1, 2], id='array'), + pytest.param({'n': 1}, id='object'), +] + + +# UTS: objects/unit/RTLC12e1/increment-invalid-amounts-table-0 +@pytest.mark.parametrize('amount', INVALID_AMOUNTS) +async def test_rtlc12e1_increment_invalid_amounts_table(amount): + ctx = await setup_synced_channel('test') + + with pytest.raises(AblyException) as excinfo: + await ctx.root.get('score').as_live_counter().increment(amount) + + assert excinfo.value.code == 40003 diff --git a/test/uts/objects/unit/internal_live_counter_test.py b/test/uts/objects/unit/internal_live_counter_test.py new file mode 100644 index 00000000..2376671e --- /dev/null +++ b/test/uts/objects/unit/internal_live_counter_test.py @@ -0,0 +1,471 @@ +"""Derived from uts/objects/unit/internal_live_counter.md in ably/specification. + +Spec points: RTLC1, RTLC3, RTLC4, RTLC6, RTLC7, RTLC8, RTLC9, RTLC14, RTLC16, RTLO3, RTLO4a, +RTLO4b4d, RTLO4b4e, RTLO4e, RTLO5, RTLO6 + +The specification drives an `InternalLiveCounter` directly and reads the update from +`update = counter.applyOperation(...)`, or `false` when the operation is rejected. +`apply_operation` returns whether the operation applied (RTLC7g) and emits the update +through `notify_updated` (RTLC7d1a, RTLC7d5a, RTLC7d4c), so each test records the +emitted updates with `capture_updates` (shape deviation S-1): where the specification +reads `update`, the operation applied and emitted exactly one update, a no-op included; +where it reads `result == false`, the operation did not apply and emitted nothing. +`replace_data` returns its update (RTLC6h), as the specification has it. +""" + +from ably.pubsub.objects.enums import ObjectsOperationSource +from ably.pubsub.objects.livecounter import InternalLiveCounter +from ably.pubsub.objects.objectmessage import ObjectOperationAction +from test.uts.helpers.clock import FakeClock +from test.uts.objects.helpers.standard_test_pool import ( + build_counter_create, + build_counter_inc, + build_map_set, + build_object_delete, + build_object_state, + capture_updates, + object_message, +) + +COUNTER_ID = 'counter:abc@1000' + +CHANNEL = ObjectsOperationSource.CHANNEL +LOCAL = ObjectsOperationSource.LOCAL + + +def _emitted_update(applied, updates): + """S-1: the specification's `update = counter.applyOperation(...)`. + + An operation that applies returns True and emits exactly one update, a no-op included. + """ + assert applied is True, 'expected the operation to apply' + assert len(updates) == 1, f'expected the operation to emit exactly one update, got {updates!r}' + return updates[0] + + +def _assert_not_applied(applied, updates): + """S-1: the specification's `result == false`: the operation did not apply and emitted nothing.""" + assert applied is False + assert updates == [] + + +# UTS: objects/unit/RTLC4/zero-value-0 +def test_rtlc4_zero_value(): + counter = InternalLiveCounter(COUNTER_ID) + + assert counter.data == 0 + assert counter.object_id == COUNTER_ID + assert counter.is_tombstone is False + assert counter.tombstoned_at is None + assert counter.create_operation_is_merged is False + assert counter.site_timeserials == {} + + +# UTS: objects/unit/RTLC9/counter-inc-basic-0 +def test_rtlc9_counter_inc_basic(): + counter = InternalLiveCounter(COUNTER_ID) + updates = capture_updates(counter) + + msg = object_message(build_counter_inc(COUNTER_ID, 5, '01', 'site1')) + applied = counter.apply_operation(msg, CHANNEL) + + assert counter.data == 5 + update = _emitted_update(applied, updates) + assert update.noop is False + assert update.update.amount == 5 + assert update.object_message is msg + + +# UTS: objects/unit/RTLC9/counter-inc-negative-0 +def test_rtlc9_counter_inc_negative(): + counter = InternalLiveCounter(COUNTER_ID) + counter.data = 10 + counter.site_timeserials = {'site1': '00'} + updates = capture_updates(counter) + + msg = object_message(build_counter_inc(COUNTER_ID, -3, '01', 'site1')) + applied = counter.apply_operation(msg, CHANNEL) + + assert counter.data == 7 + update = _emitted_update(applied, updates) + assert update.update.amount == -3 + assert update.object_message is msg + + +# UTS: objects/unit/RTLC9/counter-inc-missing-number-0 +def test_rtlc9_counter_inc_missing_number(): + counter = InternalLiveCounter(COUNTER_ID) + counter.data = 10 + updates = capture_updates(counter) + + msg = object_message({ + 'serial': '01', + 'siteCode': 'site1', + 'operation': { + 'action': int(ObjectOperationAction.COUNTER_INC), + 'objectId': COUNTER_ID, + 'counterInc': {}, + }, + }) + applied = counter.apply_operation(msg, CHANNEL) + + assert counter.data == 10 + update = _emitted_update(applied, updates) + assert update.noop is True + + +# UTS: objects/unit/RTLC9/counter-inc-accumulate-0 +def test_rtlc9_counter_inc_accumulate(): + counter = InternalLiveCounter(COUNTER_ID) + + counter.apply_operation(object_message(build_counter_inc(COUNTER_ID, 10, '01', 'site1')), CHANNEL) + counter.apply_operation(object_message(build_counter_inc(COUNTER_ID, 20, '02', 'site1')), CHANNEL) + counter.apply_operation(object_message(build_counter_inc(COUNTER_ID, -5, '01', 'site2')), CHANNEL) + + assert counter.data == 25 + + +# UTS: objects/unit/RTLC8/counter-create-merge-0 +def test_rtlc8_counter_create_merge(): + counter = InternalLiveCounter(COUNTER_ID) + updates = capture_updates(counter) + + msg = object_message(build_counter_create(COUNTER_ID, {'count': 42}, '01', 'site1')) + applied = counter.apply_operation(msg, CHANNEL) + + assert counter.data == 42 + assert counter.create_operation_is_merged is True + update = _emitted_update(applied, updates) + assert update.update.amount == 42 + assert update.object_message is msg + + +# UTS: objects/unit/RTLC8/counter-create-already-merged-0 +def test_rtlc8_counter_create_already_merged(): + counter = InternalLiveCounter(COUNTER_ID) + counter.data = 42 + counter.create_operation_is_merged = True + counter.site_timeserials = {'site1': '00'} + updates = capture_updates(counter) + + msg = object_message(build_counter_create(COUNTER_ID, {'count': 99}, '01', 'site1')) + applied = counter.apply_operation(msg, CHANNEL) + + assert counter.data == 42 + update = _emitted_update(applied, updates) + assert update.noop is True + + +# UTS: objects/unit/RTLC16/counter-create-no-count-0 +def test_rtlc16_counter_create_no_count(): + counter = InternalLiveCounter(COUNTER_ID) + updates = capture_updates(counter) + + msg = object_message(build_counter_create(COUNTER_ID, {}, '01', 'site1')) + applied = counter.apply_operation(msg, CHANNEL) + + assert counter.data == 0 + assert counter.create_operation_is_merged is True + update = _emitted_update(applied, updates) + assert update.noop is True + + +# UTS: objects/unit/RTLO4a/apply-empty-site-serial-0 +def test_rtlo4a_apply_empty_site_serial(): + counter = InternalLiveCounter(COUNTER_ID) + + msg = object_message(build_counter_inc(COUNTER_ID, 5, '01', 'site1')) + result = counter.apply_operation(msg, CHANNEL) + + # S-1: the specification's `result IS NOT false` is a True return + assert result is True + assert counter.data == 5 + + +# UTS: objects/unit/RTLO4a/reject-stale-serial-0 +def test_rtlo4a_reject_stale_serial(): + counter = InternalLiveCounter(COUNTER_ID) + counter.site_timeserials = {'site1': '05'} + counter.data = 10 + updates = capture_updates(counter) + + msg = object_message(build_counter_inc(COUNTER_ID, 99, '03', 'site1')) + result = counter.apply_operation(msg, CHANNEL) + + _assert_not_applied(result, updates) + assert counter.data == 10 + + +# UTS: objects/unit/RTLO4a/reject-equal-serial-0 +def test_rtlo4a_reject_equal_serial(): + counter = InternalLiveCounter(COUNTER_ID) + counter.site_timeserials = {'site1': '05'} + counter.data = 10 + updates = capture_updates(counter) + + msg = object_message(build_counter_inc(COUNTER_ID, 99, '05', 'site1')) + result = counter.apply_operation(msg, CHANNEL) + + _assert_not_applied(result, updates) + assert counter.data == 10 + + +# UTS: objects/unit/RTLO4a/warn-invalid-serial-0 +def test_rtlo4a_warn_invalid_serial(): + counter = InternalLiveCounter(COUNTER_ID) + updates = capture_updates(counter) + + msg_no_serial = object_message(build_counter_inc(COUNTER_ID, 5, '', 'site1')) + result1 = counter.apply_operation(msg_no_serial, CHANNEL) + + msg_no_site = object_message(build_counter_inc(COUNTER_ID, 5, '01', '')) + result2 = counter.apply_operation(msg_no_site, CHANNEL) + + assert counter.data == 0 + assert result1 is False + assert result2 is False + # S-1: neither rejected operation emitted an update + assert updates == [] + + +# UTS: objects/unit/RTLC7c/channel-source-updates-serials-0 +def test_rtlc7c_channel_source_updates_serials(): + counter = InternalLiveCounter(COUNTER_ID) + + msg = object_message(build_counter_inc(COUNTER_ID, 5, '01', 'site1')) + counter.apply_operation(msg, CHANNEL) + + assert counter.site_timeserials['site1'] == '01' + + +# UTS: objects/unit/RTLC7c/local-source-no-serial-update-0 +def test_rtlc7c_local_source_no_serial_update(): + counter = InternalLiveCounter(COUNTER_ID) + + msg = object_message(build_counter_inc(COUNTER_ID, 5, '01', 'site1')) + counter.apply_operation(msg, LOCAL) + + assert counter.site_timeserials == {} + assert counter.data == 5 + + +# UTS: objects/unit/RTLC7g/apply-returns-true-0 +def test_rtlc7g_apply_returns_true(): + counter = InternalLiveCounter(COUNTER_ID) + + msg = object_message(build_counter_inc(COUNTER_ID, 5, '01', 'site1')) + result = counter.apply_operation(msg, CHANNEL) + + assert result is True + + +# UTS: objects/unit/RTLO5/object-delete-tombstones-0 +def test_rtlo5_object_delete_tombstones(): + counter = InternalLiveCounter(COUNTER_ID) + counter.data = 42 + counter.site_timeserials = {'site1': '00'} + updates = capture_updates(counter) + + msg = object_message(build_object_delete(COUNTER_ID, '01', 'site1', 1700000000000)) + applied = counter.apply_operation(msg, CHANNEL) + + assert counter.is_tombstone is True + assert counter.data == 0 + assert counter.tombstoned_at == 1700000000000 + update = _emitted_update(applied, updates) + assert update.update.amount == -42 + assert update.tombstone is True + assert update.object_message is msg + + +# UTS: objects/unit/RTLO5/tombstone-zero-value-counter-emits-update-0 +def test_rtlo5_tombstone_zero_value_counter_emits_update(): + counter = InternalLiveCounter(COUNTER_ID) + counter.data = 0 + counter.site_timeserials = {'site1': '00'} + updates = capture_updates(counter) + + msg = object_message(build_object_delete(COUNTER_ID, '01', 'site1', 1700000000000)) + applied = counter.apply_operation(msg, CHANNEL) + + assert counter.is_tombstone is True + assert counter.data == 0 + update = _emitted_update(applied, updates) + # The RTLC14c zero-delta exception does not apply to a tombstone diff (RTLO4e5) + assert update.noop is False + assert update.tombstone is True + assert update.update.amount == 0 + assert update.object_message is msg + + +# UTS: objects/unit/RTLC7e/tombstoned-reject-ops-0 +def test_rtlc7e_tombstoned_reject_ops(): + counter = InternalLiveCounter(COUNTER_ID) + counter.is_tombstone = True + counter.tombstoned_at = 1700000000000 + updates = capture_updates(counter) + + msg = object_message(build_counter_inc(COUNTER_ID, 5, '01', 'site1')) + result = counter.apply_operation(msg, CHANNEL) + + _assert_not_applied(result, updates) + assert counter.data == 0 + + +# UTS: objects/unit/RTLO6/tombstoned-at-from-serial-timestamp-0 +def test_rtlo6_tombstoned_at_from_serial_timestamp(): + counter = InternalLiveCounter(COUNTER_ID) + + msg = object_message(build_object_delete(COUNTER_ID, '01', 'site1', 1700000050000)) + counter.apply_operation(msg, CHANNEL) + + assert counter.tombstoned_at == 1700000050000 + + +# UTS: objects/unit/RTLO6/tombstoned-at-local-clock-0 +def test_rtlo6_tombstoned_at_local_clock(): + # The local clock is the counter's own. A fake one, at an epoch no fixture uses, holds + # still between the two readings, so the bounds pin the value RTLO6b must take. + clock = FakeClock(epoch_ms=1_700_000_123_456) + counter = InternalLiveCounter(COUNTER_ID, clock=clock) + before_time = clock.now_ms() + + msg = object_message(build_object_delete(COUNTER_ID, '01', 'site1')) + counter.apply_operation(msg, CHANNEL) + + after_time = clock.now_ms() + assert counter.tombstoned_at >= before_time + assert counter.tombstoned_at <= after_time + + +# UTS: objects/unit/RTLC7d3/unsupported-action-0 +def test_rtlc7d3_unsupported_action(): + counter = InternalLiveCounter(COUNTER_ID) + updates = capture_updates(counter) + + msg = object_message(build_map_set(COUNTER_ID, 'x', {'string': 'y'}, '01', 'site1')) + result = counter.apply_operation(msg, CHANNEL) + + _assert_not_applied(result, updates) + assert counter.data == 0 + + +# UTS: objects/unit/RTLC6/replace-data-basic-0 +def test_rtlc6_replace_data_basic(): + counter = InternalLiveCounter(COUNTER_ID) + counter.data = 10 + counter.create_operation_is_merged = True + counter.site_timeserials = {'site1': '00'} + + state_msg = object_message(build_object_state(COUNTER_ID, {'site2': '05'}, counter={'count': 50})) + update = counter.replace_data(state_msg) + + assert counter.data == 50 + assert counter.site_timeserials == {'site2': '05'} + assert counter.create_operation_is_merged is False + assert update.update.amount == 40 + assert update.object_message is state_msg + + +# UTS: objects/unit/RTLC6/replace-data-with-create-op-0 +def test_rtlc6_replace_data_with_create_op(): + counter = InternalLiveCounter(COUNTER_ID) + + state_msg = object_message(build_object_state( + COUNTER_ID, {'site1': '01'}, counter={'count': 100}, create_op={'counterCreate': {'count': 50}})) + update = counter.replace_data(state_msg) + + assert counter.data == 150 + assert counter.create_operation_is_merged is True + assert update.update.amount == 150 + assert update.object_message is state_msg + + +# UTS: objects/unit/RTLC6e/replace-data-tombstoned-noop-0 +def test_rtlc6e_replace_data_tombstoned_noop(): + counter = InternalLiveCounter(COUNTER_ID) + counter.is_tombstone = True + counter.tombstoned_at = 1700000000000 + counter.data = 0 + + state_msg = object_message(build_object_state(COUNTER_ID, {'site1': '01'}, counter={'count': 999})) + update = counter.replace_data(state_msg) + + assert counter.data == 0 + assert update.noop is True + + +# UTS: objects/unit/RTLC6f/replace-data-tombstone-flag-0 +def test_rtlc6f_replace_data_tombstone_flag(): + counter = InternalLiveCounter(COUNTER_ID) + counter.data = 30 + + state_msg = object_message(build_object_state( + COUNTER_ID, {'site1': '01'}, counter={'count': 0}, tombstone=True)) + update = counter.replace_data(state_msg) + + assert counter.is_tombstone is True + assert counter.data == 0 + assert update.update.amount == -30 + assert update.tombstone is True + assert update.object_message is state_msg + + +# UTS: objects/unit/RTLC6/replace-data-missing-count-0 +def test_rtlc6_replace_data_missing_count(): + counter = InternalLiveCounter(COUNTER_ID) + counter.data = 42 + + state_msg = object_message(build_object_state(COUNTER_ID, {'site1': '01'}, counter={})) + update = counter.replace_data(state_msg) + + assert counter.data == 0 + assert update.update.amount == -42 + assert update.object_message is state_msg + + +# UTS: objects/unit/RTLC14/diff-calculation-0 +def test_rtlc14_diff_calculation(): + counter = InternalLiveCounter(COUNTER_ID) + counter.data = 20 + + state_msg = object_message(build_object_state(COUNTER_ID, {'site1': '01'}, counter={'count': 75})) + update = counter.replace_data(state_msg) + + assert update.update.amount == 55 + assert update.object_message is state_msg + + +# UTS: objects/unit/RTLC14c/zero-delta-diff-is-noop-0 +def test_rtlc14c_zero_delta_diff_is_noop(): + counter = InternalLiveCounter(COUNTER_ID) + counter.data = 100 + + state_msg = object_message(build_object_state(COUNTER_ID, {'site1': '01'}, counter={'count': 100})) + update = counter.replace_data(state_msg) + + assert update.noop is True + assert counter.data == 100 + + +# UTS: objects/unit/RTLC8/create-then-inc-0 +def test_rtlc8_create_then_inc(): + counter = InternalLiveCounter(COUNTER_ID) + + counter.apply_operation( + object_message(build_counter_create(COUNTER_ID, {'count': 100}, '01', 'site1')), CHANNEL) + counter.apply_operation(object_message(build_counter_inc(COUNTER_ID, 25, '02', 'site1')), CHANNEL) + + assert counter.data == 125 + assert counter.create_operation_is_merged is True + + +# UTS: objects/unit/RTLO3/live-object-init-properties-0 +def test_rtlo3_live_object_init_properties(): + counter = InternalLiveCounter('counter:test@2000') + + assert counter.object_id == 'counter:test@2000' + assert counter.site_timeserials == {} + assert counter.create_operation_is_merged is False + assert counter.is_tombstone is False + assert counter.tombstoned_at is None diff --git a/test/uts/objects/unit/internal_live_map_api_test.py b/test/uts/objects/unit/internal_live_map_api_test.py new file mode 100644 index 00000000..37c23d42 --- /dev/null +++ b/test/uts/objects/unit/internal_live_map_api_test.py @@ -0,0 +1,300 @@ +"""Derived from uts/objects/unit/internal_live_map_api.md in ably/specification. + +Spec points: RTLM5, RTLM5d2, RTLM10, RTLM10d, RTLM11, RTLM11d, RTLM12, RTLM20, RTLM20a3, +RTLM20e1, RTLM20e2, RTLM20e3, RTLM20e6, RTLM20e7b, RTLM20e7c, RTLM20e7d, RTLM20e7e, +RTLM20e7f, RTLM20e7g, RTLM20e7g1, RTLM20e7g2, RTLM20h1, RTLM20h2, RTLM21, RTLM21e1, +RTLM21e2, RTLM21e5, RTLMV4c, RTLMV4d1, RTLMV4d2, RTLCV4 + +The specification reaches `InternalLiveMap` through the untyped `PathObject`; ably-python +partitions it by type (RTTS). `root` is already a `LiveMapPathObject`, so `size`, +`entries`, `keys`, `set` and `remove` are called on it directly, and a primitive or counter +below it is read through `as_primitive()` or `as_live_counter()`. + +The tests that capture what the client sends use the standard synced-channel mock, which +is the specification's hand-written one: it records each OBJECT message and then ACKs it +with `ack_serial(msgSerial, i)`, so every awaited write resolves (RTO20). Captured messages +are the decoded JSON wire, where a `json` value is a JSON-encoded string (OD2g), a `bytes` +value is base64, and an action is its number. Each captured operation is also checked +against the whole protocol v6 shape: a primitive MAP_SET or a MAP_REMOVE is compared +exactly, and a create carries only its `*CreateWithObjectId` payload, the `CounterCreate` +or `MapCreate` it was derived from staying local (RTLCV4g5, RTLMV4j5). +""" + +import json + +import pytest + +from ably.pubsub.objects.objectmessage import ObjectOperationAction +from ably.pubsub.objects.valuetypes import LiveCounter, LiveMap +from ably.pubsub.util.exceptions import AblyException +from test.uts.objects.helpers.standard_test_pool import ( + LWW, + objects_connected_message, + setup_synced_channel, + standard_mock_websocket, +) + + +def _capturing_mock_websocket(captured): + """The specification's capturing mock: the standard synced-channel conversation, with each + OBJECT message the client sends appended to `captured` before it is ACKed.""" + return standard_mock_websocket( + on_object=captured.append, + connected=objects_connected_message(connection_key='key-1'), + ) + + +def _map_set_operation(object_id, key, value): + """The v6 wire form of a MAP_SET operation; `value` is the `ObjectData` wire dictionary.""" + return { + 'action': int(ObjectOperationAction.MAP_SET), + 'objectId': object_id, + 'mapSet': {'key': key, 'value': value}, + } + + +def _assert_create_wire_shape(operation, payload_name): + """Asserts that a captured create carries only its v6 `*CreateWithObjectId` payload, with + an `initialValue` string and a nonce, and returns the decoded initial value.""" + assert set(operation) == {'action', 'objectId', payload_name} + payload = operation[payload_name] + assert set(payload) == {'initialValue', 'nonce'} + assert isinstance(payload['nonce'], str) + assert isinstance(payload['initialValue'], str) + return json.loads(payload['initialValue']) + + +# UTS: objects/unit/RTLM5/get-string-value-0 +async def test_rtlm5_get_string_value(): + ctx = await setup_synced_channel('test') + + assert ctx.root.get('name').as_primitive().value() == 'Alice' + assert ctx.root.get('age').as_primitive().value() == 30 + assert ctx.root.get('active').as_primitive().value() is True + + +# UTS: objects/unit/RTLM5/get-nonexistent-key-0 +async def test_rtlm5_get_nonexistent_key(): + ctx = await setup_synced_channel('test') + + # The specification's untyped `value()`: a path that resolves to nothing answers None + # through every typed view (RTTS5d1) + assert ctx.root.get('nonexistent').as_primitive().value() is None + assert ctx.root.get('nonexistent').as_live_counter().value() is None + + +# UTS: objects/unit/RTLM5/get-objectid-reference-0 +async def test_rtlm5_get_objectid_reference(): + ctx = await setup_synced_channel('test') + + assert ctx.root.get('score').as_live_counter().value() == 100 + assert ctx.root.get('profile').get('email').as_primitive().value() == 'alice@example.com' + + +# UTS: objects/unit/RTLM10/size-non-tombstoned-0 +async def test_rtlm10_size_non_tombstoned(): + ctx = await setup_synced_channel('test') + + assert ctx.root.size() == 7 + + +# UTS: objects/unit/RTLM11/entries-yields-pairs-0 +async def test_rtlm11_entries_yields_pairs(): + ctx = await setup_synced_channel('test') + + entries = [] + for key, _path_object in ctx.root.entries(): + entries.append(key) + + assert 'name' in entries + assert 'age' in entries + assert 'active' in entries + assert 'score' in entries + assert 'profile' in entries + assert 'data' in entries + assert 'avatar' in entries + assert len(entries) == 7 + + +# UTS: objects/unit/RTLM12/keys-0 +async def test_rtlm12_keys(): + ctx = await setup_synced_channel('test') + + keys = list(ctx.root.keys()) + + assert len(keys) == 7 + assert 'name' in keys + + +# UTS: objects/unit/RTLM20/set-sends-map-set-0 +async def test_rtlm20_set_sends_map_set(): + captured = [] + ctx = await setup_synced_channel('test', mock_ws=_capturing_mock_websocket(captured)) + + await ctx.root.set('name', 'Bob') + + assert len(captured) == 1 + obj_msg = captured[0]['state'][0] + assert obj_msg['operation']['action'] == ObjectOperationAction.MAP_SET + assert obj_msg['operation']['objectId'] == 'root' + assert obj_msg['operation']['mapSet']['key'] == 'name' + assert obj_msg['operation']['mapSet']['value']['string'] == 'Bob' + assert obj_msg['operation'] == _map_set_operation('root', 'name', {'string': 'Bob'}) + + +# UTS: objects/unit/RTLM20/set-value-types-0 +async def test_rtlm20_set_value_types(): + captured = [] + ctx = await setup_synced_channel('test', mock_ws=_capturing_mock_websocket(captured)) + + await ctx.root.set('num_key', 42) + await ctx.root.set('bool_key', False) + await ctx.root.set('json_key', {'nested': True}) + + assert captured[0]['state'][0]['operation']['mapSet']['value']['number'] == 42 + assert captured[1]['state'][0]['operation']['mapSet']['value']['boolean'] is False + # A `json` value travels as a JSON-encoded string (OD2g); the specification compares the + # decoded value + json_value = captured[2]['state'][0]['operation']['mapSet']['value'] + assert json.loads(json_value['json']) == {'nested': True} + + assert captured[0]['state'][0]['operation'] == _map_set_operation('root', 'num_key', {'number': 42}) + assert captured[1]['state'][0]['operation'] == _map_set_operation('root', 'bool_key', {'boolean': False}) + assert set(json_value) == {'json'} + assert isinstance(json_value['json'], str) + + +# UTS: objects/unit/RTLM20e7g/set-counter-value-type-0 +async def test_rtlm20e7g_set_counter_value_type(): + captured = [] + ctx = await setup_synced_channel('test', mock_ws=_capturing_mock_websocket(captured)) + + await ctx.root.set('new_counter', LiveCounter.create(50)) + + assert len(captured) == 1 + state = captured[0]['state'] + assert len(state) == 2 + assert state[0]['operation']['action'] == ObjectOperationAction.COUNTER_CREATE + assert state[0]['operation']['objectId'].startswith('counter:') + assert state[1]['operation']['action'] == ObjectOperationAction.MAP_SET + assert state[1]['operation']['mapSet']['value']['objectId'] == state[0]['operation']['objectId'] + + initial_value = _assert_create_wire_shape(state[0]['operation'], 'counterCreateWithObjectId') + assert initial_value == {'count': 50} + assert state[1]['operation'] == _map_set_operation( + 'root', 'new_counter', {'objectId': state[0]['operation']['objectId']}) + + +# UTS: objects/unit/RTLM20e7g/set-map-value-type-0 +async def test_rtlm20e7g_set_map_value_type(): + captured = [] + ctx = await setup_synced_channel('test', mock_ws=_capturing_mock_websocket(captured)) + + await ctx.root.set('nested_map', LiveMap.create({'key1': 'value1'})) + + assert len(captured) == 1 + state = captured[0]['state'] + assert len(state) == 2 + assert state[0]['operation']['action'] == ObjectOperationAction.MAP_CREATE + assert state[0]['operation']['objectId'].startswith('map:') + assert state[1]['operation']['action'] == ObjectOperationAction.MAP_SET + assert state[1]['operation']['mapSet']['key'] == 'nested_map' + assert state[1]['operation']['mapSet']['value']['objectId'] == state[0]['operation']['objectId'] + + initial_value = _assert_create_wire_shape(state[0]['operation'], 'mapCreateWithObjectId') + assert initial_value['semantics'] == LWW + assert set(initial_value['entries']) == {'key1'} + assert initial_value['entries']['key1']['data'] == {'string': 'value1'} + assert state[1]['operation'] == _map_set_operation( + 'root', 'nested_map', {'objectId': state[0]['operation']['objectId']}) + + +# UTS: objects/unit/RTLM20h1/set-nested-value-types-0 +async def test_rtlm20h1_set_nested_value_types(): + captured = [] + ctx = await setup_synced_channel('test', mock_ws=_capturing_mock_websocket(captured)) + + await ctx.root.set('stats', LiveMap.create({ + 'count': LiveCounter.create(0), + 'label': 'test', + })) + + assert len(captured) == 1 + state = captured[0]['state'] + # COUNTER_CREATE, MAP_CREATE, MAP_SET: depth-first, then the MAP_SET at root + assert len(state) == 3 + assert state[0]['operation']['action'] == ObjectOperationAction.COUNTER_CREATE + assert state[0]['operation']['objectId'].startswith('counter:') + assert state[1]['operation']['action'] == ObjectOperationAction.MAP_CREATE + assert state[1]['operation']['objectId'].startswith('map:') + assert state[2]['operation']['action'] == ObjectOperationAction.MAP_SET + assert state[2]['operation']['mapSet']['key'] == 'stats' + assert state[2]['operation']['mapSet']['value']['objectId'] == state[1]['operation']['objectId'] + + counter_initial_value = _assert_create_wire_shape(state[0]['operation'], 'counterCreateWithObjectId') + assert counter_initial_value == {'count': 0} + # RTLMV4d1: the nested counter's entry references the counter the first message creates + map_initial_value = _assert_create_wire_shape(state[1]['operation'], 'mapCreateWithObjectId') + assert map_initial_value['entries']['count']['data'] == {'objectId': state[0]['operation']['objectId']} + assert map_initial_value['entries']['label']['data'] == {'string': 'test'} + assert state[2]['operation'] == _map_set_operation( + 'root', 'stats', {'objectId': state[1]['operation']['objectId']}) + + +# UTS: objects/unit/RTLM21/remove-sends-map-remove-0 +async def test_rtlm21_remove_sends_map_remove(): + captured = [] + ctx = await setup_synced_channel('test', mock_ws=_capturing_mock_websocket(captured)) + + await ctx.root.remove('name') + + obj_msg = captured[0]['state'][0] + assert obj_msg['operation']['action'] == ObjectOperationAction.MAP_REMOVE + assert obj_msg['operation']['objectId'] == 'root' + assert obj_msg['operation']['mapRemove']['key'] == 'name' + assert obj_msg['operation'] == { + 'action': int(ObjectOperationAction.MAP_REMOVE), + 'objectId': 'root', + 'mapRemove': {'key': 'name'}, + } + + +# UTS: objects/unit/RTLM20/set-applies-locally-0 +async def test_rtlm20_set_applies_locally(): + ctx = await setup_synced_channel('test') + + await ctx.root.set('name', 'Bob') + + assert ctx.root.get('name').as_primitive().value() == 'Bob' + + +# The specification's rows are JavaScript values. Their Python counterparts: a function is a +# lambda, `undefined` is `None`, and a symbol is an arbitrary `object()`. None of them is a +# value a map entry can hold (RTLM20a3). +INVALID_VALUES = [ + pytest.param(lambda: None, id='function'), + pytest.param(None, id='undefined'), + pytest.param(object(), id='symbol'), +] + + +# UTS: objects/unit/RTLM20/set-invalid-values-table-0 +@pytest.mark.parametrize('value', INVALID_VALUES) +async def test_rtlm20_set_invalid_values_table(value): + ctx = await setup_synced_channel('test') + + with pytest.raises(AblyException) as excinfo: + await ctx.root.set('key', value) + + assert excinfo.value.code == 40013 + + +# UTS: objects/unit/RTLM20/set-bytes-value-0 +async def test_rtlm20_set_bytes_value(): + captured = [] + ctx = await setup_synced_channel('test', mock_ws=_capturing_mock_websocket(captured)) + + await ctx.root.set('binary_data', bytes([1, 2, 3])) + + assert captured[0]['state'][0]['operation']['mapSet']['value']['bytes'] == 'AQID' + assert captured[0]['state'][0]['operation'] == _map_set_operation('root', 'binary_data', {'bytes': 'AQID'}) diff --git a/test/uts/objects/unit/internal_live_map_test.py b/test/uts/objects/unit/internal_live_map_test.py new file mode 100644 index 00000000..4bb1e873 --- /dev/null +++ b/test/uts/objects/unit/internal_live_map_test.py @@ -0,0 +1,928 @@ +"""Derived from uts/objects/unit/internal_live_map.md in ably/specification. + +Spec points: RTLM1-RTLM9, RTLM14-RTLM16, RTLM18, RTLM19, RTLM22-RTLM25, RTLO3, RTLO4a, RTLO4e, +RTLO4g, RTLO4h, RTLO5, RTLO6 + +The specification drives an `InternalLiveMap` directly and reads the update from +`update = map.applyOperation(...)`, or `false` when the operation is rejected. +`apply_operation` returns whether the operation applied (RTLM15g) and emits the update +through `notify_updated` (RTLM15d1a and its siblings), so each test records the emitted +updates with `capture_updates` (shape deviation S-1): where the specification reads +`update`, the operation applied and emitted exactly one update, a no-op included; where +it reads `result == false` or `update == false`, the operation did not apply and emitted +nothing. `replace_data` returns its update (RTLM6h), and `diff` is the static +`InternalLiveMap.diff`, as the specification has them. + +Where the specification passes a pool, the map under test reads it but is not the pool's +own root: `ObjectsPool()` creates that (RTO3b1), and the specification builds a second map +with the id `root` alongside it. +""" + +from ably.pubsub.objects.enums import ObjectsOperationSource +from ably.pubsub.objects.livecounter import InternalLiveCounter +from ably.pubsub.objects.livemap import InternalLiveMap +from ably.pubsub.objects.objectmessage import ObjectData, ObjectsMapEntry, ObjectsMapSemantics +from ably.pubsub.objects.objectspool import ObjectsPool +from test.uts.objects.helpers.standard_test_pool import ( + LWW, + build_counter_inc, + build_map_clear, + build_map_create, + build_map_remove, + build_map_set, + build_object_delete, + build_object_state, + capture_updates, + object_message, +) + +CHANNEL = ObjectsOperationSource.CHANNEL + + +def _emitted_update(applied, updates): + """S-1: the specification's `update = map.applyOperation(...)`. + + An operation that applies returns True and emits exactly one update, a no-op included. + """ + assert applied is True, 'expected the operation to apply' + assert len(updates) == 1, f'expected the operation to emit exactly one update, got {updates!r}' + return updates[0] + + +def _assert_not_applied(applied, updates): + """S-1: the specification's `result == false`: the operation did not apply and emitted nothing.""" + assert applied is False + assert updates == [] + + +def _references(child, parent_id, key): + """Whether `child.parent_references` records the map `parent_id` referencing it at `key`. + + `not _references(...)` is the specification's + `parent_id NOT IN child.parentReferences OR key NOT IN child.parentReferences[parent_id]`. + """ + return key in child.parent_references.get(parent_id, set()) + + +# UTS: objects/unit/RTLM4/zero-value-0 +def test_rtlm4_zero_value(): + live_map = InternalLiveMap('root', ObjectsMapSemantics.LWW) + + assert live_map.data == {} + assert live_map.clear_timeserial is None + assert live_map.is_tombstone is False + assert live_map.create_operation_is_merged is False + assert live_map.site_timeserials == {} + + +# UTS: objects/unit/RTLM7/map-set-new-entry-0 +def test_rtlm7_map_set_new_entry(): + live_map = InternalLiveMap('root', ObjectsMapSemantics.LWW) + updates = capture_updates(live_map) + + msg = object_message(build_map_set('root', 'name', {'string': 'Alice'}, '01', 'site1')) + applied = live_map.apply_operation(msg, CHANNEL) + + assert live_map.data['name'].data == ObjectData(string='Alice') + assert live_map.data['name'].timeserial == '01' + assert live_map.data['name'].tombstone is False + update = _emitted_update(applied, updates) + assert update.update == {'name': 'updated'} + assert update.object_message is msg + + +# UTS: objects/unit/RTLM7/map-set-update-entry-0 +def test_rtlm7_map_set_update_entry(): + live_map = InternalLiveMap('root', ObjectsMapSemantics.LWW) + live_map.data = { + 'name': ObjectsMapEntry(data=ObjectData(string='Alice'), timeserial='01', tombstone=False), + } + updates = capture_updates(live_map) + + msg = object_message(build_map_set('root', 'name', {'string': 'Bob'}, '02', 'site1')) + applied = live_map.apply_operation(msg, CHANNEL) + + assert live_map.data['name'].data == ObjectData(string='Bob') + assert live_map.data['name'].timeserial == '02' + update = _emitted_update(applied, updates) + assert update.update == {'name': 'updated'} + assert update.object_message is msg + + +# UTS: objects/unit/RTLM9/lww-reject-stale-0 +def test_rtlm9_lww_reject_stale(): + live_map = InternalLiveMap('root', ObjectsMapSemantics.LWW) + live_map.data = { + 'name': ObjectsMapEntry(data=ObjectData(string='Alice'), timeserial='05', tombstone=False), + } + updates = capture_updates(live_map) + + msg = object_message(build_map_set('root', 'name', {'string': 'Bob'}, '03', 'site1')) + applied = live_map.apply_operation(msg, CHANNEL) + + assert live_map.data['name'].data == ObjectData(string='Alice') + update = _emitted_update(applied, updates) + assert update.noop is True + + +# UTS: objects/unit/RTLM9/lww-reject-equal-0 +def test_rtlm9_lww_reject_equal(): + live_map = InternalLiveMap('root', ObjectsMapSemantics.LWW) + live_map.data = { + 'name': ObjectsMapEntry(data=ObjectData(string='Alice'), timeserial='05', tombstone=False), + } + updates = capture_updates(live_map) + + msg = object_message(build_map_set('root', 'name', {'string': 'Bob'}, '05', 'site1')) + applied = live_map.apply_operation(msg, CHANNEL) + + assert live_map.data['name'].data == ObjectData(string='Alice') + update = _emitted_update(applied, updates) + assert update.noop is True + + +# UTS: objects/unit/RTLM9b/both-empty-reject-0 +def test_rtlm9b_both_empty_reject(): + live_map = InternalLiveMap('root', ObjectsMapSemantics.LWW) + live_map.data = { + 'name': ObjectsMapEntry(data=ObjectData(string='Alice'), timeserial='', tombstone=False), + } + updates = capture_updates(live_map) + + msg = object_message(build_map_set('root', 'name', {'string': 'Bob'}, '', 'site1')) + applied = live_map.apply_operation(msg, CHANNEL) + + assert live_map.data['name'].data == ObjectData(string='Alice') + # The empty ObjectMessage.serial fails the object-level gate (RTLO4a3) before the + # entry-level RTLM9b comparison is reached, so apply_operation returns False (RTLM15b). + _assert_not_applied(applied, updates) + # RTLM9b itself, which apply_operation cannot reach: an empty or missing entry serial + # and operation serial are equal, so the operation is not applied + assert InternalLiveMap.can_apply_map_operation('', '') is False + assert InternalLiveMap.can_apply_map_operation(None, None) is False + + +# UTS: objects/unit/RTLM9d/missing-entry-serial-allows-0 +def test_rtlm9d_missing_entry_serial_allows(): + live_map = InternalLiveMap('root', ObjectsMapSemantics.LWW) + live_map.data = { + 'name': ObjectsMapEntry(data=ObjectData(string='Alice'), timeserial=None, tombstone=False), + } + updates = capture_updates(live_map) + + msg = object_message(build_map_set('root', 'name', {'string': 'Bob'}, '01', 'site1')) + applied = live_map.apply_operation(msg, CHANNEL) + + assert live_map.data['name'].data == ObjectData(string='Bob') + update = _emitted_update(applied, updates) + assert update.update == {'name': 'updated'} + assert update.object_message is msg + + +# UTS: objects/unit/RTLM7h/map-set-clear-timeserial-floor-0 +def test_rtlm7h_map_set_clear_timeserial_floor(): + live_map = InternalLiveMap('root', ObjectsMapSemantics.LWW) + live_map.clear_timeserial = '05' + updates = capture_updates(live_map) + + msg = object_message(build_map_set('root', 'name', {'string': 'Alice'}, '03', 'site1')) + applied = live_map.apply_operation(msg, CHANNEL) + + assert 'name' not in live_map.data + update = _emitted_update(applied, updates) + assert update.noop is True + + +# UTS: objects/unit/RTLM7g/map-set-objectid-creates-zero-value-0 +def test_rtlm7g_map_set_objectid_creates_zero_value(): + pool = ObjectsPool() + live_map = InternalLiveMap('root', ObjectsMapSemantics.LWW, pool=pool) + + msg = object_message(build_map_set('root', 'score', {'objectId': 'counter:new@2000'}, '01', 'site1')) + live_map.apply_operation(msg, CHANNEL) + + assert 'counter:new@2000' in pool + assert isinstance(pool['counter:new@2000'], InternalLiveCounter) + assert pool['counter:new@2000'].data == 0 + + +# UTS: objects/unit/RTLM8/map-remove-existing-0 +def test_rtlm8_map_remove_existing(): + live_map = InternalLiveMap('root', ObjectsMapSemantics.LWW) + live_map.data = { + 'name': ObjectsMapEntry(data=ObjectData(string='Alice'), timeserial='01', tombstone=False), + } + updates = capture_updates(live_map) + + msg = object_message(build_map_remove('root', 'name', '02', 'site1', 1700000000000)) + applied = live_map.apply_operation(msg, CHANNEL) + + assert live_map.data['name'].data is None + assert live_map.data['name'].tombstone is True + assert live_map.data['name'].timeserial == '02' + assert live_map.data['name'].tombstoned_at == 1700000000000 + update = _emitted_update(applied, updates) + assert update.update == {'name': 'removed'} + assert update.object_message is msg + + +# UTS: objects/unit/RTLM8/map-remove-nonexistent-0 +def test_rtlm8_map_remove_nonexistent(): + live_map = InternalLiveMap('root', ObjectsMapSemantics.LWW) + updates = capture_updates(live_map) + + msg = object_message(build_map_remove('root', 'ghost', '01', 'site1', 1700000000000)) + applied = live_map.apply_operation(msg, CHANNEL) + + assert live_map.data['ghost'].tombstone is True + assert live_map.data['ghost'].tombstoned_at == 1700000000000 + update = _emitted_update(applied, updates) + assert update.update == {'ghost': 'removed'} + assert update.object_message is msg + + +# UTS: objects/unit/RTLM8g/map-remove-clear-timeserial-floor-0 +def test_rtlm8g_map_remove_clear_timeserial_floor(): + live_map = InternalLiveMap('root', ObjectsMapSemantics.LWW) + live_map.clear_timeserial = '05' + live_map.data = { + 'name': ObjectsMapEntry(data=ObjectData(string='Alice'), timeserial='04', tombstone=False), + } + updates = capture_updates(live_map) + + msg = object_message(build_map_remove('root', 'name', '03', 'site1', 1700000000000)) + applied = live_map.apply_operation(msg, CHANNEL) + + assert live_map.data['name'].data == ObjectData(string='Alice') + assert live_map.data['name'].tombstone is False + update = _emitted_update(applied, updates) + assert update.noop is True + + # UTS SPEC ERROR: the entry's own timeserial ('04') is later than the operation's ('03'), + # so LWW (RTLM8a1, RTLM9e) rejects the remove with or without the clear floor, and the + # steps above pass either way. A remove for a key with no entry, still at or below the + # floor, reaches RTLM8g alone: without it the key would gain a tombstoned entry (RTLM8b). + ghost_updates = capture_updates(live_map) + ghost_msg = object_message(build_map_remove('root', 'ghost', '04', 'site1', 1700000000000)) + ghost_applied = live_map.apply_operation(ghost_msg, CHANNEL) + + assert 'ghost' not in live_map.data + ghost_update = _emitted_update(ghost_applied, ghost_updates) + assert ghost_update.noop is True + + +# UTS: objects/unit/RTLM24/map-clear-basic-0 +def test_rtlm24_map_clear_basic(): + live_map = InternalLiveMap('root', ObjectsMapSemantics.LWW) + live_map.data = { + 'old': ObjectsMapEntry(data=ObjectData(string='old'), timeserial='02', tombstone=False), + 'new': ObjectsMapEntry(data=ObjectData(string='new'), timeserial='06', tombstone=False), + 'same': ObjectsMapEntry(data=ObjectData(string='same'), timeserial='04', tombstone=False), + } + updates = capture_updates(live_map) + + msg = object_message(build_map_clear('root', '04', 'site1')) + applied = live_map.apply_operation(msg, CHANNEL) + + assert live_map.clear_timeserial == '04' + assert 'old' not in live_map.data + # RTLM24e1: an entry is removed only if the clear serial is lexicographically greater + # than the entry's timeserial; 'same' has the clear serial itself, so it is kept + assert 'same' in live_map.data + assert 'new' in live_map.data + update = _emitted_update(applied, updates) + assert update.update == {'old': 'removed'} + assert update.object_message is msg + + +# UTS: objects/unit/RTLM24c/map-clear-stale-0 +def test_rtlm24c_map_clear_stale(): + live_map = InternalLiveMap('root', ObjectsMapSemantics.LWW) + live_map.clear_timeserial = '10' + updates = capture_updates(live_map) + + msg = object_message(build_map_clear('root', '05', 'site1')) + applied = live_map.apply_operation(msg, CHANNEL) + + assert live_map.clear_timeserial == '10' + update = _emitted_update(applied, updates) + assert update.noop is True + + +# UTS: objects/unit/RTLM16/map-create-merge-0 +def test_rtlm16_map_create_merge(): + live_map = InternalLiveMap('map:test@1000', ObjectsMapSemantics.LWW) + updates = capture_updates(live_map) + + msg = object_message(build_map_create('map:test@1000', { + 'semantics': LWW, + 'entries': { + 'name': {'data': {'string': 'Alice'}, 'timeserial': '01'}, + 'removed_key': {'tombstone': True, 'timeserial': '01', 'serialTimestamp': 1700000000000}, + }, + }, '02', 'site1')) + applied = live_map.apply_operation(msg, CHANNEL) + + assert live_map.data['name'].data == ObjectData(string='Alice') + assert live_map.data['removed_key'].tombstone is True + assert live_map.create_operation_is_merged is True + update = _emitted_update(applied, updates) + assert update.update == {'name': 'updated', 'removed_key': 'removed'} + assert update.object_message is msg + + +# UTS: objects/unit/RTLM16b/map-create-already-merged-0 +def test_rtlm16b_map_create_already_merged(): + live_map = InternalLiveMap('map:test@1000', ObjectsMapSemantics.LWW) + live_map.create_operation_is_merged = True + live_map.site_timeserials = {'site1': '00'} + updates = capture_updates(live_map) + + msg = object_message(build_map_create('map:test@1000', { + 'semantics': LWW, + 'entries': {'name': {'data': {'string': 'Bob'}, 'timeserial': '01'}}, + }, '01', 'site1')) + applied = live_map.apply_operation(msg, CHANNEL) + + assert 'name' not in live_map.data + update = _emitted_update(applied, updates) + assert update.noop is True + + +# UTS: objects/unit/RTLM15c/channel-source-updates-serials-0 +def test_rtlm15c_channel_source_updates_serials(): + live_map = InternalLiveMap('root', ObjectsMapSemantics.LWW) + + msg = object_message(build_map_set('root', 'x', {'number': 1}, '01', 'site1')) + live_map.apply_operation(msg, CHANNEL) + + assert live_map.site_timeserials['site1'] == '01' + + +# UTS: objects/unit/RTLM15e/tombstoned-reject-ops-0 +def test_rtlm15e_tombstoned_reject_ops(): + live_map = InternalLiveMap('root', ObjectsMapSemantics.LWW) + live_map.is_tombstone = True + updates = capture_updates(live_map) + + msg = object_message(build_map_set('root', 'x', {'number': 1}, '01', 'site1')) + result = live_map.apply_operation(msg, CHANNEL) + + _assert_not_applied(result, updates) + assert live_map.data == {} + + +# UTS: objects/unit/RTLO5/object-delete-tombstones-map-0 +def test_rtlo5_object_delete_tombstones_map(): + live_map = InternalLiveMap('map:test@1000', ObjectsMapSemantics.LWW) + live_map.data = { + 'name': ObjectsMapEntry(data=ObjectData(string='Alice'), timeserial='01', tombstone=False), + 'age': ObjectsMapEntry(data=ObjectData(number=30), timeserial='01', tombstone=False), + } + live_map.site_timeserials = {'site1': '00'} + updates = capture_updates(live_map) + + msg = object_message(build_object_delete('map:test@1000', '01', 'site1', 1700000000000)) + applied = live_map.apply_operation(msg, CHANNEL) + + assert live_map.is_tombstone is True + assert live_map.data == {} + update = _emitted_update(applied, updates) + assert update.update == {'name': 'removed', 'age': 'removed'} + assert update.tombstone is True + assert update.object_message is msg + + +# UTS: objects/unit/RTLO5/tombstone-empty-map-emits-update-0 +def test_rtlo5_tombstone_empty_map_emits_update(): + live_map = InternalLiveMap('map:test@1000', ObjectsMapSemantics.LWW) + live_map.data = { + 'name': ObjectsMapEntry(data=ObjectData(string='Alice'), timeserial='01', tombstone=True, + tombstoned_at=1600000000000), + 'age': ObjectsMapEntry(data=ObjectData(number=30), timeserial='01', tombstone=True, + tombstoned_at=1600000000000), + } + live_map.site_timeserials = {'site1': '00'} + updates = capture_updates(live_map) + + msg = object_message(build_object_delete('map:test@1000', '01', 'site1', 1700000000000)) + applied = live_map.apply_operation(msg, CHANNEL) + + assert live_map.is_tombstone is True + assert live_map.data == {} + update = _emitted_update(applied, updates) + # The RTLM22c empty-diff exception does not apply to a tombstone diff (RTLO4e5) + assert update.noop is False + assert update.tombstone is True + assert update.update == {} + assert update.object_message is msg + + +# UTS: objects/unit/RTLO4e10/object-delete-root-noop-0 +def test_rtlo4e10_object_delete_root_noop(): + live_map = InternalLiveMap('root', ObjectsMapSemantics.LWW) + live_map.data = { + 'name': ObjectsMapEntry(data=ObjectData(string='Alice'), timeserial='01', tombstone=False), + } + live_map.site_timeserials = {'site1': '00'} + updates = capture_updates(live_map) + + msg = object_message(build_object_delete('root', '01', 'site1', 1700000000000)) + applied = live_map.apply_operation(msg, CHANNEL) + + assert live_map.is_tombstone is False + # The data is untouched + assert live_map.data['name'].data.string == 'Alice' + update = _emitted_update(applied, updates) + assert update.noop is True + + +# UTS: objects/unit/RTLM14/tombstone-check-objectid-ref-0 +def test_rtlm14_tombstone_check_objectid_ref(): + pool = ObjectsPool() + tombstoned_counter = InternalLiveCounter('counter:dead@1000') + tombstoned_counter.is_tombstone = True + pool['counter:dead@1000'] = tombstoned_counter + + live_map = InternalLiveMap('root', ObjectsMapSemantics.LWW, pool=pool) + live_map.data = { + 'alive': ObjectsMapEntry(data=ObjectData(string='ok'), timeserial='01', tombstone=False), + 'dead_entry': ObjectsMapEntry(data=None, timeserial='01', tombstone=True), + 'dead_ref': ObjectsMapEntry(data=ObjectData(object_id='counter:dead@1000'), timeserial='01', + tombstone=False), + } + + assert live_map.is_entry_tombstoned(live_map.data['alive']) is False + assert live_map.is_entry_tombstoned(live_map.data['dead_entry']) is True + assert live_map.is_entry_tombstoned(live_map.data['dead_ref']) is True + + +# UTS: objects/unit/RTLM6/replace-data-basic-0 +def test_rtlm6_replace_data_basic(): + live_map = InternalLiveMap('root', ObjectsMapSemantics.LWW) + live_map.data = { + 'old': ObjectsMapEntry(data=ObjectData(string='old'), timeserial='01', tombstone=False), + } + live_map.create_operation_is_merged = True + + state_msg = object_message(build_object_state('root', {'site2': '05'}, map={ + 'semantics': LWW, + 'clearTimeserial': '03', + 'entries': { + 'new': {'data': {'string': 'new'}, 'timeserial': '04', 'tombstone': False}, + }, + })) + update = live_map.replace_data(state_msg) + + assert live_map.site_timeserials == {'site2': '05'} + assert live_map.create_operation_is_merged is False + assert live_map.clear_timeserial == '03' + assert 'old' not in live_map.data + assert live_map.data['new'].data == ObjectData(string='new') + assert update.update == {'old': 'removed', 'new': 'updated'} + assert update.object_message is state_msg + + +# UTS: objects/unit/RTLM6c1/replace-data-tombstoned-entries-0 +def test_rtlm6c1_replace_data_tombstoned_entries(): + live_map = InternalLiveMap('root', ObjectsMapSemantics.LWW) + + state_msg = object_message(build_object_state('root', {'site1': '01'}, map={ + 'semantics': LWW, + 'entries': { + 'dead': {'tombstone': True, 'timeserial': '01', 'serialTimestamp': 1700000050000}, + }, + })) + live_map.replace_data(state_msg) + + assert live_map.data['dead'].tombstoned_at == 1700000050000 + + +# UTS: objects/unit/RTLM6d/replace-data-with-create-op-0 +def test_rtlm6d_replace_data_with_create_op(): + live_map = InternalLiveMap('map:test@1000', ObjectsMapSemantics.LWW) + + state_msg = object_message(build_object_state('map:test@1000', {'site1': '01'}, map={ + 'semantics': LWW, + 'entries': { + 'from_sync': {'data': {'string': 'synced'}, 'timeserial': '01'}, + }, + }, create_op={ + 'mapCreate': { + 'semantics': LWW, + 'entries': { + 'from_create': {'data': {'string': 'created'}, 'timeserial': '00'}, + }, + }, + })) + live_map.replace_data(state_msg) + + assert live_map.data['from_sync'].data == ObjectData(string='synced') + assert live_map.data['from_create'].data == ObjectData(string='created') + assert live_map.create_operation_is_merged is True + + +# UTS: objects/unit/RTLM6f/replace-data-tombstone-flag-0 +def test_rtlm6f_replace_data_tombstone_flag(): + live_map = InternalLiveMap('map:test@1000', ObjectsMapSemantics.LWW) + live_map.data = { + 'name': ObjectsMapEntry(data=ObjectData(string='Alice'), timeserial='01', tombstone=False), + } + + state_msg = object_message(build_object_state( + 'map:test@1000', {'site1': '01'}, map={'semantics': LWW, 'entries': {}}, tombstone=True)) + update = live_map.replace_data(state_msg) + + assert live_map.is_tombstone is True + assert live_map.data == {} + assert update.update == {'name': 'removed'} + assert update.tombstone is True + assert update.object_message is state_msg + + +# UTS: objects/unit/RTLO4e10/replace-data-tombstone-root-noop-0 +def test_rtlo4e10_replace_data_tombstone_root_noop(): + live_map = InternalLiveMap('root', ObjectsMapSemantics.LWW) + live_map.data = { + 'name': ObjectsMapEntry(data=ObjectData(string='Alice'), timeserial='01', tombstone=False), + } + + state_msg = object_message(build_object_state( + 'root', {'site1': '01'}, map={'semantics': LWW, 'entries': {}}, tombstone=True)) + update = live_map.replace_data(state_msg) + + assert live_map.is_tombstone is False + # The data is untouched + assert live_map.data['name'].data.string == 'Alice' + assert update.noop is True + + +# UTS: objects/unit/RTLM19/gc-tombstoned-entries-0 +def test_rtlm19_gc_tombstoned_entries(): + live_map = InternalLiveMap('root', ObjectsMapSemantics.LWW) + grace_period = 86400000 + now = 1700100000000 + + live_map.data = { + 'recent_dead': ObjectsMapEntry(data=None, timeserial='01', tombstone=True, tombstoned_at=now - 1000), + 'old_dead': ObjectsMapEntry(data=None, timeserial='01', tombstone=True, + tombstoned_at=now - grace_period - 1), + 'alive': ObjectsMapEntry(data=ObjectData(string='ok'), timeserial='01', tombstone=False), + } + + live_map.gc_tombstoned_entries(grace_period, now) + + assert 'recent_dead' in live_map.data + assert 'old_dead' not in live_map.data + assert 'alive' in live_map.data + + +# UTS: objects/unit/RTLM22/diff-calculation-0 +def test_rtlm22_diff_calculation(): + previous_data = { + 'removed': ObjectsMapEntry(data=ObjectData(string='gone'), timeserial='01', tombstone=False), + 'changed': ObjectsMapEntry(data=ObjectData(string='old'), timeserial='01', tombstone=False), + 'unchanged': ObjectsMapEntry(data=ObjectData(string='same'), timeserial='01', tombstone=False), + 'was_dead': ObjectsMapEntry(data=None, timeserial='01', tombstone=True), + } + new_data = { + 'added': ObjectsMapEntry(data=ObjectData(string='new'), timeserial='02', tombstone=False), + 'changed': ObjectsMapEntry(data=ObjectData(string='new_val'), timeserial='02', tombstone=False), + 'unchanged': ObjectsMapEntry(data=ObjectData(string='same'), timeserial='01', tombstone=False), + 'now_dead': ObjectsMapEntry(data=None, timeserial='02', tombstone=True), + } + + update = InternalLiveMap.diff(previous_data, new_data) + + assert update.update['removed'] == 'removed' + assert update.update['added'] == 'updated' + assert update.update['changed'] == 'updated' + assert 'unchanged' not in update.update + assert 'was_dead' not in update.update + assert 'now_dead' not in update.update + + +# UTS: objects/unit/RTLM22c/empty-diff-is-noop-0 +def test_rtlm22c_empty_diff_is_noop(): + live_map = InternalLiveMap('root', ObjectsMapSemantics.LWW) + live_map.data = { + 'name': ObjectsMapEntry(data=ObjectData(string='alice'), timeserial='01', tombstone=False), + } + + state_msg = object_message(build_object_state('root', {'site1': '02'}, map={ + 'semantics': LWW, + 'entries': { + 'name': {'data': {'string': 'alice'}, 'timeserial': '02', 'tombstone': False}, + }, + })) + update = live_map.replace_data(state_msg) + + assert update.noop is True + assert live_map.data['name'].data == ObjectData(string='alice') + + +# UTS: objects/unit/RTLM15d4/unsupported-action-0 +def test_rtlm15d4_unsupported_action(): + live_map = InternalLiveMap('root', ObjectsMapSemantics.LWW) + updates = capture_updates(live_map) + + msg = object_message(build_counter_inc('root', 5, '01', 'site1')) + result = live_map.apply_operation(msg, CHANNEL) + + _assert_not_applied(result, updates) + + +# UTS: objects/unit/RTLM6i/replace-data-resets-clear-timeserial-0 +def test_rtlm6i_replace_data_resets_clear_timeserial(): + live_map = InternalLiveMap('root', ObjectsMapSemantics.LWW) + live_map.clear_timeserial = '05' + live_map.data = { + 'x': ObjectsMapEntry(data=ObjectData(number=1), timeserial='03', tombstone=False), + } + + state_msg = object_message(build_object_state('root', {'site1': '01'}, map={ + 'semantics': LWW, + 'entries': { + 'y': {'data': {'number': 2}, 'timeserial': '01'}, + }, + })) + live_map.replace_data(state_msg) + + assert live_map.clear_timeserial is None + assert 'y' in live_map.data + + +# UTS: objects/unit/RTLM14c/tombstoned-ref-yields-null-0 +def test_rtlm14c_tombstoned_ref_yields_null(): + pool = ObjectsPool() + tombstoned_counter = InternalLiveCounter('counter:dead@1000') + tombstoned_counter.is_tombstone = True + pool['counter:dead@1000'] = tombstoned_counter + + live_map = InternalLiveMap('root', ObjectsMapSemantics.LWW, pool=pool) + live_map.data = { + 'ref': ObjectsMapEntry(data=ObjectData(object_id='counter:dead@1000'), timeserial='01', tombstone=False), + } + + # The entry itself is not tombstoned, but the object it references is + assert live_map.data['ref'].tombstone is False + # RTLM14c makes the entry tombstoned, so size() does not count it + assert live_map.size() == 0 + assert live_map.get('ref') is None + + +# UTS: objects/unit/RTLM7/map-set-revives-tombstoned-0 +def test_rtlm7_map_set_revives_tombstoned(): + live_map = InternalLiveMap('root', ObjectsMapSemantics.LWW) + live_map.data = { + 'name': ObjectsMapEntry(data=None, timeserial='01', tombstone=True, tombstoned_at=1700000000000), + } + updates = capture_updates(live_map) + + msg = object_message(build_map_set('root', 'name', {'string': 'Alice'}, '02', 'site1')) + applied = live_map.apply_operation(msg, CHANNEL) + + assert live_map.data['name'].data == ObjectData(string='Alice') + assert live_map.data['name'].tombstone is False + assert live_map.data['name'].tombstoned_at is None + update = _emitted_update(applied, updates) + assert update.update == {'name': 'updated'} + assert update.object_message is msg + + +# UTS: objects/unit/RTLM24/map-clear-preserves-newer-0 +def test_rtlm24_map_clear_preserves_newer(): + live_map = InternalLiveMap('root', ObjectsMapSemantics.LWW) + live_map.data = { + 'before': ObjectsMapEntry(data=ObjectData(string='a'), timeserial='03', tombstone=False), + 'after': ObjectsMapEntry(data=ObjectData(string='b'), timeserial='07', tombstone=False), + 'no_ts': ObjectsMapEntry(data=ObjectData(string='c'), timeserial=None, tombstone=False), + } + updates = capture_updates(live_map) + + msg = object_message(build_map_clear('root', '05', 'site1')) + applied = live_map.apply_operation(msg, CHANNEL) + + assert 'before' not in live_map.data + assert 'no_ts' not in live_map.data + assert live_map.data['after'].data == ObjectData(string='b') + update = _emitted_update(applied, updates) + assert 'before' in update.update + assert 'no_ts' in update.update + assert 'after' not in update.update + assert update.object_message is msg + + +# UTS: objects/unit/RTLM7a3/map-set-overwrite-objectid-parent-refs-0 +def test_rtlm7a3_map_set_overwrite_objectid_parent_refs(): + pool = ObjectsPool() + old_counter = InternalLiveCounter('counter:old@1000') + new_counter = InternalLiveCounter('counter:new@2000') + pool['counter:old@1000'] = old_counter + pool['counter:new@2000'] = new_counter + + live_map = InternalLiveMap('root', ObjectsMapSemantics.LWW, pool=pool) + live_map.data = { + 'ref': ObjectsMapEntry(data=ObjectData(object_id='counter:old@1000'), timeserial='01', tombstone=False), + } + # The existing reference, as the map would have recorded it + old_counter.parent_references = {'root': {'ref'}} + updates = capture_updates(live_map) + + msg = object_message(build_map_set('root', 'ref', {'objectId': 'counter:new@2000'}, '02', 'site1')) + applied = live_map.apply_operation(msg, CHANNEL) + + assert live_map.data['ref'].data == ObjectData(object_id='counter:new@2000') + # removeParentReference was called on the old child + assert not _references(old_counter, 'root', 'ref') + # addParentReference was called on the new child + assert 'root' in new_counter.parent_references + assert 'ref' in new_counter.parent_references['root'] + update = _emitted_update(applied, updates) + assert update.update == {'ref': 'updated'} + assert update.object_message is msg + + +# UTS: objects/unit/RTLM7g2/map-set-new-entry-add-parent-ref-0 +def test_rtlm7g2_map_set_new_entry_add_parent_ref(): + pool = ObjectsPool() + child_counter = InternalLiveCounter('counter:child@1000') + pool['counter:child@1000'] = child_counter + + live_map = InternalLiveMap('root', ObjectsMapSemantics.LWW, pool=pool) + updates = capture_updates(live_map) + + msg = object_message(build_map_set('root', 'score', {'objectId': 'counter:child@1000'}, '01', 'site1')) + applied = live_map.apply_operation(msg, CHANNEL) + + assert live_map.data['score'].data == ObjectData(object_id='counter:child@1000') + assert 'root' in child_counter.parent_references + assert 'score' in child_counter.parent_references['root'] + update = _emitted_update(applied, updates) + assert update.object_message is msg + + +# UTS: objects/unit/RTLM7/map-set-primitive-no-parent-refs-0 +def test_rtlm7_map_set_primitive_no_parent_refs(): + pool = ObjectsPool() + old_counter = InternalLiveCounter('counter:old@1000') + pool['counter:old@1000'] = old_counter + + live_map = InternalLiveMap('root', ObjectsMapSemantics.LWW, pool=pool) + live_map.data = { + 'ref': ObjectsMapEntry(data=ObjectData(object_id='counter:old@1000'), timeserial='01', tombstone=False), + } + old_counter.parent_references = {'root': {'ref'}} + updates = capture_updates(live_map) + + msg = object_message(build_map_set('root', 'ref', {'string': 'plain_value'}, '02', 'site1')) + applied = live_map.apply_operation(msg, CHANNEL) + + assert live_map.data['ref'].data == ObjectData(string='plain_value') + # removeParentReference was called on the old child, as the entry referenced it; the new + # value is a primitive, so there is no child to call addParentReference on + assert not _references(old_counter, 'root', 'ref') + update = _emitted_update(applied, updates) + assert update.update == {'ref': 'updated'} + assert update.object_message is msg + + +# UTS: objects/unit/RTLM8a3/map-remove-objectid-parent-refs-0 +def test_rtlm8a3_map_remove_objectid_parent_refs(): + pool = ObjectsPool() + child_counter = InternalLiveCounter('counter:child@1000') + pool['counter:child@1000'] = child_counter + + live_map = InternalLiveMap('root', ObjectsMapSemantics.LWW, pool=pool) + live_map.data = { + 'score': ObjectsMapEntry(data=ObjectData(object_id='counter:child@1000'), timeserial='01', + tombstone=False), + } + child_counter.parent_references = {'root': {'score'}} + updates = capture_updates(live_map) + + msg = object_message(build_map_remove('root', 'score', '02', 'site1', 1700000000000)) + applied = live_map.apply_operation(msg, CHANNEL) + + assert live_map.data['score'].tombstone is True + # removeParentReference was called on the child + assert not _references(child_counter, 'root', 'score') + update = _emitted_update(applied, updates) + assert update.update == {'score': 'removed'} + assert update.object_message is msg + + +# UTS: objects/unit/RTLM8/map-remove-primitive-no-parent-refs-0 +def test_rtlm8_map_remove_primitive_no_parent_refs(): + pool = ObjectsPool() + live_map = InternalLiveMap('root', ObjectsMapSemantics.LWW, pool=pool) + live_map.data = { + 'name': ObjectsMapEntry(data=ObjectData(string='Alice'), timeserial='01', tombstone=False), + } + updates = capture_updates(live_map) + + msg = object_message(build_map_remove('root', 'name', '02', 'site1', 1700000000000)) + applied = live_map.apply_operation(msg, CHANNEL) + + assert live_map.data['name'].tombstone is True + update = _emitted_update(applied, updates) + assert update.update == {'name': 'removed'} + assert update.object_message is msg + + +# UTS: objects/unit/RTLM24e1c/map-clear-parent-refs-0 +def test_rtlm24e1c_map_clear_parent_refs(): + pool = ObjectsPool() + counter_a = InternalLiveCounter('counter:a@1000') + counter_b = InternalLiveCounter('counter:b@1000') + pool['counter:a@1000'] = counter_a + pool['counter:b@1000'] = counter_b + + live_map = InternalLiveMap('root', ObjectsMapSemantics.LWW, pool=pool) + live_map.data = { + 'ref_a': ObjectsMapEntry(data=ObjectData(object_id='counter:a@1000'), timeserial='02', tombstone=False), + 'ref_b': ObjectsMapEntry(data=ObjectData(object_id='counter:b@1000'), timeserial='02', tombstone=False), + 'primitive': ObjectsMapEntry(data=ObjectData(string='hello'), timeserial='02', tombstone=False), + 'newer': ObjectsMapEntry(data=ObjectData(string='kept'), timeserial='09', tombstone=False), + } + counter_a.parent_references = {'root': {'ref_a'}} + counter_b.parent_references = {'root': {'ref_b'}} + updates = capture_updates(live_map) + + msg = object_message(build_map_clear('root', '05', 'site1')) + applied = live_map.apply_operation(msg, CHANNEL) + + # ref_a, ref_b and primitive are removed (timeserial '02' < '05'); newer is kept ('09' > '05') + assert 'ref_a' not in live_map.data + assert 'ref_b' not in live_map.data + assert 'primitive' not in live_map.data + assert 'newer' in live_map.data + # removeParentReference was called on both child counters + assert not _references(counter_a, 'root', 'ref_a') + assert not _references(counter_b, 'root', 'ref_b') + update = _emitted_update(applied, updates) + assert update.update == {'ref_a': 'removed', 'ref_b': 'removed', 'primitive': 'removed'} + assert update.object_message is msg + + +# UTS: objects/unit/RTLO4e9/tombstone-map-parent-refs-0 +def test_rtlo4e9_tombstone_map_parent_refs(): + pool = ObjectsPool() + child_counter = InternalLiveCounter('counter:child@1000') + child_map = InternalLiveMap('map:child@1000', ObjectsMapSemantics.LWW) + pool['counter:child@1000'] = child_counter + pool['map:child@1000'] = child_map + + live_map = InternalLiveMap('map:test@1000', ObjectsMapSemantics.LWW, pool=pool) + live_map.data = { + 'counter_ref': ObjectsMapEntry(data=ObjectData(object_id='counter:child@1000'), timeserial='01', + tombstone=False), + 'map_ref': ObjectsMapEntry(data=ObjectData(object_id='map:child@1000'), timeserial='01', tombstone=False), + 'name': ObjectsMapEntry(data=ObjectData(string='Alice'), timeserial='01', tombstone=False), + } + live_map.site_timeserials = {'site1': '00'} + child_counter.parent_references = {'map:test@1000': {'counter_ref'}} + child_map.parent_references = {'map:test@1000': {'map_ref'}} + updates = capture_updates(live_map) + + msg = object_message(build_object_delete('map:test@1000', '01', 'site1', 1700000000000)) + applied = live_map.apply_operation(msg, CHANNEL) + + assert live_map.is_tombstone is True + assert live_map.data == {} + # removeParentReference was called on both children + assert not _references(child_counter, 'map:test@1000', 'counter_ref') + assert not _references(child_map, 'map:test@1000', 'map_ref') + update = _emitted_update(applied, updates) + assert update.update == {'counter_ref': 'removed', 'map_ref': 'removed', 'name': 'removed'} + assert update.tombstone is True + assert update.object_message is msg + + +# UTS: objects/unit/RTLM7a3/map-set-replace-objectid-both-refs-0 +def test_rtlm7a3_map_set_replace_objectid_both_refs(): + pool = ObjectsPool() + old_map = InternalLiveMap('map:old@1000', ObjectsMapSemantics.LWW) + new_map = InternalLiveMap('map:new@2000', ObjectsMapSemantics.LWW) + pool['map:old@1000'] = old_map + pool['map:new@2000'] = new_map + + live_map = InternalLiveMap('root', ObjectsMapSemantics.LWW, pool=pool) + live_map.data = { + 'child': ObjectsMapEntry(data=ObjectData(object_id='map:old@1000'), timeserial='01', tombstone=False), + } + old_map.parent_references = {'root': {'child'}} + updates = capture_updates(live_map) + + msg = object_message(build_map_set('root', 'child', {'objectId': 'map:new@2000'}, '02', 'site1')) + applied = live_map.apply_operation(msg, CHANNEL) + + assert live_map.data['child'].data == ObjectData(object_id='map:new@2000') + # The old child no longer records the reference from root + assert not _references(old_map, 'root', 'child') + # The new child records it + assert 'root' in new_map.parent_references + assert 'child' in new_map.parent_references['root'] + update = _emitted_update(applied, updates) + assert update.update == {'child': 'updated'} + assert update.object_message is msg diff --git a/test/uts/objects/unit/live_object_subscribe_test.py b/test/uts/objects/unit/live_object_subscribe_test.py new file mode 100644 index 00000000..d240e8c8 --- /dev/null +++ b/test/uts/objects/unit/live_object_subscribe_test.py @@ -0,0 +1,321 @@ +"""Derived from uts/objects/unit/live_object_subscribe.md in ably/specification. + +Spec points: RTLO4b, RTLO4b3, RTLO4b4c1, RTLO4b4c3a, RTLO4b4c3c, RTLO4b4d, RTLO4b4e, +RTLO4b6, RTLO4b7, RTINS16e, RTLC14c + +`LiveObject#subscribe` is exercised through `Instance#subscribe` (RTINS16), as the +specification does. ably-python partitions `Instance` by type (RTTS) and only the live +object views have `subscribe`, so a counter's instance is subscribed through +`as_live_counter()` and the root map's through `as_live_map()`. Each listener receives an +`InstanceSubscriptionEvent`, whose `message` is the public `ObjectMessage`. + +Inbound frames are processed on the transport's read task, so a positive is awaited with +`poll_until`, and an exact count or a "listener did not fire" is asserted only once a +control listener on a later dispatch has fired and the loop has settled (the +specification's negative-assertion quiescence pattern). +""" + +from ably.pubsub.objects import publicmessage +from ably.pubsub.objects.objectmessage import ObjectOperationAction +from ably.pubsub.objects.subscription import Subscription +from test.uts.helpers.client import poll_until +from test.uts.helpers.clock import settle +from test.uts.objects.helpers.standard_test_pool import ( + assert_unchanged_after_quiescence, + build_counter_inc, + build_map_set, + build_object_delete, + build_object_message, + build_public_object_message, + remote_serial, + setup_synced_channel, +) + + +# UTS: objects/unit/RTLO4b/subscribe-receives-updates-0 +async def test_rtlo4b_subscribe_receives_updates(): + ctx = await setup_synced_channel('test') + updates = [] + instance = ctx.root.get('score').instance() + sub = instance.as_live_counter().subscribe(lambda event: updates.append(event)) + + ctx.mock_ws.send_to_client(build_object_message('test', [ + build_counter_inc('counter:score@1000', 7, '99', 'remote'), + ])) + await poll_until(lambda: len(updates) >= 1, timeout=5, description='the counter update to be delivered') + await settle() + + assert isinstance(sub, Subscription) + assert len(updates) == 1 + + +# UTS: objects/unit/RTLO4b7/subscribe-returns-subscription-0 +async def test_rtlo4b7_subscribe_returns_subscription(): + ctx = await setup_synced_channel('test') + instance = ctx.root.get('score').instance() + + sub = instance.as_live_counter().subscribe(lambda event: None) + + assert isinstance(sub, Subscription) + assert callable(sub.unsubscribe) + + +# UTS: objects/unit/RTLO4b7/subscription-unsubscribe-stops-delivery-0 +async def test_rtlo4b7_subscription_unsubscribe_stops_delivery(): + ctx = await setup_synced_channel('test') + updates = [] + control = [] + instance = ctx.root.get('score').instance() + sub = instance.as_live_counter().subscribe(lambda event: updates.append(event)) + + ctx.mock_ws.send_to_client(build_object_message('test', [ + build_counter_inc('counter:score@1000', 5, '01', 'remote'), + ])) + await poll_until(lambda: len(updates) >= 1, timeout=5, description='the first update to be delivered') + + sub.unsubscribe() + + # A control listener fires on the same dispatch as the message under test, so once it + # has fired the unsubscribed listener would also have run had it still been registered + instance.as_live_counter().subscribe(lambda event: control.append(event)) + ctx.mock_ws.send_to_client(build_object_message('test', [ + build_counter_inc('counter:score@1000', 10, '02', 'remote'), + ])) + await assert_unchanged_after_quiescence(lambda: len(updates), lambda: len(control) >= 1, + description='the control listener received the second update') + + assert len(updates) == 1 + + +# UTS: objects/unit/RTLO4b7/subscription-unsubscribe-idempotent-0 +async def test_rtlo4b7_subscription_unsubscribe_idempotent(): + ctx = await setup_synced_channel('test') + instance = ctx.root.get('score').instance() + sub = instance.as_live_counter().subscribe(lambda event: None) + + # Neither call raises + sub.unsubscribe() + sub.unsubscribe() + + +# UTS: objects/unit/RTLO4b4c1/noop-no-trigger-0 +async def test_rtlo4b4c1_noop_no_trigger(): + ctx = await setup_synced_channel('test') + updates = [] + control = [] + instance = ctx.root.get('score').instance() + instance.as_live_counter().subscribe(lambda event: updates.append(event)) + + ctx.mock_ws.send_to_client(build_object_message('test', [ + build_counter_inc('counter:score@1000', 5, '01', 'remote'), + ])) + await poll_until(lambda: len(updates) >= 1, timeout=5, description='the first update to be delivered') + + # Serial "02" passes the newness check (RTLO4a6), and an increment with no `number` is + # the no-op (RTLC9h). `number: 0` would be present per RTLC9g and give a real update of 0. + ctx.mock_ws.send_to_client(build_object_message('test', [{ + 'serial': '02', + 'siteCode': 'remote', + 'operation': { + 'action': int(ObjectOperationAction.COUNTER_INC), + 'objectId': 'counter:score@1000', + 'counterInc': {}, + }, + }])) + # The follow-up "03" is dispatched after the no-op, so once its separate control + # listener fires the no-op has certainly been processed + control_sub = instance.as_live_counter().subscribe(lambda event: control.append(event)) + ctx.mock_ws.send_to_client(build_object_message('test', [ + build_counter_inc('counter:score@1000', 3, '03', 'remote'), + ])) + await poll_until(lambda: len(control) >= 1, timeout=5, description='the control listener to receive "03"') + await settle() + control_sub.unsubscribe() + + # The original listener fired for "01" and "03" only; had the no-op fired it would be 3 + assert len(updates) == 2 + + +# UTS: objects/unit/RTLO4b6/subscribe-no-side-effects-0 +async def test_rtlo4b6_subscribe_no_side_effects(): + ctx = await setup_synced_channel('test') + state_before = ctx.channel.state + sync_state_before = ctx.channel.object._sync_state + sent_before = len(ctx.mock_ws.messages_from_client) + instance = ctx.root.get('score').instance() + + instance.as_live_counter().subscribe(lambda event: None) + await settle() + + assert ctx.channel.state == state_before + # Nor does it change the objects' sync state or send anything on the channel + assert ctx.channel.object._sync_state == sync_state_before + assert len(ctx.mock_ws.messages_from_client) == sent_before + + +# UTS: objects/unit/RTLO4b/subscribe-map-update-0 +async def test_rtlo4b_subscribe_map_update(): + ctx = await setup_synced_channel('test') + updates = [] + instance = ctx.root.instance() + instance.as_live_map().subscribe(lambda event: updates.append(event)) + + ctx.mock_ws.send_to_client(build_object_message('test', [ + build_map_set('root', 'name', {'string': 'Bob'}, remote_serial(0), 'remote'), + ])) + await poll_until(lambda: len(updates) >= 1, timeout=5, description='the map update to be delivered') + await settle() + + assert len(updates) == 1 + + +# UTS: objects/unit/RTLO4b4c3c/tombstone-deregisters-listeners-0 +async def test_rtlo4b4c3c_tombstone_deregisters_listeners(): + ctx = await setup_synced_channel('test') + updates_a = [] + updates_b = [] + control = [] + instance = ctx.root.get('score').instance() + instance.as_live_counter().subscribe(lambda event: updates_a.append(event)) + instance.as_live_counter().subscribe(lambda event: updates_b.append(event)) + + # An OBJECT_DELETE tombstones the counter + ctx.mock_ws.send_to_client(build_object_message('test', [ + build_object_delete('counter:score@1000', '50', 'remote'), + ])) + # Both listeners are awaited on this dispatch before either count is asserted + await poll_until(lambda: len(updates_a) >= 1, timeout=5, description='listener A to receive the tombstone') + await poll_until(lambda: len(updates_b) >= 1, timeout=5, description='listener B to receive the tombstone') + await settle() + + # Both listeners received the tombstone update + assert len(updates_a) == 1 + assert updates_a[0].message.operation.action == ObjectOperationAction.OBJECT_DELETE + assert len(updates_b) == 1 + assert updates_b[0].message.operation.action == ObjectOperationAction.OBJECT_DELETE + + # A tombstoned object ignores further operations (RTLC7e), so no listener on the counter + # can serve as the control. A listener on another live object can: the "52" MAP_SET on + # the profile map is processed after the "51" increment, so once it fires "51" has been too. + control_inst = ctx.root.get('profile').instance() + control_inst.as_live_map().subscribe(lambda event: control.append(event)) + ctx.mock_ws.send_to_client(build_object_message('test', [ + build_counter_inc('counter:score@1000', 3, '51', 'remote'), + ])) + ctx.mock_ws.send_to_client(build_object_message('test', [ + build_map_set('map:profile@1000', 'quiescence_probe', {'string': 'x'}, '52', 'remote'), + ])) + await assert_unchanged_after_quiescence(lambda: (len(updates_a), len(updates_b)), lambda: len(control) >= 1, + description='the control listener on the profile map fired') + + # The tombstone deregistered both listeners (RTLO4b4c3c) + assert len(updates_a) == 1 + assert len(updates_b) == 1 + + +# UTS: objects/unit/RTLO4b4c3c/tombstone-zero-value-counter-tears-down-0 +async def test_rtlo4b4c3c_tombstone_zero_value_counter_tears_down(): + ctx = await setup_synced_channel('test') + updates_a = [] + updates_b = [] + control = [] + instance = ctx.root.get('score').instance() + + # The counter (100 in the standard pool) is driven to 0 before the listeners under test + # are registered, so they observe only the tombstone + ctx.mock_ws.send_to_client(build_object_message('test', [ + build_counter_inc('counter:score@1000', -100, '40', 'remote'), + ])) + await poll_until(lambda: ctx.root.get('score').as_live_counter().value() == 0, timeout=5, + description='the counter to reach 0') + + instance.as_live_counter().subscribe(lambda event: updates_a.append(event)) + instance.as_live_counter().subscribe(lambda event: updates_b.append(event)) + + # The OBJECT_DELETE tombstones an already-zero counter: a zero-delta diff, which for a + # tombstone is not a no-op (RTLC14c) + ctx.mock_ws.send_to_client(build_object_message('test', [ + build_object_delete('counter:score@1000', '50', 'remote'), + ])) + await poll_until(lambda: len(updates_a) >= 1, timeout=5, description='listener A to receive the tombstone') + await poll_until(lambda: len(updates_b) >= 1, timeout=5, description='listener B to receive the tombstone') + await settle() + + # Both listeners received the tombstone update although the counter's data did not change + assert len(updates_a) == 1 + assert updates_a[0].message.operation.action == ObjectOperationAction.OBJECT_DELETE + assert len(updates_b) == 1 + assert updates_b[0].message.operation.action == ObjectOperationAction.OBJECT_DELETE + + # A separate live object is the quiescence barrier, as in the populated case + control_inst = ctx.root.get('profile').instance() + control_inst.as_live_map().subscribe(lambda event: control.append(event)) + ctx.mock_ws.send_to_client(build_object_message('test', [ + build_counter_inc('counter:score@1000', 3, '51', 'remote'), + ])) + ctx.mock_ws.send_to_client(build_object_message('test', [ + build_map_set('map:profile@1000', 'quiescence_probe', {'string': 'x'}, '52', 'remote'), + ])) + await assert_unchanged_after_quiescence(lambda: (len(updates_a), len(updates_b)), lambda: len(control) >= 1, + description='the control listener on the profile map fired') + + # The tombstone deregistered both listeners (RTLO4b4c3c) + assert len(updates_a) == 1 + assert len(updates_b) == 1 + + +# UTS: objects/unit/RTLO4b4d/update-has-object-message-0 +async def test_rtlo4b4d_update_has_object_message(): + ctx = await setup_synced_channel('test') + updates = [] + instance = ctx.root.get('score').instance() + instance.as_live_counter().subscribe(lambda event: updates.append(event)) + + counter_inc = build_counter_inc('counter:score@1000', 7, '99', 'remote') + ctx.mock_ws.send_to_client(build_object_message('test', [counter_inc])) + await poll_until(lambda: len(updates) >= 1, timeout=5, description='the counter update to be delivered') + await settle() + + assert len(updates) == 1 + assert updates[0].message is not None + assert updates[0].message.serial == '99' + assert updates[0].message.site_code == 'remote' + assert updates[0].message.operation.action == ObjectOperationAction.COUNTER_INC + assert updates[0].message.operation.object_id == 'counter:score@1000' + # RTINS16e2: the message is the public ObjectMessage derived from the source per PAOM3 + assert isinstance(updates[0].message, publicmessage.ObjectMessage) + assert updates[0].message == build_public_object_message(counter_inc, 'test') + + +# UTS: objects/unit/RTLO4b4e/tombstone-flag-true-0 +async def test_rtlo4b4e_tombstone_flag_true(): + ctx = await setup_synced_channel('test') + updates = [] + instance = ctx.root.get('score').instance() + instance.as_live_counter().subscribe(lambda event: updates.append(event)) + + ctx.mock_ws.send_to_client(build_object_message('test', [ + build_object_delete('counter:score@1000', '50', 'remote'), + ])) + await poll_until(lambda: len(updates) >= 1, timeout=5, description='the tombstone update to be delivered') + await settle() + + assert len(updates) == 1 + assert updates[0].message.operation.action == ObjectOperationAction.OBJECT_DELETE + + +# UTS: objects/unit/RTLO4b4e/tombstone-flag-false-0 +async def test_rtlo4b4e_tombstone_flag_false(): + ctx = await setup_synced_channel('test') + updates = [] + instance = ctx.root.get('score').instance() + instance.as_live_counter().subscribe(lambda event: updates.append(event)) + + ctx.mock_ws.send_to_client(build_object_message('test', [ + build_counter_inc('counter:score@1000', 7, '99', 'remote'), + ])) + await poll_until(lambda: len(updates) >= 1, timeout=5, description='the counter update to be delivered') + await settle() + + assert len(updates) == 1 + assert updates[0].message.operation.action == ObjectOperationAction.COUNTER_INC diff --git a/test/uts/objects/unit/object_id_test.py b/test/uts/objects/unit/object_id_test.py new file mode 100644 index 00000000..d6f43184 --- /dev/null +++ b/test/uts/objects/unit/object_id_test.py @@ -0,0 +1,99 @@ +"""Derived from uts/objects/unit/object_id.md in ably/specification. + +Spec points: RTO14, RTO14a1, RTO14b, RTO14b1, RTO14b2, RTO14c + +A pure function: `generate_object_id(object_type, initial_value, nonce, timestamp_ms)` is the +specification's `generateObjectId(type:, initialValue:, nonce:, timestamp:)`, producing +`[type]:[hash]@[timestamp]`. +""" + +import base64 +import hashlib +import re + +from ably.pubsub.objects.objectid import generate_object_id + +# RFC 4648 s.5's URL-safe alphabet, with no padding +BASE64URL_UNPADDED = re.compile(r'[A-Za-z0-9_-]+') + + +def _is_base64url(value): + """Whether `value` is unpadded base64url that decodes: the alphabet, and a length no + encoding produces (one more than a multiple of four) excluded.""" + if not BASE64URL_UNPADDED.fullmatch(value) or len(value) % 4 == 1: + return False + base64.urlsafe_b64decode(value + '=' * (-len(value) % 4)) + return True + + +def _expected_hash(initial_value, nonce): + """RTO14b, computed here independently: the unpadded base64url SHA-256 digest of the + UTF-8 string `initial_value:nonce`.""" + digest = hashlib.sha256(f'{initial_value}:{nonce}'.encode()).digest() + return base64.urlsafe_b64encode(digest).decode('ascii').rstrip('=') + + +# UTS: objects/unit/RTO14/objectid-format-counter-0 +def test_rto14_objectid_format_counter(): + object_id = generate_object_id('counter', '{"counter":{"count":42}}', 'test-nonce-12345678', 1700000000000) + + assert object_id.startswith('counter:') + assert '@1700000000000' in object_id + parts = object_id.split(':') + # The shape is asserted before indexing, so a malformed id fails on an assertion + assert len(parts) == 2 + type_part = parts[0] + rest = parts[1] + hash_and_ts = rest.split('@') + assert len(hash_and_ts) == 2 + hash_part = hash_and_ts[0] + ts_part = hash_and_ts[1] + assert type_part == 'counter' + assert ts_part == '1700000000000' + assert _is_base64url(hash_part) + assert '+' not in hash_part + assert '/' not in hash_part + assert '=' not in hash_part + # The hash itself, which RTO14b1 and RTO14b2 determine and the specification's + # assertions leave open: the digest of the initial value and the nonce, joined by ':' + assert hash_part == _expected_hash('{"counter":{"count":42}}', 'test-nonce-12345678') + + +# UTS: objects/unit/RTO14/objectid-format-map-0 +def test_rto14_objectid_format_map(): + object_id = generate_object_id( + 'map', '{"map":{"semantics":"LWW","entries":{}}}', 'test-nonce-12345678', 1700000000000) + + assert object_id.startswith('map:') + assert '@1700000000000' in object_id + + +# UTS: objects/unit/RTO14/deterministic-0 +def test_rto14_deterministic(): + id1 = generate_object_id('counter', '{"counter":{"count":0}}', 'same-nonce-1234567', 1700000000000) + id2 = generate_object_id('counter', '{"counter":{"count":0}}', 'same-nonce-1234567', 1700000000000) + + assert id1 == id2 + + +# UTS: objects/unit/RTO14/different-nonce-0 +def test_rto14_different_nonce(): + id1 = generate_object_id('counter', '{"counter":{"count":0}}', 'nonce-aaaaaaaaaaaaa', 1700000000000) + id2 = generate_object_id('counter', '{"counter":{"count":0}}', 'nonce-bbbbbbbbbbbbb', 1700000000000) + + assert id1 != id2 + + +# UTS: objects/unit/RTO14b/base64url-encoding-0 +def test_rto14b_base64url_encoding(): + object_id = generate_object_id('counter', '{"counter":{"count":0}}', 'test-nonce-12345678', 1700000000000) + parts = object_id.split(':') + # The shape is asserted before indexing, so a malformed id fails on an assertion + assert len(parts) == 2 + hash_and_ts = parts[1].split('@') + assert len(hash_and_ts) == 2 + hash_part = hash_and_ts[0] + + assert '+' not in hash_part + assert '/' not in hash_part + assert not hash_part.endswith('=') diff --git a/test/uts/objects/unit/objects_pool_test.py b/test/uts/objects/unit/objects_pool_test.py new file mode 100644 index 00000000..04843e5e --- /dev/null +++ b/test/uts/objects/unit/objects_pool_test.py @@ -0,0 +1,687 @@ +"""Derived from uts/objects/unit/objects_pool.md in ably/specification. + +Spec points: RTO3, RTO3a, RTO3b, RTO3b1, RTO4, RTO4a, RTO4b, RTO4b1, RTO4b2, RTO4b2a, RTO4b4, +RTO4c, RTO4d, RTO5, RTO5a1, RTO5a2, RTO5a2a, RTO5a4, RTO5a5, RTO5a6, RTO5c, RTO5c2, RTO5c2a, +RTO5c6, RTO5c7, RTO5c8, RTO5c9, RTO5c10, RTO5c10a, RTO5c10b, RTO5d, RTO5e, RTO5f1, RTO5f2, +RTO5f2a2, RTO5f2b, RTO5f3, RTO6, RTO6a, RTO6b1, RTO6b2, RTO6b3, RTO7, RTO7a, RTO8a, RTO8b, +RTO9a1, RTO9a2a4, RTO9a2b, RTO9a3, RTLM23 + +A pure unit specification: nothing is mocked and nothing is connected. The specification +puts the sync and apply state machine on `ObjectsPool` (`processAttached`, +`processObjectSync`, `processObjectMessage`, `applyObjectMessages`, `syncState`). Here, as +in features.md (RTO4-RTO9, RTO17), it is `RealtimeObject`'s, and the pool is a mapping of +object ids to objects (RTO3a); this is shape deviation S-2. A test that builds only +`pool = ObjectsPool()` and then drives the state machine works on the pool a standalone +`RealtimeObject()` holds (`_driven_pool()` below); one that builds +`RealtimeObject(pool: pool)` does the same with `RealtimeObject(pool=pool)`. The +`_process_*` helpers below are the specification's three `process*` calls. + +The channel hands `RealtimeObject` an ATTACHED's HAS_OBJECTS flag and nothing else (RTO4), +so the `channelSerial` the specification gives its ATTACHED messages goes no further than +the message dictionary. An OBJECT_SYNC's `channelSerial` is the sync cursor (RTO5a1) and is +passed on. + +Updates are emitted synchronously (`LiveObject.notify_updated`), so a listener registered +with `subscribe` has seen every update by the time the call that caused it returns. The +tests that drive a `RealtimeObject` are coroutines only so that an implementation is free to +touch the event loop. +""" + +from ably.pubsub.objects.enums import ObjectsOperationSource, ObjectsSyncState +from ably.pubsub.objects.livecounter import InternalLiveCounter +from ably.pubsub.objects.livemap import InternalLiveMap +from ably.pubsub.objects.objectmessage import ObjectData, ObjectOperationAction, ObjectsMapEntry +from ably.pubsub.objects.objectspool import ObjectsPool +from ably.pubsub.objects.realtimeobject import RealtimeObject +from test.uts.objects.helpers.standard_test_pool import ( + ATTACHED, + HAS_OBJECTS, + LWW, + build_counter_inc, + build_map_set, + build_object_message, + build_object_state, + build_object_sync_message, + object_message, + object_messages, + objects_attached_message, +) + + +def _driven_pool(): + """The specification's `pool = ObjectsPool()`, in a test that then drives the sync state machine. + + S-2: the state machine is `RealtimeObject`'s, so the pool is the one a standalone + `RealtimeObject` holds. Returns `(realtime_object, pool)`. + """ + realtime_object = RealtimeObject() + return realtime_object, realtime_object._objects_pool + + +def _process_attached(realtime_object, attached): + """The specification's `pool.processAttached(attached)` (S-2). + + `RealtimeObject` is given the ATTACHED's HAS_OBJECTS flag; a message with no `flags` + has the flag unset (RTO4b). + """ + realtime_object._on_attached(has_objects=bool(attached.get('flags', 0) & HAS_OBJECTS)) + + +def _process_object_sync(realtime_object, protocol_message): + """The specification's `pool.processObjectSync(protocol_message)` (S-2).""" + realtime_object._handle_object_sync_messages( + object_messages(protocol_message), protocol_message.get('channelSerial')) + + +def _process_object_message(realtime_object, protocol_message): + """The specification's `pool.processObjectMessage(protocol_message)` (S-2).""" + realtime_object._handle_object_messages(object_messages(protocol_message)) + + +def _empty_root_state(): + """The root state most syncs carry: an empty LWW map whose create operation adds nothing.""" + return build_object_state('root', {'aaa': 't:0'}, map={'semantics': LWW, 'entries': {}}, + create_op={'mapCreate': {'semantics': LWW, 'entries': {}}}) + + +# UTS: objects/unit/RTO3/pool-init-root-0 +def test_rto3_pool_init_root(): + pool = ObjectsPool() + + assert 'root' in pool + assert isinstance(pool['root'], InternalLiveMap) + assert pool['root'].data == {} + assert pool['root'].object_id == 'root' + + +# UTS: objects/unit/RTO4/attached-has-objects-syncing-0 +async def test_rto4_attached_has_objects_syncing(): + realtime_object, pool = _driven_pool() + + _process_attached(realtime_object, objects_attached_message('test', 'sync1:cursor', flags=HAS_OBJECTS)) + + assert realtime_object._sync_state == ObjectsSyncState.SYNCING + + +# UTS: objects/unit/RTO4b/attached-no-objects-synced-0 +async def test_rto4b_attached_no_objects_synced(): + realtime_object, pool = _driven_pool() + pool['counter:abc@1000'] = InternalLiveCounter('counter:abc@1000') + pool['root'].data = { + 'name': ObjectsMapEntry(data=ObjectData(string='Alice'), timeserial='01', tombstone=False), + } + root = pool['root'] + + updates = [] + pool['root'].subscribe(updates.append) + + _process_attached(realtime_object, {'action': ATTACHED, 'channel': 'test', 'flags': 0}) + + assert realtime_object._sync_state == ObjectsSyncState.SYNCED + assert 'counter:abc@1000' not in pool + assert 'root' in pool + # RTO4b2: the root map's data is cleared in place; the root is never replaced + assert pool['root'] is root + assert pool['root'].data == {} + assert len(updates) >= 1 + assert updates[0].update == {'name': 'removed'} + assert updates[0].object_message is None + + +# UTS: objects/unit/RTO4b2a/reset-of-empty-root-emits-no-update-0 +async def test_rto4b2a_reset_of_empty_root_emits_no_update(): + realtime_object, pool = _driven_pool() + pool['counter:abc@1000'] = InternalLiveCounter('counter:abc@1000') + # The root is already empty, the zero value of RTLM4c + pool['root'].data = {} + + updates = [] + pool['root'].subscribe(updates.append) + + _process_attached(realtime_object, {'action': ATTACHED, 'channel': 'test', 'flags': 0}) + + assert realtime_object._sync_state == ObjectsSyncState.SYNCED + # RTO4b1: the objects other than the root are still removed + assert 'counter:abc@1000' not in pool + assert 'root' in pool + assert pool['root'].data == {} + # RTO4b2a: no key was removed, so the update has no changed keys, is a no-op and is + # not delivered + assert len(updates) == 0 + + # Liveness control: a reset that does remove a key is delivered, so the count of zero + # above is the empty root's no-op and not a dead subscription. Emission is synchronous + # at this tier, so there is nothing to wait for. + realtime_object2, pool2 = _driven_pool() + pool2['root'].data = { + 'name': ObjectsMapEntry(data=ObjectData(string='Alice'), timeserial='01', tombstone=False), + } + control = [] + pool2['root'].subscribe(control.append) + _process_attached(realtime_object2, {'action': ATTACHED, 'channel': 'test', 'flags': 0}) + assert len(control) >= 1 + assert control[0].update == {'name': 'removed'} + + +# UTS: objects/unit/RTO5/sync-complete-sequence-0 +async def test_rto5_sync_complete_sequence(): + realtime_object, pool = _driven_pool() + _process_attached(realtime_object, objects_attached_message('test', 'sync1:cursor', flags=HAS_OBJECTS)) + + _process_object_sync(realtime_object, build_object_sync_message('test', 'sync1:', [ + build_object_state('root', {'aaa': 't:0'}, map={ + 'semantics': LWW, + 'entries': {'name': {'data': {'string': 'Alice'}, 'timeserial': 't:0'}}, + }, create_op={'mapCreate': {'semantics': LWW, 'entries': {}}}), + build_object_state('counter:abc@1000', {'aaa': 't:0'}, counter={'count': 0}, + create_op={'counterCreate': {'count': 42}}), + ])) + + assert realtime_object._sync_state == ObjectsSyncState.SYNCED + assert 'root' in pool + assert 'counter:abc@1000' in pool + assert pool['root'].data['name'].data == ObjectData(string='Alice') + assert pool['counter:abc@1000'].data == 42 + + +# UTS: objects/unit/RTO5a2/new-sequence-discards-old-0 +async def test_rto5a2_new_sequence_discards_old(): + realtime_object, pool = _driven_pool() + _process_attached(realtime_object, objects_attached_message('test', 'seq1:cursor', flags=HAS_OBJECTS)) + _process_object_sync(realtime_object, build_object_sync_message('test', 'seq1:more', [ + build_object_state('counter:old@1000', {'aaa': 't:0'}, counter={'count': 10}), + ])) + + _process_object_sync(realtime_object, build_object_sync_message('test', 'seq2:', [ + _empty_root_state(), + build_object_state('counter:new@1000', {'aaa': 't:0'}, counter={'count': 99}), + ])) + + assert realtime_object._sync_state == ObjectsSyncState.SYNCED + assert 'counter:old@1000' not in pool + assert 'counter:new@1000' in pool + + +# UTS: objects/unit/RTO5a5/absent-channel-serial-0 +async def test_rto5a5_absent_channel_serial(): + realtime_object, pool = _driven_pool() + _process_attached(realtime_object, objects_attached_message('test', 'sync1:cursor', flags=HAS_OBJECTS)) + + # No channelSerial: the whole sync is contained in this one message (RTO5a5) + _process_object_sync(realtime_object, build_object_sync_message('test', None, [ + build_object_state('counter:new@1000', {'aaa': 't:0'}, counter={'count': 99}), + ])) + + assert realtime_object._sync_state == ObjectsSyncState.SYNCED + assert 'counter:new@1000' in pool + + +# UTS: objects/unit/RTO5a6/malformed-channel-serial-treated-as-absent-0 +async def test_rto5a6_malformed_channel_serial_treated_as_absent(): + realtime_object, pool = _driven_pool() + _process_attached(realtime_object, objects_attached_message('test', 'sync1:cursor', flags=HAS_OBJECTS)) + + # 'malformedserialnocolon' has no ':' separator, so it cannot be parsed per RTO5a1; + # RTO5a6 handles it as if the channelSerial were absent (RTO5a5) + _process_object_sync(realtime_object, build_object_sync_message('test', 'malformedserialnocolon', [ + build_object_state('counter:new@1000', {'aaa': 't:0'}, counter={'count': 99}), + ])) + + # Treated as absent (RTO5a5): the message was applied and the sync ended + assert realtime_object._sync_state == ObjectsSyncState.SYNCED + assert 'counter:new@1000' in pool + + +# UTS: objects/unit/RTO5f2a/partial-map-merge-0 +async def test_rto5f2a_partial_map_merge(): + realtime_object, pool = _driven_pool() + _process_attached(realtime_object, objects_attached_message('test', 'sync1:cursor', flags=HAS_OBJECTS)) + + _process_object_sync(realtime_object, build_object_sync_message('test', 'sync1:more', [ + build_object_state('root', {'aaa': 't:0'}, map={ + 'semantics': LWW, + 'entries': {'name': {'data': {'string': 'Alice'}, 'timeserial': 't:0'}}, + }), + ])) + _process_object_sync(realtime_object, build_object_sync_message('test', 'sync1:', [ + build_object_state('root', {'aaa': 't:0'}, map={ + 'semantics': LWW, + 'entries': {'age': {'data': {'number': 30}, 'timeserial': 't:0'}}, + }, create_op={'mapCreate': {'semantics': LWW, 'entries': {}}}), + ])) + + assert pool['root'].data['name'].data == ObjectData(string='Alice') + assert pool['root'].data['age'].data == ObjectData(number=30) + + +# UTS: objects/unit/RTO5c2/remove-absent-objects-0 +async def test_rto5c2_remove_absent_objects(): + realtime_object, pool = _driven_pool() + pool['counter:old@1000'] = InternalLiveCounter('counter:old@1000') + pool['counter:old@1000'].data = 99 + _process_attached(realtime_object, objects_attached_message('test', 'sync1:cursor', flags=HAS_OBJECTS)) + + _process_object_sync(realtime_object, build_object_sync_message('test', 'sync1:', [_empty_root_state()])) + + assert 'counter:old@1000' not in pool + assert 'root' in pool + + +# UTS: objects/unit/RTO5c9/clear-applied-on-ack-serials-0 +async def test_rto5c9_clear_applied_on_ack_serials(): + pool = ObjectsPool() + realtime_object = RealtimeObject(pool=pool) + realtime_object._applied_on_ack_serials = {'serial-1', 'serial-2'} + _process_attached(realtime_object, objects_attached_message('test', 'sync1:cursor', flags=HAS_OBJECTS)) + + _process_object_sync(realtime_object, build_object_sync_message('test', 'sync1:', [_empty_root_state()])) + + # The specification's `{}` is the empty set + assert realtime_object._applied_on_ack_serials == set() + + +# UTS: objects/unit/RTO8a/buffer-during-syncing-0 +async def test_rto8a_buffer_during_syncing(): + pool = ObjectsPool() + realtime_object = RealtimeObject(pool=pool) + _process_attached(realtime_object, objects_attached_message('test', 'sync1:cursor', flags=HAS_OBJECTS)) + + _process_object_message(realtime_object, build_object_message('test', [ + build_counter_inc('counter:abc@1000', 5, '01', 'site1'), + ])) + + assert realtime_object._sync_state == ObjectsSyncState.SYNCING + assert len(realtime_object._buffered_object_operations) == 1 + assert 'counter:abc@1000' not in pool + + +# UTS: objects/unit/RTO5c6/apply-buffered-on-sync-0 +async def test_rto5c6_apply_buffered_on_sync(): + pool = ObjectsPool() + realtime_object = RealtimeObject(pool=pool) + _process_attached(realtime_object, objects_attached_message('test', 'sync1:cursor', flags=HAS_OBJECTS)) + _process_object_message(realtime_object, build_object_message('test', [ + build_counter_inc('counter:abc@1000', 10, '02', 'site1'), + ])) + + _process_object_sync(realtime_object, build_object_sync_message('test', 'sync1:', [ + _empty_root_state(), + build_object_state('counter:abc@1000', {'aaa': 't:0'}, counter={'count': 0}, + create_op={'counterCreate': {'count': 100}}), + ])) + + assert pool['counter:abc@1000'].data == 110 + assert len(realtime_object._buffered_object_operations) == 0 + + +# UTS: objects/unit/RTO9a1/null-operation-warning-0 +async def test_rto9a1_null_operation_warning(): + realtime_object, pool = _driven_pool() + realtime_object._sync_state = ObjectsSyncState.SYNCED + + # The specification's `ObjectMessage(serial: "01", siteCode: "site1", operation: null)` + _process_object_message(realtime_object, build_object_message('test', [ + {'serial': '01', 'siteCode': 'site1'}, + ])) + + assert len(pool) == 1 + + +# UTS: objects/unit/RTO9a3/dedup-applied-on-ack-0 +async def test_rto9a3_dedup_applied_on_ack(): + pool = ObjectsPool() + realtime_object = RealtimeObject(pool=pool) + realtime_object._sync_state = ObjectsSyncState.SYNCED + pool['counter:abc@1000'] = InternalLiveCounter('counter:abc@1000') + pool['counter:abc@1000'].data = 10 + realtime_object._applied_on_ack_serials = {'echo-serial-1'} + + _process_object_message(realtime_object, build_object_message('test', [ + { + 'serial': 'echo-serial-1', + 'siteCode': 'site1', + 'operation': { + 'action': int(ObjectOperationAction.COUNTER_INC), + 'objectId': 'counter:abc@1000', + 'counterInc': {'number': 5}, + }, + }, + ])) + + assert pool['counter:abc@1000'].data == 10 + assert 'echo-serial-1' not in realtime_object._applied_on_ack_serials + + +# UTS: objects/unit/RTO9a2a4/local-source-adds-serial-0 +async def test_rto9a2a4_local_source_adds_serial(): + pool = ObjectsPool() + realtime_object = RealtimeObject(pool=pool) + realtime_object._sync_state = ObjectsSyncState.SYNCED + pool['counter:abc@1000'] = InternalLiveCounter('counter:abc@1000') + + realtime_object._apply_object_messages([ + object_message(build_counter_inc('counter:abc@1000', 5, 'local-serial-1', 'test-site')), + ], ObjectsOperationSource.LOCAL) + + assert 'local-serial-1' in realtime_object._applied_on_ack_serials + assert pool['counter:abc@1000'].data == 5 + + +# UTS: objects/unit/RTO9a2b/unsupported-action-warning-0 +async def test_rto9a2b_unsupported_action_warning(): + realtime_object, pool = _driven_pool() + realtime_object._sync_state = ObjectsSyncState.SYNCED + + # The specification's `action: "UNKNOWN_ACTION"`: 99 is no ObjectOperationAction, so it + # decodes to UNKNOWN (OOP2a) + _process_object_message(realtime_object, build_object_message('test', [ + {'serial': '01', 'siteCode': 'site1', 'operation': {'action': 99, 'objectId': 'counter:abc@1000'}}, + ])) + + assert len(pool) == 1 + + +# UTS: objects/unit/RTO6/zero-value-from-prefix-0 +async def test_rto6_zero_value_from_prefix(): + realtime_object, pool = _driven_pool() + realtime_object._sync_state = ObjectsSyncState.SYNCED + + _process_object_message(realtime_object, build_object_message('test', [ + build_counter_inc('counter:new@2000', 5, '01', 'site1'), + ])) + _process_object_message(realtime_object, build_object_message('test', [ + build_map_set('map:new@2000', 'key', {'string': 'val'}, '01', 'site1'), + ])) + + assert 'counter:new@2000' in pool + assert isinstance(pool['counter:new@2000'], InternalLiveCounter) + assert pool['counter:new@2000'].data == 5 + + assert 'map:new@2000' in pool + assert isinstance(pool['map:new@2000'], InternalLiveMap) + assert pool['map:new@2000'].data['key'].data == ObjectData(string='val') + + +# UTS: objects/unit/RTO5d/null-object-skipped-0 +async def test_rto5d_null_object_skipped(): + realtime_object, pool = _driven_pool() + _process_attached(realtime_object, objects_attached_message('test', 'sync1:cursor', flags=HAS_OBJECTS)) + + # The specification's `ObjectMessage(object: null)` is the empty message `{}` + _process_object_sync(realtime_object, build_object_sync_message('test', 'sync1:', [ + {}, + _empty_root_state(), + ])) + + assert realtime_object._sync_state == ObjectsSyncState.SYNCED + + +# UTS: objects/unit/RTO5f3/unsupported-type-skipped-0 +async def test_rto5f3_unsupported_type_skipped(): + realtime_object, pool = _driven_pool() + _process_attached(realtime_object, objects_attached_message('test', 'sync1:cursor', flags=HAS_OBJECTS)) + + _process_object_sync(realtime_object, build_object_sync_message('test', 'sync1:', [ + _empty_root_state(), + {'object': {'objectId': 'unknown:xyz@1000', 'siteTimeserials': {}}}, + ])) + + assert realtime_object._sync_state == ObjectsSyncState.SYNCED + assert 'unknown:xyz@1000' not in pool + + +# UTS: objects/unit/RTO5e/object-sync-transitions-syncing-0 +async def test_rto5e_object_sync_transitions_syncing(): + realtime_object, pool = _driven_pool() + + _process_object_sync(realtime_object, build_object_sync_message('test', 'sync1:more', [ + build_object_state('root', {'aaa': 't:0'}, map={'semantics': LWW, 'entries': {}}), + ])) + + assert realtime_object._sync_state == ObjectsSyncState.SYNCING + + +# UTS: objects/unit/RTO5c7/sync-emits-updates-0 +async def test_rto5c7_sync_emits_updates(): + realtime_object, pool = _driven_pool() + pool['root'].data = { + 'name': ObjectsMapEntry(data=ObjectData(string='Old'), timeserial='01', tombstone=False), + } + + updates = [] + pool['root'].subscribe(updates.append) + + _process_attached(realtime_object, objects_attached_message('test', 'sync1:cursor', flags=HAS_OBJECTS)) + + _process_object_sync(realtime_object, build_object_sync_message('test', 'sync1:', [ + build_object_state('root', {'aaa': 't:0'}, map={ + 'semantics': LWW, + 'entries': {'name': {'data': {'string': 'New'}, 'timeserial': 't:0'}}, + }, create_op={'mapCreate': {'semantics': LWW, 'entries': {}}}), + ])) + + assert len(updates) >= 1 + assert 'name' in updates[0].update + assert updates[0].update['name'] == 'updated' + + +# UTS: objects/unit/RTO5f2b/partial-counter-error-0 +async def test_rto5f2b_partial_counter_error(): + realtime_object, pool = _driven_pool() + _process_attached(realtime_object, objects_attached_message('test', 'sync1:cursor', flags=HAS_OBJECTS)) + + _process_object_sync(realtime_object, build_object_sync_message('test', 'sync1:more', [ + build_object_state('counter:abc@1000', {'aaa': 't:0'}, counter={'count': 10}), + ])) + _process_object_sync(realtime_object, build_object_sync_message('test', 'sync1:', [ + _empty_root_state(), + build_object_state('counter:abc@1000', {'aaa': 't:0'}, counter={'count': 5}), + ])) + + assert pool['counter:abc@1000'].data == 10 + + +# UTS: objects/unit/RTO4d/attached-clears-buffer-0 +async def test_rto4d_attached_clears_buffer(): + pool = ObjectsPool() + realtime_object = RealtimeObject(pool=pool) + _process_attached(realtime_object, objects_attached_message('test', 'sync1:cursor', flags=HAS_OBJECTS)) + _process_object_message(realtime_object, build_object_message('test', [ + build_counter_inc('counter:abc@1000', 5, '01', 'site1'), + ])) + assert len(realtime_object._buffered_object_operations) == 1 + + _process_attached(realtime_object, objects_attached_message('test', 'sync2:cursor', flags=HAS_OBJECTS)) + + assert len(realtime_object._buffered_object_operations) == 0 + + +# UTS: objects/unit/RTO4-RTO5/attached-during-syncing-resets-0 +async def test_rto4_rto5_attached_during_syncing_resets(): + realtime_object, pool = _driven_pool() + _process_attached(realtime_object, objects_attached_message('test', 'sync1:cursor', flags=HAS_OBJECTS)) + _process_object_sync(realtime_object, build_object_sync_message('test', 'sync1:more', [ + build_object_state('counter:old@1000', {'aaa': 't:0'}, counter={'count': 10}), + ])) + assert realtime_object._sync_state == ObjectsSyncState.SYNCING + + _process_attached(realtime_object, objects_attached_message('test', 'sync2:cursor', flags=HAS_OBJECTS)) + _process_object_sync(realtime_object, build_object_sync_message('test', 'sync2:', [ + _empty_root_state(), + build_object_state('counter:new@1000', {'aaa': 't:0'}, counter={'count': 99}), + ])) + + assert realtime_object._sync_state == ObjectsSyncState.SYNCED + assert 'counter:old@1000' not in pool + assert 'counter:new@1000' in pool + + +# UTS: objects/unit/RTO5-RTO7/new-sync-keeps-buffer-0 +async def test_rto5_rto7_new_sync_keeps_buffer(): + pool = ObjectsPool() + realtime_object = RealtimeObject(pool=pool) + _process_attached(realtime_object, objects_attached_message('test', 'sync1:cursor', flags=HAS_OBJECTS)) + _process_object_message(realtime_object, build_object_message('test', [ + build_counter_inc('counter:abc@1000', 5, '01', 'site1'), + ])) + assert len(realtime_object._buffered_object_operations) == 1 + + _process_object_sync(realtime_object, build_object_sync_message('test', 'seq2:', [ + _empty_root_state(), + build_object_state('counter:abc@1000', {'aaa': 't:0'}, counter={'count': 0}, + create_op={'counterCreate': {'count': 100}}), + ])) + + assert realtime_object._sync_state == ObjectsSyncState.SYNCED + assert pool['counter:abc@1000'].data == 105 + + +# UTS: objects/unit/RTO7-RTO8/buffer-without-attached-0 +async def test_rto7_rto8_buffer_without_attached(): + pool = ObjectsPool() + realtime_object = RealtimeObject(pool=pool) + assert realtime_object._sync_state == ObjectsSyncState.INITIALIZED + + _process_object_message(realtime_object, build_object_message('test', [ + build_counter_inc('counter:abc@1000', 5, '01', 'site1'), + ])) + + assert len(realtime_object._buffered_object_operations) == 1 + + +# UTS: objects/unit/RTO5c-RTLM23/sync-clear-timeserial-hides-create-entries-0 +async def test_rto5c_rtlm23_sync_clear_timeserial_hides_create_entries(): + realtime_object, pool = _driven_pool() + _process_attached(realtime_object, objects_attached_message('test', 'sync1:cursor', flags=HAS_OBJECTS)) + + _process_object_sync(realtime_object, build_object_sync_message('test', 'sync1:', [ + build_object_state('root', {'aaa': 't:0'}, map={ + 'semantics': LWW, + 'entries': {}, + 'clearTimeserial': '05', + }, create_op={'mapCreate': { + 'semantics': LWW, + 'entries': { + 'old_key': {'data': {'string': 'old'}, 'timeserial': '03'}, + 'new_key': {'data': {'string': 'new'}, 'timeserial': '07'}, + }, + }}), + ])) + + assert realtime_object._sync_state == ObjectsSyncState.SYNCED + assert 'old_key' not in pool['root'].data + assert pool['root'].data['new_key'].data == ObjectData(string='new') + + +# UTS: objects/unit/RTO5c10/sync-rebuilds-parent-refs-0 +async def test_rto5c10_sync_rebuilds_parent_refs(): + realtime_object, pool = _driven_pool() + _process_attached(realtime_object, objects_attached_message('test', 'sync1:cursor', flags=HAS_OBJECTS)) + + _process_object_sync(realtime_object, build_object_sync_message('test', 'sync1:', [ + build_object_state('root', {'aaa': 't:0'}, map={ + 'semantics': LWW, + 'entries': { + 'score': {'data': {'objectId': 'counter:score@1000'}, 'timeserial': 't:0'}, + 'profile': {'data': {'objectId': 'map:profile@1000'}, 'timeserial': 't:0'}, + 'name': {'data': {'string': 'Alice'}, 'timeserial': 't:0'}, + }, + }, create_op={'mapCreate': {'semantics': LWW, 'entries': {}}}), + build_object_state('counter:score@1000', {'aaa': 't:0'}, counter={'count': 0}, + create_op={'counterCreate': {'count': 100}}), + build_object_state('map:profile@1000', {'aaa': 't:0'}, map={ + 'semantics': LWW, + 'entries': { + 'nested_counter': {'data': {'objectId': 'counter:nested@1000'}, 'timeserial': 't:0'}, + }, + }, create_op={'mapCreate': {'semantics': LWW, 'entries': {}}}), + build_object_state('counter:nested@1000', {'aaa': 't:0'}, counter={'count': 0}, + create_op={'counterCreate': {'count': 5}}), + ])) + + # root is not referenced by any parent + assert pool['root'].parent_references == {} + # counter:score@1000 is referenced by root at key 'score' + assert pool['counter:score@1000'].parent_references == {'root': {'score'}} + # map:profile@1000 is referenced by root at key 'profile' + assert pool['map:profile@1000'].parent_references == {'root': {'profile'}} + # counter:nested@1000 is referenced by map:profile@1000 at key 'nested_counter' + assert pool['counter:nested@1000'].parent_references == {'map:profile@1000': {'nested_counter'}} + # The primitive-valued entry 'name' appears in no parent_references, which the + # equalities above already establish + + +# UTS: objects/unit/RTO5c10/resync-rebuilds-parent-refs-0 +async def test_rto5c10_resync_rebuilds_parent_refs(): + realtime_object, pool = _driven_pool() + _process_attached(realtime_object, objects_attached_message('test', 'sync1:cursor', flags=HAS_OBJECTS)) + + # First sync: counter:abc@1000 is a child of root + _process_object_sync(realtime_object, build_object_sync_message('test', 'sync1:', [ + build_object_state('root', {'aaa': 't:0'}, map={ + 'semantics': LWW, + 'entries': { + 'counter_key': {'data': {'objectId': 'counter:abc@1000'}, 'timeserial': 't:0'}, + }, + }, create_op={'mapCreate': {'semantics': LWW, 'entries': {}}}), + build_object_state('counter:abc@1000', {'aaa': 't:0'}, counter={'count': 0}, + create_op={'counterCreate': {'count': 10}}), + ])) + assert pool['counter:abc@1000'].parent_references == {'root': {'counter_key'}} + + # Second sync: counter:abc@1000 is a child of map:wrapper@1000, not of root + _process_attached(realtime_object, objects_attached_message('test', 'sync2:cursor', flags=HAS_OBJECTS)) + _process_object_sync(realtime_object, build_object_sync_message('test', 'sync2:', [ + build_object_state('root', {'aaa': 't:1'}, map={ + 'semantics': LWW, + 'entries': { + 'wrapper': {'data': {'objectId': 'map:wrapper@1000'}, 'timeserial': 't:1'}, + }, + }, create_op={'mapCreate': {'semantics': LWW, 'entries': {}}}), + build_object_state('map:wrapper@1000', {'aaa': 't:1'}, map={ + 'semantics': LWW, + 'entries': { + 'moved_counter': {'data': {'objectId': 'counter:abc@1000'}, 'timeserial': 't:1'}, + }, + }, create_op={'mapCreate': {'semantics': LWW, 'entries': {}}}), + build_object_state('counter:abc@1000', {'aaa': 't:1'}, counter={'count': 0}, + create_op={'counterCreate': {'count': 20}}), + ])) + + assert realtime_object._sync_state == ObjectsSyncState.SYNCED + # root is not referenced by any parent + assert pool['root'].parent_references == {} + # map:wrapper@1000 is a child of root at key 'wrapper' + assert pool['map:wrapper@1000'].parent_references == {'root': {'wrapper'}} + # counter:abc@1000 is a child of map:wrapper@1000, and no longer of root + assert pool['counter:abc@1000'].parent_references == {'map:wrapper@1000': {'moved_counter'}} + + +# UTS: objects/unit/RTO5c10/empty-sync-parent-refs-0 +async def test_rto5c10_empty_sync_parent_refs(): + realtime_object, pool = _driven_pool() + + # A normal sync first, to populate parent_references + _process_attached(realtime_object, objects_attached_message('test', 'sync1:cursor', flags=HAS_OBJECTS)) + _process_object_sync(realtime_object, build_object_sync_message('test', 'sync1:', [ + build_object_state('root', {'aaa': 't:0'}, map={ + 'semantics': LWW, + 'entries': { + 'child': {'data': {'objectId': 'counter:child@1000'}, 'timeserial': 't:0'}, + }, + }, create_op={'mapCreate': {'semantics': LWW, 'entries': {}}}), + build_object_state('counter:child@1000', {'aaa': 't:0'}, counter={'count': 0}, + create_op={'counterCreate': {'count': 1}}), + ])) + assert pool['counter:child@1000'].parent_references == {'root': {'child'}} + + # An empty sync: ATTACHED without HAS_OBJECTS + _process_attached(realtime_object, {'action': ATTACHED, 'channel': 'test', 'flags': 0}) + + assert realtime_object._sync_state == ObjectsSyncState.SYNCED + # counter:child@1000 was removed from the pool (RTO4b1) + assert 'counter:child@1000' not in pool + # root remains, with empty data and empty parent_references + assert 'root' in pool + assert pool['root'].data == {} + assert pool['root'].parent_references == {} diff --git a/test/uts/objects/unit/parent_references_test.py b/test/uts/objects/unit/parent_references_test.py new file mode 100644 index 00000000..20697c74 --- /dev/null +++ b/test/uts/objects/unit/parent_references_test.py @@ -0,0 +1,433 @@ +"""Derived from uts/objects/unit/parent_references.md in ably/specification. + +Spec points: RTLO3f, RTLO3f2, RTLO4g, RTLO4g1, RTLO4g2, RTLO4h, RTLO4h1, RTLO4h2, RTLO4h3, +RTLO4f, RTLO4f1, RTLO4f2, RTLO4f3, RTLO4f4, RTO5c10, RTO5c10a, RTO5c10b + +A pure unit specification: nothing is mocked and nothing is connected. Objects are built on +their own and then put in a pool with `pool[object_id] = obj`, which adopts them, so +`get_full_paths()` resolves their parents' ids through that pool. A path is a list of keys, +so the specification's `paths CONTAINS ["a", "x"]` is `['a', 'x'] in paths`. + +The three RTO5c10 cases drive a sync, which the specification does through +`ObjectsPool.processAttached` and `processObjectSync`. The sync state machine is +`RealtimeObject`'s (shape deviation S-2), so those cases work on the pool a standalone +`RealtimeObject()` holds and drive the `RealtimeObject`, through the helpers below; they are +coroutines only so that an implementation is free to touch the event loop. +""" + +from ably.pubsub.objects.enums import ObjectsSyncState +from ably.pubsub.objects.livecounter import InternalLiveCounter +from ably.pubsub.objects.livemap import InternalLiveMap +from ably.pubsub.objects.objectmessage import ObjectsMapSemantics +from ably.pubsub.objects.objectspool import ObjectsPool +from ably.pubsub.objects.realtimeobject import RealtimeObject +from test.uts.objects.helpers.standard_test_pool import ( + HAS_OBJECTS, + LWW, + build_object_state, + build_object_sync_message, + object_messages, + objects_attached_message, +) + + +def _driven_pool(): + """The specification's `pool = ObjectsPool()`, in a test that then drives the sync state machine. + + S-2: the state machine is `RealtimeObject`'s, so the pool is the one a standalone + `RealtimeObject` holds. Returns `(realtime_object, pool)`. + """ + realtime_object = RealtimeObject() + return realtime_object, realtime_object._objects_pool + + +def _process_attached(realtime_object, attached): + """The specification's `pool.processAttached(attached)` (S-2). + + `RealtimeObject` is given the ATTACHED's HAS_OBJECTS flag and nothing else (RTO4). + """ + realtime_object._on_attached(has_objects=bool(attached.get('flags', 0) & HAS_OBJECTS)) + + +def _process_object_sync(realtime_object, protocol_message): + """The specification's `pool.processObjectSync(protocol_message)` (S-2).""" + realtime_object._handle_object_sync_messages( + object_messages(protocol_message), protocol_message.get('channelSerial')) + + +# UTS: objects/unit/RTLO3f2/init-empty-counter-0 +def test_rtlo3f2_init_empty_counter(): + counter = InternalLiveCounter('counter:abc@1000') + + assert counter.parent_references == {} + + +# UTS: objects/unit/RTLO3f2/init-empty-map-0 +def test_rtlo3f2_init_empty_map(): + live_map = InternalLiveMap('map:abc@1000', ObjectsMapSemantics.LWW) + + assert live_map.parent_references == {} + + +# UTS: objects/unit/RTLO4g2/first-reference-new-entry-0 +def test_rtlo4g2_first_reference_new_entry(): + child = InternalLiveCounter('counter:child@1000') + parent = InternalLiveMap('map:parent@1000', ObjectsMapSemantics.LWW) + + child.add_parent_reference(parent, 'score') + + assert 'map:parent@1000' in child.parent_references + assert child.parent_references['map:parent@1000'] == {'score'} + + +# UTS: objects/unit/RTLO4g1/second-key-same-parent-0 +def test_rtlo4g1_second_key_same_parent(): + child = InternalLiveCounter('counter:child@1000') + parent = InternalLiveMap('map:parent@1000', ObjectsMapSemantics.LWW) + child.parent_references = {'map:parent@1000': {'score'}} + + child.add_parent_reference(parent, 'points') + + assert child.parent_references['map:parent@1000'] == {'score', 'points'} + + +# UTS: objects/unit/RTLO4g/different-parent-separate-entry-0 +def test_rtlo4g_different_parent_separate_entry(): + child = InternalLiveCounter('counter:child@1000') + parent_a = InternalLiveMap('map:a@1000', ObjectsMapSemantics.LWW) + parent_b = InternalLiveMap('map:b@1000', ObjectsMapSemantics.LWW) + + child.add_parent_reference(parent_a, 'x') + child.add_parent_reference(parent_b, 'y') + + assert child.parent_references['map:a@1000'] == {'x'} + assert child.parent_references['map:b@1000'] == {'y'} + + +# UTS: objects/unit/RTLO4g/multiple-parents-multiple-keys-0 +def test_rtlo4g_multiple_parents_multiple_keys(): + child = InternalLiveCounter('counter:child@1000') + parent_a = InternalLiveMap('map:a@1000', ObjectsMapSemantics.LWW) + parent_b = InternalLiveMap('map:b@1000', ObjectsMapSemantics.LWW) + + child.add_parent_reference(parent_a, 'x') + child.add_parent_reference(parent_a, 'y') + child.add_parent_reference(parent_b, 'p') + child.add_parent_reference(parent_b, 'q') + + assert child.parent_references['map:a@1000'] == {'x', 'y'} + assert child.parent_references['map:b@1000'] == {'p', 'q'} + + +# UTS: objects/unit/RTLO4h1/nonexistent-parent-noop-0 +def test_rtlo4h1_nonexistent_parent_noop(): + child = InternalLiveCounter('counter:child@1000') + parent = InternalLiveMap('map:parent@1000', ObjectsMapSemantics.LWW) + + child.remove_parent_reference(parent, 'score') + + assert child.parent_references == {} + + +# UTS: objects/unit/RTLO4h2/remove-key-leaves-others-0 +def test_rtlo4h2_remove_key_leaves_others(): + child = InternalLiveCounter('counter:child@1000') + parent = InternalLiveMap('map:parent@1000', ObjectsMapSemantics.LWW) + child.parent_references = {'map:parent@1000': {'score', 'points'}} + + child.remove_parent_reference(parent, 'score') + + assert child.parent_references['map:parent@1000'] == {'points'} + + +# UTS: objects/unit/RTLO4h3/remove-last-key-removes-entry-0 +def test_rtlo4h3_remove_last_key_removes_entry(): + child = InternalLiveCounter('counter:child@1000') + parent = InternalLiveMap('map:parent@1000', ObjectsMapSemantics.LWW) + child.parent_references = {'map:parent@1000': {'score'}} + + child.remove_parent_reference(parent, 'score') + + assert 'map:parent@1000' not in child.parent_references + assert child.parent_references == {} + + +# UTS: objects/unit/RTLO4h/remove-nonexistent-key-0 +def test_rtlo4h_remove_nonexistent_key(): + child = InternalLiveCounter('counter:child@1000') + parent = InternalLiveMap('map:parent@1000', ObjectsMapSemantics.LWW) + child.parent_references = {'map:parent@1000': {'score'}} + + child.remove_parent_reference(parent, 'nonexistent') + + assert child.parent_references['map:parent@1000'] == {'score'} + + +# UTS: objects/unit/RTLO4f2/root-returns-empty-path-0 +def test_rtlo4f2_root_returns_empty_path(): + pool = ObjectsPool() + root = pool['root'] + + paths = root.get_full_paths() + assert len(paths) == 1 + assert [] in paths + + +# UTS: objects/unit/RTLO4f/direct-child-single-path-0 +def test_rtlo4f_direct_child_single_path(): + pool = ObjectsPool() + counter = InternalLiveCounter('counter:score@1000') + pool['counter:score@1000'] = counter + + root = pool['root'] + counter.add_parent_reference(root, 'score') + + paths = counter.get_full_paths() + assert len(paths) == 1 + assert ['score'] in paths + + +# UTS: objects/unit/RTLO4f/deep-nesting-0 +def test_rtlo4f_deep_nesting(): + # root --'profile'--> map:profile --'prefs'--> map:prefs --'theme_counter'--> counter:theme + pool = ObjectsPool() + root = pool['root'] + + profile = InternalLiveMap('map:profile@1000', ObjectsMapSemantics.LWW) + pool['map:profile@1000'] = profile + profile.add_parent_reference(root, 'profile') + + prefs = InternalLiveMap('map:prefs@1000', ObjectsMapSemantics.LWW) + pool['map:prefs@1000'] = prefs + prefs.add_parent_reference(profile, 'prefs') + + theme_counter = InternalLiveCounter('counter:theme@1000') + pool['counter:theme@1000'] = theme_counter + theme_counter.add_parent_reference(prefs, 'theme_counter') + + paths = theme_counter.get_full_paths() + assert len(paths) == 1 + assert ['profile', 'prefs', 'theme_counter'] in paths + + +# UTS: objects/unit/RTLO4f/diamond-graph-0 +def test_rtlo4f_diamond_graph(): + # root --'a'--> map:A --'x'--> counter:leaf, and root --'b'--> map:B --'y'--> counter:leaf + pool = ObjectsPool() + root = pool['root'] + + map_a = InternalLiveMap('map:a@1000', ObjectsMapSemantics.LWW) + pool['map:a@1000'] = map_a + map_a.add_parent_reference(root, 'a') + + map_b = InternalLiveMap('map:b@1000', ObjectsMapSemantics.LWW) + pool['map:b@1000'] = map_b + map_b.add_parent_reference(root, 'b') + + leaf = InternalLiveCounter('counter:leaf@1000') + pool['counter:leaf@1000'] = leaf + leaf.add_parent_reference(map_a, 'x') + leaf.add_parent_reference(map_b, 'y') + + paths = leaf.get_full_paths() + assert len(paths) == 2 + assert ['a', 'x'] in paths + assert ['b', 'y'] in paths + + +# UTS: objects/unit/RTLO4f/single-parent-multiple-keys-0 +def test_rtlo4f_single_parent_multiple_keys(): + pool = ObjectsPool() + root = pool['root'] + + child = InternalLiveCounter('counter:child@1000') + pool['counter:child@1000'] = child + child.add_parent_reference(root, 'primary') + child.add_parent_reference(root, 'alias') + + paths = child.get_full_paths() + assert len(paths) == 2 + assert ['primary'] in paths + assert ['alias'] in paths + + +# UTS: objects/unit/RTLO4f/orphan-returns-empty-0 +def test_rtlo4f_orphan_returns_empty(): + pool = ObjectsPool() + + orphan = InternalLiveCounter('counter:orphan@1000') + pool['counter:orphan@1000'] = orphan + + paths = orphan.get_full_paths() + assert len(paths) == 0 + + +# UTS: objects/unit/RTLO4f/cycle-suppression-0 +def test_rtlo4f_cycle_suppression(): + # root --'a'--> map:A --'b'--> map:B --'a'--> map:A, a cycle. The only simple path to + # map:B is ['a', 'b']. + pool = ObjectsPool() + root = pool['root'] + + map_a = InternalLiveMap('map:a@1000', ObjectsMapSemantics.LWW) + pool['map:a@1000'] = map_a + map_a.add_parent_reference(root, 'a') + + map_b = InternalLiveMap('map:b@1000', ObjectsMapSemantics.LWW) + pool['map:b@1000'] = map_b + map_b.add_parent_reference(map_a, 'b') + + # The cycle: map:A also has map:B as a parent + map_a.add_parent_reference(map_b, 'a') + + paths_b = map_b.get_full_paths() + assert len(paths_b) == 1 + assert ['a', 'b'] in paths_b + + paths_a = map_a.get_full_paths() + assert len(paths_a) == 1 + assert ['a'] in paths_a + + +# UTS: objects/unit/RTLO4f/complex-diamond-deep-0 +def test_rtlo4f_complex_diamond_deep(): + # root --'left'--> map:L --'mid'--> map:M --'target'--> counter:T, and + # root --'right'--> map:R --'target'--> counter:T + pool = ObjectsPool() + root = pool['root'] + + map_l = InternalLiveMap('map:l@1000', ObjectsMapSemantics.LWW) + pool['map:l@1000'] = map_l + map_l.add_parent_reference(root, 'left') + + map_r = InternalLiveMap('map:r@1000', ObjectsMapSemantics.LWW) + pool['map:r@1000'] = map_r + map_r.add_parent_reference(root, 'right') + + map_m = InternalLiveMap('map:m@1000', ObjectsMapSemantics.LWW) + pool['map:m@1000'] = map_m + map_m.add_parent_reference(map_l, 'mid') + + target = InternalLiveCounter('counter:t@1000') + pool['counter:t@1000'] = target + target.add_parent_reference(map_m, 'target') + target.add_parent_reference(map_r, 'target') + + paths = target.get_full_paths() + assert len(paths) == 2 + assert ['left', 'mid', 'target'] in paths + assert ['right', 'target'] in paths + + +# UTS: objects/unit/RTO5c10/rebuild-from-sync-0 +async def test_rto5c10_rebuild_from_sync(): + realtime_object, pool = _driven_pool() + _process_attached(realtime_object, objects_attached_message('test', 'sync1:cursor', flags=HAS_OBJECTS)) + + _process_object_sync(realtime_object, build_object_sync_message('test', 'sync1:', [ + build_object_state('root', {'aaa': 't:0'}, map={ + 'semantics': LWW, + 'entries': { + 'score': {'data': {'objectId': 'counter:score@1000'}, 'timeserial': 't:0'}, + 'profile': {'data': {'objectId': 'map:profile@1000'}, 'timeserial': 't:0'}, + }, + }, create_op={'mapCreate': {'semantics': LWW, 'entries': {}}}), + build_object_state('counter:score@1000', {'aaa': 't:0'}, counter={'count': 0}, + create_op={'counterCreate': {'count': 100}}), + build_object_state('map:profile@1000', {'aaa': 't:0'}, map={ + 'semantics': LWW, + 'entries': { + 'nested': {'data': {'objectId': 'counter:nested@1000'}, 'timeserial': 't:0'}, + }, + }, create_op={'mapCreate': {'semantics': LWW, 'entries': {}}}), + build_object_state('counter:nested@1000', {'aaa': 't:0'}, counter={'count': 0}, + create_op={'counterCreate': {'count': 5}}), + ])) + + assert realtime_object._sync_state == ObjectsSyncState.SYNCED + + # counter:score@1000 is referenced by root at key 'score' + score = pool['counter:score@1000'] + assert score.parent_references['root'] == {'score'} + + # map:profile@1000 is referenced by root at key 'profile' + profile = pool['map:profile@1000'] + assert profile.parent_references['root'] == {'profile'} + + # counter:nested@1000 is referenced by map:profile@1000 at key 'nested' + nested = pool['counter:nested@1000'] + assert nested.parent_references['map:profile@1000'] == {'nested'} + + # root has no parent references + assert pool['root'].parent_references == {} + + # get_full_paths resolves through the rebuilt references + assert ['score'] in score.get_full_paths() + assert ['profile', 'nested'] in nested.get_full_paths() + + +# UTS: objects/unit/RTO5c10a/rebuild-clears-stale-refs-0 +async def test_rto5c10a_rebuild_clears_stale_refs(): + realtime_object, pool = _driven_pool() + + # First sync: root --'score'--> counter:abc@1000 + _process_attached(realtime_object, objects_attached_message('test', 'sync1:cursor', flags=HAS_OBJECTS)) + _process_object_sync(realtime_object, build_object_sync_message('test', 'sync1:', [ + build_object_state('root', {'aaa': 't:0'}, map={ + 'semantics': LWW, + 'entries': { + 'score': {'data': {'objectId': 'counter:abc@1000'}, 'timeserial': 't:0'}, + }, + }, create_op={'mapCreate': {'semantics': LWW, 'entries': {}}}), + build_object_state('counter:abc@1000', {'aaa': 't:0'}, counter={'count': 0}, + create_op={'counterCreate': {'count': 10}}), + ])) + assert pool['counter:abc@1000'].parent_references['root'] == {'score'} + + # Second sync: root --'points'--> counter:abc@1000, the key changed from 'score' to 'points' + _process_attached(realtime_object, objects_attached_message('test', 'sync2:cursor', flags=HAS_OBJECTS)) + _process_object_sync(realtime_object, build_object_sync_message('test', 'sync2:', [ + build_object_state('root', {'aaa': 't:1'}, map={ + 'semantics': LWW, + 'entries': { + 'points': {'data': {'objectId': 'counter:abc@1000'}, 'timeserial': 't:1'}, + }, + }, create_op={'mapCreate': {'semantics': LWW, 'entries': {}}}), + build_object_state('counter:abc@1000', {'aaa': 't:1'}, counter={'count': 0}, + create_op={'counterCreate': {'count': 20}}), + ])) + + counter = pool['counter:abc@1000'] + + # The 'score' reference is gone, replaced by 'points' + assert counter.parent_references['root'] == {'points'} + assert ['points'] in counter.get_full_paths() + + paths = counter.get_full_paths() + assert len(paths) == 1 + + +# UTS: objects/unit/RTO5c10/unreferenced-empty-refs-0 +async def test_rto5c10_unreferenced_empty_refs(): + realtime_object, pool = _driven_pool() + _process_attached(realtime_object, objects_attached_message('test', 'sync1:cursor', flags=HAS_OBJECTS)) + + _process_object_sync(realtime_object, build_object_sync_message('test', 'sync1:', [ + build_object_state('root', {'aaa': 't:0'}, map={ + 'semantics': LWW, + 'entries': { + 'name': {'data': {'string': 'Alice'}, 'timeserial': 't:0'}, + }, + }, create_op={'mapCreate': {'semantics': LWW, 'entries': {}}}), + build_object_state('counter:orphan@1000', {'aaa': 't:0'}, counter={'count': 0}, + create_op={'counterCreate': {'count': 42}}), + ])) + + assert realtime_object._sync_state == ObjectsSyncState.SYNCED + + # The counter is in the pool, but no InternalLiveMap entry refers to it + orphan = pool['counter:orphan@1000'] + assert orphan.parent_references == {} + + # get_full_paths returns no key-path for an unreferenced object + assert len(orphan.get_full_paths()) == 0 diff --git a/test/uts/objects/unit/path_object_mutations_test.py b/test/uts/objects/unit/path_object_mutations_test.py new file mode 100644 index 00000000..0f62f201 --- /dev/null +++ b/test/uts/objects/unit/path_object_mutations_test.py @@ -0,0 +1,152 @@ +"""Derived from uts/objects/unit/path_object_mutations.md in ably/specification. + +Spec points: RTPO3c2, RTPO15, RTPO15a2, RTPO15b, RTPO15c, RTPO15d, RTPO15e, RTPO16, RTPO16b, RTPO16c, +RTPO16d, RTPO16e, RTPO17, RTPO17a1, RTPO17b, RTPO17c, RTPO17d, RTPO17e, RTPO18, RTPO18a1, RTPO18b, +RTPO18c, RTPO18d, RTPO18e, RTTS5d2 + +The specification writes through the untyped `PathObject`. ably-python partitions it (LODR-061, +RTTS3-RTTS6): `set` and `remove` are on `LiveMapPathObject` and `increment` and `decrement` on +`LiveCounterPathObject`, reached through the unchecked view helpers. `root` is already a +`LiveMapPathObject`, so `root.set(...)` needs no view; `root.get('score').increment(25)` is +`root.get('score').as_live_counter().increment(25)`. A view never raises for the type at the path; +the write through it raises 92007 for a path resolving to the wrong type and 92005 for one that does +not resolve (RTTS5d2), which is what the specification's failure cases assert. + +Every write is awaited against the standard mock, which ACKs each OBJECT message, so the operation +has been applied locally (RTO20) by the time the write returns and the read that follows needs no +wait. Reads go through the view matching the type written: `as_primitive()` for map entries, +`as_live_counter()` for the counter. +""" + +import pytest + +from ably.pubsub.util.exceptions import AblyException +from test.uts.objects.helpers.standard_test_pool import setup_synced_channel + + +# UTS: objects/unit/RTPO15/set-delegates-to-map-0 +async def test_rtpo15_set_delegates_to_map(): + client, channel, root, mock_ws = await setup_synced_channel('test') + + await root.set('name', 'Bob') + + assert root.get('name').as_primitive().value() == 'Bob' + + +# UTS: objects/unit/RTPO15/set-nested-path-0 +async def test_rtpo15_set_nested_path(): + client, channel, root, mock_ws = await setup_synced_channel('test') + + await root.get('profile').as_live_map().set('email', 'bob@example.com') + + assert root.get('profile').get('email').as_primitive().value() == 'bob@example.com' + + +# UTS: objects/unit/RTPO15d/set-non-map-throws-0 +async def test_rtpo15d_set_non_map_throws(): + client, channel, root, mock_ws = await setup_synced_channel('test') + + # The map view is unchecked (RTTS5d); `set` through it finds a counter at the path + with pytest.raises(AblyException) as excinfo: + await root.get('score').as_live_map().set('key', 'value') + + assert excinfo.value.code == 92007 + + +# UTS: objects/unit/RTPO16/remove-delegates-to-map-0 +async def test_rtpo16_remove_delegates_to_map(): + client, channel, root, mock_ws = await setup_synced_channel('test') + + await root.remove('name') + + assert root.get('name').as_primitive().value() is None + + +# UTS: objects/unit/RTPO16d/remove-non-map-throws-0 +async def test_rtpo16d_remove_non_map_throws(): + client, channel, root, mock_ws = await setup_synced_channel('test') + + with pytest.raises(AblyException) as excinfo: + await root.get('score').as_live_map().remove('key') + + assert excinfo.value.code == 92007 + + +# UTS: objects/unit/RTPO17/increment-delegates-to-counter-0 +async def test_rtpo17_increment_delegates_to_counter(): + client, channel, root, mock_ws = await setup_synced_channel('test') + + await root.get('score').as_live_counter().increment(25) + + assert root.get('score').as_live_counter().value() == 125 + + +# UTS: objects/unit/RTPO17/increment-default-amount-0 +async def test_rtpo17_increment_default_amount(): + client, channel, root, mock_ws = await setup_synced_channel('test') + + await root.get('score').as_live_counter().increment() + + assert root.get('score').as_live_counter().value() == 101 + + +# UTS: objects/unit/RTPO17d/increment-non-counter-throws-0 +async def test_rtpo17d_increment_non_counter_throws(): + client, channel, root, mock_ws = await setup_synced_channel('test') + + # `root` is a `LiveMapPathObject` and has no `increment`; the counter view over the root + # path is the translation, and the root map is the wrong type for it + with pytest.raises(AblyException) as excinfo: + await root.as_live_counter().increment(5) + + assert excinfo.value.code == 92007 + + +# UTS: objects/unit/RTPO18/decrement-delegates-to-counter-0 +async def test_rtpo18_decrement_delegates_to_counter(): + client, channel, root, mock_ws = await setup_synced_channel('test') + + await root.get('score').as_live_counter().decrement(10) + + assert root.get('score').as_live_counter().value() == 90 + + +# UTS: objects/unit/RTPO18/decrement-default-amount-0 +async def test_rtpo18_decrement_default_amount(): + client, channel, root, mock_ws = await setup_synced_channel('test') + + await root.get('score').as_live_counter().decrement() + + assert root.get('score').as_live_counter().value() == 99 + + +# UTS: objects/unit/RTPO18d/decrement-non-counter-throws-0 +async def test_rtpo18d_decrement_non_counter_throws(): + client, channel, root, mock_ws = await setup_synced_channel('test') + + with pytest.raises(AblyException) as excinfo: + await root.as_live_counter().decrement(5) + + assert excinfo.value.code == 92007 + + +# UTS: objects/unit/RTPO3c2/set-unresolvable-throws-0 +async def test_rtpo3c2_set_unresolvable_throws(): + client, channel, root, mock_ws = await setup_synced_channel('test') + + with pytest.raises(AblyException) as excinfo: + await root.get('nonexistent').get('deep').as_live_map().set('key', 'value') + + assert excinfo.value.code == 92005 + assert excinfo.value.status_code == 400 + + +# UTS: objects/unit/RTPO3c2/increment-unresolvable-throws-0 +async def test_rtpo3c2_increment_unresolvable_throws(): + client, channel, root, mock_ws = await setup_synced_channel('test') + + with pytest.raises(AblyException) as excinfo: + await root.get('nonexistent').as_live_counter().increment(5) + + assert excinfo.value.code == 92005 + assert excinfo.value.status_code == 400 diff --git a/test/uts/objects/unit/path_object_subscribe_test.py b/test/uts/objects/unit/path_object_subscribe_test.py new file mode 100644 index 00000000..06e3a62c --- /dev/null +++ b/test/uts/objects/unit/path_object_subscribe_test.py @@ -0,0 +1,553 @@ +"""Derived from uts/objects/unit/path_object_subscribe.md in ably/specification. + +Spec points: RTPO19, RTPO19b, RTPO19c1, RTPO19c1a, RTPO19d, RTPO19e1, RTPO19e2, RTPO19f, RTPO19g, +RTO24b1, RTO24b2, RTO24b2a, RTO24b2a1, RTO24b2a2, RTO24b2b, RTO24b2b2, RTO24b2c, RTO24c1, RTO24c2a, +RTO24c2b, RTO24c2c, RTO24c2d, RTO25b, RTLM24 + +`subscribe` is on the base `PathObject` (LODR-061, RTTS3d) and is synchronous; the specification's +`{ depth: n }` option is the keyword argument `depth=n`. The listener receives a +`PathObjectSubscriptionEvent` whose `object` is a base `PathObject`, so the counter value RTPO19e1 +reads is `event.object.as_live_counter().value()`, and whose `message` is the public `ObjectMessage` +with snake_case fields (`site_code`, `operation.object_id`, `operation.counter_inc`) and the action as +an `ObjectOperationAction`. + +Each stimulus is an inbound OBJECT or OBJECT_SYNC message, which the client applies on the +transport's read task, so every positive expectation is reached with `poll_until`. Where the +specification then asserts an exact count, or that a listener stayed silent, the test first awaits +the delivery of a control (a further event, or an unlimited-depth listener on the same dispatch) and +settles, so that a late or spurious callback would already have run. Where the specification +subscribes straight after sending a seeding message, the test waits for the seed to be applied +first, since a seed applied after the subscription would itself be delivered to it. +""" + +import pytest + +from ably.pubsub.objects.objectmessage import ObjectOperationAction +from ably.pubsub.objects.pathobject import PathObject +from ably.pubsub.objects.subscription import Subscription +from ably.pubsub.types.channelstate import ChannelState +from ably.pubsub.util.exceptions import AblyException +from test.uts.helpers.client import await_channel_state, poll_until +from test.uts.helpers.clock import settle +from test.uts.objects.helpers.standard_test_pool import ( + build_counter_inc, + build_map_clear, + build_map_set, + build_object_message, + build_object_state, + build_object_sync_message, + objects_channel_options, + objects_client, + remote_serial, + setup_synced_channel, + standard_mock_websocket, +) + + +# UTS: objects/unit/RTPO19/subscribe-receives-events-0 +async def test_rtpo19_subscribe_receives_events(): + client, channel, root, mock_ws = await setup_synced_channel('test') + events = [] + sub = root.get('score').subscribe(events.append) + + mock_ws.send_to_client(build_object_message('test', [ + build_counter_inc('counter:score@1000', 7, '99', 'remote'), + ])) + await poll_until(lambda: len(events) >= 1, description='the COUNTER_INC event') + await settle() + + assert isinstance(sub, Subscription) + assert len(events) == 1 + assert isinstance(events[0].object, PathObject) + assert events[0].object.path() == 'score' + assert events[0].message is not None + assert events[0].message.serial == '99' + assert events[0].message.site_code == 'remote' + assert events[0].message.operation is not None + assert events[0].message.operation.action == ObjectOperationAction.COUNTER_INC + assert events[0].message.channel == 'test' + + +# UTS: objects/unit/RTPO19b/subscribe-precondition-detached-0 +async def test_rtpo19b_subscribe_precondition_detached(): + # The specification's hand-written mock is the standard one: CONNECTED with siteCode and + # objectsGCGracePeriod, ATTACH answered with ATTACHED and the standard pool's OBJECT_SYNC, + # DETACH answered with DETACHED + mock_ws = standard_mock_websocket() + client = objects_client(mock_ws) + channel = client.channels.get('test', objects_channel_options()) + root = await channel.object.get() + + await channel.detach() + await await_channel_state(channel, ChannelState.DETACHED) + + with pytest.raises(AblyException) as excinfo: + root.subscribe(lambda event: None) + + assert excinfo.value.code == 90001 + assert excinfo.value.status_code == 400 + + +# UTS: objects/unit/RTPO19c1a/subscribe-non-positive-depth-throws-0 +async def test_rtpo19c1a_subscribe_non_positive_depth_throws(): + client, channel, root, mock_ws = await setup_synced_channel('test') + + with pytest.raises(AblyException) as excinfo: + root.subscribe(lambda event: None, depth=0) + + assert excinfo.value.code == 40003 + + +# UTS: objects/unit/RTPO19c1a/subscribe-negative-depth-throws-0 +async def test_rtpo19c1a_subscribe_negative_depth_throws(): + client, channel, root, mock_ws = await setup_synced_channel('test') + + with pytest.raises(AblyException) as excinfo: + root.subscribe(lambda event: None, depth=-1) + + assert excinfo.value.code == 40003 + + +# UTS: objects/unit/RTPO19c1/subscribe-depth-1-self-only-0 +async def test_rtpo19c1_subscribe_depth_1_self_only(): + client, channel, root, mock_ws = await setup_synced_channel('test') + events = [] + root.subscribe(events.append, depth=1) + # Quiescence control: an unlimited-depth root listener covers the out-of-scope child path + control = [] + root.subscribe(control.append) + + mock_ws.send_to_client(build_object_message('test', [ + build_map_set('root', 'name', {'string': 'Bob'}, remote_serial(0), 'remote'), + ])) + await poll_until(lambda: len(events) >= 1, description='the self event') + await settle() + + control_before = len(control) + mock_ws.send_to_client(build_object_message('test', [ + build_counter_inc('counter:score@1000', 7, '100', 'remote'), + ])) + await poll_until(lambda: len(control) > control_before, description='the control to see the child event') + await settle() + + assert len(events) == 1 + + +# UTS: objects/unit/RTPO19c1/subscribe-depth-2-children-0 +async def test_rtpo19c1_subscribe_depth_2_children(): + client, channel, root, mock_ws = await setup_synced_channel('test') + events = [] + root.subscribe(events.append, depth=2) + # Quiescence control: an unlimited-depth root listener covers the out-of-scope grandchild path + control = [] + root.subscribe(control.append) + + # Self event (root map update): the candidate [] is covered at depth 2 + mock_ws.send_to_client(build_object_message('test', [ + build_map_set('root', 'name', {'string': 'Bob'}, remote_serial(0), 'remote'), + ])) + await poll_until(lambda: len(events) >= 1, description='the self event') + + # Child event (the counter at ["score"]): relative depth 1 - 0 + 1 = 2 <= 2, covered + mock_ws.send_to_client(build_object_message('test', [ + build_counter_inc('counter:score@1000', 7, '100', 'remote'), + ])) + await poll_until(lambda: len(events) >= 2, description='the child event') + await settle() + + # Grandchild event (the counter at ["profile", "nested_counter"]): relative depth + # 2 - 0 + 1 = 3 > 2, not covered. A COUNTER_INC yields this single candidate only. + control_before = len(control) + mock_ws.send_to_client(build_object_message('test', [ + build_counter_inc('counter:nested@1000', 1, '101', 'remote'), + ])) + await poll_until(lambda: len(control) > control_before, description='the control to see the grandchild event') + await settle() + + assert len(events) == 2 + + +# UTS: objects/unit/RTPO19c1/subscribe-unlimited-depth-0 +async def test_rtpo19c1_subscribe_unlimited_depth(): + client, channel, root, mock_ws = await setup_synced_channel('test') + events = [] + root.subscribe(events.append) + + mock_ws.send_to_client(build_object_message('test', [ + build_map_set('root', 'name', {'string': 'Bob'}, remote_serial(0), 'remote'), + ])) + await poll_until(lambda: len(events) >= 1, description='the root event') + + mock_ws.send_to_client(build_object_message('test', [ + build_counter_inc('counter:score@1000', 7, '100', 'remote'), + ])) + await poll_until(lambda: len(events) >= 2, description='the child event') + + mock_ws.send_to_client(build_object_message('test', [ + build_map_set('map:prefs@1000', 'theme', {'string': 'light'}, remote_serial(1), 'remote'), + ])) + await poll_until(lambda: len(events) >= 3, description='the descendant event') + + assert len(events) >= 3 + + +# UTS: objects/unit/RTPO19d/subscribe-returns-subscription-0 +async def test_rtpo19d_subscribe_returns_subscription(): + client, channel, root, mock_ws = await setup_synced_channel('test') + events = [] + sub = root.get('score').subscribe(events.append) + # Quiescence control: a separate listener at the same path, which stays subscribed + control = [] + root.get('score').subscribe(control.append) + + assert isinstance(sub, Subscription) + sub.unsubscribe() + + mock_ws.send_to_client(build_object_message('test', [ + build_counter_inc('counter:score@1000', 7, '99', 'remote'), + ])) + await poll_until(lambda: len(control) >= 1, description='the control listener to fire') + await settle() + + assert len(events) == 0 + + +# UTS: objects/unit/RTPO19e1/event-path-object-correct-0 +async def test_rtpo19e1_event_path_object_correct(): + client, channel, root, mock_ws = await setup_synced_channel('test') + events = [] + root.subscribe(events.append) + + mock_ws.send_to_client(build_object_message('test', [ + build_counter_inc('counter:score@1000', 7, '99', 'remote'), + ])) + await poll_until(lambda: len(events) >= 1, description='the COUNTER_INC event') + + assert isinstance(events[0].object, PathObject) + assert events[0].object.path() == 'score' + assert events[0].object.as_live_counter().value() == 107 + + +# UTS: objects/unit/RTPO19e2/event-message-delivery-0 +async def test_rtpo19e2_event_message_delivery(): + client, channel, root, mock_ws = await setup_synced_channel('test') + events = [] + root.get('score').subscribe(events.append) + + mock_ws.send_to_client(build_object_message('test', [ + build_counter_inc('counter:score@1000', 42, 'serial-1', 'site-a'), + ])) + await poll_until(lambda: len(events) >= 1, description='the COUNTER_INC event') + + assert events[0].message is not None + assert events[0].message.channel == 'test' + assert events[0].message.serial == 'serial-1' + assert events[0].message.site_code == 'site-a' + assert events[0].message.operation is not None + assert events[0].message.operation.action == ObjectOperationAction.COUNTER_INC + assert events[0].message.operation.object_id == 'counter:score@1000' + assert events[0].message.operation.counter_inc.number == 42 + + +# UTS: objects/unit/RTPO19e2/event-message-omitted-no-operation-0 +async def test_rtpo19e2_event_message_omitted_no_operation(): + client, channel, root, mock_ws = await setup_synced_channel('test') + events = [] + root.subscribe(events.append) + + # An OBJECT_SYNC that moves counter:score@1000 from 100 to 200 through replaceData (RTLC6), so + # the update's ObjectMessage has no operation. It omits root, which is never removed from the + # pool (RTO5c2a) and still references "score", so the counter stays reachable from the root + # subscription. + mock_ws.send_to_client(build_object_sync_message('test', 'sync2:', [ + build_object_state('counter:score@1000', {'aaa': 't:1'}, counter={'count': 0}, + create_op={'counterCreate': {'count': 200}}), + ])) + await poll_until(lambda: len(events) >= 1, description='the sync-triggered event') + await settle() + + # Events from sync-triggered updates carry no message + for event in events: + assert event.message is None + + +# UTS: objects/unit/RTPO19f/subscribe-follows-path-0 +async def test_rtpo19f_subscribe_follows_path(): + client, channel, root, mock_ws = await setup_synced_channel('test') + events = [] + root.get('score').subscribe(events.append) + + # Replace the counter at "score" with a new counter + mock_ws.send_to_client(build_object_message('test', [ + build_map_set('root', 'score', {'objectId': 'counter:new@2000'}, remote_serial(0), 'remote'), + ])) + + # Increment the new counter at "score" + mock_ws.send_to_client(build_object_message('test', [ + build_counter_inc('counter:new@2000', 10, '100', 'remote'), + ])) + await poll_until(lambda: len(events) >= 2, description='the MAP_SET and COUNTER_INC events') + await settle() + + # The subscription follows the path, so it sees the new counter + found_new = False + for event in events: + if event.object.path() == 'score': + found_new = True + assert found_new is True + # The second dispatch is the increment on the new counter, which is what shows the + # subscription followed the path to the replacement rather than staying bound to the old + # object; the MAP_SET dispatch alone would satisfy found_new + assert len(events) == 2 + assert events[1].object.path() == 'score' + assert events[1].message.operation.action == ObjectOperationAction.COUNTER_INC + assert events[1].message.operation.object_id == 'counter:new@2000' + + +# UTS: objects/unit/RTPO19g/subscribe-no-side-effects-0 +async def test_rtpo19g_subscribe_no_side_effects(): + client, channel, root, mock_ws = await setup_synced_channel('test') + state_before = channel.state + # RTPO19g also rules out side effects on the RealtimeObject and on the channel itself, which + # the channel state alone does not show while it is attached + sync_state_before = channel.object._sync_state + sent_before = len(mock_ws.messages_from_client) + + root.get('score').subscribe(lambda event: None) + await settle() + + assert channel.state == state_before + assert channel.object._sync_state == sync_state_before + assert len(mock_ws.messages_from_client) == sent_before + + +# UTS: objects/unit/RTPO19/subscribe-primitive-path-0 +async def test_rtpo19_subscribe_primitive_path(): + client, channel, root, mock_ws = await setup_synced_channel('test') + events = [] + root.get('name').subscribe(events.append) + + mock_ws.send_to_client(build_object_message('test', [ + build_map_set('root', 'name', {'string': 'Bob'}, remote_serial(0), 'remote'), + ])) + await poll_until(lambda: len(events) >= 1, description='the MAP_SET event at "name"') + await settle() + + assert len(events) == 1 + assert events[0].object.path() == 'name' + + +# UTS: objects/unit/RTPO19/map-clear-triggers-child-events-0 +async def test_rtpo19_map_clear_triggers_child_events(): + client, channel, root, mock_ws = await setup_synced_channel('test') + events = [] + root.subscribe(events.append) + # UTS SPEC ERROR: the specification clears with serial "99", which sorts before the pool's + # entry serial "t:0", so RTLM24e1 removes no entry and the update carries no key; and it + # subscribes at the root only, which fires for any root update. Neither reaches the child + # paths the test is named for. The clear uses a serial after the pool's, and a subscription + # at the child path "name" is asserted as well. + child_events = [] + root.get('name').subscribe(child_events.append) + + mock_ws.send_to_client(build_object_message('test', [ + build_map_clear('root', remote_serial(0), 'remote'), + ])) + await poll_until(lambda: len(events) >= 1, description='the MAP_CLEAR event at the root') + + assert len(events) >= 1 + await poll_until(lambda: len(child_events) >= 1, description='the MAP_CLEAR event at "name"') + assert child_events[0].object.path() == 'name' + + +# UTS: objects/unit/RTPO19/child-events-bubble-0 +async def test_rtpo19_child_events_bubble(): + client, channel, root, mock_ws = await setup_synced_channel('test') + events = [] + root.get('profile').subscribe(events.append) + + mock_ws.send_to_client(build_object_message('test', [ + build_map_set('map:profile@1000', 'email', {'string': 'bob@example.com'}, remote_serial(0), 'remote'), + ])) + await poll_until(lambda: len(events) >= 1, description='the profile event') + + mock_ws.send_to_client(build_object_message('test', [ + build_counter_inc('counter:nested@1000', 3, '100', 'remote'), + ])) + await poll_until(lambda: len(events) >= 2, description='the nested counter event') + + assert len(events) >= 2 + + +# UTS: objects/unit/RTO24c1/depth-filtering-formula-0 +async def test_rto24c1_depth_filtering_formula(): + client, channel, root, mock_ws = await setup_synced_channel('test') + # Seed a grandchild object under profile.prefs (path ["profile", "prefs", "deep"]), so the + # grandchild stimulus below can be a COUNTER_INC yielding only that depth-3 candidate (RTO6 + # creates counter:deep@3000 at its zero value) + mock_ws.send_to_client(build_object_message('test', [ + build_map_set('map:prefs@1000', 'deep', {'objectId': 'counter:deep@3000'}, '50', 'remote'), + ])) + # The specification subscribes straight after the send, relying on the seed not reaching + # the listener under test; it is applied on the read task, and its ["profile", "prefs"] + # candidate is covered at depth 2, so it must land before the subscription + await poll_until(lambda: 'deep' in root.get('profile').get('prefs').as_live_map().keys(), + description='the seeding MAP_SET to be applied') + events = [] + # Subscribed at "profile" with depth 2: + # self (profile) -> ["profile"], 1 - 1 + 1 = 1 <= 2, covered + # child (profile.nested) -> ["profile", "nested_counter"], 2 - 1 + 1 = 2 <= 2, covered + # grandchild (prefs.deep) -> ["profile", "prefs", "deep"], 3 - 1 + 1 = 3 > 2, not covered + root.get('profile').subscribe(events.append, depth=2) + # Quiescence control: an unlimited-depth root listener covers the out-of-scope grandchild path + control = [] + root.subscribe(control.append) + + # Self event (profile map update): the first covered candidate is ["profile"] + mock_ws.send_to_client(build_object_message('test', [ + build_map_set('map:profile@1000', 'email', {'string': 'bob@example.com'}, remote_serial(0), 'remote'), + ])) + await poll_until(lambda: len(events) >= 1, description='the self event') + + # Child event (the nested counter, relative depth 2): covered + mock_ws.send_to_client(build_object_message('test', [ + build_counter_inc('counter:nested@1000', 3, '100', 'remote'), + ])) + await poll_until(lambda: len(events) >= 2, description='the child event') + await settle() + + # Grandchild event (counter:deep, relative depth 3): not covered. A COUNTER_INC yields only + # this candidate, where a MAP_SET on map:prefs would also yield the covered ["profile", "prefs"]. + control_before = len(control) + mock_ws.send_to_client(build_object_message('test', [ + build_counter_inc('counter:deep@3000', 1, '101', 'remote'), + ])) + await poll_until(lambda: len(control) > control_before, description='the control to see the grandchild event') + await settle() + + assert len(events) == 2 + + +# UTS: objects/unit/RTO24c1/prefix-mismatch-0 +async def test_rto24c1_prefix_mismatch(): + client, channel, root, mock_ws = await setup_synced_channel('test') + profile_events = [] + root.get('profile').subscribe(profile_events.append) + # Control listener at the root, which fires on both out-of-scope sends below + control_events = [] + root.subscribe(control_events.append) + + # Change at "score": "profile" is not a prefix of "score" + mock_ws.send_to_client(build_object_message('test', [ + build_counter_inc('counter:score@1000', 7, '99', 'remote'), + ])) + + # Change at "name": "profile" is not a prefix of "name" + mock_ws.send_to_client(build_object_message('test', [ + build_map_set('root', 'name', {'string': 'Bob'}, remote_serial(0), 'remote'), + ])) + await poll_until(lambda: len(control_events) >= 2, description='the control to see both events') + await settle() + + assert len(profile_events) == 0 + + +# UTS: objects/unit/RTO24b2a/candidate-paths-map-keys-0 +async def test_rto24b2a_candidate_paths_map_keys(): + client, channel, root, mock_ws = await setup_synced_channel('test') + score_events = [] + root_events = [] + # Subscribed at the child path ["score"] + root.get('score').subscribe(score_events.append) + # Subscribed at the root path [] + root.subscribe(root_events.append) + + # A MAP_SET on root with key "score" yields the candidates [] (root itself) and ["score"] + # (from the map update key), so both subscriptions fire + mock_ws.send_to_client(build_object_message('test', [ + build_map_set('root', 'score', {'objectId': 'counter:new@2000'}, remote_serial(0), 'remote'), + ])) + await poll_until(lambda: len(score_events) >= 1, description='the event at "score"') + await poll_until(lambda: len(root_events) >= 1, description='the event at the root') + await settle() + + assert len(score_events) == 1 + assert score_events[0].object.path() == 'score' + assert len(root_events) == 1 + + +# UTS: objects/unit/RTO24b2c/listener-exception-caught-0 +async def test_rto24b2c_listener_exception_caught(): + client, channel, root, mock_ws = await setup_synced_channel('test') + events = [] + + def boom(event): + raise Exception('boom') + + root.subscribe(boom) + root.subscribe(events.append) + + mock_ws.send_to_client(build_object_message('test', [ + build_map_set('root', 'name', {'string': 'Bob'}, remote_serial(0), 'remote'), + ])) + await poll_until(lambda: len(events) >= 1, description='the second listener to fire') + await settle() + + assert len(events) == 1 + + +# UTS: objects/unit/RTO24b1/multi-path-dispatch-0 +async def test_rto24b1_multi_path_dispatch(): + client, channel, root, mock_ws = await setup_synced_channel('test') + events_score = [] + events_alias = [] + + # "score" already points to counter:score@1000; a second reference "alias" gives it two paths + mock_ws.send_to_client(build_object_message('test', [ + build_map_set('root', 'alias', {'objectId': 'counter:score@1000'}, '98', 'remote'), + ])) + # The specification subscribes straight after the send; the MAP_SET is applied on the read + # task, and its ["alias"] candidate would reach the "alias" subscription if it landed after it + await poll_until(lambda: 'alias' in root.keys(), description='the alias MAP_SET to be applied') + + root.get('score').subscribe(events_score.append) + root.get('alias').subscribe(events_alias.append) + + # Increment counter:score@1000, whose full paths are ["score"] and ["alias"] + mock_ws.send_to_client(build_object_message('test', [ + build_counter_inc('counter:score@1000', 5, '99', 'remote'), + ])) + await poll_until(lambda: len(events_score) >= 1, description='the event at "score"') + await poll_until(lambda: len(events_alias) >= 1, description='the event at "alias"') + await settle() + + assert len(events_score) == 1 + assert events_score[0].object.path() == 'score' + assert len(events_alias) == 1 + assert events_alias[0].object.path() == 'alias' + + +# UTS: objects/unit/RTO24b2b/fires-once-per-dispatch-0 +async def test_rto24b2b_fires_once_per_dispatch(): + client, channel, root, mock_ws = await setup_synced_channel('test') + events = [] + # Subscribed at the root with unlimited depth, covering both [] and ["score"] + root.subscribe(events.append) + + # A MAP_SET on root with key "score" yields the candidates [] and ["score"]; the root + # subscription covers both and fires once, with the first + mock_ws.send_to_client(build_object_message('test', [ + build_map_set('root', 'score', {'objectId': 'counter:new@2000'}, remote_serial(0), 'remote'), + ])) + await poll_until(lambda: len(events) >= 1, description='the MAP_SET event') + + # Quiescence: a second, single-candidate dispatch is the control delivery, so a spurious + # second callback from the first dispatch would already have run + mock_ws.send_to_client(build_object_message('test', [ + build_counter_inc('counter:new@2000', 1, '100', 'remote'), + ])) + await poll_until(lambda: len(events) >= 2, description='the control COUNTER_INC event') + await settle() + + # One event per dispatch, though the first had two covered candidates + assert len(events) == 2 diff --git a/test/uts/objects/unit/path_object_test.py b/test/uts/objects/unit/path_object_test.py new file mode 100644 index 00000000..2172f068 --- /dev/null +++ b/test/uts/objects/unit/path_object_test.py @@ -0,0 +1,363 @@ +"""Derived from uts/objects/unit/path_object.md in ably/specification. + +Spec points: RTPO3, RTPO3a1, RTPO3b, RTPO3c1, RTPO4, RTPO4a, RTPO4b, RTPO4c, RTPO5, RTPO5b, RTPO5c, +RTPO5d, RTPO6, RTPO6b, RTPO6d, RTPO7, RTPO7a, RTPO7c, RTPO7d, RTPO7e, RTPO7f, RTPO8, RTPO8a, RTPO8c, +RTPO8f, RTPO9, RTPO9a, RTPO9c, RTPO9d, RTPO10, RTPO10a, RTPO10c, RTPO10d, RTPO11, RTPO11a, RTPO11c, +RTPO11d, RTPO12, RTPO12a, RTPO12c, RTPO12d, RTPO13, RTPO13a, RTPO13c1, RTPO13c2, RTPO13c3, RTPO13c4, +RTPO13c5, RTPO13d, RTPO14, RTPO14a, RTPO14b1, RTPO14b2, RTINS3b, RTINS4c, RTTS5d1 + +The specification reads through the untyped `PathObject`, which carries every method. ably-python +partitions it (LODR-061, RTTS3-RTTS6): navigation, `instance`, `compact` and `compact_json` are on the +base `PathObject`, and the type-specific reads are reached through the unchecked view helpers, so +`po.value()` on a counter is `po.as_live_counter().value()`, on a primitive `po.as_primitive().value()`, +and `po.keys()` is `po.as_live_map().keys()`. A view whose type does not match what the path resolves +to answers None, or `[]` for the collection reads (RTTS5d1), which is what the specification's +null and empty-array expectations become. Where the specification expects the untyped `value()` to +be null, neither value view may answer, so both are asserted. + +Every test opens with the standard synced channel; the module issues no writes, so no ACK is involved. +""" + +import pytest + +from ably.pubsub.objects.instance import Instance +from ably.pubsub.util.exceptions import AblyException +from test.uts.helpers.client import poll_until +from test.uts.objects.helpers.standard_test_pool import ( + build_map_set, + build_object_message, + setup_synced_channel, +) + + +async def _apply_back_ref(root, mock_ws): + """Sends the specification's `prefs.back_ref -> profile` MAP_SET, closing a cycle, and waits for it. + + The MAP_SET is an inbound OBJECT message, applied on the transport's read task, so the + specification's quiescence barrier is a poll until the new key is visible. + """ + mock_ws.send_to_client(build_object_message('test', [ + build_map_set('map:prefs@1000', 'back_ref', {'objectId': 'map:profile@1000'}, '99', 'remote'), + ])) + await poll_until(lambda: 'back_ref' in root.get('profile').get('prefs').as_live_map().keys(), + description='the back_ref MAP_SET to be applied') + + +# UTS: objects/unit/RTPO4/path-string-representation-0 +async def test_rtpo4_path_string_representation(): + client, channel, root, mock_ws = await setup_synced_channel('test') + + assert root.path() == '' + assert root.get('profile').path() == 'profile' + assert root.get('profile').get('email').path() == 'profile.email' + + +# UTS: objects/unit/RTPO4b/path-escapes-dots-0 +async def test_rtpo4b_path_escapes_dots(): + client, channel, root, mock_ws = await setup_synced_channel('test') + + po = root.get('a.b').get('c') + + # The specification's "a\\.b.c" is the string a\.b.c: one backslash before the dot in the segment + assert po.path() == 'a\\.b.c' + + +# UTS: objects/unit/RTPO5/get-appends-key-0 +async def test_rtpo5_get_appends_key(): + client, channel, root, mock_ws = await setup_synced_channel('test') + + child = root.get('profile') + grandchild = child.get('email') + + assert child.path() == 'profile' + assert grandchild.path() == 'profile.email' + assert child is not root + + +# UTS: objects/unit/RTPO5b/get-non-string-throws-0 +async def test_rtpo5b_get_non_string_throws(): + client, channel, root, mock_ws = await setup_synced_channel('test') + + with pytest.raises(AblyException) as excinfo: + root.get(123) + + assert excinfo.value.code == 40003 + + +# UTS: objects/unit/RTPO6/at-parses-path-0 +async def test_rtpo6_at_parses_path(): + client, channel, root, mock_ws = await setup_synced_channel('test') + + po = root.at('profile.email') + + assert po.path() == 'profile.email' + assert po.as_primitive().value() == 'alice@example.com' + + +# UTS: objects/unit/RTPO6/at-escaped-dots-0 +async def test_rtpo6_at_escaped_dots(): + client, channel, root, mock_ws = await setup_synced_channel('test') + + po = root.at('a\\.b.c') + + assert po.path() == 'a\\.b.c' + # UTS SPEC ERROR: the round trip through path() alone does not show that `\.` stayed inside + # its segment: an `at` that split on every dot gives the segments ['a\\', 'b', 'c'], which + # path() renders as the same string. RTPO6b makes `\.` a literal dot within a segment, so + # the parsed segments are asserted as well. + assert po._path == ['a.b', 'c'] + + +# UTS: objects/unit/RTPO6b/at-non-string-throws-0 +async def test_rtpo6b_at_non_string_throws(): + client, channel, root, mock_ws = await setup_synced_channel('test') + + # NOTE: the features spec's RTPO6 defines no error for a path that is not a string (RTPO5b + # covers `get` only); 40003 is asserted as the specification has it, consistently with RTPO5b. + # ably-python's `at` also takes a sequence of segments (LODR-061), and an int is neither. + with pytest.raises(AblyException) as excinfo: + root.at(123) + + assert excinfo.value.code == 40003 + + +# UTS: objects/unit/RTPO7/value-counter-0 +async def test_rtpo7_value_counter(): + client, channel, root, mock_ws = await setup_synced_channel('test') + + assert root.get('score').as_live_counter().value() == 100 + + +# UTS: objects/unit/RTPO7/value-primitive-0 +async def test_rtpo7_value_primitive(): + client, channel, root, mock_ws = await setup_synced_channel('test') + + assert root.get('name').as_primitive().value() == 'Alice' + assert root.get('age').as_primitive().value() == 30 + assert root.get('active').as_primitive().value() is True + + +# UTS: objects/unit/RTPO7d/value-livemap-null-0 +async def test_rtpo7d_value_livemap_null(): + client, channel, root, mock_ws = await setup_synced_channel('test') + + profile = root.get('profile') + + # value() on a map is null (RTPO7e); neither value view answers for a map (RTTS6b, RTTS6c) + assert profile.as_primitive().value() is None + assert profile.as_live_counter().value() is None + + +# UTS: objects/unit/RTPO7e/value-unresolvable-null-0 +async def test_rtpo7e_value_unresolvable_null(): + client, channel, root, mock_ws = await setup_synced_channel('test') + + deep = root.get('nonexistent').get('deep') + + assert deep.as_primitive().value() is None + assert deep.as_live_counter().value() is None + + +# UTS: objects/unit/RTPO8/instance-live-object-0 +async def test_rtpo8_instance_live_object(): + client, channel, root, mock_ws = await setup_synced_channel('test') + + counter_inst = root.get('score').instance() + assert isinstance(counter_inst, Instance) + assert counter_inst.id == 'counter:score@1000' + + map_inst = root.get('profile').instance() + assert isinstance(map_inst, Instance) + assert map_inst.id == 'map:profile@1000' + + +# UTS: objects/unit/RTPO8f/instance-primitive-wrapped-0 +async def test_rtpo8f_instance_primitive_wrapped(): + client, channel, root, mock_ws = await setup_synced_channel('test') + + name_inst = root.get('name').instance() + + assert isinstance(name_inst, Instance) + assert name_inst.id is None + assert name_inst.as_primitive().value() == 'Alice' + + +# UTS: objects/unit/RTPO9/entries-yields-pairs-0 +async def test_rtpo9_entries_yields_pairs(): + client, channel, root, mock_ws = await setup_synced_channel('test') + + entries = {} + for key, path_obj in root.entries(): + entries[key] = path_obj.path() + + assert entries['name'] == 'name' + assert entries['profile'] == 'profile' + assert len(entries) == 7 + + +# UTS: objects/unit/RTPO9d/entries-non-map-empty-0 +async def test_rtpo9d_entries_non_map_empty(): + client, channel, root, mock_ws = await setup_synced_channel('test') + + entries = root.get('score').as_live_map().entries() + + assert len(entries) == 0 + + +# UTS: objects/unit/RTPO10/keys-returns-array-0 +async def test_rtpo10_keys_returns_array(): + client, channel, root, mock_ws = await setup_synced_channel('test') + + keys = root.keys() + + assert isinstance(keys, list) + assert len(keys) == 7 + assert 'name' in keys + assert 'profile' in keys + assert 'score' in keys + + +# UTS: objects/unit/RTPO10d/keys-non-map-empty-0 +async def test_rtpo10d_keys_non_map_empty(): + client, channel, root, mock_ws = await setup_synced_channel('test') + + keys = root.get('score').as_live_map().keys() + + assert isinstance(keys, list) + assert len(keys) == 0 + + +# UTS: objects/unit/RTPO11/values-returns-array-0 +async def test_rtpo11_values_returns_array(): + client, channel, root, mock_ws = await setup_synced_channel('test') + + vals = root.values() + + assert isinstance(vals, list) + assert len(vals) == 7 + # Each element is a PathObject whose path is the key + paths = {} + for v in vals: + paths[v.path()] = True + assert paths['name'] is True + assert paths['profile'] is True + assert paths['score'] is True + + +# UTS: objects/unit/RTPO11d/values-non-map-empty-0 +async def test_rtpo11d_values_non_map_empty(): + client, channel, root, mock_ws = await setup_synced_channel('test') + + vals = root.get('score').as_live_map().values() + + assert isinstance(vals, list) + assert len(vals) == 0 + + +# UTS: objects/unit/RTPO12/size-count-0 +async def test_rtpo12_size_count(): + client, channel, root, mock_ws = await setup_synced_channel('test') + + assert root.size() == 7 + assert root.get('profile').as_live_map().size() == 3 + + +# UTS: objects/unit/RTPO12c/size-non-map-null-0 +async def test_rtpo12c_size_non_map_null(): + client, channel, root, mock_ws = await setup_synced_channel('test') + + assert root.get('score').as_live_map().size() is None + assert root.get('name').as_live_map().size() is None + + +# UTS: objects/unit/RTPO13/compact-recursive-0 +async def test_rtpo13_compact_recursive(): + client, channel, root, mock_ws = await setup_synced_channel('test') + + result = root.compact() + + assert result['name'] == 'Alice' + assert result['age'] == 30 + assert result['active'] is True + assert result['score'] == 100 + assert result['data'] == {'tags': ['a', 'b']} + assert result['avatar'] == bytes([1, 2, 3]) + assert result['profile']['email'] == 'alice@example.com' + assert result['profile']['nested_counter'] == 5 + assert result['profile']['prefs']['theme'] == 'dark' + + +# UTS: objects/unit/RTPO13c5/compact-cycle-detection-0 +async def test_rtpo13c5_compact_cycle_detection(): + client, channel, root, mock_ws = await setup_synced_channel('test') + await _apply_back_ref(root, mock_ws) + + result = root.get('profile').compact() + + assert result['prefs']['back_ref'] is result + + +# UTS: objects/unit/RTPO13c/compact-counter-0 +async def test_rtpo13c_compact_counter(): + client, channel, root, mock_ws = await setup_synced_channel('test') + + assert root.get('score').compact() == 100 + + +# UTS: objects/unit/RTPO14/compact-json-0 +async def test_rtpo14_compact_json(): + client, channel, root, mock_ws = await setup_synced_channel('test') + await _apply_back_ref(root, mock_ws) + + result = root.get('profile').compact_json() + + assert result['prefs']['back_ref'] == {'objectId': 'map:profile@1000'} + + +# UTS: objects/unit/RTPO3/path-resolution-walk-0 +async def test_rtpo3_path_resolution_walk(): + client, channel, root, mock_ws = await setup_synced_channel('test') + + # The empty path resolves to the root map (RTPO3b), for which value() is null (RTPO7e) + assert root.as_primitive().value() is None + assert root.as_live_counter().value() is None + assert root.get('profile').get('prefs').get('theme').as_primitive().value() == 'dark' + + +# UTS: objects/unit/RTPO3a1/intermediate-not-map-0 +async def test_rtpo3a1_intermediate_not_map(): + client, channel, root, mock_ws = await setup_synced_channel('test') + + something = root.get('score').get('something') + + assert something.as_primitive().value() is None + assert something.as_live_counter().value() is None + + +# UTS: objects/unit/RTPO3c1/read-null-on-failure-0 +async def test_rtpo3c1_read_null_on_failure(): + client, channel, root, mock_ws = await setup_synced_channel('test') + + nonexistent = root.get('nonexistent') + + assert nonexistent.as_primitive().value() is None + assert nonexistent.as_live_counter().value() is None + assert nonexistent.instance() is None + assert nonexistent.as_live_map().size() is None + assert nonexistent.compact() is None + + +# UTS: objects/unit/RTPO7/value-bytes-0 +async def test_rtpo7_value_bytes(): + client, channel, root, mock_ws = await setup_synced_channel('test') + + assert root.get('avatar').as_primitive().value() == bytes([1, 2, 3]) + + +# UTS: objects/unit/RTPO14/compact-json-bytes-0 +async def test_rtpo14_compact_json_bytes(): + client, channel, root, mock_ws = await setup_synced_channel('test') + + result = root.compact_json() + + assert result['avatar'] == 'AQID' diff --git a/test/uts/objects/unit/public_object_message_test.py b/test/uts/objects/unit/public_object_message_test.py new file mode 100644 index 00000000..1e412c77 --- /dev/null +++ b/test/uts/objects/unit/public_object_message_test.py @@ -0,0 +1,345 @@ +"""Derived from uts/objects/unit/public_object_message.md in ably/specification. + +Spec points: PAOM1, PAOM2, PAOM3, PAOOP1, PAOOP2, PAOOP3 + +The specification's `PublicObjectMessage.fromObjectMessage(source, channel)` is +`publicmessage.ObjectMessage._from_internal(source, channel_name)`, which takes the +channel's name rather than the channel (PAOM3b reads only `channel.name`), and +`PublicObjectOperation.fromObjectOperation(op)` is +`publicmessage.ObjectOperation._from_internal(op)`. The public types share their names with +the internal wire types, so the public pair is reached through the `publicmessage` module. + +Each source is built as the internal dataclasses directly. An attribute the specification +writes as `null` or omitted is None, operation actions are `ObjectOperationAction` members +and map semantics `ObjectsMapSemantics` members. + +The specification gives `mapCreateWithObjectId` and `counterCreateWithObjectId` the fields +of the payload they were derived from (`objectId`, `semantics`, `entries`, `count`), which +those payloads do not carry: MCRO2 and CCRO2 define only `initialValue` and `nonce`. Here +they carry those two, with `initialValue` the JSON encoding of the derived payload as +RTLMV4f and RTLCV4c produce it, and the derived payload in `derived_from` (RTLMV4j5, +RTLCV4g5). +""" + +import json + +from ably.pubsub.objects import publicmessage +from ably.pubsub.objects.objectmessage import ( + CounterCreate, + CounterCreateWithObjectId, + CounterInc, + MapClear, + MapCreate, + MapCreateWithObjectId, + MapRemove, + MapSet, + ObjectData, + ObjectDelete, + ObjectMessage, + ObjectOperation, + ObjectOperationAction, + ObjectsMapEntry, + ObjectsMapSemantics, +) + +NONCE = 'test-nonce-1234567890' + + +# UTS: objects/unit/PAOM3/construction-all-fields-0 +def test_paom3_construction_all_fields(): + source = ObjectMessage( + id='msg-id-1', + client_id='client-1', + connection_id='conn-1', + timestamp=1700000000000, + serial='01', + serial_timestamp=1700000001000, + site_code='site1', + extras={'key': 'value'}, + operation=ObjectOperation( + action=ObjectOperationAction.MAP_SET, + object_id='map:abc@1000', + map_set=MapSet(key='name', value=ObjectData(string='Alice')), + ), + ) + + public_msg = publicmessage.ObjectMessage._from_internal(source, 'test-channel') + + assert isinstance(public_msg, publicmessage.ObjectMessage) + assert public_msg.id == 'msg-id-1' + assert public_msg.client_id == 'client-1' + assert public_msg.connection_id == 'conn-1' + assert public_msg.timestamp == 1700000000000 + assert public_msg.channel == 'test-channel' + assert public_msg.serial == '01' + assert public_msg.serial_timestamp == 1700000001000 + assert public_msg.site_code == 'site1' + assert public_msg.extras == {'key': 'value'} + assert public_msg.operation is not None + assert isinstance(public_msg.operation, publicmessage.ObjectOperation) + assert public_msg.operation.action == ObjectOperationAction.MAP_SET + assert public_msg.operation.object_id == 'map:abc@1000' + assert public_msg.operation.map_set.key == 'name' + assert public_msg.operation.map_set.value.string == 'Alice' + + +# UTS: objects/unit/PAOM3/construction-optional-fields-missing-0 +def test_paom3_construction_optional_fields_missing(): + source = ObjectMessage( + operation=ObjectOperation( + action=ObjectOperationAction.COUNTER_INC, + object_id='counter:abc@1000', + counter_inc=CounterInc(number=5.0), + ), + ) + + public_msg = publicmessage.ObjectMessage._from_internal(source, 'my-channel') + + assert public_msg.id is None + assert public_msg.client_id is None + assert public_msg.connection_id is None + assert public_msg.timestamp is None + assert public_msg.channel == 'my-channel' + assert public_msg.serial is None + assert public_msg.serial_timestamp is None + assert public_msg.site_code is None + assert public_msg.extras is None + assert public_msg.operation is not None + assert public_msg.operation.action == ObjectOperationAction.COUNTER_INC + + +# UTS: objects/unit/PAOM3/channel-from-channel-name-0 +def test_paom3_channel_from_channel_name(): + source = ObjectMessage( + operation=ObjectOperation( + action=ObjectOperationAction.OBJECT_DELETE, + object_id='counter:abc@1000', + ), + ) + + public_msg = publicmessage.ObjectMessage._from_internal(source, 'different-channel-name') + + assert public_msg.channel == 'different-channel-name' + + +# UTS: objects/unit/PAOOP3/map-set-copies-fields-0 +def test_paoop3_map_set_copies_fields(): + source_operation = ObjectOperation( + action=ObjectOperationAction.MAP_SET, + object_id='map:abc@1000', + map_set=MapSet(key='color', value=ObjectData(string='blue')), + ) + + public_op = publicmessage.ObjectOperation._from_internal(source_operation) + + assert public_op.action == ObjectOperationAction.MAP_SET + assert public_op.object_id == 'map:abc@1000' + assert public_op.map_set.key == 'color' + assert public_op.map_set.value.string == 'blue' + assert public_op.map_create is None + assert public_op.map_remove is None + assert public_op.counter_create is None + assert public_op.counter_inc is None + assert public_op.object_delete is None + assert public_op.map_clear is None + + +# UTS: objects/unit/PAOOP3/map-remove-copies-fields-0 +def test_paoop3_map_remove_copies_fields(): + source_operation = ObjectOperation( + action=ObjectOperationAction.MAP_REMOVE, + object_id='map:abc@1000', + map_remove=MapRemove(key='old-key'), + ) + + public_op = publicmessage.ObjectOperation._from_internal(source_operation) + + assert public_op.action == ObjectOperationAction.MAP_REMOVE + assert public_op.object_id == 'map:abc@1000' + assert public_op.map_remove.key == 'old-key' + assert public_op.map_create is None + assert public_op.map_set is None + assert public_op.counter_create is None + assert public_op.counter_inc is None + assert public_op.object_delete is None + assert public_op.map_clear is None + + +# UTS: objects/unit/PAOOP3/counter-inc-copies-fields-0 +def test_paoop3_counter_inc_copies_fields(): + source_operation = ObjectOperation( + action=ObjectOperationAction.COUNTER_INC, + object_id='counter:abc@1000', + counter_inc=CounterInc(number=42.0), + ) + + public_op = publicmessage.ObjectOperation._from_internal(source_operation) + + assert public_op.action == ObjectOperationAction.COUNTER_INC + assert public_op.object_id == 'counter:abc@1000' + assert public_op.counter_inc.number == 42 + assert public_op.map_create is None + assert public_op.map_set is None + assert public_op.map_remove is None + assert public_op.counter_create is None + assert public_op.object_delete is None + assert public_op.map_clear is None + + +# UTS: objects/unit/PAOOP3/object-delete-copies-fields-0 +def test_paoop3_object_delete_copies_fields(): + source_operation = ObjectOperation( + action=ObjectOperationAction.OBJECT_DELETE, + object_id='counter:abc@1000', + object_delete=ObjectDelete(), + ) + + public_op = publicmessage.ObjectOperation._from_internal(source_operation) + + assert public_op.action == ObjectOperationAction.OBJECT_DELETE + assert public_op.object_id == 'counter:abc@1000' + assert public_op.object_delete is not None + assert public_op.map_create is None + assert public_op.map_set is None + assert public_op.map_remove is None + assert public_op.counter_create is None + assert public_op.counter_inc is None + assert public_op.map_clear is None + + +# UTS: objects/unit/PAOOP3/map-clear-copies-fields-0 +def test_paoop3_map_clear_copies_fields(): + source_operation = ObjectOperation( + action=ObjectOperationAction.MAP_CLEAR, + object_id='map:abc@1000', + map_clear=MapClear(), + ) + + public_op = publicmessage.ObjectOperation._from_internal(source_operation) + + assert public_op.action == ObjectOperationAction.MAP_CLEAR + assert public_op.object_id == 'map:abc@1000' + assert public_op.map_clear is not None + assert public_op.map_create is None + assert public_op.map_set is None + assert public_op.map_remove is None + assert public_op.counter_create is None + assert public_op.counter_inc is None + assert public_op.object_delete is None + + +# UTS: objects/unit/PAOOP3/map-create-direct-0 +def test_paoop3_map_create_direct(): + source_operation = ObjectOperation( + action=ObjectOperationAction.MAP_CREATE, + object_id='map:new@2000', + map_create=MapCreate( + semantics=ObjectsMapSemantics.LWW, + entries={'key1': ObjectsMapEntry(data=ObjectData(string='val1'))}, + ), + ) + + public_op = publicmessage.ObjectOperation._from_internal(source_operation) + + assert public_op.action == ObjectOperationAction.MAP_CREATE + assert public_op.object_id == 'map:new@2000' + assert public_op.map_create is not None + assert public_op.map_create.semantics == ObjectsMapSemantics.LWW + assert public_op.map_create.entries['key1'].data.string == 'val1' + assert public_op.counter_create is None + + +# UTS: objects/unit/PAOOP3/map-create-from-with-object-id-0 +def test_paoop3_map_create_from_with_object_id(): + derived_map_create = MapCreate( + semantics=ObjectsMapSemantics.LWW, + entries={'x': ObjectsMapEntry(data=ObjectData(number=10.0))}, + ) + + # The specification also lists `objectId`, `semantics` and `entries` inside + # `mapCreateWithObjectId`, which MCRO2 does not define; see the module docstring + source_operation = ObjectOperation( + action=ObjectOperationAction.MAP_CREATE, + object_id='map:derived@3000', + map_create_with_object_id=MapCreateWithObjectId( + initial_value=json.dumps(derived_map_create.to_dict()), + nonce=NONCE, + derived_from=derived_map_create, + ), + ) + + public_op = publicmessage.ObjectOperation._from_internal(source_operation) + + assert public_op.action == ObjectOperationAction.MAP_CREATE + assert public_op.object_id == 'map:derived@3000' + assert public_op.map_create is not None + assert public_op.map_create.semantics == ObjectsMapSemantics.LWW + assert public_op.map_create.entries['x'].data.number == 10 + assert public_op.counter_create is None + # PAOOP1: the public operation carries the derived payload, never the WithObjectId one + assert public_op.map_create == derived_map_create + assert not hasattr(public_op, 'map_create_with_object_id') + + +# UTS: objects/unit/PAOOP3/counter-create-from-with-object-id-0 +def test_paoop3_counter_create_from_with_object_id(): + derived_counter_create = CounterCreate(count=100.0) + + # The specification also lists `objectId` and `count` inside + # `counterCreateWithObjectId`, which CCRO2 does not define; see the module docstring + source_operation = ObjectOperation( + action=ObjectOperationAction.COUNTER_CREATE, + object_id='counter:derived@3000', + counter_create_with_object_id=CounterCreateWithObjectId( + initial_value=json.dumps(derived_counter_create.to_dict()), + nonce=NONCE, + derived_from=derived_counter_create, + ), + ) + + public_op = publicmessage.ObjectOperation._from_internal(source_operation) + + assert public_op.action == ObjectOperationAction.COUNTER_CREATE + assert public_op.object_id == 'counter:derived@3000' + assert public_op.counter_create is not None + assert public_op.counter_create.count == 100 + assert public_op.map_create is None + # PAOOP1: the public operation carries the derived payload, never the WithObjectId one + assert public_op.counter_create == derived_counter_create + assert not hasattr(public_op, 'counter_create_with_object_id') + + +# UTS: objects/unit/PAOOP3/create-payloads-omitted-0 +def test_paoop3_create_payloads_omitted(): + source_operation = ObjectOperation( + action=ObjectOperationAction.MAP_SET, + object_id='map:abc@1000', + map_set=MapSet(key='k', value=ObjectData(string='v')), + ) + + public_op = publicmessage.ObjectOperation._from_internal(source_operation) + + assert public_op.map_create is None + assert public_op.counter_create is None + + +# UTS: objects/unit/PAOOP3/only-relevant-field-per-action-0 +def test_paoop3_only_relevant_field_per_action(): + source_operation = ObjectOperation( + action=ObjectOperationAction.COUNTER_CREATE, + object_id='counter:new@2000', + counter_create=CounterCreate(count=50.0), + ) + + public_op = publicmessage.ObjectOperation._from_internal(source_operation) + + assert public_op.action == ObjectOperationAction.COUNTER_CREATE + assert public_op.object_id == 'counter:new@2000' + assert public_op.counter_create is not None + assert public_op.counter_create.count == 50 + assert public_op.map_create is None + assert public_op.map_set is None + assert public_op.map_remove is None + assert public_op.counter_inc is None + assert public_op.object_delete is None + assert public_op.map_clear is None diff --git a/test/uts/objects/unit/realtime_object_test.py b/test/uts/objects/unit/realtime_object_test.py new file mode 100644 index 00000000..33252655 --- /dev/null +++ b/test/uts/objects/unit/realtime_object_test.py @@ -0,0 +1,951 @@ +"""Derived from uts/objects/unit/realtime_object.md in ably/specification. + +Spec points: RTO2, RTO2a, RTO2b, RTO5c9, RTO10, RTO10a, RTO10b1, RTO10c1b, RTO10c1b1, RTO15, +RTO15e1, RTO15e2, RTO15e3, RTO17, RTO17b, RTO18, RTO18b, RTO18d, RTO18e, RTO19, RTO20, RTO20b, +RTO20c1, RTO20d1, RTO20d2, RTO20d4, RTO20e, RTO20e1, RTO20f, RTO23, RTO23a, RTO23c, RTO23c1, +RTO23d, RTO23e, RTO24a, RTO24c1, RTO25a, RTO25b, RTO26a, RTO26b, RTO26c, RTO27a, RTO27b, RTL33b, +RTL33c, RTLO4e10 + +The specification drives `channel.object` against the mock websocket, and reaches into the +RealtimeObject where nothing public observes what it asserts: the channel-state handler +(`_act_on_channel_state`), the pool (`_objects_pool`), the sync state (`_sync_state`) and the +GC interval (`_gc_interval_ms`). + +Several translations recur through the file. + +Every OBJECT the client publishes is ACKed, by the standard mock or by a test's own handler, +because a publish resolves only on its ACK (RTO15g). The one test whose specification sends +the ACK by hand, `ack-after-echo-no-double-apply`, drives the write as a task and ACKs it once +the OBJECT has left the client. + +The specifications' untyped `PathObject` calls go through the typed views: +`root.get("score").value()` is `root.get('score').as_live_counter().value()`, and +`root.get("name").value()` is `root.get('name').as_primitive().value()`. + +The modes a server grants travel as bits of the ATTACHED `flags`, so the specification's +`modes: ["OBJECT_SUBSCRIBE"]` is `flags=HAS_OBJECTS | OBJECT_SUBSCRIBE_FLAG`. + +An inbound frame is processed on the transport's read task, so state is read after a +`poll_until` or a `settle()`, and a negative assertion ("the listener did not fire", "the echo +did not apply") follows a positive control delivered behind the message under test. + +`enable_fake_timers()` is a `FakeClock` given to the client, which is where the GC timer +(RTO10a) is scheduled and where `now_ms()` is read for a sweep (RTO10c1b). +""" + +import asyncio + +import pytest + +from ably.pubsub.objects.enums import ObjectsEvent, ObjectsSyncState +from ably.pubsub.objects.objectmessage import ObjectOperationAction +from ably.pubsub.objects.pathobject import PathObject +from ably.pubsub.objects.valuetypes import LiveCounter +from ably.pubsub.transport.websockettransport import ProtocolMessageAction +from ably.pubsub.types.channelmode import ChannelMode +from ably.pubsub.types.channelstate import ChannelState +from ably.pubsub.util.exceptions import AblyException +from test.uts.helpers.client import await_channel_state, poll_until +from test.uts.helpers.clock import FakeClock, settle +from test.uts.helpers.mock_websocket import MockWebSocket, channel_error_message, contains_in_order +from test.uts.objects.helpers.standard_test_pool import ( + ATTACH, + HAS_OBJECTS, + OBJECT, + OBJECT_PUBLISH_FLAG, + OBJECT_SUBSCRIBE_FLAG, + SITE_CODE, + STANDARD_POOL_OBJECTS, + ack_serial, + assert_unchanged_after_quiescence, + below_ack_serial, + build_ack_message, + build_counter_inc, + build_map_set, + build_object_delete, + build_object_message, + build_object_sync_message, + objects_attached_message, + objects_channel_options, + objects_client, + objects_connected_message, + remote_serial, + setup_synced_channel, + setup_synced_channel_no_ack, + standard_mock_websocket, +) + +# How long a test waits on an operation it expects to complete or fail, so that one which +# never settles fails with a timeout rather than running into the suite's own +OPERATION_TIMEOUT = 5 + +# RTO10b3's default grace period and RTO10a's example interval, which the GC tests advance past +DEFAULT_GC_GRACE_PERIOD_MS = 86_400_000 +DEFAULT_GC_INTERVAL_MS = 300_000 + + +def _score(root): + """The value of the standard pool's `score` counter, read through the counter view.""" + return root.get('score').as_live_counter().value() + + +def _nested_counter(root): + """The value of the standard pool's `profile.nested_counter` counter.""" + return root.get('profile').get('nested_counter').as_live_counter().value() + + +def _name(root): + """The standard pool's `name` primitive, read through the primitive view.""" + return root.get('name').as_primitive().value() + + +def _sent(mock_ws, action): + """How many protocol messages carrying `action` the client has sent.""" + return sum(1 for message in mock_ws.messages_from_client if message.get('action') == int(action)) + + +def _raise_if_failed(task): + """Re-raises the exception `task` finished with, so that an early failure reports itself.""" + if task.done() and not task.cancelled() and task.exception() is not None: + raise task.exception() + + +def _assert_pending(task, description): + """Asserts `task` is still waiting; a task that already failed re-raises its own error.""" + _raise_if_failed(task) + assert not task.done(), f'{description} completed while it should still be waiting' + + +async def _await_sent(mock_ws, action, task): + """Waits for the client to send a message carrying `action` on behalf of `task`.""" + await poll_until(lambda: task.done() or _sent(mock_ws, action) >= 1, + description=f'the client to send a message with action {int(action)}') + _raise_if_failed(task) + + +async def _restart_sync(channel, mock_ws, channel_serial='sync2:cursor'): + """Sends an ATTACHED carrying a sync cursor and waits for the sync state to leave SYNCED. + + A new ATTACHED moves the sync state to SYNCING (RTO4c), and the cursor keeps it there + until an OBJECT_SYNC completes the sequence. + """ + mock_ws.send_to_client(objects_attached_message(channel.name, channel_serial)) + await poll_until(lambda: channel.object._sync_state == ObjectsSyncState.SYNCING, + description='the ATTACHED to restart the objects sync') + + +def _acking_mock_websocket(serials, captured=None, **kwargs): + """The standard mock, ACKing each OBJECT with the serials `serials(message)` gives. + + Each OBJECT is recorded in `captured` first, if given. `kwargs` go to + `standard_mock_websocket`. + """ + mock_ws = None + + def on_object(message): + if captured is not None: + captured.append(message) + mock_ws.send_to_client(build_ack_message(message['msgSerial'], serials(message))) + + mock_ws = standard_mock_websocket(auto_ack=False, on_object=on_object, **kwargs) + return mock_ws + + +# UTS: objects/unit/RTO23/get-returns-path-object-0 +async def test_rto23_get_returns_path_object(): + client, channel, root, mock_ws = await setup_synced_channel('test') + + assert isinstance(root, PathObject) + assert root._path == [] + # RTO23d: the root is the pool's InternalLiveMap with id `root` + assert root._root is channel.object._objects_pool['root'] + + +# UTS: objects/unit/RTO23a/get-requires-subscribe-mode-0 +async def test_rto23a_get_requires_subscribe_mode(): + mock_ws = standard_mock_websocket(sync_objects=None) + client = objects_client(mock_ws) + channel = client.channels.get('test', objects_channel_options(ChannelMode.OBJECT_PUBLISH)) + + with pytest.raises(AblyException) as excinfo: + await channel.object.get() + + assert excinfo.value.code == 40024 + + +# UTS: objects/unit/RTO23e/get-reattaches-detached-0 +async def test_rto23e_get_reattaches_detached(): + client, channel, root, mock_ws = await setup_synced_channel('test', modes=(ChannelMode.OBJECT_SUBSCRIBE,)) + + await channel.detach() + assert channel.state == ChannelState.DETACHED + + root = await asyncio.wait_for(channel.object.get(), OPERATION_TIMEOUT) + + assert isinstance(root, PathObject) + assert root._path == [] + assert channel.state == ChannelState.ATTACHED + + +# UTS: objects/unit/RTO23c/get-waits-for-synced-0 +async def test_rto23c_get_waits_for_synced(): + mock_ws = standard_mock_websocket(attached_channel_serial='sync1:cursor', sync_objects=None) + client = objects_client(mock_ws) + channel = client.channels.get('test', objects_channel_options()) + + task = asyncio.ensure_future(channel.object.get()) + await _await_sent(mock_ws, ProtocolMessageAction.ATTACH, task) + + # The ATTACHED carries a sync cursor and no OBJECT_SYNC has arrived, so get() must still + # be waiting for SYNCED + await settle() + _assert_pending(task, 'get()') + + mock_ws.send_to_client(build_object_sync_message('test', 'sync1:', STANDARD_POOL_OBJECTS)) + root = await asyncio.wait_for(task, OPERATION_TIMEOUT) + + assert isinstance(root, PathObject) + assert root._path == [] + + +# UTS: objects/unit/RTO23c1/fails-on-channel-detached-0 +async def test_rto23c1_fails_on_channel_detached(): + client, channel, root, mock_ws = await setup_synced_channel('test') + await _restart_sync(channel, mock_ws) + + task = asyncio.ensure_future(channel.object.get()) + await settle() + _assert_pending(task, 'get()') + + await channel.detach() + + with pytest.raises(AblyException) as excinfo: + await asyncio.wait_for(task, OPERATION_TIMEOUT) + + assert excinfo.value.code == 92008 + assert excinfo.value.status_code == 400 + + +# UTS: objects/unit/RTO23c1/fails-on-channel-suspended-0 +async def test_rto23c1_fails_on_channel_suspended(): + client, channel, root, mock_ws = await setup_synced_channel('test') + await _restart_sync(channel, mock_ws) + + task = asyncio.ensure_future(channel.object.get()) + await settle() + _assert_pending(task, 'get()') + + # The mock cannot drive the channel to SUSPENDED, so the RealtimeObject's channel-state + # handler is called directly, as the RTO27 test does + channel.object._act_on_channel_state(ChannelState.SUSPENDED) + + with pytest.raises(AblyException) as excinfo: + await asyncio.wait_for(task, OPERATION_TIMEOUT) + + assert excinfo.value.code == 92008 + assert excinfo.value.status_code == 400 + + +# UTS: objects/unit/RTO23c1/fails-on-channel-failed-0 +async def test_rto23c1_fails_on_channel_failed(): + client, channel, root, mock_ws = await setup_synced_channel('test') + await _restart_sync(channel, mock_ws) + + task = asyncio.ensure_future(channel.object.get()) + await settle() + _assert_pending(task, 'get()') + + mock_ws.send_to_client(channel_error_message('test', 90000, 'Channel failed', 400)) + + with pytest.raises(AblyException) as excinfo: + await asyncio.wait_for(task, OPERATION_TIMEOUT) + + assert excinfo.value.code == 92008 + assert excinfo.value.status_code == 400 + # The cause is the channel's errorReason, the injected channel error + assert excinfo.value.cause.code == 90000 + + +# UTS: objects/unit/RTO15/publish-sends-object-pm-0 +async def test_rto15_publish_sends_object_pm(): + captured = [] + mock_ws = _acking_mock_websocket(lambda message: ['serial-0'], captured) + client, channel, root, mock_ws = await setup_synced_channel('test', mock_ws=mock_ws) + + await root.get('score').as_live_counter().increment(5) + + assert len(captured) == 1 + assert captured[0]['action'] == OBJECT + assert captured[0]['channel'] == 'test' + assert len(captured[0]['state']) == 1 + # RTO15e3: the state entry is the encoded ObjectMessage for the increment + operation = captured[0]['state'][0]['operation'] + assert operation['action'] == ObjectOperationAction.COUNTER_INC + assert operation['objectId'] == 'counter:score@1000' + assert operation['counterInc']['number'] == 5 + + +# UTS: objects/unit/RTO20/publish-and-apply-local-0 +async def test_rto20_publish_and_apply_local(): + client, channel, root, mock_ws = await setup_synced_channel('test') + + await root.get('score').as_live_counter().increment(10) + + assert _score(root) == 110 + + +# UTS: objects/unit/RTO20c/missing-site-code-0 +async def test_rto20c_missing_site_code(): + captured = [] + mock_ws = _acking_mock_websocket(lambda message: ['serial-0'], captured, + connected=objects_connected_message(site_code=None)) + client, channel, root, mock_ws = await setup_synced_channel('test', mock_ws=mock_ws) + + await root.get('score').as_live_counter().increment(10) + + # The increment was published and ACKed, so its not being applied is the siteCode's doing + assert len(captured) == 1 + assert _score(root) == 100 + + +# UTS: objects/unit/RTO20d1/null-serial-skipped-0 +async def test_rto20d1_null_serial_skipped(): + captured = [] + mock_ws = _acking_mock_websocket(lambda message: [None], captured) + client, channel, root, mock_ws = await setup_synced_channel('test', mock_ws=mock_ws) + + await root.get('score').as_live_counter().increment(10) + + # The increment was published and ACKed, so its not being applied is the null serial's doing + assert len(captured) == 1 + assert _score(root) == 100 + + +# UTS: objects/unit/RTO20d4/empty-synthetic-list-skips-sync-wait-0 +async def test_rto20d4_empty_synthetic_list_skips_sync_wait(): + mock_ws = _acking_mock_websocket(lambda message: [None]) + client, channel, root, mock_ws = await setup_synced_channel('test', mock_ws=mock_ws) + + # Back to SYNCING, where a write with something to apply would wait for SYNCED (RTO20e). + # No OBJECT_SYNC is ever sent, so the increment completing at all shows the wait was skipped. + await _restart_sync(channel, mock_ws) + + await asyncio.wait_for(root.get('score').as_live_counter().increment(10), OPERATION_TIMEOUT) + + assert channel.object._sync_state == ObjectsSyncState.SYNCING + assert _score(root) == 100 + + +# UTS: objects/unit/RTO20d4/mixed-null-serials-applies-non-null-0 +async def test_rto20d4_mixed_null_serials_applies_non_null(): + # The set() publishes [COUNTER_CREATE, MAP_SET] (RTLM20h1); the CREATE is ACKed with a + # null serial and skipped (RTO20d1), and the MAP_SET with a real one and applied + mock_ws = _acking_mock_websocket(lambda message: [None, ack_serial(message['msgSerial'], 1)]) + client, channel, root, mock_ws = await setup_synced_channel('test', mock_ws=mock_ws) + + await root.set('child', LiveCounter.create(0)) + + assert 'child' in root.keys() + # The seven standard-pool entries and `child` + assert root.size() == 8 + + +# UTS: objects/unit/RTO20e/waits-for-synced-0 +async def test_rto20e_waits_for_synced(): + client, channel, root, mock_ws = await setup_synced_channel('test') + await _restart_sync(channel, mock_ws) + + task = asyncio.ensure_future(root.get('score').as_live_counter().increment(10)) + + # The OBJECT is published and ACKed; while still SYNCING the increment must wait, unapplied + await _await_sent(mock_ws, ProtocolMessageAction.OBJECT, task) + await settle() + _assert_pending(task, 'increment()') + assert _score(root) == 100 + + mock_ws.send_to_client(build_object_sync_message('test', 'sync2:', STANDARD_POOL_OBJECTS)) + await asyncio.wait_for(task, OPERATION_TIMEOUT) + + assert _score(root) == 110 + + +# UTS: objects/unit/RTO20e1/fails-on-channel-detached-0 +async def test_rto20e1_fails_on_channel_detached(): + client, channel, root, mock_ws = await setup_synced_channel('test') + await _restart_sync(channel, mock_ws) + + task = asyncio.ensure_future(root.get('score').as_live_counter().increment(10)) + + # The publish and its ACK complete against the mock, and the increment waits for SYNCED + await _await_sent(mock_ws, ProtocolMessageAction.OBJECT, task) + await settle() + _assert_pending(task, 'increment()') + + await channel.detach() + + with pytest.raises(AblyException) as excinfo: + await asyncio.wait_for(task, OPERATION_TIMEOUT) + + assert excinfo.value.code == 92008 + # RTO20e1 gives the status code as well + assert excinfo.value.status_code == 400 + + +# UTS: objects/unit/RTO20e1/fails-on-channel-failed-0 +async def test_rto20e1_fails_on_channel_failed(): + client, channel, root, mock_ws = await setup_synced_channel('test') + await _restart_sync(channel, mock_ws) + + task = asyncio.ensure_future(root.get('score').as_live_counter().increment(10)) + + # The publish and its ACK complete against the mock, and the increment waits for SYNCED + await _await_sent(mock_ws, ProtocolMessageAction.OBJECT, task) + await settle() + _assert_pending(task, 'increment()') + + mock_ws.send_to_client(channel_error_message('test', 90000, 'Channel failed', 400)) + + with pytest.raises(AblyException) as excinfo: + await asyncio.wait_for(task, OPERATION_TIMEOUT) + + assert excinfo.value.code == 92008 + # RTO20e1 gives the status code, and the channel's errorReason as the cause + assert excinfo.value.status_code == 400 + assert excinfo.value.cause.code == 90000 + + +# UTS: objects/unit/RTO17/sync-state-events-0 +async def test_rto17_sync_state_events(): + mock_ws = standard_mock_websocket(attached_channel_serial='sync1:cursor', sync_objects=None) + client = objects_client(mock_ws) + channel = client.channels.get('test', objects_channel_options()) + + events = [] + channel.object.on(ObjectsEvent.SYNCING, lambda: events.append('SYNCING')) + channel.object.on(ObjectsEvent.SYNCED, lambda: events.append('SYNCED')) + + task = asyncio.ensure_future(channel.object.get()) + await poll_until(lambda: task.done() or len(events) >= 1, description='the SYNCING event') + _raise_if_failed(task) + + mock_ws.send_to_client(build_object_sync_message('test', 'sync1:', STANDARD_POOL_OBJECTS)) + await asyncio.wait_for(task, OPERATION_TIMEOUT) + + assert contains_in_order(events, ['SYNCING', 'SYNCED']), events + + +# UTS: objects/unit/RTO18d/duplicate-listener-0 +async def test_rto18d_duplicate_listener(): + client, channel, root, mock_ws = await setup_synced_channel('test') + calls = [] + + def listener(): + calls.append('SYNCED') + + channel.object.on(ObjectsEvent.SYNCED, listener) + channel.object.on(ObjectsEvent.SYNCED, listener) + + mock_ws.send_to_client(objects_attached_message('test', 'sync2:cursor')) + mock_ws.send_to_client(build_object_sync_message('test', 'sync2:', STANDARD_POOL_OBJECTS)) + await poll_until(lambda: len(calls) >= 2, description='the listener to be called twice') + await settle() + + assert len(calls) == 2 + + +# UTS: objects/unit/RTO19/off-deregisters-0 +async def test_rto19_off_deregisters(): + client, channel, root, mock_ws = await setup_synced_channel('test') + calls = [] + + def listener(): + calls.append('SYNCED') + + subscription = channel.object.on(ObjectsEvent.SYNCED, listener) + subscription.off() + + # A second listener on the same event is the control which shows the SYNCED was emitted + control = [] + channel.object.on(ObjectsEvent.SYNCED, lambda: control.append('SYNCED')) + + mock_ws.send_to_client(objects_attached_message('test', 'sync2:cursor')) + mock_ws.send_to_client(build_object_sync_message('test', 'sync2:', STANDARD_POOL_OBJECTS)) + await assert_unchanged_after_quiescence(lambda: len(calls), lambda: len(control) >= 1, + description='the control listener receives SYNCED') + + assert len(calls) == 0 + + +# UTS: objects/unit/RTO2/mode-enforcement-0 +async def test_rto2_mode_enforcement(): + # The server grants OBJECT_SUBSCRIBE only, where the channel requested both modes + mock_ws = standard_mock_websocket(attached_flags=HAS_OBJECTS | OBJECT_SUBSCRIBE_FLAG) + client, channel, root, mock_ws = await setup_synced_channel('test', mock_ws=mock_ws) + + with pytest.raises(AblyException) as excinfo: + await root.set('name', 'Bob') + + assert excinfo.value.code == 40024 + + +# UTS: objects/unit/RTO23e/get-rejects-failed-0 +async def test_rto23e_get_rejects_failed(): + mock_ws = None + + def on_message_from_client(message): + if message.get('action') == ATTACH: + mock_ws.send_to_client(channel_error_message(message.get('channel'), 90000, 'Channel error', 400)) + + mock_ws = MockWebSocket( + on_connection_attempt=lambda conn: conn.respond_with_success(objects_connected_message()), + on_message_from_client=on_message_from_client, + ) + client = objects_client(mock_ws) + channel = client.channels.get('test', objects_channel_options(ChannelMode.OBJECT_SUBSCRIBE)) + + # The attach fails, which leaves the channel FAILED + with pytest.raises(AblyException): + await channel.attach() + assert channel.state == ChannelState.FAILED + + with pytest.raises(AblyException) as excinfo: + await channel.object.get() + + assert excinfo.value.code == 90001 + assert excinfo.value.status_code == 400 + + +# UTS: objects/unit/RTO25a/access-requires-subscribe-mode-0 +async def test_rto25a_access_requires_subscribe_mode(): + mock_ws = standard_mock_websocket(attached_flags=HAS_OBJECTS | OBJECT_PUBLISH_FLAG) + client = objects_client(mock_ws) + channel = client.channels.get('test', objects_channel_options(ChannelMode.OBJECT_PUBLISH)) + + with pytest.raises(AblyException) as excinfo: + await channel.object.get() + + assert excinfo.value.code == 40024 + assert excinfo.value.status_code == 400 + + +# UTS: objects/unit/RTO25b/access-throws-detached-0 +async def test_rto25b_access_throws_detached(): + client, channel, root, mock_ws = await setup_synced_channel('test') + + await channel.detach() + assert channel.state == ChannelState.DETACHED + + with pytest.raises(AblyException) as excinfo: + root.keys() + + assert excinfo.value.code == 90001 + assert excinfo.value.status_code == 400 + + +# UTS: objects/unit/RTO25b/access-throws-failed-0 +async def test_rto25b_access_throws_failed(): + client, channel, root, mock_ws = await setup_synced_channel('test') + + mock_ws.send_to_client(channel_error_message('test', 90000, 'Channel error', 400)) + await await_channel_state(channel, ChannelState.FAILED) + + with pytest.raises(AblyException) as excinfo: + root.keys() + + assert excinfo.value.code == 90001 + assert excinfo.value.status_code == 400 + + +# UTS: objects/unit/RTO26a/write-requires-publish-mode-0 +async def test_rto26a_write_requires_publish_mode(): + mock_ws = standard_mock_websocket(attached_flags=HAS_OBJECTS | OBJECT_SUBSCRIBE_FLAG) + client, channel, root, mock_ws = await setup_synced_channel('test', mock_ws=mock_ws, + modes=(ChannelMode.OBJECT_SUBSCRIBE,)) + + with pytest.raises(AblyException) as excinfo: + await root.set('name', 'Bob') + + assert excinfo.value.code == 40024 + assert excinfo.value.status_code == 400 + + +# UTS: objects/unit/RTO26b/write-throws-detached-0 +async def test_rto26b_write_throws_detached(): + client, channel, root, mock_ws = await setup_synced_channel('test') + + await channel.detach() + assert channel.state == ChannelState.DETACHED + + with pytest.raises(AblyException) as excinfo: + await root.set('name', 'Bob') + + assert excinfo.value.code == 90001 + assert excinfo.value.status_code == 400 + + +# UTS: objects/unit/RTO26b/write-throws-failed-0 +async def test_rto26b_write_throws_failed(): + client, channel, root, mock_ws = await setup_synced_channel('test') + + mock_ws.send_to_client(channel_error_message('test', 90000, 'Channel error', 400)) + await await_channel_state(channel, ChannelState.FAILED) + + with pytest.raises(AblyException) as excinfo: + await root.set('name', 'Bob') + + assert excinfo.value.code == 90001 + assert excinfo.value.status_code == 400 + + +# UTS: objects/unit/RTO26c/write-throws-echo-disabled-0 +async def test_rto26c_write_throws_echo_disabled(): + client, channel, root, mock_ws = await setup_synced_channel('test', echo_messages=False) + + with pytest.raises(AblyException) as excinfo: + await root.set('name', 'Bob') + + assert excinfo.value.code == 40000 + assert excinfo.value.status_code == 400 + + +# UTS: objects/unit/RTO24a/single-register-instance-0 +async def test_rto24a_single_register_instance(): + client, channel, root, mock_ws = await setup_synced_channel('test') + events_root = [] + events_score = [] + + root.subscribe(events_root.append) + score_path = root.get('score') + score_path.subscribe(events_score.append) + + # `remote` has no entry in the counter's siteTimeserials, so the increment is new (RTLO4a) + mock_ws.send_to_client(build_object_message('test', [ + build_counter_inc('counter:score@1000', 5, 't:1', 'remote'), + ])) + await poll_until(lambda: len(events_score) >= 1, description='the score subscription to fire') + + # Both subscriptions are held by the one register, so both fire + assert len(events_root) >= 1 + assert len(events_score) >= 1 + + +# UTS: objects/unit/RTO24c1/coverage-prefix-depth-0 +async def test_rto24c1_coverage_prefix_depth(): + client, channel, root, mock_ws = await setup_synced_channel('test') + shallow_events = [] + deep_events = [] + + # With depth 1 a subscription at the root covers the root's own path only; a child such + # as ['score'] is at relative depth 2 (RTO24c2b) + root.subscribe(shallow_events.append, depth=1) + root.subscribe(deep_events.append) + + # A MAP_SET on the root itself, at path [] + mock_ws.send_to_client(build_object_message('test', [ + build_map_set('root', 'name', {'string': 'Bob'}, remote_serial(0), 'remote'), + ])) + await poll_until(lambda: len(deep_events) >= 1, description='the deep subscription to see the root update') + + # An increment on the root's child, at path ['score'] + mock_ws.send_to_client(build_object_message('test', [ + build_counter_inc('counter:score@1000', 5, 't:2', 'remote'), + ])) + await poll_until(lambda: len(deep_events) >= 2, description='the deep subscription to see the child update') + await poll_until(lambda: len(shallow_events) >= 1, + description='the shallow subscription to see the root update') + await settle() + + # The shallow subscription sees the root update only; the deep one both + assert len(shallow_events) == 1 + assert len(deep_events) >= 2 + + +# UTS: objects/unit/RTO10/gc-tombstoned-objects-0 +async def test_rto10_gc_tombstoned_objects(): + clock = FakeClock() + client, channel, root, mock_ws = await setup_synced_channel('test', clock=clock) + pool = channel.object._objects_pool + + # A tombstone stamped now, which only the advance below makes eligible for collection + mock_ws.send_to_client(build_object_message('test', [ + build_object_delete('counter:score@1000', '99', 'site1', clock.now_ms()), + ])) + await poll_until(lambda: pool['counter:score@1000'].is_tombstone, + description='the OBJECT_DELETE to tombstone the score counter') + + await clock.advance(DEFAULT_GC_GRACE_PERIOD_MS + DEFAULT_GC_INTERVAL_MS) + + assert _score(root) is None + # UTS SPEC ERROR: the specification asserts only that `score` reads null, which holds + # from the moment the counter is tombstoned (RTLM14c, RTLM5d2h), with or without a GC + # sweep. The sweep itself is observed as the counter leaving the pool (RTO10c1b). + assert 'counter:score@1000' not in pool + + +# UTS: objects/unit/RTO10c1b1/gc-root-never-removed-0 +async def test_rto10c1b1_gc_root_never_removed(): + clock = FakeClock() + client, channel, root, mock_ws = await setup_synced_channel('test', clock=clock) + pool = channel.object._objects_pool + root_map = pool['root'] + + # A rogue OBJECT_DELETE targeting the root, which RTLO4e10 rejects + mock_ws.send_to_client(build_object_message('test', [ + build_object_delete('root', remote_serial(0), 'remote', clock.now_ms()), + ])) + # The score counter is tombstoned behind it: the control which shows the rogue delete has + # been processed, and, once collected, that a GC sweep has run past the grace period + mock_ws.send_to_client(build_object_message('test', [ + build_object_delete('counter:score@1000', '99', 'site1', clock.now_ms()), + ])) + await poll_until(lambda: pool['counter:score@1000'].is_tombstone, + description='the control OBJECT_DELETE to tombstone the score counter') + + # The root is not tombstoned and its data is untouched + assert _name(root) == 'Alice' + assert root_map.is_tombstone is False + + await clock.advance(DEFAULT_GC_GRACE_PERIOD_MS + DEFAULT_GC_INTERVAL_MS) + + assert 'counter:score@1000' not in pool + assert pool['root'] is root_map + + # The root is still live: an operation still applies to the root object the client holds + mock_ws.send_to_client(build_object_message('test', [ + build_map_set('root', 'name', {'string': 'Bob'}, remote_serial(1), 'remote'), + ])) + await poll_until(lambda: _name(root) == 'Bob', description='the MAP_SET to apply to the root') + + assert _name(root) == 'Bob' + + +# UTS: objects/unit/RTO20/echo-dedup-0 +async def test_rto20_echo_dedup(): + client, channel, root, mock_ws = await setup_synced_channel('test') + + await root.get('score').as_live_counter().increment(10) + score_after_apply = _score(root) + + # The echo of the increment, carrying the serial it was ACKed and applied with + mock_ws.send_to_client(build_object_message('test', [ + build_counter_inc('counter:score@1000', 10, ack_serial(0, 0), SITE_CODE), + ])) + # A control behind the echo on the same connection, which shows the echo has been processed + mock_ws.send_to_client(build_object_message('test', [ + build_counter_inc('counter:nested@1000', 1, remote_serial(0), 'remote'), + ])) + await assert_unchanged_after_quiescence(lambda: _score(root), lambda: _nested_counter(root) == 6, + description='the control increment has applied') + score_after_echo = _score(root) + + assert score_after_apply == 110 + assert score_after_echo == 110 + + +# UTS: objects/unit/RTO20f/ack-no-site-timeserials-update-0 +async def test_rto20f_ack_no_site_timeserials_update(): + client, channel, root, mock_ws = await setup_synced_channel('test') + + await root.get('score').as_live_counter().increment(10) + assert _score(root) == 110 + + # `below_ack_serial(9)` is not the ACK serial, so the echo dedup (RTO9a3) leaves it alone, + # and it sorts below `ack_serial(0, 0)`, so it is rejected as stale (RTLO4a) if and only if + # the LOCAL apply recorded the ACK serial in siteTimeserials + mock_ws.send_to_client(build_object_message('test', [ + build_counter_inc('counter:score@1000', 10, below_ack_serial(9), SITE_CODE), + ])) + await poll_until(lambda: _score(root) == 120, description='the inbound increment to apply') + + assert _score(root) == 120 + + +# UTS: objects/unit/RTO20/ack-after-echo-no-double-apply-0 +async def test_rto20_ack_after_echo_no_double_apply(): + client, channel, root, mock_ws = await setup_synced_channel_no_ack('test') + + task = asyncio.ensure_future(root.get('score').as_live_counter().increment(10)) + + # The OBJECT must be pending on the connection before its ACK can be matched to it + await _await_sent(mock_ws, ProtocolMessageAction.OBJECT, task) + + # The echo arrives before the ACK + mock_ws.send_to_client(build_object_message('test', [ + build_counter_inc('counter:score@1000', 10, ack_serial(0, 0), SITE_CODE), + ])) + mock_ws.send_to_client(build_ack_message(0, [ack_serial(0, 0)])) + + await asyncio.wait_for(task, OPERATION_TIMEOUT) + + assert _score(root) == 110 + + +# UTS: objects/unit/RTO5c9-RTO20/ack-serials-cleared-on-resync-0 +async def test_rto5c9_rto20_ack_serials_cleared_on_resync(): + client, channel, root, mock_ws = await setup_synced_channel('test') + + await root.get('score').as_live_counter().increment(10) + assert _score(root) == 110 + + # A re-sync replaces the data and clears appliedOnAckSerials (RTO5c9) + mock_ws.send_to_client(objects_attached_message('test', 'sync2:cursor')) + mock_ws.send_to_client(build_object_sync_message('test', 'sync2:', STANDARD_POOL_OBJECTS)) + await poll_until(lambda: _score(root) == 100, description='the re-sync to replace the score') + assert _score(root) == 100 + + # The serial the increment was applied with on ACK applies normally once the set is cleared; + # were it still recorded, the dedup (RTO9a3) would discard it and the score would stay 100 + mock_ws.send_to_client(build_object_message('test', [ + build_counter_inc('counter:score@1000', 10, ack_serial(0, 0), SITE_CODE), + ])) + await poll_until(lambda: _score(root) == 110, description='the replayed increment to apply') + + assert _score(root) == 110 + + +# UTS: objects/unit/RTO20/subscription-fires-on-ack-apply-0 +async def test_rto20_subscription_fires_on_ack_apply(): + client, channel, root, mock_ws = await setup_synced_channel('test') + events = [] + root.get('score').subscribe(events.append) + + await root.get('score').as_live_counter().increment(10) + + assert len(events) >= 1 + assert _score(root) == 110 + + +# UTS: objects/unit/RTO23/get-implicit-attach-0 +async def test_rto23_get_implicit_attach(): + mock_ws = standard_mock_websocket() + client = objects_client(mock_ws) + channel = client.channels.get('test', objects_channel_options()) + + assert channel.state == ChannelState.INITIALIZED + root = await asyncio.wait_for(channel.object.get(), OPERATION_TIMEOUT) + + assert isinstance(root, PathObject) + assert root._path == [] + assert channel.state == ChannelState.ATTACHED + + +# UTS: objects/unit/RTO23d/get-resolves-immediately-synced-0 +async def test_rto23d_get_resolves_immediately_synced(): + client, channel, root, mock_ws = await setup_synced_channel('test') + + root2 = await asyncio.wait_for(channel.object.get(), OPERATION_TIMEOUT) + + assert isinstance(root2, PathObject) + assert root2._path == [] + + +# UTS: objects/unit/RTO10b1/gc-grace-period-source-0 +async def test_rto10b1_gc_grace_period_source(): + clock = FakeClock() + mock_ws = standard_mock_websocket(connected=objects_connected_message(objects_gc_grace_period=5000)) + client = objects_client(mock_ws, clock=clock) + channel = client.channels.get('test', objects_channel_options()) + # UTS SPEC ERROR: the specification advances 6000 ms, which RTO10a's example interval of + # five minutes never sweeps within, so as written no GC check runs at all. The interval is + # shortened before the first ATTACHED schedules the GC timer, so that sweeps fall inside it. + channel.object._gc_interval_ms = 1000 + root = await channel.object.get() + pool = channel.object._objects_pool + + assert channel.object._gc_grace_period_ms == 5000 + + # A tombstone stamped now: eligible after 6000 ms under the 5000 ms grace period from + # ConnectionDetails, but not under the 24 hour default + mock_ws.send_to_client(build_object_message('test', [ + build_object_delete('counter:score@1000', '99', 'site1', clock.now_ms()), + ])) + await poll_until(lambda: pool['counter:score@1000'].is_tombstone, + description='the OBJECT_DELETE to tombstone the score counter') + + # The advance is taken in two steps so that the object is seen to survive the sweeps that + # fall inside the grace period (RTO10c1b) + await clock.advance(4000) + assert 'counter:score@1000' in pool + + await clock.advance(2000) + + assert _score(root) is None + # UTS SPEC ERROR: as in RTO10, `score` reads null from the moment the counter is + # tombstoned; the collection is observed as the counter leaving the pool + assert 'counter:score@1000' not in pool + + +# UTS: objects/unit/RTO17-RTO18/sync-event-sequences-0 +@pytest.mark.parametrize('scenario', [ + 'initial attach', + 're-sync on new ATTACHED', + 'ATTACHED without HAS_OBJECTS', +]) +async def test_rto17_rto18_sync_event_sequences(scenario): + if scenario == 'initial attach': + # A first attach is only observable on a channel that has not attached, with the + # listeners registered before attach() is called + mock_ws = standard_mock_websocket() + client = objects_client(mock_ws) + channel = client.channels.get('test', objects_channel_options()) + else: + client, channel, root, mock_ws = await setup_synced_channel('test') + + events = [] + channel.object.on(ObjectsEvent.SYNCING, lambda: events.append('SYNCING')) + channel.object.on(ObjectsEvent.SYNCED, lambda: events.append('SYNCED')) + + if scenario == 'initial attach': + await channel.attach() + elif scenario == 're-sync on new ATTACHED': + mock_ws.send_to_client(objects_attached_message('test', 'sync3:cursor')) + mock_ws.send_to_client(build_object_sync_message('test', 'sync3:', STANDARD_POOL_OBJECTS)) + else: + # RTO4c moves the sync state to SYNCING on any ATTACHED, and with no HAS_OBJECTS the + # sync completes at once (RTO4b4), moving it to SYNCED + mock_ws.send_to_client(objects_attached_message('test', 'sync4:', flags=0)) + + expected_events = ['SYNCING', 'SYNCED'] + await poll_until(lambda: len(events) >= len(expected_events), description=f'the {scenario} events') + await settle() + + assert events == expected_events + + +# UTS: objects/unit/RTO27/channel-state-data-lifecycle-0 +@pytest.mark.parametrize('state', [ChannelState.DETACHED, ChannelState.FAILED, ChannelState.SUSPENDED], + ids=['RTO27a-detached', 'RTO27a-failed', 'RTO27b-suspended']) +async def test_rto27_channel_state_data_lifecycle(state): + client, channel, root, mock_ws = await setup_synced_channel('test') + pool = channel.object._objects_pool + + # The standard pool has been materialised: the root map, a counter and a nested map + assert 'name' in pool['root'].data + assert pool['counter:score@1000'].value() == 100 + assert 'email' in pool['map:profile@1000'].data + + updates = [] + pool['root'].subscribe(updates.append) + pool['counter:score@1000'].subscribe(updates.append) + pool['map:profile@1000'].subscribe(updates.append) + + # The mock cannot drive every state, so the channel-state handler is called directly + channel.object._act_on_channel_state(state) + + if state == ChannelState.SUSPENDED: + # RTO27b: the data is retained unchanged + assert 'name' in pool['root'].data + assert pool['counter:score@1000'].value() == 100 + assert 'email' in pool['map:profile@1000'].data + else: + # RTO27a1: every object's data is cleared, the nested map's checked through the pool + # rather than through the root, while the objects themselves stay in the pool + assert pool['root'].data == {} + assert 'counter:score@1000' in pool + assert pool['counter:score@1000'].value() == 0 + assert 'map:profile@1000' in pool + assert pool['map:profile@1000'].data == {} + + # RTO27a1: and no update is emitted for the clear + await settle() + assert updates == [] diff --git a/test/uts/objects/unit/value_types_test.py b/test/uts/objects/unit/value_types_test.py new file mode 100644 index 00000000..a424b108 --- /dev/null +++ b/test/uts/objects/unit/value_types_test.py @@ -0,0 +1,291 @@ +"""Derived from uts/objects/unit/value_types.md in ably/specification. + +Spec points: RTLCV1, RTLCV2, RTLCV3, RTLCV3a1, RTLCV3b, RTLCV3c, RTLCV3d, RTLCV4, RTLCV4a, +RTLCV4b1, RTLCV4c, RTLCV4d, RTLCV4f, RTLCV4g1, RTLCV4g2, RTLCV4g3, RTLCV4g4, RTLCV4g5, +RTLMV1, RTLMV2, RTLMV3, RTLMV3a1, RTLMV3b, RTLMV3d, RTLMV4, RTLMV4a, RTLMV4b, RTLMV4c, +RTLMV4d, RTLMV4d1, RTLMV4d2, RTLMV4d3, RTLMV4d4, RTLMV4d5, RTLMV4d6, RTLMV4d7, RTLMV4e1, +RTLMV4e2, RTLMV4f, RTLMV4g, RTLMV4i, RTLMV4j1, RTLMV4j3, RTLMV4j4, RTLMV4j5, RTLMV4k + +Pure tests: no client and no mock. The blueprints' internal `count` and `entries` are +`_count` and `_entries`. The specification's `evaluate(vt)` is `evaluate(vt, timestamp_ms)`: +evaluation reads the server time (RTLCV4e, RTLMV4h), which the caller fetches and passes +in (S-3), so these tests pass a fixed one. + +Where the specification reads `operation.counterCreate` or `operation.mapCreate` of an +evaluated message, it means the create retained alongside the `*CreateWithObjectId` +payload (RTLCV4g5, RTLMV4j5). The evaluated operation carries only the +`*CreateWithObjectId` payload, and the retained create is read through +`resolved_counter_create` / `resolved_map_create`, its `derived_from` (S-4). Its entries +hold decoded values, so a `bytes` entry holds bytes; the base64 form is in the +`initialValue` JSON, which is encoded for the JSON wire (RTLMV4f1). +""" + +import json + +import pytest + +from ably.pubsub.objects.objectmessage import ObjectOperationAction, ObjectsMapSemantics +from ably.pubsub.objects.valuetypes import LiveCounter, LiveMap, evaluate +from ably.pubsub.util.exceptions import AblyException + +# The server time evaluation generates object ids with (S-3) +SERVER_TIME_MS = 1_700_000_000_000 + + +# UTS: objects/unit/RTLCV3/create-with-count-0 +def test_rtlcv3_create_with_count(): + vt = LiveCounter.create(42) + + assert isinstance(vt, LiveCounter) + assert vt._count == 42 + + +# UTS: objects/unit/RTLCV3/create-default-zero-0 +def test_rtlcv3_create_default_zero(): + vt = LiveCounter.create() + + assert vt._count == 0 + + +# UTS: objects/unit/RTLCV3c/no-validation-at-create-0 +def test_rtlcv3c_no_validation_at_create(): + # Does not raise + vt = LiveCounter.create('not_a_number') + + assert isinstance(vt, LiveCounter) + + +# UTS: objects/unit/RTLCV4/evaluate-generates-message-0 +def test_rtlcv4_evaluate_generates_message(): + vt = LiveCounter.create(42) + messages = evaluate(vt, SERVER_TIME_MS) + + assert len(messages) == 1 + msg = messages[0] + assert msg.operation.action == ObjectOperationAction.COUNTER_CREATE + assert msg.operation.object_id.startswith('counter:') + assert '@' in msg.operation.object_id + assert msg.operation.counter_create_with_object_id is not None + assert msg.operation.counter_create_with_object_id.nonce is not None + assert len(msg.operation.counter_create_with_object_id.nonce) >= 16 + assert msg.operation.counter_create_with_object_id.initial_value is not None + # RTLCV4c: the initial value is the JSON of the CounterCreate + assert json.loads(msg.operation.counter_create_with_object_id.initial_value) == {'count': 42} + + +# UTS: objects/unit/RTLCV4g5/retains-local-counter-create-0 +def test_rtlcv4g5_retains_local_counter_create(): + vt = LiveCounter.create(42) + messages = evaluate(vt, SERVER_TIME_MS) + + msg = messages[0] + # S-4: the specification's `operation.counterCreate` is the retained CounterCreate + assert msg.operation.resolved_counter_create is not None + assert msg.operation.resolved_counter_create.count == 42 + # The retained CounterCreate is local only and never sent (RTLCV4g5) + assert 'counterCreate' not in msg.operation.to_dict() + + +# UTS: objects/unit/RTLCV4a/evaluate-validates-count-0 +def test_rtlcv4a_evaluate_validates_count(): + vt = LiveCounter.create('not_a_number') + + with pytest.raises(AblyException) as excinfo: + evaluate(vt, SERVER_TIME_MS) + + assert excinfo.value.code == 40003 + + +# UTS: objects/unit/RTLCV4/evaluate-zero-count-0 +def test_rtlcv4_evaluate_zero_count(): + vt = LiveCounter.create(0) + messages = evaluate(vt, SERVER_TIME_MS) + + msg = messages[0] + # S-4: the retained CounterCreate + assert msg.operation.resolved_counter_create.count == 0 + + +# UTS: objects/unit/RTLMV3/create-with-entries-0 +def test_rtlmv3_create_with_entries(): + vt = LiveMap.create({ + 'name': 'Alice', + 'age': 30, + }) + + assert isinstance(vt, LiveMap) + assert vt._entries['name'] == 'Alice' + assert vt._entries['age'] == 30 + + +# UTS: objects/unit/RTLMV3/create-no-entries-0 +def test_rtlmv3_create_no_entries(): + vt = LiveMap.create() + + assert isinstance(vt, LiveMap) + + +# UTS: objects/unit/RTLMV4/evaluate-generates-message-0 +def test_rtlmv4_evaluate_generates_message(): + vt = LiveMap.create({'name': 'Alice'}) + messages = evaluate(vt, SERVER_TIME_MS) + + assert len(messages) == 1 + msg = messages[0] + assert msg.operation.action == ObjectOperationAction.MAP_CREATE + assert msg.operation.object_id.startswith('map:') + assert msg.operation.map_create_with_object_id is not None + assert len(msg.operation.map_create_with_object_id.nonce) >= 16 + assert msg.operation.map_create_with_object_id.initial_value is not None + # RTLMV4f: the initial value is the JSON of the encoded MapCreate + initial_value = json.loads(msg.operation.map_create_with_object_id.initial_value) + assert initial_value['semantics'] == ObjectsMapSemantics.LWW + assert initial_value['entries']['name']['data'] == {'string': 'Alice'} + + +# UTS: objects/unit/RTLMV4j5/retains-local-map-create-0 +def test_rtlmv4j5_retains_local_map_create(): + vt = LiveMap.create({'name': 'Alice'}) + messages = evaluate(vt, SERVER_TIME_MS) + + msg = messages[0] + # S-4: the specification's `operation.mapCreate` is the retained MapCreate + assert msg.operation.resolved_map_create is not None + assert msg.operation.resolved_map_create.semantics == ObjectsMapSemantics.LWW + assert msg.operation.resolved_map_create.entries['name'].data.string == 'Alice' + # The retained MapCreate is local only and never sent (RTLMV4j5) + assert 'mapCreate' not in msg.operation.to_dict() + + +# UTS: objects/unit/RTLMV4d/entry-value-types-0 +def test_rtlmv4d_entry_value_types(): + vt = LiveMap.create({ + 'str': 'hello', + 'num': 42, + 'bool': True, + 'json_arr': [1, 2, 3], + 'json_obj': {'key': 'value'}, + }) + messages = evaluate(vt, SERVER_TIME_MS) + + msg = messages[0] + # S-4: the retained MapCreate + entries = msg.operation.resolved_map_create.entries + assert entries['str'].data.string == 'hello' + assert entries['num'].data.number == 42 + # `bool` is an `int` in Python, so a boolean is told apart from a number by identity + assert entries['bool'].data.boolean is True + assert entries['json_arr'].data.json == [1, 2, 3] + assert entries['json_obj'].data.json == {'key': 'value'} + + +# UTS: objects/unit/RTLMV4d1/nested-value-types-0 +def test_rtlmv4d1_nested_value_types(): + inner_counter = LiveCounter.create(10) + inner_map = LiveMap.create({ + 'nested_count': inner_counter, + }) + outer = LiveMap.create({ + 'child': inner_map, + }) + messages = evaluate(outer, SERVER_TIME_MS) + + assert len(messages) == 3 + assert messages[0].operation.action == ObjectOperationAction.COUNTER_CREATE + assert messages[0].operation.object_id.startswith('counter:') + assert messages[1].operation.action == ObjectOperationAction.MAP_CREATE + assert messages[1].operation.object_id.startswith('map:') + assert messages[2].operation.action == ObjectOperationAction.MAP_CREATE + assert messages[2].operation.object_id.startswith('map:') + + inner_counter_id = messages[0].operation.object_id + inner_map_id = messages[1].operation.object_id + + # S-4: the retained MapCreates + assert messages[1].operation.resolved_map_create.entries['nested_count'].data.object_id == inner_counter_id + assert messages[2].operation.resolved_map_create.entries['child'].data.object_id == inner_map_id + + +# UTS: objects/unit/RTLMV4a/evaluate-validates-entries-0 +def test_rtlmv4a_evaluate_validates_entries(): + # The specification's input is `LiveMap.create(null)`. `LiveMap.create` takes None as its + # default, so `None` cannot be told apart from an omitted argument and the null input is + # not applicable. The other half of RTLMV4a, entries that are not a dict, is the + # reachable failure. + vt = LiveMap.create(['not', 'a', 'dict']) + + with pytest.raises(AblyException) as excinfo: + evaluate(vt, SERVER_TIME_MS) + + assert excinfo.value.code == 40003 + + # `None` is the omitted argument, and evaluates as an empty map (RTLMV4e2) + messages = evaluate(LiveMap.create(None), SERVER_TIME_MS) + assert messages[-1].operation.resolved_map_create.entries == {} + + +# UTS: objects/unit/RTLMV4b/evaluate-validates-keys-0 +def test_rtlmv4b_evaluate_validates_keys(): + vt = LiveMap.create({123: 'value'}) + + with pytest.raises(AblyException) as excinfo: + evaluate(vt, SERVER_TIME_MS) + + assert excinfo.value.code == 40003 + + +# UTS: objects/unit/RTLMV4c/evaluate-validates-values-0 +def test_rtlmv4c_evaluate_validates_values(): + vt = LiveMap.create({'fn': lambda: None}) + + with pytest.raises(AblyException) as excinfo: + evaluate(vt, SERVER_TIME_MS) + + assert excinfo.value.code == 40013 + + +# UTS: objects/unit/RTLMV4e2/empty-entries-0 +def test_rtlmv4e2_empty_entries(): + vt = LiveMap.create() + messages = evaluate(vt, SERVER_TIME_MS) + + msg = messages[0] + # S-4: the retained MapCreate + assert msg.operation.resolved_map_create.entries == {} + + +# The specification's `expected_value` for the bytes row, "AQID", is the JSON-wire form of +# bytes([1, 2, 3]). The retained MapCreate holds the decoded bytes, and the base64 is what the +# `initialValue` JSON carries, so the test asserts both. +MAP_SET_TYPE_SCENARIOS = [ + pytest.param('hello', 'string', 'hello', id='string'), + pytest.param(42, 'number', 42, id='number-42'), + pytest.param(3.14, 'number', 3.14, id='number-3.14'), + pytest.param(0, 'number', 0, id='number-0'), + pytest.param(-1, 'number', -1, id='number-minus-1'), + pytest.param(True, 'boolean', True, id='boolean-true'), + pytest.param(False, 'boolean', False, id='boolean-false'), + pytest.param([1, 'a', None], 'json', [1, 'a', None], id='json-array'), + pytest.param({'k': 'v'}, 'json', {'k': 'v'}, id='json-object'), + pytest.param(bytes([1, 2, 3]), 'bytes', bytes([1, 2, 3]), id='bytes'), +] + + +# UTS: objects/unit/RTLMV4d/map-set-all-types-table-0 +@pytest.mark.parametrize('value,expected_field,expected_value', MAP_SET_TYPE_SCENARIOS) +def test_rtlmv4d_map_set_all_types_table(value, expected_field, expected_value): + vt = LiveMap.create({'test_key': value}) + messages = evaluate(vt, SERVER_TIME_MS) + + # S-4: the retained MapCreate + entry = messages[0].operation.resolved_map_create.entries['test_key'] + actual = getattr(entry.data, expected_field) + if isinstance(expected_value, bool): + # `bool` is an `int` in Python, so a boolean is told apart from a number by identity + assert actual is expected_value + else: + assert actual == expected_value + + if expected_field == 'bytes': + initial_value = json.loads(messages[0].operation.map_create_with_object_id.initial_value) + assert initial_value['entries']['test_key']['data']['bytes'] == 'AQID' From 580a9809e053d55f7bc9f7810bfbd2f3b70675c3 Mon Sep 17 00:00:00 2001 From: owenpearson Date: Fri, 9 Oct 2026 13:09:28 +0100 Subject: [PATCH 14/71] test: derive the LiveObjects integration specifications Derives the three sandbox specifications under uts/objects/integration and the proxy specification under uts/objects/integration/proxy, 17 tests. The sandbox ones run under both protocols, against an app of their own; the proxy ones authenticate with a locally signed JWT, as the realtime proxy tier does, and fault the objects sync through uts-proxy. Co-Authored-By: Claude Opus 5.5 (1M context) --- test/uts/objects/integration/__init__.py | 0 test/uts/objects/integration/conftest.py | 61 +++ .../objects/integration/objects_gc_test.py | 109 +++++ .../integration/objects_lifecycle_test.py | 167 +++++++ .../objects/integration/objects_sync_test.py | 123 +++++ .../uts/objects/integration/proxy/__init__.py | 0 .../uts/objects/integration/proxy/conftest.py | 78 +++ .../integration/proxy/objects_faults_test.py | 450 ++++++++++++++++++ 8 files changed, 988 insertions(+) create mode 100644 test/uts/objects/integration/__init__.py create mode 100644 test/uts/objects/integration/conftest.py create mode 100644 test/uts/objects/integration/objects_gc_test.py create mode 100644 test/uts/objects/integration/objects_lifecycle_test.py create mode 100644 test/uts/objects/integration/objects_sync_test.py create mode 100644 test/uts/objects/integration/proxy/__init__.py create mode 100644 test/uts/objects/integration/proxy/conftest.py create mode 100644 test/uts/objects/integration/proxy/objects_faults_test.py diff --git a/test/uts/objects/integration/__init__.py b/test/uts/objects/integration/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/test/uts/objects/integration/conftest.py b/test/uts/objects/integration/conftest.py new file mode 100644 index 00000000..62ee721c --- /dev/null +++ b/test/uts/objects/integration/conftest.py @@ -0,0 +1,61 @@ +"""Fixtures the LiveObjects integration specifications share. + +The sandbox app is provisioned once and read by every test that asks for it, as +it is for the realtime integration tier: each specification's `BEFORE ALL TESTS` +provisions an app, and provisioning one per file would make the tier slower and +invite the sandbox's rate limiting. The fixtures in +`test/uts/realtime/integration/conftest.py` are not visible from this package, +so the same shape is defined here; the app is a separate one from the realtime +tier's, so that a channel one tier writes objects to is never one the other +reads. +""" + +import os + +import pytest +import pytest_asyncio + +from test.uts.helpers.sandbox import delete_app, provision_app + +# `integration-testing.md` puts a suite of this size at 120 seconds. A LiveObjects +# test opens one or two connections, attaches, waits for the objects sync and then +# waits again for a mutation to reach a second client, each bounded at 10 to 30 +# seconds. The repository default in `pyproject.toml` is 30 seconds, which suits a +# test served from a mock; the marker applies to this package alone. +SUITE_TIMEOUT = 120 + +__package_dir = os.path.dirname(os.path.abspath(__file__)) + + +def pytest_collection_modifyitems(items): + for item in items: + if os.path.abspath(str(item.fspath)).startswith(__package_dir + os.sep): + item.add_marker(pytest.mark.timeout(SUITE_TIMEOUT)) + + +@pytest_asyncio.fixture(scope='session') +async def realtime_sandbox(): + """The provisioned sandbox app, as a specification's `app_config`. + + This is the specifications' `BEFORE ALL TESTS` / `AFTER ALL TESTS` pair. + `realtime_sandbox.key_str` is the full-access key they call `api_key`. + """ + app = await provision_app() + yield app + await delete_app(app) + + +@pytest.fixture(params=[False, True], ids=['json', 'msgpack']) +def use_binary_protocol(request): + """Runs the test once per protocol, which is the specifications' `PROTOCOL`. + + The three specifications directly in this package carry a + `## Protocol Variants` section, so their tests run against both json and + msgpack and pass this straight to every client they build: + + client = sandbox_realtime_client(api_key, use_binary_protocol=use_binary_protocol) + + The proxy specification has no such section and runs json only, which the + proxy requires in any case. + """ + return request.param diff --git a/test/uts/objects/integration/objects_gc_test.py b/test/uts/objects/integration/objects_gc_test.py new file mode 100644 index 00000000..416cb31c --- /dev/null +++ b/test/uts/objects/integration/objects_gc_test.py @@ -0,0 +1,109 @@ +"""Derived from uts/objects/integration/objects_gc_test.md in ably/specification. + +Spec points: RTO10, RTLM19, RTLM5d2h, RTLM7 + +Tombstone semantics end to end against the sandbox: removing a map entry tombstones it +(RTLM7), a tombstoned entry reads back as absent (RTLM5d2h), and the key can be set again, +with a fresh object id where the new value is an object (RTO10, RTLM19). Only the public +API is used. + +The GC sweep itself is not exercised here, as the specification's Scope section says: its +five-minute cadence and the server's grace period are not observable within an +integration test's budget, so the sweep is covered at the unit tier against a `FakeClock`. +Every wait here is on real time. + +The specification carries a `## Protocol Variants` section, so every test runs once per +protocol and passes `use_binary_protocol` to the client it builds. + +The specification's `value() == null` is `None` in Python. A counter's `value()` is reached +through `as_live_counter()` and a primitive's through `as_primitive()`, since ably-python's +`PathObject` is partitioned by type (RTTS); either view answers `None` once the path no +longer resolves (RTTS5d1), and each test has already seen a value at that path, so the +`None` is the removal rather than a value that never arrived. Clients are closed by the +suite's fixture. +""" + +import asyncio + +from ably.pubsub.realtime.connection import ConnectionState +from ably.pubsub.server import LiveCounter +from test.uts.helpers.client import await_connection_state, sandbox_realtime_client, wall_clock_poll_until +from test.uts.helpers.sandbox import random_id +from test.uts.objects.helpers.standard_test_pool import objects_channel_options + +# The specification's `AWAIT_STATE ... WITH timeout` and `AWAIT channel.object.get() WITH +# timeout`, in seconds. +CONNECT_TIMEOUT = 15.0 +SYNC_TIMEOUT = 15.0 + + +async def synced_root(api_key, channel_name, use_binary_protocol): + """The setup both tests share: a CONNECTED client and the synced root of `channel_name`.""" + client = sandbox_realtime_client(api_key, auto_connect=False, use_binary_protocol=use_binary_protocol) + client.connect() + await await_connection_state(client, ConnectionState.CONNECTED, timeout=CONNECT_TIMEOUT) + + channel = client.channels.get(channel_name, objects_channel_options()) + try: + return await asyncio.wait_for(channel.object.get(), SYNC_TIMEOUT) + except asyncio.TimeoutError: + raise AssertionError( + f'Timed out after {SYNC_TIMEOUT}s waiting for channel.object.get() on {channel_name!r}') from None + + +# UTS: objects/integration/RTO10/tombstoned-object-gc-recreate-0 +async def test_rto10_tombstoned_object_gc_recreate(realtime_sandbox, use_binary_protocol): + channel_name = 'objects-gc-object-' + random_id() + root = await synced_root(realtime_sandbox.key_str, channel_name, use_binary_protocol) + + # Create a counter + await root.set('counter', LiveCounter.create(42)) + await wall_clock_poll_until( + lambda: root.get('counter').as_live_counter().value() == 42, + description='counter to read 42') + + counter_id = root.get('counter').instance().id + + # Remove it (tombstones the entry and the object, RTLM7) + await root.remove('counter') + + # RTLM5d2h: tombstoned entries read back as None + await wall_clock_poll_until( + lambda: root.get('counter').as_live_counter().value() is None, + description='the removed counter to read None') + + # Create a new counter at the same key + await root.set('counter', LiveCounter.create(99)) + await wall_clock_poll_until( + lambda: root.get('counter').as_live_counter().value() == 99, + description='the new counter to read 99') + + assert root.get('counter').as_live_counter().value() == 99 + assert root.get('counter').instance().id != counter_id + + +# UTS: objects/integration/RTLM19/tombstoned-entry-gc-reset-0 +async def test_rtlm19_tombstoned_entry_gc_reset(realtime_sandbox, use_binary_protocol): + channel_name = 'objects-gc-entry-' + random_id() + root = await synced_root(realtime_sandbox.key_str, channel_name, use_binary_protocol) + + # Set then remove a key + await root.set('ephemeral', 'temporary') + await wall_clock_poll_until( + lambda: root.get('ephemeral').as_primitive().value() == 'temporary', + description="ephemeral to read 'temporary'") + + await root.remove('ephemeral') + + # RTLM5d2h: tombstoned entries read back as None + await wall_clock_poll_until( + lambda: root.get('ephemeral').as_primitive().value() is None, + description='the removed entry to read None') + + # Set the same key again + await root.set('ephemeral', 'revived') + await wall_clock_poll_until( + lambda: root.get('ephemeral').as_primitive().value() == 'revived', + description="ephemeral to read 'revived'") + + assert root.get('ephemeral').as_primitive().value() == 'revived' diff --git a/test/uts/objects/integration/objects_lifecycle_test.py b/test/uts/objects/integration/objects_lifecycle_test.py new file mode 100644 index 00000000..e43edf2a --- /dev/null +++ b/test/uts/objects/integration/objects_lifecycle_test.py @@ -0,0 +1,167 @@ +"""Derived from uts/objects/integration/objects_lifecycle_test.md in ably/specification. + +Spec points: RTO23, RTPO15, RTPO17 + +End to end against the sandbox: connect, sync, create and mutate objects through the +root `PathObject`, and read the result from a second client. Only the public API is used. + +The specification carries a `## Protocol Variants` section, so every test runs once per +protocol and passes `use_binary_protocol` to each client it builds. Provisioning over REST +is a JSON request whichever protocol the realtime client speaks. + +The specification writes its reads and writes against an untyped `PathObject`. +ably-python's `PathObject` is partitioned by type (RTTS), so a counter's `value()` and +`increment()` are reached through `as_live_counter()` and a primitive's `value()` through +`as_primitive()`; `root` is already a `LiveMapPathObject`, so `set` and `size` are called on +it directly. That is a difference of spelling, not of behaviour. + +`AWAIT_STATE` for CONNECTED waits ten seconds, the figure the realtime integration tier +uses for a connect over the network. The specification leaves `channel.object.get()` +unbounded; it is bounded here at the fifteen seconds `objects_gc_test.md` gives the same +call, so that a sync that never completes fails with a message naming it rather than at the +package timeout. Clients are closed by the suite's fixture. +""" + +import asyncio + +from ably.pubsub.realtime.connection import ConnectionState +from ably.pubsub.server import LiveCounter, LiveMap, PathObject +from test.uts.helpers.client import await_connection_state, sandbox_realtime_client, wall_clock_poll_until +from test.uts.helpers.sandbox import random_id +from test.uts.objects.helpers.standard_test_pool import objects_channel_options, provision_objects_via_rest + +CONNECT_TIMEOUT = 10.0 +SYNC_TIMEOUT = 15.0 + + +async def connected_client(api_key, use_binary_protocol): + """The specification's `Realtime(options: {...})`, connected and CONNECTED.""" + client = sandbox_realtime_client(api_key, auto_connect=False, use_binary_protocol=use_binary_protocol) + client.connect() + await await_connection_state(client, ConnectionState.CONNECTED, timeout=CONNECT_TIMEOUT) + return client + + +async def synced_root(channel): + """The specification's `AWAIT channel.object.get()`, bounded at `SYNC_TIMEOUT`.""" + try: + return await asyncio.wait_for(channel.object.get(), SYNC_TIMEOUT) + except asyncio.TimeoutError: + raise AssertionError( + f'Timed out after {SYNC_TIMEOUT}s waiting for channel.object.get() on {channel.name!r}') from None + + +async def synced_pair(api_key, channel_name, use_binary_protocol): + """The two clients, channels and roots every propagation test opens with.""" + client_a = await connected_client(api_key, use_binary_protocol) + client_b = await connected_client(api_key, use_binary_protocol) + + channel_a = client_a.channels.get(channel_name, objects_channel_options()) + channel_b = client_b.channels.get(channel_name, objects_channel_options()) + + root_a = await synced_root(channel_a) + root_b = await synced_root(channel_b) + return root_a, root_b + + +# UTS: objects/integration/RTO23-RTPO15/set-primitive-propagates-0 +async def test_rto23_rtpo15_set_primitive_propagates(realtime_sandbox, use_binary_protocol): + channel_name = 'objects-lifecycle-' + random_id() + root_a, root_b = await synced_pair(realtime_sandbox.key_str, channel_name, use_binary_protocol) + + # Client A sets a value + await root_a.set('greeting', 'hello') + + # Client B subscribes and waits for the update + events_b = [] + root_b.subscribe(events_b.append) + await wall_clock_poll_until( + lambda: root_b.get('greeting').as_primitive().value() == 'hello', + description="client B to read greeting == 'hello'") + + assert root_b.get('greeting').as_primitive().value() == 'hello' + + +# UTS: objects/integration/RTPO15/set-counter-value-type-0 +async def test_rtpo15_set_counter_value_type(realtime_sandbox, use_binary_protocol): + channel_name = 'objects-counter-create-' + random_id() + root_a, root_b = await synced_pair(realtime_sandbox.key_str, channel_name, use_binary_protocol) + + await root_a.set('my_counter', LiveCounter.create(42)) + await wall_clock_poll_until( + lambda: root_b.get('my_counter').as_live_counter().value() == 42, + description='client B to read my_counter == 42') + + assert root_b.get('my_counter').as_live_counter().value() == 42 + assert root_b.get('my_counter').instance() is not None + + +# UTS: objects/integration/RTPO17/increment-propagates-0 +async def test_rtpo17_increment_propagates(realtime_sandbox, use_binary_protocol): + channel_name = 'objects-increment-' + random_id() + root_a, root_b = await synced_pair(realtime_sandbox.key_str, channel_name, use_binary_protocol) + + # Create a counter first + await root_a.set('hits', LiveCounter.create(0)) + await wall_clock_poll_until( + lambda: root_b.get('hits').as_live_counter().value() == 0, + description='client B to read hits == 0') + + # Increment it + await root_a.get('hits').as_live_counter().increment(10) + await wall_clock_poll_until( + lambda: root_b.get('hits').as_live_counter().value() == 10, + description='client B to read hits == 10') + + assert root_a.get('hits').as_live_counter().value() == 10 + assert root_b.get('hits').as_live_counter().value() == 10 + + +# UTS: objects/integration/RTPO15/set-map-value-type-0 +async def test_rtpo15_set_map_value_type(realtime_sandbox, use_binary_protocol): + channel_name = 'objects-map-create-' + random_id() + root_a, root_b = await synced_pair(realtime_sandbox.key_str, channel_name, use_binary_protocol) + + await root_a.set('settings', LiveMap.create({ + 'theme': 'dark', + 'fontSize': 14, + })) + await wall_clock_poll_until( + lambda: root_b.get('settings').get('theme').as_primitive().value() == 'dark', + description="client B to read settings.theme == 'dark'") + + assert root_b.get('settings').get('theme').as_primitive().value() == 'dark' + assert root_b.get('settings').get('fontSize').as_primitive().value() == 14 + + +# UTS: objects/integration/RTO23/get-returns-path-object-0 +async def test_rto23_get_returns_path_object(realtime_sandbox, use_binary_protocol): + channel_name = 'objects-get-root-' + random_id() + + client = await connected_client(realtime_sandbox.key_str, use_binary_protocol) + channel = client.channels.get(channel_name, objects_channel_options()) + + root = await synced_root(channel) + + assert isinstance(root, PathObject) + assert root.path() == '' + assert root.size() == 0 + + +# UTS: objects/integration/RTPO15/rest-provisioned-data-sync-0 +async def test_rtpo15_rest_provisioned_data_sync(realtime_sandbox, use_binary_protocol): + channel_name = 'objects-rest-provision-' + random_id() + + # Provision data via REST before any realtime client connects + await provision_objects_via_rest(realtime_sandbox.key_str, channel_name, [ + { + 'mapSet': {'key': 'provisioned', 'value': {'string': 'from_rest'}}, + 'objectId': 'root', + }, + ]) + + client = await connected_client(realtime_sandbox.key_str, use_binary_protocol) + channel = client.channels.get(channel_name, objects_channel_options()) + root = await synced_root(channel) + + assert root.get('provisioned').as_primitive().value() == 'from_rest' diff --git a/test/uts/objects/integration/objects_sync_test.py b/test/uts/objects/integration/objects_sync_test.py new file mode 100644 index 00000000..338c395a --- /dev/null +++ b/test/uts/objects/integration/objects_sync_test.py @@ -0,0 +1,123 @@ +"""Derived from uts/objects/integration/objects_sync_test.md in ably/specification. + +Spec points: RTO4, RTO5, RTO17 + +The objects sync sequence against the sandbox: attaching with object modes gets an +ATTACHED carrying HAS_OBJECTS and an OBJECT_SYNC, after which `channel.object.get()` +resolves; a second client syncs what a first one wrote; and a detach and re-attach syncs +the pool afresh. Only the public API is used. + +The specification carries a `## Protocol Variants` section, so every test runs once per +protocol and passes `use_binary_protocol` to each client it builds. + +A primitive's `value()` is reached through `as_primitive()`, since ably-python's +`PathObject` is partitioned by type (RTTS); `root` is a `LiveMapPathObject`, so `set` and +`size` are called on it directly. + +`AWAIT_STATE` for CONNECTED waits ten seconds, the figure the realtime integration tier +uses for a connect over the network, and `channel.object.get()` is bounded at the fifteen +seconds `objects_gc_test.md` gives the same call. Clients are closed by the suite's fixture. +""" + +import asyncio + +from ably.pubsub.realtime.connection import ConnectionState +from ably.pubsub.server import ChannelMode, PathObject +from test.uts.helpers.client import await_connection_state, sandbox_realtime_client, wall_clock_poll_until +from test.uts.helpers.sandbox import random_id +from test.uts.objects.helpers.standard_test_pool import objects_channel_options + +CONNECT_TIMEOUT = 10.0 +SYNC_TIMEOUT = 15.0 + + +async def connected_client(api_key, use_binary_protocol): + """The specification's `Realtime(options: {...})`, connected and CONNECTED.""" + client = sandbox_realtime_client(api_key, auto_connect=False, use_binary_protocol=use_binary_protocol) + client.connect() + await await_connection_state(client, ConnectionState.CONNECTED, timeout=CONNECT_TIMEOUT) + return client + + +async def synced_root(channel): + """The specification's `AWAIT channel.object.get()`, bounded at `SYNC_TIMEOUT`.""" + try: + return await asyncio.wait_for(channel.object.get(), SYNC_TIMEOUT) + except asyncio.TimeoutError: + raise AssertionError( + f'Timed out after {SYNC_TIMEOUT}s waiting for channel.object.get() on {channel.name!r}') from None + + +# UTS: objects/integration/RTO4-RTO5/attach-sync-get-0 +async def test_rto4_rto5_attach_sync_get(realtime_sandbox, use_binary_protocol): + channel_name = 'objects-sync-' + random_id() + + client = await connected_client(realtime_sandbox.key_str, use_binary_protocol) + channel = client.channels.get(channel_name, objects_channel_options()) + + root = await synced_root(channel) + + assert isinstance(root, PathObject) + assert root.path() == '' + + +# UTS: objects/integration/RTO5-RTO17/two-clients-sync-0 +async def test_rto5_rto17_two_clients_sync(realtime_sandbox, use_binary_protocol): + channel_name = 'objects-two-sync-' + random_id() + + client_a = await connected_client(realtime_sandbox.key_str, use_binary_protocol) + client_b = await connected_client(realtime_sandbox.key_str, use_binary_protocol) + + channel_a = client_a.channels.get(channel_name, objects_channel_options()) + channel_b = client_b.channels.get(channel_name, objects_channel_options()) + + # Client A creates data + root_a = await synced_root(channel_a) + await root_a.set('key1', 'value1') + + # Client B attaches and syncs, and should see the data + root_b = await synced_root(channel_b) + await wall_clock_poll_until( + lambda: root_b.get('key1').as_primitive().value() == 'value1', + description="client B to read key1 == 'value1'") + + assert root_b.get('key1').as_primitive().value() == 'value1' + + +# UTS: objects/integration/RTO17/reattach-resyncs-0 +async def test_rto17_reattach_resyncs(realtime_sandbox, use_binary_protocol): + channel_name = 'objects-reattach-' + random_id() + + client = await connected_client(realtime_sandbox.key_str, use_binary_protocol) + channel = client.channels.get(channel_name, objects_channel_options()) + root = await synced_root(channel) + + # Set some data. The write is applied locally once the server ACKs it (RTO20), so + # it reads back as soon as `set` returns. + await root.set('before_detach', 'hello') + assert root.get('before_detach').as_primitive().value() == 'hello' + + # Detach and re-attach + await channel.detach() + await channel.attach() + + # Re-sync should restore data + root = await synced_root(channel) + await wall_clock_poll_until( + lambda: root.get('before_detach').as_primitive().value() == 'hello', + description="before_detach to read 'hello' after the re-sync") + + assert root.get('before_detach').as_primitive().value() == 'hello' + + +# UTS: objects/integration/RTO4/attach-subscribe-only-0 +async def test_rto4_attach_subscribe_only(realtime_sandbox, use_binary_protocol): + channel_name = 'objects-subscribe-only-' + random_id() + + client = await connected_client(realtime_sandbox.key_str, use_binary_protocol) + channel = client.channels.get(channel_name, objects_channel_options(ChannelMode.OBJECT_SUBSCRIBE)) + + root = await synced_root(channel) + + assert isinstance(root, PathObject) + assert root.size() == 0 diff --git a/test/uts/objects/integration/proxy/__init__.py b/test/uts/objects/integration/proxy/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/test/uts/objects/integration/proxy/conftest.py b/test/uts/objects/integration/proxy/conftest.py new file mode 100644 index 00000000..9491c7ce --- /dev/null +++ b/test/uts/objects/integration/proxy/conftest.py @@ -0,0 +1,78 @@ +"""Fixtures the LiveObjects proxy integration specification shares. + +The specification in this package runs its traffic through `uts-proxy`, so each +test opens a proxy session and closes it again afterwards. The `realtime_sandbox` +app the clients connect to comes from the parent package, which provisions it +once for the whole LiveObjects integration tier. The fixtures in +`test/uts/realtime/integration/proxy/conftest.py` are not visible from here, so +the same two are defined again; `ensure_proxy()` hands every caller the one +control process, so a run that collects both packages still starts only one. + +`test/uts/helpers/proxy.py` describes the proxy itself and the two environment +variables that change how it is obtained and started. +""" + +import os + +import pytest +import pytest_asyncio + +from test.uts.helpers.proxy import create_proxy_session, ensure_proxy, stop_proxy + +# What a test in this package gets, in seconds, in place of the 120 the parent +# package gives the rest of the tier. The first proxy test to run waits for the +# binary to be downloaded on a cold cache and for the control process to come up, +# and a fault test sits through a disconnect, a reconnection and a second objects +# sync before it can assert anything. +SUITE_TIMEOUT = 300 + +__package_dir = os.path.dirname(os.path.abspath(__file__)) + + +def pytest_collection_modifyitems(items): + # The parent package marks everything beneath it, this package included, with + # its own shorter timeout, and pytest-timeout reads the first of the item's own + # markers. Putting this one at the front is what makes it the one read, + # whichever order the two hooks happen to run in. + for item in items: + if os.path.abspath(str(item.fspath)).startswith(__package_dir + os.sep): + item.add_marker(pytest.mark.timeout(SUITE_TIMEOUT), append=False) + + +@pytest_asyncio.fixture(scope='session') +async def proxy_control(): + """The running `uts-proxy` control API, shared by every test in the package. + + One control process serves any number of sessions, each on a port of its own, + so it is started once and reaped when the run ends. `proxy_session` asks for + this fixture, so a test that opens sessions does not have to. + """ + await ensure_proxy() + yield + stop_proxy() + + +@pytest_asyncio.fixture +async def proxy_session(proxy_control): + """Opens proxy sessions, and closes every one of them when the test ends. + + This is the specification's `create_proxy_session(...)` together with its + `AFTER EACH TEST: IF session IS NOT null: session.close()`. A test calls it as + it would the function: + + session = await proxy_session(rules=[...]) + + and leaves the closing alone, so a test that fails part way through still + gives its sessions back. + """ + sessions = [] + + async def open_session(**options): + session = await create_proxy_session(**options) + sessions.append(session) + return session + + yield open_session + + for session in reversed(sessions): + await session.close() diff --git a/test/uts/objects/integration/proxy/objects_faults_test.py b/test/uts/objects/integration/proxy/objects_faults_test.py new file mode 100644 index 00000000..75afbda9 --- /dev/null +++ b/test/uts/objects/integration/proxy/objects_faults_test.py @@ -0,0 +1,450 @@ +"""Derived from uts/objects/integration/proxy/objects_faults.md in ably/specification. + +Spec points: RTO5a2, RTO7, RTO8, RTO17, RTO20e, RTO20e1 + +Each test routes a client through a `uts-proxy` session and faults the objects sync +against the real sandbox: the connection dropped on the first OBJECT_SYNC, a disconnect +while another client writes, a DETACHED injected onto an attached channel, an ATTACHED +injected to restart the sync followed by a channel ERROR while a write waits for it, and +the first OBJECT_SYNC delayed while another client writes. Only the public API is used; +the proxy's event log is the second witness where the specification reads it. + +**Clients.** A client under test points at the session (`endpoint='localhost'`, +`port=session.proxy_port`, `tls=False`, `use_binary_protocol=False`) and authenticates with +an `auth_callback` returning a locally signed Ably JWT: the session speaks plain WebSocket, +over which basic auth is refused, and a JWT signed here puts nothing in the event log. The +specification's `key: api_key` on those clients is that callback. Client A, where a test +has one, goes straight to the sandbox with the key, as the specification builds it. There +is no `## Protocol Variants` section, so every client speaks JSON, which the proxy requires +in any case. + +**Rules and the log.** A frame rule matches `action` as a string, and the proxy resolves +names only up to AUTH, so OBJECT_SYNC is matched as `'20'`. The log's frames carry +`direction` and a decoded `message` whose `action` is an integer. + +**Transient and already-held states.** The specification's `AWAIT_STATE ... DISCONNECTED` +waits for a state the client leaves at once: it reconnects immediately (RTN15a), so the +state can be gone before a wait that starts after the fault is registered. A recorder is +registered on the connection before it connects and the waits read the recorded list, +which also tells the reconnection's CONNECTED from the first one. Likewise RTO17's +`AWAIT_STATE channel.state == attached` follows a DETACHED injected onto an attached +channel, so the wait is for ATTACHING followed by ATTACHED in the recorded sequence. + +**Un-awaited calls.** `channel.attach()` and RTO20e's `pending = root.set(...)` are written +without `AWAIT`; they run as tasks. `poll_until_success` is the specification's: a read that +raises `AblyException` (the channel transiently not attached, RTO25b) means "not yet", and +the most recent such error is raised if the wait runs out. Any other error fails at once, +since it means the read is wrong rather than early. + +**Cleanup.** The specification's `AFTER EACH TEST` is the fixtures': the suite closes every +client a test built and `proxy_session` closes every session it opened. + +A primitive's `value()` is reached through `as_primitive()`, since ably-python's +`PathObject` is partitioned by type (RTTS); `root` is a `LiveMapPathObject`, so `set` is +called on it directly. +""" + +import asyncio + +import pytest + +from ably.pubsub.realtime.connection import ConnectionState +from ably.pubsub.server import AblyException, PathObject +from ably.pubsub.types.channelstate import ChannelState +from test.uts.helpers.client import ( + await_channel_state, + await_connection_state, + sandbox_realtime_client, + wall_clock_poll_until, +) +from test.uts.helpers.sandbox import extract_key_name, extract_key_secret, generate_jwt, random_id +from test.uts.objects.helpers.standard_test_pool import objects_channel_options + +# The specification's `WITH timeout` values, as wall-clock seconds. +CONNECT_TIMEOUT = 15.0 +DISCONNECT_TIMEOUT = 15.0 +RECONNECT_TIMEOUT = 30.0 +SYNC_TIMEOUT = 15.0 +RESYNC_TIMEOUT = 30.0 +REATTACH_TIMEOUT = 30.0 +FAILED_TIMEOUT = 15.0 +PENDING_TIMEOUT = 15.0 +ACK_TIMEOUT = 10.0 +VALUE_TIMEOUT = 15.0 + +# `poll_until_success`'s defaults, as `uts/README.md` gives them. +POLL_SUCCESS_TIMEOUT = 10.0 +POLL_SUCCESS_INTERVAL = 0.5 + +# How often a recorded state list is re-read. The list is append-only, so nothing is +# missed at any interval; a short one keeps a wait from outlasting the reconnection. +STATE_POLL_INTERVAL = 0.05 + +# RTO20e's `WAIT 500ms`, for the ACKed operation to reach the wait for SYNCED. +PARK_DELAY = 0.5 + +# The actions `uts/docs/proxy.md` tabulates, as a logged frame's `message.action`. +ACK_ACTION = 1 +OBJECT_ACTION = 19 + + +def jwt_auth_callback(api_key): + """The credentials of a proxied client: an Ably JWT signed locally for `api_key`.""" + key_name = extract_key_name(api_key) + key_secret = extract_key_secret(api_key) + + async def auth_callback(params): + return generate_jwt(key_name, key_secret) + + return auth_callback + + +def proxied_client(session, api_key): + """The specification's `Realtime(options: ClientOptions(...))` pointed at `session`.""" + return sandbox_realtime_client( + auth_callback=jwt_auth_callback(api_key), + endpoint='localhost', + port=session.proxy_port, + tls=False, + use_binary_protocol=False, + auto_connect=False, + ) + + +def direct_client(api_key): + """The specification's client A, which goes straight to the sandbox with the key.""" + return sandbox_realtime_client(api_key, auto_connect=False) + + +def record_states(emitter): + """Records every state `emitter` enters from now on, in order.""" + states = [] + + def on_state_change(change): + states.append(change.current) + + emitter.on(on_state_change) + return states + + +async def await_recorded_state(states, state, count=1, timeout=CONNECT_TIMEOUT): + """This file's `AWAIT_STATE` for a connection: waits until `states` holds `count` entries of `state`.""" + try: + await wall_clock_poll_until( + lambda: states.count(state) >= count, + timeout=timeout, + description=f'{count} state change(s) to {state.value}', + interval=STATE_POLL_INTERVAL, + ) + except AssertionError as error: + recorded = [recorded_state.value for recorded_state in states] + raise AssertionError(f'{error}; the states recorded were {recorded}') from None + + +def contains_in_order(recorded, expected): + """The specifications' `CONTAINS_IN_ORDER`: `expected` as a subsequence of `recorded`.""" + remaining = list(expected) + for item in recorded: + if remaining and item == remaining[0]: + remaining.pop(0) + return not remaining + + +async def bounded(awaitable, timeout, description): + """An `AWAIT ... WITH timeout`, failing with `description` when the time runs out.""" + try: + return await asyncio.wait_for(awaitable, timeout) + except asyncio.TimeoutError: + raise AssertionError(f'Timed out after {timeout}s waiting for {description}') from None + + +def in_background(coroutine): + """Starts `coroutine` without awaiting it, as an un-awaited call in the specification. + + What it ends with is retrieved when it ends, so an outcome the specification does not + assert on is not reported again as an exception nobody retrieved. + """ + task = asyncio.ensure_future(coroutine) + task.add_done_callback(lambda done: done.cancelled() or done.exception()) + return task + + +async def poll_until_success(condition, timeout=POLL_SUCCESS_TIMEOUT, description='condition'): + """The specification's `poll_until_success`, for a read across a fault and its recovery.""" + loop = asyncio.get_running_loop() + deadline = loop.time() + timeout + last_error = None + while True: + try: + result = condition() + if result: + return result + except AblyException as error: + last_error = error + if loop.time() >= deadline: + if last_error is not None: + raise last_error + raise AssertionError(f'Timed out after {timeout}s waiting for {description}') + await asyncio.sleep(POLL_SUCCESS_INTERVAL) + + +def object_publish_acked(log): + """Whether the log holds the server's ACK for an OBJECT frame the client sent. + + The specification polls for any server-to-client ACK; the client sends nothing else + that is ACKed before the `set`, so requiring the OBJECT frame first names the same + frame and cannot be satisfied by anything else. + """ + published = False + for event in log: + if event['type'] != 'ws_frame': + continue + action = (event.get('message') or {}).get('action') + if event.get('direction') == 'client_to_server' and action == OBJECT_ACTION: + published = True + elif published and event.get('direction') == 'server_to_client' and action == ACK_ACTION: + return True + return False + + +# UTS: objects/proxy/RTO5a2-RTO17/sync-interrupted-reconnect-0 +async def test_rto5a2_rto17_sync_interrupted_reconnect(realtime_sandbox, proxy_session): + channel_name = 'objects-sync-interrupt-' + random_id() + + # Disconnect after first OBJECT_SYNC frame + session = await proxy_session(rules=[{ + 'match': {'type': 'ws_frame_to_client', 'action': '20'}, + 'action': {'type': 'disconnect'}, + 'times': 1, + 'comment': 'RTO5a2: Disconnect after first OBJECT_SYNC to interrupt sync', + }]) + + client = proxied_client(session, realtime_sandbox.key_str) + channel = client.channels.get(channel_name, objects_channel_options()) + states = record_states(client.connection) + + client.connect() + await await_recorded_state(states, ConnectionState.CONNECTED, timeout=CONNECT_TIMEOUT) + + # First attach triggers sync; proxy disconnects mid-sync + in_background(channel.attach()) + await await_recorded_state(states, ConnectionState.DISCONNECTED, timeout=DISCONNECT_TIMEOUT) + + # Client auto-reconnects; re-attach triggers fresh sync + await await_recorded_state(states, ConnectionState.CONNECTED, count=2, timeout=RECONNECT_TIMEOUT) + + # get() waits for SYNCED, and will only resolve if the re-sync completes + root = await bounded(channel.object.get(), RESYNC_TIMEOUT, 'channel.object.get() after the re-sync') + + assert isinstance(root, PathObject) + assert root.path() == '' + + +# UTS: objects/proxy/RTO7-RTO8/mutations-buffered-during-resync-0 +async def test_rto7_rto8_mutations_buffered_during_resync(realtime_sandbox, proxy_session): + channel_name = 'objects-buffer-resync-' + random_id() + + # Client A: direct connection (no proxy), publishes mutations + client_a = direct_client(realtime_sandbox.key_str) + client_a.connect() + await await_connection_state(client_a, ConnectionState.CONNECTED, timeout=CONNECT_TIMEOUT) + + channel_a = client_a.channels.get(channel_name, objects_channel_options()) + root_a = await bounded(channel_a.object.get(), SYNC_TIMEOUT, "client A's channel.object.get()") + + # Set initial data + await root_a.set('key1', 'initial') + + # Client B: through proxy, will be disconnected + session = await proxy_session(rules=[]) + + client_b = proxied_client(session, realtime_sandbox.key_str) + channel_b = client_b.channels.get(channel_name, objects_channel_options()) + states_b = record_states(client_b.connection) + + # Client B connects and syncs + client_b.connect() + await await_recorded_state(states_b, ConnectionState.CONNECTED, timeout=CONNECT_TIMEOUT) + + root_b = await bounded(channel_b.object.get(), SYNC_TIMEOUT, "client B's channel.object.get()") + await poll_until_success( + lambda: root_b.get('key1').as_primitive().value() == 'initial', + description="client B to read key1 == 'initial'") + + # Disconnect client B + await session.trigger_action({'type': 'disconnect'}) + await await_recorded_state(states_b, ConnectionState.DISCONNECTED, timeout=DISCONNECT_TIMEOUT) + + # While B is disconnected, A publishes a mutation + await root_a.set('key1', 'updated_during_disconnect') + + # Client B reconnects and re-syncs; the mutation should be visible + await await_recorded_state(states_b, ConnectionState.CONNECTED, count=2, timeout=RECONNECT_TIMEOUT) + + root_b = await bounded( + channel_b.object.get(), SYNC_TIMEOUT, "client B's channel.object.get() after reconnecting") + await poll_until_success( + lambda: root_b.get('key1').as_primitive().value() == 'updated_during_disconnect', + timeout=VALUE_TIMEOUT, + description="client B to read key1 == 'updated_during_disconnect'") + + assert root_b.get('key1').as_primitive().value() == 'updated_during_disconnect' + + +# UTS: objects/proxy/RTO17/server-detach-resync-0 +async def test_rto17_server_detach_resync(realtime_sandbox, proxy_session): + channel_name = 'objects-detach-resync-' + random_id() + + session = await proxy_session(rules=[]) + + client = proxied_client(session, realtime_sandbox.key_str) + channel = client.channels.get(channel_name, objects_channel_options()) + + client.connect() + await await_connection_state(client, ConnectionState.CONNECTED, timeout=CONNECT_TIMEOUT) + + root = await bounded(channel.object.get(), SYNC_TIMEOUT, 'channel.object.get()') + + # Set some data. The write is applied locally once the server ACKs it (RTO20), so it + # reads back as soon as `set` returns. + await root.set('before_detach', 'hello') + assert root.get('before_detach').as_primitive().value() == 'hello' + + channel_states = record_states(channel) + + # Inject server-initiated DETACHED + await session.trigger_action({ + 'type': 'inject_to_client', + 'message': { + 'action': 13, + 'channel': channel_name, + }, + }) + + # Client should auto-re-attach (RTL13a). The channel is ATTACHED when the wait begins, + # so it is the recorded re-attach that is waited for. + try: + await wall_clock_poll_until( + lambda: contains_in_order(channel_states, [ChannelState.ATTACHING, ChannelState.ATTACHED]), + timeout=REATTACH_TIMEOUT, + description='the channel to re-attach after the injected DETACHED', + interval=STATE_POLL_INTERVAL) + except AssertionError as error: + recorded = [recorded_state.value for recorded_state in channel_states] + raise AssertionError(f'{error}; the channel states recorded were {recorded}') from None + assert channel.state == ChannelState.ATTACHED + + # Re-sync should restore data + root = await bounded(channel.object.get(), SYNC_TIMEOUT, 'channel.object.get() after the re-attach') + await poll_until_success( + lambda: root.get('before_detach').as_primitive().value() == 'hello', + timeout=VALUE_TIMEOUT, + description="before_detach to read 'hello' after the re-sync") + + assert root.get('before_detach').as_primitive().value() == 'hello' + + +# UTS: objects/proxy/RTO20e/publish-fails-on-channel-failed-0 +async def test_rto20e_publish_fails_on_channel_failed(realtime_sandbox, proxy_session): + channel_name = 'objects-publish-failed-' + random_id() + + session = await proxy_session(rules=[]) + + client = proxied_client(session, realtime_sandbox.key_str) + channel = client.channels.get(channel_name, objects_channel_options()) + + client.connect() + await await_connection_state(client, ConnectionState.CONNECTED, timeout=CONNECT_TIMEOUT) + + root = await bounded(channel.object.get(), SYNC_TIMEOUT, 'channel.object.get()') + + # Force the objects back into SYNCING: inject an ATTACHED (action 11) carrying the + # HAS_OBJECTS flag (bit 7, i.e. flags: 128). RTO4c starts a new sync sequence on every + # ATTACHED protocol message; the server never sent this ATTACHED, so no OBJECT_SYNC + # follows and the objects remain SYNCING. The channel itself stays ATTACHED. + await session.trigger_action({ + 'type': 'inject_to_client', + 'message': {'action': 11, 'channel': channel_name, 'flags': 128}, + }) + + # Mutate WHILE SYNCING: the channel is ATTACHED so the write preconditions (RTO26) + # pass and the publish + ACK complete against the real server; publishAndApply then + # waits for a SYNCED that will never arrive (RTO20e). Do not await yet. + pending = asyncio.ensure_future(root.set('key', 'value')) + + # Ensure the operation is in the RTO20e sync-wait, not still publishing: wait until + # the proxy log shows the server's ACK for the OBJECT publish, then allow a brief + # real-time yield for the client to move the ACKed operation into the wait. + async def acked(): + return object_publish_acked(await session.get_log()) + + await wall_clock_poll_until(acked, timeout=ACK_TIMEOUT, description='the server to ACK the OBJECT publish') + await asyncio.sleep(PARK_DELAY) + + # The channel enters FAILED whilst the operation waits for SYNCED (RTO20e1) + await session.trigger_action({ + 'type': 'inject_to_client', + 'message': { + 'action': 9, + 'channel': channel_name, + 'error': {'statusCode': 400, 'code': 90000, 'message': 'injected error'}, + }, + }) + + await await_channel_state(channel, ChannelState.FAILED, timeout=FAILED_TIMEOUT) + + with pytest.raises(AblyException) as excinfo: + await bounded(pending, PENDING_TIMEOUT, 'the pending set() to fail') + + error = excinfo.value + assert error.code == 92008 + assert error.status_code == 400 + # RTO20e1: cause is set to RealtimeChannel.errorReason, the injected channel ERROR + assert error.cause is not None + assert error.cause.code == 90000 + + +# UTS: objects/proxy/RTO5-RTO7/publish-during-sync-echo-after-0 +async def test_rto5_rto7_publish_during_sync_echo_after(realtime_sandbox, proxy_session): + channel_name = 'objects-publish-during-sync-' + random_id() + + # Client A: direct, no proxy + client_a = direct_client(realtime_sandbox.key_str) + client_a.connect() + await await_connection_state(client_a, ConnectionState.CONNECTED, timeout=CONNECT_TIMEOUT) + + channel_a = client_a.channels.get(channel_name, objects_channel_options()) + root_a = await bounded(channel_a.object.get(), SYNC_TIMEOUT, "client A's channel.object.get()") + + # Set up initial data + await root_a.set('existing', 'before') + + # Client B: through proxy with delayed OBJECT_SYNC + session = await proxy_session(rules=[{ + 'match': {'type': 'ws_frame_to_client', 'action': '20'}, + 'action': {'type': 'delay', 'delayMs': 3000}, + 'times': 1, + 'comment': 'Delay first OBJECT_SYNC to keep B in SYNCING state', + }]) + + client_b = proxied_client(session, realtime_sandbox.key_str) + channel_b = client_b.channels.get(channel_name, objects_channel_options()) + + # Start client B, which stays SYNCING while its first OBJECT_SYNC is delayed + client_b.connect() + await await_connection_state(client_b, ConnectionState.CONNECTED, timeout=CONNECT_TIMEOUT) + in_background(channel_b.attach()) + + # While B is syncing, A publishes a mutation + await root_a.set('existing', 'after') + + # B's get() will resolve once the delayed sync completes + root_b = await bounded(channel_b.object.get(), RESYNC_TIMEOUT, "client B's channel.object.get()") + + # The mutation from A should be visible (either in sync data or buffered OBJECT) + await poll_until_success( + lambda: root_b.get('existing').as_primitive().value() == 'after', + timeout=VALUE_TIMEOUT, + description="client B to read existing == 'after'") + + assert root_b.get('existing').as_primitive().value() == 'after' From af9b3db4a5a8c3f0e05a0f40f92a3f9e994e0ce5 Mon Sep 17 00:00:00 2001 From: owenpearson Date: Fri, 9 Oct 2026 13:28:00 +0100 Subject: [PATCH 15/71] objects: add the LiveObjects data model and objects sync Adds the ably.pubsub.objects package: the protocol v6 object message wire types, with their JSON and msgpack encodings; the LWW map and counter CRDTs with their tombstones, parent references and garbage collection; the objects pool and the sync sequence that replaces it on attach, buffering operations until the sync completes; object id generation; the evaluation of LiveMap and LiveCounter creation values into create operations; and the public ObjectMessage the subscription events carry. A RealtimeObject can be built with no channel, which is how the pure unit specifications drive the sync state machine. The path-based public API, publishing and the channel integration follow in later commits and are stubs here. Also adds the HAS_OBJECTS flag, the OBJECT_SUBSCRIBE and OBJECT_PUBLISH channel modes, and the siteCode and objectsGCGracePeriod connection details. Co-Authored-By: Claude Opus 5.5 (1M context) --- ably/pubsub/objects/__init__.py | 7 + ably/pubsub/objects/batch.py | 161 +++++ ably/pubsub/objects/defaults.py | 10 + ably/pubsub/objects/enums.py | 39 ++ ably/pubsub/objects/instance.py | 162 +++++ ably/pubsub/objects/livecounter.py | 156 +++++ ably/pubsub/objects/livemap.py | 387 +++++++++++ ably/pubsub/objects/liveobject.py | 255 +++++++ ably/pubsub/objects/objectid.py | 31 + ably/pubsub/objects/objectmessage.py | 657 ++++++++++++++++++ ably/pubsub/objects/objectspool.py | 114 +++ ably/pubsub/objects/pathobject.py | 185 +++++ .../objects/pathobjectsubscriptionregister.py | 35 + ably/pubsub/objects/publicmessage.py | 102 +++ ably/pubsub/objects/realtimeobject.py | 386 ++++++++++ ably/pubsub/objects/subscription.py | 31 + ably/pubsub/objects/syncobjectspool.py | 83 +++ ably/pubsub/objects/valuetypes.py | 229 ++++++ ably/pubsub/types/channelmode.py | 2 + ably/pubsub/types/connectiondetails.py | 10 +- ably/pubsub/types/flags.py | 3 + 21 files changed, 3043 insertions(+), 2 deletions(-) create mode 100644 ably/pubsub/objects/__init__.py create mode 100644 ably/pubsub/objects/batch.py create mode 100644 ably/pubsub/objects/defaults.py create mode 100644 ably/pubsub/objects/enums.py create mode 100644 ably/pubsub/objects/instance.py create mode 100644 ably/pubsub/objects/livecounter.py create mode 100644 ably/pubsub/objects/livemap.py create mode 100644 ably/pubsub/objects/liveobject.py create mode 100644 ably/pubsub/objects/objectid.py create mode 100644 ably/pubsub/objects/objectmessage.py create mode 100644 ably/pubsub/objects/objectspool.py create mode 100644 ably/pubsub/objects/pathobject.py create mode 100644 ably/pubsub/objects/pathobjectsubscriptionregister.py create mode 100644 ably/pubsub/objects/publicmessage.py create mode 100644 ably/pubsub/objects/realtimeobject.py create mode 100644 ably/pubsub/objects/subscription.py create mode 100644 ably/pubsub/objects/syncobjectspool.py create mode 100644 ably/pubsub/objects/valuetypes.py diff --git a/ably/pubsub/objects/__init__.py b/ably/pubsub/objects/__init__.py new file mode 100644 index 00000000..621bd62b --- /dev/null +++ b/ably/pubsub/objects/__init__.py @@ -0,0 +1,7 @@ +"""LiveObjects: shared, conflict-free data structures stored on a realtime channel. + +The public names are re-exported from :mod:`ably.pubsub.server`; the modules here +hold the implementation behind them, and the internal classes the specification +names (``ObjectsPool``, ``InternalLiveMap``, ``InternalLiveCounter`` and the wire +types). +""" diff --git a/ably/pubsub/objects/batch.py b/ably/pubsub/objects/batch.py new file mode 100644 index 00000000..1a9f6f49 --- /dev/null +++ b/ably/pubsub/objects/batch.py @@ -0,0 +1,161 @@ +"""Batched writes, published as one message (RTPO20, RTINS17, RTBC*). + +`batch()` on a typed path or instance returns a `Batch`, an async context manager whose +`__aenter__` checks the write preconditions and resolves the target (RTPO20b-d, +RTINS17b-d) and returns the matching `BatchContext`. The context's write methods are +synchronous and only queue; `__aexit__` publishes everything queued as one message +(RTBC16d) when the block exits without raising, and closes the batch either way +(RTPO20g). Any use of a context after its batch has closed raises `AblyException` 40000 +(RTBC16e). +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING, Any, Awaitable, Callable, Generic, TypeVar, overload + +if TYPE_CHECKING: + from ably.pubsub.objects.instance import Instance + from ably.pubsub.objects.objectmessage import ObjectMessage + from ably.pubsub.objects.realtimeobject import RealtimeObject + from ably.pubsub.objects.valuetypes import LiveMapValue, Primitive, T + +C = TypeVar('C', bound='BatchContext') + + +class Batch(Generic[C]): + """RTBC16: the `async with` block a `batch()` call returns.""" + + def __init__(self, realtime_object: RealtimeObject, resolve: Callable[[], Instance]): + self._realtime_object = realtime_object + # Resolves the batch's target when the block is entered (RTPO20c, RTINS17c) + self._resolve = resolve + self._root_context: RootBatchContext | None = None + + async def __aenter__(self) -> C: + """RTPO20b-d, RTINS17b-d: checks the write preconditions, resolves the target and opens + the batch.""" + raise NotImplementedError + + async def __aexit__(self, exc_type, exc, tb) -> None: + """RTPO20f, RTPO20g, RTBC16d: publishes the queued writes unless the block raised, and + closes the batch either way.""" + raise NotImplementedError + + +class RootBatchContext: + """RTBC16: the state one batch shares across every context opened from it.""" + + def __init__(self, realtime_object: RealtimeObject, instance: Instance): + self.realtime_object = realtime_object + self.instance = instance + self.wrapped_instances: dict[str, BatchContext] = {} # RTBC16a + self.queued_message_constructors: list[Callable[[], Awaitable[list[ObjectMessage]]]] = [] # RTBC16b + self.closed = False + + def wrap_instance(self, instance: Instance) -> BatchContext: + """RTBC16c: the context for `instance`, one per object id.""" + raise NotImplementedError + + async def flush(self) -> None: + """RTBC16d: closes the batch, builds the queued messages and publishes them as one message, + if there are any.""" + raise NotImplementedError + + def close(self) -> None: + """RTBC16e: closes the batch, so that every later use of its contexts raises 40000.""" + raise NotImplementedError + + +class BatchContext: + """RTBC1: a synchronous view of an `Instance` inside a batch (RTBC2a).""" + + def __init__(self, root_context: RootBatchContext, instance: Instance): + self._root_context = root_context # RTBC2b + self._instance = instance # RTBC2a + + @property + def id(self) -> str | None: + """RTBC3: the wrapped object's id.""" + raise NotImplementedError + + def get(self, key: str) -> BatchContext | None: + """RTBC4: the context for the value at `key` of the wrapped map, or None.""" + raise NotImplementedError + + def compact(self) -> Any: + """RTBC10: `Instance.compact` on the wrapped instance.""" + raise NotImplementedError + + def compact_json(self) -> Any: + """RTBC11: `Instance.compact_json` on the wrapped instance.""" + raise NotImplementedError + + def as_live_map(self) -> LiveMapBatchContext: + """RTBC1a: this context as a map. Raises AblyException 92007 if it wraps something else.""" + raise NotImplementedError + + def as_live_counter(self) -> LiveCounterBatchContext: + """RTBC1a: this context as a counter. Raises AblyException 92007 if it wraps something else.""" + raise NotImplementedError + + def as_primitive(self) -> PrimitiveBatchContext: + """RTBC1a: this context as a primitive. Raises AblyException 92007 if it wraps something else.""" + raise NotImplementedError + + +class LiveMapBatchContext(BatchContext): + """RTBC1a: a batch context wrapping a map.""" + + def entries(self) -> list[tuple[str, BatchContext]]: + """RTBC6: `(key, context)` for each entry of the map.""" + raise NotImplementedError + + def keys(self) -> list[str]: + """RTBC7: the keys of the map.""" + raise NotImplementedError + + def values(self) -> list[BatchContext]: + """RTBC8: a context for each entry of the map.""" + raise NotImplementedError + + def size(self) -> int | None: + """RTBC9: the number of entries in the map.""" + raise NotImplementedError + + def set(self, key: str, value: LiveMapValue) -> None: + """RTBC12: queues a MAP_SET of `key`.""" + raise NotImplementedError + + def remove(self, key: str) -> None: + """RTBC13: queues a MAP_REMOVE of `key`.""" + raise NotImplementedError + + +class LiveCounterBatchContext(BatchContext): + """RTBC1a: a batch context wrapping a counter.""" + + def value(self) -> float: + """RTBC5: the counter's value.""" + raise NotImplementedError + + def increment(self, amount: float = 1) -> None: + """RTBC14: queues a COUNTER_INC of `amount`.""" + raise NotImplementedError + + def decrement(self, amount: float = 1) -> None: + """RTBC15: queues a COUNTER_INC of `-amount`.""" + raise NotImplementedError + + +class PrimitiveBatchContext(BatchContext): + """RTBC1a: a batch context wrapping a primitive.""" + + @overload + def value(self) -> Primitive: ... + + @overload + def value(self, expected: type[T]) -> T | None: ... + + def value(self, expected: type | None = None) -> Any: + """RTBC5: the wrapped primitive; with `expected`, only if it is of that type.""" + raise NotImplementedError diff --git a/ably/pubsub/objects/defaults.py b/ably/pubsub/objects/defaults.py new file mode 100644 index 00000000..fbfd3c1e --- /dev/null +++ b/ably/pubsub/objects/defaults.py @@ -0,0 +1,10 @@ +"""Default values for the LiveObjects client.""" + +# RTO10a: how often tombstoned objects and map entries are checked for release +GC_INTERVAL_MS = 5 * 60 * 1000 + +# RTO10b3: the grace period used when ConnectionDetails.objectsGCGracePeriod is absent +GC_GRACE_PERIOD_MS = 24 * 60 * 60 * 1000 + +# RTO3b: the object id of the root map, which every ObjectsPool holds +ROOT_OBJECT_ID = 'root' diff --git a/ably/pubsub/objects/enums.py b/ably/pubsub/objects/enums.py new file mode 100644 index 00000000..b42e90ee --- /dev/null +++ b/ably/pubsub/objects/enums.py @@ -0,0 +1,39 @@ +"""The enumerations LiveObjects shares across its modules.""" + +from enum import Enum + + +class ObjectsSyncState(str, Enum): + """RTO17a: the state of synchronising a channel's objects with Ably.""" + + INITIALIZED = 'initialized' # RTO17a1 + SYNCING = 'syncing' # RTO17a2 + SYNCED = 'synced' # RTO17a3 + + +class ObjectsEvent(str, Enum): + """RTO18b: the sync state events `RealtimeObject.on` registers listeners for.""" + + SYNCING = 'syncing' # RTO18b1 + SYNCED = 'synced' # RTO18b2 + + +class ObjectsOperationSource(str, Enum): + """RTO22: where an operation being applied came from.""" + + LOCAL = 'local' # RTO22a: applied on receipt of the ACK for a local publish + CHANNEL = 'channel' # RTO22b: received over the channel + + +class ValueType(Enum): + """RTTS2: the category of value a path or an instance holds.""" + + STRING = 'string' # RTTS2a1 + NUMBER = 'number' # RTTS2a2 + BOOLEAN = 'boolean' # RTTS2a3 + BINARY = 'binary' # RTTS2a4 + JSON_OBJECT = 'json_object' # RTTS2a5 + JSON_ARRAY = 'json_array' # RTTS2a6 + LIVE_MAP = 'live_map' # RTTS2a7 + LIVE_COUNTER = 'live_counter' # RTTS2a8 + UNKNOWN = 'unknown' # RTTS2a9 diff --git a/ably/pubsub/objects/instance.py b/ably/pubsub/objects/instance.py new file mode 100644 index 00000000..c8112cb2 --- /dev/null +++ b/ably/pubsub/objects/instance.py @@ -0,0 +1,162 @@ +"""Identity-addressed views onto a channel's objects (RTINS*, RTTS7-RTTS10). + +An `Instance` wraps one resolved value, a live object or a primitive, and follows that +object wherever it sits in the graph. Every instance is one of `LiveMapInstance`, +`LiveCounterInstance` or `PrimitiveInstance`, matching the value it wraps. The view +helpers are checked: asking for a type the instance does not wrap raises `AblyException` +92007 (RTTS9d). +""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import TYPE_CHECKING, Any, Callable, overload + +if TYPE_CHECKING: + from ably.pubsub.objects.batch import Batch, LiveCounterBatchContext, LiveMapBatchContext + from ably.pubsub.objects.enums import ValueType + from ably.pubsub.objects.liveobject import LiveObject + from ably.pubsub.objects.publicmessage import ObjectMessage + from ably.pubsub.objects.realtimeobject import RealtimeObject + from ably.pubsub.objects.subscription import Subscription + from ably.pubsub.objects.valuetypes import LiveMapValue, Primitive, T + + +@dataclass +class InstanceSubscriptionEvent: + """RTINS16e: what an `Instance.subscribe` listener receives.""" + + object: Instance # RTINS16e1 + message: ObjectMessage | None = None # RTINS16e2 + + +class Instance: + """RTINS1, RTTS7: a reference to a specific live object or primitive value.""" + + def __init__(self, realtime_object: RealtimeObject, value: LiveObject | Primitive): + self._realtime_object = realtime_object + self._value = value # RTINS2a + + @staticmethod + def _wrap(realtime_object: RealtimeObject, value: LiveObject | Primitive) -> Instance: + """The `Instance` subclass matching `value`.""" + raise NotImplementedError + + @property + def id(self) -> str | None: + """RTINS3: the wrapped object's id, or None for a primitive.""" + raise NotImplementedError + + @property + def type(self) -> ValueType: + """RTTS8a: the type of the wrapped value.""" + raise NotImplementedError + + def get(self, key: str) -> Instance | None: + """RTINS5: an instance wrapping the value at `key` of the wrapped map, or None.""" + raise NotImplementedError + + def compact(self) -> Any: + """RTINS10: a plain snapshot of the wrapped value.""" + raise NotImplementedError + + def compact_json(self) -> Any: + """RTINS11: `compact`, with binary as base64 and cycles as `{'objectId': ...}`. Never None.""" + raise NotImplementedError + + def as_live_map(self) -> LiveMapInstance: + """RTTS9a: this instance as a map. Raises AblyException 92007 if it wraps something else.""" + raise NotImplementedError + + def as_live_counter(self) -> LiveCounterInstance: + """RTTS9b: this instance as a counter. Raises AblyException 92007 if it wraps something else.""" + raise NotImplementedError + + def as_primitive(self) -> PrimitiveInstance: + """RTTS9c: this instance as a primitive. Raises AblyException 92007 if it wraps something else.""" + raise NotImplementedError + + +class LiveMapInstance(Instance): + """RTTS10a: an instance wrapping an `InternalLiveMap`.""" + + @property + def id(self) -> str: + """RTINS3a: the wrapped map's id.""" + raise NotImplementedError + + def batch(self) -> Batch[LiveMapBatchContext]: + """RTINS17: a block whose queued writes are published as one message when it exits.""" + raise NotImplementedError + + def entries(self) -> list[tuple[str, Instance]]: + """RTINS6: `(key, instance)` for each entry of the map.""" + raise NotImplementedError + + def keys(self) -> list[str]: + """RTINS7: the keys of the map.""" + raise NotImplementedError + + def values(self) -> list[Instance]: + """RTINS8: an instance for each entry of the map.""" + raise NotImplementedError + + def size(self) -> int: + """RTINS9: the number of entries in the map.""" + raise NotImplementedError + + async def set(self, key: str, value: LiveMapValue) -> None: + """RTINS12: sets `key` in the wrapped map.""" + raise NotImplementedError + + async def remove(self, key: str) -> None: + """RTINS13: removes `key` from the wrapped map.""" + raise NotImplementedError + + def subscribe(self, listener: Callable[[InstanceSubscriptionEvent], None]) -> Subscription: + """RTINS16: calls `listener` for each update to the wrapped map, wherever it sits (RTINS16g).""" + raise NotImplementedError + + +class LiveCounterInstance(Instance): + """RTTS10b: an instance wrapping an `InternalLiveCounter`.""" + + @property + def id(self) -> str: + """RTINS3a: the wrapped counter's id.""" + raise NotImplementedError + + def batch(self) -> Batch[LiveCounterBatchContext]: + """RTINS17: a block whose queued writes are published as one message when it exits.""" + raise NotImplementedError + + def value(self) -> float: + """RTINS4b: the wrapped counter's value.""" + raise NotImplementedError + + async def increment(self, amount: float = 1) -> None: + """RTINS14: increments the wrapped counter by `amount`.""" + raise NotImplementedError + + async def decrement(self, amount: float = 1) -> None: + """RTINS15: decrements the wrapped counter by `amount`.""" + raise NotImplementedError + + def subscribe(self, listener: Callable[[InstanceSubscriptionEvent], None]) -> Subscription: + """RTINS16: calls `listener` for each update to the wrapped counter, wherever it sits (RTINS16g).""" + raise NotImplementedError + + +class PrimitiveInstance(Instance): + """RTTS10c, RTTS10d: an instance wrapping a primitive value. It has no `subscribe` (RTTS7b).""" + + @overload + def value(self) -> Primitive: ... + + @overload + def value(self, expected: type[T]) -> T | None: ... + + def value(self, expected: type | None = None) -> Any: + """RTINS4c: the wrapped primitive; with `expected`, only if it is of that type, as for + `PrimitivePathObject.value`.""" + raise NotImplementedError diff --git a/ably/pubsub/objects/livecounter.py b/ably/pubsub/objects/livecounter.py new file mode 100644 index 00000000..64d2425d --- /dev/null +++ b/ably/pubsub/objects/livecounter.py @@ -0,0 +1,156 @@ +"""The counter CRDT (RTLC*).""" + +from __future__ import annotations + +import logging +from typing import TYPE_CHECKING + +from ably.pubsub.objects.enums import ObjectsOperationSource +from ably.pubsub.objects.liveobject import CounterUpdate, LiveCounterUpdate, LiveObject +from ably.pubsub.objects.objectmessage import ObjectOperationAction +from ably.pubsub.util.clock import Clock + +if TYPE_CHECKING: + from ably.pubsub.objects.objectmessage import CounterInc, ObjectMessage, ObjectOperation + from ably.pubsub.objects.objectspool import ObjectsPool + +log = logging.getLogger(__name__) + + +class InternalLiveCounter(LiveObject): + """RTLC1: a counter holding a 64-bit float (RTLC3). + + A new counter has `data` 0 (RTLC4). + """ + + _update_type = LiveCounterUpdate + + def __init__(self, object_id: str, *, pool: ObjectsPool | None = None, clock: Clock | None = None): + super().__init__(object_id, pool=pool, clock=clock) + self.data: float = 0.0 # RTLC3, RTLC4b + + def value(self) -> float: + """RTLC5: the current count.""" + return self.data # RTLC5c + + async def increment(self, amount: float) -> None: + """RTLC12: publishes a COUNTER_INC of `amount` through `RealtimeObject._publish_and_apply`. + + Raises AblyException 40003 if `amount` is not a finite number (RTLC12e1); a bool + is not a number. + """ + raise NotImplementedError + + async def decrement(self, amount: float) -> None: + """RTLC13: `increment` by `-amount`, after the same validation (RTLC13c).""" + raise NotImplementedError + + def apply_operation(self, object_message: ObjectMessage, source: ObjectsOperationSource) -> bool: + """RTLC7: applies `object_message.operation`, returning whether it was applied (RTLC7g). + + The update is emitted through `notify_updated` (RTLC7d1a, RTLC7d5a, RTLC7d4c). + """ + if not self.can_apply_operation(object_message): + # RTLC7b; an operation with invalid serials has already been logged by can_apply_operation + if object_message.serial and object_message.site_code: + log.debug(f'InternalLiveCounter.apply_operation(): skipping an operation whose serial is not ' + f'newer than the one recorded for its site; serial={object_message.serial}, ' + f'site_code={object_message.site_code}, object_id={self.object_id}') + return False + + if source == ObjectsOperationSource.CHANNEL: + # RTLC7c: the serial is recorded whether or not the operation then applies + self.site_timeserials[object_message.site_code] = object_message.serial + + if self.is_tombstone: + return False # RTLC7e + + operation = object_message.operation + if operation.action == ObjectOperationAction.COUNTER_CREATE: + update = self.apply_counter_create(operation, object_message) # RTLC7d1 + elif operation.action == ObjectOperationAction.COUNTER_INC: + update = self.apply_counter_inc(operation.counter_inc, object_message) # RTLC7d5 + elif operation.action == ObjectOperationAction.OBJECT_DELETE: + update = self.tombstone(object_message) # RTLC7d4, RTLO5 + else: + # RTLC7d3 + log.warning(f'InternalLiveCounter.apply_operation(): skipping an object operation message with ' + f'an unsupported action; action={operation.action!r}, object_id={self.object_id}') + return False + + self.notify_updated(update) # RTLC7d1a, RTLC7d5a, RTLC7d4c + return True # RTLC7d1b, RTLC7d5b, RTLC7d4b + + def replace_data(self, object_message: ObjectMessage) -> LiveCounterUpdate: + """RTLC6: replaces this counter's data with `object_message.object`, returning the diff.""" + object_state = object_message.object + self.site_timeserials = dict(object_state.site_timeserials) # RTLC6a + + if self.is_tombstone: + return LiveCounterUpdate(noop=True) # RTLC6e, RTLC6e1 + + if object_state.tombstone: + return self.tombstone(object_message) # RTLC6f, RTLC6f2 + + previous_data = self.data # RTLC6g + self.create_operation_is_merged = False # RTLC6b + count = object_state.counter.count if object_state.counter is not None else None + self.data = count if count is not None else 0.0 # RTLC6c + if object_state.create_op is not None: + self.merge_initial_value(object_state.create_op, object_message) # RTLC6d + + update = self.diff(previous_data, self.data) # RTLC6h + update.object_message = object_message + return update + + def apply_counter_create(self, operation: ObjectOperation, object_message: ObjectMessage) -> LiveCounterUpdate: + """RTLC8: applies a COUNTER_CREATE.""" + if self.create_operation_is_merged: + # RTLC8b + log.debug(f'InternalLiveCounter.apply_counter_create(): skipping a COUNTER_CREATE for a counter ' + f'whose create operation is already merged; object_id={self.object_id}') + return LiveCounterUpdate(noop=True) + return self.merge_initial_value(operation, object_message) # RTLC8c, RTLC8e + + def apply_counter_inc(self, counter_inc: CounterInc | None, + object_message: ObjectMessage) -> LiveCounterUpdate: + """RTLC9: applies a COUNTER_INC. + + A COUNTER_INC with no `number`, or with no payload at all, is a no-op (RTLC9h). + """ + number = counter_inc.number if counter_inc is not None else None + if number is None: + return LiveCounterUpdate(noop=True) # RTLC9h + + self.data += number # RTLC9f + return LiveCounterUpdate(update=CounterUpdate(amount=number), object_message=object_message) # RTLC9g + + def merge_initial_value(self, operation: ObjectOperation, object_message: ObjectMessage) -> LiveCounterUpdate: + """RTLC16: merges the initial value of a create operation into this counter. + + The `CounterCreate` is `operation.resolved_counter_create`. + """ + counter_create = operation.resolved_counter_create + count = counter_create.count if counter_create is not None else None + self.create_operation_is_merged = True # RTLC16b + if count is None: + return LiveCounterUpdate(noop=True) # RTLC16d + + self.data += count # RTLC16a + return LiveCounterUpdate(update=CounterUpdate(amount=count), object_message=object_message) # RTLC16c + + def clear_data(self) -> None: + """RTO27a1: resets `data` to 0, emitting nothing.""" + self.data = 0.0 # RTLC4b + + @staticmethod + def diff(previous_data: float, new_data: float, *, for_tombstone: bool = False) -> LiveCounterUpdate: + """RTLC14: the update between two counter values. + + A zero difference is a no-op (RTLC14c), unless the diff is computed for a tombstone + (`for_tombstone`, RTLO4e5). + """ + amount = new_data - previous_data # RTLC14b + if amount == 0 and not for_tombstone: + return LiveCounterUpdate(noop=True) # RTLC14c + return LiveCounterUpdate(update=CounterUpdate(amount=amount)) diff --git a/ably/pubsub/objects/livemap.py b/ably/pubsub/objects/livemap.py new file mode 100644 index 00000000..74a0e3ac --- /dev/null +++ b/ably/pubsub/objects/livemap.py @@ -0,0 +1,387 @@ +"""The last-write-wins map CRDT (RTLM*).""" + +from __future__ import annotations + +import dataclasses +import logging +from typing import TYPE_CHECKING, Any + +from ably.pubsub.objects.enums import ObjectsOperationSource +from ably.pubsub.objects.liveobject import MAP_KEY_REMOVED, MAP_KEY_UPDATED, LiveMapUpdate, LiveObject +from ably.pubsub.objects.objectmessage import ( + MapRemove, + MapSet, + ObjectOperationAction, + ObjectsMapEntry, + ObjectsMapSemantics, +) +from ably.pubsub.util.clock import Clock + +if TYPE_CHECKING: + from ably.pubsub.objects.objectmessage import ObjectData, ObjectMessage, ObjectOperation + from ably.pubsub.objects.objectspool import ObjectsPool + from ably.pubsub.objects.valuetypes import LiveMapValue + +log = logging.getLogger(__name__) + + +class InternalLiveMap(LiveObject): + """RTLM1: a map of `ObjectsMapEntry` values by key, with per-entry last-write-wins (RTLM3). + + A new map has empty `data` and no `clear_timeserial` (RTLM4). A value read from the + map is a primitive, an `InternalLiveMap` or `InternalLiveCounter` looked up in `pool`, + or None. + """ + + _update_type = LiveMapUpdate + + def __init__(self, object_id: str, semantics: ObjectsMapSemantics = ObjectsMapSemantics.LWW, *, + pool: ObjectsPool | None = None, clock: Clock | None = None): + super().__init__(object_id, pool=pool, clock=clock) + self.semantics: ObjectsMapSemantics = semantics # RTLM4b + self.data: dict[str, ObjectsMapEntry] = {} # RTLM3, RTLM4c + self.clear_timeserial: str | None = None # RTLM25, RTLM4d + + def get(self, key: str) -> Any: + """RTLM5: the value at `key`, or None if there is none, it is tombstoned, or this map is.""" + if self.is_tombstone: + return None # RTLM5e + entry = self.data.get(key) + if entry is None: + return None # RTLM5d1 + if self.is_entry_tombstoned(entry): + return None # RTLM5d2h + return self._resolve(entry.data) + + def size(self) -> int: + """RTLM10: the number of entries that are not tombstoned (RTLM14).""" + return sum(1 for entry in self.data.values() if not self.is_entry_tombstoned(entry)) # RTLM10d + + def entries(self) -> list[tuple[str, Any]]: + """RTLM11: `(key, value)` for each entry that is not tombstoned, values as `get` returns them.""" + # RTLM11d1, RTLM11d3; a value that resolves to None is still returned (RTLM11d3a) + return [(key, self._resolve(entry.data)) for key, entry in self.data.items() + if not self.is_entry_tombstoned(entry)] + + def keys(self) -> list[str]: + """RTLM12: the keys `entries` returns.""" + return [key for key, _ in self.entries()] + + def values(self) -> list[Any]: + """RTLM13: the values `entries` returns.""" + return [value for _, value in self.entries()] + + async def set(self, key: str, value: LiveMapValue) -> None: + """RTLM20: publishes a MAP_SET, preceded by the creates a `LiveMap` or `LiveCounter` value + evaluates to, through `RealtimeObject._publish_and_apply`. + + Raises AblyException 40003 for a key that is not a string and 40013 for a value of + an unsupported type (RTLM20e1, RTLMV4b, RTLMV4c). + """ + raise NotImplementedError + + async def remove(self, key: str) -> None: + """RTLM21: publishes a MAP_REMOVE through `RealtimeObject._publish_and_apply`. + + Raises AblyException 40003 for a key that is not a string (RTLM21e1). + """ + raise NotImplementedError + + def is_entry_tombstoned(self, entry: ObjectsMapEntry) -> bool: + """RTLM14: whether `entry` is tombstoned, or references a tombstoned object in `pool`.""" + if entry.tombstone: + return True # RTLM14a + referenced = self._referenced_object(entry.data) + return referenced is not None and referenced.is_tombstone # RTLM14c, RTLM14b + + def apply_operation(self, object_message: ObjectMessage, source: ObjectsOperationSource) -> bool: + """RTLM15: applies `object_message.operation`, returning whether it was applied (RTLM15g). + + The update is emitted through `notify_updated` (RTLM15d1a and its siblings). + """ + if not self.can_apply_operation(object_message): + # RTLM15b; an operation with invalid serials has already been logged by can_apply_operation + if object_message.serial and object_message.site_code: + log.debug(f'InternalLiveMap.apply_operation(): skipping an operation whose serial is not ' + f'newer than the one recorded for its site; serial={object_message.serial}, ' + f'site_code={object_message.site_code}, object_id={self.object_id}') + return False + + if source == ObjectsOperationSource.CHANNEL: + # RTLM15c: the serial is recorded whether or not the operation then applies + self.site_timeserials[object_message.site_code] = object_message.serial + + if self.is_tombstone: + return False # RTLM15e + + operation = object_message.operation + action = operation.action + if action == ObjectOperationAction.MAP_CREATE: + update = self.apply_map_create(operation, object_message) # RTLM15d1 + elif action == ObjectOperationAction.MAP_SET: + map_set = operation.map_set + if map_set is None or map_set.key is None or map_set.value is None: + self._log_missing_payload(action) + return False + update = self.apply_map_set(map_set, object_message.serial, object_message) # RTLM15d6 + elif action == ObjectOperationAction.MAP_REMOVE: + map_remove = operation.map_remove + if map_remove is None or map_remove.key is None: + self._log_missing_payload(action) + return False + update = self.apply_map_remove(map_remove, object_message.serial, object_message.serial_timestamp, + object_message) # RTLM15d7 + elif action == ObjectOperationAction.OBJECT_DELETE: + update = self.tombstone(object_message) # RTLM15d5, RTLO5 + elif action == ObjectOperationAction.MAP_CLEAR: + update = self.apply_map_clear(object_message.serial, object_message) # RTLM15d8 + else: + # RTLM15d4 + log.warning(f'InternalLiveMap.apply_operation(): skipping an object operation message with an ' + f'unsupported action; action={action!r}, object_id={self.object_id}') + return False + + self.notify_updated(update) # RTLM15d1a, RTLM15d6a, RTLM15d7a, RTLM15d5c, RTLM15d8a + return True # RTLM15d1b, RTLM15d6b, RTLM15d7b, RTLM15d5b, RTLM15d8b + + def replace_data(self, object_message: ObjectMessage) -> LiveMapUpdate: + """RTLM6: replaces this map's data with `object_message.object`, returning the diff.""" + object_state = object_message.object + self.site_timeserials = dict(object_state.site_timeserials) # RTLM6a + + if self.is_tombstone: + return LiveMapUpdate(noop=True) # RTLM6e, RTLM6e1 + + if object_state.tombstone: + return self.tombstone(object_message) # RTLM6f, RTLM6f2 + + previous_data = self.data # RTLM6g + self.create_operation_is_merged = False # RTLM6b + objects_map = object_state.map + self.clear_timeserial = objects_map.clear_timeserial if objects_map is not None else None # RTLM6i + entries = objects_map.entries if objects_map is not None and objects_map.entries is not None else {} + # RTLM6c. Each entry is copied, so that applying operations to this map never alters the + # ObjectState it was replaced from. + self.data = {} + for key, entry in entries.items(): + entry = dataclasses.replace(entry, tombstoned_at=None) + if entry.tombstone: + entry.tombstoned_at = self._tombstoned_at(entry.serial_timestamp) # RTLM6c1 + self.data[key] = entry + if object_state.create_op is not None: + self.merge_initial_value(object_state.create_op, object_message) # RTLM6d + + update = self.diff(previous_data, self.data) # RTLM6h + update.object_message = object_message + return update + + def apply_map_create(self, operation: ObjectOperation, object_message: ObjectMessage) -> LiveMapUpdate: + """RTLM16: applies a MAP_CREATE.""" + if self.create_operation_is_merged: + # RTLM16b + log.debug(f'InternalLiveMap.apply_map_create(): skipping a MAP_CREATE for a map whose create ' + f'operation is already merged; object_id={self.object_id}') + return LiveMapUpdate(noop=True) + return self.merge_initial_value(operation, object_message) # RTLM16d, RTLM16f + + def apply_map_set(self, map_set: MapSet, serial: str | None, object_message: ObjectMessage) -> LiveMapUpdate: + """RTLM7: applies a MAP_SET for `map_set.key`.""" + key = map_set.key + if self._is_at_or_before_clear(serial): + # RTLM7h + log.debug(f'InternalLiveMap.apply_map_set(): skipping a MAP_SET at or before the map\'s clear ' + f'serial; key={key!r}, serial={serial}, clear_timeserial={self.clear_timeserial}, ' + f'object_id={self.object_id}') + return LiveMapUpdate(noop=True) + + entry = self.data.get(key) + if entry is not None: + if not self.can_apply_map_operation(entry.timeserial, serial): + # RTLM7a1 + log.debug(f'InternalLiveMap.apply_map_set(): skipping a MAP_SET not newer than the entry; ' + f'key={key!r}, serial={serial}, entry serial={entry.timeserial}, ' + f'object_id={self.object_id}') + return LiveMapUpdate(noop=True) + self._release_reference(entry.data, key) # RTLM7a3 + entry.data = map_set.value # RTLM7a2e + entry.timeserial = serial # RTLM7a2b + entry.tombstone = False # RTLM7a2c + entry.tombstoned_at = None # RTLM7a2d + else: + # RTLM7b4, RTLM7b2, RTLM7b3 + self.data[key] = ObjectsMapEntry(data=map_set.value, timeserial=serial, tombstone=False, + tombstoned_at=None) + + value = map_set.value + if value is not None and value.object_id and self.pool is not None: + # RTLM7g + referenced = self.pool.create_zero_value_object_if_not_exists(value.object_id) # RTLM7g1 + if referenced is not None: + referenced.add_parent_reference(self, key) # RTLM7g2 + + return LiveMapUpdate(update={key: MAP_KEY_UPDATED}, object_message=object_message) # RTLM7f + + def apply_map_remove(self, map_remove: MapRemove, serial: str | None, serial_timestamp: int | None, + object_message: ObjectMessage) -> LiveMapUpdate: + """RTLM8: applies a MAP_REMOVE for `map_remove.key`.""" + key = map_remove.key + if self._is_at_or_before_clear(serial): + # RTLM8g + log.debug(f'InternalLiveMap.apply_map_remove(): skipping a MAP_REMOVE at or before the map\'s ' + f'clear serial; key={key!r}, serial={serial}, clear_timeserial={self.clear_timeserial}, ' + f'object_id={self.object_id}') + return LiveMapUpdate(noop=True) + + entry = self.data.get(key) + if entry is not None: + if not self.can_apply_map_operation(entry.timeserial, serial): + # RTLM8a1 + log.debug(f'InternalLiveMap.apply_map_remove(): skipping a MAP_REMOVE not newer than the ' + f'entry; key={key!r}, serial={serial}, entry serial={entry.timeserial}, ' + f'object_id={self.object_id}') + return LiveMapUpdate(noop=True) + self._release_reference(entry.data, key) # RTLM8a3 + entry.data = None # RTLM8a2a + entry.timeserial = serial # RTLM8a2b + entry.tombstone = True # RTLM8a2c + entry.tombstoned_at = self._tombstoned_at(serial_timestamp) # RTLM8a2d + else: + # RTLM8b1, RTLM8b2, RTLM8b3 + self.data[key] = ObjectsMapEntry(data=None, timeserial=serial, tombstone=True, + tombstoned_at=self._tombstoned_at(serial_timestamp)) + + return LiveMapUpdate(update={key: MAP_KEY_REMOVED}, object_message=object_message) # RTLM8e + + def apply_map_clear(self, serial: str | None, object_message: ObjectMessage) -> LiveMapUpdate: + """RTLM24: applies a MAP_CLEAR.""" + if not serial: + # A MAP_CLEAR with no serial cannot be ordered against the map's entries + log.warning(f'InternalLiveMap.apply_map_clear(): skipping a MAP_CLEAR with no serial; ' + f'object_id={self.object_id}') + return LiveMapUpdate(noop=True) + + if self.clear_timeserial is not None and self.clear_timeserial > serial: + # RTLM24c + log.debug(f'InternalLiveMap.apply_map_clear(): skipping a MAP_CLEAR older than the map\'s clear ' + f'serial; serial={serial}, clear_timeserial={self.clear_timeserial}, ' + f'object_id={self.object_id}') + return LiveMapUpdate(noop=True) + + self.clear_timeserial = serial # RTLM24d + removed: dict[str, str] = {} + for key, entry in list(self.data.items()): + # RTLM24e1 + if entry.timeserial is None or serial > entry.timeserial: + self._release_reference(entry.data, key) # RTLM24e1c + del self.data[key] # RTLM24e1a + removed[key] = MAP_KEY_REMOVED # RTLM24e1b + + return LiveMapUpdate(update=removed, object_message=object_message) # RTLM24f + + def merge_initial_value(self, operation: ObjectOperation, object_message: ObjectMessage) -> LiveMapUpdate: + """RTLM23: merges the initial entries of a create operation into this map. + + The `MapCreate` is `operation.resolved_map_create`. + """ + map_create = operation.resolved_map_create + merged: dict[str, str] = {} + entries = map_create.entries if map_create is not None else {} + for key, entry in entries.items(): + # The serial an entry is applied with is the entry's own, not the message's + if entry.tombstone: + # RTLM23a2 + update = self.apply_map_remove(MapRemove(key=key), entry.timeserial, entry.serial_timestamp, + object_message) + else: + # RTLM23a1 + update = self.apply_map_set(MapSet(key=key, value=entry.data), entry.timeserial, object_message) + if not update.noop: + merged.update(update.update) # RTLM23c + + self.create_operation_is_merged = True # RTLM23b + return LiveMapUpdate(update=merged, object_message=object_message) # RTLM23c + + def gc_tombstoned_entries(self, grace_period_ms: int, now_ms: int) -> None: + """RTLM19: removes tombstoned entries whose `tombstoned_at` is `grace_period_ms` or more + before `now_ms`.""" + for key, entry in list(self.data.items()): + # RTLM19a1 + if (entry.tombstone and entry.tombstoned_at is not None + and now_ms - entry.tombstoned_at >= grace_period_ms): + del self.data[key] + + def clear_data(self) -> None: + """RTO27a1: resets `data` to empty and `clear_timeserial` to None, emitting nothing. + + Each object an entry references stops recording this map as a parent (RTLO4e9). + """ + for key, entry in self.data.items(): + self._release_reference(entry.data, key) + self.data = {} # RTLM4c + self.clear_timeserial = None # RTLM4d + + @staticmethod + def can_apply_map_operation(entry_serial: str | None, operation_serial: str | None) -> bool: + """RTLM9: whether an operation with `operation_serial` may replace an entry with `entry_serial`.""" + if not entry_serial and not operation_serial: + return False # RTLM9b + if not entry_serial: + return True # RTLM9d + if not operation_serial: + return False # RTLM9c + return operation_serial > entry_serial # RTLM9a, RTLM9e + + @staticmethod + def diff(previous_data: dict[str, ObjectsMapEntry], new_data: dict[str, ObjectsMapEntry], *, + for_tombstone: bool = False) -> LiveMapUpdate: + """RTLM22: the update between two versions of a map's data. + + Only entries whose own `tombstone` flag is false are compared. An empty diff is a + no-op (RTLM22c), unless it is computed for a tombstone (`for_tombstone`, RTLO4e5). + """ + previous_live = {key: entry for key, entry in previous_data.items() if not entry.tombstone} + new_live = {key: entry for key, entry in new_data.items() if not entry.tombstone} + update: dict[str, str] = {} + for key in previous_live: + if key not in new_live: + update[key] = MAP_KEY_REMOVED # RTLM22b1 + for key, entry in new_live.items(): + if key not in previous_live or previous_live[key].data != entry.data: + update[key] = MAP_KEY_UPDATED # RTLM22b2, RTLM22b3 + + if not update and not for_tombstone: + return LiveMapUpdate(noop=True) # RTLM22c + return LiveMapUpdate(update=update) + + def _resolve(self, data: ObjectData | None) -> Any: + """RTLM5d2: the value `data` holds: a primitive, the object it references, or None.""" + if data is None: + return None + # RTLM5d2b-RTLM5d2e, and a JSON value (OD2g) + primitive = data.value + if primitive is not None: + return primitive + if data.object_id is not None: + return self._referenced_object(data) # RTLM5d2f1, RTLM5d2f2 + return None # RTLM5d2g + + def _referenced_object(self, data: ObjectData | None) -> LiveObject | None: + """The object in `pool` that `data` references, if there is one.""" + if data is None or data.object_id is None or self.pool is None: + return None + return self.pool.get(data.object_id) + + def _release_reference(self, data: ObjectData | None, key: str) -> None: + """RTLM7a3, RTLM8a3, RTLM24e1c, RTLO4e9: removes this map at `key` from the parent references of + the object `data` references, if it is in `pool`.""" + referenced = self._referenced_object(data) + if referenced is not None: + referenced.remove_parent_reference(self, key) + + def _is_at_or_before_clear(self, serial: str | None) -> bool: + """RTLM7h, RTLM8g: whether an operation with `serial` is no later than the map's last MAP_CLEAR.""" + return self.clear_timeserial is not None and (not serial or self.clear_timeserial >= serial) + + def _log_missing_payload(self, action: ObjectOperationAction) -> None: + log.warning(f'InternalLiveMap.apply_operation(): skipping a {action.name} operation with no key or ' + f'value; object_id={self.object_id}') diff --git a/ably/pubsub/objects/liveobject.py b/ably/pubsub/objects/liveobject.py new file mode 100644 index 00000000..cc75a3d5 --- /dev/null +++ b/ably/pubsub/objects/liveobject.py @@ -0,0 +1,255 @@ +"""The behaviour every live object shares (RTLO*), and the updates objects emit.""" + +from __future__ import annotations + +import logging +from abc import ABC, abstractmethod +from dataclasses import dataclass, field +from typing import TYPE_CHECKING, Any, Callable + +from ably.pubsub.objects.defaults import ROOT_OBJECT_ID +from ably.pubsub.objects.subscription import Subscription +from ably.pubsub.util.clock import Clock + +if TYPE_CHECKING: + from ably.pubsub.objects.enums import ObjectsOperationSource + from ably.pubsub.objects.livemap import InternalLiveMap + from ably.pubsub.objects.objectmessage import ObjectMessage + from ably.pubsub.objects.objectspool import ObjectsPool + from ably.pubsub.objects.realtimeobject import RealtimeObject + +log = logging.getLogger(__name__) + +# RTLM18b: the change recorded against a map key in a LiveMapUpdate +MAP_KEY_UPDATED = 'updated' +MAP_KEY_REMOVED = 'removed' + + +@dataclass +class LiveObjectUpdate: + """RTLO4b4: a change to a live object's data, as the object emits it.""" + + update: Any = None # RTLO4b4a + noop: bool = False # RTLO4b4b + object_message: ObjectMessage | None = None # RTLO4b4d + tombstone: bool = False # RTLO4b4e + + +@dataclass +class CounterUpdate: + """RTLC11b: what a counter update changed.""" + + amount: float # RTLC11b1 + + +@dataclass +class LiveCounterUpdate(LiveObjectUpdate): + """RTLC11: an update emitted by an `InternalLiveCounter`. + + `update` is a `CounterUpdate`, or None on a no-op. + """ + + update: CounterUpdate | None = None + + +@dataclass +class LiveMapUpdate(LiveObjectUpdate): + """RTLM18: an update emitted by an `InternalLiveMap`. + + `update` maps each changed key to `'updated'` or `'removed'` (RTLM18b). + """ + + update: dict[str, str] = field(default_factory=dict) + + +class LiveObject(ABC): + """RTLO1: the state and behaviour `InternalLiveCounter` and `InternalLiveMap` share. + + A live object is constructible on its own, with no channel, so that its CRDT + behaviour can be driven directly. `pool` is the `ObjectsPool` it reads other objects + from (RTLM5d2f, RTLM14c, RTLO4f); an object added to a pool with `pool[object_id] = obj` + is adopted by it. `clock` is what RTLO6b reads the local time from; without one the + object uses its pool's clock, and with neither, the system clock. + """ + + # The update type this object emits, which a no-op it returns is also an instance of + _update_type: type[LiveObjectUpdate] = LiveObjectUpdate + + def __init__(self, object_id: str, *, pool: ObjectsPool | None = None, clock: Clock | None = None): + self.object_id: str = object_id # RTLO3a + self.site_timeserials: dict[str, str] = {} # RTLO3b + self.create_operation_is_merged: bool = False # RTLO3c + self.is_tombstone: bool = False # RTLO3d + self.tombstoned_at: int | None = None # RTLO3e + self.parent_references: dict[str, set[str]] = {} # RTLO3f + self.pool: ObjectsPool | None = pool + self.data: Any = None + self._clock: Clock | None = clock + # (token, listener) for each subscribe call, so that one registration can be removed + # even when the same listener is registered more than once + self._listeners: list[tuple[object, Callable[[LiveObjectUpdate], None]]] = [] + + @property + def clock(self) -> Clock: + """The clock RTLO6b reads the local time from.""" + if self._clock is not None: + return self._clock + if self.pool is not None: + return self.pool.clock + return Clock() + + @property + def realtime_object(self) -> RealtimeObject | None: + """The `RealtimeObject` this object's pool belongs to, if any.""" + return self.pool.realtime_object if self.pool is not None else None + + def subscribe(self, listener: Callable[[LiveObjectUpdate], None]) -> Subscription: + """RTLO4b: registers `listener` for the updates this object emits. + + Listeners are called synchronously from `notify_updated`, in registration order. + """ + token = object() + self._listeners.append((token, listener)) + + def deregister() -> None: + self._listeners = [entry for entry in self._listeners if entry[0] is not token] + + return Subscription(deregister) + + def notify_updated(self, update: LiveObjectUpdate) -> None: + """RTLO4b4c: emits `update`. + + Every update an operation produces passes through here, a no-op included, and a + no-op stops here (RTLO4b4c1). Otherwise the `subscribe` listeners are called + (RTLO4b4c3a), path subscriptions are dispatched through the `RealtimeObject`, if + there is one (RTLO4b4c3b), and a tombstone update then deregisters the `subscribe` + listeners (RTLO4b4c3c). + """ + if update.noop: + return + + for _, listener in list(self._listeners): + try: + listener(update) + except Exception: + log.exception(f'LiveObject.notify_updated(): a subscription listener raised; ' + f'object_id={self.object_id}') + + realtime_object = self.realtime_object + if realtime_object is not None: + realtime_object._path_object_subscription_register.dispatch(self, update) + + if update.tombstone: + self._listeners = [] + + def can_apply_operation(self, object_message: ObjectMessage) -> bool: + """RTLO4a: whether `object_message`'s serial is newer than this object's for its site.""" + serial = object_message.serial + site_code = object_message.site_code + # RTLO4a3 + if not isinstance(serial, str) or not serial or not isinstance(site_code, str) or not site_code: + log.warning(f'LiveObject.can_apply_operation(): object operation message has invalid serial ' + f'values, skipping it; serial={serial!r}, site_code={site_code!r}, ' + f'object_id={self.object_id}') + return False + + site_serial = self.site_timeserials.get(site_code) # RTLO4a4 + if not site_serial: + return True # RTLO4a5 + return serial > site_serial # RTLO4a6 + + @abstractmethod + def apply_operation(self, object_message: ObjectMessage, source: ObjectsOperationSource) -> bool: + """RTLC7, RTLM15: applies `object_message.operation`, returning whether it was applied. + + The resulting update is emitted through `notify_updated`, not returned. + """ + + @abstractmethod + def replace_data(self, object_message: ObjectMessage) -> LiveObjectUpdate: + """RTLC6, RTLM6: replaces this object's data with `object_message.object`. + + The update is returned, not emitted; the caller emits it (RTO5c7). + """ + + def tombstone(self, object_message: ObjectMessage) -> LiveObjectUpdate: + """RTLO4e: tombstones this object, returning the resulting update. + + The update has `tombstone` set and `object_message` populated, and is never a no-op, + except for the root object, which is never tombstoned (RTLO4e10). + """ + if self.object_id == ROOT_OBJECT_ID: + log.warning(f'LiveObject.tombstone(): attempt to tombstone the root object was rejected; ' + f'serial={object_message.serial}, site_code={object_message.site_code}, ' + f'message id={object_message.id}') + return self._update_type(noop=True) + + self.is_tombstone = True # RTLO4e2 + self.tombstoned_at = self._tombstoned_at(object_message.serial_timestamp) # RTLO4e3 + previous_data = self.data + self.clear_data() # RTLO4e9, RTLO4e4 + update = self.diff(previous_data, self.data, for_tombstone=True) # RTLO4e5 + update.tombstone = True # RTLO4e6 + update.object_message = object_message # RTLO4e7 + return update + + @abstractmethod + def clear_data(self) -> None: + """RTO27a1: resets this object's data to that of a new empty object, emitting nothing.""" + + @staticmethod + @abstractmethod + def diff(previous_data: Any, new_data: Any, *, for_tombstone: bool = False) -> LiveObjectUpdate: + """RTLC14, RTLM22: the update between two versions of this type of object's data.""" + + def add_parent_reference(self, parent: InternalLiveMap, key: str) -> None: + """RTLO4g: records that `parent` references this object at `key`.""" + self.parent_references.setdefault(parent.object_id, set()).add(key) + + def remove_parent_reference(self, parent: InternalLiveMap, key: str) -> None: + """RTLO4h: removes the record that `parent` references this object at `key`.""" + keys = self.parent_references.get(parent.object_id) + if keys is None: + return # RTLO4h1 + keys.discard(key) # RTLO4h2 + if not keys: + del self.parent_references[parent.object_id] # RTLO4h3 + + def get_full_paths(self) -> list[list[str]]: + """RTLO4f: every key-path from the root map to this object, each once, in no set order. + + Parents are looked up by object id in this object's pool. + """ + paths: list[list[str]] = [] + seen: set[tuple[str, ...]] = set() + # Each item is an object, the keys from it down to this object, and the ids of the + # objects already on that path, which a simple path does not revisit (RTLO4f2) + stack: list[tuple[LiveObject, list[str], frozenset[str]]] = [(self, [], frozenset())] + while stack: + live_object, path, visited = stack.pop() + if live_object.object_id in visited: + continue + if live_object.object_id == ROOT_OBJECT_ID: + if tuple(path) not in seen: + seen.add(tuple(path)) + paths.append(path) + continue + if self.pool is None: + continue + visited = visited | {live_object.object_id} + for parent_id, keys in live_object.parent_references.items(): + parent = self.pool.get(parent_id) + if parent is None: + continue + for key in keys: + stack.append((parent, [key, *path], visited)) + return paths + + def _tombstoned_at(self, serial_timestamp: int | None) -> int: + """RTLO6: the time something is tombstoned at, given the operation's `serial_timestamp`.""" + if serial_timestamp is not None: + return serial_timestamp # RTLO6a + # RTLO6b, RTLO6b1 + log.debug(f'LiveObject._tombstoned_at(): no serial_timestamp for the operation, using the local ' + f'clock instead; object_id={self.object_id}') + return self.clock.now_ms() diff --git a/ably/pubsub/objects/objectid.py b/ably/pubsub/objects/objectid.py new file mode 100644 index 00000000..8a7a349a --- /dev/null +++ b/ably/pubsub/objects/objectid.py @@ -0,0 +1,31 @@ +"""Object ids generated by the client (RTO14) and parsed by it (RTO6b).""" + +from __future__ import annotations + +import base64 +import hashlib +import os + +# 12 random bytes base64-encode to 16 characters, the RTLCV4d/RTLMV4g minimum +_NONCE_ENTROPY_BYTES = 12 + + +def generate_object_id(object_type: str, initial_value: str, nonce: str, timestamp_ms: int) -> str: + """RTO14: `[type]:[hash]@[timestamp]`, where the hash is the URL-safe, unpadded base64 of + the SHA-256 of `initial_value:nonce` (RTO14b). + + `object_type` is `'map'` or `'counter'` (RTO14a1). + """ + digest = hashlib.sha256(f'{initial_value}:{nonce}'.encode()).digest() # RTO14b1 + hash_part = base64.urlsafe_b64encode(digest).decode('ascii').rstrip('=') # RTO14b2 + return f'{object_type}:{hash_part}@{int(timestamp_ms)}' # RTO14c + + +def generate_nonce() -> str: + """RTLCV4d, RTLMV4g: a random string of at least 16 characters.""" + return base64.urlsafe_b64encode(os.urandom(_NONCE_ENTROPY_BYTES)).decode('ascii') + + +def object_type_of(object_id: str) -> str: + """RTO6b1: the type an object id names, the part before the first `:`.""" + return object_id.split(':', 1)[0] diff --git a/ably/pubsub/objects/objectmessage.py b/ably/pubsub/objects/objectmessage.py new file mode 100644 index 00000000..3183199d --- /dev/null +++ b/ably/pubsub/objects/objectmessage.py @@ -0,0 +1,657 @@ +"""The internal object message types and their wire encoding. + +These are the `ObjectMessage` (OM*), `ObjectOperation` (OOP*), `ObjectState` (OST*) +and `ObjectData` (OD*) types of features.md, with the protocol v6 operation payloads +(`MapCreate`, `MapSet`, `MapRemove`, `CounterCreate`, `CounterInc`, `ObjectDelete`, +`MapClear`, `MapCreateWithObjectId`, `CounterCreateWithObjectId`) and the `ObjectsMap`, +`ObjectsMapEntry` and `ObjectsCounter` state types. + +Every type decodes from a wire dictionary with `from_dict(obj, format)` and encodes to +one with `to_dict(format)`, where `format` is `'json'` or `'msgpack'`. The two forms +differ only in how values are carried: + +- `ObjectData.bytes` is a base64 string on the JSON wire and raw binary on msgpack + (OD2d, OD4c2, OD4d2, OD5a1, OD5b2). +- `ObjectData.json` is a JSON-encoded string on both wires (OD2g), and a decoded `dict` + or `list` here. +- Numbers are float64 (OD4c3) and decode to `float`. +- Enums are carried as their integer wire values (OOP2, OMP2). + +Decoded values are what the rest of the package works with: `ObjectData.bytes` holds +`bytes`, `ObjectData.json` holds the parsed value, and wire names are snake_cased. +Fields that are only ever held locally (`ObjectsMapEntry.tombstoned_at`, RTLM3a1, and +the `derived_from` of the `*CreateWithObjectId` payloads, RTLMV4j5 and RTLCV4g5) are +never encoded. +""" + +from __future__ import annotations + +import base64 +import json +from dataclasses import dataclass, field +from enum import IntEnum +from typing import Any + +WIRE_FORMAT_JSON = 'json' +WIRE_FORMAT_MSGPACK = 'msgpack' + + +class ObjectOperationAction(IntEnum): + """OOP2: the operation an `ObjectOperation` describes, by its wire value. + + `UNKNOWN` stands for a wire value this library does not recognise (OOP2a). Such an + operation decodes without error, is never applied and is never encoded. + """ + + UNKNOWN = -1 + MAP_CREATE = 0 + MAP_SET = 1 + MAP_REMOVE = 2 + COUNTER_CREATE = 3 + COUNTER_INC = 4 + OBJECT_DELETE = 5 + MAP_CLEAR = 6 + + @classmethod + def from_wire(cls, value: Any) -> ObjectOperationAction: + """The member for a wire value, or `UNKNOWN` for one that is not recognised (OOP2a).""" + if isinstance(value, bool): + return cls.UNKNOWN + try: + return cls(value) + except ValueError: + return cls.UNKNOWN + + +class ObjectsMapSemantics(IntEnum): + """OMP2: the conflict-resolution semantics of a map, by its wire value. + + `UNKNOWN` stands for a wire value this library does not recognise (OMP2a). + """ + + UNKNOWN = -1 + LWW = 0 + + @classmethod + def from_wire(cls, value: Any) -> ObjectsMapSemantics: + """The member for a wire value, or `UNKNOWN` for one that is not recognised (OMP2a).""" + if isinstance(value, bool): + return cls.UNKNOWN + try: + return cls(value) + except ValueError: + return cls.UNKNOWN + + +def _check_format(format: str) -> None: + if format not in (WIRE_FORMAT_JSON, WIRE_FORMAT_MSGPACK): + raise ValueError(f"Unknown wire format {format!r}; expected 'json' or 'msgpack'") + + +def _decode_number(value: Any) -> Any: + # OD4c3: numbers are float64 on the wire, so a decoded number is always a float + if isinstance(value, (int, float)) and not isinstance(value, bool): + return float(value) + return value + + +def _encode_number(value: Any, format: str) -> Any: + if format == WIRE_FORMAT_MSGPACK and isinstance(value, int) and not isinstance(value, bool): + return float(value) + return value + + +def _decode_bytes(value: Any) -> Any: + # OD5a1, OD5b2: raw binary on msgpack, a base64 string on JSON + if isinstance(value, (bytes, bytearray)): + return bytes(value) + if isinstance(value, str): + return base64.b64decode(value) + return value + + +def _encode_bytes(value: Any, format: str) -> Any: + # OD4c2, OD4d2 + if format == WIRE_FORMAT_MSGPACK: + return bytes(value) + return base64.b64encode(value).decode('ascii') + + +def _encode_json(value: Any) -> str: + return json.dumps(value, separators=(',', ':')) + + +@dataclass +class ObjectData: + """OD1: a value held in an object, either a primitive or a reference to another object. + + At most one of the value fields is set (OD2). + """ + + object_id: str | None = None # OD2a + encoding: str | None = None # OD2b + boolean: bool | None = None # OD2c + bytes: bytes | None = None # OD2d + number: float | None = None # OD2e + string: str | None = None # OD2f + json: dict | list | None = None # OD2g + + @property + def value(self) -> Any: + """The primitive this data holds, or None when it holds a reference or nothing.""" + for candidate in (self.boolean, self.bytes, self.number, self.string, self.json): + if candidate is not None: + return candidate + return None + + @staticmethod + def from_dict(obj: dict, format: str = WIRE_FORMAT_JSON) -> ObjectData: + """Decodes an `ObjectData` per OD5.""" + _check_format(format) + json_value = obj.get('json') + if isinstance(json_value, str): + json_value = json.loads(json_value) + string_value = obj.get('string') + encoding = obj.get('encoding') + # OD5a2, OD5b3: a JSON payload carried in `string` under the legacy `json` encoding + if encoding == 'json' and string_value is not None and json_value is None: + json_value = json.loads(string_value) + string_value = None + encoding = None + return ObjectData( + object_id=obj.get('objectId'), + encoding=encoding, + boolean=obj.get('boolean'), + bytes=_decode_bytes(obj.get('bytes')), + number=_decode_number(obj.get('number')), + string=string_value, + json=json_value, + ) + + def to_dict(self, format: str = WIRE_FORMAT_JSON) -> dict: + """Encodes this `ObjectData` per OD4.""" + _check_format(format) + result: dict = {} + if self.object_id is not None: + result['objectId'] = self.object_id + if self.encoding is not None: + result['encoding'] = self.encoding + if self.boolean is not None: + result['boolean'] = self.boolean + if self.bytes is not None: + result['bytes'] = _encode_bytes(self.bytes, format) + if self.number is not None: + result['number'] = _encode_number(self.number, format) + if self.string is not None: + result['string'] = self.string + if self.json is not None: + result['json'] = _encode_json(self.json) + return result + + +@dataclass +class ObjectsMapEntry: + """OME1: the value at one key of a map. + + `tombstoned_at` (RTLM3a1) is held locally by an `InternalLiveMap` and is never sent + or received. + """ + + tombstone: bool = False # OME2a + timeserial: str | None = None # OME2b + serial_timestamp: int | None = None # OME2d + data: ObjectData | None = None # OME2c + tombstoned_at: int | None = None # RTLM3a1 + + @staticmethod + def from_dict(obj: dict, format: str = WIRE_FORMAT_JSON) -> ObjectsMapEntry: + _check_format(format) + data = obj.get('data') + return ObjectsMapEntry( + tombstone=bool(obj.get('tombstone', False)), + timeserial=obj.get('timeserial'), + serial_timestamp=obj.get('serialTimestamp'), + data=ObjectData.from_dict(data, format) if data is not None else None, + ) + + def to_dict(self, format: str = WIRE_FORMAT_JSON) -> dict: + _check_format(format) + result: dict = {} + if self.tombstone: + result['tombstone'] = True + if self.timeserial is not None: + result['timeserial'] = self.timeserial + if self.serial_timestamp is not None: + result['serialTimestamp'] = self.serial_timestamp + if self.data is not None: + result['data'] = self.data.to_dict(format) + return result + + +def _decode_entries(entries: dict | None, format: str) -> dict[str, ObjectsMapEntry] | None: + if entries is None: + return None + return {key: ObjectsMapEntry.from_dict(entry or {}, format) for key, entry in entries.items()} + + +def _encode_entries(entries: dict[str, ObjectsMapEntry], format: str) -> dict: + return {key: entry.to_dict(format) for key, entry in entries.items()} + + +@dataclass +class ObjectsMap: + """OMP1: the state of a map object, as an `ObjectState` carries it.""" + + semantics: ObjectsMapSemantics | None = None # OMP3a + entries: dict[str, ObjectsMapEntry] | None = None # OMP3b + clear_timeserial: str | None = None # OMP3c + + @staticmethod + def from_dict(obj: dict, format: str = WIRE_FORMAT_JSON) -> ObjectsMap: + _check_format(format) + semantics = obj.get('semantics') + return ObjectsMap( + semantics=ObjectsMapSemantics.from_wire(semantics) if semantics is not None else None, + entries=_decode_entries(obj.get('entries'), format), + clear_timeserial=obj.get('clearTimeserial'), + ) + + def to_dict(self, format: str = WIRE_FORMAT_JSON) -> dict: + _check_format(format) + result: dict = {} + if self.semantics is not None: + result['semantics'] = int(self.semantics) + if self.entries is not None: + result['entries'] = _encode_entries(self.entries, format) + if self.clear_timeserial is not None: + result['clearTimeserial'] = self.clear_timeserial + return result + + +@dataclass +class ObjectsCounter: + """OCN1: the state of a counter object, as an `ObjectState` carries it.""" + + count: float | None = None # OCN2a + + @staticmethod + def from_dict(obj: dict, format: str = WIRE_FORMAT_JSON) -> ObjectsCounter: + _check_format(format) + return ObjectsCounter(count=_decode_number(obj.get('count'))) + + def to_dict(self, format: str = WIRE_FORMAT_JSON) -> dict: + _check_format(format) + return {} if self.count is None else {'count': _encode_number(self.count, format)} + + +@dataclass +class MapCreate: + """MCR1: the payload of a MAP_CREATE operation.""" + + semantics: ObjectsMapSemantics = ObjectsMapSemantics.LWW # MCR2a + entries: dict[str, ObjectsMapEntry] = field(default_factory=dict) # MCR2b + + @staticmethod + def from_dict(obj: dict, format: str = WIRE_FORMAT_JSON) -> MapCreate: + _check_format(format) + semantics = obj.get('semantics') + return MapCreate( + semantics=(ObjectsMapSemantics.from_wire(semantics) if semantics is not None + else ObjectsMapSemantics.LWW), + entries=_decode_entries(obj.get('entries'), format) or {}, + ) + + def to_dict(self, format: str = WIRE_FORMAT_JSON) -> dict: + _check_format(format) + return {'semantics': int(self.semantics), 'entries': _encode_entries(self.entries, format)} + + +@dataclass +class MapSet: + """MST1: the payload of a MAP_SET operation.""" + + key: str | None = None # MST2a + value: ObjectData | None = None # MST2b + + @staticmethod + def from_dict(obj: dict, format: str = WIRE_FORMAT_JSON) -> MapSet: + _check_format(format) + value = obj.get('value') + return MapSet(key=obj.get('key'), value=ObjectData.from_dict(value, format) if value is not None else None) + + def to_dict(self, format: str = WIRE_FORMAT_JSON) -> dict: + _check_format(format) + result: dict = {} + if self.key is not None: + result['key'] = self.key + if self.value is not None: + result['value'] = self.value.to_dict(format) + return result + + +@dataclass +class MapRemove: + """MRM1: the payload of a MAP_REMOVE operation.""" + + key: str | None = None # MRM2a + + @staticmethod + def from_dict(obj: dict, format: str = WIRE_FORMAT_JSON) -> MapRemove: + _check_format(format) + return MapRemove(key=obj.get('key')) + + def to_dict(self, format: str = WIRE_FORMAT_JSON) -> dict: + _check_format(format) + return {} if self.key is None else {'key': self.key} + + +@dataclass +class CounterCreate: + """CCR1: the payload of a COUNTER_CREATE operation.""" + + count: float | None = None # CCR2a + + @staticmethod + def from_dict(obj: dict, format: str = WIRE_FORMAT_JSON) -> CounterCreate: + _check_format(format) + return CounterCreate(count=_decode_number(obj.get('count'))) + + def to_dict(self, format: str = WIRE_FORMAT_JSON) -> dict: + _check_format(format) + return {} if self.count is None else {'count': _encode_number(self.count, format)} + + +@dataclass +class CounterInc: + """CIN1: the payload of a COUNTER_INC operation.""" + + number: float | None = None # CIN2a + + @staticmethod + def from_dict(obj: dict, format: str = WIRE_FORMAT_JSON) -> CounterInc: + _check_format(format) + return CounterInc(number=_decode_number(obj.get('number'))) + + def to_dict(self, format: str = WIRE_FORMAT_JSON) -> dict: + _check_format(format) + return {} if self.number is None else {'number': _encode_number(self.number, format)} + + +@dataclass +class ObjectDelete: + """ODE1: the payload of an OBJECT_DELETE operation, which has no attributes (ODE2).""" + + @staticmethod + def from_dict(obj: dict, format: str = WIRE_FORMAT_JSON) -> ObjectDelete: + _check_format(format) + return ObjectDelete() + + def to_dict(self, format: str = WIRE_FORMAT_JSON) -> dict: + _check_format(format) + return {} + + +@dataclass +class MapClear: + """MCL1: the payload of a MAP_CLEAR operation, which has no attributes (MCL2).""" + + @staticmethod + def from_dict(obj: dict, format: str = WIRE_FORMAT_JSON) -> MapClear: + _check_format(format) + return MapClear() + + def to_dict(self, format: str = WIRE_FORMAT_JSON) -> dict: + _check_format(format) + return {} + + +@dataclass +class MapCreateWithObjectId: + """MCRO1: the payload of a MAP_CREATE operation sent with a client-generated object id. + + `derived_from` is the `MapCreate` the payload was built from, kept for local + application and size calculation (RTLMV4j5) and never sent. + """ + + initial_value: str | None = None # MCRO2a + nonce: str | None = None # MCRO2b + derived_from: MapCreate | None = None # RTLMV4j5 + + @staticmethod + def from_dict(obj: dict, format: str = WIRE_FORMAT_JSON) -> MapCreateWithObjectId: + _check_format(format) + return MapCreateWithObjectId(initial_value=obj.get('initialValue'), nonce=obj.get('nonce')) + + def to_dict(self, format: str = WIRE_FORMAT_JSON) -> dict: + _check_format(format) + result: dict = {} + if self.initial_value is not None: + result['initialValue'] = self.initial_value + if self.nonce is not None: + result['nonce'] = self.nonce + return result + + +@dataclass +class CounterCreateWithObjectId: + """CCRO1: the payload of a COUNTER_CREATE operation sent with a client-generated object id. + + `derived_from` is the `CounterCreate` the payload was built from, kept for local + application and size calculation (RTLCV4g5) and never sent. + """ + + initial_value: str | None = None # CCRO2a + nonce: str | None = None # CCRO2b + derived_from: CounterCreate | None = None # RTLCV4g5 + + @staticmethod + def from_dict(obj: dict, format: str = WIRE_FORMAT_JSON) -> CounterCreateWithObjectId: + _check_format(format) + return CounterCreateWithObjectId(initial_value=obj.get('initialValue'), nonce=obj.get('nonce')) + + def to_dict(self, format: str = WIRE_FORMAT_JSON) -> dict: + _check_format(format) + result: dict = {} + if self.initial_value is not None: + result['initialValue'] = self.initial_value + if self.nonce is not None: + result['nonce'] = self.nonce + return result + + +# The operation payloads, by the wire name each is carried under +_PAYLOADS = ( + ('map_create', 'mapCreate', MapCreate), + ('map_set', 'mapSet', MapSet), + ('map_remove', 'mapRemove', MapRemove), + ('counter_create', 'counterCreate', CounterCreate), + ('counter_inc', 'counterInc', CounterInc), + ('object_delete', 'objectDelete', ObjectDelete), + ('map_create_with_object_id', 'mapCreateWithObjectId', MapCreateWithObjectId), + ('counter_create_with_object_id', 'counterCreateWithObjectId', CounterCreateWithObjectId), + ('map_clear', 'mapClear', MapClear), +) + + +@dataclass +class ObjectOperation: + """OOP1: an operation to apply to an object on a channel.""" + + action: ObjectOperationAction # OOP3a + object_id: str # OOP3b + map_create: MapCreate | None = None # OOP3j + map_set: MapSet | None = None # OOP3k + map_remove: MapRemove | None = None # OOP3l + counter_create: CounterCreate | None = None # OOP3m + counter_inc: CounterInc | None = None # OOP3n + object_delete: ObjectDelete | None = None # OOP3o + map_create_with_object_id: MapCreateWithObjectId | None = None # OOP3p + counter_create_with_object_id: CounterCreateWithObjectId | None = None # OOP3q + map_clear: MapClear | None = None # OOP3r + + @property + def resolved_map_create(self) -> MapCreate | None: + """`map_create` if present, else the `MapCreate` `map_create_with_object_id` was derived from. + + This is the `mapCreate` RTLM23 merges and PAOOP3b exposes. + """ + if self.map_create is not None: + return self.map_create + if self.map_create_with_object_id is not None: + return self.map_create_with_object_id.derived_from + return None + + @property + def resolved_counter_create(self) -> CounterCreate | None: + """`counter_create` if present, else the `CounterCreate` `counter_create_with_object_id` was + derived from. + + This is the `counterCreate` RTLC16 merges and PAOOP3c exposes. + """ + if self.counter_create is not None: + return self.counter_create + if self.counter_create_with_object_id is not None: + return self.counter_create_with_object_id.derived_from + return None + + @staticmethod + def from_dict(obj: dict, format: str = WIRE_FORMAT_JSON) -> ObjectOperation: + _check_format(format) + operation = ObjectOperation( + action=ObjectOperationAction.from_wire(obj.get('action')), + object_id=obj.get('objectId'), + ) + for attribute, wire_name, payload_type in _PAYLOADS: + payload = obj.get(wire_name) + if payload is not None: + setattr(operation, attribute, payload_type.from_dict(payload, format)) + return operation + + def to_dict(self, format: str = WIRE_FORMAT_JSON) -> dict: + _check_format(format) + if self.action is ObjectOperationAction.UNKNOWN: + # OOP2a: an unrecognised action is never sent + raise ValueError('An ObjectOperation with an unrecognised action cannot be encoded') + result: dict = {'action': int(self.action)} + if self.object_id is not None: + result['objectId'] = self.object_id + for attribute, wire_name, _ in _PAYLOADS: + payload = getattr(self, attribute) + if payload is not None: + result[wire_name] = payload.to_dict(format) + return result + + +@dataclass +class ObjectState: + """OST1: the instantaneous state of an object, as an OBJECT_SYNC carries it.""" + + object_id: str # OST2a + site_timeserials: dict[str, str] = field(default_factory=dict) # OST2b + tombstone: bool = False # OST2c + create_op: ObjectOperation | None = None # OST2d + map: ObjectsMap | None = None # OST2e + counter: ObjectsCounter | None = None # OST2f + + @staticmethod + def from_dict(obj: dict, format: str = WIRE_FORMAT_JSON) -> ObjectState: + _check_format(format) + create_op = obj.get('createOp') + objects_map = obj.get('map') + counter = obj.get('counter') + return ObjectState( + object_id=obj.get('objectId'), + site_timeserials=dict(obj.get('siteTimeserials') or {}), + tombstone=bool(obj.get('tombstone', False)), + create_op=ObjectOperation.from_dict(create_op, format) if create_op is not None else None, + map=ObjectsMap.from_dict(objects_map, format) if objects_map is not None else None, + counter=ObjectsCounter.from_dict(counter, format) if counter is not None else None, + ) + + def to_dict(self, format: str = WIRE_FORMAT_JSON) -> dict: + _check_format(format) + result: dict = {'objectId': self.object_id, 'siteTimeserials': dict(self.site_timeserials)} + if self.tombstone: + result['tombstone'] = True + if self.create_op is not None: + result['createOp'] = self.create_op.to_dict(format) + if self.map is not None: + result['map'] = self.map.to_dict(format) + if self.counter is not None: + result['counter'] = self.counter.to_dict(format) + return result + + +@dataclass +class ObjectMessage: + """OM1: an object message sent or received over a channel. + + This is the internal type. The type delivered to listeners is the public + `ably.pubsub.objects.publicmessage.ObjectMessage` (PAOM1). + """ + + id: str | None = None # OM2a + client_id: str | None = None # OM2b + connection_id: str | None = None # OM2c + extras: dict | None = None # OM2d + timestamp: int | None = None # OM2e + operation: ObjectOperation | None = None # OM2f + object: ObjectState | None = None # OM2g + serial: str | None = None # OM2h + serial_timestamp: int | None = None # OM2j + site_code: str | None = None # OM2i + + @staticmethod + def from_dict(obj: dict, format: str = WIRE_FORMAT_JSON) -> ObjectMessage: + """Decodes one entry of a ProtocolMessage's `state` array (OM5).""" + _check_format(format) + operation = obj.get('operation') + state = obj.get('object') + return ObjectMessage( + id=obj.get('id'), + client_id=obj.get('clientId'), + connection_id=obj.get('connectionId'), + extras=obj.get('extras'), + timestamp=obj.get('timestamp'), + operation=ObjectOperation.from_dict(operation, format) if operation is not None else None, + object=ObjectState.from_dict(state, format) if state is not None else None, + serial=obj.get('serial'), + serial_timestamp=obj.get('serialTimestamp'), + site_code=obj.get('siteCode'), + ) + + @staticmethod + def from_protocol_message(protocol_message: dict, format: str = WIRE_FORMAT_JSON) -> list[ObjectMessage]: + """Decodes the `state` array of an OBJECT or OBJECT_SYNC ProtocolMessage. + + A message with no `id`, `connectionId` or `timestamp` takes them from the + ProtocolMessage that carried it (OM2a, OM2c, OM2e). + """ + messages = [] + protocol_id = protocol_message.get('id') + for index, entry in enumerate(protocol_message.get('state') or []): + message = ObjectMessage.from_dict(entry, format) + if message.id is None and protocol_id is not None: + message.id = f'{protocol_id}:{index}' + if message.connection_id is None: + message.connection_id = protocol_message.get('connectionId') + if message.timestamp is None: + message.timestamp = protocol_message.get('timestamp') + messages.append(message) + return messages + + def to_dict(self, format: str = WIRE_FORMAT_JSON) -> dict: + """Encodes this message for the `state` array of a ProtocolMessage (OM4).""" + _check_format(format) + result: dict = {} + for attribute, wire_name in (('id', 'id'), ('client_id', 'clientId'), ('connection_id', 'connectionId'), + ('extras', 'extras'), ('timestamp', 'timestamp'), ('serial', 'serial'), + ('serial_timestamp', 'serialTimestamp'), ('site_code', 'siteCode')): + value = getattr(self, attribute) + if value is not None: + result[wire_name] = value + if self.operation is not None: + result['operation'] = self.operation.to_dict(format) + if self.object is not None: + result['object'] = self.object.to_dict(format) + return result diff --git a/ably/pubsub/objects/objectspool.py b/ably/pubsub/objects/objectspool.py new file mode 100644 index 00000000..41919bc3 --- /dev/null +++ b/ably/pubsub/objects/objectspool.py @@ -0,0 +1,114 @@ +"""The objects held for a channel (RTO3).""" + +from __future__ import annotations + +import logging +from collections.abc import Iterator, MutableMapping +from typing import TYPE_CHECKING + +from ably.pubsub.objects.defaults import ROOT_OBJECT_ID +from ably.pubsub.objects.livecounter import InternalLiveCounter +from ably.pubsub.objects.livemap import InternalLiveMap +from ably.pubsub.objects.liveobject import LiveObject +from ably.pubsub.objects.objectid import object_type_of +from ably.pubsub.util.clock import Clock + +if TYPE_CHECKING: + from ably.pubsub.objects.realtimeobject import RealtimeObject + +log = logging.getLogger(__name__) + + +class ObjectsPool(MutableMapping): + """RTO3: the live objects on a channel, by object id (RTO3a). + + A pool always holds an `InternalLiveMap` with id `root`, created with the pool + (RTO3b1). It is a mutable mapping, so `pool[object_id]`, `object_id in pool`, + `len(pool)` and `pool.keys()` work as they do on a dict. Setting `pool[object_id] = obj` + adopts `obj`: its `pool` becomes this pool. + + A pool is constructible on its own. `realtime_object` is None for a pool built without + one, in which case objects in it dispatch no path subscriptions and publish nothing. + """ + + def __init__(self, realtime_object: RealtimeObject | None = None, *, clock: Clock | None = None): + self.realtime_object: RealtimeObject | None = realtime_object + self.clock: Clock = clock if clock is not None else Clock() + self._objects: dict[str, LiveObject] = {} + self[ROOT_OBJECT_ID] = InternalLiveMap(ROOT_OBJECT_ID, pool=self) + + @property + def root(self) -> InternalLiveMap: + """The root map (RTO3b).""" + return self._objects[ROOT_OBJECT_ID] + + def __getitem__(self, object_id: str) -> LiveObject: + return self._objects[object_id] + + def __setitem__(self, object_id: str, live_object: LiveObject) -> None: + live_object.pool = self + self._objects[object_id] = live_object + + def __delitem__(self, object_id: str) -> None: + del self._objects[object_id] + + def __iter__(self) -> Iterator[str]: + return iter(self._objects) + + def __len__(self) -> int: + return len(self._objects) + + def __repr__(self) -> str: + return f'ObjectsPool({list(self._objects)!r})' + + def create_zero_value_object_if_not_exists(self, object_id: str) -> LiveObject | None: + """RTO6: the object with `object_id`, created empty from the type its id names if absent. + + Returns None, creating nothing, for an id that names neither a map nor a counter. + """ + existing = self._objects.get(object_id) + if existing is not None: + return existing # RTO6a + + object_type = object_type_of(object_id) if isinstance(object_id, str) else None # RTO6b1 + if object_type == 'map': + live_object: LiveObject = InternalLiveMap(object_id) # RTO6b2 + elif object_type == 'counter': + live_object = InternalLiveCounter(object_id) # RTO6b3 + else: + log.warning(f'ObjectsPool.create_zero_value_object_if_not_exists(): cannot create an object for ' + f'an id of unknown type; object_id={object_id!r}') + return None + + self[object_id] = live_object + return live_object + + def clear_all_data(self) -> None: + """RTO27a1: resets every object's data to empty, keeping the objects, emitting nothing.""" + for live_object in list(self._objects.values()): + live_object.clear_data() + + def rebuild_parent_references(self) -> None: + """RTO5c10: clears every object's `parent_references` and rebuilds them from the map entries.""" + for live_object in self._objects.values(): + live_object.parent_references = {} # RTO5c10a + + # RTO5c10b + for live_object in list(self._objects.values()): + if not isinstance(live_object, InternalLiveMap): + continue + for key, value in live_object.entries(): + if isinstance(value, LiveObject): + value.add_parent_reference(live_object, key) + + def collect_garbage(self, grace_period_ms: int, now_ms: int) -> None: + """RTO10c: releases tombstoned map entries (RTLM19) and objects tombstoned `grace_period_ms` + or more before `now_ms`, never the root (RTO10c1b1).""" + for object_id, live_object in list(self._objects.items()): + # RTO10c1b, RTO10c1b1 + if (object_id != ROOT_OBJECT_ID and live_object.is_tombstone and live_object.tombstoned_at is not None + and now_ms - live_object.tombstoned_at >= grace_period_ms): + del self._objects[object_id] + continue + if isinstance(live_object, InternalLiveMap): + live_object.gc_tombstoned_entries(grace_period_ms, now_ms) # RTO10c1a diff --git a/ably/pubsub/objects/pathobject.py b/ably/pubsub/objects/pathobject.py new file mode 100644 index 00000000..e395d69f --- /dev/null +++ b/ably/pubsub/objects/pathobject.py @@ -0,0 +1,185 @@ +"""Path-addressed views onto a channel's objects (RTPO*, RTTS3-RTTS6). + +A `PathObject` holds a path from the root map and resolves it each time a method reads +or writes through it, so it follows whatever object is at that path at the time. The +base class carries what does not depend on the type at the path, navigation included; +`as_live_map()`, `as_live_counter()` and `as_primitive()` return the typed views, without +checking what the path resolves to (RTTS5d). + +Reads never raise for a path that does not resolve or that resolves to another type: +they return None, or an empty list for the collection methods (RTPO3c1, RTTS5d1). Writes +raise `AblyException` 92005 for a path that does not resolve and 92007 for one that +resolves to the wrong type (RTPO3c2, RTTS5d2). Every read except `path`, `get` and `at` +checks the access preconditions (RTO25), and every write the write preconditions (RTO26). +""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import TYPE_CHECKING, Any, Callable, Sequence, overload + +if TYPE_CHECKING: + from ably.pubsub.objects.batch import Batch, LiveCounterBatchContext, LiveMapBatchContext + from ably.pubsub.objects.enums import ValueType + from ably.pubsub.objects.instance import Instance + from ably.pubsub.objects.livemap import InternalLiveMap + from ably.pubsub.objects.publicmessage import ObjectMessage + from ably.pubsub.objects.realtimeobject import RealtimeObject + from ably.pubsub.objects.subscription import Subscription + from ably.pubsub.objects.valuetypes import LiveMapValue, Primitive, T + + +@dataclass +class PathObjectSubscriptionEvent: + """RTPO19e: what a `PathObject.subscribe` listener receives.""" + + object: PathObject # RTPO19e1: the path where the change occurred + message: ObjectMessage | None = None # RTPO19e2 + + +class PathObject: + """RTPO1, RTTS3: a lazy reference to whatever is at a path from the root map.""" + + def __init__(self, realtime_object: RealtimeObject, root: InternalLiveMap, path: Sequence[str]): + self._realtime_object = realtime_object + self._root = root # RTPO2b + self._path: list[str] = list(path) # RTPO2a + + def path(self) -> str: + """RTPO4: the path as a dotted string, with dots in segments escaped as `\\.`.""" + raise NotImplementedError + + def type(self) -> ValueType | None: + """RTTS4b: the type of what the path resolves to, or None if it resolves to nothing.""" + raise NotImplementedError + + def exists(self) -> bool: + """RTTS4a: whether the path resolves to anything.""" + raise NotImplementedError + + def get(self, key: str) -> PathObject: + """RTPO5: the path one key further down. Navigational only (RTPO5d). + + Raises AblyException 40003 if `key` is not a string (RTPO5b). + """ + raise NotImplementedError + + def at(self, path: str | Sequence[str]) -> PathObject: + """RTPO6: the path further down by a dotted string, `\\.` escaping a literal dot (RTPO6b), + or by a sequence of segments taken as they are. + + Raises AblyException 40003 for anything else. + """ + raise NotImplementedError + + def instance(self) -> Instance | None: + """RTPO8: an `Instance` wrapping what the path resolves to, or None if it resolves to nothing. + + The instance is the subclass matching the resolved value (`LiveMapInstance`, + `LiveCounterInstance` or `PrimitiveInstance`). + """ + raise NotImplementedError + + def compact(self) -> Any: + """RTPO13: a plain snapshot of what the path resolves to, or None.""" + raise NotImplementedError + + def compact_json(self) -> Any: + """RTPO14: `compact`, with binary as base64 and cycles as `{'objectId': ...}`.""" + raise NotImplementedError + + def subscribe(self, listener: Callable[[PathObjectSubscriptionEvent], None], *, + depth: int | None = None) -> Subscription: + """RTPO19: calls `listener` for changes at this path or, within `depth` levels, below it. + + Raises AblyException 40003 if `depth` is given and is not a positive integer + (RTPO19c1a). Has no effect on the channel (RTPO19g). + """ + raise NotImplementedError + + def as_live_map(self) -> LiveMapPathObject: + """RTTS5a: this path, viewed as a map.""" + raise NotImplementedError + + def as_live_counter(self) -> LiveCounterPathObject: + """RTTS5b: this path, viewed as a counter.""" + raise NotImplementedError + + def as_primitive(self) -> PrimitivePathObject: + """RTTS5c, RTTS6h: this path, viewed as a primitive.""" + raise NotImplementedError + + +class LiveMapPathObject(PathObject): + """RTTS6a: a path expected to resolve to a map.""" + + def batch(self) -> Batch[LiveMapBatchContext]: + """RTPO20: a block whose queued writes are published as one message when it exits. + + Entering raises AblyException 92007 if the path does not resolve to a live object + (RTPO20c). + """ + raise NotImplementedError + + def entries(self) -> list[tuple[str, PathObject]]: + """RTPO9: `(key, path)` for each key of the map, or `[]`.""" + raise NotImplementedError + + def keys(self) -> list[str]: + """RTPO10: the keys of the map, or `[]`.""" + raise NotImplementedError + + def values(self) -> list[PathObject]: + """RTPO11: a path for each key of the map, or `[]`.""" + raise NotImplementedError + + def size(self) -> int | None: + """RTPO12: the number of entries in the map, or None.""" + raise NotImplementedError + + async def set(self, key: str, value: LiveMapValue) -> None: + """RTPO15: sets `key` in the map at this path.""" + raise NotImplementedError + + async def remove(self, key: str) -> None: + """RTPO16: removes `key` from the map at this path.""" + raise NotImplementedError + + +class LiveCounterPathObject(PathObject): + """RTTS6b: a path expected to resolve to a counter.""" + + def batch(self) -> Batch[LiveCounterBatchContext]: + """RTPO20: a block whose queued writes are published as one message when it exits.""" + raise NotImplementedError + + def value(self) -> float | None: + """RTTS6b: the counter's value, or None if the path does not resolve to a counter.""" + raise NotImplementedError + + async def increment(self, amount: float = 1) -> None: + """RTPO17: increments the counter at this path by `amount`.""" + raise NotImplementedError + + async def decrement(self, amount: float = 1) -> None: + """RTPO18: decrements the counter at this path by `amount`.""" + raise NotImplementedError + + +class PrimitivePathObject(PathObject): + """RTTS6c, RTTS6h: a path expected to resolve to a primitive.""" + + @overload + def value(self) -> Primitive | None: ... + + @overload + def value(self, expected: type[T]) -> T | None: ... + + def value(self, expected: type | None = None) -> Any: + """RTPO7, RTTS6c: the primitive at this path, or None. + + With `expected` (one of `str`, `float`, `bool`, `bytes`, `list`, `dict`) the value is + returned only if it is of that type, judged by its wire type, so a boolean is never + a number. Any other `expected` raises TypeError. + """ + raise NotImplementedError diff --git a/ably/pubsub/objects/pathobjectsubscriptionregister.py b/ably/pubsub/objects/pathobjectsubscriptionregister.py new file mode 100644 index 00000000..460f130a --- /dev/null +++ b/ably/pubsub/objects/pathobjectsubscriptionregister.py @@ -0,0 +1,35 @@ +"""The register of path subscriptions a channel's `RealtimeObject` keeps (RTO24).""" + +from __future__ import annotations + +from typing import TYPE_CHECKING, Callable + +if TYPE_CHECKING: + from ably.pubsub.objects.liveobject import LiveObject, LiveObjectUpdate + from ably.pubsub.objects.pathobject import PathObjectSubscriptionEvent + from ably.pubsub.objects.realtimeobject import RealtimeObject + from ably.pubsub.objects.subscription import Subscription + + +class PathObjectSubscriptionRegister: + """RTO24: every subscription made through `PathObject.subscribe` on one channel (RTO24a).""" + + def __init__(self, realtime_object: RealtimeObject): + self.realtime_object = realtime_object + + def subscribe(self, path: list[str], listener: Callable[[PathObjectSubscriptionEvent], None], + depth: int | None = None) -> Subscription: + """RTPO19f: registers `listener` for changes covered by `path` and `depth` (RTO24c1).""" + raise NotImplementedError + + def dispatch(self, live_object: LiveObject, update: LiveObjectUpdate) -> None: + """RTO24b: calls each subscription covering a path to `live_object` once. + + A listener that raises is logged and does not stop the others (RTO24b2c). + """ + raise NotImplementedError + + @staticmethod + def covers(subscription_path: list[str], depth: int | None, event_path: list[str]) -> bool: + """RTO24c1: whether a subscription at `subscription_path` with `depth` covers `event_path`.""" + raise NotImplementedError diff --git a/ably/pubsub/objects/publicmessage.py b/ably/pubsub/objects/publicmessage.py new file mode 100644 index 00000000..8f603c0e --- /dev/null +++ b/ably/pubsub/objects/publicmessage.py @@ -0,0 +1,102 @@ +"""The object message delivered to subscription listeners (PAOM*, PAOOP*). + +The specification calls these `PublicAPI::ObjectMessage` and `PublicAPI::ObjectOperation`. +They are exposed to users as `ObjectMessage` and `ObjectOperation`, sharing their names +with the internal wire types in `ably.pubsub.objects.objectmessage`; code that needs both +imports one of the pairs under a module-qualified name. +""" + +from __future__ import annotations + +import copy +from dataclasses import dataclass +from typing import TYPE_CHECKING + +from ably.pubsub.objects.objectmessage import ( + CounterCreate, + CounterInc, + MapClear, + MapCreate, + MapRemove, + MapSet, + ObjectDelete, + ObjectOperationAction, +) + +if TYPE_CHECKING: + from ably.pubsub.objects import objectmessage + + +@dataclass +class ObjectOperation: + """PAOOP1: the operation that caused an object change. + + Unlike the wire operation it never carries the `*CreateWithObjectId` payloads: a + create sent with a client-generated object id is shown as the `MapCreate` or + `CounterCreate` it was derived from. + """ + + action: ObjectOperationAction # PAOOP2a + object_id: str # PAOOP2b + map_create: MapCreate | None = None # PAOOP2c + map_set: MapSet | None = None # PAOOP2d + map_remove: MapRemove | None = None # PAOOP2e + counter_create: CounterCreate | None = None # PAOOP2f + counter_inc: CounterInc | None = None # PAOOP2g + object_delete: ObjectDelete | None = None # PAOOP2h + map_clear: MapClear | None = None # PAOOP2i + + @staticmethod + def _from_internal(operation: objectmessage.ObjectOperation) -> ObjectOperation: + """PAOOP3: the public form of an internal `ObjectOperation`. + + The payloads are copies, so a listener that changes them leaves the objects the + operation was applied to unchanged. + """ + return ObjectOperation( + action=operation.action, # PAOOP3a + object_id=operation.object_id, + map_create=copy.deepcopy(operation.resolved_map_create), # PAOOP3b + map_set=copy.deepcopy(operation.map_set), + map_remove=copy.deepcopy(operation.map_remove), + counter_create=copy.deepcopy(operation.resolved_counter_create), # PAOOP3c + counter_inc=copy.deepcopy(operation.counter_inc), + object_delete=copy.deepcopy(operation.object_delete), + map_clear=copy.deepcopy(operation.map_clear), + ) + + +@dataclass +class ObjectMessage: + """PAOM1: the inbound object message that caused an object change.""" + + channel: str # PAOM2e + operation: ObjectOperation # PAOM2f + id: str | None = None # PAOM2a + client_id: str | None = None # PAOM2b + connection_id: str | None = None # PAOM2c + timestamp: int | None = None # PAOM2d + serial: str | None = None # PAOM2g + serial_timestamp: int | None = None # PAOM2h + site_code: str | None = None # PAOM2i + extras: dict | None = None # PAOM2j + + @staticmethod + def _from_internal(message: objectmessage.ObjectMessage, channel_name: str) -> ObjectMessage: + """PAOM3: the public form of an internal `ObjectMessage` received on `channel_name`. + + The caller ensures `message.operation` is populated (PAOM3a1). + """ + return ObjectMessage( + channel=channel_name, # PAOM3b + operation=ObjectOperation._from_internal(message.operation), # PAOM3d + # PAOM3c + id=message.id, + client_id=message.client_id, + connection_id=message.connection_id, + timestamp=message.timestamp, + serial=message.serial, + serial_timestamp=message.serial_timestamp, + site_code=message.site_code, + extras=copy.deepcopy(message.extras), + ) diff --git a/ably/pubsub/objects/realtimeobject.py b/ably/pubsub/objects/realtimeobject.py new file mode 100644 index 00000000..8cf75406 --- /dev/null +++ b/ably/pubsub/objects/realtimeobject.py @@ -0,0 +1,386 @@ +"""The LiveObjects entry point for a channel (RTO*), reached as `channel.object`.""" + +from __future__ import annotations + +import logging +from typing import TYPE_CHECKING, Callable + +from ably.pubsub.objects.defaults import GC_GRACE_PERIOD_MS, GC_INTERVAL_MS, ROOT_OBJECT_ID +from ably.pubsub.objects.enums import ObjectsEvent, ObjectsOperationSource, ObjectsSyncState +from ably.pubsub.objects.livecounter import InternalLiveCounter +from ably.pubsub.objects.livemap import InternalLiveMap +from ably.pubsub.objects.liveobject import LiveObject, LiveObjectUpdate +from ably.pubsub.objects.objectmessage import ( + WIRE_FORMAT_JSON, + WIRE_FORMAT_MSGPACK, + ObjectOperationAction, + ObjectsMapSemantics, +) +from ably.pubsub.objects.objectspool import ObjectsPool +from ably.pubsub.objects.pathobjectsubscriptionregister import PathObjectSubscriptionRegister +from ably.pubsub.objects.subscription import StatusSubscription +from ably.pubsub.objects.syncobjectspool import SyncObjectsPool +from ably.pubsub.util.clock import Clock, select_clock + +if TYPE_CHECKING: + from ably.pubsub.objects.objectmessage import ObjectMessage + from ably.pubsub.objects.pathobject import LiveMapPathObject + from ably.pubsub.realtime.channel import RealtimeChannel + from ably.pubsub.types.channelmode import ChannelMode + from ably.pubsub.types.channelstate import ChannelState + from ably.pubsub.types.operations import PublishResult + from ably.pubsub.util.exceptions import AblyException + from ably.pubsub.util.helper import Timer + +log = logging.getLogger(__name__) + +# RTO9a2a: the operation actions `_apply_object_messages` applies +_SUPPORTED_ACTIONS = frozenset(( + ObjectOperationAction.MAP_CREATE, + ObjectOperationAction.MAP_SET, + ObjectOperationAction.MAP_REMOVE, + ObjectOperationAction.COUNTER_CREATE, + ObjectOperationAction.COUNTER_INC, + ObjectOperationAction.OBJECT_DELETE, + ObjectOperationAction.MAP_CLEAR, +)) + +# RTO17b: the event each sync state emits when it is entered +_SYNC_STATE_EVENTS = { + ObjectsSyncState.SYNCING: ObjectsEvent.SYNCING, + ObjectsSyncState.SYNCED: ObjectsEvent.SYNCED, +} + + +class RealtimeObject: + """RTO*: the objects on one channel, and the entry point to them. + + Each `RealtimeChannel` builds one, reachable as `channel.object`. One can also be built + with no channel, so that its sync and apply behaviour can be driven directly; such a + `RealtimeObject` publishes nothing, checks no channel preconditions and schedules no + timers. `pool` adopts an existing `ObjectsPool`; `clock` defaults to the channel's + client's clock. + + Construction is cheap and needs no running event loop: the GC timer (RTO10) is + scheduled on the first ATTACHED, not here. + """ + + def __init__(self, channel: RealtimeChannel | None = None, *, pool: ObjectsPool | None = None, + clock: Clock | None = None): + self._channel = channel + if clock is None: + clock = select_clock(channel.ably.options) if channel is not None else Clock() + self._clock: Clock = clock + if pool is None: + pool = ObjectsPool(self, clock=clock) + else: + pool.realtime_object = self + self._objects_pool: ObjectsPool = pool # RTO3 + self._sync_objects_pool = SyncObjectsPool() # RTO5f + self._sync_state = ObjectsSyncState.INITIALIZED # RTO17a1 + self._buffered_object_operations: list[ObjectMessage] = [] # RTO7a, RTO7a1 + self._applied_on_ack_serials: set[str] = set() # RTO7b, RTO7b1 + self._current_sync_id: str | None = None # RTO5a1 + self._current_sync_cursor: str | None = None # RTO5a1 + self._path_object_subscription_register = PathObjectSubscriptionRegister(self) # RTO24a + self._gc_interval_ms: int = GC_INTERVAL_MS # RTO10a, read each time the GC timer is scheduled + self._gc_timer: Timer | None = None + self._last_gc_grace_period_ms: int = GC_GRACE_PERIOD_MS # RTO10b + # RTO18: (token, callback) for each `on` call, by event, so that a callback registered + # twice is called twice (RTO18d) and each StatusSubscription removes only its own + self._sync_state_listeners: dict[ObjectsEvent, list[tuple[object, Callable[[], None]]]] = { + event: [] for event in ObjectsEvent + } + + # Public API + + async def get(self) -> LiveMapPathObject: + """RTO23: the root of the channel's objects, once they have synced. + + Requires the OBJECT_SUBSCRIBE mode (RTO23a, 40024), attaches the channel if it is not + attached (RTO23e, RTL33), and waits for the sync state to reach SYNCED (RTO23c). + Raises AblyException 92008 if the channel enters DETACHED, SUSPENDED or FAILED while + waiting (RTO23c1), and 90001 if the channel is FAILED (RTL33c). + """ + raise NotImplementedError + + def on(self, event: ObjectsEvent, callback: Callable[[], None]) -> StatusSubscription: + """RTO18: calls `callback`, with no arguments, whenever the sync state reaches `event`. + + Registering one callback twice calls it twice (RTO18d). + """ + event = ObjectsEvent(event) + token = object() + self._sync_state_listeners[event].append((token, callback)) # RTO18c + + def deregister() -> None: + self._sync_state_listeners[event] = [ + entry for entry in self._sync_state_listeners[event] if entry[0] is not token] + + return StatusSubscription(deregister) # RTO18f + + def off(self, event: ObjectsEvent, callback: Callable[[], None]) -> None: + """RTO19: deregisters `callback` from `event`, however many times it was registered.""" + event = ObjectsEvent(event) + self._sync_state_listeners[event] = [ + entry for entry in self._sync_state_listeners[event] if entry[1] != callback] + + # Internal API: inbound protocol messages and channel state + + @property + def _channel_name(self) -> str | None: + """The channel's name, which PAOM3b puts on every public message.""" + return self._channel.name if self._channel is not None else None + + @property + def _wire_format(self) -> str: + """The wire format the channel's connection decodes with.""" + if self._channel is not None and self._channel.ably.options.use_binary_protocol: + return WIRE_FORMAT_MSGPACK + return WIRE_FORMAT_JSON + + def _on_attached(self, has_objects: bool) -> None: + """RTO4: handles an ATTACHED ProtocolMessage, whatever the channel's state was. + + `has_objects` is the HAS_OBJECTS flag. Starts the GC timer if it is not running + and there is a channel (RTO10). + """ + log.debug(f'RealtimeObject._on_attached(): channel={self._channel_name}, has_objects={has_objects}') + self._set_sync_state(ObjectsSyncState.SYNCING) # RTO4c + self._buffered_object_operations = [] # RTO4d + # The objects an ATTACHED announces arrive in a sync sequence of their own (RTO4a), so a + # sequence still in flight from before it is abandoned + self._start_sync_sequence(None) + + if not has_objects: + # RTO4b + pool = self._objects_pool + for object_id in list(pool): + if object_id != ROOT_OBJECT_ID: + del pool[object_id] # RTO4b1 + + # RTO4b2: the root is cleared in place, never replaced + root = pool.root + previous_data = root.data + root.clear_data() + # RTO4b2a: the removed keys, with no object message; a no-op if the root was empty + root.notify_updated(InternalLiveMap.diff(previous_data, root.data)) + + self._sync_objects_pool.clear() # RTO4b3 + self._complete_sync() # RTO4b4 + + self._schedule_gc_timer() # RTO10a + + def _handle_object_sync_messages(self, object_messages: list[ObjectMessage], + sync_channel_serial: str | None) -> None: + """RTO5: handles the decoded `state` of an OBJECT_SYNC ProtocolMessage.""" + sync_id, sync_cursor = self._parse_sync_channel_serial(sync_channel_serial) # RTO5a + self._set_sync_state(ObjectsSyncState.SYNCING) # RTO5e + + # RTO5a2: a new sequence id starts a new sequence. A message with no sequence id holds a + # whole sync of its own (RTO5a5), so it too discards anything accumulated before it. + if sync_id is None or sync_id != self._current_sync_id: + self._start_sync_sequence(sync_id) + self._current_sync_cursor = sync_cursor + + self._sync_objects_pool.apply_object_sync_messages(object_messages) # RTO5f + + # RTO5a4, RTO5a5: the sequence is complete once its cursor is empty + if not sync_cursor: + self._complete_sync() + + def _handle_object_messages(self, object_messages: list[ObjectMessage]) -> None: + """RTO8: handles the decoded `state` of an OBJECT ProtocolMessage, buffering while not SYNCED.""" + if self._sync_state != ObjectsSyncState.SYNCED: + self._buffered_object_operations.extend(object_messages) # RTO8a + return + self._apply_object_messages(object_messages, ObjectsOperationSource.CHANNEL) # RTO8b + + def _apply_object_messages(self, object_messages: list[ObjectMessage], + source: ObjectsOperationSource) -> None: + """RTO9: applies operations to the pool, recording LOCAL serials in `_applied_on_ack_serials`.""" + for object_message in object_messages: + operation = object_message.operation + if operation is None: + # RTO9a1 + log.warning(f'RealtimeObject._apply_object_messages(): skipping an object message with no ' + f'operation; message id={object_message.id}, channel={self._channel_name}') + continue + + serial = object_message.serial + if serial is not None and serial in self._applied_on_ack_serials: + # RTO9a3 + log.debug(f'RealtimeObject._apply_object_messages(): skipping an operation already applied ' + f'on ACK; serial={serial}, channel={self._channel_name}') + self._applied_on_ack_serials.discard(serial) + continue + + if operation.action not in _SUPPORTED_ACTIONS: + # RTO9a2b + log.warning(f'RealtimeObject._apply_object_messages(): skipping an object message with an ' + f'unsupported action; action={operation.action!r}, ' + f'message id={object_message.id}, channel={self._channel_name}') + continue + + # RTO9a2a1, RTO9a2a2 + live_object = self._objects_pool.create_zero_value_object_if_not_exists(operation.object_id) + if live_object is None: + continue + applied = live_object.apply_operation(object_message, source) # RTO9a2a3 + if source == ObjectsOperationSource.LOCAL and applied: + self._applied_on_ack_serials.add(serial) # RTO9a2a4 + + def _act_on_channel_state(self, state: ChannelState, reason: AblyException | None = None) -> None: + """RTO27: handles the channel entering `state`, for every state but ATTACHED. + + DETACHED and FAILED clear every object's data (RTO27a); every other state keeps it + (RTO27b). DETACHED, SUSPENDED and FAILED fail a `get` or a `_publish_and_apply` + waiting for SYNCED with 92008 (RTO23c1, RTO20e1), with the channel's `error_reason` + as the cause. + """ + raise NotImplementedError + + def _set_sync_state(self, state: ObjectsSyncState) -> None: + """RTO17: moves to `state`, emitting the matching `ObjectsEvent` to `on` listeners (RTO17b).""" + if state == self._sync_state: + return + self._sync_state = state + + event = _SYNC_STATE_EVENTS.get(state) + if event is None: + return + for _, callback in list(self._sync_state_listeners[event]): + try: + callback() # RTO18e + except Exception: + log.exception(f'RealtimeObject._set_sync_state(): a {event.value} listener raised; ' + f'channel={self._channel_name}') + + # Internal API: the sync sequence + + @staticmethod + def _parse_sync_channel_serial(sync_channel_serial: str | None) -> tuple[str | None, str | None]: + """RTO5a1: the `(sequence id, cursor)` of an OBJECT_SYNC's `channelSerial`. + + Both are None when the serial is absent (RTO5a5) or has no `:` separator (RTO5a6). + """ + if sync_channel_serial is None: + return None, None + sync_id, separator, sync_cursor = sync_channel_serial.partition(':') + if not separator: + log.warning(f'RealtimeObject._parse_sync_channel_serial(): handling an OBJECT_SYNC whose ' + f'channelSerial has no ":" as having none; channelSerial={sync_channel_serial!r}') + return None, None + return sync_id, sync_cursor + + def _start_sync_sequence(self, sync_id: str | None) -> None: + """RTO5a2: discards whatever an earlier sync sequence accumulated and starts `sync_id`.""" + self._sync_objects_pool.clear() # RTO5a2a + self._current_sync_id = sync_id + self._current_sync_cursor = None + + def _complete_sync(self) -> None: + """RTO5c: applies the objects the sync sequence delivered, then the operations buffered meanwhile.""" + pool = self._objects_pool + received_object_ids: set[str] = set() + updates: list[tuple[LiveObject, LiveObjectUpdate]] = [] + + # RTO5c1 + for object_id, object_message in self._sync_objects_pool.entries.items(): + received_object_ids.add(object_id) + existing = pool.get(object_id) + if existing is not None: + # RTO5c1a1, RTO5c1a2 + updates.append((existing, existing.replace_data(object_message))) + continue + + # RTO5c1b1 + object_state = object_message.object + if object_state.counter is not None: + live_object: LiveObject = InternalLiveCounter(object_id) # RTO5c1b1a + else: + semantics = object_state.map.semantics + live_object = InternalLiveMap( + object_id, semantics if semantics is not None else ObjectsMapSemantics.LWW) # RTO5c1b1b + pool[object_id] = live_object + live_object.replace_data(object_message) + + # RTO5c2, RTO5c2a + for object_id in list(pool): + if object_id not in received_object_ids and object_id != ROOT_OBJECT_ID: + del pool[object_id] + + pool.rebuild_parent_references() # RTO5c10 + + # RTO5c7: emitted once every object is in place, so that listeners see the synced state + for live_object, update in updates: + live_object.notify_updated(update) + + self._apply_object_messages(self._buffered_object_operations, ObjectsOperationSource.CHANNEL) # RTO5c6 + + self._current_sync_id = None # RTO5c3 + self._current_sync_cursor = None # RTO5c3 + self._sync_objects_pool.clear() # RTO5c4 + self._buffered_object_operations = [] # RTO5c5 + self._applied_on_ack_serials.clear() # RTO5c9 + self._set_sync_state(ObjectsSyncState.SYNCED) # RTO5c8 + + # Internal API: publishing + + async def _publish(self, object_messages: list[ObjectMessage]) -> PublishResult: + """RTO15: sends `object_messages` in one OBJECT ProtocolMessage and returns the ACK's result.""" + raise NotImplementedError + + async def _publish_and_apply(self, object_messages: list[ObjectMessage]) -> None: + """RTO20: publishes `object_messages`, then applies them locally with the serials the ACK + assigned, as LOCAL operations, once synced.""" + raise NotImplementedError + + async def _get_server_time_ms(self) -> int: + """RTO16: the current server time, from a persisted offset where one is known (RTO16a).""" + raise NotImplementedError + + # Internal API: preconditions + + def _throw_if_missing_channel_mode(self, mode: ChannelMode) -> None: + """RTO2: raises AblyException 40024 if `mode` is neither granted nor, before ATTACHED, requested.""" + raise NotImplementedError + + def _check_access_preconditions(self) -> None: + """RTO25: OBJECT_SUBSCRIBE (40024), and 90001 if the channel is DETACHED or FAILED.""" + raise NotImplementedError + + def _check_write_preconditions(self) -> None: + """RTO26: OBJECT_PUBLISH (40024), 90001 if the channel is DETACHED, FAILED or SUSPENDED, + and 40000 if `echo_messages` is disabled.""" + raise NotImplementedError + + # Internal API: garbage collection + + @property + def _gc_grace_period_ms(self) -> int: + """RTO10b: `ConnectionDetails.objects_gc_grace_period` from the latest CONNECTED, or the default.""" + connection_details = None + if self._channel is not None: + connection_details = self._channel.ably.connection.connection_details + if connection_details is not None: + # RTO10b1, RTO10b2, RTO10b3 + grace_period_ms = connection_details.objects_gc_grace_period + self._last_gc_grace_period_ms = grace_period_ms if grace_period_ms is not None else GC_GRACE_PERIOD_MS + return self._last_gc_grace_period_ms + + def _schedule_gc_timer(self) -> None: + """RTO10a: schedules `_on_gc_interval` on the clock, `_gc_interval_ms` from now, if there is a + channel and no sweep is scheduled already. + + A `RealtimeObject` with no channel schedules nothing. + """ + if self._channel is None: + return + raise NotImplementedError + + def _on_gc_interval(self) -> None: + """RTO10c: releases what has been tombstoned for the grace period, and reschedules itself.""" + self._gc_timer = None + self._objects_pool.collect_garbage(self._gc_grace_period_ms, self._clock.now_ms()) + self._schedule_gc_timer() diff --git a/ably/pubsub/objects/subscription.py b/ably/pubsub/objects/subscription.py new file mode 100644 index 00000000..e8082758 --- /dev/null +++ b/ably/pubsub/objects/subscription.py @@ -0,0 +1,31 @@ +"""The handles a subscribe call returns.""" + +from __future__ import annotations + +from typing import Callable + + +class Subscription: + """SUB1: a registration for receiving events from a `subscribe` call.""" + + def __init__(self, deregister: Callable[[], None]): + self._deregister: Callable[[], None] | None = deregister + + def unsubscribe(self) -> None: + """SUB2a: deregisters the listener. Calling it again does nothing (SUB2b).""" + deregister, self._deregister = self._deregister, None + if deregister is not None: + deregister() + + +class StatusSubscription: + """RTO18f: the registration `RealtimeObject.on` returns.""" + + def __init__(self, deregister: Callable[[], None]): + self._deregister: Callable[[], None] | None = deregister + + def off(self) -> None: + """RTO18f1, RTO18f2: deregisters the listener the corresponding `on` call registered.""" + deregister, self._deregister = self._deregister, None + if deregister is not None: + deregister() diff --git a/ably/pubsub/objects/syncobjectspool.py b/ably/pubsub/objects/syncobjectspool.py new file mode 100644 index 00000000..254ba592 --- /dev/null +++ b/ably/pubsub/objects/syncobjectspool.py @@ -0,0 +1,83 @@ +"""The object states accumulated during an objects sync sequence (RTO5f).""" + +from __future__ import annotations + +import logging +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from ably.pubsub.objects.objectmessage import ObjectMessage + +log = logging.getLogger(__name__) + + +class SyncObjectsPool: + """RTO5f: one `ObjectMessage` per object id, received over OBJECT_SYNC and not yet applied. + + A partial map state for an id already held is merged into it (RTO5f2). + """ + + def __init__(self): + self.entries: dict[str, ObjectMessage] = {} + + def apply_object_sync_messages(self, object_messages: list[ObjectMessage]) -> None: + """RTO5f: stores each message with an `object`, merging partial map states (RTO5f2a) and + skipping messages with neither `map` nor `counter` (RTO5f3) and partial counter states + (RTO5f2b).""" + for object_message in object_messages: + object_state = object_message.object + if object_state is None: + # RTO5d: only this message is skipped, so the rest of the sequence still applies + log.warning(f'SyncObjectsPool.apply_object_sync_messages(): skipping an OBJECT_SYNC message ' + f'with no object; message id={object_message.id}') + continue + + if object_state.map is None and object_state.counter is None: + # RTO5f3 + log.warning(f'SyncObjectsPool.apply_object_sync_messages(): skipping an OBJECT_SYNC message ' + f'with an unsupported object type; object_id={object_state.object_id}, ' + f'message id={object_message.id}') + continue + + existing = self.entries.get(object_state.object_id) + if existing is None: + self.entries[object_state.object_id] = object_message # RTO5f1 + continue + + # RTO5f2: a partial state of an object split across several OBJECT_SYNC messages + if object_state.map is not None: + self._merge_partial_map_state(existing, object_message) # RTO5f2a + else: + # RTO5f2b + log.error(f'SyncObjectsPool.apply_object_sync_messages(): skipping an unexpected partial ' + f'state for a counter; object_id={object_state.object_id}, ' + f'message id={object_message.id}') + + def clear(self) -> None: + """RTO5a2a, RTO5c4, RTO27a2: discards every stored message.""" + self.entries.clear() + + def _merge_partial_map_state(self, existing: ObjectMessage, object_message: ObjectMessage) -> None: + """RTO5f2a: merges the partial map state in `object_message` into the stored `existing`.""" + object_state = object_message.object + if object_state.tombstone: + self.entries[object_state.object_id] = object_message # RTO5f2a1 + return + + existing_map = existing.object.map + if existing_map is None: + log.error(f'SyncObjectsPool.apply_object_sync_messages(): skipping a partial map state for an ' + f'object first received as a counter; object_id={object_state.object_id}, ' + f'message id={object_message.id}') + return + + # RTO5f2a2: no two partial states of one map carry the same key + if existing_map.entries is None: + existing_map.entries = {} + existing_map.entries.update(object_state.map.entries or {}) + + def __len__(self) -> int: + return len(self.entries) + + def __contains__(self, object_id: object) -> bool: + return object_id in self.entries diff --git a/ably/pubsub/objects/valuetypes.py b/ably/pubsub/objects/valuetypes.py new file mode 100644 index 00000000..40c1dadf --- /dev/null +++ b/ably/pubsub/objects/valuetypes.py @@ -0,0 +1,229 @@ +"""The creation blueprints `LiveCounter` and `LiveMap` (RTLCV*, RTLMV*), and the value unions. + +A blueprint is an inert, immutable description of an object to create. It becomes +ObjectMessages only when a mutation evaluates it (RTLCV4, RTLMV4), so `create` performs +no validation (RTLCV3c, RTLMV3c). + +Evaluation is pure: the caller fetches the server time (RTO16) and passes it in, and +each evaluation generates a fresh nonce, so a blueprint used twice creates two objects. +""" + +from __future__ import annotations + +import json +import math +from dataclasses import dataclass +from typing import Any, TypeVar, Union + +from ably.pubsub.objects.objectid import generate_nonce, generate_object_id +from ably.pubsub.objects.objectmessage import ( + WIRE_FORMAT_JSON, + CounterCreate, + CounterCreateWithObjectId, + MapCreate, + MapCreateWithObjectId, + ObjectData, + ObjectMessage, + ObjectOperation, + ObjectOperationAction, + ObjectsMapEntry, + ObjectsMapSemantics, +) +from ably.pubsub.util.exceptions import AblyException + +# The primitive values a map entry can hold: String, Number, Boolean, Binary, JsonArray +# and JsonObject (RTLM20a3) +Primitive = Union[str, float, bool, bytes, list, dict] + +T = TypeVar('T', bound=Primitive) + + +@dataclass(frozen=True) +class LiveCounter: + """RTLCV1: the intent to create a counter with an initial count.""" + + _count: Any = 0 # RTLCV2a + + @staticmethod + def create(initial_count: float = 0) -> LiveCounter: + """RTLCV3: a blueprint for a counter starting at `initial_count`.""" + return LiveCounter(initial_count) + + +@dataclass(frozen=True) +class LiveMap: + """RTLMV1: the intent to create a map with initial entries.""" + + _entries: Any = None # RTLMV2a + + @staticmethod + def create(entries: dict[str, LiveMapValue] | None = None) -> LiveMap: + """RTLMV3: a blueprint for a map holding `entries`. + + A dict is copied, so the blueprint does not change when the caller's dict does + (RTLMV3d); anything else is kept for evaluation to reject (RTLMV3c). + """ + return LiveMap(dict(entries) if isinstance(entries, dict) else entries) + + +# The values `set` accepts: a primitive, or a blueprint for a new object (RTTS11) +LiveMapValue = Union[Primitive, LiveMap, LiveCounter] + + +def validate_key(key: Any) -> None: + """RTLMV4b: raises AblyException 40003 unless `key` is a string. + + A map's `set` and `remove` validate their key the same way (RTLM20e1, RTLM21e1). + """ + if not isinstance(key, str): + raise AblyException('Map key should be string', 400, 40003) + + +def validate_value(value: Any) -> None: + """RTLMV4c: raises AblyException 40013 unless `value` is a primitive or a blueprint. + + A map's `set` validates its value the same way (RTLM20e1). A blueprint's own contents + are validated when it is evaluated. + """ + if not isinstance(value, (LiveCounter, LiveMap)): + primitive_to_object_data(value) + + +def primitive_to_object_data(value: Any) -> ObjectData: + """RTLMV4d3-RTLMV4d7: the `ObjectData` holding the primitive `value`. + + It holds the value as it decodes on receipt: a number as a float (OD4c3), binary as + `bytes`, and a JSON object or array as a copy parsed back from its JSON encoding. + Raises AblyException 40013 for any other value (RTLMV4c, OD4a), which includes a + number that is not finite, a dict or list that is not JSON-encodable, a blueprint, + and a live object or a `PathObject` or `Instance` wrapping one (RTLMV4c1). + """ + if isinstance(value, bool): + return ObjectData(boolean=value) # RTLMV4d6 + if isinstance(value, (int, float)): + number = _finite_number(value) + if number is not None: + return ObjectData(number=number) # RTLMV4d5 + elif isinstance(value, str): + return ObjectData(string=value) # RTLMV4d4 + elif isinstance(value, (bytes, bytearray)): + return ObjectData(bytes=bytes(value)) # RTLMV4d7 + elif isinstance(value, (dict, list)): + try: + encoded = json.dumps(value, allow_nan=False) + except (TypeError, ValueError, RecursionError): + pass + else: + return ObjectData(json=json.loads(encoded)) # RTLMV4d3 + raise AblyException('Map value data type is unsupported', 400, 40013) + + +def evaluate_live_counter(value: LiveCounter, timestamp_ms: int) -> ObjectMessage: + """RTLCV4: the COUNTER_CREATE `ObjectMessage` a `LiveCounter` evaluates to. + + `timestamp_ms` is the server time the object id is generated with (RTLCV4e). The + operation carries `counter_create_with_object_id`, whose `derived_from` is the + `CounterCreate` it was built from (RTLCV4g5). Raises AblyException 40003 for a count + that is not a finite number (RTLCV4a). + """ + count = _finite_number(value._count) + if count is None: + raise AblyException('Counter value should be a valid number', 400, 40003) # RTLCV4a + + counter_create = CounterCreate(count=count) # RTLCV4b + initial_value = _initial_value(counter_create) # RTLCV4c + nonce = generate_nonce() # RTLCV4d + object_id = generate_object_id('counter', initial_value, nonce, timestamp_ms) # RTLCV4f + return ObjectMessage(operation=ObjectOperation( + action=ObjectOperationAction.COUNTER_CREATE, # RTLCV4g1 + object_id=object_id, # RTLCV4g2 + counter_create_with_object_id=CounterCreateWithObjectId( + initial_value=initial_value, # RTLCV4g4 + nonce=nonce, # RTLCV4g3 + derived_from=counter_create, # RTLCV4g5 + ), + )) + + +def evaluate_live_map(value: LiveMap, timestamp_ms: int) -> list[ObjectMessage]: + """RTLMV4: the ObjectMessages a `LiveMap` evaluates to, nested creates first, depth-first, + and this map's MAP_CREATE last (RTLMV4k). + + The MAP_CREATE carries `map_create_with_object_id`, whose `derived_from` is the + `MapCreate` it was built from (RTLMV4j5). Raises AblyException 40003 for entries that + are not a dict or a key that is not a string (RTLMV4a, RTLMV4b), and 40013 for a + value of an unsupported type (RTLMV4c). + """ + # `LiveMap.create()` and `LiveMap.create(None)` both leave the entries unset + entries = {} if value._entries is None else value._entries + if not isinstance(entries, dict): + raise AblyException('Map entries should be a dict', 400, 40003) # RTLMV4a + for key in entries: + validate_key(key) # RTLMV4b + + # Every value at this level is validated (RTLMV4c) before any blueprint is evaluated + data: dict[str, ObjectData | None] = { + key: None if isinstance(entry, (LiveCounter, LiveMap)) else primitive_to_object_data(entry) + for key, entry in entries.items() + } + + messages: list[ObjectMessage] = [] + for key, entry in entries.items(): + if isinstance(entry, LiveCounter): + counter_create_message = evaluate_live_counter(entry, timestamp_ms) # RTLMV4d1 + messages.append(counter_create_message) + data[key] = ObjectData(object_id=counter_create_message.operation.object_id) + elif isinstance(entry, LiveMap): + nested_messages = evaluate_live_map(entry, timestamp_ms) # RTLMV4d2 + messages.extend(nested_messages) + data[key] = ObjectData(object_id=nested_messages[-1].operation.object_id) + + map_create = MapCreate( # RTLMV4e + semantics=ObjectsMapSemantics.LWW, + entries={key: ObjectsMapEntry(data=entry_data) for key, entry_data in data.items()}, + ) + initial_value = _initial_value(map_create) # RTLMV4f + nonce = generate_nonce() # RTLMV4g + object_id = generate_object_id('map', initial_value, nonce, timestamp_ms) # RTLMV4i + messages.append(ObjectMessage(operation=ObjectOperation( + action=ObjectOperationAction.MAP_CREATE, # RTLMV4j1 + object_id=object_id, # RTLMV4j2 + map_create_with_object_id=MapCreateWithObjectId( + initial_value=initial_value, # RTLMV4j4 + nonce=nonce, # RTLMV4j3 + derived_from=map_create, # RTLMV4j5 + ), + ))) + return messages # RTLMV4k + + +def evaluate(value: LiveCounter | LiveMap, timestamp_ms: int) -> list[ObjectMessage]: + """The ObjectMessages a blueprint evaluates to, the create for `value` itself last.""" + if isinstance(value, LiveCounter): + return [evaluate_live_counter(value, timestamp_ms)] + if isinstance(value, LiveMap): + return evaluate_live_map(value, timestamp_ms) + raise TypeError(f'Expected a LiveCounter or LiveMap, got {type(value).__name__}') + + +def _finite_number(value: Any) -> float | None: + """`value` as a float if it is a finite number, else None. A boolean is not a number.""" + if isinstance(value, bool) or not isinstance(value, (int, float)): + return None + try: + number = float(value) + except OverflowError: + return None + return number if math.isfinite(number) else None + + +def _initial_value(create: CounterCreate | MapCreate) -> str: + """RTLCV4c, RTLMV4f: the JSON string of `create` in its JSON-wire encoding (OD4d), whatever + protocol the connection uses. + + The object id is the hash of this exact string (RTO14b1), and it is sent unchanged as + the `initialValue`. Keys keep the order `to_dict` builds them in, with no whitespace + between tokens, non-ASCII characters escaped, and numbers in Python's shortest + round-trip form. + """ + return json.dumps(create.to_dict(WIRE_FORMAT_JSON), separators=(',', ':'), allow_nan=False) diff --git a/ably/pubsub/types/channelmode.py b/ably/pubsub/types/channelmode.py index b16b355d..47a5f9c5 100644 --- a/ably/pubsub/types/channelmode.py +++ b/ably/pubsub/types/channelmode.py @@ -12,6 +12,8 @@ class ChannelMode(int, Enum): PRESENCE_SUBSCRIBE = Flag.PRESENCE_SUBSCRIBE ANNOTATION_PUBLISH = Flag.ANNOTATION_PUBLISH ANNOTATION_SUBSCRIBE = Flag.ANNOTATION_SUBSCRIBE + OBJECT_SUBSCRIBE = Flag.OBJECT_SUBSCRIBE + OBJECT_PUBLISH = Flag.OBJECT_PUBLISH def encode_channel_mode(modes: list[ChannelMode]) -> int: diff --git a/ably/pubsub/types/connectiondetails.py b/ably/pubsub/types/connectiondetails.py index a281daed..d1b461f7 100644 --- a/ably/pubsub/types/connectiondetails.py +++ b/ably/pubsub/types/connectiondetails.py @@ -8,13 +8,19 @@ class ConnectionDetails: connection_key: str def __init__(self, connection_state_ttl: int, max_idle_interval: int, - connection_key: str, client_id: str): + connection_key: str, client_id: str, site_code: str = None, + objects_gc_grace_period: int = None): self.connection_state_ttl = connection_state_ttl self.max_idle_interval = max_idle_interval self.connection_key = connection_key self.client_id = client_id + # CD2j: the site the connection reached, under which LiveObjects applies its own operations + self.site_code = site_code + # CD2i: how long tombstoned objects and map entries are kept before release, in milliseconds + self.objects_gc_grace_period = objects_gc_grace_period @staticmethod def from_dict(json_dict: dict): return ConnectionDetails(json_dict.get('connectionStateTtl'), json_dict.get('maxIdleInterval'), - json_dict.get('connectionKey'), json_dict.get('clientId')) + json_dict.get('connectionKey'), json_dict.get('clientId'), + json_dict.get('siteCode'), json_dict.get('objectsGCGracePeriod')) diff --git a/ably/pubsub/types/flags.py b/ably/pubsub/types/flags.py index 86666019..ff8fe3a0 100644 --- a/ably/pubsub/types/flags.py +++ b/ably/pubsub/types/flags.py @@ -8,6 +8,7 @@ class Flag(int, Enum): RESUMED = 1 << 2 TRANSIENT = 1 << 4 ATTACH_RESUME = 1 << 5 + HAS_OBJECTS = 1 << 7 # Channel mode flags PRESENCE = 1 << 16 PUBLISH = 1 << 17 @@ -15,6 +16,8 @@ class Flag(int, Enum): PRESENCE_SUBSCRIBE = 1 << 19 ANNOTATION_PUBLISH = 1 << 21 ANNOTATION_SUBSCRIBE = 1 << 22 + OBJECT_SUBSCRIBE = 1 << 24 + OBJECT_PUBLISH = 1 << 25 def has_flag(message_flags: int, flag: Flag): From a68d184ad5d40750e27b42e58def88d6f8f87c11 Mon Sep 17 00:00:00 2001 From: owenpearson Date: Fri, 9 Oct 2026 13:52:40 +0100 Subject: [PATCH 16/71] objects: sync and publish objects over the channel Routes ATTACHED, OBJECT and OBJECT_SYNC to the channel's RealtimeObject, and every other channel state change to it, so that each attach starts an objects sync and a detached or failed channel clears its objects. The transport now hands OBJECT and OBJECT_SYNC frames to the channel at all. RealtimeObject.get() attaches the channel through the RTL33 ensure-active-channel procedure and waits for the sync; a write publishes an OBJECT message and, once its ACK assigns serials, applies the operations locally rather than waiting for the echo. The internal counter and map gain their increment, decrement, set and remove, and the GC sweep runs on the client's clock while the channel is attached. Connects with protocol version 6, whose wire format the objects messages use, and identifies to REST as version 6 to match. Adds the echo_messages client option, sent as the echo connect parameter, which LiveObjects writes require. Co-Authored-By: Claude Opus 5.5 (1M context) --- ably/pubsub/objects/livecounter.py | 38 ++- ably/pubsub/objects/livemap.py | 39 +++- ably/pubsub/objects/realtimeobject.py | 219 ++++++++++++++++-- ably/pubsub/realtime/channel.py | 60 +++++ ably/pubsub/realtime/connectionmanager.py | 2 + ably/pubsub/server/__init__.py | 3 + ably/pubsub/transport/defaults.py | 2 +- ably/pubsub/transport/websockettransport.py | 4 +- ably/pubsub/types/options.py | 10 +- ably/pubsub/version.py | 2 +- test/ably/http/httphttp_test.py | 2 +- test/uts/deviations.md | 14 +- .../unit/client/realtime_client_test.py | 3 - 13 files changed, 352 insertions(+), 46 deletions(-) diff --git a/ably/pubsub/objects/livecounter.py b/ably/pubsub/objects/livecounter.py index 64d2425d..4c7c016a 100644 --- a/ably/pubsub/objects/livecounter.py +++ b/ably/pubsub/objects/livecounter.py @@ -3,15 +3,16 @@ from __future__ import annotations import logging -from typing import TYPE_CHECKING +import math +from typing import TYPE_CHECKING, Any from ably.pubsub.objects.enums import ObjectsOperationSource from ably.pubsub.objects.liveobject import CounterUpdate, LiveCounterUpdate, LiveObject -from ably.pubsub.objects.objectmessage import ObjectOperationAction +from ably.pubsub.objects.objectmessage import CounterInc, ObjectMessage, ObjectOperation, ObjectOperationAction from ably.pubsub.util.clock import Clock +from ably.pubsub.util.exceptions import AblyException if TYPE_CHECKING: - from ably.pubsub.objects.objectmessage import CounterInc, ObjectMessage, ObjectOperation from ably.pubsub.objects.objectspool import ObjectsPool log = logging.getLogger(__name__) @@ -39,11 +40,23 @@ async def increment(self, amount: float) -> None: Raises AblyException 40003 if `amount` is not a finite number (RTLC12e1); a bool is not a number. """ - raise NotImplementedError + await self._publish_counter_inc(_validate_amount(amount)) async def decrement(self, amount: float) -> None: """RTLC13: `increment` by `-amount`, after the same validation (RTLC13c).""" - raise NotImplementedError + await self._publish_counter_inc(-_validate_amount(amount)) # RTLC13b, RTLC13c + + async def _publish_counter_inc(self, number: float) -> None: + realtime_object = self.realtime_object + if realtime_object is None: + raise AblyException('Unable to increment a counter that is not on a channel', 400, 40000) + + object_message = ObjectMessage(operation=ObjectOperation( + action=ObjectOperationAction.COUNTER_INC, # RTLC12e2 + object_id=self.object_id, # RTLC12e3 + counter_inc=CounterInc(number=number), # RTLC12e5 + )) + await realtime_object._publish_and_apply([object_message]) # RTLC12g def apply_operation(self, object_message: ObjectMessage, source: ObjectsOperationSource) -> bool: """RTLC7: applies `object_message.operation`, returning whether it was applied (RTLC7g). @@ -154,3 +167,18 @@ def diff(previous_data: float, new_data: float, *, for_tombstone: bool = False) if amount == 0 and not for_tombstone: return LiveCounterUpdate(noop=True) # RTLC14c return LiveCounterUpdate(update=CounterUpdate(amount=amount)) + + +def _validate_amount(amount: Any) -> float: + """RTLC12e1: `amount` as a float, raising AblyException 40003 unless it is a finite number. + + A bool is not a number, and None is not an omitted amount. + """ + if not isinstance(amount, bool) and isinstance(amount, (int, float)): + try: + number = float(amount) + except OverflowError: + number = math.inf + if math.isfinite(number): + return number + raise AblyException('Counter value increment should be a valid number', 400, 40003) diff --git a/ably/pubsub/objects/livemap.py b/ably/pubsub/objects/livemap.py index 74a0e3ac..00d9f58b 100644 --- a/ably/pubsub/objects/livemap.py +++ b/ably/pubsub/objects/livemap.py @@ -11,15 +11,20 @@ from ably.pubsub.objects.objectmessage import ( MapRemove, MapSet, + ObjectData, + ObjectMessage, + ObjectOperation, ObjectOperationAction, ObjectsMapEntry, ObjectsMapSemantics, ) +from ably.pubsub.objects.valuetypes import LiveCounter, LiveMap, evaluate, primitive_to_object_data, validate_key from ably.pubsub.util.clock import Clock +from ably.pubsub.util.exceptions import AblyException if TYPE_CHECKING: - from ably.pubsub.objects.objectmessage import ObjectData, ObjectMessage, ObjectOperation from ably.pubsub.objects.objectspool import ObjectsPool + from ably.pubsub.objects.realtimeobject import RealtimeObject from ably.pubsub.objects.valuetypes import LiveMapValue log = logging.getLogger(__name__) @@ -78,14 +83,42 @@ async def set(self, key: str, value: LiveMapValue) -> None: Raises AblyException 40003 for a key that is not a string and 40013 for a value of an unsupported type (RTLM20e1, RTLMV4b, RTLMV4c). """ - raise NotImplementedError + validate_key(key) # RTLM20e1 + object_messages: list[ObjectMessage] = [] + if isinstance(value, (LiveCounter, LiveMap)): + # RTLM20e7g1: the creates the blueprint evaluates to, its contents validated as it is (RTLMV4c) + server_time_ms = await self._publishing_realtime_object()._get_server_time_ms() + object_messages = evaluate(value, server_time_ms) + data = ObjectData(object_id=object_messages[-1].operation.object_id) # RTLM20e7g2 + else: + data = primitive_to_object_data(value) # RTLM20e1, RTLM20e7b-RTLM20e7f + + object_messages.append(ObjectMessage(operation=ObjectOperation( + action=ObjectOperationAction.MAP_SET, # RTLM20e2 + object_id=self.object_id, # RTLM20e3 + map_set=MapSet(key=key, value=data), # RTLM20e6, RTLM20e7 + ))) + await self._publishing_realtime_object()._publish_and_apply(object_messages) # RTLM20h1, RTLM20h2 async def remove(self, key: str) -> None: """RTLM21: publishes a MAP_REMOVE through `RealtimeObject._publish_and_apply`. Raises AblyException 40003 for a key that is not a string (RTLM21e1). """ - raise NotImplementedError + validate_key(key) # RTLM21e1 + realtime_object = self._publishing_realtime_object() + object_message = ObjectMessage(operation=ObjectOperation( + action=ObjectOperationAction.MAP_REMOVE, # RTLM21e2 + object_id=self.object_id, # RTLM21e3 + map_remove=MapRemove(key=key), # RTLM21e5 + )) + await realtime_object._publish_and_apply([object_message]) # RTLM21g + + def _publishing_realtime_object(self) -> RealtimeObject: + realtime_object = self.realtime_object + if realtime_object is None: + raise AblyException('Unable to update a map that is not on a channel', 400, 40000) + return realtime_object def is_entry_tombstoned(self, entry: ObjectsMapEntry) -> bool: """RTLM14: whether `entry` is tombstoned, or references a tombstoned object in `pool`.""" diff --git a/ably/pubsub/objects/realtimeobject.py b/ably/pubsub/objects/realtimeobject.py index 8cf75406..5d61b960 100644 --- a/ably/pubsub/objects/realtimeobject.py +++ b/ably/pubsub/objects/realtimeobject.py @@ -2,6 +2,8 @@ from __future__ import annotations +import asyncio +import dataclasses import logging from typing import TYPE_CHECKING, Callable @@ -17,19 +19,20 @@ ObjectsMapSemantics, ) from ably.pubsub.objects.objectspool import ObjectsPool +from ably.pubsub.objects.pathobject import LiveMapPathObject from ably.pubsub.objects.pathobjectsubscriptionregister import PathObjectSubscriptionRegister from ably.pubsub.objects.subscription import StatusSubscription from ably.pubsub.objects.syncobjectspool import SyncObjectsPool +from ably.pubsub.transport.websockettransport import ProtocolMessageAction +from ably.pubsub.types.channelmode import ChannelMode +from ably.pubsub.types.channelstate import ChannelState from ably.pubsub.util.clock import Clock, select_clock +from ably.pubsub.util.exceptions import AblyException if TYPE_CHECKING: from ably.pubsub.objects.objectmessage import ObjectMessage - from ably.pubsub.objects.pathobject import LiveMapPathObject from ably.pubsub.realtime.channel import RealtimeChannel - from ably.pubsub.types.channelmode import ChannelMode - from ably.pubsub.types.channelstate import ChannelState from ably.pubsub.types.operations import PublishResult - from ably.pubsub.util.exceptions import AblyException from ably.pubsub.util.helper import Timer log = logging.getLogger(__name__) @@ -51,6 +54,12 @@ ObjectsSyncState.SYNCED: ObjectsEvent.SYNCED, } +# RTO23c1, RTO20e1: the channel states that fail a wait for SYNCED +_SYNC_WAIT_FAILURE_STATES = frozenset((ChannelState.DETACHED, ChannelState.SUSPENDED, ChannelState.FAILED)) + +# RTO27a: the channel states after which the objects' data can no longer be known +_DATA_CLEARING_STATES = frozenset((ChannelState.DETACHED, ChannelState.FAILED)) + class RealtimeObject: """RTO*: the objects on one channel, and the entry point to them. @@ -86,6 +95,9 @@ def __init__(self, channel: RealtimeChannel | None = None, *, pool: ObjectsPool self._gc_interval_ms: int = GC_INTERVAL_MS # RTO10a, read each time the GC timer is scheduled self._gc_timer: Timer | None = None self._last_gc_grace_period_ms: int = GC_GRACE_PERIOD_MS # RTO10b + self._server_time_offset_ms: float | None = None # RTO16a + # RTO23c, RTO20e: each pending wait for SYNCED, with the description its 92008 error gives + self._sync_waiters: dict[asyncio.Future[None], str] = {} # RTO18: (token, callback) for each `on` call, by event, so that a callback registered # twice is called twice (RTO18d) and each StatusSubscription removes only its own self._sync_state_listeners: dict[ObjectsEvent, list[tuple[object, Callable[[], None]]]] = { @@ -102,7 +114,14 @@ async def get(self) -> LiveMapPathObject: Raises AblyException 92008 if the channel enters DETACHED, SUSPENDED or FAILED while waiting (RTO23c1), and 90001 if the channel is FAILED (RTL33c). """ - raise NotImplementedError + self._throw_if_missing_channel_mode(ChannelMode.OBJECT_SUBSCRIBE) # RTO23a + if self._channel is not None: + await self._channel._ensure_active() # RTO23e + + if self._sync_state != ObjectsSyncState.SYNCED: + await self._wait_for_synced('The object could not be retrieved') # RTO23c, RTO23c1 + + return LiveMapPathObject(self, self._objects_pool.root, []) # RTO23d def on(self, event: ObjectsEvent, callback: Callable[[], None]) -> StatusSubscription: """RTO18: calls `callback`, with no arguments, whenever the sync state reaches `event`. @@ -235,10 +254,17 @@ def _act_on_channel_state(self, state: ChannelState, reason: AblyException | Non DETACHED and FAILED clear every object's data (RTO27a); every other state keeps it (RTO27b). DETACHED, SUSPENDED and FAILED fail a `get` or a `_publish_and_apply` - waiting for SYNCED with 92008 (RTO23c1, RTO20e1), with the channel's `error_reason` - as the cause. + waiting for SYNCED with 92008 (RTO23c1, RTO20e1), with `reason`, else the channel's + `error_reason`, as the cause. DETACHED and FAILED also stop the GC timer, which the + next ATTACHED starts again. """ - raise NotImplementedError + if state in _SYNC_WAIT_FAILURE_STATES: + self._fail_sync_waiters(state, reason) + + if state in _DATA_CLEARING_STATES: + self._objects_pool.clear_all_data() # RTO27a1 + self._sync_objects_pool.clear() # RTO27a2 + self._cancel_gc_timer() def _set_sync_state(self, state: ObjectsSyncState) -> None: """RTO17: moves to `state`, emitting the matching `ObjectsEvent` to `on` listeners (RTO17b).""" @@ -246,6 +272,9 @@ def _set_sync_state(self, state: ObjectsSyncState) -> None: return self._sync_state = state + if state == ObjectsSyncState.SYNCED: + self._resolve_sync_waiters() # RTO23c, RTO20e + event = _SYNC_STATE_EVENTS.get(state) if event is None: return @@ -325,35 +354,174 @@ def _complete_sync(self) -> None: self._applied_on_ack_serials.clear() # RTO5c9 self._set_sync_state(ObjectsSyncState.SYNCED) # RTO5c8 + # Internal API: waiting for SYNCED + + async def _wait_for_synced(self, failure_description: str) -> None: + """RTO23c, RTO20e: waits for the sync state to reach SYNCED. + + Raises AblyException 92008, whose message starts with `failure_description`, if the + channel enters DETACHED, SUSPENDED or FAILED first (RTO23c1, RTO20e1). + """ + future: asyncio.Future[None] = asyncio.get_running_loop().create_future() + self._sync_waiters[future] = failure_description + try: + await future + finally: + self._sync_waiters.pop(future, None) + + def _resolve_sync_waiters(self) -> None: + waiters, self._sync_waiters = self._sync_waiters, {} + for future in waiters: + if not future.done(): + future.set_result(None) + + def _fail_sync_waiters(self, state: ChannelState, reason: AblyException | None) -> None: + """RTO23c1, RTO20e1: fails every wait for SYNCED with 92008, caused by the channel's error.""" + waiters, self._sync_waiters = self._sync_waiters, {} + cause = reason + if cause is None and self._channel is not None: + cause = self._channel.error_reason + for future, failure_description in waiters.items(): + if not future.done(): + future.set_exception(AblyException( + f'{failure_description} due to the channel entering the {state.value} state ' + f'whilst waiting for objects sync to complete', 400, 92008, cause=cause)) + # Internal API: publishing async def _publish(self, object_messages: list[ObjectMessage]) -> PublishResult: - """RTO15: sends `object_messages` in one OBJECT ProtocolMessage and returns the ACK's result.""" - raise NotImplementedError + """RTO15: sends `object_messages` in one OBJECT ProtocolMessage and returns the ACK's result. + + Fails as `RealtimeChannel.publish` does when the connection or channel state does not + allow publishing (RTO15b), and with the NACK's error if the publish is rejected (RTO15g). + """ + channel = self._require_channel() + channel._throw_if_unpublishable_state() # RTO15b + + protocol_message = { + 'action': ProtocolMessageAction.OBJECT, # RTO15e1 + 'channel': channel.name, # RTO15e2 + # RTO15c, RTO15e3 + 'state': [object_message.to_dict(self._wire_format) for object_message in object_messages], + } + + log.debug(f'RealtimeObject._publish(): sending {len(object_messages)} object message(s); ' + f'channel={channel.name}') + + # RTO15f, RTO15g, RTO15h + return await channel.ably.connection.connection_manager.send_protocol_message(protocol_message) async def _publish_and_apply(self, object_messages: list[ObjectMessage]) -> None: """RTO20: publishes `object_messages`, then applies them locally with the serials the ACK - assigned, as LOCAL operations, once synced.""" - raise NotImplementedError + assigned, as LOCAL operations, once synced. + + Raises the publish's error (RTO20b), and AblyException 92008 if the channel enters + DETACHED, SUSPENDED or FAILED while waiting for SYNCED (RTO20e1). An operation that + cannot be applied locally is left for its echo to apply (RTO20c, RTO20d1). + """ + publish_result = await self._publish(object_messages) # RTO20b + + # RTO20c1 + connection_details = self._channel.ably.connection.connection_details + site_code = connection_details.site_code if connection_details is not None else None + if not site_code: + log.error(f'RealtimeObject._publish_and_apply(): the operations will not be applied locally, ' + f'as the connection details carry no siteCode; channel={self._channel_name}') + return + + # RTO20c2 + serials = publish_result.serials if publish_result is not None else None + if serials is None or len(serials) != len(object_messages): + log.error(f'RealtimeObject._publish_and_apply(): the operations will not be applied locally, as the ' + f'ACK carries {len(serials or [])} serial(s) for {len(object_messages)} object message(s); ' + f'channel={self._channel_name}') + return + + # RTO20d + synthetic_messages = [] + for index, (object_message, serial) in enumerate(zip(object_messages, serials)): + if serial is None: + # RTO20d1 + log.debug(f'RealtimeObject._publish_and_apply(): the operation at index {index} will not be ' + f'applied locally, as the ACK assigned it no serial; channel={self._channel_name}') + continue + # RTO20d2, RTO20d3 + synthetic_messages.append(dataclasses.replace(object_message, serial=serial, site_code=site_code)) + + # RTO20d4 + if not synthetic_messages: + return + + if self._sync_state != ObjectsSyncState.SYNCED: + # RTO20e, RTO20e1 + await self._wait_for_synced('The operation could not be applied locally') + + self._apply_object_messages(synthetic_messages, ObjectsOperationSource.LOCAL) # RTO20f async def _get_server_time_ms(self) -> int: - """RTO16: the current server time, from a persisted offset where one is known (RTO16a).""" - raise NotImplementedError + """RTO16: the current server time, from a persisted offset where one is known (RTO16a). + + The offset is the one this RealtimeObject persisted, else the one the client's `Auth` + persisted for token requests (RSA10k); with neither, the server is asked for its time. + A `RealtimeObject` with no channel reads its own clock. + """ + now_ms = self._clock.now_ms() + if self._channel is None: + return now_ms + + offset_ms = self._server_time_offset_ms + if offset_ms is None: + offset_ms = self._channel.ably.auth.time_offset + if offset_ms is None: + server_time_ms = await self._channel.ably.time() # RTO16 + self._server_time_offset_ms = server_time_ms - self._clock.now_ms() + return int(server_time_ms) + return int(now_ms + offset_ms) # RTO16a + + def _require_channel(self) -> RealtimeChannel: + if self._channel is None: + raise AblyException('Unable to publish object messages from a RealtimeObject with no channel', + 400, 40000) + return self._channel # Internal API: preconditions def _throw_if_missing_channel_mode(self, mode: ChannelMode) -> None: - """RTO2: raises AblyException 40024 if `mode` is neither granted nor, before ATTACHED, requested.""" - raise NotImplementedError + """RTO2: raises AblyException 40024 if `mode` is neither granted nor, before ATTACHED, requested. + + The modes the server granted are checked while the channel is ATTACHED and its ATTACHED + carried mode flags (RTO2a); otherwise, the modes requested in the channel options are + (RTO2b). + """ + if self._channel is None: + return + + modes = self._channel.modes if self._channel.state == ChannelState.ATTACHED else None # RTO2a + if not modes: + modes = (self._channel.options or {}).get('modes') or [] # RTO2b + if mode not in modes: + # RTO2a2, RTO2b2 + raise AblyException(f'"{mode.name}" channel mode must be set for this operation', 400, 40024) def _check_access_preconditions(self) -> None: """RTO25: OBJECT_SUBSCRIBE (40024), and 90001 if the channel is DETACHED or FAILED.""" - raise NotImplementedError + self._throw_if_missing_channel_mode(ChannelMode.OBJECT_SUBSCRIBE) # RTO25a + self._throw_if_in_channel_state(ChannelState.DETACHED, ChannelState.FAILED) # RTO25b def _check_write_preconditions(self) -> None: """RTO26: OBJECT_PUBLISH (40024), 90001 if the channel is DETACHED, FAILED or SUSPENDED, and 40000 if `echo_messages` is disabled.""" - raise NotImplementedError + self._throw_if_missing_channel_mode(ChannelMode.OBJECT_PUBLISH) # RTO26a + # RTO26b + self._throw_if_in_channel_state(ChannelState.DETACHED, ChannelState.FAILED, ChannelState.SUSPENDED) + if self._channel is not None and not self._channel.ably.options.echo_messages: + # RTO26c + raise AblyException('"echo_messages" client option must be enabled for this operation', 400, 40000) + + def _throw_if_in_channel_state(self, *states: ChannelState) -> None: + if self._channel is not None and self._channel.state in states: + raise AblyException(f'Channel operation failed as channel state is {self._channel.state.value}', + 400, 90001) # Internal API: garbage collection @@ -375,12 +543,19 @@ def _schedule_gc_timer(self) -> None: A `RealtimeObject` with no channel schedules nothing. """ - if self._channel is None: + if self._channel is None or self._gc_timer is not None: return - raise NotImplementedError + self._gc_timer = self._clock.timer(self._gc_interval_ms, self._on_gc_interval) + + def _cancel_gc_timer(self) -> None: + if self._gc_timer is not None: + self._gc_timer.cancel() + self._gc_timer = None def _on_gc_interval(self) -> None: """RTO10c: releases what has been tombstoned for the grace period, and reschedules itself.""" self._gc_timer = None - self._objects_pool.collect_garbage(self._gc_grace_period_ms, self._clock.now_ms()) - self._schedule_gc_timer() + try: + self._objects_pool.collect_garbage(self._gc_grace_period_ms, self._clock.now_ms()) + finally: + self._schedule_gc_timer() diff --git a/ably/pubsub/realtime/channel.py b/ably/pubsub/realtime/channel.py index 0dfd720b..7b79ca99 100644 --- a/ably/pubsub/realtime/channel.py +++ b/ably/pubsub/realtime/channel.py @@ -6,6 +6,8 @@ from ably.pubsub.http.channel import Channel from ably.pubsub.http.channel import Channels as HttpChannels +from ably.pubsub.objects.objectmessage import ObjectMessage +from ably.pubsub.objects.realtimeobject import RealtimeObject from ably.pubsub.realtime.annotations import RealtimeAnnotations from ably.pubsub.realtime.connection import ConnectionState from ably.pubsub.realtime.presence import RealtimePresence @@ -92,6 +94,9 @@ def __init__(self, realtime: DefaultPubSubRealtimeClient, name: str, # Initialize realtime annotations for this channel (override REST annotations) self._Channel__annotations = RealtimeAnnotations(self, realtime.connection.connection_manager) + # RTL27: the LiveObjects entry point for this channel + self.__object = RealtimeObject(self) + async def set_options(self, channel_options: ChannelOptions) -> None: """Set channel options""" should_reattach = self.should_reattach_to_set_options(channel_options) @@ -170,6 +175,27 @@ def _attach_impl(self): self._send_message(attach_msg) + # RTL33 + async def _ensure_active(self) -> None: + """The ensure-active-channel procedure: attaches the channel unless it is ATTACHED or SUSPENDED. + + Raises + ------ + AblyException + 90001 if the channel is FAILED (RTL33c), or the error the implicit attach failed + with (RTL33b1) + """ + # RTL33a + if self.state in (ChannelState.ATTACHED, ChannelState.SUSPENDED): + return + + # RTL33c + if self.state == ChannelState.FAILED: + raise AblyException(f"Channel operation failed as channel state is {self.state.value}", 400, 90001) + + # RTL33b, RTL33b1 + await self.attach() + # RTL5 async def detach(self) -> None: """Detach from channel @@ -706,6 +732,7 @@ def _on_message(self, proto_msg: dict) -> None: exception = None resumed = False has_presence = False + has_objects = False self.__attach_serial = channel_serial self.__channel_serial = channel_serial @@ -718,6 +745,8 @@ def _on_message(self, proto_msg: dict) -> None: resumed = has_flag(flags, Flag.RESUMED) # RTP1: Check for HAS_PRESENCE flag has_presence = has_flag(flags, Flag.HAS_PRESENCE) + # RTO4: whether an OBJECT_SYNC follows + has_objects = has_flag(flags, Flag.HAS_OBJECTS) # Store channel attach flags self.__modes = decode_channel_mode(flags) @@ -726,8 +755,11 @@ def _on_message(self, proto_msg: dict) -> None: if not resumed: state_change = ChannelStateChange(self.state, ChannelState.ATTACHED, resumed, exception) self._emit("update", state_change) + # RTO4: an ATTACHED restarts the objects sync even while attached, a resumed one included + self.__object._on_attached(has_objects) elif self.state == ChannelState.ATTACHING: self._notify_state(ChannelState.ATTACHED, resumed=resumed, has_presence=has_presence) + self.__object._on_attached(has_objects) # RTO4 else: log.warn("RealtimeChannel._on_message(): ATTACHED received while not attaching") elif action == ProtocolMessageAction.DETACHED: @@ -774,6 +806,24 @@ def _on_message(self, proto_msg: dict) -> None: self.__channel_serial = channel_serial except Exception as e: log.error(f"Annotation processing error {e}. Skip annotations {annotation_data}") + elif action == ProtocolMessageAction.OBJECT: + try: + object_messages = ObjectMessage.from_protocol_message(proto_msg, self.__object._wire_format) + except Exception as e: + log.error(f"Object message processing error {e}. Skip object messages {proto_msg.get('state')}") + else: + # RTL15b + if channel_serial: + self.__channel_serial = channel_serial + self.__object._handle_object_messages(object_messages) # RTO8 + elif action == ProtocolMessageAction.OBJECT_SYNC: + try: + object_messages = ObjectMessage.from_protocol_message(proto_msg, self.__object._wire_format) + except Exception as e: + log.error(f"Object sync processing error {e}. Skip object messages {proto_msg.get('state')}") + else: + # RTO5: the channelSerial of an OBJECT_SYNC carries the sync sequence and cursor (RTO5a1) + self.__object._handle_object_sync_messages(object_messages, channel_serial) elif action == ProtocolMessageAction.ERROR: error = AblyException.from_dict(proto_msg.get('error')) self._notify_state(ChannelState.FAILED, reason=error) @@ -825,6 +875,10 @@ def _notify_state(self, state: ChannelState, reason: AblyException | None = None # RTP5: Notify presence of channel state change self.__presence.act_on_channel_state(state, has_presence=has_presence, error=reason) + # RTO27: the objects act on every state but ATTACHED, whose ATTACHED message RTO4 handles + if state != ChannelState.ATTACHED: + self.__object._act_on_channel_state(state, reason) + def _send_message(self, msg: dict) -> None: asyncio.create_task(self.__realtime.connection.connection_manager.send_protocol_message(msg)) @@ -925,6 +979,12 @@ def presence(self): def annotations(self) -> RealtimeAnnotations: return self._Channel__annotations + # RTL27 + @property + def object(self) -> RealtimeObject: + """The LiveObjects on this channel""" + return self.__object + @property def modes(self): """Get the list of channel modes""" diff --git a/ably/pubsub/realtime/connectionmanager.py b/ably/pubsub/realtime/connectionmanager.py index 09ce5eab..407692d2 100644 --- a/ably/pubsub/realtime/connectionmanager.py +++ b/ably/pubsub/realtime/connectionmanager.py @@ -211,6 +211,8 @@ async def __get_transport_params(self) -> dict: # RTN2a: Set format to msgpack if use_binary_protocol is enabled if self.options.use_binary_protocol: params["format"] = "msgpack" + # RTN2h, RTC1a: whether the server echoes this connection's own messages back to it + params["echo"] = "true" if self.options.echo_messages else "false" # Add any custom transport params from options params.update(self.options.transport_params) diff --git a/ably/pubsub/server/__init__.py b/ably/pubsub/server/__init__.py index 034a3521..b31150ed 100644 --- a/ably/pubsub/server/__init__.py +++ b/ably/pubsub/server/__init__.py @@ -196,6 +196,9 @@ def create_realtime_client(**kwargs) -> PubSubRealtimeClient: queue_messages: bool Hold messages published while the connection is not yet established and send them once it is, rather than failing them. Defaults to True. + echo_messages: bool + Deliver the messages this client publishes back to it on its own connection. + LiveObjects writes require it. Defaults to True. transport_params: dict Additional query parameters to send when opening the realtime connection. diff --git a/ably/pubsub/transport/defaults.py b/ably/pubsub/transport/defaults.py index 40d73e08..ae5310e6 100644 --- a/ably/pubsub/transport/defaults.py +++ b/ably/pubsub/transport/defaults.py @@ -1,5 +1,5 @@ class Defaults: - protocol_version = "5" + protocol_version = "6" connectivity_check_url = "https://internet-up.ably-realtime.com/is-the-internet-up.txt" endpoint = 'main' diff --git a/ably/pubsub/transport/websockettransport.py b/ably/pubsub/transport/websockettransport.py index 7177732c..785bae46 100644 --- a/ably/pubsub/transport/websockettransport.py +++ b/ably/pubsub/transport/websockettransport.py @@ -207,7 +207,9 @@ async def on_protocol_message(self, msg): ProtocolMessageAction.MESSAGE, ProtocolMessageAction.PRESENCE, ProtocolMessageAction.ANNOTATION, - ProtocolMessageAction.SYNC + ProtocolMessageAction.SYNC, + ProtocolMessageAction.OBJECT, + ProtocolMessageAction.OBJECT_SYNC, ): self.connection_manager.on_channel_message(msg) diff --git a/ably/pubsub/types/options.py b/ably/pubsub/types/options.py index 51f6016e..0a4fc932 100644 --- a/ably/pubsub/types/options.py +++ b/ably/pubsub/types/options.py @@ -34,8 +34,8 @@ def __init__(self, client_id=None, log_level=0, tls=True, rest_host=None, realti idempotent_rest_publishing=None, loop=None, auto_connect=True, suspended_retry_timeout=None, connectivity_check_url=None, channel_retry_timeout=Defaults.channel_retry_timeout, add_request_ids=False, - vcdiff_decoder: VCDiffDecoder = None, transport_params=None, _test_options=None, - **kwargs): + vcdiff_decoder: VCDiffDecoder = None, transport_params=None, echo_messages=True, + _test_options=None, **kwargs): super().__init__(**kwargs) @@ -108,6 +108,7 @@ def __init__(self, client_id=None, log_level=0, tls=True, rest_host=None, realti self.__tls_port = tls_port self.__use_binary_protocol = use_binary_protocol self.__queue_messages = queue_messages + self.__echo_messages = echo_messages self.__recover = recover self.__endpoint = endpoint self.__http_open_timeout = http_open_timeout @@ -189,6 +190,11 @@ def queue_messages(self): def queue_messages(self, value): self.__queue_messages = value + # TO3h + @property + def echo_messages(self): + return self.__echo_messages + @property def recover(self): return self.__recover diff --git a/ably/pubsub/version.py b/ably/pubsub/version.py index 8389a068..59f747d4 100644 --- a/ably/pubsub/version.py +++ b/ably/pubsub/version.py @@ -5,5 +5,5 @@ ``ably-pubsub-*`` distributions, and so cannot carry one. """ -api_version = '5' +api_version = '6' lib_version = '3.1.4' diff --git a/test/ably/http/httphttp_test.py b/test/ably/http/httphttp_test.py index e89fb1c8..7f0bfa04 100644 --- a/test/ably/http/httphttp_test.py +++ b/test/ably/http/httphttp_test.py @@ -190,7 +190,7 @@ async def test_request_headers(self): # API assert 'X-Ably-Version' in r.request.headers - assert r.request.headers['X-Ably-Version'] == '5' + assert r.request.headers['X-Ably-Version'] == '6' # Agent assert 'Ably-Agent' in r.request.headers diff --git a/test/uts/deviations.md b/test/uts/deviations.md index 2eb9daa1..7e97de4d 100644 --- a/test/uts/deviations.md +++ b/test/uts/deviations.md @@ -31,8 +31,8 @@ Variants` section and run every one of their tests twice, once per protocol, and `rest/unit` tests are parametrized over a table of fixtures the specification gives inline. That turns 1141 derived tests into 1234 pytest cases. -Of **1132 Test IDs, derived as 1141 tests and run as 1234 pytest cases**: 947 Test IDs -(956 tests, 1047 cases) pass, 170 (170 tests, 172 cases) are gated behind +Of **1132 Test IDs, derived as 1141 tests and run as 1234 pytest cases**: 948 Test IDs +(957 tests, 1048 cases) pass, 169 (169 tests, 171 cases) are gated behind `RUN_DEVIATIONS`, and 15 (15 tests, 15 cases) cannot be run at all. The three groups are disjoint: two Test IDs, and one parametrized test, have a gated part and a passing part, and are counted with the gated. Every gated test has been confirmed to fail when @@ -40,12 +40,12 @@ enabled, so none of them passes under both behaviours. 494 of the Test IDs come `uts/rest/unit` (503 tests, 536 cases), 481 from `uts/realtime/unit` (481, 481), 84 from `uts/rest/integration` (84, 122) and 73 from `uts/realtime/integration` (73, 95); 8 of the REST integration ids (8, 8) and 30 of the realtime ones (30, 30) come from the -`proxy` package within each. Of the gated Test IDs 79 are REST and 91 realtime, which is -80 REST cases and 92 realtime. +`proxy` package within each. Of the gated Test IDs 79 are REST and 90 realtime, which is +80 REST cases and 91 realtime. A further 130 pytest cases under `helpers/` cover the mock infrastructure itself and are not derived from a specification. -The 159 gated Test IDs that record SDK non-compliance — 159 tests, 161 cases — reduce to +The 158 gated Test IDs that record SDK non-compliance — 158 tests, 160 cases — reduce to **70 distinct root causes**, 26 on the REST side and 44 on the realtime side. Three further defects are recorded below with no test of their own, because the specification's test cannot discriminate (RTP18a), has nothing to assert against (the timezone split on @@ -839,7 +839,7 @@ on the `recover` parameter the connection never sends. | TP3a, TP3d, TP3g | Presence attributes defaulted from the encapsulating ProtocolMessage. There is no ProtocolMessage type; `ably/realtime/channel.py:751-761` passes the presence array through without context. Matters for synthesized-leave detection and `memberKey` | 3 | | TB4, RTL7h, RTP6e | `attachOnSubscribe`. `ChannelOptions.__init__` (`channeloptions.py:22-26`) takes only `cipher`, `params` and `modes`, and `subscribe()` on the channel, on presence and on annotations all end unconditionally with `await attach()`. This absence also forces the largest single adaptation in the suite, below | 3 | | RSL7 | `RestChannel#setOptions`. The realtime channel implements it; the REST `options` setter expects the kwargs dict `Channels.get` collected, so a `ChannelOptions` raises `TypeError` | 2 | -| RTC1a (TO3h), RTL7f | `echoMessages`, in both the forms RTL7f allows. There is no `echo_messages` client option — passing one raises `TypeError` — and no `echo` connect parameter, so every message the server sends is delivered whatever its `connectionId` | 2 | +| RTL7f | `echoMessages` in its client-filter form. The `echo_messages` option sends `echo=false` (RTC1a), which the server honours, but the client does not itself drop a message carrying its own `connectionId`, and that is the only form a mock-backed test can observe | 1 | | RTP12, RTP12a, RTP12c | `RealtimePresence#history`. The realtime *channel* does delegate `history` to the REST implementation; only the presence object is missing it | 2 | | RTN23c1, RTN23c2 | PING/PONG. `ProtocolMessageAction` stops at `ANNOTATION` (21), so PING (22) and PONG (23) are not modelled and action 22 matches no branch of `on_protocol_message` (`websockettransport.py:37-59`, `:143-199`). The message is counted as activity and discarded | 2 | | TI4, TI1/TI5 | `href` anywhere in the SDK, and `cause` when deserialising. `AblyException.from_dict` and `raise_for_response` read only `message`, `statusCode` and `code`, so both fields are dropped from server errors | 2 | @@ -2741,7 +2741,7 @@ Each row is one feature and one issue. None is a bug in existing code. | Retry backoff, jitter and `retryIn` on both state-change types | RTB1, RTB1a, RTB1b | 4 | `connection/backoff_jitter_test.py` | | `RealtimeChannel#whenState` (the connection has a private equivalent) | RTL25, RTL25a, RTL25b | 4 | `channels/channel_when_state_test.py` | | `attachOnSubscribe` on `ChannelOptions`. Also forces the suite's largest adaptation — 21 tests attach explicitly to work around it | TB4, RTL7h, RTP6e | 3 | `channels/channel_subscribe_test.py -k rtl7h`, `channels/channel_options_test.py -k tb4`, `presence/realtime_presence_subscribe_test.py -k rtp6e` | -| `echoMessages`, in both the client-filter and `echo`-parameter forms | RTC1a, RTL7f | 2 | `client/realtime_client_test.py -k rtc1a`, `channels/channel_subscribe_test.py -k rtl7f` | +| `echoMessages` in its client-filter form; the `echo` connect parameter is sent | RTL7f | 1 | `channels/channel_subscribe_test.py -k rtl7f` | | `RealtimePresence#history` (the realtime *channel* does delegate `history`) | RTP12, RTP12a, RTP12c | 2 | `presence/realtime_presence_history_test.py` | | PING/PONG handling — actions 22 and 23 are not modelled | RTN23c1, RTN23c2 | 2 | `connection/heartbeat_test.py -k rtn23c1` | | The `heartbeats` connect parameter. Binding on ably-python, which cannot observe ping frames | RTN23a | 1 | `connection/heartbeat_test.py -k rtn23a_heartbeats_true` | diff --git a/test/uts/realtime/unit/client/realtime_client_test.py b/test/uts/realtime/unit/client/realtime_client_test.py index 3c548191..b680edd5 100644 --- a/test/uts/realtime/unit/client/realtime_client_test.py +++ b/test/uts/realtime/unit/client/realtime_client_test.py @@ -151,9 +151,6 @@ async def test_rtc17_client_id_attribute(): # UTS: realtime/unit/RTC1a/echo-messages-option-0 -# DEVIATION: ably-python has no `echo_messages` option and sends no `echo` query -# parameter. See deviations.md. -@deviation async def test_rtc1a_echo_messages_option(): # RTC1a_1: echoMessages defaults to true mock_ws = MockWebSocket() From e2b577a134cb6d9e2d2dab8794bc97bf08802a84 Mon Sep 17 00:00:00 2001 From: owenpearson Date: Fri, 9 Oct 2026 13:52:49 +0100 Subject: [PATCH 17/71] objects: add the path-based LiveObjects API Implements the public API LODR-061 specifies. A PathObject names a location in the channel's object tree and resolves it on every call, so it keeps working as the objects at that path are replaced; an Instance holds one object and follows it wherever it moves. Each has a base class carrying navigation, instance(), compact() and compact_json(), and three views, as_live_map(), as_live_counter() and as_primitive(), that carry the methods specific to a type. The path views are unchecked, because a path's type can change under it: a read through the wrong view answers None and a write raises 92005 or 92007. The instance views check, and raise 92007 at once. Path subscriptions are dispatched from every object update to each path the updated object can be reached by, filtered by depth, with the public ObjectMessage describing the operation. The LiveObjects public names are exported from ably.pubsub.server. Co-Authored-By: Claude Opus 5.5 (1M context) --- ably/pubsub/objects/instance.py | 268 ++++++++++++++++-- ably/pubsub/objects/pathobject.py | 178 ++++++++++-- .../objects/pathobjectsubscriptionregister.py | 72 ++++- ably/pubsub/server/__init__.py | 55 ++++ 4 files changed, 511 insertions(+), 62 deletions(-) diff --git a/ably/pubsub/objects/instance.py b/ably/pubsub/objects/instance.py index c8112cb2..17fa2ba5 100644 --- a/ably/pubsub/objects/instance.py +++ b/ably/pubsub/objects/instance.py @@ -5,22 +5,152 @@ `LiveCounterInstance` or `PrimitiveInstance`, matching the value it wraps. The view helpers are checked: asking for a type the instance does not wrap raises `AblyException` 92007 (RTTS9d). + +The module also holds what the path and instance views share about resolved values: +their `ValueType`, the type filter `value(expected)` applies, and compaction. """ from __future__ import annotations +import base64 +import copy from dataclasses import dataclass from typing import TYPE_CHECKING, Any, Callable, overload +from ably.pubsub.objects.enums import ValueType +from ably.pubsub.objects.livecounter import InternalLiveCounter +from ably.pubsub.objects.livemap import InternalLiveMap +from ably.pubsub.objects.liveobject import LiveObject +from ably.pubsub.objects.publicmessage import ObjectMessage +from ably.pubsub.util.exceptions import AblyException + if TYPE_CHECKING: from ably.pubsub.objects.batch import Batch, LiveCounterBatchContext, LiveMapBatchContext - from ably.pubsub.objects.enums import ValueType - from ably.pubsub.objects.liveobject import LiveObject - from ably.pubsub.objects.publicmessage import ObjectMessage + from ably.pubsub.objects.liveobject import LiveObjectUpdate from ably.pubsub.objects.realtimeobject import RealtimeObject from ably.pubsub.objects.subscription import Subscription from ably.pubsub.objects.valuetypes import LiveMapValue, Primitive, T +# The types `value(expected)` accepts as `expected`, and the category each one selects +_EXPECTED_VALUE_TYPES: tuple[tuple[type, ValueType], ...] = ( + (str, ValueType.STRING), + (float, ValueType.NUMBER), + (bool, ValueType.BOOLEAN), + (bytes, ValueType.BINARY), + (list, ValueType.JSON_ARRAY), + (dict, ValueType.JSON_OBJECT), +) + +_PRIMITIVE_VALUE_TYPES = frozenset(value_type for _, value_type in _EXPECTED_VALUE_TYPES) + + +def value_type_of(value: Any) -> ValueType: + """RTTS2: the category of a resolved value, `UNKNOWN` for one in no known category (RTTS2a9). + + A boolean is never a number. + """ + if isinstance(value, InternalLiveMap): + return ValueType.LIVE_MAP + if isinstance(value, InternalLiveCounter): + return ValueType.LIVE_COUNTER + if isinstance(value, bool): + return ValueType.BOOLEAN + if isinstance(value, (int, float)): + return ValueType.NUMBER + if isinstance(value, str): + return ValueType.STRING + if isinstance(value, (bytes, bytearray)): + return ValueType.BINARY + if isinstance(value, dict): + return ValueType.JSON_OBJECT + if isinstance(value, list): + return ValueType.JSON_ARRAY + return ValueType.UNKNOWN + + +def expected_value_type(expected: type | None) -> ValueType | None: + """The category `value(expected)` filters on, or None for no filter. + + Raises TypeError for an `expected` other than `str`, `float`, `bool`, `bytes`, `list` + or `dict`. + """ + if expected is None: + return None + for python_type, value_type in _EXPECTED_VALUE_TYPES: + if expected is python_type: + return value_type + raise TypeError(f'expected must be one of str, float, bool, bytes, list or dict, not {expected!r}') + + +def primitive_value(value: Any, value_type: ValueType | None = None) -> Any: + """RTPO7d, RTTS6c: `value` if it is a primitive, of category `value_type` when one is given, + else None. + + A number is a float, and a JSON object or array is a copy, so that changing it leaves + the object it was read from unchanged. + """ + if value is None: + return None + actual_type = value_type_of(value) + if actual_type not in _PRIMITIVE_VALUE_TYPES or (value_type is not None and actual_type is not value_type): + return None + if actual_type is ValueType.NUMBER: + return float(value) + if actual_type is ValueType.BINARY: + return bytes(value) + if actual_type in (ValueType.JSON_OBJECT, ValueType.JSON_ARRAY): + return copy.deepcopy(value) + return value + + +def compact_value(value: Any, *, for_json: bool = False) -> Any: + """RTPO13, RTPO14: a plain snapshot of a resolved value. + + A map becomes a dict of its entries that are not tombstoned (RTPO13c1), nested maps + becoming nested dicts (RTPO13c2) and counters their values (RTPO13c3). Maps are visited + depth first in entry order, and a map reached again in the same snapshot, through a + cycle or a second reference, is the dict already built for it (RTPO13c5). With + `for_json` it is `{'objectId': id}` instead (RTPO14b2), and binary is a base64 string + (RTPO14b1), so that the result is JSON-serializable. + """ + if not isinstance(value, InternalLiveMap): + return _compact_leaf(value, for_json) + + result: dict[str, Any] = {} + built: dict[str, dict[str, Any]] = {value.object_id: result} + # A stack in place of recursion, so that the depth of the tree is not limited by the + # interpreter's. Each item is a dict being built and the entries of its map still to add. + stack = [(result, iter(value.entries()))] + while stack: + target, entries = stack[-1] + for key, entry_value in entries: + if not isinstance(entry_value, InternalLiveMap): + target[key] = _compact_leaf(entry_value, for_json) + continue + seen = built.get(entry_value.object_id) + if seen is not None: + target[key] = {'objectId': entry_value.object_id} if for_json else seen + continue + child: dict[str, Any] = {} + built[entry_value.object_id] = child + target[key] = child + stack.append((child, iter(entry_value.entries()))) + break + else: + stack.pop() + return result + + +def _compact_leaf(value: Any, for_json: bool) -> Any: + """RTPO13c3, RTPO13c4, RTPO13d, RTPO13e, RTPO14b1: the snapshot of a value that is not a map.""" + if isinstance(value, InternalLiveCounter): + return value.value() + if for_json and isinstance(value, (bytes, bytearray)): + return base64.b64encode(value).decode('ascii') + if isinstance(value, (dict, list)): + return copy.deepcopy(value) + return value + @dataclass class InstanceSubscriptionEvent: @@ -37,94 +167,149 @@ def __init__(self, realtime_object: RealtimeObject, value: LiveObject | Primitiv self._realtime_object = realtime_object self._value = value # RTINS2a + def __repr__(self) -> str: + if isinstance(self._value, LiveObject): + return f'{type(self).__name__}(id={self._value.object_id!r})' + return f'{type(self).__name__}({self._value!r})' + @staticmethod def _wrap(realtime_object: RealtimeObject, value: LiveObject | Primitive) -> Instance: """The `Instance` subclass matching `value`.""" - raise NotImplementedError + if isinstance(value, InternalLiveMap): + return LiveMapInstance(realtime_object, value) + if isinstance(value, InternalLiveCounter): + return LiveCounterInstance(realtime_object, value) + return PrimitiveInstance(realtime_object, value) @property def id(self) -> str | None: """RTINS3: the wrapped object's id, or None for a primitive.""" - raise NotImplementedError + if isinstance(self._value, LiveObject): + return self._value.object_id # RTINS3a + return None # RTINS3b @property def type(self) -> ValueType: """RTTS8a: the type of the wrapped value.""" - raise NotImplementedError + return value_type_of(self._value) def get(self, key: str) -> Instance | None: - """RTINS5: an instance wrapping the value at `key` of the wrapped map, or None.""" - raise NotImplementedError + """RTINS5: an instance wrapping the value at `key` of the wrapped map, or None. + + Raises AblyException 40003 if `key` is not a string. + """ + self._realtime_object._check_access_preconditions() # RTINS5b + if not isinstance(key, str): + raise AblyException(f'Map key must be a string, not {type(key).__name__}', 400, 40003) + if not isinstance(self._value, InternalLiveMap): + return None # RTINS5d + value = self._value.get(key) + if value is None: + return None + return Instance._wrap(self._realtime_object, value) # RTINS5c def compact(self) -> Any: """RTINS10: a plain snapshot of the wrapped value.""" - raise NotImplementedError + self._realtime_object._check_access_preconditions() # RTINS10a + return compact_value(self._value) # RTINS10b def compact_json(self) -> Any: """RTINS11: `compact`, with binary as base64 and cycles as `{'objectId': ...}`. Never None.""" - raise NotImplementedError + self._realtime_object._check_access_preconditions() # RTINS11a + return compact_value(self._value, for_json=True) # RTINS11b def as_live_map(self) -> LiveMapInstance: """RTTS9a: this instance as a map. Raises AblyException 92007 if it wraps something else.""" - raise NotImplementedError + if not isinstance(self._value, InternalLiveMap): + raise self._view_mismatch('a map') # RTTS9d + if isinstance(self, LiveMapInstance): + return self + return LiveMapInstance(self._realtime_object, self._value) def as_live_counter(self) -> LiveCounterInstance: """RTTS9b: this instance as a counter. Raises AblyException 92007 if it wraps something else.""" - raise NotImplementedError + if not isinstance(self._value, InternalLiveCounter): + raise self._view_mismatch('a counter') # RTTS9d + if isinstance(self, LiveCounterInstance): + return self + return LiveCounterInstance(self._realtime_object, self._value) def as_primitive(self) -> PrimitiveInstance: """RTTS9c: this instance as a primitive. Raises AblyException 92007 if it wraps something else.""" - raise NotImplementedError + if isinstance(self._value, LiveObject): + raise self._view_mismatch('a primitive') # RTTS9d + if isinstance(self, PrimitiveInstance): + return self + return PrimitiveInstance(self._realtime_object, self._value) + + def _view_mismatch(self, requested: str) -> AblyException: + return AblyException(f'Cannot view an instance of type {self.type.value} as {requested}', 400, 92007) class LiveMapInstance(Instance): """RTTS10a: an instance wrapping an `InternalLiveMap`.""" + _value: InternalLiveMap + @property def id(self) -> str: """RTINS3a: the wrapped map's id.""" - raise NotImplementedError + return self._value.object_id def batch(self) -> Batch[LiveMapBatchContext]: """RTINS17: a block whose queued writes are published as one message when it exits.""" raise NotImplementedError def entries(self) -> list[tuple[str, Instance]]: - """RTINS6: `(key, instance)` for each entry of the map.""" - raise NotImplementedError + """RTINS6: `(key, instance)` for each entry of the map. + + An entry referencing an object this client does not hold has no value to wrap, and + is left out. + """ + self._realtime_object._check_access_preconditions() # RTINS6a + # RTINS6b + return [(key, Instance._wrap(self._realtime_object, value)) + for key, value in self._value.entries() if value is not None] def keys(self) -> list[str]: """RTINS7: the keys of the map.""" - raise NotImplementedError + self._realtime_object._check_access_preconditions() # RTINS7a + return self._value.keys() # RTINS7b def values(self) -> list[Instance]: - """RTINS8: an instance for each entry of the map.""" - raise NotImplementedError + """RTINS8: an instance for each entry of the map, as `entries` wraps them.""" + return [instance for _, instance in self.entries()] # RTINS8a, RTINS8b def size(self) -> int: """RTINS9: the number of entries in the map.""" - raise NotImplementedError + self._realtime_object._check_access_preconditions() # RTINS9a + return self._value.size() # RTINS9b async def set(self, key: str, value: LiveMapValue) -> None: """RTINS12: sets `key` in the wrapped map.""" - raise NotImplementedError + self._realtime_object._check_write_preconditions() # RTINS12b + await self._value.set(key, value) # RTINS12c async def remove(self, key: str) -> None: """RTINS13: removes `key` from the wrapped map.""" - raise NotImplementedError + self._realtime_object._check_write_preconditions() # RTINS13b + await self._value.remove(key) # RTINS13c def subscribe(self, listener: Callable[[InstanceSubscriptionEvent], None]) -> Subscription: """RTINS16: calls `listener` for each update to the wrapped map, wherever it sits (RTINS16g).""" - raise NotImplementedError + self._realtime_object._check_access_preconditions() # RTINS16b + return _subscribe(self, listener) class LiveCounterInstance(Instance): """RTTS10b: an instance wrapping an `InternalLiveCounter`.""" + _value: InternalLiveCounter + @property def id(self) -> str: """RTINS3a: the wrapped counter's id.""" - raise NotImplementedError + return self._value.object_id def batch(self) -> Batch[LiveCounterBatchContext]: """RTINS17: a block whose queued writes are published as one message when it exits.""" @@ -132,19 +317,23 @@ def batch(self) -> Batch[LiveCounterBatchContext]: def value(self) -> float: """RTINS4b: the wrapped counter's value.""" - raise NotImplementedError + self._realtime_object._check_access_preconditions() # RTINS4a + return self._value.value() async def increment(self, amount: float = 1) -> None: """RTINS14: increments the wrapped counter by `amount`.""" - raise NotImplementedError + self._realtime_object._check_write_preconditions() # RTINS14b + await self._value.increment(amount) # RTINS14c async def decrement(self, amount: float = 1) -> None: """RTINS15: decrements the wrapped counter by `amount`.""" - raise NotImplementedError + self._realtime_object._check_write_preconditions() # RTINS15b + await self._value.decrement(amount) # RTINS15c def subscribe(self, listener: Callable[[InstanceSubscriptionEvent], None]) -> Subscription: """RTINS16: calls `listener` for each update to the wrapped counter, wherever it sits (RTINS16g).""" - raise NotImplementedError + self._realtime_object._check_access_preconditions() # RTINS16b + return _subscribe(self, listener) class PrimitiveInstance(Instance): @@ -159,4 +348,25 @@ def value(self, expected: type[T]) -> T | None: ... def value(self, expected: type | None = None) -> Any: """RTINS4c: the wrapped primitive; with `expected`, only if it is of that type, as for `PrimitivePathObject.value`.""" - raise NotImplementedError + value_type = expected_value_type(expected) + self._realtime_object._check_access_preconditions() # RTINS4a + return primitive_value(self._value, value_type) + + +def _subscribe(instance: LiveMapInstance | LiveCounterInstance, + listener: Callable[[InstanceSubscriptionEvent], None]) -> Subscription: + """RTINS16d: subscribes `listener` to the updates of the live object `instance` wraps. + + The live object calls its listeners synchronously, logging any that raises, and drops + them once it is tombstoned (RTLO4b4c3c). + """ + realtime_object = instance._realtime_object + + def on_update(update: LiveObjectUpdate) -> None: + object_message = update.object_message + message = None + if object_message is not None and object_message.operation is not None: + message = ObjectMessage._from_internal(object_message, realtime_object._channel_name) # RTINS16e2 + listener(InstanceSubscriptionEvent(instance, message)) # RTINS16e1 + + return instance._value.subscribe(on_update) diff --git a/ably/pubsub/objects/pathobject.py b/ably/pubsub/objects/pathobject.py index e395d69f..0086d37d 100644 --- a/ably/pubsub/objects/pathobject.py +++ b/ably/pubsub/objects/pathobject.py @@ -9,20 +9,31 @@ Reads never raise for a path that does not resolve or that resolves to another type: they return None, or an empty list for the collection methods (RTPO3c1, RTTS5d1). Writes raise `AblyException` 92005 for a path that does not resolve and 92007 for one that -resolves to the wrong type (RTPO3c2, RTTS5d2). Every read except `path`, `get` and `at` -checks the access preconditions (RTO25), and every write the write preconditions (RTO26). +resolves to the wrong type (RTPO3c2, RTTS5d2). Every read and `subscribe` checks the +access preconditions (RTO25), and every write the write preconditions (RTO26), before +resolving the path; navigation and the view helpers check nothing. """ from __future__ import annotations +from collections.abc import Sequence from dataclasses import dataclass -from typing import TYPE_CHECKING, Any, Callable, Sequence, overload +from typing import TYPE_CHECKING, Any, Callable, overload + +from ably.pubsub.objects.instance import ( + Instance, + compact_value, + expected_value_type, + primitive_value, + value_type_of, +) +from ably.pubsub.objects.livecounter import InternalLiveCounter +from ably.pubsub.objects.livemap import InternalLiveMap +from ably.pubsub.util.exceptions import AblyException if TYPE_CHECKING: from ably.pubsub.objects.batch import Batch, LiveCounterBatchContext, LiveMapBatchContext from ably.pubsub.objects.enums import ValueType - from ably.pubsub.objects.instance import Instance - from ably.pubsub.objects.livemap import InternalLiveMap from ably.pubsub.objects.publicmessage import ObjectMessage from ably.pubsub.objects.realtimeobject import RealtimeObject from ably.pubsub.objects.subscription import Subscription @@ -45,24 +56,33 @@ def __init__(self, realtime_object: RealtimeObject, root: InternalLiveMap, path: self._root = root # RTPO2b self._path: list[str] = list(path) # RTPO2a + def __repr__(self) -> str: + return f'{type(self).__name__}({self.path()!r})' + def path(self) -> str: """RTPO4: the path as a dotted string, with dots in segments escaped as `\\.`.""" - raise NotImplementedError + # RTPO4a, RTPO4b; the empty path is the empty string (RTPO4c) + return '.'.join(segment.replace('.', '\\.') for segment in self._path) def type(self) -> ValueType | None: """RTTS4b: the type of what the path resolves to, or None if it resolves to nothing.""" - raise NotImplementedError + self._realtime_object._check_access_preconditions() # RTTS4b1 + resolved = self._resolve() # RTTS4b2 + return value_type_of(resolved) if resolved is not None else None # RTTS4b3 def exists(self) -> bool: """RTTS4a: whether the path resolves to anything.""" - raise NotImplementedError + self._realtime_object._check_access_preconditions() # RTTS4a1 + return self._resolve() is not None # RTTS4a2, RTTS4a3 def get(self, key: str) -> PathObject: """RTPO5: the path one key further down. Navigational only (RTPO5d). Raises AblyException 40003 if `key` is not a string (RTPO5b). """ - raise NotImplementedError + if not isinstance(key, str): + raise AblyException(f'Path key must be a string, not {type(key).__name__}', 400, 40003) + return PathObject(self._realtime_object, self._root, [*self._path, key]) # RTPO5c def at(self, path: str | Sequence[str]) -> PathObject: """RTPO6: the path further down by a dotted string, `\\.` escaping a literal dot (RTPO6b), @@ -70,7 +90,15 @@ def at(self, path: str | Sequence[str]) -> PathObject: Raises AblyException 40003 for anything else. """ - raise NotImplementedError + if isinstance(path, str): + segments = _parse_path(path) # RTPO6b + elif (isinstance(path, Sequence) and not isinstance(path, (bytes, bytearray)) + and all(isinstance(segment, str) for segment in path)): + segments = list(path) + else: + raise AblyException(f'Path must be a string or a sequence of strings, not {type(path).__name__}', + 400, 40003) + return PathObject(self._realtime_object, self._root, [*self._path, *segments]) # RTPO6c def instance(self) -> Instance | None: """RTPO8: an `Instance` wrapping what the path resolves to, or None if it resolves to nothing. @@ -78,15 +106,27 @@ def instance(self) -> Instance | None: The instance is the subclass matching the resolved value (`LiveMapInstance`, `LiveCounterInstance` or `PrimitiveInstance`). """ - raise NotImplementedError + self._realtime_object._check_access_preconditions() # RTPO8a + resolved = self._resolve() # RTPO8b + if resolved is None: + return None # RTPO8e + return Instance._wrap(self._realtime_object, resolved) # RTPO8c, RTPO8f def compact(self) -> Any: """RTPO13: a plain snapshot of what the path resolves to, or None.""" - raise NotImplementedError + self._realtime_object._check_access_preconditions() # RTPO13a + resolved = self._resolve() # RTPO13b + if resolved is None: + return None # RTPO13f + return compact_value(resolved) # RTPO13c, RTPO13d, RTPO13e def compact_json(self) -> Any: """RTPO14: `compact`, with binary as base64 and cycles as `{'objectId': ...}`.""" - raise NotImplementedError + self._realtime_object._check_access_preconditions() # RTPO14a + resolved = self._resolve() + if resolved is None: + return None + return compact_value(resolved, for_json=True) # RTPO14b def subscribe(self, listener: Callable[[PathObjectSubscriptionEvent], None], *, depth: int | None = None) -> Subscription: @@ -95,19 +135,50 @@ def subscribe(self, listener: Callable[[PathObjectSubscriptionEvent], None], *, Raises AblyException 40003 if `depth` is given and is not a positive integer (RTPO19c1a). Has no effect on the channel (RTPO19g). """ - raise NotImplementedError + self._realtime_object._check_access_preconditions() # RTPO19b + if depth is not None and (isinstance(depth, bool) or not isinstance(depth, int) or depth < 1): + # RTPO19c1a + raise AblyException(f'Subscription depth must be a positive integer, or None for any depth; ' + f'got {depth!r}', 400, 40003) + register = self._realtime_object._path_object_subscription_register + return register.subscribe(self._path, listener, depth) # RTPO19f def as_live_map(self) -> LiveMapPathObject: """RTTS5a: this path, viewed as a map.""" - raise NotImplementedError + return LiveMapPathObject(self._realtime_object, self._root, self._path) def as_live_counter(self) -> LiveCounterPathObject: """RTTS5b: this path, viewed as a counter.""" - raise NotImplementedError + return LiveCounterPathObject(self._realtime_object, self._root, self._path) def as_primitive(self) -> PrimitivePathObject: """RTTS5c, RTTS6h: this path, viewed as a primitive.""" - raise NotImplementedError + return PrimitivePathObject(self._realtime_object, self._root, self._path) + + def _resolve(self) -> Any: + """RTPO3: what the path resolves to, or None if it does not resolve (RTPO3c).""" + current: Any = self._root # RTPO3b + for segment in self._path: # RTPO3a + if not isinstance(current, InternalLiveMap): + return None # RTPO3a1 + current = current.get(segment) # RTPO3a2, RTPO3a3 + if current is None: + return None + return current + + def _resolve_for_write(self, expected_type: type, description: str) -> Any: + """RTPO3c2, RTTS5d2: what the path resolves to, for a write that requires an `expected_type`. + + Raises AblyException 92005 if the path does not resolve and 92007 if it resolves to + something else. + """ + resolved = self._resolve() + if resolved is None: + raise AblyException(f'Could not resolve a value at path {self.path()!r}', 400, 92005) + if not isinstance(resolved, expected_type): + raise AblyException(f'Cannot write to the value at path {self.path()!r} as {description}: it is of ' + f'type {value_type_of(resolved).value}', 400, 92007) + return resolved class LiveMapPathObject(PathObject): @@ -123,27 +194,43 @@ def batch(self) -> Batch[LiveMapBatchContext]: def entries(self) -> list[tuple[str, PathObject]]: """RTPO9: `(key, path)` for each key of the map, or `[]`.""" - raise NotImplementedError + self._realtime_object._check_access_preconditions() # RTPO9a + resolved = self._resolve() # RTPO9b + if not isinstance(resolved, InternalLiveMap): + return [] # RTPO9d + return [(key, self.get(key)) for key in resolved.keys()] # RTPO9c def keys(self) -> list[str]: """RTPO10: the keys of the map, or `[]`.""" - raise NotImplementedError + self._realtime_object._check_access_preconditions() # RTPO10a + resolved = self._resolve() # RTPO10b + if not isinstance(resolved, InternalLiveMap): + return [] # RTPO10d + return resolved.keys() # RTPO10c def values(self) -> list[PathObject]: """RTPO11: a path for each key of the map, or `[]`.""" - raise NotImplementedError + return [path_object for _, path_object in self.entries()] # RTPO11a-RTPO11d def size(self) -> int | None: """RTPO12: the number of entries in the map, or None.""" - raise NotImplementedError + self._realtime_object._check_access_preconditions() # RTPO12a + resolved = self._resolve() # RTPO12b + if not isinstance(resolved, InternalLiveMap): + return None # RTPO12d + return resolved.size() # RTPO12c async def set(self, key: str, value: LiveMapValue) -> None: """RTPO15: sets `key` in the map at this path.""" - raise NotImplementedError + self._realtime_object._check_write_preconditions() # RTPO15b + live_map = self._resolve_for_write(InternalLiveMap, 'a map') # RTPO15c, RTPO15e + await live_map.set(key, value) # RTPO15d async def remove(self, key: str) -> None: """RTPO16: removes `key` from the map at this path.""" - raise NotImplementedError + self._realtime_object._check_write_preconditions() # RTPO16b + live_map = self._resolve_for_write(InternalLiveMap, 'a map') # RTPO16c, RTPO16e + await live_map.remove(key) # RTPO16d class LiveCounterPathObject(PathObject): @@ -155,15 +242,23 @@ def batch(self) -> Batch[LiveCounterBatchContext]: def value(self) -> float | None: """RTTS6b: the counter's value, or None if the path does not resolve to a counter.""" - raise NotImplementedError + self._realtime_object._check_access_preconditions() # RTPO7a + resolved = self._resolve() # RTPO7b + if not isinstance(resolved, InternalLiveCounter): + return None # RTPO7f, RTTS6b + return resolved.value() # RTPO7c async def increment(self, amount: float = 1) -> None: """RTPO17: increments the counter at this path by `amount`.""" - raise NotImplementedError + self._realtime_object._check_write_preconditions() # RTPO17b + counter = self._resolve_for_write(InternalLiveCounter, 'a counter') # RTPO17c, RTPO17e + await counter.increment(amount) # RTPO17d async def decrement(self, amount: float = 1) -> None: """RTPO18: decrements the counter at this path by `amount`.""" - raise NotImplementedError + self._realtime_object._check_write_preconditions() # RTPO18b + counter = self._resolve_for_write(InternalLiveCounter, 'a counter') # RTPO18c, RTPO18e + await counter.decrement(amount) # RTPO18d class PrimitivePathObject(PathObject): @@ -182,4 +277,33 @@ def value(self, expected: type | None = None) -> Any: returned only if it is of that type, judged by its wire type, so a boolean is never a number. Any other `expected` raises TypeError. """ - raise NotImplementedError + value_type = expected_value_type(expected) + self._realtime_object._check_access_preconditions() # RTPO7a + return primitive_value(self._resolve(), value_type) # RTPO7b, RTPO7d-RTPO7f, RTTS6c + + +def _parse_path(path: str) -> list[str]: + """RTPO6b: the segments of a dotted path, in which `\\.` is a literal dot. + + A backslash before anything other than a dot is kept as it is. + """ + segments: list[str] = [] + segment: list[str] = [] + escaping = False + for char in path: + if escaping: + if char != '.': + segment.append('\\') + segment.append(char) + escaping = False + elif char == '\\': + escaping = True + elif char == '.': + segments.append(''.join(segment)) + segment = [] + else: + segment.append(char) + if escaping: + segment.append('\\') + segments.append(''.join(segment)) + return segments diff --git a/ably/pubsub/objects/pathobjectsubscriptionregister.py b/ably/pubsub/objects/pathobjectsubscriptionregister.py index 460f130a..d33fbf3e 100644 --- a/ably/pubsub/objects/pathobjectsubscriptionregister.py +++ b/ably/pubsub/objects/pathobjectsubscriptionregister.py @@ -2,13 +2,30 @@ from __future__ import annotations +import logging from typing import TYPE_CHECKING, Callable +from ably.pubsub.objects.liveobject import LiveMapUpdate +from ably.pubsub.objects.pathobject import PathObject, PathObjectSubscriptionEvent +from ably.pubsub.objects.publicmessage import ObjectMessage +from ably.pubsub.objects.subscription import Subscription + if TYPE_CHECKING: from ably.pubsub.objects.liveobject import LiveObject, LiveObjectUpdate - from ably.pubsub.objects.pathobject import PathObjectSubscriptionEvent from ably.pubsub.objects.realtimeobject import RealtimeObject - from ably.pubsub.objects.subscription import Subscription + +log = logging.getLogger(__name__) + + +class _PathSubscription: + """One `PathObject.subscribe` registration.""" + + def __init__(self, path: list[str], listener: Callable[[PathObjectSubscriptionEvent], None], + depth: int | None): + self.path = path + self.listener = listener + self.depth = depth + self.active = True class PathObjectSubscriptionRegister: @@ -16,20 +33,63 @@ class PathObjectSubscriptionRegister: def __init__(self, realtime_object: RealtimeObject): self.realtime_object = realtime_object + self._subscriptions: list[_PathSubscription] = [] def subscribe(self, path: list[str], listener: Callable[[PathObjectSubscriptionEvent], None], depth: int | None = None) -> Subscription: """RTPO19f: registers `listener` for changes covered by `path` and `depth` (RTO24c1).""" - raise NotImplementedError + subscription = _PathSubscription(list(path), listener, depth) + self._subscriptions.append(subscription) + + def deregister() -> None: + subscription.active = False + self._subscriptions = [entry for entry in self._subscriptions if entry is not subscription] + + return Subscription(deregister) def dispatch(self, live_object: LiveObject, update: LiveObjectUpdate) -> None: """RTO24b: calls each subscription covering a path to `live_object` once. - A listener that raises is logged and does not stop the others (RTO24b2c). + A listener that raises is logged and does not stop the others (RTO24b2c). A + subscription made during a dispatch is not called by it, and one removed during a + dispatch is not called again. """ - raise NotImplementedError + if not self._subscriptions: + return + + realtime_object = self.realtime_object + root = realtime_object._objects_pool.root + object_message = update.object_message + # RTO24b2b2: an update from a sync carries the object's state, not an operation + has_operation = object_message is not None and object_message.operation is not None + updated_keys = list(update.update) if isinstance(update, LiveMapUpdate) else [] + + for path_to_this in live_object.get_full_paths(): # RTO24b1, RTO24b2 + # RTO24b2a1, RTO24b2a2 + candidate_paths = [path_to_this, *([*path_to_this, key] for key in updated_keys)] + for subscription in list(self._subscriptions): + if not subscription.active: + continue + # RTO24b2b + event_path = next((candidate for candidate in candidate_paths + if self.covers(subscription.path, subscription.depth, candidate)), None) + if event_path is None: + continue + try: + message = None + if has_operation: + message = ObjectMessage._from_internal(object_message, realtime_object._channel_name) + event = PathObjectSubscriptionEvent(PathObject(realtime_object, root, event_path), message) + subscription.listener(event) + except Exception: + # RTO24b2c + log.exception(f'PathObjectSubscriptionRegister.dispatch(): a path subscription listener ' + f'raised; path={event_path}, channel={realtime_object._channel_name}') @staticmethod def covers(subscription_path: list[str], depth: int | None, event_path: list[str]) -> bool: """RTO24c1: whether a subscription at `subscription_path` with `depth` covers `event_path`.""" - raise NotImplementedError + prefix_length = len(subscription_path) + if len(event_path) < prefix_length or list(event_path[:prefix_length]) != list(subscription_path): + return False + return depth is None or len(event_path) - prefix_length + 1 <= depth diff --git a/ably/pubsub/server/__init__.py b/ably/pubsub/server/__init__.py index b31150ed..20747fd9 100644 --- a/ably/pubsub/server/__init__.py +++ b/ably/pubsub/server/__init__.py @@ -22,6 +22,33 @@ from ably.pubsub.http.channel import Channels as HttpChannels from ably.pubsub.http.http import DefaultPubSubHttpClient as _DefaultPubSubHttpClient from ably.pubsub.http.push import Push +from ably.pubsub.objects.batch import ( + Batch, + BatchContext, + LiveCounterBatchContext, + LiveMapBatchContext, + PrimitiveBatchContext, +) +from ably.pubsub.objects.enums import ObjectsEvent, ValueType +from ably.pubsub.objects.instance import ( + Instance, + InstanceSubscriptionEvent, + LiveCounterInstance, + LiveMapInstance, + PrimitiveInstance, +) +from ably.pubsub.objects.objectmessage import ObjectOperationAction, ObjectsMapSemantics +from ably.pubsub.objects.pathobject import ( + LiveCounterPathObject, + LiveMapPathObject, + PathObject, + PathObjectSubscriptionEvent, + PrimitivePathObject, +) +from ably.pubsub.objects.publicmessage import ObjectMessage, ObjectOperation +from ably.pubsub.objects.realtimeobject import RealtimeObject +from ably.pubsub.objects.subscription import StatusSubscription, Subscription +from ably.pubsub.objects.valuetypes import LiveCounter, LiveMap, LiveMapValue, Primitive from ably.pubsub.prototypes import PubSubHttpClient, PubSubRealtimeClient from ably.pubsub.realtime.channel import Channels as RealtimeChannels from ably.pubsub.realtime.connection import Connection @@ -259,6 +286,8 @@ def create_realtime_client(**kwargs) -> PubSubRealtimeClient: 'Annotation', 'AnnotationAction', 'Auth', + 'Batch', + 'BatchContext', 'BatchPresenceFailureResult', 'BatchPresenceSuccessResult', 'BatchPublishFailureResult', @@ -274,21 +303,47 @@ def create_realtime_client(**kwargs) -> PubSubRealtimeClient: 'HttpChannels', 'HttpPaginatedResponse', 'IncompatibleClientIdException', + 'Instance', + 'InstanceSubscriptionEvent', + 'LiveCounter', + 'LiveCounterBatchContext', + 'LiveCounterInstance', + 'LiveCounterPathObject', + 'LiveMap', + 'LiveMapBatchContext', + 'LiveMapInstance', + 'LiveMapPathObject', + 'LiveMapValue', 'Message', 'MessageAction', 'MessageOperation', 'MessageVersion', + 'ObjectMessage', + 'ObjectOperation', + 'ObjectOperationAction', + 'ObjectsEvent', + 'ObjectsMapSemantics', 'Options', 'PaginatedResult', + 'PathObject', + 'PathObjectSubscriptionEvent', + 'Primitive', + 'PrimitiveBatchContext', + 'PrimitiveInstance', + 'PrimitivePathObject', 'PubSubHttpClient', 'PubSubRealtimeClient', 'PublishResult', 'Push', 'PushChannelSubscription', 'RealtimeChannels', + 'RealtimeObject', + 'StatusSubscription', + 'Subscription', 'TokenDetails', 'UpdateDeleteResult', 'VCDiffDecoder', + 'ValueType', 'api_version', 'create_http_client', 'create_realtime_client', From a73ab4bd8056f7a3110bb0e42aefced58d7fb763 Mon Sep 17 00:00:00 2001 From: owenpearson Date: Fri, 9 Oct 2026 14:15:37 +0100 Subject: [PATCH 18/71] objects: batch several writes into one message Adds the batch API LODR-061 specifies, from the RTBC section of ably/specification#471: `async with root.batch() as ctx:` resolves the target once, and the writes made through the context inside the block are queued and published as a single OBJECT message when the block exits, so other clients see them as one update. Reads inside the block resolve against the local tree; a block that raises publishes nothing, and a context used after its block has closed raises 40000. batch() is on the map and counter views of both PathObject and Instance, which is what types the context. LiveMap and LiveCounter values are evaluated when the batch is published, since creating them reads the server time. No UTS specification covers the batch API or the typed views of LODR-061 yet, so test/unit/objects tests both against their spec points. Co-Authored-By: Claude Opus 5.5 (1M context) --- ably/pubsub/objects/batch.py | 342 +++++++++++-- ably/pubsub/objects/instance.py | 20 +- ably/pubsub/objects/pathobject.py | 16 +- test/unit/objects/__init__.py | 0 test/unit/objects/batch_test.py | 700 ++++++++++++++++++++++++++ test/unit/objects/conftest.py | 10 + test/unit/objects/typed_views_test.py | 386 ++++++++++++++ 7 files changed, 1414 insertions(+), 60 deletions(-) create mode 100644 test/unit/objects/__init__.py create mode 100644 test/unit/objects/batch_test.py create mode 100644 test/unit/objects/conftest.py create mode 100644 test/unit/objects/typed_views_test.py diff --git a/ably/pubsub/objects/batch.py b/ably/pubsub/objects/batch.py index 1a9f6f49..ddb1d1d6 100644 --- a/ably/pubsub/objects/batch.py +++ b/ably/pubsub/objects/batch.py @@ -2,44 +2,108 @@ `batch()` on a typed path or instance returns a `Batch`, an async context manager whose `__aenter__` checks the write preconditions and resolves the target (RTPO20b-d, -RTINS17b-d) and returns the matching `BatchContext`. The context's write methods are -synchronous and only queue; `__aexit__` publishes everything queued as one message -(RTBC16d) when the block exits without raising, and closes the batch either way -(RTPO20g). Any use of a context after its batch has closed raises `AblyException` 40000 -(RTBC16e). +RTINS17b-d) and returns the matching `BatchContext`. Inside the block every context +method is synchronous: reads resolve against the objects as they are held locally, so +they do not see the writes queued before them, and writes only queue. `__aexit__` +publishes everything queued as one message (RTBC16d) when the block exits without +raising, and closes the batch either way (RTPO20g). Any use of a context after its batch +has closed raises `AblyException` 40000 (RTBC16e). + +A write validates its arguments when it is called, so an invalid one raises inside the +block and nothing is published. A `LiveMap` or `LiveCounter` value is evaluated when the +batch is published, because its object id needs the server time (RTO16): an invalid +value inside one raises from the end of the block, and nothing is published then either. """ from __future__ import annotations from typing import TYPE_CHECKING, Any, Awaitable, Callable, Generic, TypeVar, overload +from ably.pubsub.objects.enums import ValueType +from ably.pubsub.objects.instance import ( + Instance, + LiveCounterInstance, + LiveMapInstance, + PrimitiveInstance, + expected_value_type, + primitive_value, + value_type_of, +) +from ably.pubsub.objects.livecounter import _validate_amount +from ably.pubsub.objects.objectmessage import ( + CounterInc, + MapRemove, + MapSet, + ObjectData, + ObjectMessage, + ObjectOperation, + ObjectOperationAction, +) +from ably.pubsub.objects.valuetypes import LiveCounter, LiveMap, evaluate, primitive_to_object_data, validate_key +from ably.pubsub.util.exceptions import AblyException + if TYPE_CHECKING: - from ably.pubsub.objects.instance import Instance - from ably.pubsub.objects.objectmessage import ObjectMessage from ably.pubsub.objects.realtimeobject import RealtimeObject from ably.pubsub.objects.valuetypes import LiveMapValue, Primitive, T + # RTBC16b: builds the ObjectMessages of one queued write when the batch is flushed + MessageConstructor = Callable[[], Awaitable[list[ObjectMessage]]] + C = TypeVar('C', bound='BatchContext') +V = TypeVar('V', bound='BatchContext') class Batch(Generic[C]): - """RTBC16: the `async with` block a `batch()` call returns.""" + """RTBC16: the `async with` block a `batch()` call returns. + + `resolve` returns the value the batch acts on, or None, and is called when the block + is entered; the value must be the live object a `context_type` wraps. `description` + names the target in the error raised when it is not. A `Batch` opens one batch, so it + can be entered once. + """ - def __init__(self, realtime_object: RealtimeObject, resolve: Callable[[], Instance]): + def __init__(self, realtime_object: RealtimeObject, resolve: Callable[[], Any], context_type: type[C], + description: str): self._realtime_object = realtime_object - # Resolves the batch's target when the block is entered (RTPO20c, RTINS17c) self._resolve = resolve + self._context_type = context_type + self._description = description self._root_context: RootBatchContext | None = None async def __aenter__(self) -> C: """RTPO20b-d, RTINS17b-d: checks the write preconditions, resolves the target and opens - the batch.""" - raise NotImplementedError + the batch. + + Raises AblyException 92007 if the target is not a live object of the type the batch + was taken on (RTPO20c, RTINS17c), and RuntimeError if this batch was entered before. + """ + if self._root_context is not None: + raise RuntimeError('This batch has already been entered; call batch() again for another') + self._realtime_object._check_write_preconditions() # RTPO20b, RTINS17b + + value = self._resolve() + expected = self._context_type._value_type + actual = value_type_of(value) if value is not None else None + if actual is not expected: + # RTPO20c, RTINS17c: nothing, a primitive or a live object of another type + reason = 'it does not resolve' if actual is None else f'it is of type {actual.value}' + raise AblyException(f'Cannot batch operations on {self._description} as a {expected.value}: ' + f'{reason}', 400, 92007) + + instance = Instance._wrap(self._realtime_object, value) + self._root_context = RootBatchContext(self._realtime_object, instance) # RTPO20d, RTINS17d + return self._root_context.wrap_instance(instance) async def __aexit__(self, exc_type, exc, tb) -> None: """RTPO20f, RTPO20g, RTBC16d: publishes the queued writes unless the block raised, and - closes the batch either way.""" - raise NotImplementedError + closes the batch either way. + + An exception raised in the block propagates, and nothing is published. + """ + if exc_type is not None: + self._root_context.close() # RTPO20g + return + await self._root_context.flush() # RTPO20f, RTINS17f class RootBatchContext: @@ -49,25 +113,69 @@ def __init__(self, realtime_object: RealtimeObject, instance: Instance): self.realtime_object = realtime_object self.instance = instance self.wrapped_instances: dict[str, BatchContext] = {} # RTBC16a - self.queued_message_constructors: list[Callable[[], Awaitable[list[ObjectMessage]]]] = [] # RTBC16b + self.queued_message_constructors: list[MessageConstructor] = [] # RTBC16b self.closed = False def wrap_instance(self, instance: Instance) -> BatchContext: - """RTBC16c: the context for `instance`, one per object id.""" - raise NotImplementedError + """RTBC16c: the context for `instance`, one per object id. + + The context is the subclass matching the `Instance` subclass, so a `LiveMapInstance` + is wrapped in a `LiveMapBatchContext`. + """ + object_id = instance.id + if object_id is not None: + context = self.wrapped_instances.get(object_id) + if context is not None: + return context + + context = _context_type_for(instance)(self, instance) + if object_id is not None: + self.wrapped_instances[object_id] = context + return context + + def queue_message_constructor(self, constructor: MessageConstructor) -> None: + """RTBC16b: queues `constructor`, which builds the ObjectMessages of one write when the + batch is flushed.""" + self.queued_message_constructors.append(constructor) async def flush(self) -> None: - """RTBC16d: closes the batch, builds the queued messages and publishes them as one message, - if there are any.""" - raise NotImplementedError + """RTBC16d: closes the batch, builds the queued messages in the order they were queued and + publishes them in one message through `RealtimeObject._publish_and_apply`, if there are + any. + + Raises whatever building or publishing the messages raises. Nothing is published if + building any of them fails. + """ + try: + self.close() + if self.queued_message_constructors: + # The block may have awaited since its writes checked the write preconditions (RTO26) + self.realtime_object._check_write_preconditions() + object_messages: list[ObjectMessage] = [] + for construct in self.queued_message_constructors: + object_messages.extend(await construct()) + if object_messages: + await self.realtime_object._publish_and_apply(object_messages) + finally: + self.wrapped_instances.clear() + self.queued_message_constructors.clear() def close(self) -> None: """RTBC16e: closes the batch, so that every later use of its contexts raises 40000.""" - raise NotImplementedError + self.closed = True class BatchContext: - """RTBC1: a synchronous view of an `Instance` inside a batch (RTBC2a).""" + """RTBC1: a synchronous view of an `Instance` inside a batch (RTBC2a). + + As on an `Instance`, the view helpers are checked: asking for a type the context does + not wrap raises `AblyException` 92007 (RTBC1a, RTTS9d). Reads check the access + preconditions (RTO25), and writes the write preconditions (RTO26), before checking + that the batch is still open. + """ + + # The type of live object that a batch opening this subclass of context acts on + _value_type: ValueType | None = None def __init__(self, root_context: RootBatchContext, instance: Instance): self._root_context = root_context # RTBC2b @@ -75,76 +183,186 @@ def __init__(self, root_context: RootBatchContext, instance: Instance): @property def id(self) -> str | None: - """RTBC3: the wrapped object's id.""" - raise NotImplementedError + """RTBC3: the wrapped object's id, or None for a primitive.""" + self._throw_if_closed() # RTBC3b + return self._instance.id # RTBC3a def get(self, key: str) -> BatchContext | None: - """RTBC4: the context for the value at `key` of the wrapped map, or None.""" - raise NotImplementedError + """RTBC4: the context for the value at `key` of the wrapped map, or None. + + Raises AblyException 40003 if `key` is not a string. + """ + self._realtime_object._check_access_preconditions() # RTBC4b + self._throw_if_closed() # RTBC4c + instance = self._instance.get(key) # RTBC4d + if instance is None: + return None + return self._root_context.wrap_instance(instance) # RTBC4e def compact(self) -> Any: """RTBC10: `Instance.compact` on the wrapped instance.""" - raise NotImplementedError + self._realtime_object._check_access_preconditions() # RTBC10a + self._throw_if_closed() # RTBC10b + return self._instance.compact() # RTBC10c def compact_json(self) -> Any: """RTBC11: `Instance.compact_json` on the wrapped instance.""" - raise NotImplementedError + self._realtime_object._check_access_preconditions() # RTBC11a + self._throw_if_closed() # RTBC11b + return self._instance.compact_json() # RTBC11c def as_live_map(self) -> LiveMapBatchContext: """RTBC1a: this context as a map. Raises AblyException 92007 if it wraps something else.""" - raise NotImplementedError + self._throw_if_closed() # RTBC16e + return self._view(LiveMapBatchContext, self._instance.as_live_map()) # RTTS9d def as_live_counter(self) -> LiveCounterBatchContext: """RTBC1a: this context as a counter. Raises AblyException 92007 if it wraps something else.""" - raise NotImplementedError + self._throw_if_closed() # RTBC16e + return self._view(LiveCounterBatchContext, self._instance.as_live_counter()) # RTTS9d def as_primitive(self) -> PrimitiveBatchContext: """RTBC1a: this context as a primitive. Raises AblyException 92007 if it wraps something else.""" - raise NotImplementedError + self._throw_if_closed() # RTBC16e + return self._view(PrimitiveBatchContext, self._instance.as_primitive()) # RTTS9d + + @property + def _realtime_object(self) -> RealtimeObject: + return self._root_context.realtime_object + + def _view(self, context_type: type[V], instance: Instance) -> V: + """This context if it is a `context_type`, else a `context_type` wrapping `instance`.""" + if isinstance(self, context_type): + return self + return context_type(self._root_context, instance) + + def _throw_if_closed(self) -> None: + """RTBC16e: raises AblyException 40000 once the batch has closed.""" + if self._root_context.closed: + raise AblyException('Batch is closed: a batch context can only be used inside its batch block', + 400, 40000) + + def _queue(self, *object_messages: ObjectMessage) -> None: + """Queues a write whose ObjectMessages are already built.""" + async def construct() -> list[ObjectMessage]: + return list(object_messages) + + self._root_context.queue_message_constructor(construct) class LiveMapBatchContext(BatchContext): - """RTBC1a: a batch context wrapping a map.""" + """RTBC1a: a batch context wrapping a map. + + A context of this type only ever wraps a map, so its writes never find another type + (RTBC12d, RTBC13d). + """ + + _instance: LiveMapInstance + _value_type = ValueType.LIVE_MAP def entries(self) -> list[tuple[str, BatchContext]]: - """RTBC6: `(key, context)` for each entry of the map.""" - raise NotImplementedError + """RTBC6: `(key, context)` for each entry of the map, as `LiveMapInstance.entries` gives them.""" + self._realtime_object._check_access_preconditions() # RTBC6a + self._throw_if_closed() # RTBC6b + wrap_instance = self._root_context.wrap_instance + return [(key, wrap_instance(instance)) for key, instance in self._instance.entries()] # RTBC6c, RTBC6d def keys(self) -> list[str]: """RTBC7: the keys of the map.""" - raise NotImplementedError + self._realtime_object._check_access_preconditions() # RTBC7a + self._throw_if_closed() # RTBC7b + return self._instance.keys() # RTBC7c def values(self) -> list[BatchContext]: - """RTBC8: a context for each entry of the map.""" - raise NotImplementedError + """RTBC8: a context for each entry of the map, as `entries` wraps them.""" + return [context for _, context in self.entries()] # RTBC8a-RTBC8c def size(self) -> int | None: """RTBC9: the number of entries in the map.""" - raise NotImplementedError + self._realtime_object._check_access_preconditions() # RTBC9a + self._throw_if_closed() # RTBC9b + return self._instance.size() # RTBC9c def set(self, key: str, value: LiveMapValue) -> None: - """RTBC12: queues a MAP_SET of `key`.""" - raise NotImplementedError + """RTBC12: queues a MAP_SET of `key`, preceded by the creates a `LiveMap` or `LiveCounter` + value evaluates to. + + Raises AblyException 40003 for a key that is not a string and 40013 for a value of + an unsupported type (RTLM20e1). A `LiveMap` or `LiveCounter` value is evaluated when + the batch is flushed, so it is the end of the block that raises for an invalid value + inside one (RTLMV4). + """ + realtime_object = self._realtime_object + realtime_object._check_write_preconditions() # RTBC12b + self._throw_if_closed() # RTBC12c + validate_key(key) # RTLM20e1 + object_id = self._instance.id + + if not isinstance(value, (LiveCounter, LiveMap)): + self._queue(_map_set(object_id, key, primitive_to_object_data(value))) # RTBC12e, RTLM20e7b-f + return + + async def construct() -> list[ObjectMessage]: + # RTLM20e7g1: the creates the value evaluates to, their object ids from the server time + server_time_ms = await realtime_object._get_server_time_ms() + object_messages = evaluate(value, server_time_ms) + data = ObjectData(object_id=object_messages[-1].operation.object_id) # RTLM20e7g2 + return [*object_messages, _map_set(object_id, key, data)] + + self._root_context.queue_message_constructor(construct) # RTBC12e def remove(self, key: str) -> None: - """RTBC13: queues a MAP_REMOVE of `key`.""" - raise NotImplementedError + """RTBC13: queues a MAP_REMOVE of `key`. + + Raises AblyException 40003 for a key that is not a string (RTLM21e1). + """ + self._realtime_object._check_write_preconditions() # RTBC13b + self._throw_if_closed() # RTBC13c + validate_key(key) # RTLM21e1 + # RTBC13e + self._queue(ObjectMessage(operation=ObjectOperation( + action=ObjectOperationAction.MAP_REMOVE, # RTLM21e2 + object_id=self._instance.id, # RTLM21e3 + map_remove=MapRemove(key=key), # RTLM21e5 + ))) class LiveCounterBatchContext(BatchContext): - """RTBC1a: a batch context wrapping a counter.""" + """RTBC1a: a batch context wrapping a counter. + + A context of this type only ever wraps a counter, so its writes never find another + type (RTBC14d, RTBC15d). + """ + + _instance: LiveCounterInstance + _value_type = ValueType.LIVE_COUNTER def value(self) -> float: """RTBC5: the counter's value.""" - raise NotImplementedError + self._realtime_object._check_access_preconditions() # RTBC5a + self._throw_if_closed() # RTBC5b + return self._instance.value() # RTBC5c def increment(self, amount: float = 1) -> None: - """RTBC14: queues a COUNTER_INC of `amount`.""" - raise NotImplementedError + """RTBC14: queues a COUNTER_INC of `amount`. + + Raises AblyException 40003 unless `amount` is a finite number (RTLC12e1); a bool is + not a number. + """ + self._realtime_object._check_write_preconditions() # RTBC14b + self._throw_if_closed() # RTBC14c + # RTBC14e + self._queue(ObjectMessage(operation=ObjectOperation( + action=ObjectOperationAction.COUNTER_INC, # RTLC12e2 + object_id=self._instance.id, # RTLC12e3 + counter_inc=CounterInc(number=_validate_amount(amount)), # RTLC12e1, RTLC12e5 + ))) def decrement(self, amount: float = 1) -> None: - """RTBC15: queues a COUNTER_INC of `-amount`.""" - raise NotImplementedError + """RTBC15: queues a COUNTER_INC of `-amount`, after the same validation as `increment`.""" + self._realtime_object._check_write_preconditions() # RTBC15b + self._throw_if_closed() # RTBC15c + self.increment(-_validate_amount(amount)) # RTBC15e class PrimitiveBatchContext(BatchContext): @@ -157,5 +375,29 @@ def value(self) -> Primitive: ... def value(self, expected: type[T]) -> T | None: ... def value(self, expected: type | None = None) -> Any: - """RTBC5: the wrapped primitive; with `expected`, only if it is of that type.""" - raise NotImplementedError + """RTBC5: the wrapped primitive; with `expected`, only if it is of that type, as for + `PrimitiveInstance.value`.""" + value_type = expected_value_type(expected) + self._realtime_object._check_access_preconditions() # RTBC5a + self._throw_if_closed() # RTBC5b + return primitive_value(self._instance._value, value_type) # RTBC5c + + +def _context_type_for(instance: Instance) -> type[BatchContext]: + """RTBC1a: the `BatchContext` subclass matching the `Instance` subclass of `instance`.""" + if isinstance(instance, LiveMapInstance): + return LiveMapBatchContext + if isinstance(instance, LiveCounterInstance): + return LiveCounterBatchContext + if isinstance(instance, PrimitiveInstance): + return PrimitiveBatchContext + return BatchContext + + +def _map_set(object_id: str, key: str, data: ObjectData) -> ObjectMessage: + """RTLM20e: the ObjectMessage setting `key` of the map `object_id` to `data`.""" + return ObjectMessage(operation=ObjectOperation( + action=ObjectOperationAction.MAP_SET, # RTLM20e2 + object_id=object_id, # RTLM20e3 + map_set=MapSet(key=key, value=data), # RTLM20e6, RTLM20e7 + )) diff --git a/ably/pubsub/objects/instance.py b/ably/pubsub/objects/instance.py index 17fa2ba5..54ab3ac8 100644 --- a/ably/pubsub/objects/instance.py +++ b/ably/pubsub/objects/instance.py @@ -257,8 +257,14 @@ def id(self) -> str: return self._value.object_id def batch(self) -> Batch[LiveMapBatchContext]: - """RTINS17: a block whose queued writes are published as one message when it exits.""" - raise NotImplementedError + """RTINS17: a block whose queued writes are published as one message when it exits. + + Entering checks the write preconditions (RTINS17b). + """ + # Imported here, as the batch module builds on this one + from ably.pubsub.objects.batch import Batch, LiveMapBatchContext + + return Batch(self._realtime_object, lambda: self._value, LiveMapBatchContext, f'object {self.id!r}') def entries(self) -> list[tuple[str, Instance]]: """RTINS6: `(key, instance)` for each entry of the map. @@ -312,8 +318,14 @@ def id(self) -> str: return self._value.object_id def batch(self) -> Batch[LiveCounterBatchContext]: - """RTINS17: a block whose queued writes are published as one message when it exits.""" - raise NotImplementedError + """RTINS17: a block whose queued writes are published as one message when it exits. + + Entering checks the write preconditions (RTINS17b). + """ + # Imported here, as the batch module builds on this one + from ably.pubsub.objects.batch import Batch, LiveCounterBatchContext + + return Batch(self._realtime_object, lambda: self._value, LiveCounterBatchContext, f'object {self.id!r}') def value(self) -> float: """RTINS4b: the wrapped counter's value.""" diff --git a/ably/pubsub/objects/pathobject.py b/ably/pubsub/objects/pathobject.py index 0086d37d..43f9d6f9 100644 --- a/ably/pubsub/objects/pathobject.py +++ b/ably/pubsub/objects/pathobject.py @@ -20,6 +20,7 @@ from dataclasses import dataclass from typing import TYPE_CHECKING, Any, Callable, overload +from ably.pubsub.objects.batch import Batch, LiveCounterBatchContext, LiveMapBatchContext from ably.pubsub.objects.instance import ( Instance, compact_value, @@ -32,7 +33,6 @@ from ably.pubsub.util.exceptions import AblyException if TYPE_CHECKING: - from ably.pubsub.objects.batch import Batch, LiveCounterBatchContext, LiveMapBatchContext from ably.pubsub.objects.enums import ValueType from ably.pubsub.objects.publicmessage import ObjectMessage from ably.pubsub.objects.realtimeobject import RealtimeObject @@ -187,10 +187,10 @@ class LiveMapPathObject(PathObject): def batch(self) -> Batch[LiveMapBatchContext]: """RTPO20: a block whose queued writes are published as one message when it exits. - Entering raises AblyException 92007 if the path does not resolve to a live object - (RTPO20c). + Entering checks the write preconditions (RTPO20b) and raises AblyException 92007 if + the path does not resolve to a map (RTPO20c). """ - raise NotImplementedError + return Batch(self._realtime_object, self._resolve, LiveMapBatchContext, f'path {self.path()!r}') def entries(self) -> list[tuple[str, PathObject]]: """RTPO9: `(key, path)` for each key of the map, or `[]`.""" @@ -237,8 +237,12 @@ class LiveCounterPathObject(PathObject): """RTTS6b: a path expected to resolve to a counter.""" def batch(self) -> Batch[LiveCounterBatchContext]: - """RTPO20: a block whose queued writes are published as one message when it exits.""" - raise NotImplementedError + """RTPO20: a block whose queued writes are published as one message when it exits. + + Entering checks the write preconditions (RTPO20b) and raises AblyException 92007 if + the path does not resolve to a counter (RTPO20c). + """ + return Batch(self._realtime_object, self._resolve, LiveCounterBatchContext, f'path {self.path()!r}') def value(self) -> float | None: """RTTS6b: the counter's value, or None if the path does not resolve to a counter.""" diff --git a/test/unit/objects/__init__.py b/test/unit/objects/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/test/unit/objects/batch_test.py b/test/unit/objects/batch_test.py new file mode 100644 index 00000000..46d10d69 --- /dev/null +++ b/test/unit/objects/batch_test.py @@ -0,0 +1,700 @@ +"""Batched writes: `batch()` on the typed path and instance views, and the batch contexts. + +Spec points: RTPO20, RTINS17 and RTBC1-RTBC16 (ably/specification#471), with RTO26 as the +precondition a batch checks. No UTS specification covers batch, so these tests are +written against the features specification and LODR-061. + +Each test drives the standard synced pool over the mock websocket, which ACKs every OBJECT +message, so a batch's writes have been applied locally (RTO20) by the time its block exits. +""" + +import json +from typing import NamedTuple + +import pytest + +from ably.pubsub.objects.objectmessage import ObjectOperationAction +from ably.pubsub.server import ( + LiveCounter, + LiveCounterBatchContext, + LiveMap, + LiveMapBatchContext, + PrimitiveBatchContext, +) +from ably.pubsub.transport.websockettransport import ProtocolMessageAction +from ably.pubsub.types.channelmode import ChannelMode +from ably.pubsub.types.channelstate import ChannelState +from ably.pubsub.util.exceptions import AblyException +from test.uts.helpers.client import poll_until +from test.uts.helpers.clock import FakeClock, settle +from test.uts.objects.helpers.standard_test_pool import ( + HAS_OBJECTS, + OBJECT, + OBJECT_SUBSCRIBE_FLAG, + build_map_set, + build_object_message, + remote_serial, + setup_synced_channel, + standard_mock_websocket, +) + +MAP_CREATE = ObjectOperationAction.MAP_CREATE +MAP_SET = ObjectOperationAction.MAP_SET +MAP_REMOVE = ObjectOperationAction.MAP_REMOVE +COUNTER_CREATE = ObjectOperationAction.COUNTER_CREATE +COUNTER_INC = ObjectOperationAction.COUNTER_INC + +# The server time a test with a fake clock creates objects at +SERVER_TIME_MS = 1_700_000_123_000 + + +async def _synced(**client_options): + """The standard synced channel, and a list of each OBJECT ProtocolMessage the client sends on it.""" + published = [] + mock_ws = standard_mock_websocket(on_object=published.append) + synced = await setup_synced_channel('test', mock_ws=mock_ws, **client_options) + return synced, published + + +def _operations(protocol_message): + """The wire operations of an OBJECT ProtocolMessage the client sent, in order.""" + return [object_message['operation'] for object_message in protocol_message['state']] + + +def _map_set(object_id, key, value): + return {'action': MAP_SET, 'objectId': object_id, 'mapSet': {'key': key, 'value': value}} + + +def _counter_inc(object_id, number): + return {'action': COUNTER_INC, 'objectId': object_id, 'counterInc': {'number': number}} + + +def _assert_error(excinfo, code): + assert excinfo.value.code == code + assert excinfo.value.status_code == 400 + + +# --- Publishing --------------------------------------------------------------------------- + +async def test_rtpo20f_batch_publishes_one_object_message(): + """RTPO20f, RTBC16d: the writes queued in a batch are published in order as one OBJECT + ProtocolMessage, then applied locally.""" + (client, channel, root, mock_ws), published = await _synced() + + async with root.batch() as ctx: + ctx.set('name', 'Bob') + ctx.remove('age') + ctx.get('score').as_live_counter().increment(5) + ctx.get('profile').as_live_map().set('email', 'bob@example.com') + + assert len(published) == 1 + assert published[0]['action'] == OBJECT + assert published[0]['channel'] == 'test' + assert _operations(published[0]) == [ + _map_set('root', 'name', {'string': 'Bob'}), + {'action': MAP_REMOVE, 'objectId': 'root', 'mapRemove': {'key': 'age'}}, + _counter_inc('counter:score@1000', 5), + _map_set('map:profile@1000', 'email', {'string': 'bob@example.com'}), + ] + + assert root.get('name').as_primitive().value() == 'Bob' + assert root.get('age').exists() is False + assert root.get('score').as_live_counter().value() == 105 + assert root.at('profile.email').as_primitive().value() == 'bob@example.com' + + +async def test_rtbc12e_nested_value_types_published_in_the_batch(): + """RTBC12e, RTLM20e7g: a `LiveMap` value is published as the creates it evaluates to, nested + ones first, ahead of the MAP_SET that references it, all in the batch's one message. Their + object ids carry the server time (RTO16), which the mock answers from the client's clock.""" + (client, channel, root, mock_ws), published = await _synced(clock=FakeClock(epoch_ms=SERVER_TIME_MS)) + + async with root.batch() as ctx: + ctx.set('team', LiveMap.create({'lead': 'Carol', 'points': LiveCounter.create(10)})) + ctx.get('score').as_live_counter().increment(1) + + assert len(published) == 1 + counter_create, map_create, map_set, counter_inc = _operations(published[0]) + + assert counter_create['action'] == COUNTER_CREATE + assert counter_create['objectId'].startswith('counter:') + assert counter_create['objectId'].endswith(f'@{SERVER_TIME_MS}') + assert json.loads(counter_create['counterCreateWithObjectId']['initialValue']) == {'count': 10} + + assert map_create['action'] == MAP_CREATE + assert map_create['objectId'].startswith('map:') + assert map_create['objectId'].endswith(f'@{SERVER_TIME_MS}') + assert json.loads(map_create['mapCreateWithObjectId']['initialValue'])['entries'] == { + 'lead': {'data': {'string': 'Carol'}}, + 'points': {'data': {'objectId': counter_create['objectId']}}, + } + + assert map_set == _map_set('root', 'team', {'objectId': map_create['objectId']}) + assert counter_inc == _counter_inc('counter:score@1000', 1) + + assert root.get('team').instance().id == map_create['objectId'] + assert root.at('team.lead').as_primitive().value() == 'Carol' + assert root.at('team.points').as_live_counter().value() == 10 + + +async def test_rtbc12e_value_type_set_twice_creates_two_objects(): + """RTBC12e, RTLMV4d1: each write of a `LiveCounter` value is evaluated on its own, so one + value set at two keys creates two counters.""" + (client, channel, root, mock_ws), published = await _synced() + counter = LiveCounter.create(3) + + async with root.batch() as ctx: + ctx.set('first', counter) + ctx.set('second', counter) + + first_create, first_set, second_create, second_set = _operations(published[0]) + assert first_create['objectId'] != second_create['objectId'] + assert first_set == _map_set('root', 'first', {'objectId': first_create['objectId']}) + assert second_set == _map_set('root', 'second', {'objectId': second_create['objectId']}) + assert root.get('first').as_live_counter().value() == 3 + assert root.get('second').as_live_counter().value() == 3 + + +async def test_rtbc16d_empty_batch_publishes_nothing(): + """RTBC16d: a batch that queues nothing publishes nothing.""" + (client, channel, root, mock_ws), published = await _synced() + + async with root.batch() as ctx: + assert ctx.get('name').as_primitive().value() == 'Alice' + + assert published == [] + + +async def test_rtbc16b_reads_inside_the_block_see_the_state_before_the_batch(): + """RTBC4-RTBC9, RTBC16b: the writes in a block only queue, so reads inside it resolve against + the objects as they were, and nothing is sent until the block exits.""" + (client, channel, root, mock_ws), published = await _synced() + + async with root.batch() as ctx: + score = ctx.get('score').as_live_counter() + ctx.set('name', 'Bob') + ctx.set('city', 'Paris') + ctx.remove('age') + score.increment(5) + + assert ctx.get('name').as_primitive().value() == 'Alice' + assert ctx.get('city') is None + assert ctx.get('age').as_primitive().value() == 30 + assert ctx.keys() == ['name', 'age', 'active', 'score', 'profile', 'data', 'avatar'] + assert ctx.size() == 7 + assert score.value() == 100 + assert root.get('name').as_primitive().value() == 'Alice' + + # Nothing is sent while the block runs, however long it yields to the event loop + await settle() + assert published == [] + + assert root.get('name').as_primitive().value() == 'Bob' + assert root.get('city').as_primitive().value() == 'Paris' + assert root.get('age').exists() is False + assert root.get('score').as_live_counter().value() == 105 + + +async def test_rtpo20g_exception_in_the_block_publishes_nothing(): + """RTPO20g: an exception raised in the block propagates, nothing queued is published, and the + batch is closed.""" + (client, channel, root, mock_ws), published = await _synced() + + with pytest.raises(ValueError, match='abandoned'): + async with root.batch() as ctx: + ctx.set('name', 'Bob') + ctx.get('score').as_live_counter().increment(5) + raise ValueError('abandoned') + + assert published == [] + assert root.get('name').as_primitive().value() == 'Alice' + assert root.get('score').as_live_counter().value() == 100 + + with pytest.raises(AblyException) as excinfo: + ctx.keys() + _assert_error(excinfo, 40000) + + +async def test_rtbc16d_invalid_value_inside_a_value_type_publishes_nothing(): + """RTBC16d, RTLMV4c: a `LiveMap` value is evaluated when the batch is flushed, so an + unsupported value inside one raises from the end of the block, and nothing in the batch + is published.""" + (client, channel, root, mock_ws), published = await _synced() + + with pytest.raises(AblyException) as excinfo: + async with root.batch() as ctx: + ctx.set('name', 'Bob') + ctx.set('team', LiveMap.create({'lead': object()})) + + _assert_error(excinfo, 40013) + assert published == [] + assert root.get('name').as_primitive().value() == 'Alice' + + with pytest.raises(AblyException) as closed: + ctx.keys() + _assert_error(closed, 40000) + + +async def test_rtpo20f_batch_checks_preconditions_again_before_publishing(): + """RTPO20f, RTO26b: a block can await after its writes, so the batch checks the write + preconditions again before publishing; a channel detached in the meantime makes the end of + the block raise 90001, with nothing published.""" + (client, channel, root, mock_ws), published = await _synced() + + with pytest.raises(AblyException) as excinfo: + async with root.batch() as ctx: + ctx.set('name', 'Bob') + await channel.detach() + + _assert_error(excinfo, 90001) + assert published == [] + + +async def test_rtpo20f_rejected_publish_raises_from_the_end_of_the_block(): + """RTPO20f, RTPO20g, RTO20b: when the batch's publish is rejected, the end of the block raises + the rejection's error, nothing is applied locally, and the batch is closed.""" + published = [] + + def reject(message): + published.append(message) + mock_ws.send_to_client({ + 'action': int(ProtocolMessageAction.NACK), + 'msgSerial': message['msgSerial'], + 'count': 1, + 'error': {'code': 40160, 'statusCode': 401, 'message': 'Operation not permitted'}, + }) + + mock_ws = standard_mock_websocket(auto_ack=False, on_object=reject) + client, channel, root, mock_ws = await setup_synced_channel('test', mock_ws=mock_ws) + + with pytest.raises(AblyException) as excinfo: + async with root.batch() as ctx: + ctx.set('name', 'Bob') + ctx.get('score').as_live_counter().increment(5) + + assert excinfo.value.code == 40160 + assert len(published) == 1 + assert len(_operations(published[0])) == 2 + assert root.get('name').as_primitive().value() == 'Alice' + assert root.get('score').as_live_counter().value() == 100 + + with pytest.raises(AblyException) as closed: + ctx.keys() + _assert_error(closed, 40000) + + +# --- Entering ----------------------------------------------------------------------------- + +@pytest.mark.parametrize('open_batch', [ + lambda root: root.get('name').as_live_map().batch(), + lambda root: root.get('missing').as_live_map().batch(), + lambda root: root.get('name').get('first').as_live_counter().batch(), + lambda root: root.get('score').as_live_map().batch(), + lambda root: root.as_live_counter().batch(), +], ids=['primitive', 'missing-key', 'below-primitive', 'counter-as-map', 'map-as-counter']) +async def test_rtpo20c_batch_on_a_path_without_the_live_object_raises(open_batch): + """RTPO20c: entering a batch on a path that does not resolve to a live object of the view's + type raises 92007, and publishes nothing.""" + (client, channel, root, mock_ws), published = await _synced() + batch = open_batch(root) + + with pytest.raises(AblyException) as excinfo: + async with batch: + pytest.fail('the batch was entered') + + _assert_error(excinfo, 92007) + assert published == [] + + +async def test_rtpo20c_path_is_resolved_when_the_block_is_entered(): + """RTPO20c: a path batch resolves its path when the block is entered, not when `batch()` is + called.""" + (client, channel, root, mock_ws), published = await _synced() + batch = root.get('visits').as_live_counter().batch() + + await root.set('visits', LiveCounter.create(1)) + async with batch as visits: + assert isinstance(visits, LiveCounterBatchContext) + visits.increment(2) + + assert root.get('visits').as_live_counter().value() == 3 + + +async def test_rtpo20b_batch_requires_publish_mode(): + """RTPO20b, RTO26a: entering a batch on a channel without the OBJECT_PUBLISH mode raises 40024.""" + mock_ws = standard_mock_websocket(attached_flags=HAS_OBJECTS | OBJECT_SUBSCRIBE_FLAG) + client, channel, root, mock_ws = await setup_synced_channel('test', mock_ws=mock_ws, + modes=(ChannelMode.OBJECT_SUBSCRIBE,)) + + with pytest.raises(AblyException) as excinfo: + async with root.batch(): + pytest.fail('the batch was entered') + + _assert_error(excinfo, 40024) + + +async def test_rtpo20b_batch_requires_echo_messages(): + """RTPO20b, RTO26c: entering a batch on a client with `echo_messages` disabled raises 40000.""" + (client, channel, root, mock_ws), published = await _synced(echo_messages=False) + + with pytest.raises(AblyException) as excinfo: + async with root.batch(): + pytest.fail('the batch was entered') + + _assert_error(excinfo, 40000) + + +async def test_rtpo20b_batch_checks_preconditions_before_resolving(): + """RTPO20b, RTO26b: entering a batch on a detached channel raises 90001, which is checked + before the path is resolved.""" + (client, channel, root, mock_ws), published = await _synced() + score = root.get('score').as_live_counter() + + await channel.detach() + assert channel.state == ChannelState.DETACHED + + # The detach cleared the objects (RTO27a), so `score` no longer resolves: a 92007 here + # would mean the path was resolved first + with pytest.raises(AblyException) as excinfo: + async with score.batch(): + pytest.fail('the batch was entered') + + _assert_error(excinfo, 90001) + + +async def test_rtins17b_instance_batch_checks_preconditions(): + """RTINS17b, RTO26b: entering a batch on an instance of a detached channel raises 90001.""" + (client, channel, root, mock_ws), published = await _synced() + score = root.get('score').instance().as_live_counter() + + await channel.detach() + + with pytest.raises(AblyException) as excinfo: + async with score.batch(): + pytest.fail('the batch was entered') + + _assert_error(excinfo, 90001) + + +async def test_rtpo20d_batch_can_be_entered_once(): + """RTPO20d: each `batch()` call opens one batch, so its block can only be entered once.""" + (client, channel, root, mock_ws), published = await _synced() + batch = root.batch() + + async with batch as ctx: + ctx.set('name', 'Bob') + + with pytest.raises(RuntimeError): + async with batch: + pytest.fail('the batch was entered again') + + assert len(published) == 1 + + +# --- Instances ---------------------------------------------------------------------------- + +async def test_rtins17_batch_on_a_map_instance(): + """RTINS17d-RTINS17f: a batch on a map instance acts on the map it wraps, and is published as + one message.""" + (client, channel, root, mock_ws), published = await _synced() + profile = root.get('profile').instance().as_live_map() + + async with profile.batch() as ctx: + assert isinstance(ctx, LiveMapBatchContext) + assert ctx.id == 'map:profile@1000' + ctx.set('email', 'carol@example.com') + ctx.remove('prefs') + ctx.get('nested_counter').as_live_counter().increment(3) + + assert len(published) == 1 + assert _operations(published[0]) == [ + _map_set('map:profile@1000', 'email', {'string': 'carol@example.com'}), + {'action': MAP_REMOVE, 'objectId': 'map:profile@1000', 'mapRemove': {'key': 'prefs'}}, + _counter_inc('counter:nested@1000', 3), + ] + assert root.at('profile.email').as_primitive().value() == 'carol@example.com' + assert root.at('profile.prefs').exists() is False + assert root.at('profile.nested_counter').as_live_counter().value() == 8 + + +async def test_rtins17_batch_on_a_counter_instance(): + """RTINS17d-RTINS17f: a batch on a counter instance increments the counter it wraps.""" + (client, channel, root, mock_ws), published = await _synced() + score = root.get('score').instance().as_live_counter() + + async with score.batch() as ctx: + assert isinstance(ctx, LiveCounterBatchContext) + ctx.increment(10) + ctx.decrement(4) + + assert _operations(published[0]) == [ + _counter_inc('counter:score@1000', 10), + _counter_inc('counter:score@1000', -4), + ] + assert score.value() == 106 + + +async def test_rtins17d_instance_batch_follows_the_object_not_the_path(): + """RTINS17d, RTPO20c: an instance batch acts on the object the instance wraps wherever it + sits, while a path batch acts on what the path resolves to when it is entered.""" + (client, channel, root, mock_ws), published = await _synced() + profile = root.get('profile').instance().as_live_map() + + await root.remove('profile') + + with pytest.raises(AblyException) as excinfo: + async with root.get('profile').as_live_map().batch(): + pytest.fail('the path batch was entered') + _assert_error(excinfo, 92007) + + async with profile.batch() as ctx: + ctx.set('email', 'dana@example.com') + + assert _operations(published[-1]) == [_map_set('map:profile@1000', 'email', {'string': 'dana@example.com'})] + assert profile.get('email').as_primitive().value() == 'dana@example.com' + + +# --- Counter contexts --------------------------------------------------------------------- + +async def test_rtbc14_rtbc15_counter_batch_increments_and_decrements(): + """RTBC14, RTBC15: a counter context queues a COUNTER_INC for each increment, and one of the + negated amount for each decrement, the amount defaulting to 1.""" + (client, channel, root, mock_ws), published = await _synced() + + async with root.get('score').as_live_counter().batch() as score: + assert isinstance(score, LiveCounterBatchContext) + score.increment(5) + score.decrement(2) + score.increment() + score.decrement() + score.increment(0.5) + assert score.value() == 100 + + assert len(published) == 1 + assert _operations(published[0]) == [ + _counter_inc('counter:score@1000', number) for number in (5, -2, 1, -1, 0.5) + ] + assert root.get('score').as_live_counter().value() == 103.5 + + +@pytest.mark.parametrize('write, code', [ + (lambda ctx, score: ctx.set(1, 'value'), 40003), + (lambda ctx, score: ctx.set('key', object()), 40013), + (lambda ctx, score: ctx.set('key', None), 40013), + (lambda ctx, score: ctx.set('key', float('nan')), 40013), + (lambda ctx, score: ctx.remove(None), 40003), + (lambda ctx, score: score.increment('5'), 40003), + (lambda ctx, score: score.increment(None), 40003), + (lambda ctx, score: score.increment(float('inf')), 40003), + (lambda ctx, score: score.decrement(True), 40003), +], ids=['key-not-str', 'value-object', 'value-none', 'value-nan', 'remove-key-none', 'amount-str', + 'amount-none', 'amount-inf', 'amount-bool']) +async def test_rtbc12_rtbc15_invalid_arguments_raise_at_the_call(write, code): + """RTBC12-RTBC15, RTLM20e1, RTLM21e1, RTLC12e1: a write with an invalid key, value or amount + raises where it is called, and queues nothing.""" + (client, channel, root, mock_ws), published = await _synced() + + async with root.batch() as ctx: + with pytest.raises(AblyException) as excinfo: + write(ctx, ctx.get('score').as_live_counter()) + _assert_error(excinfo, code) + ctx.set('name', 'Bob') + + assert _operations(published[0]) == [_map_set('root', 'name', {'string': 'Bob'})] + + +# --- Reads and views ---------------------------------------------------------------------- + +async def test_rtbc3_rtbc4_id_and_get(): + """RTBC3, RTBC4: `id` is the wrapped object's id, None for a primitive; `get` wraps the value + at a key in the context of its type, and is None for a key with no value or on anything but + a map.""" + (client, channel, root, mock_ws), published = await _synced() + + async with root.batch() as ctx: + assert ctx.id == 'root' + profile, score, name = ctx.get('profile'), ctx.get('score'), ctx.get('name') + assert isinstance(profile, LiveMapBatchContext) + assert isinstance(score, LiveCounterBatchContext) + assert isinstance(name, PrimitiveBatchContext) + assert profile.id == 'map:profile@1000' + assert score.id == 'counter:score@1000' + assert name.id is None + + assert ctx.get('missing') is None + assert name.get('first') is None + assert score.get('first') is None + + with pytest.raises(AblyException) as excinfo: + ctx.get(1) + _assert_error(excinfo, 40003) + + +async def test_rtbc5_value(): + """RTBC5: a counter context's value is the count; a primitive context's is the primitive, + filtered by `expected` as `PrimitiveInstance.value` filters it.""" + (client, channel, root, mock_ws), published = await _synced() + + async with root.batch() as ctx: + assert ctx.get('score').as_live_counter().value() == 100 + assert ctx.get('name').as_primitive().value() == 'Alice' + assert ctx.get('name').as_primitive().value(str) == 'Alice' + assert ctx.get('name').as_primitive().value(float) is None + assert ctx.get('active').as_primitive().value(bool) is True + assert ctx.get('active').as_primitive().value(float) is None + assert ctx.get('avatar').as_primitive().value(bytes) == b'\x01\x02\x03' + assert ctx.get('data').as_primitive().value(dict) == {'tags': ['a', 'b']} + + age = ctx.get('age').as_primitive().value(float) + assert age == 30 + assert type(age) is float + + with pytest.raises(TypeError): + ctx.get('age').as_primitive().value(int) + + +async def test_rtbc6_rtbc9_map_enumeration(): + """RTBC6-RTBC9: a map context's entries and values are contexts of each entry's type, its keys + and size those of the map.""" + (client, channel, root, mock_ws), published = await _synced() + + async with root.get('profile').as_live_map().batch() as profile: + entries = profile.entries() + assert [key for key, _ in entries] == ['email', 'nested_counter', 'prefs'] + assert [type(context) for _, context in entries] == [ + PrimitiveBatchContext, LiveCounterBatchContext, LiveMapBatchContext] + values = profile.values() + assert [type(context) for context in values] == [type(context) for _, context in entries] + # A live object has one context per batch (RTBC16c); a primitive gets a new one each time + assert values[1:] == [context for _, context in entries[1:]] + assert values[0].value() == 'alice@example.com' + assert profile.keys() == ['email', 'nested_counter', 'prefs'] + assert profile.size() == 3 + assert entries[1][1].value() == 5 + assert entries[2][1].get('theme').as_primitive().value() == 'dark' + + +async def test_rtbc10_rtbc11_compact(): + """RTBC10, RTBC11: `compact` and `compact_json` snapshot the wrapped value as `Instance` does.""" + (client, channel, root, mock_ws), published = await _synced() + + async with root.batch() as ctx: + assert ctx.get('profile').compact() == { + 'email': 'alice@example.com', 'nested_counter': 5, 'prefs': {'theme': 'dark'}} + assert ctx.compact()['avatar'] == b'\x01\x02\x03' + assert ctx.compact_json()['avatar'] == 'AQID' + assert ctx.get('score').compact_json() == 100 + + +async def test_rtbc1a_view_helpers_are_checked(): + """RTBC1a, RTTS9d: a context's view helpers return the context itself for the type it wraps, + and raise 92007 for any other.""" + (client, channel, root, mock_ws), published = await _synced() + + async with root.batch() as ctx: + score, name = ctx.get('score'), ctx.get('name') + assert ctx.as_live_map() is ctx + assert score.as_live_counter() is score + assert name.as_primitive() is name + + for view_helper in (ctx.as_live_counter, ctx.as_primitive, score.as_live_map, score.as_primitive, + name.as_live_map, name.as_live_counter): + with pytest.raises(AblyException) as excinfo: + view_helper() + _assert_error(excinfo, 92007) + + assert published == [] + + +async def test_rtbc16c_one_context_per_object(): + """RTBC16c: a batch wraps each object in one context however it is reached, the object the + batch was opened on included.""" + (client, channel, root, mock_ws), published = await _synced() + # A reference from `profile.prefs` back to the root, so that the root is reachable from inside + mock_ws.send_to_client(build_object_message('test', [ + build_map_set('map:prefs@1000', 'home', {'objectId': 'root'}, remote_serial(0), 'remote'), + ])) + await poll_until(lambda: 'home' in root.at('profile.prefs').as_live_map().keys(), + description='the back reference to be applied') + + async with root.batch() as ctx: + profile = ctx.get('profile') + assert ctx.get('profile') is profile + assert dict(ctx.entries())['profile'] is profile + assert ctx.get('score') is ctx.get('score') + assert profile.get('prefs').get('home') is ctx + + +class _Contexts(NamedTuple): + root: LiveMapBatchContext + profile: LiveMapBatchContext + score: LiveCounterBatchContext + name: PrimitiveBatchContext + + +# A call of each context method that reads the objects, and of each that writes +_READS = { + 'get': lambda c: c.root.get('name'), + 'compact': lambda c: c.root.compact(), + 'compact_json': lambda c: c.root.compact_json(), + 'entries': lambda c: c.root.entries(), + 'keys': lambda c: c.root.keys(), + 'values': lambda c: c.root.values(), + 'size': lambda c: c.root.size(), + 'child-get': lambda c: c.profile.get('email'), + 'counter-value': lambda c: c.score.value(), + 'primitive-value': lambda c: c.name.value(), +} +_WRITES = { + 'set': lambda c: c.root.set('name', 'Bob'), + 'set-value-type': lambda c: c.root.set('team', LiveMap.create()), + 'remove': lambda c: c.root.remove('name'), + 'child-set': lambda c: c.profile.set('email', 'bob@example.com'), + 'increment': lambda c: c.score.increment(1), + 'decrement': lambda c: c.score.decrement(1), +} +# The calls that check whether the batch is open, but no channel preconditions +_VIEWS = { + 'id': lambda c: c.root.id, + 'primitive-id': lambda c: c.name.id, + 'as_live_map': lambda c: c.root.as_live_map(), + 'as_live_counter': lambda c: c.score.as_live_counter(), + 'as_primitive': lambda c: c.name.as_primitive(), +} + + +def _contexts(ctx): + return _Contexts(ctx, ctx.get('profile'), ctx.get('score'), ctx.get('name')) + + +@pytest.mark.parametrize('call', [*_READS.values(), *_WRITES.values()], ids=[*_READS, *_WRITES]) +async def test_rtbc4b_rtbc15b_contexts_check_preconditions_on_each_call(call): + """RTBC4b-RTBC15b: the contexts check the channel preconditions on every read (RTO25) and + write (RTO26), so once the channel detaches inside the block each of them raises 90001, and + nothing is published.""" + (client, channel, root, mock_ws), published = await _synced() + + async with root.batch() as ctx: + contexts = _contexts(ctx) + await channel.detach() + + with pytest.raises(AblyException) as excinfo: + call(contexts) + _assert_error(excinfo, 90001) + + assert published == [] + + +@pytest.mark.parametrize('call', [*_VIEWS.values(), *_READS.values(), *_WRITES.values()], + ids=[*_VIEWS, *_READS, *_WRITES]) +async def test_rtbc16e_contexts_raise_once_the_batch_has_closed(call): + """RTBC16e, RTBC3b-RTBC15c: once its block has exited, every method of a batch's contexts, + those opened inside the block included, raises 40000.""" + (client, channel, root, mock_ws), published = await _synced() + + async with root.batch() as ctx: + contexts = _contexts(ctx) + + with pytest.raises(AblyException) as excinfo: + call(contexts) + _assert_error(excinfo, 40000) diff --git a/test/unit/objects/conftest.py b/test/unit/objects/conftest.py new file mode 100644 index 00000000..6b7cf0d3 --- /dev/null +++ b/test/unit/objects/conftest.py @@ -0,0 +1,10 @@ +import pytest + +from test.uts.helpers.client import close_open_clients + + +@pytest.fixture(autouse=True) +async def close_clients(): + """Closes the clients a test built through the UTS helpers, whether or not its assertions held.""" + yield + await close_open_clients() diff --git a/test/unit/objects/typed_views_test.py b/test/unit/objects/typed_views_test.py new file mode 100644 index 00000000..6121dc0f --- /dev/null +++ b/test/unit/objects/typed_views_test.py @@ -0,0 +1,386 @@ +"""The typed views: `type()` and `exists()`, the unchecked path views and the checked instance views. + +Spec points: RTTS2-RTTS10 (ably/specification#491), as LODR-061 binds them: one +`PrimitivePathObject` and `PrimitiveInstance` whose `value(expected)` stands for the six +per-primitive classes (RTTS6h), and `ValueType.UNKNOWN` for a value in none of the known +categories. No UTS specification covers the typed views, so these tests are written against +the features specification and LODR-061. + +Each test drives the standard synced pool over the mock websocket. +""" + +import typing + +import pytest + +from ably.pubsub.objects.liveobject import LiveObject, LiveObjectUpdate +from ably.pubsub.server import ( + Instance, + LiveCounterInstance, + LiveCounterPathObject, + LiveMapInstance, + LiveMapPathObject, + PathObject, + PrimitiveInstance, + PrimitivePathObject, + ValueType, +) +from ably.pubsub.types.channelstate import ChannelState +from ably.pubsub.util.exceptions import AblyException +from test.uts.helpers.client import poll_until +from test.uts.objects.helpers.standard_test_pool import ( + build_map_set, + build_object_message, + remote_serial, + setup_synced_channel, +) + +# The type of each value in the standard pool, and of the JSON array each test adds to it +POOL_VALUE_TYPES = { + '': ValueType.LIVE_MAP, + 'name': ValueType.STRING, + 'age': ValueType.NUMBER, + 'active': ValueType.BOOLEAN, + 'avatar': ValueType.BINARY, + 'data': ValueType.JSON_OBJECT, + 'tags': ValueType.JSON_ARRAY, + 'score': ValueType.LIVE_COUNTER, + 'profile': ValueType.LIVE_MAP, + 'profile.prefs.theme': ValueType.STRING, +} + + +async def _synced_root(): + """The standard synced root, with a JSON array added at `tags`.""" + client, channel, root, mock_ws = await setup_synced_channel('test') + await root.set('tags', ['a', 'b']) + return root + + +def _at(root, path): + """The path object at the dotted `path` from `root`, `root` itself for the empty path.""" + return root.at(path) if path else root + + +def _assert_error(excinfo, code): + assert excinfo.value.code == code + assert excinfo.value.status_code == 400 + + +class _FutureLiveObject(LiveObject): + """A live object of a type this SDK does not know, standing in for one a later protocol adds.""" + + def apply_operation(self, object_message, source): + return False + + def replace_data(self, object_message): + return LiveObjectUpdate(noop=True) + + def clear_data(self): + pass + + @staticmethod + def diff(previous_data, new_data, *, for_tombstone=False): + return LiveObjectUpdate(noop=True) + + +# --- type() and exists() ------------------------------------------------------------------ + +@pytest.mark.parametrize('path, value_type', POOL_VALUE_TYPES.items(), ids=[p or 'root' for p in POOL_VALUE_TYPES]) +async def test_rtts4b_type_is_the_type_of_the_resolved_value(path, value_type): + """RTTS4a, RTTS4b, RTTS2a1-RTTS2a8: `type()` is the `ValueType` of the value the path resolves to, + and `exists()` is True for it.""" + root = await _synced_root() + + assert _at(root, path).type() is value_type + assert _at(root, path).exists() is True + + +@pytest.mark.parametrize('path', ['missing', 'name.first', 'score.value', 'data.tags', 'profile.prefs.missing']) +async def test_rtts4b_type_is_none_where_nothing_resolves(path): + """RTTS4a3, RTTS4b3: `type()` is None and `exists()` False for a path that does not resolve, + including one that continues below a primitive or a counter.""" + root = await _synced_root() + + assert root.at(path).type() is None + assert root.at(path).exists() is False + + +async def test_rtts4b_type_follows_the_value_at_the_path(): + """RTTS4b, RTTS8a: a path's type is read when it is asked for, so it follows a change to the + value at the path; an instance's type is fixed by the value it wraps.""" + root = await _synced_root() + score = root.get('score') + instance = score.instance() + + await root.set('score', 'retired') + assert score.type() is ValueType.STRING + assert instance.type is ValueType.LIVE_COUNTER + + await root.remove('score') + assert score.type() is None + assert score.exists() is False + + +async def test_rtts2a9_type_is_unknown_for_a_value_in_no_known_category(): + """RTTS2a9: a value that resolves but falls into none of the known categories is + `ValueType.UNKNOWN`, which is distinct from None for no value, and the typed views read + None from it.""" + client, channel, root, mock_ws = await setup_synced_channel('test') + channel.object._objects_pool['future:thing@1000'] = _FutureLiveObject('future:thing@1000') + mock_ws.send_to_client(build_object_message('test', [ + build_map_set('root', 'thing', {'objectId': 'future:thing@1000'}, remote_serial(0), 'remote'), + ])) + await poll_until(lambda: 'thing' in root.keys(), description='the reference to be applied') + + thing = root.get('thing') + assert thing.type() is ValueType.UNKNOWN + assert thing.exists() is True + assert thing.as_primitive().value() is None + assert thing.as_live_counter().value() is None + assert thing.as_live_map().size() is None + assert thing.as_live_map().keys() == [] + assert thing.instance().type is ValueType.UNKNOWN + + +async def test_rtts4a1_rtts4b1_type_and_exists_check_access_preconditions(): + """RTTS4a1, RTTS4b1, RTO25b: `type()` and `exists()` raise 90001 on a detached channel.""" + client, channel, root, mock_ws = await setup_synced_channel('test') + + await channel.detach() + assert channel.state == ChannelState.DETACHED + + for read in (root.get('name').type, root.get('name').exists): + with pytest.raises(AblyException) as excinfo: + read() + _assert_error(excinfo, 90001) + + +# --- The unchecked path views ------------------------------------------------------------- + +async def test_rtts5_path_views_rewrap_the_same_path(): + """RTTS5, RTTS6d: `channel.object.get()` is a `LiveMapPathObject`; navigation gives base + `PathObject`s, and each view helper gives the typed view of the same path, without reading + anything.""" + root = await _synced_root() + assert isinstance(root, LiveMapPathObject) + + path = root.get('profile').get('prefs') + assert type(path) is PathObject + for view, view_type in ((path.as_live_map(), LiveMapPathObject), + (path.as_live_counter(), LiveCounterPathObject), + (path.as_primitive(), PrimitivePathObject)): + assert type(view) is view_type + assert view.path() == 'profile.prefs' + + assert path.as_live_map().get('theme').as_primitive().value() == 'dark' + # A view of a path that does not resolve is still a view + assert root.get('missing').as_live_counter().path() == 'missing' + + +async def test_rtts5d1_mismatched_path_views_read_none(): + """RTTS5d, RTTS5d1: a path view never raises for the type at its path; a read through a view + that does not match it gives None, or [] for the collections.""" + root = await _synced_root() + + name_as_map = root.get('name').as_live_map() + assert name_as_map.keys() == [] + assert name_as_map.entries() == [] + assert name_as_map.values() == [] + assert name_as_map.size() is None + + assert root.get('name').as_live_counter().value() is None + assert root.get('score').as_primitive().value() is None + assert root.get('score').as_live_map().size() is None + assert root.as_primitive().value() is None + assert root.as_live_counter().value() is None + assert root.get('missing').as_primitive().value() is None + + +async def test_rtts5d2_writes_through_mismatched_path_views_raise(): + """RTTS5d2: a write through a path view raises 92007 when the path resolves to another type, + and 92005 when it does not resolve.""" + root = await _synced_root() + + for write, code in ((lambda: root.get('name').as_live_map().set('k', 'v'), 92007), + (lambda: root.get('profile').as_live_counter().increment(), 92007), + (lambda: root.get('score').as_live_map().remove('k'), 92007), + (lambda: root.get('missing').as_live_counter().decrement(), 92005), + (lambda: root.at('name.first').as_live_map().set('k', 'v'), 92005)): + with pytest.raises(AblyException) as excinfo: + await write() + _assert_error(excinfo, code) + + assert root.get('name').as_primitive().value() == 'Alice' + + +async def test_rtts6b_counter_view_value_is_only_a_counters(): + """RTTS6b: `LiveCounterPathObject.value()` is the counter's value as a float, and None for + anything else, a number included.""" + root = await _synced_root() + + score = root.get('score').as_live_counter().value() + assert score == 100 + assert type(score) is float + assert root.get('age').as_live_counter().value() is None + assert root.as_live_counter().value() is None + + +@pytest.mark.parametrize('path, expected, value', [ + ('name', str, 'Alice'), + ('name', float, None), + ('age', float, 30.0), + ('age', str, None), + ('age', bool, None), + ('active', bool, True), + ('active', float, None), + ('avatar', bytes, b'\x01\x02\x03'), + ('avatar', str, None), + ('data', dict, {'tags': ['a', 'b']}), + ('data', list, None), + ('tags', list, ['a', 'b']), + ('tags', dict, None), + ('score', float, None), + ('profile', dict, None), + ('missing', str, None), +], ids=lambda param: param.__name__ if isinstance(param, type) else None) +async def test_rtts6c_value_expected_filters_by_type(path, expected, value): + """RTTS6c, RTTS6h: `value(expected)` is the primitive at the path only if it is of the expected + type, judged by its wire type, so a boolean is never a number and a counter or map is never + a primitive.""" + root = await _synced_root() + + result = root.get(path).as_primitive().value(expected) + assert result == value + assert type(result) is type(value) + + +async def test_rtts6c_value_without_expected_is_any_primitive(): + """RTTS6c, RTTS6h: `value()` with no `expected` is the primitive at the path whatever its type, + and None for a live object.""" + root = await _synced_root() + + assert {path: _at(root, path).as_primitive().value() for path in POOL_VALUE_TYPES} == { + '': None, + 'name': 'Alice', + 'age': 30.0, + 'active': True, + 'avatar': b'\x01\x02\x03', + 'data': {'tags': ['a', 'b']}, + 'tags': ['a', 'b'], + 'score': None, + 'profile': None, + 'profile.prefs.theme': 'dark', + } + + +async def test_rtts6c_value_rejects_an_unsupported_expected_type(): + """RTTS6c: `expected` is one of str, float, bool, bytes, list and dict; anything else, a + subclass of one of them included, is a TypeError, raised before the preconditions are + checked.""" + client, channel, root, mock_ws = await setup_synced_channel('test') + age = root.get('age').as_primitive() + name = root.get('name').as_primitive() + + for expected in (int, object, tuple, 'str', typing.List, type('Text', (str,), {})): + with pytest.raises(TypeError): + name.value(expected) + + await channel.detach() + with pytest.raises(TypeError): + age.value(int) + with pytest.raises(AblyException) as excinfo: + age.value(float) + _assert_error(excinfo, 90001) + + +async def test_rtts6c_value_is_a_copy_of_a_json_value(): + """RTTS6c: a JSON value read through a view is the reader's own, so changing it leaves the + object it was read from unchanged.""" + root = await _synced_root() + + data = root.get('data').as_primitive().value(dict) + data['tags'].append('c') + + assert root.get('data').as_primitive().value(dict) == {'tags': ['a', 'b']} + + +# --- The checked instance views ----------------------------------------------------------- + +@pytest.mark.parametrize('path, value_type', POOL_VALUE_TYPES.items(), ids=[p or 'root' for p in POOL_VALUE_TYPES]) +async def test_rtts8a_instance_type_is_the_type_of_the_wrapped_value(path, value_type): + """RTTS7e, RTTS8a: every instance is the typed subclass matching the value it wraps, and its + `type` property is that value's `ValueType`.""" + root = await _synced_root() + + instance = _at(root, path).instance() + + assert instance.type is value_type + expected_class = {ValueType.LIVE_MAP: LiveMapInstance, ValueType.LIVE_COUNTER: LiveCounterInstance}.get( + value_type, PrimitiveInstance) + assert type(instance) is expected_class + + +async def test_rtts9d_instance_views_are_checked(): + """RTTS9, RTTS9d: an instance's view helper returns the instance for the type it wraps and + raises 92007 for any other, where the same helper on a path never raises.""" + root = await _synced_root() + instances = { + 'profile': (root.get('profile').instance(), 'as_live_map'), + 'score': (root.get('score').instance(), 'as_live_counter'), + 'name': (root.get('name').instance(), 'as_primitive'), + } + + for path, (instance, matching_helper) in instances.items(): + assert getattr(instance, matching_helper)() is instance + for helper in {'as_live_map', 'as_live_counter', 'as_primitive'} - {matching_helper}: + with pytest.raises(AblyException) as excinfo: + getattr(instance, helper)() + _assert_error(excinfo, 92007) + # The same request on the path is an unchecked expectation + assert getattr(root.get(path), helper)().path() == path + + +async def test_rtts10_instance_id_is_a_property(): + """RTTS10a, RTTS10b, RTINS3: `id` is a property: the object id of a map or counter instance, + and None for a primitive one.""" + root = await _synced_root() + + assert root.instance().id == 'root' + assert root.get('profile').instance().id == 'map:profile@1000' + assert root.get('score').instance().id == 'counter:score@1000' + assert root.get('name').instance().id is None + + +@pytest.mark.parametrize('typed_class, present, absent', [ + (PathObject, + {'path', 'type', 'exists', 'get', 'at', 'instance', 'compact', 'compact_json', 'subscribe', + 'as_live_map', 'as_live_counter', 'as_primitive'}, + {'value', 'entries', 'keys', 'values', 'size', 'set', 'remove', 'increment', 'decrement', 'batch'}), + (LiveMapPathObject, + {'entries', 'keys', 'values', 'size', 'set', 'remove', 'batch'}, + {'value', 'increment', 'decrement'}), + (LiveCounterPathObject, + {'value', 'increment', 'decrement', 'batch'}, + {'entries', 'keys', 'values', 'size', 'set', 'remove'}), + (PrimitivePathObject, + {'value'}, + {'entries', 'keys', 'values', 'size', 'set', 'remove', 'increment', 'decrement', 'batch'}), + (Instance, + {'id', 'type', 'get', 'compact', 'compact_json', 'as_live_map', 'as_live_counter', 'as_primitive'}, + {'value', 'entries', 'size', 'set', 'remove', 'increment', 'decrement', 'subscribe', 'batch'}), + (LiveMapInstance, + {'entries', 'keys', 'values', 'size', 'set', 'remove', 'subscribe', 'batch'}, + {'value', 'increment', 'decrement'}), + (LiveCounterInstance, + {'value', 'increment', 'decrement', 'subscribe', 'batch'}, + {'entries', 'keys', 'values', 'size', 'set', 'remove'}), + (PrimitiveInstance, + {'value'}, + {'entries', 'keys', 'values', 'size', 'set', 'remove', 'increment', 'decrement', 'subscribe', 'batch'}), +], ids=lambda param: param.__name__ if isinstance(param, type) else '') +def test_rtts3_rtts10_methods_are_partitioned_by_type(typed_class, present, absent): + """RTTS3, RTTS6, RTTS7, RTTS10: each typed view carries the methods of its own type only, so + `subscribe` is on map and counter instances alone and only live objects can be batched.""" + assert {name for name in present if not hasattr(typed_class, name)} == set() + assert {name for name in absent if hasattr(typed_class, name)} == set() From d2325eaee5e46a20959f0ea2fe9725b03601360b Mon Sep 17 00:00:00 2001 From: owenpearson Date: Fri, 9 Oct 2026 14:42:45 +0100 Subject: [PATCH 19/71] docs: record the LiveObjects derivation Adds the objects tier to deviations.md: the faults in the uts/objects specifications the derived tests correct or work around, the shape adaptations the Python API forces, the claims investigated and refuted against the sandbox, seven candidate upstream issues, and how the tier is adopted here. The header counts are re-measured with objects as a tier of its own, which also corrects the gated count: the RSC7d spec-error gate was never counted in it. The UTS README and the uts-to-python skill gain the objects layout, its helpers, the typed-view translation and the traps met deriving it. Co-Authored-By: Claude Opus 5.5 (1M context) --- .claude/skills/uts-to-python/SKILL.md | 111 ++++- test/uts/README.md | 62 ++- test/uts/deviations.md | 586 ++++++++++++++++++++++++-- 3 files changed, 715 insertions(+), 44 deletions(-) diff --git a/.claude/skills/uts-to-python/SKILL.md b/.claude/skills/uts-to-python/SKILL.md index 5a93bf7a..a5930b02 100644 --- a/.claude/skills/uts-to-python/SKILL.md +++ b/.claude/skills/uts-to-python/SKILL.md @@ -20,6 +20,9 @@ gh api 'repos/ably/specification/contents/uts/realtime/integration/.md' -- gh api repos/ably/specification/contents/uts/docs/proxy.md --jq '.content' | base64 -d gh api 'repos/ably/specification/contents/uts/rest/integration/proxy/.md' --jq '.content' | base64 -d gh api 'repos/ably/specification/contents/uts/realtime/integration/proxy/.md' --jq '.content' | base64 -d +gh api 'repos/ably/specification/contents/uts/objects/unit/.md' --jq '.content' | base64 -d +gh api 'repos/ably/specification/contents/uts/objects/integration/.md' --jq '.content' | base64 -d +gh api repos/ably/specification/contents/uts/objects/helpers/standard_test_pool.md --jq '.content' | base64 -d ``` The realtime integration tier nests, so list a directory before fetching from it: @@ -39,9 +42,10 @@ carries over as it stands: `channels/channel_publish_test.md` becomes `channels/channel_publish_test.py`. Every directory needs an `__init__.py`, as `test` is a package. -There are two kinds of tier. `rest/unit` and `realtime/unit` serve every request from a -mock and reach no network; `rest/integration` and `realtime/integration` run against the -real Ably sandbox and have no mock at all. See **The integration tier** below. +There are two kinds of tier. `rest/unit`, `realtime/unit` and `objects/unit` reach no +network; `rest/integration`, `realtime/integration` and `objects/integration` run against +the real Ably sandbox and have no mock at all. See **The integration tier** below, and +**The objects tier** for LiveObjects. `test/uts/rest/unit/time_test.py` is the reference example for REST unit, `test/uts/realtime/unit/connection/auto_connect_test.py` for realtime unit, @@ -451,6 +455,103 @@ async def test_rsc15l4_cloudfront_header_fallback(sandbox, proxy_session): assert http_responses(log)[0]['status'] == 403 ``` +## The objects tier + +`uts/objects/unit/.md` becomes `test/uts/objects/unit/_test.py`, the integration +specifications keep their `_test` suffix under `test/uts/objects/integration/`, and +`integration/proxy/objects_faults.md` becomes +`test/uts/objects/integration/proxy/objects_faults_test.py`. Seven unit specifications are +pure and construct internal objects; the other eight drive `channel.object` over the mock +websocket. Follow `internal_live_map_test.py` (pure), `path_object_mutations_test.py` (mock) +and `objects_lifecycle_test.py` (integration). The integration package's own `conftest.py` +provisions a separate app under the realtime tier's fixture name, `realtime_sandbox`, and +all three of its specifications take `use_binary_protocol`. The package is +`ably.pubsub.objects`; the public names are exported from `ably.pubsub.server`. + +**Translate the untyped API through the typed views** (LODR-061, the RTTS partition). The +specifications call every method on one `PathObject` or `Instance` class; ably-python +reaches a type's methods through its view: + +| Pseudocode | ably-python | +|---|---| +| `root.get("score").value()` | `root.get('score').as_live_counter().value()` | +| `root.get("name").value()` | `root.get('name').as_primitive().value()` | +| `pathObject.set("k", v)` | `await path_object.as_live_map().set('k', v)`; `root` is already a `LiveMapPathObject` | +| untyped `value() == null` | `as_primitive().value() is None` **and** `as_live_counter().value() is None` | +| `inst.id()` | `inst.id`, a property, as is `inst.type` | +| `{ depth: n }` | `subscribe(listener, depth=n)`, keyword-only | + +Mutations are `async`; reads, navigation, views and `subscribe` are not. `entries()` yields +tuples. A path's views never raise, and a write through the wrong one raises 92007, or 92005 +for a path that does not resolve; an `Instance`'s views raise 92007. A spelling difference is +not a deviation. Where the partition makes a read unreachable — `value()` on a +`LiveMapInstance` — assert the `type`, `not hasattr(...)` and the 92007, and cite S-5. + +**The pure tier adapts to five shapes**, defined in `deviations.md` and cited by label in the +module docstring: S-1 `apply_operation` returns a boolean, so record updates with +`capture_updates(obj)`; S-2 the sync state machine is a standalone `RealtimeObject()`'s +(`_on_attached(has_objects)`, `_handle_object_sync_messages(msgs, channel_serial)`, +`_sync_state`); S-3 `evaluate(vt, timestamp_ms)`; S-4 the retained create is +`operation.resolved_counter_create` / `resolved_map_create`; S-5 above. Members of a public +class that LODR-061 does not name carry a leading underscore (`channel.object._objects_pool`). + +`test/uts/objects/helpers/standard_test_pool.py` holds what the specifications share: +`setup_synced_channel` (and `_no_ack`), `standard_mock_websocket`, `objects_client`, +`objects_channel_options`, `objects_connected_message`, `objects_attached_message`, the +`build_*` builders, `json_value`, `bytes_value`, `ack_serial`, `remote_serial`, +`below_ack_serial`, `object_message(s)`, `capture_updates`, `build_public_object_message`, +`assert_unchanged_after_quiescence`, `provision_objects_via_rest`, and the wire constants +(`HAS_OBJECTS`, `OBJECT_SUBSCRIBE_FLAG`, `OBJECT_PUBLISH_FLAG`, `LWW`, the actions). +`test/uts/README.md` says what each is. + +### Traps found while deriving and implementing the objects tier + +- **An injected frame is applied on the transport's read task, not by `send_to_client`.** + Read state only after a `poll_until` on its effect. A negative — "did not fire", "the echo + was not applied" — needs a positive control sent *behind* the message under test and + `assert_unchanged_after_quiescence`; an exact count after `poll_until(>= n)` needs + `await settle()` first; and a subscription made straight after a seeding message receives + the seed, so poll for the seed before subscribing. The specifications often skip all three, + and their negatives then pass whatever the SDK does. +- **Process a re-sync ATTACHED before starting what it should hold back.** Send it, poll until + `channel.object._sync_state == ObjectsSyncState.SYNCING`, then start `get()` or the write as + a task; started earlier, it sees SYNCED and resolves at once. +- **A mock ATTACHED grants no modes.** The standard one carries `HAS_OBJECTS` alone, which + empties `channel.modes`, and RTO2 then checks the modes the channel *requested* — so request + them with `objects_channel_options()` or `get()` raises 40024. Grant modes as `flags` bits, + never as the specifications' `modes: [...]`. An injected `flags: 128` at the integration tier + empties them the same way. +- **A write resolves on its ACK.** The standard mock ACKs each OBJECT with + `ack_serial(msgSerial, i)`; under `setup_synced_channel_no_ack`, drive the write as a task. + Those serials land in `appliedOnAckSerials`, so never reuse one as an inbound serial, and + use `remote_serial(n)` for a remote write: a bare `'99'` sorts before `POOL_SERIAL` and is + stale. +- **`json` values are JSON strings on the wire** (OD2g), so compare a captured + `mapSet.value.json` after `json.loads`; `bytes` are base64. Actions are + `ObjectOperationAction`, an `IntEnum`. +- **Numbers decode to `float`.** Read them with `value(float)`; `value(int)` raises + `TypeError`, since `expected` must be exactly one of `str, float, bool, bytes, list, dict`. + Assert booleans with `is True`, since `True == 1`. +- **Creating a `LiveCounter` or `LiveMap` reads `/time`.** `objects_client` answers it; a + client built any other way needs `mock_http=time_mock_http(clock)`. +- **A tombstoned object reads null with no GC at all**, so a GC test asserts that the object + left `_objects_pool`. The GC interval, 300000 ms, is read when the first ATTACHED schedules + the timer on the client's clock: set `channel.object._gc_interval_ms` before `get()`. +- **The public and internal `ObjectMessage` share a name.** Reach the public one as + `publicmessage.ObjectMessage`, from `ably.pubsub.objects`. +- **Check that a test can fail.** Several objects specifications assert something that holds + with the behaviour removed. The derivation caught them by running each module against a + throwaway reference implementation with one fault injected; add the discriminating + assertion under `# UTS SPEC ERROR:` and record the fault. +- **At the sandbox, the echo arrives before the ACK**, so RTO9a3's dedup is only reachable at + the mock tier, and **ACKs are paced at about one per 500 ms per connection**, so each + awaited write after the first can take half a second. +- **Every objects channel gets an OBJECT_SYNC**, an empty one included, and a resumed + ATTACHED on an attached channel restarts the sync without a state change: wait on + `get()`, not on a channel state. A sync cursor can itself contain `:`. +- **uts-proxy matches `action` as a string and names actions only up to AUTH**: OBJECT_SYNC + is `'20'`, OBJECT `'19'`. Its `delay` holds every later frame behind the delayed one. + ## Traps that cost the most time Ordered by how much they cost, not by subject. Every one was hit for real while @@ -896,8 +997,8 @@ the reasoning. The next reader will otherwise reach the same first conclusion. ```bash uv run --frozen --extra crypto --extra dev ruff check ably/ test/ -uv run --frozen --extra crypto --extra dev pytest test/uts/rest/unit test/uts/realtime/unit test/uts/helpers -q -uv run --frozen --extra crypto --extra dev pytest test/uts/rest/integration test/uts/realtime/integration -q +uv run --frozen --extra crypto --extra dev pytest test/uts/rest/unit test/uts/realtime/unit test/uts/objects/unit test/uts/helpers test/uts/objects/helpers -q +uv run --frozen --extra crypto --extra dev pytest test/uts/rest/integration test/uts/realtime/integration test/uts/objects/integration -q RUN_DEVIATIONS=1 uv run --frozen --extra crypto --extra dev pytest test/uts -q ``` diff --git a/test/uts/README.md b/test/uts/README.md index 8690d290..2a7e5976 100644 --- a/test/uts/README.md +++ b/test/uts/README.md @@ -20,12 +20,13 @@ helpers/ shared infrastructure the specifications assume, and its own tests assets/ fixtures the specifications name, vendored from elsewhere rest/ specifications under uts/rest realtime/ specifications under uts/realtime +objects/ specifications under uts/objects (LiveObjects), with helpers/ of its own ``` Every directory holding tests needs an `__init__.py`, because `test` is a package. Unit tests serve every request from a mock and reach no network — neither the REST -suite nor the realtime one. The seams are installed per client, so a test holds to +suite, the realtime one nor the objects one. The seams are installed per client, so a test holds to that by installing them; one that omits a seam, or that lets the host fallback loop run, reaches the real internet. Integration tests run against a sandbox app. @@ -86,10 +87,11 @@ that closes its own leaves nothing to clean up if it fails first. | [helpers/presence.py](helpers/presence.py) | the presence-map stubs and wire-message builders the presence specifications share | | [helpers/sandbox.py](helpers/sandbox.py) | the sandbox app the integration tier provisions, the presence-fixture cipher, `random_id()` and the JWT signing the auth specification asks a library for | | [helpers/deviations.py](helpers/deviations.py) | the `@deviation` and `@spec_error` gates | +| [objects/helpers/standard_test_pool.py](objects/helpers/standard_test_pool.py) | the LiveObjects fixtures, from `uts/objects/helpers/standard_test_pool.md`; see the objects tier below | `SKILL.md` lists every name in each. The helpers have their own tests -(`helpers/*_test.py`), which are not derived from a specification and are not counted -in the derived-test totals. +(`helpers/*_test.py`, `objects/helpers/standard_test_pool_test.py`), which are not derived +from a specification and are not counted in the derived-test totals. ## The integration tier @@ -253,6 +255,55 @@ the session; the modules here sign an Ably JWT locally instead, through a file-l nothing to the log a test is counting. Tests in this package are given 300 seconds each, as in the REST proxy package. +## The objects tier + +`objects/` holds the LiveObjects specifications, laid out as the other two: +`objects/unit/`, fifteen specifications that reach no network; `objects/integration/`, three +against the sandbox; and `objects/integration/proxy/`, one through `uts-proxy`. Seven of the +unit specifications are pure — they build `InternalLiveCounter`, `InternalLiveMap`, +`ObjectsPool` or a channel-less `RealtimeObject()` and connect nothing — and the other eight +drive `channel.object` over the mock websocket. The specifications are written against an +untyped `PathObject` and `Instance`; the tests reach a type's methods through LODR-061's +views, `root.get('score').as_live_counter().value()`, which [deviations.md](deviations.md) +explains along with the shapes the pure tier adapts to (S-1 to S-5). + +Everything the objects specifications share is in +[objects/helpers/standard_test_pool.py](objects/helpers/standard_test_pool.py), named as +`standard_test_pool.md` names it: + +| Helper | Is | +|---|---| +| `setup_synced_channel(channel_name='test', mock_ws=None, clock=None, modes=OBJECTS_MODES, **client_options)` | the specifications' synced channel: a client on `standard_mock_websocket()`, the channel with both object modes, and `await channel.object.get()`. Unpacks as `client, channel, root, mock_ws`. `setup_synced_channel_no_ack` records OBJECT messages without ACKing them | +| `standard_mock_websocket(auto_ack=True, on_object=None, connected=None, ...)` | answers an ATTACH with an ATTACHED and an OBJECT_SYNC of `STANDARD_POOL_OBJECTS`, a DETACH with a DETACHED, and each OBJECT, after `on_object`, with an ACK | +| `objects_client(mock_ws, clock=None, mock_http=None, **kwargs)` | the client underneath: it connects on its own, speaks JSON, and has `GET /time` answered by `time_mock_http(clock)`, since creating an object reads the server time | +| `objects_channel_options(*modes)`, `objects_connected_message(...)`, `objects_attached_message(channel, channel_serial, flags)` | the channel options and harness messages. Granted modes are `flags` bits, `HAS_OBJECTS \| OBJECT_SUBSCRIBE_FLAG`, not a `modes` list | +| `build_counter_inc`, `build_map_set`, `build_map_remove`, `build_map_clear`, `build_object_delete`, `build_counter_create`, `build_map_create`, `build_object_state`, `build_object_message`, `build_object_sync_message`, `build_ack_message`, `json_value`, `bytes_value` | the builders, each returning the JSON-wire dictionary `send_to_client` takes: camelCase keys, numeric actions, `json` values as JSON strings, `bytes` as base64 | +| `ack_serial(msg_serial, index)`, `remote_serial(index)`, `below_ack_serial(index)`, `POOL_SERIAL` | serials that sort where the specifications need them. A bare `'99'` sorts before `POOL_SERIAL` and is rejected as stale | +| `object_message(wire)`, `object_messages(protocol_message)`, `capture_updates(obj)` | the pure tier's: a builder's output decoded to the internal type, and the updates an object emits, since `apply_operation` returns a boolean | +| `build_public_object_message(message, channel_name)` | the public `ObjectMessage` a subscription event should carry, built independently of the library | +| `assert_unchanged_after_quiescence(count_under_test, control_delivered)` | the specifications' negative-assertion pattern | +| `provision_objects_via_rest(api_key, channel_name, operations)` | REST provisioning for the integration tier, over `X-Ably-Version: 6` | + +The client reads frames on a task of its own, so a test reads state after a `poll_until` on +the frame's effect, never straight after `send_to_client`, and asserts a negative or an exact +count only once a positive control behind it has arrived and the loop has settled. + +```python +async def test_rtpo17_increment_delegates_to_counter(): + client, channel, root, mock_ws = await setup_synced_channel('test') + + await root.get('score').as_live_counter().increment(25) + + assert root.get('score').as_live_counter().value() == 125 +``` + +`objects/integration/` provisions its own sandbox app from its own `conftest.py`, separate +from the realtime tier's but under the same fixture name, `realtime_sandbox`. All three of +its specifications carry Protocol Variants, so every test takes `use_binary_protocol`, and +each is given 120 seconds, as in the other integration tiers. Its `proxy/` package repeats +the realtime proxy package's `proxy_control` and `proxy_session` fixtures and 300-second +timeout, and its clients sign an Ably JWT locally, as the realtime proxy modules do. + ## Running ``` @@ -262,15 +313,16 @@ uv run --frozen --extra crypto --extra dev pytest test/uts -q The offline tiers alone, which need no network: ``` -uv run --frozen --extra crypto --extra dev pytest test/uts/rest/unit test/uts/realtime/unit test/uts/helpers -q +uv run --frozen --extra crypto --extra dev pytest test/uts/rest/unit test/uts/realtime/unit test/uts/objects/unit test/uts/helpers test/uts/objects/helpers -q ``` -Either integration tier alone, each of which provisions a sandbox app and needs network +Any integration tier alone, each of which provisions a sandbox app and needs network access: ``` uv run --frozen --extra crypto --extra dev pytest test/uts/rest/integration -q uv run --frozen --extra crypto --extra dev pytest test/uts/realtime/integration -q +uv run --frozen --extra crypto --extra dev pytest test/uts/objects/integration -q ``` `--frozen` is required: without it dependency resolution reaches past the diff --git a/test/uts/deviations.md b/test/uts/deviations.md index 7e97de4d..5c70d912 100644 --- a/test/uts/deviations.md +++ b/test/uts/deviations.md @@ -24,26 +24,32 @@ One Test ID can become more than one derived test: five Test IDs in `rest/unit` `error_types_test.py`, `fallback_test.py`, `rest_client_test.py` (two) and `paginated_result_test.py` — assert several independent things under a single id, and the derivation writes a function for each rather than one function with an unrelated -second half. That turns 1132 Test IDs into 1141 derived tests. Going the other way, one +second half. That turns 1471 Test IDs into 1480 derived tests. Going the other way, one derived test can become more than one case: five of the twelve `rest/integration` -specifications and five of the twenty `realtime/integration` ones carry a `## Protocol -Variants` section and run every one of their tests twice, once per protocol, and nine -`rest/unit` tests are parametrized over a table of fixtures the specification gives -inline. That turns 1141 derived tests into 1234 pytest cases. - -Of **1132 Test IDs, derived as 1141 tests and run as 1234 pytest cases**: 948 Test IDs -(957 tests, 1048 cases) pass, 169 (169 tests, 171 cases) are gated behind +specifications, five of the twenty `realtime/integration` ones and the three +`objects/integration` ones outside `proxy` carry a `## Protocol Variants` section and run +every one of their tests twice, once per protocol; nine `rest/unit` tests are +parametrized over a table of fixtures the specification gives inline; and five +`objects/unit` tests are parametrized, three over a table and two over the scenarios the +specification loops through. That turns 1480 derived tests into 1607 pytest cases. + +Of **1471 Test IDs, derived as 1480 tests and run as 1607 pytest cases**: 1287 Test IDs +(1296 tests, 1421 cases) pass, 169 (169 tests, 171 cases) are gated behind `RUN_DEVIATIONS`, and 15 (15 tests, 15 cases) cannot be run at all. The three groups are -disjoint: two Test IDs, and one parametrized test, have a gated part and a passing part, -and are counted with the gated. Every gated test has been confirmed to fail when -enabled, so none of them passes under both behaviours. 494 of the Test IDs come from -`uts/rest/unit` (503 tests, 536 cases), 481 from `uts/realtime/unit` (481, 481), 84 -from `uts/rest/integration` (84, 122) and 73 from `uts/realtime/integration` (73, 95); 8 -of the REST integration ids (8, 8) and 30 of the realtime ones (30, 30) come from the -`proxy` package within each. Of the gated Test IDs 79 are REST and 90 realtime, which is -80 REST cases and 91 realtime. -A further 130 pytest cases under `helpers/` cover the mock infrastructure itself and are -not derived from a specification. +disjoint: two Test IDs have a gated part and a passing part — `TI1`, as two functions, and +`TO/endpoint-affects-host`, as one parametrized test — and are counted with the gated, +while their passing parts count among the passing tests and cases. Every gated test has +been confirmed to fail when enabled, so none of them passes under both behaviours. 494 of +the Test IDs come from `uts/rest/unit` (503 tests, 536 cases), 481 from +`uts/realtime/unit` (481, 481), 322 from `uts/objects/unit` (322, 344), 84 from +`uts/rest/integration` (84, 122), 73 from `uts/realtime/integration` (73, 95) and 17 from +`uts/objects/integration` (17, 29); 8 of the REST integration ids (8, 8), 30 of the +realtime ones (30, 30) and 5 of the objects ones (5, 5) come from the `proxy` package +within each. Of the gated Test IDs 79 are REST and 90 realtime, which is 80 REST cases +and 91 realtime. None of the 339 objects Test IDs is gated or unrunnable. +A further 148 pytest cases cover the harness itself and are not derived from a +specification: 130 under `helpers/`, for the mocks, and 18 under `objects/helpers/`, for +the objects builders and the synced-channel mock. The 158 gated Test IDs that record SDK non-compliance — 158 tests, 160 cases — reduce to **70 distinct root causes**, 26 on the REST side and 44 on the realtime side. Three further @@ -51,8 +57,10 @@ defects are recorded below with no test of their own, because the specification' cannot discriminate (RTP18a), has nothing to assert against (the timezone split on synthesized LEAVE timestamps), or is worked around in the setup of every test that would otherwise trip over it (`enterClient` on an anonymous connection), so the file -carries **73 SDK root causes** in all. The remaining 10 gated Test IDs are -specification faults, and reduce to 7. +carries **73 SDK root causes** in all. The remaining 11 gated Test IDs are +specification faults, and reduce to 8. The objects tier records no SDK non-compliance; +its specification faults are all corrected or worked around in tests that pass, and +reduce to the seven candidate issues O.1–O.7, none of them filed. Entries closed by a fix are removed rather than kept as history; `git log` holds that. @@ -75,7 +83,7 @@ from the specification text, so that correcting the specification is all it take it pass, and it is marked `@spec_error` — a skip gated on `RUN_DEVIATIONS`, the same gate `@deviation` uses, with a reason naming the specification rather than the SDK. The suite stays green, a real regression still shows, and the failure is one environment variable -away. Each is filed upstream, in the issues named below. Eleven tests are gated +away. Each but RSC7d's is filed upstream, in the issues named below. Eleven tests are gated this way: | Test | Spec error | @@ -99,7 +107,8 @@ site. The entries below cover both kinds and say which applies. Almost every rea fault is of the second kind, which is why only one realtime test is gated as a spec error while fourteen `realtime/unit` entries appear below. The eight `realtime/integration` faults are of that kind without exception, so none of them is -gated either. +gated either, and so is every `uts/objects` fault: the objects entries close this +section, and not one objects test is gated. The three sections that follow this one record SDK behaviour rather than specification faults. @@ -144,8 +153,11 @@ step or a header label — so the derived test keeps the corrected fixture and p none of them is among the eleven gated above. Not every entry has an issue of its own: the URL-safe base64 alphabet is recorded below -and not filed, because ably-python's own encoding settles the tests either way. Line -references in these entries are against `ably/specification@d9a04ca`. +and not filed, because ably-python's own encoding settles the tests either way. None of +the `uts/objects` faults is filed yet; *Candidate issues* classifies them as O.1–O.7 for a +maintainer to file. Line references in these entries are against +`ably/specification@d9a04ca`, and in the `uts/objects` entries against +`ably/specification@747796f`. ### `/time` is stubbed as an object rather than an array @@ -799,10 +811,213 @@ Filed as [#554](https://github.com/ably/specification/issues/554). | `batch_presence.md` | The restricted-key setup's comment reads "only has access to \"batch-allowed\" channel" while the setup fixes `allowed_channel = "channel6"`; `batch-allowed` appears nowhere in the file. Filed with [#548](https://github.com/ably/specification/issues/548), whose fix replaces the same lines | | `batch_presence.md` | BGR2 says a channel with no members "returns a success result with an empty `presence` array", and the unit tier's mocks all send `'presence': []`. The server sends no `presence` key at all, so an implementation has to default the field for the assertion to hold. The derived test asserts the specification's `length == 0`, with the wire shape in a comment. Filed as [#550](https://github.com/ably/specification/issues/550) | +### The objects unit specifications model three internal interfaces against `objects-features.md` + +**Spec points:** RTLC7, RTLM15, RTO3–RTO9, RTO17, RTLCV4g5, RTLMV4j5, PAOOP3. Not yet +filed; candidate O.1. + +A conforming SDK passes these only by adapting, so each is also a shape adaptation under +*Adapted Tests*, which defines the labels S-1, S-2 and S-4. + +- `internal_live_counter.md` and `internal_live_map.md` read + `update = obj.applyOperation(...)` and, of the same call elsewhere in the same files, + assert `result == true` or `false`. RTLC7g and RTLM15g make the return a boolean and + have the update *emitted* (RTLC7d1a, RTLM15d1a and their siblings), so the files + contradict themselves and `objects-features.md` settles it. (S-1) +- `objects_pool.md`, and the three RTO5c10 cases of `parent_references.md`, put the sync + and apply state machine — `processAttached`, `processObjectSync`, + `processObjectMessage`, `applyObjectMessages`, `syncState` — on `ObjectsPool`, and build + `RealtimeObject(pool: pool)`. RTO3a makes the pool a `Dict`, and + RTO7a, RTO7b and RTO17 give the sync state, the buffer and `appliedOnAckSerials` to + `RealtimeObject`. `uts/objects/PLAN.md` repeats the pool model. (S-2) +- `value_types.md` reads the create an evaluated value type retains as + `operation.counterCreate` / `operation.mapCreate`, and `public_object_message.md` reads + it as `derivedFrom` on the `*CreateWithObjectId` payload. RTLCV4g5 and RTLMV4j5 say only + that it is retained alongside, locally, and is not sent — which the first reading would + put on the wire. The same file's PAOOP3b2 and PAOOP3c2 sources give + `mapCreateWithObjectId` the fields `objectId`, `semantics` and `entries`, and + `counterCreateWithObjectId` `objectId` and `count`, where MCRO2 and CCRO2 define only + `initialValue` and `nonce`. (S-4) + +**Tests affected:** the 42 tests in `internal_live_counter_test.py` (13) and +`internal_live_map_test.py` (29) that read an update or a `false`; 30 in +`objects_pool_test.py` and 3 in `parent_references_test.py`; 8 in `value_types_test.py`; +and `test_paoop3_map_create_from_with_object_id` and +`test_paoop3_counter_create_from_with_object_id`, which build the payload MCRO2 and CCRO2 +describe, with `initialValue` the JSON of the derived payload and the derived payload in +`derived_from`. All pass. + +### Objects fixtures that put a value where the protocol does not + +**Spec points:** RTO2, RTO25a, RTO26a, RTLMV4d, RTLMV4f1, and `standard_test_pool.md`. Not yet +filed; candidate O.2. + +| Spec | Fixture | Why it cannot hold | Derived as | +|---|---|---|---| +| `realtime_object.md` RTO2 `mode-enforcement-0`, RTO25a `access-requires-subscribe-mode-0`, RTO26a `write-requires-publish-mode-0` | An ATTACHED carrying `modes: ["OBJECT_SUBSCRIBE"]` | The modes a server grants travel as bits of the ATTACHED's `flags` (RTL4m, TR3), which is where a client decodes them; a `modes` field is read by nothing | `flags: HAS_OBJECTS \| OBJECT_SUBSCRIBE_FLAG`, through `objects_attached_message(..., flags=...)` | +| `standard_test_pool.md`'s CONNECTED, and the hand-written ones in `internal_live_counter_api.md` and `internal_live_map_api.md` | `connectionId` inside `connectionDetails` | `connectionId` is a ProtocolMessage field, which is where a client reads it | `objects_connected_message()` puts it on the message | +| `value_types.md` `RTLMV4d/map-set-all-types-table-0` | Each row compares the in-memory MapCreate's entry with a decoded value — the json rows expect `[1, "a", null]` and `{"k": "v"}` — except the bytes row, whose `"AQID"` is the JSON wire's base64 | In memory the entry holds bytes; base64 is what `initialValue` carries (RTLMV4f1). ably-js's derived test reads `initialValue` for that reason | `bytes([1, 2, 3])` on the retained MapCreate and `"AQID"` in the decoded `initialValue`, so both readings are pinned. The row should read `bytes([1, 2, 3])` | +| `standard_test_pool.md`'s `provision_objects_via_rest` | "a JSON array of result objects (one per batch entry)" | Measured: one result object for the whole batch, whose `objectIds` lists every object created or updated. `X-Ably-Version` 5 and 6 both accept the v6 operation format | `provision_objects_via_rest` flattens either shape | + +### Objects mock-tier steps read state as though an injected frame were applied at once + +**Spec points:** RTO5c9, RTO10c1b1, RTO17, RTO18d, RTO19, RTO20, RTO20d4, RTO20e, RTO20e1, +RTO23c1, RTO24b1, RTO24c1, RTINS16, RTINS16h, RTLO4b, RTPO19, and at the integration tier +RTO5a2, RTO7, RTO8 and RTO17. Not yet filed; candidate O.3. + +`path_object.md`'s own compact tests say that an SDK may apply an inbound OBJECT message +asynchronously, and wait for it; `standard_test_pool.md` defines a quiescence pattern for +negative assertions. The steps below use neither, and are written as though +`send_to_client` returned with the frame applied. A client that reads frames on a task of +its own, as ably-python does, meets four shapes: + +| Shape | Where | What it costs | +|---|---|---| +| A read or a negative straight after `send_to_client` | `RTO20/echo-dedup-0`, `RTO5c9-RTO20/ack-serials-cleared-on-resync-0`, `RTO19/off-deregisters-0`, `RTO10c1b1/gc-root-never-removed-0`; and `RTINS16h/subscribe-no-side-effects-0`, straight after `subscribe` | Each reads the state from before the frame, so the negatives — no double apply, no callback, the root not removed, no side effect — pass whether or not the SDK is right | +| An operation started before the ATTACHED that should hold it back has been processed | the three `RTO23c1` tests, the two `RTO20e1` tests, `RTO20e/waits-for-synced-0`, `RTO20d4/empty-synthetic-list-skips-sync-wait-0` | `get()` or the write still sees SYNCED and resolves at once, so a **conforming** SDK fails `IS NOT complete`. ably-js's derived tests insert `flushAsync()` at that point | +| A seed sent on the line before the subscription it must not reach | `RTO24c1/depth-filtering-formula-0`, `RTO24b1/multi-path-dispatch-0` | A seed applied after the subscription is delivered to it, and `events.length == 2` reads 3, `events_alias.length == 1` reads 2 | +| An exact count straight after `poll_until(count >= n)` | throughout `live_object_subscribe.md` and `path_object_subscribe.md`, `RTINS16/subscribe-receives-events-0`, and `RTO18d`, `RTO24c1` and `RTO17-RTO18` in `realtime_object.md` | The "exactly" half is a negative, read before a second, wrong event could arrive | + +`objects_faults.md` has the integration tier's version. `RTO5a2-RTO17/sync-interrupted-reconnect-0` +and `RTO7-RTO8/mutations-buffered-during-resync-0` `AWAIT_STATE` DISCONNECTED, which RTN15a +leaves at once — `writing-test-specs.md`'s *Verifying Transient States* says not to — and then +CONNECTED, which the client may still hold from before the drop. `RTO17/server-detach-resync-0` +awaits ATTACHED on a channel that has not yet processed the DETACHED injected onto it, so the +wait returns at once and says nothing about the re-attach. + +The derived tests keep every assertion and add the barrier: a `poll_until` on the frame's +effect before a read; for a negative, a positive control delivered behind the message under +test and `assert_unchanged_after_quiescence` (echo-dedup's control is an increment of the +nested counter, RTO10c1b1's an OBJECT_DELETE of the score counter, RTO19's a second SYNCED +listener); `_restart_sync`, which polls until `_sync_state` is SYNCING before the operation +starts; a poll on the seeded key before subscribing; and `await settle()` between a positive +poll and an exact count. The integration tests wait on a state recorder registered before the +fault, as the realtime tier does. The specifications want `process_pending_events()` after +each injected frame they then read from, and their own quiescence pattern before each negative. + +### Objects tests that pass without the behaviour they are named for + +Not yet filed; candidate O.4, the same class as +[#543](https://github.com/ably/specification/issues/543). + +Where the fix is an assertion the specification omits, the derived test carries every one of +the specification's assertions and adds it, with a `# UTS SPEC ERROR:` comment or a comment +citing the spec point at the site — the RSL2b3 precedent above. "A mutant passed" means +measured: a throwaway reference implementation with that single fault passed every assertion +the specification makes. + +| Test ID | Why it cannot fail | What the derived test adds | +|---|---|---| +| `RTLM8g/map-remove-clear-timeserial-floor-0` | The entry's own timeserial, `"04"`, is later than the MAP_REMOVE's `"03"`, so RTLM8a1 and RTLM9e reject it whether or not the RTLM8g floor exists. A mutant ignoring the floor passed | A MAP_REMOVE of a key with no entry at `"04"`: at or below the floor and newer than the site's serial, so only RTLM8g can reject it | +| `RTLM9b/both-empty-reject-0` | Its own note: RTLO4a3 rejects the empty `ObjectMessage.serial` first, so RTLM9b is never reached | `InternalLiveMap.can_apply_map_operation('', '')` and `(None, None)` are False, checked directly | +| `RTO10/gc-tombstoned-objects-0`, `RTO10c1b1/gc-root-never-removed-0` | `score` reads null from the moment the OBJECT_DELETE applies (RTLM14c, RTLM5d2h), so both pass with no GC sweep at all | That the counter has left `_objects_pool`; for RTO10c1b1, a control OBJECT_DELETE whose removal shows a sweep ran | +| `RTO10b1/gc-grace-period-source-0` | As RTO10, and it advances 6000 ms, less than one GC interval — RTO10a's example is five minutes — so no sweep runs. Its stated claim, that it fails if `objectsGCGracePeriod` is ignored, cannot hold | `_gc_interval_ms = 1000` before `get()`, `_gc_grace_period_ms == 5000`, and the counter still in the pool at 4000 ms and gone at 6000 ms | +| `RTPO6/at-escaped-dots-0` | `at("a\\.b.c").path() == "a\\.b.c"` holds for an `at` that splits on every dot too, since `path()` renders `['a\\', 'b', 'c']` back the same way | `_path == ['a.b', 'c']` | +| `RTPO19/map-clear-triggers-child-events-0` | The MAP_CLEAR's `"99"` sorts before the pool's `"t:0"`, so RTLM24e1 removes nothing; and nothing subscribes at a child path, which is what the title is about | The clear at `remote_serial(0)`, and a subscription at `name` that must receive an event for `name` | +| `RTO4b/attached-no-objects-synced-0` | Lists RTO4b2's "must not create a new root"; a mutant replacing the root passed | `pool['root'] is root` | +| `RTO14/objectid-format-counter-0` | Lists RTO14b1's SHA-256 of `[initialValue]:[nonce]` and asserts only the alphabet; a mutant hashing the wrong input passed | The digest, computed in the test, and that the hash decodes | +| `RTINS16g/subscription-follows-identity-0` | Nothing checks that the MAP_SET repointing `score` applied, so a rejected repoint passes | That `score` resolves to the new counter, that the event is the increment of 10, and that the counter reads 110 | +| `RTO20e/waits-for-synced-0` | `inc_future IS NOT complete` is asserted before the publish could have been sent | Pending only once the OBJECT has left the client and its ACK has been processed | +| `RTO23c/get-waits-for-synced-0` | Asserts only the eventual result, which a `get()` that does not wait also produces | `get()` still pending after the ATTACHED has been processed | +| `RTO20c/missing-site-code-0`, `RTO20d1/null-serial-skipped-0` | "The value stays 100" also holds if nothing was published | Exactly one OBJECT was published | +| `RTO20e1/fails-on-channel-detached-0`, `RTO20e1/fails-on-channel-failed-0` | RTO20e1's text gives the status code, and for FAILED the cause, and the assertions read neither | `status_code == 400`, and `cause.code == 90000` for FAILED | + +Six more are derived as written, there being no assertion to add that is not the +specification's to choose: + +- `objects_pool.md`'s `RTO4-RTO5/attached-during-syncing-resets-0` gives the second sync a new + sequence id, so RTO5a2 discards the old objects whether or not the ATTACHED reset anything; + only removing RTO5a2a's clear made it fail. `objects-features.md` does not say that an + ATTACHED with HAS_OBJECTS resets a sync in flight either; see the features entry below. +- `objects_faults.md`'s `RTO7-RTO8/mutations-buffered-during-resync-0` and + `RTO5-RTO7/publish-during-sync-echo-after-0` are titled for the buffering of RTO7 and RTO8 + and never exercise it. Measured: in the first, client B resumes about 60 ms after the + proxy's disconnect and the server delivers A's write inside the resync data, with no OBJECT + after the resume; in the second, uts-proxy's `delay` holds every later frame behind the + delayed OBJECT_SYNC, so A's OBJECT reaches B once B is SYNCED and is applied directly. + Neither can fail on a missing RTO8a buffer. +- `internal_live_counter_api.md`'s `RTLC11/counter-update-on-inc-0` cites RTLC11b1, the + update's `amount`, and asserts the inbound operation's `counterInc.number`, which is what + `Instance#subscribe`'s event carries; RTLC11b1 is covered at the pure tier only. +- `live_object_subscribe.md`'s `RTLO4b/subscribe-map-update-0` states that the update names + the key and asserts only `updates.length == 1`. +- `objects_pool.md`'s `RTO5c10/empty-sync-parent-refs-0` asserts + `pool["root"].parentReferences == {}`, which holds in every state the test can build; the + rest of the test has substance. +- `objects_lifecycle_test.md`'s `RTO23-RTPO15/set-primitive-propagates-0` subscribes + `events_b` and never reads it. + +### Boundaries no objects test reaches + +**Spec points:** RTLM7h, RTLM8g, RTLM24c, RTLM19a1, RTLC7b, RTLC7c, RTLC7e, RTLM15b, RTLM15c, +RTLM15e, RTLC6a, RTLC6e, RTLM6a, RTLM6e, RTLM7a1, RTLM7a3, RTLM7g, RTLM8a3, RTLM23c, RTLO6b, +RTINS3b. Not yet filed; candidate O.5. + +Missing tests rather than faulty ones. Each but the last was found as a single-point mutant +of a reference implementation that every test in `internal_live_counter.md` and +`internal_live_map.md` let through; the last is a point a specification's table lists and no +assertion reads. The implementation was checked against all of them outside the suite. The +derived tests do not add them, since a test with no Test ID cannot be traced back. + +| Spec point | Not covered | +|---|---| +| RTLM7h, RTLM8g | A clear timeserial **equal** to the operation's serial rejects the operation | +| RTLM24c | An equal clear serial is **not** stale; only a strictly greater one is | +| RTLM19a1 | An entry exactly `gracePeriod` old is released; the fixture uses `gracePeriod + 1` | +| RTLC7c before RTLC7e, RTLM15c before RTLM15e | A tombstoned object still records the CHANNEL serial before it rejects the operation | +| RTLC6a before RTLC6e, RTLM6a before RTLM6e | `replaceData` on a tombstoned object still replaces `siteTimeserials` | +| RTLM7a1 and RTLM7h before RTLM7a3 and RTLM7g, and RTLM8a3 | A MAP_SET or MAP_REMOVE rejected by LWW or by the clear floor keeps the old child's parent reference and creates no zero-value object | +| RTLC7b, RTLM15b | A rejected operation leaves `siteTimeserials` untouched | +| RTLM23c | A no-op entry inside a MAP_CREATE is left out of the merged update | +| RTLO6b | A MAP_REMOVE or `replaceData` tombstoning a map entry with no `serialTimestamp` reads the local clock | +| RTINS3b | `instance.md`'s `RTINS3/id-returns-objectid-0` lists it — `id` is null for a primitive — and asserts nothing about it | + +### Objects labels, commentary and harmless fixture slips + +Not yet filed; candidate O.6, the housekeeping class of +[#532](https://github.com/ably/specification/issues/532). No assertion depends on any of them. + +| Spec | Fault | +|---|---| +| `internal_live_map.md` `RTLM24/map-clear-preserves-newer-0` | The "Spec requirement" line removes entries with a timeserial `<=` the clear's serial; RTLM24e1 removes only strictly older ones, and the same file's `map-clear-basic-0` keeps the equal entry. The fixture has no equal entry | +| `internal_live_map.md` `RTLM14c/tombstoned-ref-yields-null-0` | Titled "MAP_SET referencing tombstoned objectId"; the body seeds `map.data` directly and performs no MAP_SET | +| `realtime_object.md` `RTO20d4/mixed-null-serials-applies-non-null-0` | The commentary has `child` dangle and read undefined per RTLM5d2f1. Applying the MAP_SET creates the zero-value counter (RTLM7g1) and references it (RTLM7g2), so `child` resolves to a counter reading 0. `LiveCounter.create(5)` in the fixture would let the test show the create was skipped | +| `realtime_object.md` `RTO19/off-deregisters-0` | Calls `sub.off()` on what `on()` returned, which is RTO18f2. `RealtimeObject#off` itself, RTO19, is exercised by no test in the file | +| `realtime_object.md` `RTO20/ack-after-echo-no-double-apply-0` | The requirement line credits RTO9a3 with the single apply; with the echo first it is the RTLO4a newness check on the LOCAL apply that rejects the second. The assertion is the same either way | +| `path_object.md` `RTPO6b/at-non-string-throws-0` | Asserts 40003 for a path that is not a string, where `objects-features.md`'s RTPO6 defines no error; RTPO5b gives one for `get` only. Derived as written, with a note at the site, 40003 being what RTPO5b gives the same mistake | +| `internal_live_map.md` `RTLM15e/tombstoned-reject-ops-0` | Sets `isTombstone` without `tombstonedAt`, breaking RTLO3e's invariant that one is set exactly when the other is. Kept as written; the counter twin sets both | +| `internal_live_map.md`, the pool-backed tests (RTLM7g, RTLM14, RTLM14c, every `parentReferences` test) | Build `InternalLiveMap(objectId: "root", pool: pool)` beside the root `ObjectsPool()` already holds (RTO3b1), so the map under test is a second `root`, and the parent references it writes resolve to the pool's. The module docstring says so | +| `objects_pool.md`, `parent_references.md` | Every ATTACHED carries a `channelSerial`, and `RTO5-RTO7/new-sync-keeps-buffer-0` calls `seq2:` new relative to it. RTO4 reads only the HAS_OBJECTS flag; the sync cursor is the OBJECT_SYNC's (RTO5a1) | +| `path_object_subscribe.md` | Mixes `remote_serial(n)` with bare `"98"`, `"99"`, `"100"`, `"50"` and `"serial-1"` from one site. Each applies only because no test sends two of them to one object: `"100"` sorts before `"99"` | +| `value_types.md` `RTLMV4a/evaluate-validates-entries-0` | Carries no language note for `LiveMap.create(null)`, where RTLC12e1's table and RTLMV4b both carry one; see *Adapted Tests* | +| `internal_live_map_api.md` `RTLM20/set-invalid-values-table-0` | Its invalid values — a function, `undefined`, a symbol — are JavaScript's; see *Adapted Tests* | +| `uts/docs/integration-testing.md` | Names `objects_lifecycle_test.md` and `objects_sync_test.md` as the objects specifications with Protocol Variants; `objects_gc_test.md` carries the section too, and runs both | + +### `objects-features.md` gaps the derivation and implementation met + +Not faults in a UTS specification, and not yet filed; candidate O.7. The right-hand column is +what ably-python does, and names ably-js where it agrees. + +| Spec point | Gap | ably-python | +|---|---|---| +| RTLM5d2 | Lists boolean, bytes, number, string and objectId, so an `ObjectData.json` entry falls to RTLM5d2g and reads null, though OD2g defines `json` and `path_object.md`'s compact test expects the decoded value | The decoded JSON, as ably-js | +| OD4c5, OD4d5 | Still put a JSON payload in `string` with `encoding: "json"`, where OD2g and ably-js use the `json` field, a JSON string on both wires | Sends `json`, decodes both | +| RTO4 | With HAS_OBJECTS set, does RTO4c and RTO4d only; whether a sync in flight, its SyncObjectsPool and its sequence id are abandoned is unsaid | Abandons them, as ably-js | +| RTO5a5, RTO5a6 | Whether an OBJECT_SYNC with no parseable `channelSerial` discards a sequence already accumulating | Discards it | +| RTO5d | "Skip processing that `ProtocolMessage`" when an ObjectMessage has no `object`. Skipping the ProtocolMessage loses its cursor and the sync never completes; the UTS and ably-js skip the ObjectMessage | Skips the ObjectMessage | +| RTO4b2 | Resets the root's data to RTLM4c's zero value, and is silent on `clearTimeserial` | Resets it with the data | +| RTO14b2 | Cites RFC 4648 §5, which allows padding; the UTS and ably-js require none | No padding | +| RTO2a | Checks the granted modes when they are "populated"; an ATTACHED with no mode bits decodes to none at all | No granted modes counts as unpopulated, so the requested ones are checked (RTO2b), as ably-js. A real ATTACHED always carries the bits; the standard mock's does not, nor does `objects_faults.md`'s injected `flags: 128` | +| RTPO6 | No error for a path that is not a string | 40003, as RTPO5b gives `get` | +| RTLMV4c1 | A live object, PathObject or Instance as a value is 40013 only in the unmerged [#491](https://github.com/ably/specification/pull/491) | 40013 | +| RTPO20c ([#471](https://github.com/ably/specification/pull/471)) | 92007 when the path "does not resolve to a `LiveObject`"; ably-js resolves through `_resolvePath` and gives **92005** for a path that does not resolve | 92007, as written | +| RTBC16d ([#471](https://github.com/ably/specification/pull/471)) | Publishes a batch through RTO15, so its writes would apply only on their echo | Through RTO20, applying on ACK like every other write, as ably-js | + ## Failing Tests The specification's assertion is preserved and gated behind `@deviation`. Removing -the mark is the only change needed once the SDK behaviour lands. +the mark is the only change needed once the SDK behaviour lands. Nothing derived from +`uts/objects` is gated, so no LiveObjects entry appears in this section. ### Unimplemented features @@ -2199,10 +2414,56 @@ pseudocode is not mistaken for non-compliance. | RTC17 (RSA7b1) | `client.clientId == client.auth.clientId` | `AblyRealtime.client_id` reads `options.client_id` and returns the configured value, while `Auth.__init__` sets `self.__client_id = None` whenever `ably._is_realtime` (`rest/auth.py:34-41`), deferring it to whatever a CONNECTED confirms. The two disagree on a client that has not connected | Open bug. RSA12b only allows the realtime clientId to be unknown while it has not been *configured* | | RTC1f | a `transportParams` boolean appears as `"true"` / `"false"` | `True` / `False`, because `WebSocketTransport.connect` builds the query string with `urllib.parse.urlencode`, which renders each value through `str()` (`websockettransport.py:89`). Integers are unaffected | Open bug. A caller can pass the strings directly, but a bool is what the spec's Stringifiable type admits | +### LiveObjects: the internal shapes the unit specifications assume — S-1 to S-5 + +The objects unit specifications drive internal classes, and ably-python's differ in shape from +the ones they are written against — three of them because the specification contradicts +`objects-features.md` (the first UTS Spec Error entry on the objects tier), two because of how +ably-python is built. Each shape has a label, cited in the module docstring and at the site, so +that one entry here covers every test it affects. The behaviour each test asserts is the +specification's; only the reach into the object differs. + +| Label | The specification | ably-python | Tests | +|---|---|---|---| +| S-1 | `update = obj.applyOperation(...)`, or `result == false` | `apply_operation` returns whether it applied (RTLC7g, RTLM15g) and emits the update through `notify_updated`, no-ops included. `capture_updates(obj)` records them: a specification's `update` is the one update an applied operation emitted, and its `false` is a False return with nothing emitted | the 42 in `internal_live_counter_test.py` and `internal_live_map_test.py` that read an update or a `false` | +| S-2 | The sync and apply state machine on `ObjectsPool`: `processAttached`, `processObjectSync`, `processObjectMessage`, `applyObjectMessages`, `syncState` | On `RealtimeObject`, as in `objects-features.md`: `_on_attached(has_objects)`, `_handle_object_sync_messages(messages, channel_serial)`, `_handle_object_messages`, `_apply_object_messages`, `_sync_state`. A bare `ObjectsPool()` that is then driven is the pool a standalone `RealtimeObject()` holds; `RealtimeObject(pool: pool)` is `RealtimeObject(pool=pool)` | 30 of the 31 in `objects_pool_test.py`, every one but RTO3's; the three RTO5c10 cases in `parent_references_test.py` | +| S-3 | `evaluate(vt)` | `evaluate(vt, timestamp_ms)`: evaluation needs the server time (RTLCV4e, RTLMV4h), which is an async read (RTO16), so the caller fetches it and passes it in. The tests pass a fixed one | the 13 in `value_types_test.py` that evaluate | +| S-4 | The create an evaluated value type retains, as `operation.counterCreate` / `operation.mapCreate` | `operation.resolved_counter_create` / `resolved_map_create`, the `derived_from` of the `*CreateWithObjectId` payload. The two retention tests also assert that `counterCreate` / `mapCreate` is absent from `operation.to_dict()`, which is what "not sent" means and what keeps this reading honest | `test_rtlcv4g5_retains_local_counter_create`, `test_rtlcv4_evaluate_zero_count`, `test_rtlmv4j5_retains_local_map_create`, `test_rtlmv4d_entry_value_types`, `test_rtlmv4d1_nested_value_types`, `test_rtlmv4a_evaluate_validates_entries`, `test_rtlmv4e2_empty_entries`, `test_rtlmv4d_map_set_all_types_table` | +| S-5 | Three reads on the untyped `Instance` that answer null or fail: `value()` on a map (RTINS4d), `size()` on a counter (RTINS9c), and `subscribe` on a primitive failing 92007 (RTINS16c) | The typed instances have no such method (RTTS10a–c; `PrimitiveInstance` has no `subscribe` at all, RTTS7b), and the checked views that do have it refuse the wrapped type with 92007 (RTTS9d). The tests assert the wrapped `type`, that the method is absent, and the 92007 from each view in its place | `test_rtins4_value_counter` and `test_rtins9_size` (their second halves), `test_rtins16c_subscribe_primitive_throws` | + +**Status:** shape adaptations, not defects. S-1, S-2 and S-4 close when the specifications are +corrected (candidate O.1); S-3 and S-5 are how ably-python is built. + +### LiveObjects: inputs that do not exist in Python + +| Test | The specification | The test | Status | +|---|---|---|---| +| `test_rtlmv4a_evaluate_validates_entries` | `LiveMap.create(null)` fails 40003 | `LiveMap.create(entries=None)` is LODR-061's default, so `None` is the omitted argument and evaluates to an empty map, which the test pins. The reachable half of RTLMV4a, a non-dict, is asserted as 40003 | Flips if `create` takes a sentinel default instead; the specification carries no language note here (see O.6) | +| `test_rtlm20_set_invalid_values_table` | A function, `undefined` and a symbol are 40013 | `lambda: None`, `None` and `object()`, each 40013; the parametrize ids keep the specification's labels | JavaScript values given their nearest Python counterparts | +| `test_rtlc12e1_increment_invalid_amounts_table` | The null row applies "where null is distinguishable" | It applies: Python tells `increment(None)` from `increment()`, whose default is 1. `True` is in the table and fails 40003 though `bool` is an `int`; NaN and the infinities are `float('nan')`, `float('inf')` and `float('-inf')` | The specification's own reading, for Python | + +### LiveObjects translation notes + +Not deviations, recorded so the difference from the pseudocode is not mistaken for one. The +largest, the typed views, has its own section at the end of this file. + +| Subject | Note | +|---|---| +| Granted modes | The modes a server grants are bits of the ATTACHED `flags`, so the specifications' `modes: [...]` is `flags=HAS_OBJECTS \| OBJECT_SUBSCRIBE_FLAG`. The standard mock's ATTACHED carries HAS_OBJECTS alone, which leaves `channel.modes` empty, and RTO2 then checks the modes the channel *requested* — which is why every objects channel asks for both through `objects_channel_options()`, and `get()` raises 40024 on one that asked for neither. See the RTO2a row of the features entry above | +| Wire values | The mock tier speaks JSON. A `json` value is a JSON-encoded **string** on the v6 wire (OD2g), so a captured `mapSet.value.json` is compared after `json.loads`; a `bytes` value is base64; an action the specification writes by name compares equal to its `ObjectOperationAction`, an `IntEnum` | +| Numbers | Every number decodes to a `float`, and a locally created value holds one too, so that a value reads the same before and after its echo; a number is read with `value(float)`. A boolean is asserted with `is True` / `is False`, since `True == 1` | +| The public message | `PublicObjectMessage.fromObjectMessage(source, channel)` is `publicmessage.ObjectMessage._from_internal(source, channel_name)`. The public and internal `ObjectMessage` share a name, so the tests reach the public one through the `publicmessage` module | +| Channel state | `channel.object.processChannelState(S)` is `channel.object._act_on_channel_state(ChannelState.S)`, which acts on its argument: RTO23c1's SUSPENDED case and RTO27 call it while the channel is still ATTACHED | +| Loops | RTO17-RTO18's scenario loop and RTO27's DETACHED, FAILED and SUSPENDED blocks are one parametrized test each, keeping one Test ID | +| Listeners | `subscribe` on a path or an instance takes any callable, a bound built-in such as `events.append` included — unlike the channel and connection `EventEmitter` | +| ACK serials | `ack-after-echo-no-double-apply` ACKs `msgSerial` 0, relying, as the specification does, on the first OBJECT of a fresh connection taking it (RTN7b) | +| Integration | An un-awaited `channel.attach()` or write is a task, awaited later inside `pytest.raises` where the specification expects it to fail. `poll_until_success` treats only an `AblyException` as "not yet", so that any other error fails at once. `channel.object.get()`, which the lifecycle and sync specifications leave unbounded, is bounded at the fifteen seconds `objects_gc_test.md` gives it | + ## Mock Infrastructure Limitations Tests that cannot be implemented as written, kept as skipped stubs carrying their Test -IDs so the specification's coverage is still accounted for. Fifteen in total. Two of the +IDs so the specification's coverage is still accounted for. Fifteen in total, none of +them from `uts/objects`. Two of the entries are caused by the SDK rather than by the mock, but they land here because the effect is the same: no test can observe the behaviour. @@ -2410,6 +2671,81 @@ is not shared between them. duplicates, on both protocols. The test asserts the full set rather than only the specification's `>= 5`, since the fixture is fixed. +### The sandbox delivers the echo before the ACK, so RTO9a3 is reached at the mock tier only + +Against the sandbox the echo of a client's own write arrived before its ACK in every trace, +the first write included, where both frames came in one read with the OBJECT first. The echo +is therefore applied as a CHANNEL operation (RTO8b), the ACK's LOCAL apply is rejected by the +RTLO4a site-serial check, and `_applied_on_ack_serials` (RTO9a3) is never written on that path. +Both orders are correct per the specification, and nothing is applied twice — a probe writing +every value type in every protocol combination found the writer's state unchanged by its +echoes. The ACK-first order, where RTO9a3 does the work, is covered at the mock tier, whose +standard mock ACKs before any echo is injected (`test_rto20_echo_dedup`, and RTO9a3 in +`objects_pool_test.py`). + +The same traces show the sandbox pacing ACKs at about one per 500 ms per connection: the first +ACK arrives with the echo, and later ones on a half-second tick. An awaited write after the +first can take that long without anything being wrong, which is why the GC integration tests +take over a second each. + +### A Python-formatted `initialValue` is accepted, because the server hashes the string it receives + +An object id is a hash over `[initialValue]:[nonce]` (RTO14b1), and ably-python formats +`initialValue` as Python's `json.dumps` does — `{"count":42.0}` where ably-js writes +`{"count":42}`, and non-ASCII as `\uXXXX` escapes where ably-js writes raw UTF-8 — so it looked +as though the server, or another SDK, would compute a different id. Measured against the +sandbox's REST objects endpoint with client-generated ids: Python's formatting, ably-js's +formatting and raw UTF-8 are each accepted (201) when the id is hashed over the string sent, +while an id hashed over `{"count":7.0}` and sent with `{"count":7}` is rejected 400/92000, +"object id does not match expected value". The server hashes the exact string it receives. The +evaluation computes the id from the one string it then sends, so the formatting cannot diverge, +and the integration tier confirms it over realtime in both protocols. + +### A sync cursor can itself contain `:`, and splitting on the first one is right + +A sync the sandbox split into twenty OBJECT_SYNC messages carried channelSerials of the shape +`4d58ad59:map:0TeL…@1791551583365` and ended on `4d58ad59:`, so the cursor contains `:` too. +RTO5a1 splits on the first `:`, which yields the right sequence id; late joiners on both +protocols matched the writer exactly. + +### `RTO5a2-RTO17/sync-interrupted-reconnect-0` does interrupt a sync + +Its two siblings in `objects_faults.md` never reach what they are named for (see the objects +entry under UTS Spec Errors), so this one was measured too, and it does: the proxy drops the +first OBJECT_SYNC and closes the socket, the client resumes on the same connectionId, receives +an ATTACHED carrying RESUMED and HAS_OBJECTS and a fresh OBJECT_SYNC, and `get()` resolves from +that. The ATTACHED arrives on a channel already ATTACHED, which emits no state change, so the +re-sync depends on the channel handing every ATTACHED it accepts to the objects — which it does. + +### Where ably-python departs from ably-js, it follows `objects-features.md` + +Recorded because anyone comparing the two implementations will meet these first. + +| Spec point | ably-js | ably-python | +|---|---|---| +| RTO5c2, RTO5c10 | `_applySync` returns early when the SyncObjectsPool is empty, skipping both the removal of absent objects and the parent-reference rebuild | No early return; both run, as written | +| RTO5e | Moves to SYNCING only on a new sequence, so an OBJECT_SYNC with no sequence id while SYNCED emits nothing | Applies RTO5e to every OBJECT_SYNC, emitting SYNCING then SYNCED | +| RTO5a6 | `^([\w-]+):(.*)$`, so `:x` reads as having no channelSerial | Malformed means "does not contain the `:` separator", as written | +| RTO2 | Requires a granted mode to be among the requested ones as well | RTO2b applies only "otherwise", when no modes were granted | +| RTLC9 | A COUNTER_INC with no `counterInc` is not applied | The RTLC9h no-op, applied, reading RTLC7d5's "passing in `operation.counterInc`" as written | +| RTLMV4c | A non-finite number in a map's entries is sent as `null` | 40013: it has no JSON representation (OD4d3), and `initialValue` is always JSON | + +### More `Task was destroyed` warnings, and a GC timer on every attached channel, are not faults + +A plain run of `test/uts` prints `Task was destroyed but it is pending!` 174 times where the +baseline printed 80. Every one is a `ConnectionManager.connect_base()` task, the teardown +warning the baseline prints already, and the rise comes from the objects cases that connect a +mock-backed client; no new kind of warning appears. + +The GC timer is scheduled on the first ATTACHED of every channel, whether or not it uses +objects (RTO10a; ably-js does the same whenever the plugin is loaded), and cancelled on +DETACHED and FAILED, so a client's `close()` stops it. A non-objects test advancing a +`FakeClock` by five minutes therefore runs one empty sweep. The one channel whose timer +outlives it is one passed to `Channels.release()` while attached, because release does not +detach — the RTS4a defect under *Failing Tests*. And a burst of creates on a channel that has +not yet read the server time each request `/time`, until the first answer persists the offset +(RTO16a); wasteful, and not wrong. + ## Candidate issues `writing-derived-tests.md` asks for the deviations above to be classified into distinct @@ -2845,6 +3181,87 @@ the unit tier adapts to that; RTN14b does not, which is what makes this a defect than a choice. `test/uts/realtime/integration/proxy/connection_open_failures_test.py -k rtn14b` +### From the objects derivation + +These are against `ably/specification`, not ably-python: the objects tier found no SDK +non-compliance, and each item is a fault in `uts/objects` or, for O.7, a gap in +`objects-features.md`. **None is filed.** Each matches an entry under *UTS Spec Errors*, which +carries the detail, and the command after it runs the derived tests that correct or work +around it — they pass, and are the reference for what the specification should say; +`RUN_DEVIATIONS` makes no difference to them. They are ranked by what an SDK deriving from the +specification as written would suffer: first where a conforming SDK fails or has to adapt +(O.1–O.3), then where a non-conforming one passes (O.4, O.5), then housekeeping (O.6), with the +features-spec gaps last, being questions for a different document. + +**O.1 The unit specifications model three internal interfaces against `objects-features.md`.** +RTLC7g, RTLM15g, RTO3a, RTO7, RTO17, RTLCV4g5, RTLMV4j5, MCRO2, CCRO2. +`internal_live_counter.md` and `internal_live_map.md` take an update from `applyOperation` +while asserting, of the same call, the boolean RTLC7g and RTLM15g say it returns; +`objects_pool.md` puts the sync state machine on `ObjectsPool`, and `PLAN.md` repeats it; +`value_types.md` and `public_object_message.md` read the retained create in two different +places, one of which would put it on the wire, and the latter gives `*CreateWithObjectId` fields +MCRO2 and CCRO2 do not define. Every SDK has to adapt all of it — S-1, S-2 and S-4 here, and +ably-js's derived suite likewise. One issue: write the files against the features spec's shapes. +`test/uts/objects/unit/internal_live_map_test.py test/uts/objects/unit/objects_pool_test.py test/uts/objects/unit/value_types_test.py test/uts/objects/unit/public_object_message_test.py` + +**O.2 Fixtures put values where the protocol does not.** RTO2, RTO25a, RTO26a, RTLMV4d, +RTLMV4f1. Granted modes written as an ATTACHED `modes: [...]` field rather than as `flags` bits; +`connectionId` inside `connectionDetails` in the helper specification and two hand-written +mocks; RTLMV4d's bytes row comparing an in-memory entry with its base64 wire form; and the +provisioning comment's one result per batch entry, where the server answers one result for +the batch. A conforming SDK fails the first three as written. +`test/uts/objects/unit/realtime_object_test.py test/uts/objects/unit/value_types_test.py -k "rto2_mode or rto25a or rto26a or rtlmv4d_map_set_all_types_table"` + +**O.3 Steps read state as though an injected frame were applied at once.** RTO5c9, RTO10c1b1, +RTO17, RTO18d, RTO19, RTO20, RTO20d4, RTO20e, RTO20e1, RTO23c1, RTO24b1, RTO24c1, RTINS16, +RTINS16h, RTLO4b, RTPO19, and at the integration tier RTO5a2, RTO7, RTO8, RTO17. An SDK that +processes frames on a task of its own **fails** the three RTO23c1 tests, the two RTO20e1 tests, +RTO20e and RTO20d4's empty-list test as written — the operation starts before the re-sync +ATTACHED has been processed, sees SYNCED and resolves — while the negatives of RTO20's +echo-dedup, RTO19, RTO10c1b1 and RTINS16h pass whatever the SDK does, and every exact count +after a `poll_until(>= n)` is half a negative. `path_object.md` already waits for an inbound +frame in its compact tests and `standard_test_pool.md` already defines the quiescence pattern; +the fix is to use them throughout, with `process_pending_events()` after each frame that is +read from, and, in `objects_faults.md`, a recorded state sequence in place of `AWAIT_STATE` for +DISCONNECTED and for a re-attach. +`test/uts/objects/unit/realtime_object_test.py test/uts/objects/unit/instance_test.py -k "rto23c1 or rto20e or rto20d4_empty or rto20_echo or rto19 or rto10c1b1 or rto5c9_rto20 or rtins16h"` +`test/uts/objects/integration/proxy/objects_faults_test.py` + +**O.4 Tests that pass without the behaviour they are named for.** RTLM8g, RTLM9b, RTO10, +RTO10b1, RTO10c1b1, RTPO6b, RTPO19 with RTO24b2a2, RTO4b2, RTO14b1, RTINS16g, RTO20e, RTO23c, +RTO20c, RTO20d1, RTO20e1; and, derived as written, RTO4 with RTO5a2, RTO7 and RTO8 through the +proxy, RTLC11b1, RTLO4b, RTO5c10 and the lifecycle test's unread `events_b`. Sixteen derived +tests carry the assertion or the direct check their specification omits. The sharpest are the GC +tests, which read a value that is null with no GC at all — RTO10b1 also advances less than one +GC interval — and the two `objects_faults.md` tests titled for RTO8a's buffer, which +measurement shows never buffer. Same class as +[#543](https://github.com/ably/specification/issues/543). +`test/uts/objects/unit -k "rtlm8g or rtlm9b or rto10 or rtpo6_at_escaped or rtpo19_map_clear or rto4b_attached or rto14_objectid_format_counter or rtins16g or rto20e or rto23c_get_waits or rto20c_missing or rto20d1"` + +**O.5 Boundaries no test reaches.** RTLM7h, RTLM8g, RTLM24c, RTLM19a1, RTLC7c, RTLM15c, +RTLC6a, RTLM6a, RTLM7a3, RTLC7b, RTLM15b, RTLM23c, RTLO6b, RTINS3b. Equal serials against the +clear floor, an entry exactly one grace period old, the order of serial bookkeeping against the +tombstone checks, a rejected MAP_SET's parent references, the serials a rejected operation +leaves, MAP_CREATE no-ops, the clock a map-entry tombstone reads, and a primitive instance's +`id`. Each but the last was found as a mutant every existing test lets through, so each new +test can be written against that mutant. No reproduction: there is no test to run. + +**O.6 Labels, commentary and harmless fixture slips.** The thirteen rows of the objects +housekeeping entry: RTLM24's `<=`, RTLM14c's title, RTO20d4's commentary, an RTO19 test that +exercises RTO18f2, RTO20's credit to RTO9a3, RTPO6b's undefined 40003, RTLM15e's half-set +tombstone, the second `root` beside the pool's, the ATTACHED `channelSerial`s, fragile bare +serials, two missing language notes, and `integration-testing.md`'s Protocol Variants list. +None changes an outcome. Same class as +[#532](https://github.com/ably/specification/issues/532). + +**O.7 `objects-features.md` gaps.** RTLM5d2, OD4c5, OD4d5, RTO4, RTO5a5, RTO5a6, RTO5d, RTO4b2, +RTO14b2, RTO2a, RTPO6; and, on the unmerged pull requests, RTLMV4c1 +([#491](https://github.com/ably/specification/pull/491)) and RTPO20c and RTBC16d +([#471](https://github.com/ably/specification/pull/471)). The widest reaching are RTLM5d2, +under which every JSON map entry reads null, and RTO5d, under which one ObjectMessage with no +`object` stalls the whole sync. The two #471 points want raising on that pull request before it +merges, since ably-js already disagrees with its text on the first. + ## How the specifications are adopted here Choices about the approach, as against the behaviour recorded above. @@ -3198,6 +3615,98 @@ and each message still gets a distinct id, so only the setup latency differs. Ea then polls history until the expected count is visible before paginating, which is the specifications' own `poll_until`, because history is not immediately consistent. +### The objects tier mirrors the other two, with a pure tier inside its unit tier + +`uts/objects/unit/.md` becomes `test/uts/objects/unit/_test.py`, +`uts/objects/integration/_test.md` becomes `test/uts/objects/integration/_test.py`, +and `integration/proxy/objects_faults.md` becomes `integration/proxy/objects_faults_test.py`, +whose Test IDs read `objects/proxy/...`. `uts/objects/helpers/standard_test_pool.md` becomes +`test/uts/objects/helpers/standard_test_pool.py`, with tests of its own beside it, which are +the 18 harness cases the header counts separately. + +Seven of the fifteen unit specifications — `internal_live_counter`, `internal_live_map`, +`objects_pool`, `parent_references`, `object_id`, `value_types` and `public_object_message`, +161 Test IDs — construct internal objects and connect nothing. The other eight, 161 Test IDs +more, drive `channel.object` over the mock websocket. The specifications draw no such line, +but it decides which shapes apply: S-1 to S-4 are pure-tier adaptations, and the mock tier +reaches the library only through its public surface and a handful of underscore readers. + +### The specifications' untyped `PathObject` and `Instance` become LODR-061's typed views + +The specifications are written against the merged features spec's `PathObject` and +`Instance`, each one class carrying every method. ably-python follows LODR-061 and the RTTS +partition of [#491](https://github.com/ably/specification/pull/491): navigation, `instance`, +`compact`, `compact_json` and `subscribe` are on the base `PathObject`, and a type-specific +method is reached through the view for that type. So `root.get("score").value()` is +`root.get('score').as_live_counter().value()`, `root.get("name").value()` is +`root.get('name').as_primitive().value()`, and `pathObject.set(k, v)` is +`await path_object.as_live_map().set(k, v)`; `root` is already a `LiveMapPathObject` and +needs no view. A path's views are unchecked — a view of the wrong type reads `None`, or `[]` +for a collection (RTTS5d1), and a write through it raises 92007 or 92005 (RTTS5d2) — so +where a specification expects the untyped `value()` to be null, the test asserts it through +both value views. An `Instance`'s views are checked and raise 92007 (RTTS9d). + +That is a difference of spelling, which `writing-derived-tests.md` says is not a deviation, +and nothing above records it test by test. Where the partition makes a read unreachable +rather than differently spelled, the test is an S-5 adaptation. Mutations are `async` and +awaited; reads, navigation, the views and `subscribe` are synchronous; `entries()` yields +`(key, value)` tuples; `Instance.id` and `Instance.type` are properties. + +### A standalone `RealtimeObject` drives the pure tier + +`objects-features.md` puts the sync state machine on `RealtimeObject` (S-2), so the pure +tier needs one without a channel. `RealtimeObject()` built with none, or +`RealtimeObject(pool=pool)` around a pool a test filled, is a working state machine: `get()` +waits for SYNCED and returns the root, `_publish` raises 40000, the server time is the local +clock, the channel-mode and channel-state preconditions check nothing, and a path +subscription's events carry no channel name. None of that is reachable through the public +API; it exists so that a pure test exercises the code the mock tier runs. Members of a +public class that LODR-061 does not name carry a leading underscore — `_objects_pool`, +`_sync_state`, `_on_attached`, `_gc_interval_ms` — and the tests reach them by those names. + +### The objects mock tier speaks JSON, and answers `/time` + +`setup_synced_channel` connects through `objects_client`, which sets +`use_binary_protocol=False`, so the builders' dictionaries go over the mock exactly as the +specification describes the JSON wire: camelCase keys, numeric actions and semantics, a `json` +value as a JSON-encoded string and `bytes` as base64. The msgpack wire is covered by the +integration tier, which runs its three specifications under both protocols, and by +`standard_test_pool_test.py`, which round-trips every wire type through both encodings. + +Creating a `LiveCounter` or `LiveMap` reads the server time (RTO16), which is a REST request, +so `objects_client` also installs a `MockHttpClient` answering `GET /time` from the test's +clock and 404 to anything else. Without it a mock-backed client would reach the network for +the time. The harness CONNECTED carries a `siteCode`, an `objectsGCGracePeriod` and a +`maxIdleInterval` of 0, so that a `FakeClock` test — the GC tests, chiefly — has no idle timer +to fire. + +### The objects integration tier provisions its own sandbox app, and its proxy clients sign a JWT + +`test/uts/objects/integration/conftest.py` provisions a sandbox app for the tier, separate +from the realtime tier's, whose fixtures are not visible from `test/uts/objects`; it is +called `realtime_sandbox` all the same, so the tests read as the realtime tier's do. All three +specifications outside `proxy` carry Protocol Variants, so every test there takes the +`use_binary_protocol` fixture. Objects are provisioned over REST with `X-Ably-Version: 6`, in +JSON whichever protocol the realtime client speaks. The proxy package repeats the realtime +tier's `proxy_control` and `proxy_session` fixtures and its 300-second marker, prepended so +that it wins over the parent's 120. A client under test there authenticates with an +`auth_callback` returning a locally signed Ably JWT, which costs no round trip and so puts +nothing in the event log beside the frames a test counts; a second client, where a test has +one, goes straight to the sandbox with the key, as the specification builds it. + +### Batch and the typed views are tested outside `test/uts`, for want of a specification + +No UTS specification covers batched writes (RTPO20, RTINS17, RTBC1–RTBC16, in the unmerged +[#471](https://github.com/ably/specification/pull/471)) or the typed views (RTTS2–RTTS10, in +#491). Both are implemented, so both are tested, in `test/unit/objects/`: `batch_test.py`, 29 +functions run as 76 cases, and `typed_views_test.py`, 17 functions run as 61. They are written +against `objects-features.md`, the two pull requests and LODR-061, carry no `# UTS:` comment, +and are not counted in the header. They drive the standard synced pool over the mock +websocket with this tier's helpers, and their own `conftest.py` closes clients as +`test/uts/conftest.py` does. Single-point mutants of the implementation checked that they +bite: they kill 15 of 16 in the batch code — the survivor removes a check another layer +repeats — and all 7 in the view code. + ### Deviation records are consolidated, not accumulated Each round of derivation runs a specification area per agent, and each writes its own @@ -3213,7 +3722,11 @@ new ones: `Auth#revokeTokens`, `Rest#batchPresence`, the `PushChannel` surface a same root cause as RTN15h1's, and sits in that entry. The realtime integration round did the same for five more: the RTN15h3 stall, the connection-level ERROR bypass, the unused `connectionStateTtl`, connection recovery, and the missing channel-level handling for a -decode error other than 40018. +decode error other than 40018. The objects round found its specification faults several +times over in the same way: the internal shapes the unit specifications assume were +reported by three agents, the missing barriers after an injected frame by four, and the +RTO8a buffer that `objects_faults.md` never exercises by the agent that wrote the contract, +the one that derived the tier and the one that measured it. A verdict can change the same way, when a second specification reaches a behaviour the first was content with. `errorReason` surviving a successful reconnect is permitted by @@ -3233,17 +3746,22 @@ The header states how many derived tests there are, how many pass, how many are gated and how many cannot run. Those numbers are the check that the file is still true: in pytest cases, the gated count must equal the number of failures under `RUN_DEVIATIONS=1`, and gated plus unrunnable must equal the number of skips without -it. As of this writing that is 172 failures and 15 skips with the variable set, and -187 skips and 1177 passes without it, the 1177 being 1047 derived cases and 130 -`helpers/` ones. Those figures need the `submodules` checkout CI makes: without it, three -`rest/unit/encoding` tests that read the ably-common fixtures skip as well. +it. As of this writing that is 171 failures, 1569 passes and 15 skips with the variable +set, and 186 skips and 1569 passes without it, the 1569 being 1421 derived cases and 148 +harness ones — 130 under `helpers/` and 18 under `objects/helpers/`. Those figures need +the `submodules` checkout CI makes: without it, three `rest/unit/encoding` tests that read +the ably-common fixtures skip as well. The tiers are told apart by the junit report's +`classname`, `test.uts..`, with `proxy` the package within an integration tier. The other two counts are measured from the source rather than from a run. The number of -**derived tests** is the number of `# UTS:` comments, 1141. The number of **Test IDs** is -the number of *distinct* ids in them, 1132 — not the same figure, because five ids in +**derived tests** is the number of `# UTS:` comments, 1480. The number of **Test IDs** is +the number of *distinct* ids in them, 1471 — not the same figure, because five ids in `rest/unit` are carried by more than one test function. Counting the comments and calling the result Test IDs is the easy mistake here, and it overstates the specification coverage -by nine. +by nine. The other is to gate or un-gate a test, record it in the body, and leave the +header alone. A test moved from passing to `@spec_error` moves the passing, gated, +per-side and specification-fault figures together, and this file's were a case short for a +while after `test_rsc7d_ably_agent_header_format` was gated that way. Anyone changing the suite should re-run both and update the header, rather than copying the previous numbers forward. Keep the three units apart while doing it: one Test ID is From d5481cec470944b1bfc103bd76d7dc54192fb7d8 Mon Sep 17 00:00:00 2001 From: owenpearson Date: Fri, 9 Oct 2026 14:42:45 +0100 Subject: [PATCH 20/71] docs: add LiveObjects to the README Co-Authored-By: Claude Opus 5.5 (1M context) --- README.md | 46 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/README.md b/README.md index e833df38..3d30e0a7 100644 --- a/README.md +++ b/README.md @@ -102,6 +102,52 @@ async with create_realtime_client(key='your-ably-api-key', client_id='me') as re await channel.publish('test-event', 'hello world') ``` +### LiveObjects + +LiveObjects keeps shared, mutable state on a channel: maps and counters that every client +attached to it reads, updates and subscribes to. The channel needs the object modes, and +`channel.object.get()` attaches it, waits for the objects to sync, and returns the root map. +Values are read through a typed view of their path — `as_live_map()`, `as_live_counter()` or +`as_primitive()` — and reads are synchronous; writes are awaited. + +```python +from ably.pubsub.server import ChannelMode, ChannelOptions, LiveCounter, LiveMap, create_realtime_client + +async with create_realtime_client(key='your-ably-api-key') as realtime_client: + channel = realtime_client.channels.get( + 'my-objects', + ChannelOptions(modes=[ChannelMode.OBJECT_SUBSCRIBE, ChannelMode.OBJECT_PUBLISH]), + ) + + # Attach, wait for the objects to sync, and get the root map + root = await channel.object.get() + + # Create objects by setting them on the root + await root.set('visits', LiveCounter.create(0)) + await root.set('profile', LiveMap.create({'name': 'Alice', 'theme': 'dark'})) + + # Read through a typed view of the path + visits = root.get('visits').as_live_counter() + print(visits.value()) # 0.0 + print(root.at('profile.name').as_primitive().value(str)) # Alice + + # Subscribe to changes at a path + def on_change(event): + print(f'{event.object.path()} changed') + + subscription = root.get('visits').subscribe(on_change) + + # Mutate + await visits.increment(5) + + # Batch several writes into a single message + async with root.get('profile').as_live_map().batch() as profile: + profile.set('name', 'Bob') + profile.remove('theme') + + subscription.unsubscribe() +``` + ## Releases The [CHANGELOG.md](https://github.com/ably/ably-pubsub-python/blob/main/CHANGELOG.md) contains details of the latest releases for this SDK. You can also view all Ably releases on [changelog.ably.com](https://changelog.ably.com). From 849f65e81eae4b4cdba8cb8f528e145da3b61e50 Mon Sep 17 00:00:00 2001 From: owenpearson Date: Fri, 9 Oct 2026 15:16:51 +0100 Subject: [PATCH 21/71] objects: fail sync waits on an inactive channel and skip bad messages get() and a write waiting for the objects sync now fail with 92008 when the channel is already detached, failed or released, rather than only when it moves into one of those states while they wait; before, closing the client during get()'s attach, or an ERROR arriving with the ACK, left the wait pending forever. The RTL33 ensure-active-channel step checks the state the implicit attach ended in, and a write is no longer applied locally once the channel's objects have been cleared. An object message that fails to decode, or an operation that raises while being applied, is logged and skipped on its own, so one bad value cannot drop the rest of its OBJECT message, or an OBJECT_SYNC's cursor and so the whole sync. A released channel stops its GC timer, and a sweep that raises is logged and rescheduled. Also shares the MAP_SET, MAP_REMOVE and COUNTER_INC construction between the direct writes and the batch, validates a LiveMap or LiveCounter value before reading the server time, keeps the GC grace period from each CONNECTED, and lets typing.get_type_hints resolve the wire dataclasses. Co-Authored-By: Claude Opus 5.5 (1M context) --- ably/pubsub/objects/batch.py | 106 +++--- ably/pubsub/objects/instance.py | 4 +- ably/pubsub/objects/livecounter.py | 37 +- ably/pubsub/objects/livemap.py | 79 ++-- ably/pubsub/objects/liveobject.py | 27 +- ably/pubsub/objects/objectmessage.py | 73 ++-- ably/pubsub/objects/pathobject.py | 10 +- .../objects/pathobjectsubscriptionregister.py | 26 +- ably/pubsub/objects/realtimeobject.py | 292 +++++++++------ ably/pubsub/objects/subscription.py | 81 ++++- ably/pubsub/objects/valuetypes.py | 102 ++++-- ably/pubsub/realtime/channel.py | 56 +-- test/unit/objects/batch_test.py | 28 +- test/unit/objects/realtime_object_test.py | 342 ++++++++++++++++++ test/unit/objects/subscription_test.py | 69 ++++ test/unit/objects/value_types_test.py | 43 +++ test/uts/deviations.md | 12 +- 17 files changed, 1035 insertions(+), 352 deletions(-) create mode 100644 test/unit/objects/realtime_object_test.py create mode 100644 test/unit/objects/subscription_test.py create mode 100644 test/unit/objects/value_types_test.py diff --git a/ably/pubsub/objects/batch.py b/ably/pubsub/objects/batch.py index ddb1d1d6..d704c96d 100644 --- a/ably/pubsub/objects/batch.py +++ b/ably/pubsub/objects/batch.py @@ -10,13 +10,14 @@ has closed raises `AblyException` 40000 (RTBC16e). A write validates its arguments when it is called, so an invalid one raises inside the -block and nothing is published. A `LiveMap` or `LiveCounter` value is evaluated when the -batch is published, because its object id needs the server time (RTO16): an invalid -value inside one raises from the end of the block, and nothing is published then either. +block and nothing is published. A `LiveMap` or `LiveCounter` value is validated then too, +but evaluated only when the batch is published, because its object ids need the server +time (RTO16). """ from __future__ import annotations +from functools import partial from typing import TYPE_CHECKING, Any, Awaitable, Callable, Generic, TypeVar, overload from ably.pubsub.objects.enums import ValueType @@ -29,17 +30,17 @@ primitive_value, value_type_of, ) -from ably.pubsub.objects.livecounter import _validate_amount -from ably.pubsub.objects.objectmessage import ( - CounterInc, - MapRemove, - MapSet, - ObjectData, - ObjectMessage, - ObjectOperation, - ObjectOperationAction, +from ably.pubsub.objects.livecounter import counter_inc_message +from ably.pubsub.objects.livemap import map_remove_message, map_set_message, map_set_messages +from ably.pubsub.objects.objectmessage import ObjectMessage +from ably.pubsub.objects.valuetypes import ( + LiveCounter, + LiveMap, + primitive_to_object_data, + validate_amount, + validate_key, + validate_value, ) -from ably.pubsub.objects.valuetypes import LiveCounter, LiveMap, evaluate, primitive_to_object_data, validate_key from ably.pubsub.util.exceptions import AblyException if TYPE_CHECKING: @@ -146,23 +147,23 @@ async def flush(self) -> None: Raises whatever building or publishing the messages raises. Nothing is published if building any of them fails. """ - try: - self.close() - if self.queued_message_constructors: - # The block may have awaited since its writes checked the write preconditions (RTO26) - self.realtime_object._check_write_preconditions() - object_messages: list[ObjectMessage] = [] - for construct in self.queued_message_constructors: - object_messages.extend(await construct()) - if object_messages: - await self.realtime_object._publish_and_apply(object_messages) - finally: - self.wrapped_instances.clear() - self.queued_message_constructors.clear() + constructors = self.queued_message_constructors + self.close() + if constructors: + # The block may have awaited since its writes checked the write preconditions (RTO26) + self.realtime_object._check_write_preconditions() + object_messages: list[ObjectMessage] = [] + for construct in constructors: + object_messages.extend(await construct()) + if object_messages: + await self.realtime_object._publish_and_apply(object_messages) def close(self) -> None: - """RTBC16e: closes the batch, so that every later use of its contexts raises 40000.""" + """RTBC16e: closes the batch, so that every later use of its contexts raises 40000, and + drops the contexts and writes it holds.""" self.closed = True + self.wrapped_instances = {} + self.queued_message_constructors = [] class BatchContext: @@ -278,7 +279,11 @@ def values(self) -> list[BatchContext]: return [context for _, context in self.entries()] # RTBC8a-RTBC8c def size(self) -> int | None: - """RTBC9: the number of entries in the map.""" + """RTBC9: the number of entries in the map. + + This is always an int, as the context always wraps a map; the signature is the one + `size()` has on the path views, where a path can resolve to nothing. + """ self._realtime_object._check_access_preconditions() # RTBC9a self._throw_if_closed() # RTBC9b return self._instance.size() # RTBC9c @@ -288,9 +293,10 @@ def set(self, key: str, value: LiveMapValue) -> None: value evaluates to. Raises AblyException 40003 for a key that is not a string and 40013 for a value of - an unsupported type (RTLM20e1). A `LiveMap` or `LiveCounter` value is evaluated when - the batch is flushed, so it is the end of the block that raises for an invalid value - inside one (RTLMV4). + an unsupported type (RTLM20e1), the contents of a `LiveMap` or `LiveCounter` value + included (RTLMV4). Such a value is evaluated when the batch is flushed, as its object + ids need the server time (RTO16); a primitive is encoded now, so later changes to a + dict or list the caller passed do not reach the batch. """ realtime_object = self._realtime_object realtime_object._check_write_preconditions() # RTBC12b @@ -299,17 +305,12 @@ def set(self, key: str, value: LiveMapValue) -> None: object_id = self._instance.id if not isinstance(value, (LiveCounter, LiveMap)): - self._queue(_map_set(object_id, key, primitive_to_object_data(value))) # RTBC12e, RTLM20e7b-f + self._queue(map_set_message(object_id, key, primitive_to_object_data(value))) # RTBC12e, RTLM20e1 return - async def construct() -> list[ObjectMessage]: - # RTLM20e7g1: the creates the value evaluates to, their object ids from the server time - server_time_ms = await realtime_object._get_server_time_ms() - object_messages = evaluate(value, server_time_ms) - data = ObjectData(object_id=object_messages[-1].operation.object_id) # RTLM20e7g2 - return [*object_messages, _map_set(object_id, key, data)] - - self._root_context.queue_message_constructor(construct) # RTBC12e + validate_value(value) # RTLM20e1 + self._root_context.queue_message_constructor( + partial(map_set_messages, realtime_object, object_id, key, value)) # RTBC12e def remove(self, key: str) -> None: """RTBC13: queues a MAP_REMOVE of `key`. @@ -319,12 +320,7 @@ def remove(self, key: str) -> None: self._realtime_object._check_write_preconditions() # RTBC13b self._throw_if_closed() # RTBC13c validate_key(key) # RTLM21e1 - # RTBC13e - self._queue(ObjectMessage(operation=ObjectOperation( - action=ObjectOperationAction.MAP_REMOVE, # RTLM21e2 - object_id=self._instance.id, # RTLM21e3 - map_remove=MapRemove(key=key), # RTLM21e5 - ))) + self._queue(map_remove_message(self._instance.id, key)) # RTBC13e class LiveCounterBatchContext(BatchContext): @@ -351,18 +347,13 @@ def increment(self, amount: float = 1) -> None: """ self._realtime_object._check_write_preconditions() # RTBC14b self._throw_if_closed() # RTBC14c - # RTBC14e - self._queue(ObjectMessage(operation=ObjectOperation( - action=ObjectOperationAction.COUNTER_INC, # RTLC12e2 - object_id=self._instance.id, # RTLC12e3 - counter_inc=CounterInc(number=_validate_amount(amount)), # RTLC12e1, RTLC12e5 - ))) + self._queue(counter_inc_message(self._instance.id, validate_amount(amount))) # RTBC14e, RTLC12e1 def decrement(self, amount: float = 1) -> None: """RTBC15: queues a COUNTER_INC of `-amount`, after the same validation as `increment`.""" self._realtime_object._check_write_preconditions() # RTBC15b self._throw_if_closed() # RTBC15c - self.increment(-_validate_amount(amount)) # RTBC15e + self.increment(-validate_amount(amount)) # RTBC15e class PrimitiveBatchContext(BatchContext): @@ -392,12 +383,3 @@ def _context_type_for(instance: Instance) -> type[BatchContext]: if isinstance(instance, PrimitiveInstance): return PrimitiveBatchContext return BatchContext - - -def _map_set(object_id: str, key: str, data: ObjectData) -> ObjectMessage: - """RTLM20e: the ObjectMessage setting `key` of the map `object_id` to `data`.""" - return ObjectMessage(operation=ObjectOperation( - action=ObjectOperationAction.MAP_SET, # RTLM20e2 - object_id=object_id, # RTLM20e3 - map_set=MapSet(key=key, value=data), # RTLM20e6, RTLM20e7 - )) diff --git a/ably/pubsub/objects/instance.py b/ably/pubsub/objects/instance.py index 54ab3ac8..64c238b6 100644 --- a/ably/pubsub/objects/instance.py +++ b/ably/pubsub/objects/instance.py @@ -261,7 +261,7 @@ def batch(self) -> Batch[LiveMapBatchContext]: Entering checks the write preconditions (RTINS17b). """ - # Imported here, as the batch module builds on this one + # Imported here, as the batch module builds on the views from ably.pubsub.objects.batch import Batch, LiveMapBatchContext return Batch(self._realtime_object, lambda: self._value, LiveMapBatchContext, f'object {self.id!r}') @@ -322,7 +322,7 @@ def batch(self) -> Batch[LiveCounterBatchContext]: Entering checks the write preconditions (RTINS17b). """ - # Imported here, as the batch module builds on this one + # Imported here, as the batch module builds on the views from ably.pubsub.objects.batch import Batch, LiveCounterBatchContext return Batch(self._realtime_object, lambda: self._value, LiveCounterBatchContext, f'object {self.id!r}') diff --git a/ably/pubsub/objects/livecounter.py b/ably/pubsub/objects/livecounter.py index 4c7c016a..69ce71df 100644 --- a/ably/pubsub/objects/livecounter.py +++ b/ably/pubsub/objects/livecounter.py @@ -3,12 +3,12 @@ from __future__ import annotations import logging -import math -from typing import TYPE_CHECKING, Any +from typing import TYPE_CHECKING from ably.pubsub.objects.enums import ObjectsOperationSource from ably.pubsub.objects.liveobject import CounterUpdate, LiveCounterUpdate, LiveObject from ably.pubsub.objects.objectmessage import CounterInc, ObjectMessage, ObjectOperation, ObjectOperationAction +from ably.pubsub.objects.valuetypes import validate_amount from ably.pubsub.util.clock import Clock from ably.pubsub.util.exceptions import AblyException @@ -40,23 +40,17 @@ async def increment(self, amount: float) -> None: Raises AblyException 40003 if `amount` is not a finite number (RTLC12e1); a bool is not a number. """ - await self._publish_counter_inc(_validate_amount(amount)) + await self._publish_counter_inc(validate_amount(amount)) async def decrement(self, amount: float) -> None: """RTLC13: `increment` by `-amount`, after the same validation (RTLC13c).""" - await self._publish_counter_inc(-_validate_amount(amount)) # RTLC13b, RTLC13c + await self._publish_counter_inc(-validate_amount(amount)) # RTLC13b, RTLC13c async def _publish_counter_inc(self, number: float) -> None: realtime_object = self.realtime_object if realtime_object is None: raise AblyException('Unable to increment a counter that is not on a channel', 400, 40000) - - object_message = ObjectMessage(operation=ObjectOperation( - action=ObjectOperationAction.COUNTER_INC, # RTLC12e2 - object_id=self.object_id, # RTLC12e3 - counter_inc=CounterInc(number=number), # RTLC12e5 - )) - await realtime_object._publish_and_apply([object_message]) # RTLC12g + await realtime_object._publish_and_apply([counter_inc_message(self.object_id, number)]) # RTLC12g def apply_operation(self, object_message: ObjectMessage, source: ObjectsOperationSource) -> bool: """RTLC7: applies `object_message.operation`, returning whether it was applied (RTLC7g). @@ -169,16 +163,11 @@ def diff(previous_data: float, new_data: float, *, for_tombstone: bool = False) return LiveCounterUpdate(update=CounterUpdate(amount=amount)) -def _validate_amount(amount: Any) -> float: - """RTLC12e1: `amount` as a float, raising AblyException 40003 unless it is a finite number. - - A bool is not a number, and None is not an omitted amount. - """ - if not isinstance(amount, bool) and isinstance(amount, (int, float)): - try: - number = float(amount) - except OverflowError: - number = math.inf - if math.isfinite(number): - return number - raise AblyException('Counter value increment should be a valid number', 400, 40003) +def counter_inc_message(object_id: str, number: float) -> ObjectMessage: + """RTLC12e: the COUNTER_INC of the counter `object_id` by `number`, a validated amount + (`validate_amount`).""" + return ObjectMessage(operation=ObjectOperation( + action=ObjectOperationAction.COUNTER_INC, # RTLC12e2 + object_id=object_id, # RTLC12e3 + counter_inc=CounterInc(number=number), # RTLC12e5 + )) diff --git a/ably/pubsub/objects/livemap.py b/ably/pubsub/objects/livemap.py index 00d9f58b..4b8f44f2 100644 --- a/ably/pubsub/objects/livemap.py +++ b/ably/pubsub/objects/livemap.py @@ -18,7 +18,14 @@ ObjectsMapEntry, ObjectsMapSemantics, ) -from ably.pubsub.objects.valuetypes import LiveCounter, LiveMap, evaluate, primitive_to_object_data, validate_key +from ably.pubsub.objects.valuetypes import ( + LiveCounter, + LiveMap, + evaluate, + primitive_to_object_data, + validate_key, + validate_value, +) from ably.pubsub.util.clock import Clock from ably.pubsub.util.exceptions import AblyException @@ -81,24 +88,13 @@ async def set(self, key: str, value: LiveMapValue) -> None: evaluates to, through `RealtimeObject._publish_and_apply`. Raises AblyException 40003 for a key that is not a string and 40013 for a value of - an unsupported type (RTLM20e1, RTLMV4b, RTLMV4c). + an unsupported type (RTLM20e1, RTLMV4b, RTLMV4c), before anything is requested or + published. """ validate_key(key) # RTLM20e1 - object_messages: list[ObjectMessage] = [] - if isinstance(value, (LiveCounter, LiveMap)): - # RTLM20e7g1: the creates the blueprint evaluates to, its contents validated as it is (RTLMV4c) - server_time_ms = await self._publishing_realtime_object()._get_server_time_ms() - object_messages = evaluate(value, server_time_ms) - data = ObjectData(object_id=object_messages[-1].operation.object_id) # RTLM20e7g2 - else: - data = primitive_to_object_data(value) # RTLM20e1, RTLM20e7b-RTLM20e7f - - object_messages.append(ObjectMessage(operation=ObjectOperation( - action=ObjectOperationAction.MAP_SET, # RTLM20e2 - object_id=self.object_id, # RTLM20e3 - map_set=MapSet(key=key, value=data), # RTLM20e6, RTLM20e7 - ))) - await self._publishing_realtime_object()._publish_and_apply(object_messages) # RTLM20h1, RTLM20h2 + realtime_object = self._publishing_realtime_object() + object_messages = await map_set_messages(realtime_object, self.object_id, key, value) # RTLM20e + await realtime_object._publish_and_apply(object_messages) # RTLM20h1, RTLM20h2 async def remove(self, key: str) -> None: """RTLM21: publishes a MAP_REMOVE through `RealtimeObject._publish_and_apply`. @@ -107,12 +103,7 @@ async def remove(self, key: str) -> None: """ validate_key(key) # RTLM21e1 realtime_object = self._publishing_realtime_object() - object_message = ObjectMessage(operation=ObjectOperation( - action=ObjectOperationAction.MAP_REMOVE, # RTLM21e2 - object_id=self.object_id, # RTLM21e3 - map_remove=MapRemove(key=key), # RTLM21e5 - )) - await realtime_object._publish_and_apply([object_message]) # RTLM21g + await realtime_object._publish_and_apply([map_remove_message(self.object_id, key)]) # RTLM21g def _publishing_realtime_object(self) -> RealtimeObject: realtime_object = self.realtime_object @@ -288,7 +279,8 @@ def apply_map_remove(self, map_remove: MapRemove, serial: str | None, serial_tim def apply_map_clear(self, serial: str | None, object_message: ObjectMessage) -> LiveMapUpdate: """RTLM24: applies a MAP_CLEAR.""" if not serial: - # A MAP_CLEAR with no serial cannot be ordered against the map's entries + # A MAP_CLEAR with no serial cannot be ordered against the map's entries. `apply_operation` + # never gets here, as it rejects a message with no serial (RTLO4a3); a direct call can. log.warning(f'InternalLiveMap.apply_map_clear(): skipping a MAP_CLEAR with no serial; ' f'object_id={self.object_id}') return LiveMapUpdate(noop=True) @@ -418,3 +410,42 @@ def _is_at_or_before_clear(self, serial: str | None) -> bool: def _log_missing_payload(self, action: ObjectOperationAction) -> None: log.warning(f'InternalLiveMap.apply_operation(): skipping a {action.name} operation with no key or ' f'value; object_id={self.object_id}') + + +def map_set_message(object_id: str, key: str, data: ObjectData) -> ObjectMessage: + """RTLM20e: the MAP_SET of `key` in the map `object_id` to `data`.""" + return ObjectMessage(operation=ObjectOperation( + action=ObjectOperationAction.MAP_SET, # RTLM20e2 + object_id=object_id, # RTLM20e3 + map_set=MapSet(key=key, value=data), # RTLM20e6, RTLM20e7 + )) + + +def map_remove_message(object_id: str, key: str) -> ObjectMessage: + """RTLM21e: the MAP_REMOVE of `key` from the map `object_id`.""" + return ObjectMessage(operation=ObjectOperation( + action=ObjectOperationAction.MAP_REMOVE, # RTLM21e2 + object_id=object_id, # RTLM21e3 + map_remove=MapRemove(key=key), # RTLM21e5 + )) + + +async def map_set_messages(realtime_object: RealtimeObject, object_id: str, key: str, + value: LiveMapValue) -> list[ObjectMessage]: + """RTLM20e: the ObjectMessages setting `key` of the map `object_id` to `value`. + + For a `LiveMap` or `LiveCounter`, they are the creates it evaluates to, with object ids + generated from the server time (RTLM20e7g1), followed by a MAP_SET referencing the last + of them (RTLM20e7g2); for a primitive, just the MAP_SET (RTLM20e7b-RTLM20e7f). + + Raises as `validate_value` does for an invalid value (RTLM20e1), a blueprint before the + server time is fetched. + """ + if not isinstance(value, (LiveCounter, LiveMap)): + return [map_set_message(object_id, key, primitive_to_object_data(value))] + + validate_value(value) + server_time_ms = await realtime_object._get_server_time_ms() + object_messages = evaluate(value, server_time_ms) + data = ObjectData(object_id=object_messages[-1].operation.object_id) + return [*object_messages, map_set_message(object_id, key, data)] diff --git a/ably/pubsub/objects/liveobject.py b/ably/pubsub/objects/liveobject.py index cc75a3d5..47bf61b5 100644 --- a/ably/pubsub/objects/liveobject.py +++ b/ably/pubsub/objects/liveobject.py @@ -8,13 +8,13 @@ from typing import TYPE_CHECKING, Any, Callable from ably.pubsub.objects.defaults import ROOT_OBJECT_ID -from ably.pubsub.objects.subscription import Subscription +from ably.pubsub.objects.objectmessage import ObjectMessage +from ably.pubsub.objects.subscription import Registry, Subscription from ably.pubsub.util.clock import Clock if TYPE_CHECKING: from ably.pubsub.objects.enums import ObjectsOperationSource from ably.pubsub.objects.livemap import InternalLiveMap - from ably.pubsub.objects.objectmessage import ObjectMessage from ably.pubsub.objects.objectspool import ObjectsPool from ably.pubsub.objects.realtimeobject import RealtimeObject @@ -85,9 +85,7 @@ def __init__(self, object_id: str, *, pool: ObjectsPool | None = None, clock: Cl self.pool: ObjectsPool | None = pool self.data: Any = None self._clock: Clock | None = clock - # (token, listener) for each subscribe call, so that one registration can be removed - # even when the same listener is registered more than once - self._listeners: list[tuple[object, Callable[[LiveObjectUpdate], None]]] = [] + self._listeners: Registry[Callable[[LiveObjectUpdate], None]] = Registry() @property def clock(self) -> Clock: @@ -108,13 +106,7 @@ def subscribe(self, listener: Callable[[LiveObjectUpdate], None]) -> Subscriptio Listeners are called synchronously from `notify_updated`, in registration order. """ - token = object() - self._listeners.append((token, listener)) - - def deregister() -> None: - self._listeners = [entry for entry in self._listeners if entry[0] is not token] - - return Subscription(deregister) + return Subscription(self._listeners.register(listener)) def notify_updated(self, update: LiveObjectUpdate) -> None: """RTLO4b4c: emits `update`. @@ -123,14 +115,17 @@ def notify_updated(self, update: LiveObjectUpdate) -> None: no-op stops here (RTLO4b4c1). Otherwise the `subscribe` listeners are called (RTLO4b4c3a), path subscriptions are dispatched through the `RealtimeObject`, if there is one (RTLO4b4c3b), and a tombstone update then deregisters the `subscribe` - listeners (RTLO4b4c3c). + listeners (RTLO4b4c3c). A listener registered during the dispatch is not called by it, + and one deregistered during it is not called again. """ if update.noop: return - for _, listener in list(self._listeners): + for registration in self._listeners.snapshot(): + if not registration.active: + continue try: - listener(update) + registration.item(update) except Exception: log.exception(f'LiveObject.notify_updated(): a subscription listener raised; ' f'object_id={self.object_id}') @@ -140,7 +135,7 @@ def notify_updated(self, update: LiveObjectUpdate) -> None: realtime_object._path_object_subscription_register.dispatch(self, update) if update.tombstone: - self._listeners = [] + self._listeners.clear() def can_apply_operation(self, object_message: ObjectMessage) -> bool: """RTLO4a: whether `object_message`'s serial is newer than this object's for its site.""" diff --git a/ably/pubsub/objects/objectmessage.py b/ably/pubsub/objects/objectmessage.py index 3183199d..6dce9407 100644 --- a/ably/pubsub/objects/objectmessage.py +++ b/ably/pubsub/objects/objectmessage.py @@ -22,21 +22,45 @@ Fields that are only ever held locally (`ObjectsMapEntry.tombstoned_at`, RTLM3a1, and the `derived_from` of the `*CreateWithObjectId` payloads, RTLMV4j5 and RTLCV4g5) are never encoded. + +A ProtocolMessage's `state` is decoded one `ObjectMessage` at a time: a message that fails +to decode is logged and left out, and the others are decoded. """ from __future__ import annotations import base64 +import builtins import json +import logging from dataclasses import dataclass, field from enum import IntEnum -from typing import Any +from typing import Any, TypeVar + +log = logging.getLogger(__name__) WIRE_FORMAT_JSON = 'json' WIRE_FORMAT_MSGPACK = 'msgpack' +E = TypeVar('E', bound='_WireEnum') + + +class _WireEnum(IntEnum): + """An enumeration carried as its integer wire value, whose `UNKNOWN` member stands for a wire + value this library does not recognise.""" -class ObjectOperationAction(IntEnum): + @classmethod + def from_wire(cls: type[E], value: Any) -> E: + """The member for a wire value, or `UNKNOWN` for one that is not recognised (OOP2a, OMP2a).""" + if isinstance(value, bool): + return cls.UNKNOWN + try: + return cls(value) + except ValueError: + return cls.UNKNOWN + + +class ObjectOperationAction(_WireEnum): """OOP2: the operation an `ObjectOperation` describes, by its wire value. `UNKNOWN` stands for a wire value this library does not recognise (OOP2a). Such an @@ -52,18 +76,8 @@ class ObjectOperationAction(IntEnum): OBJECT_DELETE = 5 MAP_CLEAR = 6 - @classmethod - def from_wire(cls, value: Any) -> ObjectOperationAction: - """The member for a wire value, or `UNKNOWN` for one that is not recognised (OOP2a).""" - if isinstance(value, bool): - return cls.UNKNOWN - try: - return cls(value) - except ValueError: - return cls.UNKNOWN - -class ObjectsMapSemantics(IntEnum): +class ObjectsMapSemantics(_WireEnum): """OMP2: the conflict-resolution semantics of a map, by its wire value. `UNKNOWN` stands for a wire value this library does not recognise (OMP2a). @@ -72,16 +86,6 @@ class ObjectsMapSemantics(IntEnum): UNKNOWN = -1 LWW = 0 - @classmethod - def from_wire(cls, value: Any) -> ObjectsMapSemantics: - """The member for a wire value, or `UNKNOWN` for one that is not recognised (OMP2a).""" - if isinstance(value, bool): - return cls.UNKNOWN - try: - return cls(value) - except ValueError: - return cls.UNKNOWN - def _check_format(format: str) -> None: if format not in (WIRE_FORMAT_JSON, WIRE_FORMAT_MSGPACK): @@ -131,7 +135,8 @@ class ObjectData: object_id: str | None = None # OD2a encoding: str | None = None # OD2b boolean: bool | None = None # OD2c - bytes: bytes | None = None # OD2d + # Spelled `builtins.bytes`, as the field's own name hides the builtin within the class + bytes: builtins.bytes | None = None # OD2d number: float | None = None # OD2e string: str | None = None # OD2f json: dict | list | None = None # OD2g @@ -625,12 +630,26 @@ def from_protocol_message(protocol_message: dict, format: str = WIRE_FORMAT_JSON """Decodes the `state` array of an OBJECT or OBJECT_SYNC ProtocolMessage. A message with no `id`, `connectionId` or `timestamp` takes them from the - ProtocolMessage that carried it (OM2a, OM2c, OM2e). + ProtocolMessage that carried it (OM2a, OM2c, OM2e). A message that fails to decode is + logged and left out, so that it costs neither the other messages nor the handling of + the ProtocolMessage itself, such as the sync cursor of an OBJECT_SYNC. """ + channel = protocol_message.get('channel') + entries = protocol_message.get('state') or [] + if not isinstance(entries, list): + log.error(f'ObjectMessage.from_protocol_message(): skipping a state that is not an array; ' + f'channel={channel}, state={entries!r}') + return [] + messages = [] protocol_id = protocol_message.get('id') - for index, entry in enumerate(protocol_message.get('state') or []): - message = ObjectMessage.from_dict(entry, format) + for index, entry in enumerate(entries): + try: + message = ObjectMessage.from_dict(entry, format) + except Exception as e: + log.error(f'ObjectMessage.from_protocol_message(): skipping an object message that failed to ' + f'decode; index={index}, channel={channel}, error={e!r}') + continue if message.id is None and protocol_id is not None: message.id = f'{protocol_id}:{index}' if message.connection_id is None: diff --git a/ably/pubsub/objects/pathobject.py b/ably/pubsub/objects/pathobject.py index 43f9d6f9..1c8d9928 100644 --- a/ably/pubsub/objects/pathobject.py +++ b/ably/pubsub/objects/pathobject.py @@ -20,7 +20,6 @@ from dataclasses import dataclass from typing import TYPE_CHECKING, Any, Callable, overload -from ably.pubsub.objects.batch import Batch, LiveCounterBatchContext, LiveMapBatchContext from ably.pubsub.objects.instance import ( Instance, compact_value, @@ -30,11 +29,12 @@ ) from ably.pubsub.objects.livecounter import InternalLiveCounter from ably.pubsub.objects.livemap import InternalLiveMap +from ably.pubsub.objects.publicmessage import ObjectMessage from ably.pubsub.util.exceptions import AblyException if TYPE_CHECKING: + from ably.pubsub.objects.batch import Batch, LiveCounterBatchContext, LiveMapBatchContext from ably.pubsub.objects.enums import ValueType - from ably.pubsub.objects.publicmessage import ObjectMessage from ably.pubsub.objects.realtimeobject import RealtimeObject from ably.pubsub.objects.subscription import Subscription from ably.pubsub.objects.valuetypes import LiveMapValue, Primitive, T @@ -190,6 +190,9 @@ def batch(self) -> Batch[LiveMapBatchContext]: Entering checks the write preconditions (RTPO20b) and raises AblyException 92007 if the path does not resolve to a map (RTPO20c). """ + # Imported here, as the batch module builds on the views + from ably.pubsub.objects.batch import Batch, LiveMapBatchContext + return Batch(self._realtime_object, self._resolve, LiveMapBatchContext, f'path {self.path()!r}') def entries(self) -> list[tuple[str, PathObject]]: @@ -242,6 +245,9 @@ def batch(self) -> Batch[LiveCounterBatchContext]: Entering checks the write preconditions (RTPO20b) and raises AblyException 92007 if the path does not resolve to a counter (RTPO20c). """ + # Imported here, as the batch module builds on the views + from ably.pubsub.objects.batch import Batch, LiveCounterBatchContext + return Batch(self._realtime_object, self._resolve, LiveCounterBatchContext, f'path {self.path()!r}') def value(self) -> float | None: diff --git a/ably/pubsub/objects/pathobjectsubscriptionregister.py b/ably/pubsub/objects/pathobjectsubscriptionregister.py index d33fbf3e..aa2c7ff7 100644 --- a/ably/pubsub/objects/pathobjectsubscriptionregister.py +++ b/ably/pubsub/objects/pathobjectsubscriptionregister.py @@ -8,7 +8,7 @@ from ably.pubsub.objects.liveobject import LiveMapUpdate from ably.pubsub.objects.pathobject import PathObject, PathObjectSubscriptionEvent from ably.pubsub.objects.publicmessage import ObjectMessage -from ably.pubsub.objects.subscription import Subscription +from ably.pubsub.objects.subscription import Registry, Subscription if TYPE_CHECKING: from ably.pubsub.objects.liveobject import LiveObject, LiveObjectUpdate @@ -25,7 +25,6 @@ def __init__(self, path: list[str], listener: Callable[[PathObjectSubscriptionEv self.path = path self.listener = listener self.depth = depth - self.active = True class PathObjectSubscriptionRegister: @@ -33,29 +32,23 @@ class PathObjectSubscriptionRegister: def __init__(self, realtime_object: RealtimeObject): self.realtime_object = realtime_object - self._subscriptions: list[_PathSubscription] = [] + self._subscriptions: Registry[_PathSubscription] = Registry() def subscribe(self, path: list[str], listener: Callable[[PathObjectSubscriptionEvent], None], depth: int | None = None) -> Subscription: """RTPO19f: registers `listener` for changes covered by `path` and `depth` (RTO24c1).""" - subscription = _PathSubscription(list(path), listener, depth) - self._subscriptions.append(subscription) - - def deregister() -> None: - subscription.active = False - self._subscriptions = [entry for entry in self._subscriptions if entry is not subscription] - - return Subscription(deregister) + return Subscription(self._subscriptions.register(_PathSubscription(list(path), listener, depth))) def dispatch(self, live_object: LiveObject, update: LiveObjectUpdate) -> None: - """RTO24b: calls each subscription covering a path to `live_object` once. + """RTO24b: calls each subscription once for each path to `live_object` that it covers. A listener that raises is logged and does not stop the others (RTO24b2c). A - subscription made during a dispatch is not called by it, and one removed during a - dispatch is not called again. + subscription made during a dispatch is not called by it, for any path, and one removed + during a dispatch is not called again. """ if not self._subscriptions: return + registrations = self._subscriptions.snapshot() realtime_object = self.realtime_object root = realtime_object._objects_pool.root @@ -67,9 +60,10 @@ def dispatch(self, live_object: LiveObject, update: LiveObjectUpdate) -> None: for path_to_this in live_object.get_full_paths(): # RTO24b1, RTO24b2 # RTO24b2a1, RTO24b2a2 candidate_paths = [path_to_this, *([*path_to_this, key] for key in updated_keys)] - for subscription in list(self._subscriptions): - if not subscription.active: + for registration in registrations: + if not registration.active: continue + subscription = registration.item # RTO24b2b event_path = next((candidate for candidate in candidate_paths if self.covers(subscription.path, subscription.depth, candidate)), None) diff --git a/ably/pubsub/objects/realtimeobject.py b/ably/pubsub/objects/realtimeobject.py index 5d61b960..c2276b11 100644 --- a/ably/pubsub/objects/realtimeobject.py +++ b/ably/pubsub/objects/realtimeobject.py @@ -5,7 +5,7 @@ import asyncio import dataclasses import logging -from typing import TYPE_CHECKING, Callable +from typing import TYPE_CHECKING, Any, Callable from ably.pubsub.objects.defaults import GC_GRACE_PERIOD_MS, GC_INTERVAL_MS, ROOT_OBJECT_ID from ably.pubsub.objects.enums import ObjectsEvent, ObjectsOperationSource, ObjectsSyncState @@ -21,7 +21,7 @@ from ably.pubsub.objects.objectspool import ObjectsPool from ably.pubsub.objects.pathobject import LiveMapPathObject from ably.pubsub.objects.pathobjectsubscriptionregister import PathObjectSubscriptionRegister -from ably.pubsub.objects.subscription import StatusSubscription +from ably.pubsub.objects.subscription import Registry, StatusSubscription from ably.pubsub.objects.syncobjectspool import SyncObjectsPool from ably.pubsub.transport.websockettransport import ProtocolMessageAction from ably.pubsub.types.channelmode import ChannelMode @@ -32,6 +32,7 @@ if TYPE_CHECKING: from ably.pubsub.objects.objectmessage import ObjectMessage from ably.pubsub.realtime.channel import RealtimeChannel + from ably.pubsub.types.connectiondetails import ConnectionDetails from ably.pubsub.types.operations import PublishResult from ably.pubsub.util.helper import Timer @@ -57,7 +58,8 @@ # RTO23c1, RTO20e1: the channel states that fail a wait for SYNCED _SYNC_WAIT_FAILURE_STATES = frozenset((ChannelState.DETACHED, ChannelState.SUSPENDED, ChannelState.FAILED)) -# RTO27a: the channel states after which the objects' data can no longer be known +# RTO27a: the channel states after which the objects' data can no longer be known. The channel +# leaves them only by attaching again, so a wait for SYNCED begun in one of them fails at once. _DATA_CLEARING_STATES = frozenset((ChannelState.DETACHED, ChannelState.FAILED)) @@ -90,19 +92,21 @@ def __init__(self, channel: RealtimeChannel | None = None, *, pool: ObjectsPool self._buffered_object_operations: list[ObjectMessage] = [] # RTO7a, RTO7a1 self._applied_on_ack_serials: set[str] = set() # RTO7b, RTO7b1 self._current_sync_id: str | None = None # RTO5a1 - self._current_sync_cursor: str | None = None # RTO5a1 self._path_object_subscription_register = PathObjectSubscriptionRegister(self) # RTO24a self._gc_interval_ms: int = GC_INTERVAL_MS # RTO10a, read each time the GC timer is scheduled self._gc_timer: Timer | None = None - self._last_gc_grace_period_ms: int = GC_GRACE_PERIOD_MS # RTO10b + self._gc_grace_period_ms: int = GC_GRACE_PERIOD_MS # RTO10b3, until a CONNECTED gives one self._server_time_offset_ms: float | None = None # RTO16a # RTO23c, RTO20e: each pending wait for SYNCED, with the description its 92008 error gives self._sync_waiters: dict[asyncio.Future[None], str] = {} - # RTO18: (token, callback) for each `on` call, by event, so that a callback registered - # twice is called twice (RTO18d) and each StatusSubscription removes only its own - self._sync_state_listeners: dict[ObjectsEvent, list[tuple[object, Callable[[], None]]]] = { - event: [] for event in ObjectsEvent + # RTO18: the callbacks `on` registered, by event + self._sync_state_listeners: dict[ObjectsEvent, Registry[Callable[[], None]]] = { + event: Registry() for event in ObjectsEvent } + # Whether the channel has been released, after which it receives nothing more + self._released = False + if channel is not None: + self._on_connected(channel.ably.connection.connection_details) # RTO10b1 # Public API @@ -111,8 +115,9 @@ async def get(self) -> LiveMapPathObject: Requires the OBJECT_SUBSCRIBE mode (RTO23a, 40024), attaches the channel if it is not attached (RTO23e, RTL33), and waits for the sync state to reach SYNCED (RTO23c). - Raises AblyException 92008 if the channel enters DETACHED, SUSPENDED or FAILED while - waiting (RTO23c1), and 90001 if the channel is FAILED (RTL33c). + Raises AblyException 92008 if the channel is or enters DETACHED or FAILED, or enters + SUSPENDED, while waiting (RTO23c1), 90001 if the channel is FAILED (RTL33c), and the + attach's error if attaching fails (RTO23e). """ self._throw_if_missing_channel_mode(ChannelMode.OBJECT_SUBSCRIBE) # RTO23a if self._channel is not None: @@ -126,23 +131,17 @@ async def get(self) -> LiveMapPathObject: def on(self, event: ObjectsEvent, callback: Callable[[], None]) -> StatusSubscription: """RTO18: calls `callback`, with no arguments, whenever the sync state reaches `event`. - Registering one callback twice calls it twice (RTO18d). + Registering one callback twice calls it twice (RTO18d). Raises AblyException 40003 for + an `event` that is not an `ObjectsEvent`. """ - event = ObjectsEvent(event) - token = object() - self._sync_state_listeners[event].append((token, callback)) # RTO18c - - def deregister() -> None: - self._sync_state_listeners[event] = [ - entry for entry in self._sync_state_listeners[event] if entry[0] is not token] - - return StatusSubscription(deregister) # RTO18f + return StatusSubscription(self._sync_state_listeners[_objects_event(event)].register(callback)) # RTO18c def off(self, event: ObjectsEvent, callback: Callable[[], None]) -> None: - """RTO19: deregisters `callback` from `event`, however many times it was registered.""" - event = ObjectsEvent(event) - self._sync_state_listeners[event] = [ - entry for entry in self._sync_state_listeners[event] if entry[1] != callback] + """RTO19: deregisters `callback` from `event`, however many times it was registered. + + Raises AblyException 40003 for an `event` that is not an `ObjectsEvent`. + """ + self._sync_state_listeners[_objects_event(event)].deregister(callback) # Internal API: inbound protocol messages and channel state @@ -159,7 +158,8 @@ def _wire_format(self) -> str: return WIRE_FORMAT_JSON def _on_attached(self, has_objects: bool) -> None: - """RTO4: handles an ATTACHED ProtocolMessage, whatever the channel's state was. + """RTO4: handles an ATTACHED ProtocolMessage the channel received while ATTACHING or + ATTACHED; the channel ignores one received in any other state. `has_objects` is the HAS_OBJECTS flag. Starts the GC timer if it is not running and there is a channel (RTO10). @@ -190,6 +190,27 @@ def _on_attached(self, has_objects: bool) -> None: self._schedule_gc_timer() # RTO10a + def _on_connected(self, connection_details: ConnectionDetails | None) -> None: + """RTO10b2: takes the GC grace period from the ConnectionDetails of a CONNECTED. + + With no details, as before the first CONNECTED, the grace period is left as it is. + """ + if connection_details is None: + return + grace_period_ms = connection_details.objects_gc_grace_period + # RTO10b1, RTO10b3 + self._gc_grace_period_ms = grace_period_ms if grace_period_ms is not None else GC_GRACE_PERIOD_MS + + def _release(self) -> None: + """Stops this RealtimeObject when its channel is released (RTS4). + + A released channel receives nothing more, so the GC timer stops for good, and every + wait for SYNCED, pending or later, fails with 92008. + """ + self._released = True + self._cancel_gc_timer() + self._fail_sync_waiters('the channel being released', self._channel_error_reason) + def _handle_object_sync_messages(self, object_messages: list[ObjectMessage], sync_channel_serial: str | None) -> None: """RTO5: handles the decoded `state` of an OBJECT_SYNC ProtocolMessage.""" @@ -200,7 +221,6 @@ def _handle_object_sync_messages(self, object_messages: list[ObjectMessage], # whole sync of its own (RTO5a5), so it too discards anything accumulated before it. if sync_id is None or sync_id != self._current_sync_id: self._start_sync_sequence(sync_id) - self._current_sync_cursor = sync_cursor self._sync_objects_pool.apply_object_sync_messages(object_messages) # RTO5f @@ -217,37 +237,50 @@ def _handle_object_messages(self, object_messages: list[ObjectMessage]) -> None: def _apply_object_messages(self, object_messages: list[ObjectMessage], source: ObjectsOperationSource) -> None: - """RTO9: applies operations to the pool, recording LOCAL serials in `_applied_on_ack_serials`.""" + """RTO9: applies operations to the pool, recording LOCAL serials in `_applied_on_ack_serials`. + + An operation that raises as it is applied is logged and skipped, and the rest are + applied. + """ for object_message in object_messages: - operation = object_message.operation - if operation is None: - # RTO9a1 - log.warning(f'RealtimeObject._apply_object_messages(): skipping an object message with no ' - f'operation; message id={object_message.id}, channel={self._channel_name}') - continue + try: + self._apply_object_message(object_message, source) + except Exception: + log.exception(f'RealtimeObject._apply_object_messages(): skipping an object message that ' + f'could not be applied; message id={object_message.id}, ' + f'channel={self._channel_name}') - serial = object_message.serial - if serial is not None and serial in self._applied_on_ack_serials: - # RTO9a3 - log.debug(f'RealtimeObject._apply_object_messages(): skipping an operation already applied ' - f'on ACK; serial={serial}, channel={self._channel_name}') - self._applied_on_ack_serials.discard(serial) - continue + def _apply_object_message(self, object_message: ObjectMessage, source: ObjectsOperationSource) -> None: + """RTO9a: applies one operation to the pool.""" + operation = object_message.operation + if operation is None: + # RTO9a1 + log.warning(f'RealtimeObject._apply_object_message(): skipping an object message with no ' + f'operation; message id={object_message.id}, channel={self._channel_name}') + return - if operation.action not in _SUPPORTED_ACTIONS: - # RTO9a2b - log.warning(f'RealtimeObject._apply_object_messages(): skipping an object message with an ' - f'unsupported action; action={operation.action!r}, ' - f'message id={object_message.id}, channel={self._channel_name}') - continue + serial = object_message.serial + if serial is not None and serial in self._applied_on_ack_serials: + # RTO9a3 + log.debug(f'RealtimeObject._apply_object_message(): skipping an operation already applied ' + f'on ACK; serial={serial}, channel={self._channel_name}') + self._applied_on_ack_serials.discard(serial) + return - # RTO9a2a1, RTO9a2a2 - live_object = self._objects_pool.create_zero_value_object_if_not_exists(operation.object_id) - if live_object is None: - continue - applied = live_object.apply_operation(object_message, source) # RTO9a2a3 - if source == ObjectsOperationSource.LOCAL and applied: - self._applied_on_ack_serials.add(serial) # RTO9a2a4 + if operation.action not in _SUPPORTED_ACTIONS: + # RTO9a2b + log.warning(f'RealtimeObject._apply_object_message(): skipping an object message with an ' + f'unsupported action; action={operation.action!r}, ' + f'message id={object_message.id}, channel={self._channel_name}') + return + + # RTO9a2a1, RTO9a2a2 + live_object = self._objects_pool.create_zero_value_object_if_not_exists(operation.object_id) + if live_object is None: + return + applied = live_object.apply_operation(object_message, source) # RTO9a2a3 + if source == ObjectsOperationSource.LOCAL and applied: + self._applied_on_ack_serials.add(serial) # RTO9a2a4 def _act_on_channel_state(self, state: ChannelState, reason: AblyException | None = None) -> None: """RTO27: handles the channel entering `state`, for every state but ATTACHED. @@ -259,7 +292,8 @@ def _act_on_channel_state(self, state: ChannelState, reason: AblyException | Non next ATTACHED starts again. """ if state in _SYNC_WAIT_FAILURE_STATES: - self._fail_sync_waiters(state, reason) + cause = reason if reason is not None else self._channel_error_reason + self._fail_sync_waiters(_entering(state), cause) if state in _DATA_CLEARING_STATES: self._objects_pool.clear_all_data() # RTO27a1 @@ -278,9 +312,11 @@ def _set_sync_state(self, state: ObjectsSyncState) -> None: event = _SYNC_STATE_EVENTS.get(state) if event is None: return - for _, callback in list(self._sync_state_listeners[event]): + for registration in self._sync_state_listeners[event].snapshot(): + if not registration.active: + continue try: - callback() # RTO18e + registration.item() # RTO18e except Exception: log.exception(f'RealtimeObject._set_sync_state(): a {event.value} listener raised; ' f'channel={self._channel_name}') @@ -306,10 +342,13 @@ def _start_sync_sequence(self, sync_id: str | None) -> None: """RTO5a2: discards whatever an earlier sync sequence accumulated and starts `sync_id`.""" self._sync_objects_pool.clear() # RTO5a2a self._current_sync_id = sync_id - self._current_sync_cursor = None def _complete_sync(self) -> None: - """RTO5c: applies the objects the sync sequence delivered, then the operations buffered meanwhile.""" + """RTO5c: applies the objects the sync sequence delivered, then the operations buffered meanwhile. + + An object whose state raises as it is applied is logged and left as the error left it, + so that the sync still completes. + """ pool = self._objects_pool received_object_ids: set[str] = set() updates: list[tuple[LiveObject, LiveObjectUpdate]] = [] @@ -317,22 +356,15 @@ def _complete_sync(self) -> None: # RTO5c1 for object_id, object_message in self._sync_objects_pool.entries.items(): received_object_ids.add(object_id) - existing = pool.get(object_id) - if existing is not None: - # RTO5c1a1, RTO5c1a2 - updates.append((existing, existing.replace_data(object_message))) + try: + replaced = self._replace_object_state(object_id, object_message) + except Exception: + log.exception(f'RealtimeObject._complete_sync(): skipping an object state that could not be ' + f'applied; object_id={object_id}, message id={object_message.id}, ' + f'channel={self._channel_name}') continue - - # RTO5c1b1 - object_state = object_message.object - if object_state.counter is not None: - live_object: LiveObject = InternalLiveCounter(object_id) # RTO5c1b1a - else: - semantics = object_state.map.semantics - live_object = InternalLiveMap( - object_id, semantics if semantics is not None else ObjectsMapSemantics.LWW) # RTO5c1b1b - pool[object_id] = live_object - live_object.replace_data(object_message) + if replaced is not None: + updates.append(replaced) # RTO5c2, RTO5c2a for object_id in list(pool): @@ -348,20 +380,53 @@ def _complete_sync(self) -> None: self._apply_object_messages(self._buffered_object_operations, ObjectsOperationSource.CHANNEL) # RTO5c6 self._current_sync_id = None # RTO5c3 - self._current_sync_cursor = None # RTO5c3 self._sync_objects_pool.clear() # RTO5c4 self._buffered_object_operations = [] # RTO5c5 self._applied_on_ack_serials.clear() # RTO5c9 self._set_sync_state(ObjectsSyncState.SYNCED) # RTO5c8 + def _replace_object_state(self, object_id: str, + object_message: ObjectMessage) -> tuple[LiveObject, LiveObjectUpdate] | None: + """RTO5c1: applies one synced object state to the pool. + + Returns the object and its update when it replaced an existing object's data, whose + update RTO5c7 emits, and None when it created the object. + """ + existing = self._objects_pool.get(object_id) + if existing is not None: + return existing, existing.replace_data(object_message) # RTO5c1a1, RTO5c1a2 + + # RTO5c1b1 + object_state = object_message.object + if object_state.counter is not None: + live_object: LiveObject = InternalLiveCounter(object_id) # RTO5c1b1a + else: + semantics = object_state.map.semantics + live_object = InternalLiveMap( + object_id, semantics if semantics is not None else ObjectsMapSemantics.LWW) # RTO5c1b1b + self._objects_pool[object_id] = live_object + live_object.replace_data(object_message) + return None + # Internal API: waiting for SYNCED async def _wait_for_synced(self, failure_description: str) -> None: """RTO23c, RTO20e: waits for the sync state to reach SYNCED. Raises AblyException 92008, whose message starts with `failure_description`, if the - channel enters DETACHED, SUSPENDED or FAILED first (RTO23c1, RTO20e1). + channel enters DETACHED, SUSPENDED or FAILED first (RTO23c1, RTO20e1). A channel + already DETACHED or FAILED, or released, can never sync without attaching again, so + the wait then fails at once. """ + if self._released: + raise _sync_wait_failure(failure_description, 'the channel being released', + self._channel_error_reason) + channel = self._channel + if channel is not None and channel.state in _DATA_CLEARING_STATES: + # The channel left ATTACHED before the wait began, as when an ATTACHED or an ACK and a + # channel ERROR are read together + raise _sync_wait_failure(failure_description, _entering(channel.state), channel.error_reason) + future: asyncio.Future[None] = asyncio.get_running_loop().create_future() self._sync_waiters[future] = failure_description try: @@ -375,17 +440,16 @@ def _resolve_sync_waiters(self) -> None: if not future.done(): future.set_result(None) - def _fail_sync_waiters(self, state: ChannelState, reason: AblyException | None) -> None: - """RTO23c1, RTO20e1: fails every wait for SYNCED with 92008, caused by the channel's error.""" + def _fail_sync_waiters(self, reason: str, cause: AblyException | None) -> None: + """RTO23c1, RTO20e1: fails every wait for SYNCED with 92008, `reason` saying why.""" waiters, self._sync_waiters = self._sync_waiters, {} - cause = reason - if cause is None and self._channel is not None: - cause = self._channel.error_reason for future, failure_description in waiters.items(): if not future.done(): - future.set_exception(AblyException( - f'{failure_description} due to the channel entering the {state.value} state ' - f'whilst waiting for objects sync to complete', 400, 92008, cause=cause)) + future.set_exception(_sync_wait_failure(failure_description, reason, cause)) + + @property + def _channel_error_reason(self) -> AblyException | None: + return self._channel.error_reason if self._channel is not None else None # Internal API: publishing @@ -415,9 +479,11 @@ async def _publish_and_apply(self, object_messages: list[ObjectMessage]) -> None """RTO20: publishes `object_messages`, then applies them locally with the serials the ACK assigned, as LOCAL operations, once synced. - Raises the publish's error (RTO20b), and AblyException 92008 if the channel enters - DETACHED, SUSPENDED or FAILED while waiting for SYNCED (RTO20e1). An operation that - cannot be applied locally is left for its echo to apply (RTO20c, RTO20d1). + Raises the publish's error (RTO20b), and AblyException 92008 if the channel is or + enters DETACHED or FAILED, or enters SUSPENDED, while waiting for SYNCED (RTO20e1). An + operation that cannot be applied locally is left for its echo to apply (RTO20c, + RTO20d1), and nothing is applied once the channel is DETACHED or FAILED, as its + objects' data has been cleared (RTO27a). """ publish_result = await self._publish(object_messages) # RTO20b @@ -456,6 +522,12 @@ async def _publish_and_apply(self, object_messages: list[ObjectMessage]) -> None # RTO20e, RTO20e1 await self._wait_for_synced('The operation could not be applied locally') + if self._channel.state in _DATA_CLEARING_STATES: + # The channel left ATTACHED after the ACK, and before this resumed + log.debug(f'RealtimeObject._publish_and_apply(): the operations will not be applied locally, as ' + f'the channel is {self._channel.state.value}; channel={self._channel_name}') + return + self._apply_object_messages(synthetic_messages, ObjectsOperationSource.LOCAL) # RTO20f async def _get_server_time_ms(self) -> int: @@ -525,25 +597,13 @@ def _throw_if_in_channel_state(self, *states: ChannelState) -> None: # Internal API: garbage collection - @property - def _gc_grace_period_ms(self) -> int: - """RTO10b: `ConnectionDetails.objects_gc_grace_period` from the latest CONNECTED, or the default.""" - connection_details = None - if self._channel is not None: - connection_details = self._channel.ably.connection.connection_details - if connection_details is not None: - # RTO10b1, RTO10b2, RTO10b3 - grace_period_ms = connection_details.objects_gc_grace_period - self._last_gc_grace_period_ms = grace_period_ms if grace_period_ms is not None else GC_GRACE_PERIOD_MS - return self._last_gc_grace_period_ms - def _schedule_gc_timer(self) -> None: """RTO10a: schedules `_on_gc_interval` on the clock, `_gc_interval_ms` from now, if there is a - channel and no sweep is scheduled already. + channel, it has not been released, and no sweep is scheduled already. A `RealtimeObject` with no channel schedules nothing. """ - if self._channel is None or self._gc_timer is not None: + if self._channel is None or self._released or self._gc_timer is not None: return self._gc_timer = self._clock.timer(self._gc_interval_ms, self._on_gc_interval) @@ -553,9 +613,33 @@ def _cancel_gc_timer(self) -> None: self._gc_timer = None def _on_gc_interval(self) -> None: - """RTO10c: releases what has been tombstoned for the grace period, and reschedules itself.""" + """RTO10c: releases what has been tombstoned for the grace period, and reschedules itself. + + A sweep that raises is logged, and the next one is scheduled all the same. + """ self._gc_timer = None try: self._objects_pool.collect_garbage(self._gc_grace_period_ms, self._clock.now_ms()) - finally: - self._schedule_gc_timer() + except Exception: + log.exception(f'RealtimeObject._on_gc_interval(): the GC sweep raised; channel={self._channel_name}') + self._schedule_gc_timer() + + +def _objects_event(event: Any) -> ObjectsEvent: + """`event` as an `ObjectsEvent`, raising AblyException 40003 if it is not one.""" + try: + return ObjectsEvent(event) + except ValueError: + events = ', '.join(repr(member.value) for member in ObjectsEvent) + raise AblyException(f'Objects event should be one of {events}; got {event!r}', 400, 40003) from None + + +def _entering(state: ChannelState) -> str: + """Why a wait for SYNCED fails, when the channel enters `state`.""" + return f'the channel entering the {state.value} state' + + +def _sync_wait_failure(failure_description: str, reason: str, cause: AblyException | None) -> AblyException: + """RTO23c1, RTO20e1: the 92008 error a wait for SYNCED fails with, `reason` saying why.""" + return AblyException(f'{failure_description} due to {reason} whilst waiting for objects sync to complete', + 400, 92008, cause=cause) diff --git a/ably/pubsub/objects/subscription.py b/ably/pubsub/objects/subscription.py index e8082758..63dffb1e 100644 --- a/ably/pubsub/objects/subscription.py +++ b/ably/pubsub/objects/subscription.py @@ -1,31 +1,86 @@ -"""The handles a subscribe call returns.""" +"""The handles a subscribe call returns, and the registry of listeners behind them.""" from __future__ import annotations -from typing import Callable +from typing import Callable, Generic, TypeVar +T = TypeVar('T') -class Subscription: - """SUB1: a registration for receiving events from a `subscribe` call.""" + +class _Handle: + """Calls `deregister` the first time the handle is released, and never again.""" def __init__(self, deregister: Callable[[], None]): self._deregister: Callable[[], None] | None = deregister - def unsubscribe(self) -> None: - """SUB2a: deregisters the listener. Calling it again does nothing (SUB2b).""" + def _release(self) -> None: deregister, self._deregister = self._deregister, None if deregister is not None: deregister() -class StatusSubscription: - """RTO18f: the registration `RealtimeObject.on` returns.""" +class Subscription(_Handle): + """SUB1: a registration for receiving events from a `subscribe` call.""" - def __init__(self, deregister: Callable[[], None]): - self._deregister: Callable[[], None] | None = deregister + def unsubscribe(self) -> None: + """SUB2a: deregisters the listener. Calling it again does nothing (SUB2b).""" + self._release() + + +class StatusSubscription(_Handle): + """RTO18f: the registration `RealtimeObject.on` returns.""" def off(self) -> None: """RTO18f1, RTO18f2: deregisters the listener the corresponding `on` call registered.""" - deregister, self._deregister = self._deregister, None - if deregister is not None: - deregister() + self._release() + + +class Registration(Generic[T]): + """One registration of `item` in a `Registry`, `active` until it is removed.""" + + def __init__(self, item: T): + self.item = item + self.active = True + + +class Registry(Generic[T]): + """The listeners, or other items, that `subscribe`-style calls register, in registration order. + + Registering one item twice makes two registrations, each removed on its own. A dispatch + iterates one `snapshot()` and skips each registration no longer `active` when it is + reached, so that one made during the dispatch is not called by it, and one removed during + it is not called again. + """ + + def __init__(self) -> None: + self._registrations: list[Registration[T]] = [] + + def __bool__(self) -> bool: + return bool(self._registrations) + + def register(self, item: T) -> Callable[[], None]: + """Registers `item`, returning the function that removes this one registration.""" + registration = Registration(item) + self._registrations.append(registration) + return lambda: self._remove(lambda entry: entry is registration) + + def deregister(self, item: T) -> None: + """Removes every registration of `item`.""" + self._remove(lambda entry: entry.item == item) + + def clear(self) -> None: + """Removes every registration.""" + self._remove(lambda entry: True) + + def snapshot(self) -> list[Registration[T]]: + """The registrations as they stand, for a dispatch to iterate.""" + return list(self._registrations) + + def _remove(self, matches: Callable[[Registration[T]], bool]) -> None: + kept = [] + for registration in self._registrations: + if matches(registration): + registration.active = False + else: + kept.append(registration) + self._registrations = kept diff --git a/ably/pubsub/objects/valuetypes.py b/ably/pubsub/objects/valuetypes.py index 40c1dadf..b50281fe 100644 --- a/ably/pubsub/objects/valuetypes.py +++ b/ably/pubsub/objects/valuetypes.py @@ -60,10 +60,13 @@ class LiveMap: def create(entries: dict[str, LiveMapValue] | None = None) -> LiveMap: """RTLMV3: a blueprint for a map holding `entries`. - A dict is copied, so the blueprint does not change when the caller's dict does - (RTLMV3d); anything else is kept for evaluation to reject (RTLMV3c). + A dict is copied, along with each JSON object, JSON array and binary value in it, so + the blueprint does not change when the caller's values do (RTLMV3d); anything else is + kept for evaluation to reject (RTLMV3c). """ - return LiveMap(dict(entries) if isinstance(entries, dict) else entries) + if not isinstance(entries, dict): + return LiveMap(entries) + return LiveMap({key: _snapshot(value) for key, value in entries.items()}) # The values `set` accepts: a primitive, or a blueprint for a new object (RTTS11) @@ -80,15 +83,35 @@ def validate_key(key: Any) -> None: def validate_value(value: Any) -> None: - """RTLMV4c: raises AblyException 40013 unless `value` is a primitive or a blueprint. + """RTLM20e1: raises the AblyException a map's `set` raises for `value`, without evaluating it. - A map's `set` validates its value the same way (RTLM20e1). A blueprint's own contents - are validated when it is evaluated. + That is 40013 unless `value` is a primitive or a blueprint (RTLMV4c), and for a blueprint, + whatever evaluating it raises (RTLCV4a, RTLMV4a-RTLMV4c), so that a write can reject one + before fetching the server time its object ids are generated from (RTO16). """ - if not isinstance(value, (LiveCounter, LiveMap)): + if isinstance(value, LiveCounter): + _counter_create(value) + elif isinstance(value, LiveMap): + entries, _ = _map_entries_data(value) + for entry in entries.values(): + if isinstance(entry, (LiveCounter, LiveMap)): + validate_value(entry) + else: primitive_to_object_data(value) +def validate_amount(amount: Any) -> float: + """RTLC12e1: `amount` as a float, raising AblyException 40003 unless it is a finite number. + + A bool is not a number, and None is not an omitted amount. A counter's `increment` and + `decrement` validate their amount this way (RTLC12e1, RTLC13c). + """ + number = _finite_number(amount) + if number is None: + raise AblyException('Counter value increment should be a valid number', 400, 40003) + return number + + def primitive_to_object_data(value: Any) -> ObjectData: """RTLMV4d3-RTLMV4d7: the `ObjectData` holding the primitive `value`. @@ -126,11 +149,7 @@ def evaluate_live_counter(value: LiveCounter, timestamp_ms: int) -> ObjectMessag `CounterCreate` it was built from (RTLCV4g5). Raises AblyException 40003 for a count that is not a finite number (RTLCV4a). """ - count = _finite_number(value._count) - if count is None: - raise AblyException('Counter value should be a valid number', 400, 40003) # RTLCV4a - - counter_create = CounterCreate(count=count) # RTLCV4b + counter_create = _counter_create(value) # RTLCV4a, RTLCV4b initial_value = _initial_value(counter_create) # RTLCV4c nonce = generate_nonce() # RTLCV4d object_id = generate_object_id('counter', initial_value, nonce, timestamp_ms) # RTLCV4f @@ -154,18 +173,8 @@ def evaluate_live_map(value: LiveMap, timestamp_ms: int) -> list[ObjectMessage]: are not a dict or a key that is not a string (RTLMV4a, RTLMV4b), and 40013 for a value of an unsupported type (RTLMV4c). """ - # `LiveMap.create()` and `LiveMap.create(None)` both leave the entries unset - entries = {} if value._entries is None else value._entries - if not isinstance(entries, dict): - raise AblyException('Map entries should be a dict', 400, 40003) # RTLMV4a - for key in entries: - validate_key(key) # RTLMV4b - # Every value at this level is validated (RTLMV4c) before any blueprint is evaluated - data: dict[str, ObjectData | None] = { - key: None if isinstance(entry, (LiveCounter, LiveMap)) else primitive_to_object_data(entry) - for key, entry in entries.items() - } + entries, data = _map_entries_data(value) messages: list[ObjectMessage] = [] for key, entry in entries.items(): @@ -206,6 +215,53 @@ def evaluate(value: LiveCounter | LiveMap, timestamp_ms: int) -> list[ObjectMess raise TypeError(f'Expected a LiveCounter or LiveMap, got {type(value).__name__}') +def _counter_create(value: LiveCounter) -> CounterCreate: + """RTLCV4a, RTLCV4b: the `CounterCreate` a `LiveCounter` describes, raising AblyException 40003 + for a count that is not a finite number.""" + count = _finite_number(value._count) + if count is None: + raise AblyException('Counter value should be a valid number', 400, 40003) # RTLCV4a + return CounterCreate(count=count) # RTLCV4b + + +def _map_entries_data(value: LiveMap) -> tuple[dict, dict[str, ObjectData | None]]: + """RTLMV4a-RTLMV4c: a `LiveMap`'s entries, and the `ObjectData` of each, None for a blueprint. + + Raises AblyException 40003 for entries that are not a dict or a key that is not a string, + and 40013 for a value of an unsupported type at this level; a blueprint among the entries + is not looked into. + """ + # `LiveMap.create()` and `LiveMap.create(None)` both leave the entries unset + entries = {} if value._entries is None else value._entries + if not isinstance(entries, dict): + raise AblyException('Map entries should be a dict', 400, 40003) # RTLMV4a + for key in entries: + validate_key(key) # RTLMV4b + # RTLMV4c + data: dict[str, ObjectData | None] = { + key: None if isinstance(entry, (LiveCounter, LiveMap)) else primitive_to_object_data(entry) + for key, entry in entries.items() + } + return entries, data + + +def _snapshot(value: Any) -> Any: + """A copy of a map entry's value that later changes to `value` do not reach (RTLMV3d). + + A JSON object or array is copied through its JSON encoding, which evaluation applies to + it anyway (RTLMV4d3), and binary is copied to `bytes`. A value that does not encode is + kept as it is, as no validation is done at creation (RTLMV3c); evaluation rejects it. + """ + if isinstance(value, bytearray): + return bytes(value) + if isinstance(value, (dict, list)): + try: + return json.loads(json.dumps(value, allow_nan=False)) + except (TypeError, ValueError, RecursionError): + return value + return value + + def _finite_number(value: Any) -> float | None: """`value` as a float if it is a finite number, else None. A boolean is not a number.""" if isinstance(value, bool) or not isinstance(value, (int, float)): diff --git a/ably/pubsub/realtime/channel.py b/ably/pubsub/realtime/channel.py index 7b79ca99..538222da 100644 --- a/ably/pubsub/realtime/channel.py +++ b/ably/pubsub/realtime/channel.py @@ -126,12 +126,21 @@ async def attach(self) -> None: AblyException If unable to attach channel """ + await self._attach() + async def _attach(self) -> ChannelStateChange | None: + """The attach procedure `attach` runs, returning the state change it ended with, or None if + the channel was already ATTACHED. + + Raises where `attach` does. An attach that ends in a state other than ATTACHED, + SUSPENDED or FAILED, such as DETACHED when the connection closes first, returns that + state change. + """ log.info(f'RealtimeChannel.attach() called, channel = {self.name}') # RTL4a - if channel is attached do nothing if self.state == ChannelState.ATTACHED: - return + return None self.__error_reason = None @@ -155,6 +164,7 @@ async def attach(self) -> None: if state_change.current in (ChannelState.SUSPENDED, ChannelState.FAILED): raise state_change.reason + return state_change def _attach_impl(self): log.debug("RealtimeChannel.attach_impl(): sending ATTACH protocol message") @@ -183,7 +193,7 @@ async def _ensure_active(self) -> None: ------ AblyException 90001 if the channel is FAILED (RTL33c), or the error the implicit attach failed - with (RTL33b1) + with (RTL33b1): the reason for the state it ended in, else 90001 """ # RTL33a if self.state in (ChannelState.ATTACHED, ChannelState.SUSPENDED): @@ -193,8 +203,14 @@ async def _ensure_active(self) -> None: if self.state == ChannelState.FAILED: raise AblyException(f"Channel operation failed as channel state is {self.state.value}", 400, 90001) - # RTL33b, RTL33b1 - await self.attach() + # RTL33b + state_change = await self._attach() + if state_change is not None and state_change.current != ChannelState.ATTACHED: + # RTL33b1, RTL4d: the attach ended without the channel attaching, as when the connection + # closes or `detach` is called first + raise state_change.reason or AblyException( + f'Unable to attach channel; channel state = {state_change.current.value}, ' + f'connection state = {self.__realtime.connection.state.value}', 400, 90001) # RTL5 async def detach(self) -> None: @@ -807,23 +823,17 @@ def _on_message(self, proto_msg: dict) -> None: except Exception as e: log.error(f"Annotation processing error {e}. Skip annotations {annotation_data}") elif action == ProtocolMessageAction.OBJECT: - try: - object_messages = ObjectMessage.from_protocol_message(proto_msg, self.__object._wire_format) - except Exception as e: - log.error(f"Object message processing error {e}. Skip object messages {proto_msg.get('state')}") - else: - # RTL15b - if channel_serial: - self.__channel_serial = channel_serial - self.__object._handle_object_messages(object_messages) # RTO8 + # RTL15b + if channel_serial: + self.__channel_serial = channel_serial + # An object message that fails to decode is logged and left out, and the rest are handled + object_messages = ObjectMessage.from_protocol_message(proto_msg, self.__object._wire_format) + self.__object._handle_object_messages(object_messages) # RTO8 elif action == ProtocolMessageAction.OBJECT_SYNC: - try: - object_messages = ObjectMessage.from_protocol_message(proto_msg, self.__object._wire_format) - except Exception as e: - log.error(f"Object sync processing error {e}. Skip object messages {proto_msg.get('state')}") - else: - # RTO5: the channelSerial of an OBJECT_SYNC carries the sync sequence and cursor (RTO5a1) - self.__object._handle_object_sync_messages(object_messages, channel_serial) + # RTO5: the channelSerial of an OBJECT_SYNC carries the sync sequence and cursor (RTO5a1), + # which are handled whichever of its object messages fail to decode + object_messages = ObjectMessage.from_protocol_message(proto_msg, self.__object._wire_format) + self.__object._handle_object_sync_messages(object_messages, channel_serial) elif action == ProtocolMessageAction.ERROR: error = AblyException.from_dict(proto_msg.get('error')) self._notify_state(ChannelState.FAILED, reason=error) @@ -1085,7 +1095,9 @@ def release(self, name: str) -> None: """ if name not in self.__all: return - del self.__all[name] + channel = self.__all.pop(name) + # The released channel receives nothing more, so its objects stop their GC timer and their waits + channel.object._release() def _on_channel_message(self, msg: dict) -> None: channel_name = msg.get('channel') @@ -1127,8 +1139,10 @@ def _propagate_connection_interruption(self, state: ConnectionState, reason: Abl channel._notify_state(connection_to_channel_state[state], reason) def _on_connected(self) -> None: + connection_details = self.__ably.connection.connection_details for channel_name in self.__all: channel = self.__all[channel_name] + channel.object._on_connected(connection_details) # RTO10b2 if channel.state == ChannelState.ATTACHING or channel.state == ChannelState.DETACHING: channel._check_pending_state() elif channel.state == ChannelState.SUSPENDED: diff --git a/test/unit/objects/batch_test.py b/test/unit/objects/batch_test.py index 46d10d69..5d9f703a 100644 --- a/test/unit/objects/batch_test.py +++ b/test/unit/objects/batch_test.py @@ -36,6 +36,7 @@ remote_serial, setup_synced_channel, standard_mock_websocket, + time_mock_http, ) MAP_CREATE = ObjectOperationAction.MAP_CREATE @@ -215,24 +216,23 @@ async def test_rtpo20g_exception_in_the_block_publishes_nothing(): _assert_error(excinfo, 40000) -async def test_rtbc16d_invalid_value_inside_a_value_type_publishes_nothing(): - """RTBC16d, RTLMV4c: a `LiveMap` value is evaluated when the batch is flushed, so an - unsupported value inside one raises from the end of the block, and nothing in the batch - is published.""" - (client, channel, root, mock_ws), published = await _synced() +async def test_rtbc12_invalid_value_inside_a_value_type_raises_at_the_call(): + """RTBC12, RTLMV4c: a `LiveMap` value is evaluated when the batch is flushed, but its + contents are validated when it is set, so an unsupported value inside one raises from the + `set` call, before the server time is fetched, and queues nothing; the batch's other writes + are published.""" + mock_http = time_mock_http() + (client, channel, root, mock_ws), published = await _synced(mock_http=mock_http) - with pytest.raises(AblyException) as excinfo: - async with root.batch() as ctx: - ctx.set('name', 'Bob') + async with root.batch() as ctx: + with pytest.raises(AblyException) as excinfo: ctx.set('team', LiveMap.create({'lead': object()})) + ctx.set('name', 'Bob') _assert_error(excinfo, 40013) - assert published == [] - assert root.get('name').as_primitive().value() == 'Alice' - - with pytest.raises(AblyException) as closed: - ctx.keys() - _assert_error(closed, 40000) + assert len(published) == 1 + assert _operations(published[0]) == [_map_set('root', 'name', {'string': 'Bob'})] + assert [request for request in mock_http.captured_requests if request.path == '/time'] == [] async def test_rtpo20f_batch_checks_preconditions_again_before_publishing(): diff --git a/test/unit/objects/realtime_object_test.py b/test/unit/objects/realtime_object_test.py new file mode 100644 index 00000000..bdc2184a --- /dev/null +++ b/test/unit/objects/realtime_object_test.py @@ -0,0 +1,342 @@ +"""A channel's `RealtimeObject`: waits for SYNCED the channel can no longer satisfy, object +messages that fail to decode or to apply, and the lifetime of the GC timer. + +Spec points: RTL33b1, RTO5, RTO5c, RTO8, RTO9, RTO10, RTO10c, RTO18, RTO19, RTO20e1, RTO23c1, +RTO27a. No UTS specification covers these cases, so they are written against the features +specification. + +Several cases race a channel state change against a coroutine resuming. The mock websocket +answers one client message with two frames, which the transport handles one after the other +before the coroutine awaiting the first of them resumes, as when the server's frames arrive in +one read. +""" + +import asyncio +import logging + +import pytest + +from ably.pubsub.objects.defaults import GC_INTERVAL_MS +from ably.pubsub.objects.enums import ObjectsSyncState +from ably.pubsub.objects.realtimeobject import RealtimeObject +from ably.pubsub.realtime.connection import ConnectionState +from ably.pubsub.types.channelstate import ChannelState +from ably.pubsub.util.exceptions import AblyException +from test.uts.helpers.client import await_connection_state, poll_until +from test.uts.helpers.clock import FakeClock, settle +from test.uts.helpers.mock_websocket import MockWebSocket, channel_error_message +from test.uts.objects.helpers.standard_test_pool import ( + ATTACH, + HAS_OBJECTS, + LWW, + STANDARD_POOL_OBJECTS, + build_ack_message, + build_counter_inc, + build_map_set, + build_object_message, + build_object_state, + objects_attached_message, + objects_channel_options, + objects_client, + objects_connected_message, + remote_serial, + setup_synced_channel, + standard_mock_websocket, +) + +# How long a test waits on an operation it expects to complete or fail, so that one which +# never settles fails with a timeout rather than running into the suite's own +OPERATION_TIMEOUT = 5 + +SCORE = 'counter:score@1000' + + +def _score(channel): + return channel.object._objects_pool[SCORE].data + + +def _assert_sync_wait_failure(excinfo, state_text, cause_code=None): + """Asserts an RTO23c1/RTO20e1 failure: 92008, status 400, naming why, with the channel's error.""" + assert excinfo.value.code == 92008 + assert excinfo.value.status_code == 400 + assert state_text in excinfo.value.message + if cause_code is not None: + assert excinfo.value.cause.code == cause_code + + +async def _restart_sync(channel, mock_ws): + """Sends an ATTACHED while attached, which restarts the sync (RTO4) with no OBJECT_SYNC to follow.""" + mock_ws.send_to_client(objects_attached_message('test', 'sync2:cursor', HAS_OBJECTS)) + await poll_until(lambda: channel.object._sync_state == ObjectsSyncState.SYNCING, + description='the sync to restart') + + +def _ack_then_channel_error(mock_ws_ref): + """An `on_object` handler answering each OBJECT with its ACK and, in the same read, a channel ERROR.""" + def on_object(message): + mock_ws_ref[0].send_to_client(build_ack_message(message['msgSerial'], ['t:9:0'])) + mock_ws_ref[0].send_to_client(channel_error_message('test', 90000, 'Channel error', 400)) + return on_object + + +# --- B1: a wait for SYNCED on a channel that has already left ATTACHED ---------------------- + +async def test_rto23c1_get_fails_when_the_channel_fails_with_its_attached(): + """RTO23c1: an ATTACHED and a channel ERROR read together leave the channel FAILED before + `get()` resumes from its attach, so `get()` begins its wait on a channel that will never + sync. It fails at once with 92008, the channel's error as the cause, rather than waiting + for a transition that has already happened.""" + mock_ws = None + + def on_message_from_client(message): + if message.get('action') == ATTACH: + channel_name = message.get('channel') + mock_ws.send_to_client(objects_attached_message(channel_name, 'sync1:cursor', HAS_OBJECTS)) + mock_ws.send_to_client(channel_error_message(channel_name, 90000, 'Channel error', 400)) + + mock_ws = MockWebSocket( + on_connection_attempt=lambda conn: conn.respond_with_success(objects_connected_message()), + on_message_from_client=on_message_from_client, + ) + client = objects_client(mock_ws) + channel = client.channels.get('test', objects_channel_options()) + + with pytest.raises(AblyException) as excinfo: + await asyncio.wait_for(channel.object.get(), OPERATION_TIMEOUT) + + _assert_sync_wait_failure(excinfo, 'failed', cause_code=90000) + assert channel.state == ChannelState.FAILED + assert channel.object._sync_waiters == {} + + +async def test_rto20e1_publish_and_apply_fails_when_the_channel_fails_with_the_ack(): + """RTO20e1: an ACK and a channel ERROR read together leave the channel FAILED before the + write resumes, so its wait for SYNCED begins on a channel that will never sync. It fails at + once with 92008, the channel's error as the cause.""" + mock_ws_ref = [] + mock_ws = standard_mock_websocket(auto_ack=False, on_object=_ack_then_channel_error(mock_ws_ref)) + mock_ws_ref.append(mock_ws) + client, channel, root, mock_ws = await setup_synced_channel('test', mock_ws=mock_ws) + await _restart_sync(channel, mock_ws) + + with pytest.raises(AblyException) as excinfo: + await asyncio.wait_for(root.set('name', 'Bob'), OPERATION_TIMEOUT) + + _assert_sync_wait_failure(excinfo, 'failed', cause_code=90000) + assert channel.state == ChannelState.FAILED + assert channel.object._sync_waiters == {} + + +async def test_rto20e1_publish_and_apply_fails_when_the_channel_detached_before_the_ack(): + """RTO20e1: a write whose channel detaches while its publish awaits the ACK begins its wait + for SYNCED on a DETACHED channel, and fails at once with 92008.""" + published = [] + mock_ws = standard_mock_websocket(auto_ack=False, on_object=published.append) + client, channel, root, mock_ws = await setup_synced_channel('test', mock_ws=mock_ws) + await _restart_sync(channel, mock_ws) + + task = asyncio.ensure_future(root.get('score').as_live_counter().increment(10)) + await poll_until(lambda: len(published) == 1, description='the OBJECT to be published') + await channel.detach() + mock_ws.send_to_client(build_ack_message(published[0]['msgSerial'], ['t:9:0'])) + + with pytest.raises(AblyException) as excinfo: + await asyncio.wait_for(task, OPERATION_TIMEOUT) + + _assert_sync_wait_failure(excinfo, 'detached') + assert channel.object._sync_waiters == {} + + +async def test_rto27a_publish_and_apply_does_not_apply_once_the_channel_has_failed(): + """RTO27a, RTO20f: a write on a SYNCED channel whose ACK is read together with a channel + ERROR completes, as it was published, but is not applied locally: the channel's FAILED + has cleared the objects' data, and applying it would rebuild that data and notify + subscribers on a failed channel.""" + mock_ws_ref = [] + mock_ws = standard_mock_websocket(auto_ack=False, on_object=_ack_then_channel_error(mock_ws_ref)) + mock_ws_ref.append(mock_ws) + client, channel, root, mock_ws = await setup_synced_channel('test', mock_ws=mock_ws) + calls = [] + root.get('score').subscribe(calls.append) + + await asyncio.wait_for(root.get('score').as_live_counter().increment(10), OPERATION_TIMEOUT) + await settle() + + assert channel.state == ChannelState.FAILED + assert _score(channel) == 0 + assert calls == [] + + +async def test_rtl33b1_get_fails_when_the_client_closes_during_the_attach(): + """RTL33b1, RTO23e: closing the client while `get()` attaches moves the ATTACHING channel to + DETACHED, which ends the attach without attaching. `get()` fails with the attach's error, + rather than waiting for a sync on a channel that has detached.""" + mock_ws = MockWebSocket( + on_connection_attempt=lambda conn: conn.respond_with_success(objects_connected_message()), + on_message_from_client=lambda message: None, # never answers the ATTACH + ) + client = objects_client(mock_ws) + channel = client.channels.get('test', objects_channel_options()) + await await_connection_state(client, ConnectionState.CONNECTED) + + task = asyncio.ensure_future(channel.object.get()) + await poll_until(lambda: channel.state == ChannelState.ATTACHING, description='the channel to attach') + await client.close() + + with pytest.raises(AblyException) as excinfo: + await asyncio.wait_for(task, OPERATION_TIMEOUT) + + assert excinfo.value.code == 90001 + assert excinfo.value.status_code == 400 + assert channel.state == ChannelState.DETACHED + assert channel.object._sync_waiters == {} + + +# --- R1: object messages that fail to decode or to apply ------------------------------------ + +async def test_rto5_an_object_sync_message_that_fails_to_decode_is_skipped(caplog): + """RTO5, RTO5a4: one undecodable value in the last OBJECT_SYNC of a sequence costs only the + ObjectMessage carrying it. The rest of the ProtocolMessage, its cursor included, is + handled, so the sync completes and `get()` returns.""" + bad = build_object_state('map:bad@1000', {'aaa': 't:0'}, map={ + 'semantics': LWW, 'entries': {'k': {'data': {'json': '{not json'}, 'timeserial': 't:0'}}}) + mock_ws = standard_mock_websocket(sync_objects=(*STANDARD_POOL_OBJECTS, bad)) + + with caplog.at_level(logging.ERROR, logger='ably.pubsub.objects'): + client, channel, root, mock_ws = await asyncio.wait_for( + setup_synced_channel('test', mock_ws=mock_ws), OPERATION_TIMEOUT) + + assert channel.object._sync_state == ObjectsSyncState.SYNCED + assert root.get('score').as_live_counter().value() == 100 + assert 'map:bad@1000' not in channel.object._objects_pool + assert any('failed to decode' in record.getMessage() for record in caplog.records) + + +async def test_rto8_an_object_message_that_fails_to_decode_costs_only_itself(caplog): + """RTO8: an OBJECT carrying an undecodable value and a valid operation applies the valid + one.""" + client, channel, root, mock_ws = await setup_synced_channel('test') + + with caplog.at_level(logging.ERROR, logger='ably.pubsub.objects'): + mock_ws.send_to_client(build_object_message('test', [ + build_map_set('root', 'data', {'json': '{not json'}, remote_serial(0), 'remote'), + build_counter_inc(SCORE, 1, remote_serial(1), 'remote'), + ])) + await poll_until(lambda: _score(channel) == 101, description='the valid increment to apply') + + assert root.get('data').as_primitive().value() == {'tags': ['a', 'b']} + assert any('failed to decode' in record.getMessage() for record in caplog.records) + + +async def test_rto9_an_object_message_that_fails_to_apply_costs_only_itself(caplog): + """RTO9: an operation that decodes but raises as it is applied, here an increment by a + string, is logged and skipped, and the operations after it in the OBJECT still apply.""" + client, channel, root, mock_ws = await setup_synced_channel('test') + + with caplog.at_level(logging.ERROR, logger='ably.pubsub.objects'): + mock_ws.send_to_client(build_object_message('test', [ + build_counter_inc(SCORE, 'one', remote_serial(0), 'remote'), + build_map_set('root', 'name', {'string': 'Bob'}, remote_serial(1), 'remote'), + ])) + await poll_until(lambda: root.get('name').as_primitive().value() == 'Bob', + description='the MAP_SET after the failing increment to apply') + + assert _score(channel) == 100 + assert any('could not be applied' in record.getMessage() for record in caplog.records) + + +async def test_rto5c_an_object_state_that_fails_to_apply_does_not_stall_the_sync(caplog): + """RTO5c: an object state that decodes but raises as it is applied, here a counter whose + count is a string, is logged and skipped, and the sync still completes.""" + bad = build_object_state('counter:bad@1000', {'aaa': 't:0'}, counter={'count': 'many'}) + mock_ws = standard_mock_websocket(sync_objects=(*STANDARD_POOL_OBJECTS, bad)) + + with caplog.at_level(logging.ERROR, logger='ably.pubsub.objects'): + client, channel, root, mock_ws = await asyncio.wait_for( + setup_synced_channel('test', mock_ws=mock_ws), OPERATION_TIMEOUT) + + assert channel.object._sync_state == ObjectsSyncState.SYNCED + assert root.get('score').as_live_counter().value() == 100 + assert any('could not be applied' in record.getMessage() for record in caplog.records) + + +# --- R2: the GC timer and waits of a released channel --------------------------------------- + +async def test_rto10_a_released_channel_stops_its_gc_timer(): + """RTO10a: the GC timer every ATTACHED schedules stops when its channel is released. A + released channel is out of the client's collection, so closing the client never detaches + it; the release itself stops the timer, for good.""" + clock = FakeClock() + client, channel, root, mock_ws = await setup_synced_channel('test', clock=clock) + timer = channel.object._gc_timer + assert timer is not None + + client.channels.release('test') + await client.close() + await clock.advance(GC_INTERVAL_MS) + + assert timer.cancelled + assert channel.object._gc_timer is None + + +async def test_rto23c1_a_wait_for_synced_fails_when_its_channel_is_released(): + """RTO23c1: a released channel receives nothing more, so it can never sync. A `get()` + waiting for SYNCED when the channel is released fails with 92008, and so does a later one, + at once.""" + client, channel, root, mock_ws = await setup_synced_channel('test') + await _restart_sync(channel, mock_ws) + task = asyncio.ensure_future(channel.object.get()) + await settle() + assert not task.done() + + client.channels.release('test') + + with pytest.raises(AblyException) as excinfo: + await asyncio.wait_for(task, OPERATION_TIMEOUT) + _assert_sync_wait_failure(excinfo, 'released') + + with pytest.raises(AblyException) as excinfo: + await asyncio.wait_for(channel.object.get(), OPERATION_TIMEOUT) + _assert_sync_wait_failure(excinfo, 'released') + + +# --- R4: a GC sweep that raises ------------------------------------------------------------- + +async def test_rto10c_a_gc_sweep_that_raises_is_logged_and_the_next_still_runs(caplog, monkeypatch): + """RTO10c: a sweep that raises is logged, and the next is scheduled all the same; the error + does not escape the timer's callback.""" + clock = FakeClock() + client, channel, root, mock_ws = await setup_synced_channel('test', clock=clock) + sweeps = [] + + def collect_garbage(grace_period_ms, now_ms): + sweeps.append(now_ms) + if len(sweeps) == 1: + raise RuntimeError('the sweep failed') + + monkeypatch.setattr(channel.object._objects_pool, 'collect_garbage', collect_garbage) + + with caplog.at_level(logging.ERROR, logger='ably.pubsub.objects'): + await clock.advance(GC_INTERVAL_MS) + assert len(sweeps) == 1 + assert any('GC sweep raised' in record.getMessage() for record in caplog.records) + assert channel.object._gc_timer is not None + + await clock.advance(GC_INTERVAL_MS) + assert len(sweeps) == 2 + + +# --- on and off -------------------------------------------------------------------------- + +@pytest.mark.parametrize('register', [ + lambda realtime_object: realtime_object.on('synched', lambda: None), + lambda realtime_object: realtime_object.off('synched', lambda: None), +]) +def test_rto18_rto19_an_unknown_event_raises_40003(register): + """RTO18, RTO19: `on` and `off` take an `ObjectsEvent`; any other event is an invalid + argument, raised as AblyException 40003 like the other argument errors.""" + with pytest.raises(AblyException) as excinfo: + register(RealtimeObject()) + + assert excinfo.value.code == 40003 + assert excinfo.value.status_code == 400 diff --git a/test/unit/objects/subscription_test.py b/test/unit/objects/subscription_test.py new file mode 100644 index 00000000..17fd56ab --- /dev/null +++ b/test/unit/objects/subscription_test.py @@ -0,0 +1,69 @@ +"""Which listeners a dispatch calls when listeners are added or removed during it. + +Spec points: RTLO4b4c3a (`subscribe` listeners on a live object) and RTO24b (path +subscriptions). The specification does not say how a dispatch treats registrations changed +by its own listeners; the library takes one snapshot per dispatch, so that a listener added +during it is not called by it, and skips a listener removed during it, for both kinds. +""" + +from test.uts.helpers.clock import settle +from test.uts.objects.helpers.standard_test_pool import ( + build_counter_inc, + build_map_set, + build_object_message, + remote_serial, + setup_synced_channel, +) + +SCORE = 'counter:score@1000' + + +async def test_rtlo4b4c3a_a_listener_unsubscribed_during_a_dispatch_is_not_called(): + """RTLO4b4c3a: a listener that another listener unsubscribes earlier in the same dispatch + is not called for that update.""" + client, channel, root, mock_ws = await setup_synced_channel('test') + counter = root.get('score').instance().as_live_counter() + calls = [] + second = None + + def first(event): + calls.append('first') + second.unsubscribe() + + counter.subscribe(first) + second = counter.subscribe(lambda event: calls.append('second')) + await counter.increment(1) + await settle() + + assert calls == ['first'] + + +async def test_rto24b_a_subscription_made_during_a_dispatch_is_not_called_by_it(): + """RTO24b: a path subscription made by a listener is not called by the dispatch that made + it, even for a later path to the same object, and is called by the next update.""" + client, channel, root, mock_ws = await setup_synced_channel('test') + # root.alias references the same counter as root.score, so the counter has two paths + mock_ws.send_to_client(build_object_message('test', [ + build_map_set('root', 'alias', {'objectId': SCORE}, remote_serial(0), 'remote')])) + await settle() + assert root.get('alias').as_live_counter().value() == 100 + + late_calls = [] + added = [] + + def first(event): + if not added: + added.append(root.subscribe(lambda late_event: late_calls.append(late_event.object.path()))) + + root.get('score').subscribe(first) + root.get('alias').subscribe(first) + mock_ws.send_to_client(build_object_message('test', [ + build_counter_inc(SCORE, 1, remote_serial(1), 'remote')])) + await settle() + assert added + assert late_calls == [] + + mock_ws.send_to_client(build_object_message('test', [ + build_counter_inc(SCORE, 1, remote_serial(2), 'remote')])) + await settle() + assert sorted(late_calls) == ['alias', 'score'] diff --git a/test/unit/objects/value_types_test.py b/test/unit/objects/value_types_test.py new file mode 100644 index 00000000..e4a0e368 --- /dev/null +++ b/test/unit/objects/value_types_test.py @@ -0,0 +1,43 @@ +"""The `LiveMap` and `LiveCounter` blueprints a write evaluates: what `create` captures, and when +their contents are validated. + +Spec points: RTLMV3d (a blueprint is immutable), RTLM20e1 and RTLMV4c (value validation), RTO16 +(the server time an object id is generated from). +""" + +import pytest + +from ably.pubsub.objects.valuetypes import LiveCounter, LiveMap, evaluate +from ably.pubsub.util.exceptions import AblyException +from test.uts.objects.helpers.standard_test_pool import setup_synced_channel, time_mock_http + +SERVER_TIME_MS = 1_700_000_123_000 + + +def test_rtlmv3d_a_blueprint_does_not_change_with_the_values_it_was_created_from(): + """RTLMV3d: changing a JSON value or a `bytearray` after passing it to `LiveMap.create` does + not change the blueprint, at any depth.""" + meta = {'tags': ['a']} + raw = bytearray(b'\x01') + blueprint = LiveMap.create({'meta': meta, 'raw': raw}) + + meta['tags'].append('b') + meta['owner'] = 'Bob' + raw.append(2) + + map_create = evaluate(blueprint, SERVER_TIME_MS)[-1].operation.map_create_with_object_id.derived_from + assert map_create.entries['meta'].data.json == {'tags': ['a']} + assert map_create.entries['raw'].data.bytes == b'\x01' + + +async def test_rtlm20e1_an_invalid_blueprint_is_rejected_before_the_server_time_is_fetched(): + """RTLM20e1, RTLMV4c, RTO16: `set` validates a blueprint's contents before it asks the server + for the time its object ids are generated from, so an invalid one costs no request.""" + mock_http = time_mock_http() + client, channel, root, mock_ws = await setup_synced_channel('test', mock_http=mock_http) + + with pytest.raises(AblyException) as excinfo: + await root.set('team', LiveMap.create({'lead': 'Carol', 'points': LiveCounter.create('ten')})) + + assert excinfo.value.code == 40003 + assert [request for request in mock_http.captured_requests if request.path == '/time'] == [] diff --git a/test/uts/deviations.md b/test/uts/deviations.md index 5c70d912..6131c8d4 100644 --- a/test/uts/deviations.md +++ b/test/uts/deviations.md @@ -2740,9 +2740,9 @@ mock-backed client; no new kind of warning appears. The GC timer is scheduled on the first ATTACHED of every channel, whether or not it uses objects (RTO10a; ably-js does the same whenever the plugin is loaded), and cancelled on DETACHED and FAILED, so a client's `close()` stops it. A non-objects test advancing a -`FakeClock` by five minutes therefore runs one empty sweep. The one channel whose timer -outlives it is one passed to `Channels.release()` while attached, because release does not -detach — the RTS4a defect under *Failing Tests*. And a burst of creates on a channel that has +`FakeClock` by five minutes therefore runs one empty sweep. `Channels.release()` stops it +too, although release does not detach — the RTS4a defect under *Failing Tests*. And a burst +of creates on a channel that has not yet read the server time each request `/time`, until the first answer persists the offset (RTO16a); wasteful, and not wrong. @@ -3701,7 +3701,11 @@ No UTS specification covers batched writes (RTPO20, RTINS17, RTBC1–RTBC16, in #491). Both are implemented, so both are tested, in `test/unit/objects/`: `batch_test.py`, 29 functions run as 76 cases, and `typed_views_test.py`, 17 functions run as 61. They are written against `objects-features.md`, the two pull requests and LODR-061, carry no `# UTS:` comment, -and are not counted in the header. They drive the standard synced pool over the mock +and are not counted in the header. Beside them, `realtime_object_test.py`, +`subscription_test.py` and `value_types_test.py` (17 functions, 18 cases) hold regression +tests for behaviour no specification test reaches: a sync wait on a channel that is already +detached, failed or released, an object message that fails to decode, a GC sweep that raises, +and the validation of a creation value before the server time is read. They drive the standard synced pool over the mock websocket with this tier's helpers, and their own `conftest.py` closes clients as `test/uts/conftest.py` does. Single-point mutants of the implementation checked that they bite: they kill 15 of 16 in the batch code — the survivor removes a check another layer From 1a6b82497d7c3e76dad2c4c57156bd5c7736acda Mon Sep 17 00:00:00 2001 From: owenpearson Date: Wed, 14 Oct 2026 16:13:40 +0100 Subject: [PATCH 22/71] objects: make compact_json the only snapshot ably-java and ably-cocoa expose only compactJson, and RTTS3f and RTTS7d let a typed SDK leave compact out. In Python the two would return dicts that differ only once the data holds bytes or a shared map, and json.dumps raises on compact's result at exactly that point. LODR-061 records the choice. compact_json returns JsonValue, exported with JsonObject from ably.pubsub.server, and LiveMapInstance and LiveCounterInstance narrow it per RTTS7a. The UTS compact tests read through compact_json, recorded as adaptation S-6. Co-Authored-By: Claude Opus 5.5 (1M context) --- ably/pubsub/objects/batch.py | 10 +-- ably/pubsub/objects/instance.py | 76 +++++++++++++---------- ably/pubsub/objects/pathobject.py | 26 ++++---- ably/pubsub/objects/valuetypes.py | 6 +- ably/pubsub/server/__init__.py | 11 +++- test/unit/objects/batch_test.py | 8 +-- test/unit/objects/typed_views_test.py | 46 ++++++++++++-- test/uts/deviations.md | 17 ++--- test/uts/objects/unit/instance_test.py | 7 ++- test/uts/objects/unit/path_object_test.py | 23 ++++--- 10 files changed, 146 insertions(+), 84 deletions(-) diff --git a/ably/pubsub/objects/batch.py b/ably/pubsub/objects/batch.py index d704c96d..87240627 100644 --- a/ably/pubsub/objects/batch.py +++ b/ably/pubsub/objects/batch.py @@ -45,7 +45,7 @@ if TYPE_CHECKING: from ably.pubsub.objects.realtimeobject import RealtimeObject - from ably.pubsub.objects.valuetypes import LiveMapValue, Primitive, T + from ably.pubsub.objects.valuetypes import JsonValue, LiveMapValue, Primitive, T # RTBC16b: builds the ObjectMessages of one queued write when the batch is flushed MessageConstructor = Callable[[], Awaitable[list[ObjectMessage]]] @@ -200,13 +200,7 @@ def get(self, key: str) -> BatchContext | None: return None return self._root_context.wrap_instance(instance) # RTBC4e - def compact(self) -> Any: - """RTBC10: `Instance.compact` on the wrapped instance.""" - self._realtime_object._check_access_preconditions() # RTBC10a - self._throw_if_closed() # RTBC10b - return self._instance.compact() # RTBC10c - - def compact_json(self) -> Any: + def compact_json(self) -> JsonValue: """RTBC11: `Instance.compact_json` on the wrapped instance.""" self._realtime_object._check_access_preconditions() # RTBC11a self._throw_if_closed() # RTBC11b diff --git a/ably/pubsub/objects/instance.py b/ably/pubsub/objects/instance.py index 64c238b6..5db51233 100644 --- a/ably/pubsub/objects/instance.py +++ b/ably/pubsub/objects/instance.py @@ -29,7 +29,7 @@ from ably.pubsub.objects.liveobject import LiveObjectUpdate from ably.pubsub.objects.realtimeobject import RealtimeObject from ably.pubsub.objects.subscription import Subscription - from ably.pubsub.objects.valuetypes import LiveMapValue, Primitive, T + from ably.pubsub.objects.valuetypes import JsonObject, JsonValue, LiveMapValue, Primitive, T # The types `value(expected)` accepts as `expected`, and the category each one selects _EXPECTED_VALUE_TYPES: tuple[tuple[type, ValueType], ...] = ( @@ -103,36 +103,37 @@ def primitive_value(value: Any, value_type: ValueType | None = None) -> Any: return value -def compact_value(value: Any, *, for_json: bool = False) -> Any: - """RTPO13, RTPO14: a plain snapshot of a resolved value. +def compact_json_value(value: Any) -> JsonValue: + """RTPO14, RTINS11: a JSON-serializable snapshot of a resolved value, a map as + `compact_json_map` builds it.""" + if isinstance(value, InternalLiveMap): + return compact_json_map(value) + return _compact_json_leaf(value) - A map becomes a dict of its entries that are not tombstoned (RTPO13c1), nested maps - becoming nested dicts (RTPO13c2) and counters their values (RTPO13c3). Maps are visited - depth first in entry order, and a map reached again in the same snapshot, through a - cycle or a second reference, is the dict already built for it (RTPO13c5). With - `for_json` it is `{'objectId': id}` instead (RTPO14b2), and binary is a base64 string - (RTPO14b1), so that the result is JSON-serializable. - """ - if not isinstance(value, InternalLiveMap): - return _compact_leaf(value, for_json) - result: dict[str, Any] = {} - built: dict[str, dict[str, Any]] = {value.object_id: result} +def compact_json_map(live_map: InternalLiveMap) -> JsonObject: + """RTPO13c, RTPO14b: a JSON object of a map's entries that are not tombstoned (RTPO13c1). + + Nested maps become nested objects (RTPO13c2) and counters their values (RTPO13c3). Maps + are visited depth first in entry order, and a map reached again in the same snapshot, + through a cycle or a second reference, is `{'objectId': id}` (RTPO13c5, RTPO14b2). + """ + result: JsonObject = {} + visited = {live_map.object_id} # A stack in place of recursion, so that the depth of the tree is not limited by the - # interpreter's. Each item is a dict being built and the entries of its map still to add. - stack = [(result, iter(value.entries()))] + # interpreter's. Each item is an object being built and the entries of its map still to add. + stack = [(result, iter(live_map.entries()))] while stack: target, entries = stack[-1] for key, entry_value in entries: if not isinstance(entry_value, InternalLiveMap): - target[key] = _compact_leaf(entry_value, for_json) + target[key] = _compact_json_leaf(entry_value) continue - seen = built.get(entry_value.object_id) - if seen is not None: - target[key] = {'objectId': entry_value.object_id} if for_json else seen + if entry_value.object_id in visited: + target[key] = {'objectId': entry_value.object_id} continue - child: dict[str, Any] = {} - built[entry_value.object_id] = child + child: JsonObject = {} + visited.add(entry_value.object_id) target[key] = child stack.append((child, iter(entry_value.entries()))) break @@ -141,11 +142,14 @@ def compact_value(value: Any, *, for_json: bool = False) -> Any: return result -def _compact_leaf(value: Any, for_json: bool) -> Any: - """RTPO13c3, RTPO13c4, RTPO13d, RTPO13e, RTPO14b1: the snapshot of a value that is not a map.""" +def _compact_json_leaf(value: Any) -> JsonValue: + """RTPO13c3, RTPO13c4, RTPO13d, RTPO13e, RTPO14b1: the snapshot of a value that is not a map. + + A counter is its value, binary a base64 string, and a JSON object or array a copy. + """ if isinstance(value, InternalLiveCounter): return value.value() - if for_json and isinstance(value, (bytes, bytearray)): + if isinstance(value, (bytes, bytearray)): return base64.b64encode(value).decode('ascii') if isinstance(value, (dict, list)): return copy.deepcopy(value) @@ -208,15 +212,11 @@ def get(self, key: str) -> Instance | None: return None return Instance._wrap(self._realtime_object, value) # RTINS5c - def compact(self) -> Any: - """RTINS10: a plain snapshot of the wrapped value.""" - self._realtime_object._check_access_preconditions() # RTINS10a - return compact_value(self._value) # RTINS10b - - def compact_json(self) -> Any: - """RTINS11: `compact`, with binary as base64 and cycles as `{'objectId': ...}`. Never None.""" + def compact_json(self) -> JsonValue: + """RTINS11: a JSON-serializable snapshot of the wrapped value, as + `PathObject.compact_json` builds it. Never None (RTINS11c).""" self._realtime_object._check_access_preconditions() # RTINS11a - return compact_value(self._value, for_json=True) # RTINS11b + return compact_json_value(self._value) # RTINS11b def as_live_map(self) -> LiveMapInstance: """RTTS9a: this instance as a map. Raises AblyException 92007 if it wraps something else.""" @@ -266,6 +266,11 @@ def batch(self) -> Batch[LiveMapBatchContext]: return Batch(self._realtime_object, lambda: self._value, LiveMapBatchContext, f'object {self.id!r}') + def compact_json(self) -> JsonObject: + """RTINS11, RTTS7a1: a JSON-serializable snapshot of the wrapped map.""" + self._realtime_object._check_access_preconditions() # RTINS11a + return compact_json_map(self._value) # RTINS11b + def entries(self) -> list[tuple[str, Instance]]: """RTINS6: `(key, instance)` for each entry of the map. @@ -332,6 +337,11 @@ def value(self) -> float: self._realtime_object._check_access_preconditions() # RTINS4a return self._value.value() + def compact_json(self) -> float: + """RTINS11, RTTS7a3: the wrapped counter's value.""" + self._realtime_object._check_access_preconditions() # RTINS11a + return self._value.value() # RTINS11b + async def increment(self, amount: float = 1) -> None: """RTINS14: increments the wrapped counter by `amount`.""" self._realtime_object._check_write_preconditions() # RTINS14b diff --git a/ably/pubsub/objects/pathobject.py b/ably/pubsub/objects/pathobject.py index 1c8d9928..98875002 100644 --- a/ably/pubsub/objects/pathobject.py +++ b/ably/pubsub/objects/pathobject.py @@ -22,7 +22,7 @@ from ably.pubsub.objects.instance import ( Instance, - compact_value, + compact_json_value, expected_value_type, primitive_value, value_type_of, @@ -37,7 +37,7 @@ from ably.pubsub.objects.enums import ValueType from ably.pubsub.objects.realtimeobject import RealtimeObject from ably.pubsub.objects.subscription import Subscription - from ably.pubsub.objects.valuetypes import LiveMapValue, Primitive, T + from ably.pubsub.objects.valuetypes import JsonValue, LiveMapValue, Primitive, T @dataclass @@ -112,21 +112,19 @@ def instance(self) -> Instance | None: return None # RTPO8e return Instance._wrap(self._realtime_object, resolved) # RTPO8c, RTPO8f - def compact(self) -> Any: - """RTPO13: a plain snapshot of what the path resolves to, or None.""" - self._realtime_object._check_access_preconditions() # RTPO13a - resolved = self._resolve() # RTPO13b - if resolved is None: - return None # RTPO13f - return compact_value(resolved) # RTPO13c, RTPO13d, RTPO13e + def compact_json(self) -> JsonValue: + """RTPO14: a JSON-serializable snapshot of what the path resolves to, or None. - def compact_json(self) -> Any: - """RTPO14: `compact`, with binary as base64 and cycles as `{'objectId': ...}`.""" + A map is a JSON object of its entries that are not tombstoned, nested maps becoming + nested objects and counters their values. Binary is a base64 string, and a map + reached again in the same snapshot, through a cycle or a second reference, is + `{'objectId': id}`. + """ self._realtime_object._check_access_preconditions() # RTPO14a - resolved = self._resolve() + resolved = self._resolve() # RTPO13b if resolved is None: - return None - return compact_value(resolved, for_json=True) # RTPO14b + return None # RTPO13f + return compact_json_value(resolved) # RTPO14b def subscribe(self, listener: Callable[[PathObjectSubscriptionEvent], None], *, depth: int | None = None) -> Subscription: diff --git a/ably/pubsub/objects/valuetypes.py b/ably/pubsub/objects/valuetypes.py index b50281fe..c5ed6a9c 100644 --- a/ably/pubsub/objects/valuetypes.py +++ b/ably/pubsub/objects/valuetypes.py @@ -13,7 +13,7 @@ import json import math from dataclasses import dataclass -from typing import Any, TypeVar, Union +from typing import Any, Dict, List, TypeVar, Union from ably.pubsub.objects.objectid import generate_nonce, generate_object_id from ably.pubsub.objects.objectmessage import ( @@ -37,6 +37,10 @@ T = TypeVar('T', bound=Primitive) +# A JSON value and a JSON object, as `compact_json` returns them (RTPO14, RTINS11) +JsonValue = Union[str, float, bool, None, List['JsonValue'], Dict[str, 'JsonValue']] +JsonObject = Dict[str, JsonValue] + @dataclass(frozen=True) class LiveCounter: diff --git a/ably/pubsub/server/__init__.py b/ably/pubsub/server/__init__.py index 20747fd9..572b3b5d 100644 --- a/ably/pubsub/server/__init__.py +++ b/ably/pubsub/server/__init__.py @@ -48,7 +48,14 @@ from ably.pubsub.objects.publicmessage import ObjectMessage, ObjectOperation from ably.pubsub.objects.realtimeobject import RealtimeObject from ably.pubsub.objects.subscription import StatusSubscription, Subscription -from ably.pubsub.objects.valuetypes import LiveCounter, LiveMap, LiveMapValue, Primitive +from ably.pubsub.objects.valuetypes import ( + JsonObject, + JsonValue, + LiveCounter, + LiveMap, + LiveMapValue, + Primitive, +) from ably.pubsub.prototypes import PubSubHttpClient, PubSubRealtimeClient from ably.pubsub.realtime.channel import Channels as RealtimeChannels from ably.pubsub.realtime.connection import Connection @@ -305,6 +312,8 @@ def create_realtime_client(**kwargs) -> PubSubRealtimeClient: 'IncompatibleClientIdException', 'Instance', 'InstanceSubscriptionEvent', + 'JsonObject', + 'JsonValue', 'LiveCounter', 'LiveCounterBatchContext', 'LiveCounterInstance', diff --git a/test/unit/objects/batch_test.py b/test/unit/objects/batch_test.py index 5d9f703a..3d8c4123 100644 --- a/test/unit/objects/batch_test.py +++ b/test/unit/objects/batch_test.py @@ -574,14 +574,13 @@ async def test_rtbc6_rtbc9_map_enumeration(): assert entries[2][1].get('theme').as_primitive().value() == 'dark' -async def test_rtbc10_rtbc11_compact(): - """RTBC10, RTBC11: `compact` and `compact_json` snapshot the wrapped value as `Instance` does.""" +async def test_rtbc11_compact_json(): + """RTBC11: `compact_json` snapshots the wrapped value as `Instance` does.""" (client, channel, root, mock_ws), published = await _synced() async with root.batch() as ctx: - assert ctx.get('profile').compact() == { + assert ctx.get('profile').compact_json() == { 'email': 'alice@example.com', 'nested_counter': 5, 'prefs': {'theme': 'dark'}} - assert ctx.compact()['avatar'] == b'\x01\x02\x03' assert ctx.compact_json()['avatar'] == 'AQID' assert ctx.get('score').compact_json() == 100 @@ -635,7 +634,6 @@ class _Contexts(NamedTuple): # A call of each context method that reads the objects, and of each that writes _READS = { 'get': lambda c: c.root.get('name'), - 'compact': lambda c: c.root.compact(), 'compact_json': lambda c: c.root.compact_json(), 'entries': lambda c: c.root.entries(), 'keys': lambda c: c.root.keys(), diff --git a/test/unit/objects/typed_views_test.py b/test/unit/objects/typed_views_test.py index 6121dc0f..9c46a282 100644 --- a/test/unit/objects/typed_views_test.py +++ b/test/unit/objects/typed_views_test.py @@ -1,4 +1,5 @@ -"""The typed views: `type()` and `exists()`, the unchecked path views and the checked instance views. +"""The typed views: `type()` and `exists()`, the unchecked path views, the checked instance views +and the snapshots `compact_json` takes through them. Spec points: RTTS2-RTTS10 (ably/specification#491), as LODR-061 binds them: one `PrimitivePathObject` and `PrimitiveInstance` whose `value(expected)` stands for the six @@ -9,6 +10,7 @@ Each test drives the standard synced pool over the mock websocket. """ +import json import typing import pytest @@ -352,11 +354,45 @@ async def test_rtts10_instance_id_is_a_property(): assert root.get('name').instance().id is None +async def test_rtts7a_instance_compact_json_is_of_the_wrapped_type(): + """RTINS11, RTTS7a1, RTTS7a3: a map instance's snapshot is a JSON object, a counter + instance's is its value as a float, and a primitive instance's is the primitive.""" + root = await _synced_root() + + profile = root.get('profile').instance().compact_json() + score = root.get('score').instance().compact_json() + + assert profile == {'email': 'alice@example.com', 'nested_counter': 5, 'prefs': {'theme': 'dark'}} + assert type(score) is float + assert score == 100 + assert root.get('name').instance().compact_json() == 'Alice' + assert root.get('avatar').instance().compact_json() == 'AQID' + + +async def test_rtpo14b_compact_json_is_json_serializable(): + """RTPO14b: binary is a base64 string, and a map reached a second time in one snapshot is + its object id, through a second reference as through a cycle, so `json.dumps` takes the + snapshot.""" + client, channel, root, mock_ws = await setup_synced_channel('test') + # root.alias references the same map as root.profile + mock_ws.send_to_client(build_object_message('test', [ + build_map_set('root', 'alias', {'objectId': 'map:profile@1000'}, remote_serial(0), 'remote')])) + await poll_until(lambda: 'alias' in root.keys(), description='the reference to be applied') + + result = root.compact_json() + + assert result['avatar'] == 'AQID' + assert result['profile'] == {'email': 'alice@example.com', 'nested_counter': 5, 'prefs': {'theme': 'dark'}} + assert result['alias'] == {'objectId': 'map:profile@1000'} + assert json.loads(json.dumps(result)) == result + + @pytest.mark.parametrize('typed_class, present, absent', [ (PathObject, - {'path', 'type', 'exists', 'get', 'at', 'instance', 'compact', 'compact_json', 'subscribe', + {'path', 'type', 'exists', 'get', 'at', 'instance', 'compact_json', 'subscribe', 'as_live_map', 'as_live_counter', 'as_primitive'}, - {'value', 'entries', 'keys', 'values', 'size', 'set', 'remove', 'increment', 'decrement', 'batch'}), + {'compact', 'value', 'entries', 'keys', 'values', 'size', 'set', 'remove', 'increment', 'decrement', + 'batch'}), (LiveMapPathObject, {'entries', 'keys', 'values', 'size', 'set', 'remove', 'batch'}, {'value', 'increment', 'decrement'}), @@ -367,8 +403,8 @@ async def test_rtts10_instance_id_is_a_property(): {'value'}, {'entries', 'keys', 'values', 'size', 'set', 'remove', 'increment', 'decrement', 'batch'}), (Instance, - {'id', 'type', 'get', 'compact', 'compact_json', 'as_live_map', 'as_live_counter', 'as_primitive'}, - {'value', 'entries', 'size', 'set', 'remove', 'increment', 'decrement', 'subscribe', 'batch'}), + {'id', 'type', 'get', 'compact_json', 'as_live_map', 'as_live_counter', 'as_primitive'}, + {'compact', 'value', 'entries', 'size', 'set', 'remove', 'increment', 'decrement', 'subscribe', 'batch'}), (LiveMapInstance, {'entries', 'keys', 'values', 'size', 'set', 'remove', 'subscribe', 'batch'}, {'value', 'increment', 'decrement'}), diff --git a/test/uts/deviations.md b/test/uts/deviations.md index 6131c8d4..e08d19cb 100644 --- a/test/uts/deviations.md +++ b/test/uts/deviations.md @@ -2414,11 +2414,11 @@ pseudocode is not mistaken for non-compliance. | RTC17 (RSA7b1) | `client.clientId == client.auth.clientId` | `AblyRealtime.client_id` reads `options.client_id` and returns the configured value, while `Auth.__init__` sets `self.__client_id = None` whenever `ably._is_realtime` (`rest/auth.py:34-41`), deferring it to whatever a CONNECTED confirms. The two disagree on a client that has not connected | Open bug. RSA12b only allows the realtime clientId to be unknown while it has not been *configured* | | RTC1f | a `transportParams` boolean appears as `"true"` / `"false"` | `True` / `False`, because `WebSocketTransport.connect` builds the query string with `urllib.parse.urlencode`, which renders each value through `str()` (`websockettransport.py:89`). Integers are unaffected | Open bug. A caller can pass the strings directly, but a bool is what the spec's Stringifiable type admits | -### LiveObjects: the internal shapes the unit specifications assume — S-1 to S-5 +### LiveObjects: the internal shapes the unit specifications assume — S-1 to S-6 The objects unit specifications drive internal classes, and ably-python's differ in shape from the ones they are written against — three of them because the specification contradicts -`objects-features.md` (the first UTS Spec Error entry on the objects tier), two because of how +`objects-features.md` (the first UTS Spec Error entry on the objects tier), three because of how ably-python is built. Each shape has a label, cited in the module docstring and at the site, so that one entry here covers every test it affects. The behaviour each test asserts is the specification's; only the reach into the object differs. @@ -2430,9 +2430,10 @@ specification's; only the reach into the object differs. | S-3 | `evaluate(vt)` | `evaluate(vt, timestamp_ms)`: evaluation needs the server time (RTLCV4e, RTLMV4h), which is an async read (RTO16), so the caller fetches it and passes it in. The tests pass a fixed one | the 13 in `value_types_test.py` that evaluate | | S-4 | The create an evaluated value type retains, as `operation.counterCreate` / `operation.mapCreate` | `operation.resolved_counter_create` / `resolved_map_create`, the `derived_from` of the `*CreateWithObjectId` payload. The two retention tests also assert that `counterCreate` / `mapCreate` is absent from `operation.to_dict()`, which is what "not sent" means and what keeps this reading honest | `test_rtlcv4g5_retains_local_counter_create`, `test_rtlcv4_evaluate_zero_count`, `test_rtlmv4j5_retains_local_map_create`, `test_rtlmv4d_entry_value_types`, `test_rtlmv4d1_nested_value_types`, `test_rtlmv4a_evaluate_validates_entries`, `test_rtlmv4e2_empty_entries`, `test_rtlmv4d_map_set_all_types_table` | | S-5 | Three reads on the untyped `Instance` that answer null or fail: `value()` on a map (RTINS4d), `size()` on a counter (RTINS9c), and `subscribe` on a primitive failing 92007 (RTINS16c) | The typed instances have no such method (RTTS10a–c; `PrimitiveInstance` has no `subscribe` at all, RTTS7b), and the checked views that do have it refuse the wrapped type with 92007 (RTTS9d). The tests assert the wrapped `type`, that the method is absent, and the 92007 from each view in its place | `test_rtins4_value_counter` and `test_rtins9_size` (their second halves), `test_rtins16c_subscribe_primitive_throws` | +| S-6 | `compact()` on a path or an instance (RTPO13, RTINS10) | ably-python's only snapshot is `compact_json()` (RTTS3f, RTTS7d, LODR-061), which is `compact()` with binary as a base64 string (RTPO14b1) and a map reached again as `{'objectId': id}` (RTPO14b2). The tests take the snapshot through `compact_json()`, and where the specification asserts bytes or the reused dict they assert those two forms, so `compact-cycle-detection-0` asserts what `compact-json-0` does | `test_rtpo13_compact_recursive`, `test_rtpo13c5_compact_cycle_detection`, `test_rtpo13c_compact_counter`, `test_rtpo3c1_read_null_on_failure` (its last line), `test_rtins10_compact` | **Status:** shape adaptations, not defects. S-1, S-2 and S-4 close when the specifications are -corrected (candidate O.1); S-3 and S-5 are how ably-python is built. +corrected (candidate O.1); S-3, S-5 and S-6 are how ably-python is built. ### LiveObjects: inputs that do not exist in Python @@ -3636,7 +3637,7 @@ reaches the library only through its public surface and a handful of underscore The specifications are written against the merged features spec's `PathObject` and `Instance`, each one class carrying every method. ably-python follows LODR-061 and the RTTS partition of [#491](https://github.com/ably/specification/pull/491): navigation, `instance`, -`compact`, `compact_json` and `subscribe` are on the base `PathObject`, and a type-specific +`compact_json` and `subscribe` are on the base `PathObject`, and a type-specific method is reached through the view for that type. So `root.get("score").value()` is `root.get('score').as_live_counter().value()`, `root.get("name").value()` is `root.get('name').as_primitive().value()`, and `pathObject.set(k, v)` is @@ -3648,9 +3649,11 @@ both value views. An `Instance`'s views are checked and raise 92007 (RTTS9d). That is a difference of spelling, which `writing-derived-tests.md` says is not a deviation, and nothing above records it test by test. Where the partition makes a read unreachable -rather than differently spelled, the test is an S-5 adaptation. Mutations are `async` and -awaited; reads, navigation, the views and `subscribe` are synchronous; `entries()` yields -`(key, value)` tuples; `Instance.id` and `Instance.type` are properties. +rather than differently spelled, the test is an S-5 adaptation, and where the specification +calls `compact()`, which the partition leaves out (RTTS3f, RTTS7d), it is an S-6 one. +Mutations are `async` and awaited; reads, navigation, the views and `subscribe` are +synchronous; `entries()` yields `(key, value)` tuples; `Instance.id` and `Instance.type` are +properties. ### A standalone `RealtimeObject` drives the pure tier diff --git a/test/uts/objects/unit/instance_test.py b/test/uts/objects/unit/instance_test.py index 53eb9a48..ddcaf1f6 100644 --- a/test/uts/objects/unit/instance_test.py +++ b/test/uts/objects/unit/instance_test.py @@ -5,7 +5,7 @@ The specification is written against the untyped `Instance`, one class carrying every method. ably-python partitions it per RTTS7-RTTS10: `id` and `type` are properties, `get`, -`compact` and the three view helpers are on the base `Instance`, and a type-specific method +`compact_json` and the three view helpers are on the base `Instance`, and a type-specific method is reached through the checked view helper for the wrapped type, so `inst.value()` on a counter is `inst.as_live_counter().value()` and `inst.set(k, v)` on a map is `await inst.as_live_map().set(k, v)`. A view helper asked for a type the instance does not @@ -17,6 +17,8 @@ that do have it refuse the wrapped type. Those tests assert the wrapped `type`, the absence of the method, and the 92007 the view helpers raise in its place (S-5). +The snapshot is `compact_json()` alone (RTTS7d), so RTINS10's test takes it there (S-6). + `setup_synced_channel` uses the standard mock, which ACKs every OBJECT the client sends, so each write is awaited to completion and is applied locally by the time it returns (RTO20). @@ -130,7 +132,8 @@ async def test_rtins10_compact(): client, channel, root, mock_ws = await setup_synced_channel('test') root_inst = root.instance() - result = root_inst.compact() + # S-6: the specification calls `compact()`, which reads the same for these values + result = root_inst.compact_json() assert result['name'] == 'Alice' assert result['score'] == 100 diff --git a/test/uts/objects/unit/path_object_test.py b/test/uts/objects/unit/path_object_test.py index 2172f068..f88da010 100644 --- a/test/uts/objects/unit/path_object_test.py +++ b/test/uts/objects/unit/path_object_test.py @@ -7,14 +7,17 @@ RTPO13c5, RTPO13d, RTPO14, RTPO14a, RTPO14b1, RTPO14b2, RTINS3b, RTINS4c, RTTS5d1 The specification reads through the untyped `PathObject`, which carries every method. ably-python -partitions it (LODR-061, RTTS3-RTTS6): navigation, `instance`, `compact` and `compact_json` are on the -base `PathObject`, and the type-specific reads are reached through the unchecked view helpers, so +partitions it (LODR-061, RTTS3-RTTS6): navigation, `instance` and `compact_json` are on the base +`PathObject`, and the type-specific reads are reached through the unchecked view helpers, so `po.value()` on a counter is `po.as_live_counter().value()`, on a primitive `po.as_primitive().value()`, and `po.keys()` is `po.as_live_map().keys()`. A view whose type does not match what the path resolves to answers None, or `[]` for the collection reads (RTTS5d1), which is what the specification's null and empty-array expectations become. Where the specification expects the untyped `value()` to be null, neither value view may answer, so both are asserted. +The snapshot is `compact_json()` alone (RTTS3f), so the RTPO13 tests take it there and assert its +form where the two differ: binary as base64, and a map reached again as its object id (S-6). + Every test opens with the standard synced channel; the module issues no writes, so no ACK is involved. """ @@ -274,14 +277,15 @@ async def test_rtpo12c_size_non_map_null(): async def test_rtpo13_compact_recursive(): client, channel, root, mock_ws = await setup_synced_channel('test') - result = root.compact() + result = root.compact_json() assert result['name'] == 'Alice' assert result['age'] == 30 assert result['active'] is True assert result['score'] == 100 assert result['data'] == {'tags': ['a', 'b']} - assert result['avatar'] == bytes([1, 2, 3]) + # S-6: the specification's `compact()` keeps the bytes + assert result['avatar'] == 'AQID' assert result['profile']['email'] == 'alice@example.com' assert result['profile']['nested_counter'] == 5 assert result['profile']['prefs']['theme'] == 'dark' @@ -292,16 +296,18 @@ async def test_rtpo13c5_compact_cycle_detection(): client, channel, root, mock_ws = await setup_synced_channel('test') await _apply_back_ref(root, mock_ws) - result = root.get('profile').compact() + result = root.get('profile').compact_json() - assert result['prefs']['back_ref'] is result + # S-6: the specification's `compact()` reuses the dict already built for the map + assert result['prefs']['back_ref'] == {'objectId': 'map:profile@1000'} # UTS: objects/unit/RTPO13c/compact-counter-0 async def test_rtpo13c_compact_counter(): client, channel, root, mock_ws = await setup_synced_channel('test') - assert root.get('score').compact() == 100 + # S-6: the specification calls `compact()`, which reads the same for a counter + assert root.get('score').compact_json() == 100 # UTS: objects/unit/RTPO14/compact-json-0 @@ -344,7 +350,8 @@ async def test_rtpo3c1_read_null_on_failure(): assert nonexistent.as_live_counter().value() is None assert nonexistent.instance() is None assert nonexistent.as_live_map().size() is None - assert nonexistent.compact() is None + # S-6: the specification calls `compact()`, which reads the same where nothing resolves + assert nonexistent.compact_json() is None # UTS: objects/unit/RTPO7/value-bytes-0 From 112a4d6b491383a490a98c2f46a12aba57c571ea Mon Sep 17 00:00:00 2001 From: owenpearson Date: Mon, 19 Oct 2026 10:49:03 +0100 Subject: [PATCH 23/71] liveobjects: name the package for the product Moves ably.pubsub.objects to ably.pubsub.liveobjects, and its unit tests to test/unit/liveobjects, following PDR-080: package and import names carry the product name, while the API keeps the resource name, so channel.object, RealtimeObject and ObjectMessage are unchanged. test/uts/objects keeps the specification's directory name, so each derived test still maps onto the uts/objects specification it comes from. Co-Authored-By: Claude Opus 5.5 (1M context) --- .claude/skills/uts-to-python/SKILL.md | 6 +++-- .../{objects => liveobjects}/__init__.py | 0 ably/pubsub/{objects => liveobjects}/batch.py | 16 ++++++------- .../{objects => liveobjects}/defaults.py | 0 ably/pubsub/{objects => liveobjects}/enums.py | 0 .../{objects => liveobjects}/instance.py | 24 +++++++++---------- .../{objects => liveobjects}/livecounter.py | 10 ++++---- .../{objects => liveobjects}/livemap.py | 14 +++++------ .../{objects => liveobjects}/liveobject.py | 14 +++++------ .../{objects => liveobjects}/objectid.py | 0 .../{objects => liveobjects}/objectmessage.py | 2 +- .../{objects => liveobjects}/objectspool.py | 12 +++++----- .../{objects => liveobjects}/pathobject.py | 22 ++++++++--------- .../pathobjectsubscriptionregister.py | 12 +++++----- .../{objects => liveobjects}/publicmessage.py | 6 ++--- .../realtimeobject.py | 24 +++++++++---------- .../{objects => liveobjects}/subscription.py | 0 .../syncobjectspool.py | 2 +- .../{objects => liveobjects}/valuetypes.py | 4 ++-- ably/pubsub/realtime/channel.py | 4 ++-- ably/pubsub/server/__init__.py | 18 +++++++------- .../unit/{objects => liveobjects}/__init__.py | 0 .../{objects => liveobjects}/batch_test.py | 2 +- .../unit/{objects => liveobjects}/conftest.py | 0 .../realtime_object_test.py | 16 ++++++------- .../subscription_test.py | 0 .../typed_views_test.py | 2 +- .../value_types_test.py | 2 +- test/uts/deviations.md | 2 +- .../uts/objects/helpers/standard_test_pool.py | 4 ++-- .../helpers/standard_test_pool_test.py | 6 ++--- test/uts/objects/unit/instance_test.py | 10 ++++---- .../unit/internal_live_counter_api_test.py | 2 +- .../unit/internal_live_counter_test.py | 6 ++--- .../unit/internal_live_map_api_test.py | 4 ++-- .../objects/unit/internal_live_map_test.py | 10 ++++---- .../unit/live_object_subscribe_test.py | 6 ++--- test/uts/objects/unit/object_id_test.py | 2 +- test/uts/objects/unit/objects_pool_test.py | 12 +++++----- .../objects/unit/parent_references_test.py | 12 +++++----- .../unit/path_object_subscribe_test.py | 6 ++--- test/uts/objects/unit/path_object_test.py | 2 +- .../unit/public_object_message_test.py | 4 ++-- test/uts/objects/unit/realtime_object_test.py | 8 +++---- test/uts/objects/unit/value_types_test.py | 4 ++-- 45 files changed, 157 insertions(+), 155 deletions(-) rename ably/pubsub/{objects => liveobjects}/__init__.py (100%) rename ably/pubsub/{objects => liveobjects}/batch.py (96%) rename ably/pubsub/{objects => liveobjects}/defaults.py (100%) rename ably/pubsub/{objects => liveobjects}/enums.py (100%) rename ably/pubsub/{objects => liveobjects}/instance.py (94%) rename ably/pubsub/{objects => liveobjects}/livecounter.py (95%) rename ably/pubsub/{objects => liveobjects}/livemap.py (97%) rename ably/pubsub/{objects => liveobjects}/liveobject.py (95%) rename ably/pubsub/{objects => liveobjects}/objectid.py (100%) rename ably/pubsub/{objects => liveobjects}/objectmessage.py (99%) rename ably/pubsub/{objects => liveobjects}/objectspool.py (92%) rename ably/pubsub/{objects => liveobjects}/pathobject.py (94%) rename ably/pubsub/{objects => liveobjects}/pathobjectsubscriptionregister.py (90%) rename ably/pubsub/{objects => liveobjects}/publicmessage.py (94%) rename ably/pubsub/{objects => liveobjects}/realtimeobject.py (97%) rename ably/pubsub/{objects => liveobjects}/subscription.py (100%) rename ably/pubsub/{objects => liveobjects}/syncobjectspool.py (98%) rename ably/pubsub/{objects => liveobjects}/valuetypes.py (98%) rename test/unit/{objects => liveobjects}/__init__.py (100%) rename test/unit/{objects => liveobjects}/batch_test.py (99%) rename test/unit/{objects => liveobjects}/conftest.py (100%) rename test/unit/{objects => liveobjects}/realtime_object_test.py (96%) rename test/unit/{objects => liveobjects}/subscription_test.py (100%) rename test/unit/{objects => liveobjects}/typed_views_test.py (99%) rename test/unit/{objects => liveobjects}/value_types_test.py (95%) diff --git a/.claude/skills/uts-to-python/SKILL.md b/.claude/skills/uts-to-python/SKILL.md index a5930b02..32b100c7 100644 --- a/.claude/skills/uts-to-python/SKILL.md +++ b/.claude/skills/uts-to-python/SKILL.md @@ -466,7 +466,9 @@ websocket. Follow `internal_live_map_test.py` (pure), `path_object_mutations_tes and `objects_lifecycle_test.py` (integration). The integration package's own `conftest.py` provisions a separate app under the realtime tier's fixture name, `realtime_sandbox`, and all three of its specifications take `use_binary_protocol`. The package is -`ably.pubsub.objects`; the public names are exported from `ably.pubsub.server`. +`ably.pubsub.liveobjects`, named for the product as PDR-080 has package names do, while +`test/uts/objects/` keeps the specification's directory name so that each test file maps +onto its specification; the public names are exported from `ably.pubsub.server`. **Translate the untyped API through the typed views** (LODR-061, the RTTS partition). The specifications call every method on one `PathObject` or `Instance` class; ably-python @@ -538,7 +540,7 @@ class that LODR-061 does not name carry a leading underscore (`channel.object._o left `_objects_pool`. The GC interval, 300000 ms, is read when the first ATTACHED schedules the timer on the client's clock: set `channel.object._gc_interval_ms` before `get()`. - **The public and internal `ObjectMessage` share a name.** Reach the public one as - `publicmessage.ObjectMessage`, from `ably.pubsub.objects`. + `publicmessage.ObjectMessage`, from `ably.pubsub.liveobjects`. - **Check that a test can fail.** Several objects specifications assert something that holds with the behaviour removed. The derivation caught them by running each module against a throwaway reference implementation with one fault injected; add the discriminating diff --git a/ably/pubsub/objects/__init__.py b/ably/pubsub/liveobjects/__init__.py similarity index 100% rename from ably/pubsub/objects/__init__.py rename to ably/pubsub/liveobjects/__init__.py diff --git a/ably/pubsub/objects/batch.py b/ably/pubsub/liveobjects/batch.py similarity index 96% rename from ably/pubsub/objects/batch.py rename to ably/pubsub/liveobjects/batch.py index 87240627..1d05afe0 100644 --- a/ably/pubsub/objects/batch.py +++ b/ably/pubsub/liveobjects/batch.py @@ -20,8 +20,8 @@ from functools import partial from typing import TYPE_CHECKING, Any, Awaitable, Callable, Generic, TypeVar, overload -from ably.pubsub.objects.enums import ValueType -from ably.pubsub.objects.instance import ( +from ably.pubsub.liveobjects.enums import ValueType +from ably.pubsub.liveobjects.instance import ( Instance, LiveCounterInstance, LiveMapInstance, @@ -30,10 +30,10 @@ primitive_value, value_type_of, ) -from ably.pubsub.objects.livecounter import counter_inc_message -from ably.pubsub.objects.livemap import map_remove_message, map_set_message, map_set_messages -from ably.pubsub.objects.objectmessage import ObjectMessage -from ably.pubsub.objects.valuetypes import ( +from ably.pubsub.liveobjects.livecounter import counter_inc_message +from ably.pubsub.liveobjects.livemap import map_remove_message, map_set_message, map_set_messages +from ably.pubsub.liveobjects.objectmessage import ObjectMessage +from ably.pubsub.liveobjects.valuetypes import ( LiveCounter, LiveMap, primitive_to_object_data, @@ -44,8 +44,8 @@ from ably.pubsub.util.exceptions import AblyException if TYPE_CHECKING: - from ably.pubsub.objects.realtimeobject import RealtimeObject - from ably.pubsub.objects.valuetypes import JsonValue, LiveMapValue, Primitive, T + from ably.pubsub.liveobjects.realtimeobject import RealtimeObject + from ably.pubsub.liveobjects.valuetypes import JsonValue, LiveMapValue, Primitive, T # RTBC16b: builds the ObjectMessages of one queued write when the batch is flushed MessageConstructor = Callable[[], Awaitable[list[ObjectMessage]]] diff --git a/ably/pubsub/objects/defaults.py b/ably/pubsub/liveobjects/defaults.py similarity index 100% rename from ably/pubsub/objects/defaults.py rename to ably/pubsub/liveobjects/defaults.py diff --git a/ably/pubsub/objects/enums.py b/ably/pubsub/liveobjects/enums.py similarity index 100% rename from ably/pubsub/objects/enums.py rename to ably/pubsub/liveobjects/enums.py diff --git a/ably/pubsub/objects/instance.py b/ably/pubsub/liveobjects/instance.py similarity index 94% rename from ably/pubsub/objects/instance.py rename to ably/pubsub/liveobjects/instance.py index 5db51233..3164ce4b 100644 --- a/ably/pubsub/objects/instance.py +++ b/ably/pubsub/liveobjects/instance.py @@ -17,19 +17,19 @@ from dataclasses import dataclass from typing import TYPE_CHECKING, Any, Callable, overload -from ably.pubsub.objects.enums import ValueType -from ably.pubsub.objects.livecounter import InternalLiveCounter -from ably.pubsub.objects.livemap import InternalLiveMap -from ably.pubsub.objects.liveobject import LiveObject -from ably.pubsub.objects.publicmessage import ObjectMessage +from ably.pubsub.liveobjects.enums import ValueType +from ably.pubsub.liveobjects.livecounter import InternalLiveCounter +from ably.pubsub.liveobjects.livemap import InternalLiveMap +from ably.pubsub.liveobjects.liveobject import LiveObject +from ably.pubsub.liveobjects.publicmessage import ObjectMessage from ably.pubsub.util.exceptions import AblyException if TYPE_CHECKING: - from ably.pubsub.objects.batch import Batch, LiveCounterBatchContext, LiveMapBatchContext - from ably.pubsub.objects.liveobject import LiveObjectUpdate - from ably.pubsub.objects.realtimeobject import RealtimeObject - from ably.pubsub.objects.subscription import Subscription - from ably.pubsub.objects.valuetypes import JsonObject, JsonValue, LiveMapValue, Primitive, T + from ably.pubsub.liveobjects.batch import Batch, LiveCounterBatchContext, LiveMapBatchContext + from ably.pubsub.liveobjects.liveobject import LiveObjectUpdate + from ably.pubsub.liveobjects.realtimeobject import RealtimeObject + from ably.pubsub.liveobjects.subscription import Subscription + from ably.pubsub.liveobjects.valuetypes import JsonObject, JsonValue, LiveMapValue, Primitive, T # The types `value(expected)` accepts as `expected`, and the category each one selects _EXPECTED_VALUE_TYPES: tuple[tuple[type, ValueType], ...] = ( @@ -262,7 +262,7 @@ def batch(self) -> Batch[LiveMapBatchContext]: Entering checks the write preconditions (RTINS17b). """ # Imported here, as the batch module builds on the views - from ably.pubsub.objects.batch import Batch, LiveMapBatchContext + from ably.pubsub.liveobjects.batch import Batch, LiveMapBatchContext return Batch(self._realtime_object, lambda: self._value, LiveMapBatchContext, f'object {self.id!r}') @@ -328,7 +328,7 @@ def batch(self) -> Batch[LiveCounterBatchContext]: Entering checks the write preconditions (RTINS17b). """ # Imported here, as the batch module builds on the views - from ably.pubsub.objects.batch import Batch, LiveCounterBatchContext + from ably.pubsub.liveobjects.batch import Batch, LiveCounterBatchContext return Batch(self._realtime_object, lambda: self._value, LiveCounterBatchContext, f'object {self.id!r}') diff --git a/ably/pubsub/objects/livecounter.py b/ably/pubsub/liveobjects/livecounter.py similarity index 95% rename from ably/pubsub/objects/livecounter.py rename to ably/pubsub/liveobjects/livecounter.py index 69ce71df..407eb7a9 100644 --- a/ably/pubsub/objects/livecounter.py +++ b/ably/pubsub/liveobjects/livecounter.py @@ -5,15 +5,15 @@ import logging from typing import TYPE_CHECKING -from ably.pubsub.objects.enums import ObjectsOperationSource -from ably.pubsub.objects.liveobject import CounterUpdate, LiveCounterUpdate, LiveObject -from ably.pubsub.objects.objectmessage import CounterInc, ObjectMessage, ObjectOperation, ObjectOperationAction -from ably.pubsub.objects.valuetypes import validate_amount +from ably.pubsub.liveobjects.enums import ObjectsOperationSource +from ably.pubsub.liveobjects.liveobject import CounterUpdate, LiveCounterUpdate, LiveObject +from ably.pubsub.liveobjects.objectmessage import CounterInc, ObjectMessage, ObjectOperation, ObjectOperationAction +from ably.pubsub.liveobjects.valuetypes import validate_amount from ably.pubsub.util.clock import Clock from ably.pubsub.util.exceptions import AblyException if TYPE_CHECKING: - from ably.pubsub.objects.objectspool import ObjectsPool + from ably.pubsub.liveobjects.objectspool import ObjectsPool log = logging.getLogger(__name__) diff --git a/ably/pubsub/objects/livemap.py b/ably/pubsub/liveobjects/livemap.py similarity index 97% rename from ably/pubsub/objects/livemap.py rename to ably/pubsub/liveobjects/livemap.py index 4b8f44f2..85aadd15 100644 --- a/ably/pubsub/objects/livemap.py +++ b/ably/pubsub/liveobjects/livemap.py @@ -6,9 +6,9 @@ import logging from typing import TYPE_CHECKING, Any -from ably.pubsub.objects.enums import ObjectsOperationSource -from ably.pubsub.objects.liveobject import MAP_KEY_REMOVED, MAP_KEY_UPDATED, LiveMapUpdate, LiveObject -from ably.pubsub.objects.objectmessage import ( +from ably.pubsub.liveobjects.enums import ObjectsOperationSource +from ably.pubsub.liveobjects.liveobject import MAP_KEY_REMOVED, MAP_KEY_UPDATED, LiveMapUpdate, LiveObject +from ably.pubsub.liveobjects.objectmessage import ( MapRemove, MapSet, ObjectData, @@ -18,7 +18,7 @@ ObjectsMapEntry, ObjectsMapSemantics, ) -from ably.pubsub.objects.valuetypes import ( +from ably.pubsub.liveobjects.valuetypes import ( LiveCounter, LiveMap, evaluate, @@ -30,9 +30,9 @@ from ably.pubsub.util.exceptions import AblyException if TYPE_CHECKING: - from ably.pubsub.objects.objectspool import ObjectsPool - from ably.pubsub.objects.realtimeobject import RealtimeObject - from ably.pubsub.objects.valuetypes import LiveMapValue + from ably.pubsub.liveobjects.objectspool import ObjectsPool + from ably.pubsub.liveobjects.realtimeobject import RealtimeObject + from ably.pubsub.liveobjects.valuetypes import LiveMapValue log = logging.getLogger(__name__) diff --git a/ably/pubsub/objects/liveobject.py b/ably/pubsub/liveobjects/liveobject.py similarity index 95% rename from ably/pubsub/objects/liveobject.py rename to ably/pubsub/liveobjects/liveobject.py index 47bf61b5..00a95ada 100644 --- a/ably/pubsub/objects/liveobject.py +++ b/ably/pubsub/liveobjects/liveobject.py @@ -7,16 +7,16 @@ from dataclasses import dataclass, field from typing import TYPE_CHECKING, Any, Callable -from ably.pubsub.objects.defaults import ROOT_OBJECT_ID -from ably.pubsub.objects.objectmessage import ObjectMessage -from ably.pubsub.objects.subscription import Registry, Subscription +from ably.pubsub.liveobjects.defaults import ROOT_OBJECT_ID +from ably.pubsub.liveobjects.objectmessage import ObjectMessage +from ably.pubsub.liveobjects.subscription import Registry, Subscription from ably.pubsub.util.clock import Clock if TYPE_CHECKING: - from ably.pubsub.objects.enums import ObjectsOperationSource - from ably.pubsub.objects.livemap import InternalLiveMap - from ably.pubsub.objects.objectspool import ObjectsPool - from ably.pubsub.objects.realtimeobject import RealtimeObject + from ably.pubsub.liveobjects.enums import ObjectsOperationSource + from ably.pubsub.liveobjects.livemap import InternalLiveMap + from ably.pubsub.liveobjects.objectspool import ObjectsPool + from ably.pubsub.liveobjects.realtimeobject import RealtimeObject log = logging.getLogger(__name__) diff --git a/ably/pubsub/objects/objectid.py b/ably/pubsub/liveobjects/objectid.py similarity index 100% rename from ably/pubsub/objects/objectid.py rename to ably/pubsub/liveobjects/objectid.py diff --git a/ably/pubsub/objects/objectmessage.py b/ably/pubsub/liveobjects/objectmessage.py similarity index 99% rename from ably/pubsub/objects/objectmessage.py rename to ably/pubsub/liveobjects/objectmessage.py index 6dce9407..5707a23c 100644 --- a/ably/pubsub/objects/objectmessage.py +++ b/ably/pubsub/liveobjects/objectmessage.py @@ -592,7 +592,7 @@ class ObjectMessage: """OM1: an object message sent or received over a channel. This is the internal type. The type delivered to listeners is the public - `ably.pubsub.objects.publicmessage.ObjectMessage` (PAOM1). + `ably.pubsub.liveobjects.publicmessage.ObjectMessage` (PAOM1). """ id: str | None = None # OM2a diff --git a/ably/pubsub/objects/objectspool.py b/ably/pubsub/liveobjects/objectspool.py similarity index 92% rename from ably/pubsub/objects/objectspool.py rename to ably/pubsub/liveobjects/objectspool.py index 41919bc3..741b120b 100644 --- a/ably/pubsub/objects/objectspool.py +++ b/ably/pubsub/liveobjects/objectspool.py @@ -6,15 +6,15 @@ from collections.abc import Iterator, MutableMapping from typing import TYPE_CHECKING -from ably.pubsub.objects.defaults import ROOT_OBJECT_ID -from ably.pubsub.objects.livecounter import InternalLiveCounter -from ably.pubsub.objects.livemap import InternalLiveMap -from ably.pubsub.objects.liveobject import LiveObject -from ably.pubsub.objects.objectid import object_type_of +from ably.pubsub.liveobjects.defaults import ROOT_OBJECT_ID +from ably.pubsub.liveobjects.livecounter import InternalLiveCounter +from ably.pubsub.liveobjects.livemap import InternalLiveMap +from ably.pubsub.liveobjects.liveobject import LiveObject +from ably.pubsub.liveobjects.objectid import object_type_of from ably.pubsub.util.clock import Clock if TYPE_CHECKING: - from ably.pubsub.objects.realtimeobject import RealtimeObject + from ably.pubsub.liveobjects.realtimeobject import RealtimeObject log = logging.getLogger(__name__) diff --git a/ably/pubsub/objects/pathobject.py b/ably/pubsub/liveobjects/pathobject.py similarity index 94% rename from ably/pubsub/objects/pathobject.py rename to ably/pubsub/liveobjects/pathobject.py index 98875002..2a8d15ec 100644 --- a/ably/pubsub/objects/pathobject.py +++ b/ably/pubsub/liveobjects/pathobject.py @@ -20,24 +20,24 @@ from dataclasses import dataclass from typing import TYPE_CHECKING, Any, Callable, overload -from ably.pubsub.objects.instance import ( +from ably.pubsub.liveobjects.instance import ( Instance, compact_json_value, expected_value_type, primitive_value, value_type_of, ) -from ably.pubsub.objects.livecounter import InternalLiveCounter -from ably.pubsub.objects.livemap import InternalLiveMap -from ably.pubsub.objects.publicmessage import ObjectMessage +from ably.pubsub.liveobjects.livecounter import InternalLiveCounter +from ably.pubsub.liveobjects.livemap import InternalLiveMap +from ably.pubsub.liveobjects.publicmessage import ObjectMessage from ably.pubsub.util.exceptions import AblyException if TYPE_CHECKING: - from ably.pubsub.objects.batch import Batch, LiveCounterBatchContext, LiveMapBatchContext - from ably.pubsub.objects.enums import ValueType - from ably.pubsub.objects.realtimeobject import RealtimeObject - from ably.pubsub.objects.subscription import Subscription - from ably.pubsub.objects.valuetypes import JsonValue, LiveMapValue, Primitive, T + from ably.pubsub.liveobjects.batch import Batch, LiveCounterBatchContext, LiveMapBatchContext + from ably.pubsub.liveobjects.enums import ValueType + from ably.pubsub.liveobjects.realtimeobject import RealtimeObject + from ably.pubsub.liveobjects.subscription import Subscription + from ably.pubsub.liveobjects.valuetypes import JsonValue, LiveMapValue, Primitive, T @dataclass @@ -189,7 +189,7 @@ def batch(self) -> Batch[LiveMapBatchContext]: the path does not resolve to a map (RTPO20c). """ # Imported here, as the batch module builds on the views - from ably.pubsub.objects.batch import Batch, LiveMapBatchContext + from ably.pubsub.liveobjects.batch import Batch, LiveMapBatchContext return Batch(self._realtime_object, self._resolve, LiveMapBatchContext, f'path {self.path()!r}') @@ -244,7 +244,7 @@ def batch(self) -> Batch[LiveCounterBatchContext]: the path does not resolve to a counter (RTPO20c). """ # Imported here, as the batch module builds on the views - from ably.pubsub.objects.batch import Batch, LiveCounterBatchContext + from ably.pubsub.liveobjects.batch import Batch, LiveCounterBatchContext return Batch(self._realtime_object, self._resolve, LiveCounterBatchContext, f'path {self.path()!r}') diff --git a/ably/pubsub/objects/pathobjectsubscriptionregister.py b/ably/pubsub/liveobjects/pathobjectsubscriptionregister.py similarity index 90% rename from ably/pubsub/objects/pathobjectsubscriptionregister.py rename to ably/pubsub/liveobjects/pathobjectsubscriptionregister.py index aa2c7ff7..d78cee9f 100644 --- a/ably/pubsub/objects/pathobjectsubscriptionregister.py +++ b/ably/pubsub/liveobjects/pathobjectsubscriptionregister.py @@ -5,14 +5,14 @@ import logging from typing import TYPE_CHECKING, Callable -from ably.pubsub.objects.liveobject import LiveMapUpdate -from ably.pubsub.objects.pathobject import PathObject, PathObjectSubscriptionEvent -from ably.pubsub.objects.publicmessage import ObjectMessage -from ably.pubsub.objects.subscription import Registry, Subscription +from ably.pubsub.liveobjects.liveobject import LiveMapUpdate +from ably.pubsub.liveobjects.pathobject import PathObject, PathObjectSubscriptionEvent +from ably.pubsub.liveobjects.publicmessage import ObjectMessage +from ably.pubsub.liveobjects.subscription import Registry, Subscription if TYPE_CHECKING: - from ably.pubsub.objects.liveobject import LiveObject, LiveObjectUpdate - from ably.pubsub.objects.realtimeobject import RealtimeObject + from ably.pubsub.liveobjects.liveobject import LiveObject, LiveObjectUpdate + from ably.pubsub.liveobjects.realtimeobject import RealtimeObject log = logging.getLogger(__name__) diff --git a/ably/pubsub/objects/publicmessage.py b/ably/pubsub/liveobjects/publicmessage.py similarity index 94% rename from ably/pubsub/objects/publicmessage.py rename to ably/pubsub/liveobjects/publicmessage.py index 8f603c0e..49642452 100644 --- a/ably/pubsub/objects/publicmessage.py +++ b/ably/pubsub/liveobjects/publicmessage.py @@ -2,7 +2,7 @@ The specification calls these `PublicAPI::ObjectMessage` and `PublicAPI::ObjectOperation`. They are exposed to users as `ObjectMessage` and `ObjectOperation`, sharing their names -with the internal wire types in `ably.pubsub.objects.objectmessage`; code that needs both +with the internal wire types in `ably.pubsub.liveobjects.objectmessage`; code that needs both imports one of the pairs under a module-qualified name. """ @@ -12,7 +12,7 @@ from dataclasses import dataclass from typing import TYPE_CHECKING -from ably.pubsub.objects.objectmessage import ( +from ably.pubsub.liveobjects.objectmessage import ( CounterCreate, CounterInc, MapClear, @@ -24,7 +24,7 @@ ) if TYPE_CHECKING: - from ably.pubsub.objects import objectmessage + from ably.pubsub.liveobjects import objectmessage @dataclass diff --git a/ably/pubsub/objects/realtimeobject.py b/ably/pubsub/liveobjects/realtimeobject.py similarity index 97% rename from ably/pubsub/objects/realtimeobject.py rename to ably/pubsub/liveobjects/realtimeobject.py index c2276b11..62c4792d 100644 --- a/ably/pubsub/objects/realtimeobject.py +++ b/ably/pubsub/liveobjects/realtimeobject.py @@ -7,22 +7,22 @@ import logging from typing import TYPE_CHECKING, Any, Callable -from ably.pubsub.objects.defaults import GC_GRACE_PERIOD_MS, GC_INTERVAL_MS, ROOT_OBJECT_ID -from ably.pubsub.objects.enums import ObjectsEvent, ObjectsOperationSource, ObjectsSyncState -from ably.pubsub.objects.livecounter import InternalLiveCounter -from ably.pubsub.objects.livemap import InternalLiveMap -from ably.pubsub.objects.liveobject import LiveObject, LiveObjectUpdate -from ably.pubsub.objects.objectmessage import ( +from ably.pubsub.liveobjects.defaults import GC_GRACE_PERIOD_MS, GC_INTERVAL_MS, ROOT_OBJECT_ID +from ably.pubsub.liveobjects.enums import ObjectsEvent, ObjectsOperationSource, ObjectsSyncState +from ably.pubsub.liveobjects.livecounter import InternalLiveCounter +from ably.pubsub.liveobjects.livemap import InternalLiveMap +from ably.pubsub.liveobjects.liveobject import LiveObject, LiveObjectUpdate +from ably.pubsub.liveobjects.objectmessage import ( WIRE_FORMAT_JSON, WIRE_FORMAT_MSGPACK, ObjectOperationAction, ObjectsMapSemantics, ) -from ably.pubsub.objects.objectspool import ObjectsPool -from ably.pubsub.objects.pathobject import LiveMapPathObject -from ably.pubsub.objects.pathobjectsubscriptionregister import PathObjectSubscriptionRegister -from ably.pubsub.objects.subscription import Registry, StatusSubscription -from ably.pubsub.objects.syncobjectspool import SyncObjectsPool +from ably.pubsub.liveobjects.objectspool import ObjectsPool +from ably.pubsub.liveobjects.pathobject import LiveMapPathObject +from ably.pubsub.liveobjects.pathobjectsubscriptionregister import PathObjectSubscriptionRegister +from ably.pubsub.liveobjects.subscription import Registry, StatusSubscription +from ably.pubsub.liveobjects.syncobjectspool import SyncObjectsPool from ably.pubsub.transport.websockettransport import ProtocolMessageAction from ably.pubsub.types.channelmode import ChannelMode from ably.pubsub.types.channelstate import ChannelState @@ -30,7 +30,7 @@ from ably.pubsub.util.exceptions import AblyException if TYPE_CHECKING: - from ably.pubsub.objects.objectmessage import ObjectMessage + from ably.pubsub.liveobjects.objectmessage import ObjectMessage from ably.pubsub.realtime.channel import RealtimeChannel from ably.pubsub.types.connectiondetails import ConnectionDetails from ably.pubsub.types.operations import PublishResult diff --git a/ably/pubsub/objects/subscription.py b/ably/pubsub/liveobjects/subscription.py similarity index 100% rename from ably/pubsub/objects/subscription.py rename to ably/pubsub/liveobjects/subscription.py diff --git a/ably/pubsub/objects/syncobjectspool.py b/ably/pubsub/liveobjects/syncobjectspool.py similarity index 98% rename from ably/pubsub/objects/syncobjectspool.py rename to ably/pubsub/liveobjects/syncobjectspool.py index 254ba592..88e54478 100644 --- a/ably/pubsub/objects/syncobjectspool.py +++ b/ably/pubsub/liveobjects/syncobjectspool.py @@ -6,7 +6,7 @@ from typing import TYPE_CHECKING if TYPE_CHECKING: - from ably.pubsub.objects.objectmessage import ObjectMessage + from ably.pubsub.liveobjects.objectmessage import ObjectMessage log = logging.getLogger(__name__) diff --git a/ably/pubsub/objects/valuetypes.py b/ably/pubsub/liveobjects/valuetypes.py similarity index 98% rename from ably/pubsub/objects/valuetypes.py rename to ably/pubsub/liveobjects/valuetypes.py index c5ed6a9c..176f44f5 100644 --- a/ably/pubsub/objects/valuetypes.py +++ b/ably/pubsub/liveobjects/valuetypes.py @@ -15,8 +15,8 @@ from dataclasses import dataclass from typing import Any, Dict, List, TypeVar, Union -from ably.pubsub.objects.objectid import generate_nonce, generate_object_id -from ably.pubsub.objects.objectmessage import ( +from ably.pubsub.liveobjects.objectid import generate_nonce, generate_object_id +from ably.pubsub.liveobjects.objectmessage import ( WIRE_FORMAT_JSON, CounterCreate, CounterCreateWithObjectId, diff --git a/ably/pubsub/realtime/channel.py b/ably/pubsub/realtime/channel.py index 538222da..d9b74921 100644 --- a/ably/pubsub/realtime/channel.py +++ b/ably/pubsub/realtime/channel.py @@ -6,8 +6,8 @@ from ably.pubsub.http.channel import Channel from ably.pubsub.http.channel import Channels as HttpChannels -from ably.pubsub.objects.objectmessage import ObjectMessage -from ably.pubsub.objects.realtimeobject import RealtimeObject +from ably.pubsub.liveobjects.objectmessage import ObjectMessage +from ably.pubsub.liveobjects.realtimeobject import RealtimeObject from ably.pubsub.realtime.annotations import RealtimeAnnotations from ably.pubsub.realtime.connection import ConnectionState from ably.pubsub.realtime.presence import RealtimePresence diff --git a/ably/pubsub/server/__init__.py b/ably/pubsub/server/__init__.py index 572b3b5d..1323fe36 100644 --- a/ably/pubsub/server/__init__.py +++ b/ably/pubsub/server/__init__.py @@ -22,33 +22,33 @@ from ably.pubsub.http.channel import Channels as HttpChannels from ably.pubsub.http.http import DefaultPubSubHttpClient as _DefaultPubSubHttpClient from ably.pubsub.http.push import Push -from ably.pubsub.objects.batch import ( +from ably.pubsub.liveobjects.batch import ( Batch, BatchContext, LiveCounterBatchContext, LiveMapBatchContext, PrimitiveBatchContext, ) -from ably.pubsub.objects.enums import ObjectsEvent, ValueType -from ably.pubsub.objects.instance import ( +from ably.pubsub.liveobjects.enums import ObjectsEvent, ValueType +from ably.pubsub.liveobjects.instance import ( Instance, InstanceSubscriptionEvent, LiveCounterInstance, LiveMapInstance, PrimitiveInstance, ) -from ably.pubsub.objects.objectmessage import ObjectOperationAction, ObjectsMapSemantics -from ably.pubsub.objects.pathobject import ( +from ably.pubsub.liveobjects.objectmessage import ObjectOperationAction, ObjectsMapSemantics +from ably.pubsub.liveobjects.pathobject import ( LiveCounterPathObject, LiveMapPathObject, PathObject, PathObjectSubscriptionEvent, PrimitivePathObject, ) -from ably.pubsub.objects.publicmessage import ObjectMessage, ObjectOperation -from ably.pubsub.objects.realtimeobject import RealtimeObject -from ably.pubsub.objects.subscription import StatusSubscription, Subscription -from ably.pubsub.objects.valuetypes import ( +from ably.pubsub.liveobjects.publicmessage import ObjectMessage, ObjectOperation +from ably.pubsub.liveobjects.realtimeobject import RealtimeObject +from ably.pubsub.liveobjects.subscription import StatusSubscription, Subscription +from ably.pubsub.liveobjects.valuetypes import ( JsonObject, JsonValue, LiveCounter, diff --git a/test/unit/objects/__init__.py b/test/unit/liveobjects/__init__.py similarity index 100% rename from test/unit/objects/__init__.py rename to test/unit/liveobjects/__init__.py diff --git a/test/unit/objects/batch_test.py b/test/unit/liveobjects/batch_test.py similarity index 99% rename from test/unit/objects/batch_test.py rename to test/unit/liveobjects/batch_test.py index 3d8c4123..1b4923bd 100644 --- a/test/unit/objects/batch_test.py +++ b/test/unit/liveobjects/batch_test.py @@ -13,7 +13,7 @@ import pytest -from ably.pubsub.objects.objectmessage import ObjectOperationAction +from ably.pubsub.liveobjects.objectmessage import ObjectOperationAction from ably.pubsub.server import ( LiveCounter, LiveCounterBatchContext, diff --git a/test/unit/objects/conftest.py b/test/unit/liveobjects/conftest.py similarity index 100% rename from test/unit/objects/conftest.py rename to test/unit/liveobjects/conftest.py diff --git a/test/unit/objects/realtime_object_test.py b/test/unit/liveobjects/realtime_object_test.py similarity index 96% rename from test/unit/objects/realtime_object_test.py rename to test/unit/liveobjects/realtime_object_test.py index bdc2184a..6cf2fee7 100644 --- a/test/unit/objects/realtime_object_test.py +++ b/test/unit/liveobjects/realtime_object_test.py @@ -16,9 +16,9 @@ import pytest -from ably.pubsub.objects.defaults import GC_INTERVAL_MS -from ably.pubsub.objects.enums import ObjectsSyncState -from ably.pubsub.objects.realtimeobject import RealtimeObject +from ably.pubsub.liveobjects.defaults import GC_INTERVAL_MS +from ably.pubsub.liveobjects.enums import ObjectsSyncState +from ably.pubsub.liveobjects.realtimeobject import RealtimeObject from ably.pubsub.realtime.connection import ConnectionState from ably.pubsub.types.channelstate import ChannelState from ably.pubsub.util.exceptions import AblyException @@ -202,7 +202,7 @@ async def test_rto5_an_object_sync_message_that_fails_to_decode_is_skipped(caplo 'semantics': LWW, 'entries': {'k': {'data': {'json': '{not json'}, 'timeserial': 't:0'}}}) mock_ws = standard_mock_websocket(sync_objects=(*STANDARD_POOL_OBJECTS, bad)) - with caplog.at_level(logging.ERROR, logger='ably.pubsub.objects'): + with caplog.at_level(logging.ERROR, logger='ably.pubsub.liveobjects'): client, channel, root, mock_ws = await asyncio.wait_for( setup_synced_channel('test', mock_ws=mock_ws), OPERATION_TIMEOUT) @@ -217,7 +217,7 @@ async def test_rto8_an_object_message_that_fails_to_decode_costs_only_itself(cap one.""" client, channel, root, mock_ws = await setup_synced_channel('test') - with caplog.at_level(logging.ERROR, logger='ably.pubsub.objects'): + with caplog.at_level(logging.ERROR, logger='ably.pubsub.liveobjects'): mock_ws.send_to_client(build_object_message('test', [ build_map_set('root', 'data', {'json': '{not json'}, remote_serial(0), 'remote'), build_counter_inc(SCORE, 1, remote_serial(1), 'remote'), @@ -233,7 +233,7 @@ async def test_rto9_an_object_message_that_fails_to_apply_costs_only_itself(capl string, is logged and skipped, and the operations after it in the OBJECT still apply.""" client, channel, root, mock_ws = await setup_synced_channel('test') - with caplog.at_level(logging.ERROR, logger='ably.pubsub.objects'): + with caplog.at_level(logging.ERROR, logger='ably.pubsub.liveobjects'): mock_ws.send_to_client(build_object_message('test', [ build_counter_inc(SCORE, 'one', remote_serial(0), 'remote'), build_map_set('root', 'name', {'string': 'Bob'}, remote_serial(1), 'remote'), @@ -251,7 +251,7 @@ async def test_rto5c_an_object_state_that_fails_to_apply_does_not_stall_the_sync bad = build_object_state('counter:bad@1000', {'aaa': 't:0'}, counter={'count': 'many'}) mock_ws = standard_mock_websocket(sync_objects=(*STANDARD_POOL_OBJECTS, bad)) - with caplog.at_level(logging.ERROR, logger='ably.pubsub.objects'): + with caplog.at_level(logging.ERROR, logger='ably.pubsub.liveobjects'): client, channel, root, mock_ws = await asyncio.wait_for( setup_synced_channel('test', mock_ws=mock_ws), OPERATION_TIMEOUT) @@ -316,7 +316,7 @@ def collect_garbage(grace_period_ms, now_ms): monkeypatch.setattr(channel.object._objects_pool, 'collect_garbage', collect_garbage) - with caplog.at_level(logging.ERROR, logger='ably.pubsub.objects'): + with caplog.at_level(logging.ERROR, logger='ably.pubsub.liveobjects'): await clock.advance(GC_INTERVAL_MS) assert len(sweeps) == 1 assert any('GC sweep raised' in record.getMessage() for record in caplog.records) diff --git a/test/unit/objects/subscription_test.py b/test/unit/liveobjects/subscription_test.py similarity index 100% rename from test/unit/objects/subscription_test.py rename to test/unit/liveobjects/subscription_test.py diff --git a/test/unit/objects/typed_views_test.py b/test/unit/liveobjects/typed_views_test.py similarity index 99% rename from test/unit/objects/typed_views_test.py rename to test/unit/liveobjects/typed_views_test.py index 9c46a282..a4cd7c79 100644 --- a/test/unit/objects/typed_views_test.py +++ b/test/unit/liveobjects/typed_views_test.py @@ -15,7 +15,7 @@ import pytest -from ably.pubsub.objects.liveobject import LiveObject, LiveObjectUpdate +from ably.pubsub.liveobjects.liveobject import LiveObject, LiveObjectUpdate from ably.pubsub.server import ( Instance, LiveCounterInstance, diff --git a/test/unit/objects/value_types_test.py b/test/unit/liveobjects/value_types_test.py similarity index 95% rename from test/unit/objects/value_types_test.py rename to test/unit/liveobjects/value_types_test.py index e4a0e368..2d20f893 100644 --- a/test/unit/objects/value_types_test.py +++ b/test/unit/liveobjects/value_types_test.py @@ -7,7 +7,7 @@ import pytest -from ably.pubsub.objects.valuetypes import LiveCounter, LiveMap, evaluate +from ably.pubsub.liveobjects.valuetypes import LiveCounter, LiveMap, evaluate from ably.pubsub.util.exceptions import AblyException from test.uts.objects.helpers.standard_test_pool import setup_synced_channel, time_mock_http diff --git a/test/uts/deviations.md b/test/uts/deviations.md index e08d19cb..d2ba2a93 100644 --- a/test/uts/deviations.md +++ b/test/uts/deviations.md @@ -3701,7 +3701,7 @@ one, goes straight to the sandbox with the key, as the specification builds it. No UTS specification covers batched writes (RTPO20, RTINS17, RTBC1–RTBC16, in the unmerged [#471](https://github.com/ably/specification/pull/471)) or the typed views (RTTS2–RTTS10, in -#491). Both are implemented, so both are tested, in `test/unit/objects/`: `batch_test.py`, 29 +#491). Both are implemented, so both are tested, in `test/unit/liveobjects/`: `batch_test.py`, 29 functions run as 76 cases, and `typed_views_test.py`, 17 functions run as 61. They are written against `objects-features.md`, the two pull requests and LODR-061, carry no `# UTS:` comment, and are not counted in the header. Beside them, `realtime_object_test.py`, diff --git a/test/uts/objects/helpers/standard_test_pool.py b/test/uts/objects/helpers/standard_test_pool.py index 1507b1ea..7286136a 100644 --- a/test/uts/objects/helpers/standard_test_pool.py +++ b/test/uts/objects/helpers/standard_test_pool.py @@ -26,8 +26,8 @@ from typing import NamedTuple from urllib.parse import quote -from ably.pubsub.objects import publicmessage -from ably.pubsub.objects.objectmessage import ( +from ably.pubsub.liveobjects import publicmessage +from ably.pubsub.liveobjects.objectmessage import ( WIRE_FORMAT_JSON, ObjectMessage, ObjectOperationAction, diff --git a/test/uts/objects/helpers/standard_test_pool_test.py b/test/uts/objects/helpers/standard_test_pool_test.py index 0f07d103..0b2bfbd5 100644 --- a/test/uts/objects/helpers/standard_test_pool_test.py +++ b/test/uts/objects/helpers/standard_test_pool_test.py @@ -11,8 +11,8 @@ import msgpack -from ably.pubsub.objects import publicmessage -from ably.pubsub.objects.objectmessage import ( +from ably.pubsub.liveobjects import publicmessage +from ably.pubsub.liveobjects.objectmessage import ( WIRE_FORMAT_MSGPACK, CounterCreate, MapCreate, @@ -23,7 +23,7 @@ ObjectOperationAction, ObjectsMapSemantics, ) -from ably.pubsub.objects.realtimeobject import RealtimeObject +from ably.pubsub.liveobjects.realtimeobject import RealtimeObject from ably.pubsub.realtime.connection import ConnectionState from ably.pubsub.types.channelmode import ChannelMode, decode_channel_mode from ably.pubsub.types.channelstate import ChannelState diff --git a/test/uts/objects/unit/instance_test.py b/test/uts/objects/unit/instance_test.py index ddcaf1f6..1cdefc80 100644 --- a/test/uts/objects/unit/instance_test.py +++ b/test/uts/objects/unit/instance_test.py @@ -26,11 +26,11 @@ import pytest -from ably.pubsub.objects import publicmessage -from ably.pubsub.objects.enums import ValueType -from ably.pubsub.objects.instance import Instance, InstanceSubscriptionEvent, PrimitiveInstance -from ably.pubsub.objects.objectmessage import ObjectOperationAction -from ably.pubsub.objects.subscription import Subscription +from ably.pubsub.liveobjects import publicmessage +from ably.pubsub.liveobjects.enums import ValueType +from ably.pubsub.liveobjects.instance import Instance, InstanceSubscriptionEvent, PrimitiveInstance +from ably.pubsub.liveobjects.objectmessage import ObjectOperationAction +from ably.pubsub.liveobjects.subscription import Subscription from ably.pubsub.types.channelstate import ChannelState from ably.pubsub.util.exceptions import AblyException from test.uts.helpers.client import poll_until diff --git a/test/uts/objects/unit/internal_live_counter_api_test.py b/test/uts/objects/unit/internal_live_counter_api_test.py index 00ebb54a..088c7202 100644 --- a/test/uts/objects/unit/internal_live_counter_api_test.py +++ b/test/uts/objects/unit/internal_live_counter_api_test.py @@ -16,7 +16,7 @@ import pytest -from ably.pubsub.objects.objectmessage import ObjectOperationAction +from ably.pubsub.liveobjects.objectmessage import ObjectOperationAction from ably.pubsub.util.exceptions import AblyException from test.uts.helpers.client import poll_until from test.uts.objects.helpers.standard_test_pool import ( diff --git a/test/uts/objects/unit/internal_live_counter_test.py b/test/uts/objects/unit/internal_live_counter_test.py index 2376671e..252df270 100644 --- a/test/uts/objects/unit/internal_live_counter_test.py +++ b/test/uts/objects/unit/internal_live_counter_test.py @@ -13,9 +13,9 @@ `replace_data` returns its update (RTLC6h), as the specification has it. """ -from ably.pubsub.objects.enums import ObjectsOperationSource -from ably.pubsub.objects.livecounter import InternalLiveCounter -from ably.pubsub.objects.objectmessage import ObjectOperationAction +from ably.pubsub.liveobjects.enums import ObjectsOperationSource +from ably.pubsub.liveobjects.livecounter import InternalLiveCounter +from ably.pubsub.liveobjects.objectmessage import ObjectOperationAction from test.uts.helpers.clock import FakeClock from test.uts.objects.helpers.standard_test_pool import ( build_counter_create, diff --git a/test/uts/objects/unit/internal_live_map_api_test.py b/test/uts/objects/unit/internal_live_map_api_test.py index 37c23d42..e5dc7f9a 100644 --- a/test/uts/objects/unit/internal_live_map_api_test.py +++ b/test/uts/objects/unit/internal_live_map_api_test.py @@ -24,8 +24,8 @@ import pytest -from ably.pubsub.objects.objectmessage import ObjectOperationAction -from ably.pubsub.objects.valuetypes import LiveCounter, LiveMap +from ably.pubsub.liveobjects.objectmessage import ObjectOperationAction +from ably.pubsub.liveobjects.valuetypes import LiveCounter, LiveMap from ably.pubsub.util.exceptions import AblyException from test.uts.objects.helpers.standard_test_pool import ( LWW, diff --git a/test/uts/objects/unit/internal_live_map_test.py b/test/uts/objects/unit/internal_live_map_test.py index 4bb1e873..a38d1b90 100644 --- a/test/uts/objects/unit/internal_live_map_test.py +++ b/test/uts/objects/unit/internal_live_map_test.py @@ -18,11 +18,11 @@ with the id `root` alongside it. """ -from ably.pubsub.objects.enums import ObjectsOperationSource -from ably.pubsub.objects.livecounter import InternalLiveCounter -from ably.pubsub.objects.livemap import InternalLiveMap -from ably.pubsub.objects.objectmessage import ObjectData, ObjectsMapEntry, ObjectsMapSemantics -from ably.pubsub.objects.objectspool import ObjectsPool +from ably.pubsub.liveobjects.enums import ObjectsOperationSource +from ably.pubsub.liveobjects.livecounter import InternalLiveCounter +from ably.pubsub.liveobjects.livemap import InternalLiveMap +from ably.pubsub.liveobjects.objectmessage import ObjectData, ObjectsMapEntry, ObjectsMapSemantics +from ably.pubsub.liveobjects.objectspool import ObjectsPool from test.uts.objects.helpers.standard_test_pool import ( LWW, build_counter_inc, diff --git a/test/uts/objects/unit/live_object_subscribe_test.py b/test/uts/objects/unit/live_object_subscribe_test.py index d240e8c8..437bd2b0 100644 --- a/test/uts/objects/unit/live_object_subscribe_test.py +++ b/test/uts/objects/unit/live_object_subscribe_test.py @@ -15,9 +15,9 @@ specification's negative-assertion quiescence pattern). """ -from ably.pubsub.objects import publicmessage -from ably.pubsub.objects.objectmessage import ObjectOperationAction -from ably.pubsub.objects.subscription import Subscription +from ably.pubsub.liveobjects import publicmessage +from ably.pubsub.liveobjects.objectmessage import ObjectOperationAction +from ably.pubsub.liveobjects.subscription import Subscription from test.uts.helpers.client import poll_until from test.uts.helpers.clock import settle from test.uts.objects.helpers.standard_test_pool import ( diff --git a/test/uts/objects/unit/object_id_test.py b/test/uts/objects/unit/object_id_test.py index d6f43184..fdbedbeb 100644 --- a/test/uts/objects/unit/object_id_test.py +++ b/test/uts/objects/unit/object_id_test.py @@ -11,7 +11,7 @@ import hashlib import re -from ably.pubsub.objects.objectid import generate_object_id +from ably.pubsub.liveobjects.objectid import generate_object_id # RFC 4648 s.5's URL-safe alphabet, with no padding BASE64URL_UNPADDED = re.compile(r'[A-Za-z0-9_-]+') diff --git a/test/uts/objects/unit/objects_pool_test.py b/test/uts/objects/unit/objects_pool_test.py index 04843e5e..ad908356 100644 --- a/test/uts/objects/unit/objects_pool_test.py +++ b/test/uts/objects/unit/objects_pool_test.py @@ -27,12 +27,12 @@ touch the event loop. """ -from ably.pubsub.objects.enums import ObjectsOperationSource, ObjectsSyncState -from ably.pubsub.objects.livecounter import InternalLiveCounter -from ably.pubsub.objects.livemap import InternalLiveMap -from ably.pubsub.objects.objectmessage import ObjectData, ObjectOperationAction, ObjectsMapEntry -from ably.pubsub.objects.objectspool import ObjectsPool -from ably.pubsub.objects.realtimeobject import RealtimeObject +from ably.pubsub.liveobjects.enums import ObjectsOperationSource, ObjectsSyncState +from ably.pubsub.liveobjects.livecounter import InternalLiveCounter +from ably.pubsub.liveobjects.livemap import InternalLiveMap +from ably.pubsub.liveobjects.objectmessage import ObjectData, ObjectOperationAction, ObjectsMapEntry +from ably.pubsub.liveobjects.objectspool import ObjectsPool +from ably.pubsub.liveobjects.realtimeobject import RealtimeObject from test.uts.objects.helpers.standard_test_pool import ( ATTACHED, HAS_OBJECTS, diff --git a/test/uts/objects/unit/parent_references_test.py b/test/uts/objects/unit/parent_references_test.py index 20697c74..f7077ee3 100644 --- a/test/uts/objects/unit/parent_references_test.py +++ b/test/uts/objects/unit/parent_references_test.py @@ -15,12 +15,12 @@ coroutines only so that an implementation is free to touch the event loop. """ -from ably.pubsub.objects.enums import ObjectsSyncState -from ably.pubsub.objects.livecounter import InternalLiveCounter -from ably.pubsub.objects.livemap import InternalLiveMap -from ably.pubsub.objects.objectmessage import ObjectsMapSemantics -from ably.pubsub.objects.objectspool import ObjectsPool -from ably.pubsub.objects.realtimeobject import RealtimeObject +from ably.pubsub.liveobjects.enums import ObjectsSyncState +from ably.pubsub.liveobjects.livecounter import InternalLiveCounter +from ably.pubsub.liveobjects.livemap import InternalLiveMap +from ably.pubsub.liveobjects.objectmessage import ObjectsMapSemantics +from ably.pubsub.liveobjects.objectspool import ObjectsPool +from ably.pubsub.liveobjects.realtimeobject import RealtimeObject from test.uts.objects.helpers.standard_test_pool import ( HAS_OBJECTS, LWW, diff --git a/test/uts/objects/unit/path_object_subscribe_test.py b/test/uts/objects/unit/path_object_subscribe_test.py index 06e3a62c..bdcb5603 100644 --- a/test/uts/objects/unit/path_object_subscribe_test.py +++ b/test/uts/objects/unit/path_object_subscribe_test.py @@ -22,9 +22,9 @@ import pytest -from ably.pubsub.objects.objectmessage import ObjectOperationAction -from ably.pubsub.objects.pathobject import PathObject -from ably.pubsub.objects.subscription import Subscription +from ably.pubsub.liveobjects.objectmessage import ObjectOperationAction +from ably.pubsub.liveobjects.pathobject import PathObject +from ably.pubsub.liveobjects.subscription import Subscription from ably.pubsub.types.channelstate import ChannelState from ably.pubsub.util.exceptions import AblyException from test.uts.helpers.client import await_channel_state, poll_until diff --git a/test/uts/objects/unit/path_object_test.py b/test/uts/objects/unit/path_object_test.py index f88da010..4d40bc2a 100644 --- a/test/uts/objects/unit/path_object_test.py +++ b/test/uts/objects/unit/path_object_test.py @@ -23,7 +23,7 @@ import pytest -from ably.pubsub.objects.instance import Instance +from ably.pubsub.liveobjects.instance import Instance from ably.pubsub.util.exceptions import AblyException from test.uts.helpers.client import poll_until from test.uts.objects.helpers.standard_test_pool import ( diff --git a/test/uts/objects/unit/public_object_message_test.py b/test/uts/objects/unit/public_object_message_test.py index 1e412c77..f7930bbc 100644 --- a/test/uts/objects/unit/public_object_message_test.py +++ b/test/uts/objects/unit/public_object_message_test.py @@ -23,8 +23,8 @@ import json -from ably.pubsub.objects import publicmessage -from ably.pubsub.objects.objectmessage import ( +from ably.pubsub.liveobjects import publicmessage +from ably.pubsub.liveobjects.objectmessage import ( CounterCreate, CounterCreateWithObjectId, CounterInc, diff --git a/test/uts/objects/unit/realtime_object_test.py b/test/uts/objects/unit/realtime_object_test.py index 33252655..b230be97 100644 --- a/test/uts/objects/unit/realtime_object_test.py +++ b/test/uts/objects/unit/realtime_object_test.py @@ -37,10 +37,10 @@ import pytest -from ably.pubsub.objects.enums import ObjectsEvent, ObjectsSyncState -from ably.pubsub.objects.objectmessage import ObjectOperationAction -from ably.pubsub.objects.pathobject import PathObject -from ably.pubsub.objects.valuetypes import LiveCounter +from ably.pubsub.liveobjects.enums import ObjectsEvent, ObjectsSyncState +from ably.pubsub.liveobjects.objectmessage import ObjectOperationAction +from ably.pubsub.liveobjects.pathobject import PathObject +from ably.pubsub.liveobjects.valuetypes import LiveCounter from ably.pubsub.transport.websockettransport import ProtocolMessageAction from ably.pubsub.types.channelmode import ChannelMode from ably.pubsub.types.channelstate import ChannelState diff --git a/test/uts/objects/unit/value_types_test.py b/test/uts/objects/unit/value_types_test.py index a424b108..0538c661 100644 --- a/test/uts/objects/unit/value_types_test.py +++ b/test/uts/objects/unit/value_types_test.py @@ -24,8 +24,8 @@ import pytest -from ably.pubsub.objects.objectmessage import ObjectOperationAction, ObjectsMapSemantics -from ably.pubsub.objects.valuetypes import LiveCounter, LiveMap, evaluate +from ably.pubsub.liveobjects.objectmessage import ObjectOperationAction, ObjectsMapSemantics +from ably.pubsub.liveobjects.valuetypes import LiveCounter, LiveMap, evaluate from ably.pubsub.util.exceptions import AblyException # The server time evaluation generates object ids with (S-3) From 98dedb3d0134551eefbc553504babb0692bf9ea8 Mon Sep 17 00:00:00 2001 From: owenpearson Date: Mon, 19 Oct 2026 10:51:03 +0100 Subject: [PATCH 24/71] liveobjects: export the public API from ably.pubsub.server.liveobjects Moves the thirty LiveObjects names out of ably.pubsub.server into a module of their own, alongside ably.pubsub.server.sync. Generic names such as Batch, Subscription, Instance and Primitive no longer share a namespace with the Pub/Sub ones, where Batch read as part of batch publish next to BatchResult and BatchPublishSpec, and the product name shows at the import, as PDR-080 asks: from ably.pubsub.server.liveobjects import LiveCounter, LiveMap Co-Authored-By: Claude Opus 5.5 (1M context) --- .claude/skills/uts-to-python/SKILL.md | 2 +- README.md | 3 +- ably/pubsub/liveobjects/__init__.py | 8 +- ably/pubsub/server/__init__.py | 67 +--------------- ably/pubsub/server/liveobjects.py | 76 +++++++++++++++++++ test/unit/liveobjects/batch_test.py | 2 +- test/unit/liveobjects/typed_views_test.py | 2 +- test/unit/pubsub_server_test.py | 13 ++++ .../objects/integration/objects_gc_test.py | 2 +- .../integration/objects_lifecycle_test.py | 2 +- .../objects/integration/objects_sync_test.py | 3 +- .../integration/proxy/objects_faults_test.py | 3 +- 12 files changed, 106 insertions(+), 77 deletions(-) create mode 100644 ably/pubsub/server/liveobjects.py diff --git a/.claude/skills/uts-to-python/SKILL.md b/.claude/skills/uts-to-python/SKILL.md index 32b100c7..1204ab35 100644 --- a/.claude/skills/uts-to-python/SKILL.md +++ b/.claude/skills/uts-to-python/SKILL.md @@ -468,7 +468,7 @@ provisions a separate app under the realtime tier's fixture name, `realtime_sand all three of its specifications take `use_binary_protocol`. The package is `ably.pubsub.liveobjects`, named for the product as PDR-080 has package names do, while `test/uts/objects/` keeps the specification's directory name so that each test file maps -onto its specification; the public names are exported from `ably.pubsub.server`. +onto its specification; the public names are exported from `ably.pubsub.server.liveobjects`. **Translate the untyped API through the typed views** (LODR-061, the RTTS partition). The specifications call every method on one `PathObject` or `Instance` class; ably-python diff --git a/README.md b/README.md index 3d30e0a7..7b81426f 100644 --- a/README.md +++ b/README.md @@ -111,7 +111,8 @@ Values are read through a typed view of their path — `as_live_map()`, `as_live `as_primitive()` — and reads are synchronous; writes are awaited. ```python -from ably.pubsub.server import ChannelMode, ChannelOptions, LiveCounter, LiveMap, create_realtime_client +from ably.pubsub.server import ChannelMode, ChannelOptions, create_realtime_client +from ably.pubsub.server.liveobjects import LiveCounter, LiveMap async with create_realtime_client(key='your-ably-api-key') as realtime_client: channel = realtime_client.channels.get( diff --git a/ably/pubsub/liveobjects/__init__.py b/ably/pubsub/liveobjects/__init__.py index 621bd62b..78b55146 100644 --- a/ably/pubsub/liveobjects/__init__.py +++ b/ably/pubsub/liveobjects/__init__.py @@ -1,7 +1,7 @@ """LiveObjects: shared, conflict-free data structures stored on a realtime channel. -The public names are re-exported from :mod:`ably.pubsub.server`; the modules here -hold the implementation behind them, and the internal classes the specification -names (``ObjectsPool``, ``InternalLiveMap``, ``InternalLiveCounter`` and the wire -types). +The public names are re-exported from :mod:`ably.pubsub.server.liveobjects`; the +modules here hold the implementation behind them, and the internal classes the +specification names (``ObjectsPool``, ``InternalLiveMap``, ``InternalLiveCounter`` +and the wire types). """ diff --git a/ably/pubsub/server/__init__.py b/ably/pubsub/server/__init__.py index 1323fe36..15c20641 100644 --- a/ably/pubsub/server/__init__.py +++ b/ably/pubsub/server/__init__.py @@ -15,47 +15,14 @@ ``ably`` and ``ably.pubsub`` are PEP 420 namespace packages shared with the other ``ably-*`` distributions, so neither carries an ``__init__.py`` of its own and neither exports anything; everything the library offers is reachable -from here. +from here, from :mod:`ably.pubsub.server.sync` for the synchronous client, or +from :mod:`ably.pubsub.server.liveobjects` for LiveObjects. """ from ably.pubsub.http.auth import Auth from ably.pubsub.http.channel import Channels as HttpChannels from ably.pubsub.http.http import DefaultPubSubHttpClient as _DefaultPubSubHttpClient from ably.pubsub.http.push import Push -from ably.pubsub.liveobjects.batch import ( - Batch, - BatchContext, - LiveCounterBatchContext, - LiveMapBatchContext, - PrimitiveBatchContext, -) -from ably.pubsub.liveobjects.enums import ObjectsEvent, ValueType -from ably.pubsub.liveobjects.instance import ( - Instance, - InstanceSubscriptionEvent, - LiveCounterInstance, - LiveMapInstance, - PrimitiveInstance, -) -from ably.pubsub.liveobjects.objectmessage import ObjectOperationAction, ObjectsMapSemantics -from ably.pubsub.liveobjects.pathobject import ( - LiveCounterPathObject, - LiveMapPathObject, - PathObject, - PathObjectSubscriptionEvent, - PrimitivePathObject, -) -from ably.pubsub.liveobjects.publicmessage import ObjectMessage, ObjectOperation -from ably.pubsub.liveobjects.realtimeobject import RealtimeObject -from ably.pubsub.liveobjects.subscription import StatusSubscription, Subscription -from ably.pubsub.liveobjects.valuetypes import ( - JsonObject, - JsonValue, - LiveCounter, - LiveMap, - LiveMapValue, - Primitive, -) from ably.pubsub.prototypes import PubSubHttpClient, PubSubRealtimeClient from ably.pubsub.realtime.channel import Channels as RealtimeChannels from ably.pubsub.realtime.connection import Connection @@ -293,8 +260,6 @@ def create_realtime_client(**kwargs) -> PubSubRealtimeClient: 'Annotation', 'AnnotationAction', 'Auth', - 'Batch', - 'BatchContext', 'BatchPresenceFailureResult', 'BatchPresenceSuccessResult', 'BatchPublishFailureResult', @@ -310,49 +275,21 @@ def create_realtime_client(**kwargs) -> PubSubRealtimeClient: 'HttpChannels', 'HttpPaginatedResponse', 'IncompatibleClientIdException', - 'Instance', - 'InstanceSubscriptionEvent', - 'JsonObject', - 'JsonValue', - 'LiveCounter', - 'LiveCounterBatchContext', - 'LiveCounterInstance', - 'LiveCounterPathObject', - 'LiveMap', - 'LiveMapBatchContext', - 'LiveMapInstance', - 'LiveMapPathObject', - 'LiveMapValue', 'Message', 'MessageAction', 'MessageOperation', 'MessageVersion', - 'ObjectMessage', - 'ObjectOperation', - 'ObjectOperationAction', - 'ObjectsEvent', - 'ObjectsMapSemantics', 'Options', 'PaginatedResult', - 'PathObject', - 'PathObjectSubscriptionEvent', - 'Primitive', - 'PrimitiveBatchContext', - 'PrimitiveInstance', - 'PrimitivePathObject', 'PubSubHttpClient', 'PubSubRealtimeClient', 'PublishResult', 'Push', 'PushChannelSubscription', 'RealtimeChannels', - 'RealtimeObject', - 'StatusSubscription', - 'Subscription', 'TokenDetails', 'UpdateDeleteResult', 'VCDiffDecoder', - 'ValueType', 'api_version', 'create_http_client', 'create_realtime_client', diff --git a/ably/pubsub/server/liveobjects.py b/ably/pubsub/server/liveobjects.py new file mode 100644 index 00000000..de0df2b5 --- /dev/null +++ b/ably/pubsub/server/liveobjects.py @@ -0,0 +1,76 @@ +"""LiveObjects for the Ably Pub/Sub client for servers. + +Shared, conflict-free maps and counters stored on a realtime channel. A channel +reaches them through ``channel.object``, a :class:`RealtimeObject`, once the +client from :func:`ably.pubsub.server.create_realtime_client` has attached it +with the object modes. The names here are the types that API takes and returns, +for building values to write and for annotating what it gives back. +""" + +from ably.pubsub.liveobjects.batch import ( + Batch, + BatchContext, + LiveCounterBatchContext, + LiveMapBatchContext, + PrimitiveBatchContext, +) +from ably.pubsub.liveobjects.enums import ObjectsEvent, ValueType +from ably.pubsub.liveobjects.instance import ( + Instance, + InstanceSubscriptionEvent, + LiveCounterInstance, + LiveMapInstance, + PrimitiveInstance, +) +from ably.pubsub.liveobjects.objectmessage import ObjectOperationAction, ObjectsMapSemantics +from ably.pubsub.liveobjects.pathobject import ( + LiveCounterPathObject, + LiveMapPathObject, + PathObject, + PathObjectSubscriptionEvent, + PrimitivePathObject, +) +from ably.pubsub.liveobjects.publicmessage import ObjectMessage, ObjectOperation +from ably.pubsub.liveobjects.realtimeobject import RealtimeObject +from ably.pubsub.liveobjects.subscription import StatusSubscription, Subscription +from ably.pubsub.liveobjects.valuetypes import ( + JsonObject, + JsonValue, + LiveCounter, + LiveMap, + LiveMapValue, + Primitive, +) + +__all__ = [ + 'Batch', + 'BatchContext', + 'Instance', + 'InstanceSubscriptionEvent', + 'JsonObject', + 'JsonValue', + 'LiveCounter', + 'LiveCounterBatchContext', + 'LiveCounterInstance', + 'LiveCounterPathObject', + 'LiveMap', + 'LiveMapBatchContext', + 'LiveMapInstance', + 'LiveMapPathObject', + 'LiveMapValue', + 'ObjectMessage', + 'ObjectOperation', + 'ObjectOperationAction', + 'ObjectsEvent', + 'ObjectsMapSemantics', + 'PathObject', + 'PathObjectSubscriptionEvent', + 'Primitive', + 'PrimitiveBatchContext', + 'PrimitiveInstance', + 'PrimitivePathObject', + 'RealtimeObject', + 'StatusSubscription', + 'Subscription', + 'ValueType', +] diff --git a/test/unit/liveobjects/batch_test.py b/test/unit/liveobjects/batch_test.py index 1b4923bd..fa9b1e0e 100644 --- a/test/unit/liveobjects/batch_test.py +++ b/test/unit/liveobjects/batch_test.py @@ -14,7 +14,7 @@ import pytest from ably.pubsub.liveobjects.objectmessage import ObjectOperationAction -from ably.pubsub.server import ( +from ably.pubsub.server.liveobjects import ( LiveCounter, LiveCounterBatchContext, LiveMap, diff --git a/test/unit/liveobjects/typed_views_test.py b/test/unit/liveobjects/typed_views_test.py index a4cd7c79..6f3ccb49 100644 --- a/test/unit/liveobjects/typed_views_test.py +++ b/test/unit/liveobjects/typed_views_test.py @@ -16,7 +16,7 @@ import pytest from ably.pubsub.liveobjects.liveobject import LiveObject, LiveObjectUpdate -from ably.pubsub.server import ( +from ably.pubsub.server.liveobjects import ( Instance, LiveCounterInstance, LiveCounterPathObject, diff --git a/test/unit/pubsub_server_test.py b/test/unit/pubsub_server_test.py index a806f56e..0da6f813 100644 --- a/test/unit/pubsub_server_test.py +++ b/test/unit/pubsub_server_test.py @@ -4,6 +4,7 @@ import pytest import ably.pubsub.server as server +import ably.pubsub.server.liveobjects as server_liveobjects import ably.pubsub.server.sync as server_sync from ably.pubsub.http.http import DefaultPubSubHttpClient from ably.pubsub.prototypes import PubSubHttpClient, PubSubRealtimeClient @@ -167,3 +168,15 @@ def test_message_is_exported(self): def test_crypto_helpers_are_exported(self, name): assert name in server.__all__ assert name in server_sync.__all__ + + # LiveObjects names live in their own module, named for the product (PDR-080), so + # that generic ones such as Batch and Subscription stay apart from the Pub/Sub names + @pytest.mark.parametrize('name', ['Batch', 'LiveCounter', 'LiveMap', 'PathObject', 'Subscription']) + def test_liveobjects_names_are_exported_from_their_own_module(self, name): + assert name in server_liveobjects.__all__ + assert name not in server.__all__ + assert not hasattr(server, name) + + def test_liveobjects_names_are_all_importable(self): + for name in server_liveobjects.__all__: + assert getattr(server_liveobjects, name) is not None diff --git a/test/uts/objects/integration/objects_gc_test.py b/test/uts/objects/integration/objects_gc_test.py index 416cb31c..f6182c1b 100644 --- a/test/uts/objects/integration/objects_gc_test.py +++ b/test/uts/objects/integration/objects_gc_test.py @@ -26,7 +26,7 @@ import asyncio from ably.pubsub.realtime.connection import ConnectionState -from ably.pubsub.server import LiveCounter +from ably.pubsub.server.liveobjects import LiveCounter from test.uts.helpers.client import await_connection_state, sandbox_realtime_client, wall_clock_poll_until from test.uts.helpers.sandbox import random_id from test.uts.objects.helpers.standard_test_pool import objects_channel_options diff --git a/test/uts/objects/integration/objects_lifecycle_test.py b/test/uts/objects/integration/objects_lifecycle_test.py index e43edf2a..98b87965 100644 --- a/test/uts/objects/integration/objects_lifecycle_test.py +++ b/test/uts/objects/integration/objects_lifecycle_test.py @@ -25,7 +25,7 @@ import asyncio from ably.pubsub.realtime.connection import ConnectionState -from ably.pubsub.server import LiveCounter, LiveMap, PathObject +from ably.pubsub.server.liveobjects import LiveCounter, LiveMap, PathObject from test.uts.helpers.client import await_connection_state, sandbox_realtime_client, wall_clock_poll_until from test.uts.helpers.sandbox import random_id from test.uts.objects.helpers.standard_test_pool import objects_channel_options, provision_objects_via_rest diff --git a/test/uts/objects/integration/objects_sync_test.py b/test/uts/objects/integration/objects_sync_test.py index 338c395a..23ff6872 100644 --- a/test/uts/objects/integration/objects_sync_test.py +++ b/test/uts/objects/integration/objects_sync_test.py @@ -22,7 +22,8 @@ import asyncio from ably.pubsub.realtime.connection import ConnectionState -from ably.pubsub.server import ChannelMode, PathObject +from ably.pubsub.server import ChannelMode +from ably.pubsub.server.liveobjects import PathObject from test.uts.helpers.client import await_connection_state, sandbox_realtime_client, wall_clock_poll_until from test.uts.helpers.sandbox import random_id from test.uts.objects.helpers.standard_test_pool import objects_channel_options diff --git a/test/uts/objects/integration/proxy/objects_faults_test.py b/test/uts/objects/integration/proxy/objects_faults_test.py index 75afbda9..530cd5dd 100644 --- a/test/uts/objects/integration/proxy/objects_faults_test.py +++ b/test/uts/objects/integration/proxy/objects_faults_test.py @@ -49,7 +49,8 @@ import pytest from ably.pubsub.realtime.connection import ConnectionState -from ably.pubsub.server import AblyException, PathObject +from ably.pubsub.server import AblyException +from ably.pubsub.server.liveobjects import PathObject from ably.pubsub.types.channelstate import ChannelState from test.uts.helpers.client import ( await_channel_state, From 4d416613906893df21e72450d7e5c1d5a363fb0a Mon Sep 17 00:00:00 2001 From: owenpearson Date: Mon, 19 Oct 2026 10:52:25 +0100 Subject: [PATCH 25/71] liveobjects: fail the decode of a number that is not a number _decode_number passed any value that was not an int or a float through unchanged, so a counter increment or count sent as a string decoded and only failed later, in the counter's arithmetic. It now raises ValueError for a present value of any other type, so from_protocol_message skips the ObjectMessage carrying it at decode time. The RTO9 and RTO5c tests used a string number as their operation that decodes but fails to apply; they now make the apply raise instead, and a new RTO8 test covers the string increment as a decode failure. Co-Authored-By: Claude Opus 5.5 (1M context) --- ably/pubsub/liveobjects/objectmessage.py | 11 +++-- test/unit/liveobjects/realtime_object_test.py | 44 ++++++++++++++++--- 2 files changed, 44 insertions(+), 11 deletions(-) diff --git a/ably/pubsub/liveobjects/objectmessage.py b/ably/pubsub/liveobjects/objectmessage.py index 5707a23c..cde5368a 100644 --- a/ably/pubsub/liveobjects/objectmessage.py +++ b/ably/pubsub/liveobjects/objectmessage.py @@ -92,11 +92,14 @@ def _check_format(format: str) -> None: raise ValueError(f"Unknown wire format {format!r}; expected 'json' or 'msgpack'") -def _decode_number(value: Any) -> Any: +def _decode_number(value: Any) -> float | None: # OD4c3: numbers are float64 on the wire, so a decoded number is always a float - if isinstance(value, (int, float)) and not isinstance(value, bool): - return float(value) - return value + if value is None: + return None + # A value of any other type fails the decode, so from_protocol_message skips its message + if isinstance(value, bool) or not isinstance(value, (int, float)): + raise ValueError(f'Expected a number, got {value!r}') + return float(value) def _encode_number(value: Any, format: str) -> Any: diff --git a/test/unit/liveobjects/realtime_object_test.py b/test/unit/liveobjects/realtime_object_test.py index 6cf2fee7..63e05163 100644 --- a/test/unit/liveobjects/realtime_object_test.py +++ b/test/unit/liveobjects/realtime_object_test.py @@ -18,6 +18,7 @@ from ably.pubsub.liveobjects.defaults import GC_INTERVAL_MS from ably.pubsub.liveobjects.enums import ObjectsSyncState +from ably.pubsub.liveobjects.livecounter import InternalLiveCounter from ably.pubsub.liveobjects.realtimeobject import RealtimeObject from ably.pubsub.realtime.connection import ConnectionState from ably.pubsub.types.channelstate import ChannelState @@ -228,9 +229,9 @@ async def test_rto8_an_object_message_that_fails_to_decode_costs_only_itself(cap assert any('failed to decode' in record.getMessage() for record in caplog.records) -async def test_rto9_an_object_message_that_fails_to_apply_costs_only_itself(caplog): - """RTO9: an operation that decodes but raises as it is applied, here an increment by a - string, is logged and skipped, and the operations after it in the OBJECT still apply.""" +async def test_rto8_an_increment_by_a_non_number_fails_to_decode_and_costs_only_itself(caplog): + """RTO8, OD4c3: an increment by a string fails to decode rather than reaching the counter, + and the operations after it in the OBJECT still apply.""" client, channel, root, mock_ws = await setup_synced_channel('test') with caplog.at_level(logging.ERROR, logger='ably.pubsub.liveobjects'): @@ -238,6 +239,27 @@ async def test_rto9_an_object_message_that_fails_to_apply_costs_only_itself(capl build_counter_inc(SCORE, 'one', remote_serial(0), 'remote'), build_map_set('root', 'name', {'string': 'Bob'}, remote_serial(1), 'remote'), ])) + await poll_until(lambda: root.get('name').as_primitive().value() == 'Bob', + description='the MAP_SET after the undecodable increment to apply') + + assert _score(channel) == 100 + assert any('failed to decode' in record.getMessage() for record in caplog.records) + + +async def test_rto9_an_object_message_that_fails_to_apply_costs_only_itself(caplog, monkeypatch): + """RTO9: an operation that decodes but raises as it is applied is logged and skipped, and + the operations after it in the OBJECT still apply.""" + client, channel, root, mock_ws = await setup_synced_channel('test') + + def apply_counter_inc(self, *args, **kwargs): + raise RuntimeError('apply_counter_inc failed') + monkeypatch.setattr(InternalLiveCounter, 'apply_counter_inc', apply_counter_inc) + + with caplog.at_level(logging.ERROR, logger='ably.pubsub.liveobjects'): + mock_ws.send_to_client(build_object_message('test', [ + build_counter_inc(SCORE, 1, remote_serial(0), 'remote'), + build_map_set('root', 'name', {'string': 'Bob'}, remote_serial(1), 'remote'), + ])) await poll_until(lambda: root.get('name').as_primitive().value() == 'Bob', description='the MAP_SET after the failing increment to apply') @@ -245,12 +267,20 @@ async def test_rto9_an_object_message_that_fails_to_apply_costs_only_itself(capl assert any('could not be applied' in record.getMessage() for record in caplog.records) -async def test_rto5c_an_object_state_that_fails_to_apply_does_not_stall_the_sync(caplog): - """RTO5c: an object state that decodes but raises as it is applied, here a counter whose - count is a string, is logged and skipped, and the sync still completes.""" - bad = build_object_state('counter:bad@1000', {'aaa': 't:0'}, counter={'count': 'many'}) +async def test_rto5c_an_object_state_that_fails_to_apply_does_not_stall_the_sync(caplog, monkeypatch): + """RTO5c: an object state that decodes but raises as it is applied is logged and skipped, + and the sync still completes.""" + bad = build_object_state('counter:bad@1000', {'aaa': 't:0'}, counter={'count': 1}) mock_ws = standard_mock_websocket(sync_objects=(*STANDARD_POOL_OBJECTS, bad)) + replace_data = InternalLiveCounter.replace_data + + def failing_replace_data(self, object_message): + if self.object_id == 'counter:bad@1000': + raise RuntimeError('replace_data failed') + return replace_data(self, object_message) + monkeypatch.setattr(InternalLiveCounter, 'replace_data', failing_replace_data) + with caplog.at_level(logging.ERROR, logger='ably.pubsub.liveobjects'): client, channel, root, mock_ws = await asyncio.wait_for( setup_synced_channel('test', mock_ws=mock_ws), OPERATION_TIMEOUT) From 469d22f7c6a5da48a4673190af58ecafebc24bd2 Mon Sep 17 00:00:00 2001 From: owenpearson Date: Mon, 19 Oct 2026 10:52:25 +0100 Subject: [PATCH 26/71] docs: count three integration tiers and clarify the RTO8a buffer tests Co-Authored-By: Claude Opus 5.5 (1M context) --- .claude/skills/uts-to-python/SKILL.md | 2 +- test/uts/deviations.md | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.claude/skills/uts-to-python/SKILL.md b/.claude/skills/uts-to-python/SKILL.md index 1204ab35..48f18fc9 100644 --- a/.claude/skills/uts-to-python/SKILL.md +++ b/.claude/skills/uts-to-python/SKILL.md @@ -1009,7 +1009,7 @@ environment's cutoff — and `--extra dev` carries pytest. Line length is 115. I `uv.lock` changes, revert only what the run touched; a wholesale `git checkout -- uv.lock` takes any edit of your own with it. -The second command is the offline tiers, which need no network. The third is the two +The second command is the offline tiers, which need no network. The third is the three integration tiers, each of which provisions a sandbox app of its own and does need the network; `pytest test/uts -q` runs everything together, so run the tiers separately when only one is in question. All three must pass. diff --git a/test/uts/deviations.md b/test/uts/deviations.md index d2ba2a93..3d5c5734 100644 --- a/test/uts/deviations.md +++ b/test/uts/deviations.md @@ -3234,8 +3234,8 @@ RTO20c, RTO20d1, RTO20e1; and, derived as written, RTO4 with RTO5a2, RTO7 and RT proxy, RTLC11b1, RTLO4b, RTO5c10 and the lifecycle test's unread `events_b`. Sixteen derived tests carry the assertion or the direct check their specification omits. The sharpest are the GC tests, which read a value that is null with no GC at all — RTO10b1 also advances less than one -GC interval — and the two `objects_faults.md` tests titled for RTO8a's buffer, which -measurement shows never buffer. Same class as +GC interval — and the two `objects_faults.md` tests titled for RTO8a's buffer, neither of which, by +measurement, exercises the buffer. Same class as [#543](https://github.com/ably/specification/issues/543). `test/uts/objects/unit -k "rtlm8g or rtlm9b or rto10 or rtpo6_at_escaped or rtpo19_map_clear or rto4b_attached or rto14_objectid_format_counter or rtins16g or rto20e or rto23c_get_waits or rto20c_missing or rto20d1"` From 6702f63d2d4c4f0efed7ab55ff8224fe87829602 Mon Sep 17 00:00:00 2001 From: Simon Woolf Date: Fri, 9 Oct 2026 04:32:29 +0100 Subject: [PATCH 27/71] pubsub: raise 90011 when releasing a realtime channel that isn't detached Specification 6.3.0 replaces RTS4a with RTS4c-e. Channels.release used to remove a channel whatever its state, so an attached channel was dropped from the collection while still attached in the Ably service. It now raises an AblyException with code 90011 and status 400 for a channel that is not INITIALIZED, DETACHED or FAILED, and leaves the channel as it was (RTS4e); it still returns quietly for an unknown name (RTS4c) and removes a channel in one of those states (RTS4d). This is a breaking change: callers must await channel.detach() before releasing. 3.x logs a deprecation warning in this case (RTS4b). The channels collection UTS tests are rederived for RTS4c-e, which closes the RTS4a release deviation. Co-Authored-By: Claude Opus 5.5 (1M context) --- UPDATING.md | 19 ++++ ably/pubsub/realtime/channel.py | 23 ++++- test/unit/liveobjects/realtime_object_test.py | 25 ++---- test/uts/deviations.md | 76 ++++++---------- .../unit/channels/channels_collection_test.py | 86 +++++++++++-------- 5 files changed, 127 insertions(+), 102 deletions(-) diff --git a/UPDATING.md b/UPDATING.md index a63e8ea1..64f69d56 100644 --- a/UPDATING.md +++ b/UPDATING.md @@ -105,6 +105,25 @@ async def subscribe(client: PubSubRealtimeClient, name: str) -> None: ... def publish_blocking(client: PubSubHttpClientSync, name: str) -> None: ... ``` +### Releasing a realtime channel requires it to be detached + +`channels.release(name)` on a realtime client now raises an `AblyException` with code 90011 +if the channel is in any state other than `INITIALIZED`, `DETACHED` or `FAILED`, and leaves the +channel as it was. 3.x removed the channel from the collection whatever its state, which could +leave it attached in the Ably service, and later 3.x releases log a deprecation warning when +they do so. Detach the channel, and wait for that to complete, before releasing it: + +Example 3.x code: +```python +realtime.channels.release('my-channel') +``` + +Example 4.0.0 code: +```python +await pubsub_realtime_client.channels.get('my-channel').detach() +pubsub_realtime_client.channels.release('my-channel') +``` + ## Version 2.x to 3.0.0 The 3.0.0 version of ably-python introduces several breaking changes to improve the realtime experience and align the API with the Ably specification. These include: diff --git a/ably/pubsub/realtime/channel.py b/ably/pubsub/realtime/channel.py index d9b74921..e484f656 100644 --- a/ably/pubsub/realtime/channel.py +++ b/ably/pubsub/realtime/channel.py @@ -1085,17 +1085,34 @@ def release(self, name: str) -> None: """Releases a RealtimeChannel object, deleting it, and enabling it to be garbage collected It also removes any listeners associated with the channel. - To release a channel, the channel state must be INITIALIZED, DETACHED, or FAILED. - + A realtime channel can only be released when it is in the INITIALIZED, DETACHED, or FAILED state. Parameters ---------- name: str Channel name + + Raises + ------ + AblyException + With code 90011 if the channel is in any other state. Await channel.detach() before + calling release(). """ + # RTS4c if name not in self.__all: return - channel = self.__all.pop(name) + channel = self.__all[name] + # RTS4e + if channel.state not in (ChannelState.INITIALIZED, ChannelState.DETACHED, ChannelState.FAILED): + raise AblyException( + 'Can only release a channel in a state where there is no possibility of further updates ' + 'from the server being received (initialized, detached, or failed). The current state is ' + f'{channel.state.value}', + 400, + 90011, + ) + # RTS4d + del self.__all[name] # The released channel receives nothing more, so its objects stop their GC timer and their waits channel.object._release() diff --git a/test/unit/liveobjects/realtime_object_test.py b/test/unit/liveobjects/realtime_object_test.py index 63e05163..c93786fb 100644 --- a/test/unit/liveobjects/realtime_object_test.py +++ b/test/unit/liveobjects/realtime_object_test.py @@ -293,15 +293,16 @@ def failing_replace_data(self, object_message): # --- R2: the GC timer and waits of a released channel --------------------------------------- async def test_rto10_a_released_channel_stops_its_gc_timer(): - """RTO10a: the GC timer every ATTACHED schedules stops when its channel is released. A - released channel is out of the client's collection, so closing the client never detaches - it; the release itself stops the timer, for good.""" + """RTO10a: the GC timer every ATTACHED schedules stops when its channel is detached, and + once the channel is released (RTS4) nothing schedules it again.""" clock = FakeClock() client, channel, root, mock_ws = await setup_synced_channel('test', clock=clock) timer = channel.object._gc_timer assert timer is not None + await channel.detach() client.channels.release('test') + channel.object._schedule_gc_timer() await client.close() await clock.advance(GC_INTERVAL_MS) @@ -310,23 +311,15 @@ async def test_rto10_a_released_channel_stops_its_gc_timer(): async def test_rto23c1_a_wait_for_synced_fails_when_its_channel_is_released(): - """RTO23c1: a released channel receives nothing more, so it can never sync. A `get()` - waiting for SYNCED when the channel is released fails with 92008, and so does a later one, - at once.""" + """RTO23c1: a released channel receives nothing more, so it can never sync. A wait for + SYNCED begun after the channel is released fails with 92008 at once.""" client, channel, root, mock_ws = await setup_synced_channel('test') - await _restart_sync(channel, mock_ws) - task = asyncio.ensure_future(channel.object.get()) - await settle() - assert not task.done() - + await channel.detach() client.channels.release('test') with pytest.raises(AblyException) as excinfo: - await asyncio.wait_for(task, OPERATION_TIMEOUT) - _assert_sync_wait_failure(excinfo, 'released') - - with pytest.raises(AblyException) as excinfo: - await asyncio.wait_for(channel.object.get(), OPERATION_TIMEOUT) + await asyncio.wait_for(channel.object._wait_for_synced('The object could not be retrieved'), + OPERATION_TIMEOUT) _assert_sync_wait_failure(excinfo, 'released') diff --git a/test/uts/deviations.md b/test/uts/deviations.md index 3d5c5734..28535f9f 100644 --- a/test/uts/deviations.md +++ b/test/uts/deviations.md @@ -24,40 +24,40 @@ One Test ID can become more than one derived test: five Test IDs in `rest/unit` `error_types_test.py`, `fallback_test.py`, `rest_client_test.py` (two) and `paginated_result_test.py` — assert several independent things under a single id, and the derivation writes a function for each rather than one function with an unrelated -second half. That turns 1471 Test IDs into 1480 derived tests. Going the other way, one +second half. That turns 1472 Test IDs into 1481 derived tests. Going the other way, one derived test can become more than one case: five of the twelve `rest/integration` specifications, five of the twenty `realtime/integration` ones and the three `objects/integration` ones outside `proxy` carry a `## Protocol Variants` section and run every one of their tests twice, once per protocol; nine `rest/unit` tests are parametrized over a table of fixtures the specification gives inline; and five `objects/unit` tests are parametrized, three over a table and two over the scenarios the -specification loops through. That turns 1480 derived tests into 1607 pytest cases. +specification loops through. That turns 1481 derived tests into 1608 pytest cases. -Of **1471 Test IDs, derived as 1480 tests and run as 1607 pytest cases**: 1287 Test IDs -(1296 tests, 1421 cases) pass, 169 (169 tests, 171 cases) are gated behind +Of **1472 Test IDs, derived as 1481 tests and run as 1608 pytest cases**: 1289 Test IDs +(1298 tests, 1423 cases) pass, 168 (168 tests, 170 cases) are gated behind `RUN_DEVIATIONS`, and 15 (15 tests, 15 cases) cannot be run at all. The three groups are disjoint: two Test IDs have a gated part and a passing part — `TI1`, as two functions, and `TO/endpoint-affects-host`, as one parametrized test — and are counted with the gated, while their passing parts count among the passing tests and cases. Every gated test has been confirmed to fail when enabled, so none of them passes under both behaviours. 494 of -the Test IDs come from `uts/rest/unit` (503 tests, 536 cases), 481 from -`uts/realtime/unit` (481, 481), 322 from `uts/objects/unit` (322, 344), 84 from +the Test IDs come from `uts/rest/unit` (503 tests, 536 cases), 482 from +`uts/realtime/unit` (482, 482), 322 from `uts/objects/unit` (322, 344), 84 from `uts/rest/integration` (84, 122), 73 from `uts/realtime/integration` (73, 95) and 17 from `uts/objects/integration` (17, 29); 8 of the REST integration ids (8, 8), 30 of the realtime ones (30, 30) and 5 of the objects ones (5, 5) come from the `proxy` package -within each. Of the gated Test IDs 79 are REST and 90 realtime, which is 80 REST cases -and 91 realtime. None of the 339 objects Test IDs is gated or unrunnable. +within each. Of the gated Test IDs 79 are REST and 89 realtime, which is 80 REST cases +and 90 realtime. None of the 339 objects Test IDs is gated or unrunnable. A further 148 pytest cases cover the harness itself and are not derived from a specification: 130 under `helpers/`, for the mocks, and 18 under `objects/helpers/`, for the objects builders and the synced-channel mock. -The 158 gated Test IDs that record SDK non-compliance — 158 tests, 160 cases — reduce to -**70 distinct root causes**, 26 on the REST side and 44 on the realtime side. Three further +The 157 gated Test IDs that record SDK non-compliance — 157 tests, 159 cases — reduce to +**69 distinct root causes**, 26 on the REST side and 43 on the realtime side. Three further defects are recorded below with no test of their own, because the specification's test cannot discriminate (RTP18a), has nothing to assert against (the timezone split on synthesized LEAVE timestamps), or is worked around in the setup of every test that would otherwise trip over it (`enterClient` on an anonymous connection), so the file -carries **73 SDK root causes** in all. The remaining 11 gated Test IDs are +carries **72 SDK root causes** in all. The remaining 11 gated Test IDs are specification faults, and reduce to 8. The objects tier records no SDK non-compliance; its specification faults are all corrected or worked around in tests that pass, and reduce to the seven candidate issues O.1–O.7, none of them filed. @@ -433,7 +433,7 @@ the site. | RTL4b (`channel_attach.md`) | `channelRetryTimeout: 100` ("short timeout for testing"), then `AWAIT_STATE client.connection.state == suspended` | `channelRetryTimeout` governs channel retries, not the connection's suspend timer, which runs for `connectionStateTtl`. The test does not enable fake timers either, so on real time it would wait out two minutes. Derived with a `FakeClock`, passing the named option through unchanged | | RTP5f, RTL11 (`realtime_presence_channel_state.md`) | `simulate_disconnect()` then `AWAIT_STATE channel.state == suspended` | A transport drop reaches DISCONNECTED. RTL3c propagates SUSPENDED to channels only from a SUSPENDED *connection*, so the awaited state never arrives. RTP5f's own note ("e.g. connection transitions to SUSPENDED") says as much; the steps do not carry it out | | RTP5a (`realtime_presence_channel_state.md`) | detach, then `presence.get(waitForSync: false).length == 0` | RTP11e has `get` run the ensure-active-channel procedure for any state but SUSPENDED, so the read-back re-attaches the channel and the specification's own server then repopulates the very map being checked for emptiness. The derived test reads the two maps directly | -| RTS3c1, RTL16a (`channel_options.md`), RTS4a (`channels_collection.md`) | `autoConnect: false`, no mock installed, `connect()` never called, then `AWAIT channel.attach()` and assert ATTACHED | RTL4b requires `attach()` to fail unless the connection is CONNECTING, CONNECTED or DISCONNECTED, and with no mock nothing would answer the ATTACH in any case. Derived with a mock that connects and answers each ATTACH. Upstream should give these three setups a mock, as the sibling sections of the same files do | +| RTS3c1, RTL16a (`channel_options.md`) | `autoConnect: false`, no mock installed, `connect()` never called, then `AWAIT channel.attach()` and assert ATTACHED | RTL4b requires `attach()` to fail unless the connection is CONNECTING, CONNECTED or DISCONNECTED, and with no mock nothing would answer the ATTACH in any case. Derived with a mock that connects and answers each ATTACH. Upstream should give these setups a mock, as the sibling sections of the same file do | | RTS3c1 `error-reattach-modes-1` (`channel_options.md`) | `# Put channel in attaching state (implementation detail)` | The premise the test turns on is the one step it does not give, and the setup has no mock to reach ATTACHING with | ### Fixtures written against mock methods the contract does not define @@ -1439,22 +1439,6 @@ exactly as one arriving while ATTACHED does. **Status:** open bug. -#### `Channels.release` does not detach the channel — 1 test - -**Spec point:** RTS4a. - -Release "detaches the channel and then releases the channel resource". `Channels.release` -(`channel.py:1012-1026`) is `if name not in self.__all: return` followed by -`del self.__all[name]`, and sends nothing. An attached channel is dropped from the -collection while still attached in the Ably service, and the orphaned object stays in -ATTACHED. It overrides the REST implementation, which is correct for REST, without adding -the detach. - -**Tests affected:** `test_rts4a_release_detaches_attached` — `assert 0 == 1` on the -DETACH-message count. - -**Status:** open bug. - #### A decode error other than 40018 has no channel-level handling — 2 tests, 3 cases **Spec point:** PC3. @@ -2031,7 +2015,7 @@ through an internal object to get at a value the specification makes public. | RTP2d1, RTP2h1a, and the `Interface Under Test` blocks of all three presence-map specs | `put(message) -> PresenceMessage?` and `remove(message) -> PresenceMessage?` | both return `bool` (`presencemap.py:111`, `:159`); the message to emit is the caller's own, which `set_presence` appends to `broadcast_messages` when the return is true. `IS NOT null` is read as `is True`. `put` stores a *copy* with the action rewritten to PRESENT and leaves the caller's message untouched, so RTP2d1's "emit the original action" falls out for free | all of `presence_map_test.py` | | RTP19, and the `Interface Under Test` block of `presence_sync.md` | `endSync() -> List`, the synthesized LEAVEs | `end_sync()` returns `(residual, absent)` of the *stored* members; the synthesis lives one level up in `RealtimePresence.set_presence` (`presence.py:575-587`). Tests reading only counts and clientIds concatenate the two lists exactly as `set_presence` does; tests reading the LEAVE itself drive a `RealtimePresence` and assert on what its subscribers receive | 4 in `presence_sync_test.py` | | TB2, RTS3b, RTS3c, RTS3c1, RTL16 | `channel.options` as a `ChannelOptions` | a dict keyed by wire names, because `RealtimeChannel` passes `ChannelOptions.to_dict()` to the REST `Channel` constructor (`channel.py:84`). Assertions read `channel.options['params']['rewind']`. On `ChannelOptions` itself the cipher attribute is spelled `cipher`, not `cipherParams`. `set_options_without_reattach` replaces the stored mapping wholesale rather than merging, which `test_rts3c_options_updated_existing` pins | 5 | -| RTS2, RTS4a | `channels.exists(name)`, `channels.names`, and an awaitable `release()` | `name in client.channels` (`Channels.__contains__`); the collection iterates over its channels rather than their names; `release` is synchronous. Genuinely idiomatic spelling rather than an absence — recorded only because of the `__getattr__` hazard noted below | 4 | +| RTS2, RTS4c, RTS4d, RTS4e | `channels.exists(name)` and `channels.names` | `name in client.channels` (`Channels.__contains__`); the collection iterates over its channels rather than their names. Genuinely idiomatic spelling rather than an absence — recorded only because of the `__getattr__` hazard noted below | 6 | | RSH1b1, RSH1b2, RSH1b3, RSH1b4, RSH1b5, RSH1c3 | `DevicePushDetails`. The specification builds every device as `DeviceDetails(…, push: DevicePushDetails(recipient: {…}))`; ably-python has no such type | `DeviceDetails.__init__` takes `push` as a plain dict and stores it unchanged (`ably/types/device.py:10-40`), and `DeviceDetails.push` hands that dict back, so the tests read `{'recipient': {…}}` directly. The recipient's `transportType` is still validated against `DevicePushTransportType` in the constructor, which is the only part of `DevicePushDetails` carrying behaviour | the 7 in `push_admin_test.py` that register a device, through its `apns_device()` helper | **Status:** open bugs of the missing-API kind, not of the wrong-behaviour kind. Adding the @@ -2764,7 +2748,7 @@ is missing accessors over correct behaviour. Within a tier the order is blast ra | Issue | Covers | Rows below it answers | |---|---|---| | [#706](https://github.com/ably/ably-python/issues/706) | the fabricated `"None:0"` message id | 3.3 | -| [#658](https://github.com/ably/ably-python/issues/658) | presence messages sent on reconnection before reattach | adjacent to 3.14, which is the other half of RTP17 automatic re-entry | +| [#658](https://github.com/ably/ably-python/issues/658) | presence messages sent on reconnection before reattach | adjacent to 3.13, which is the other half of RTP17 automatic re-entry | | [#656](https://github.com/ably/ably-python/issues/656) | `utcfromtimestamp` deprecation | adjacent to 2.4 | | [#709](https://github.com/ably/ably-python/issues/709)–[#712](https://github.com/ably/ably-python/issues/712) | REST request timeout, token-request nonce reuse, single-host retry, `dispose()` teardown | adjacent to I.3, which is a second defect on the line #709 is about and is not covered by it; otherwise none, these having been filed from the REST derivation | @@ -2999,33 +2983,27 @@ disconnect per RTN7e", which issue 1.2 above proves false. No gated test: the UT this cannot distinguish the behaviours (recorded under UTS Spec Errors), so the finding is the output. -**3.9 `Channels.release` does not detach the channel.** RTS4a. `channel.py:1012-1026` deletes -the entry and sends nothing, so the channel is dropped from the collection while still -attached in the Ably service — the application goes on being billed for and delivered to a -channel it believes it released. -`test/uts/realtime/unit/channels/channels_collection_test.py -k rts4a_release_detaches_attached` - -**3.10 The UPDATE event drops the CONNECTED message's error.** RTN24. `on_connected` +**3.9 The UPDATE event drops the CONNECTED message's error.** RTN24. `on_connected` (`connectionmanager.py:425-428`) builds the `ConnectionStateChange` without the `reason` it was handed, so an application never sees the error a CONNECTED carries — including the RTN15c7 failed-resume error, which RTN25 lists among those that must set `Connection#errorReason`. A one-line fix. `test/uts/realtime/unit/connection/update_events_test.py -k rtn24_update_event_with_error` -**3.11 Detach protocol faults.** RTL5i — a detach requested while already DETACHING sends a +**3.10 Detach protocol faults.** RTL5i — a detach requested while already DETACHING sends a **second** DETACH, because `_request_state` calls `_check_pending_state()` even after `_notify_state` returns early. RTL5k — an ATTACHED arriving while DETACHING or DETACHED is ignored instead of answered with a new DETACH, so the detach times out and the channel returns to ATTACHED. Same method, one issue. `test/uts/realtime/unit/channels/channel_detach_test.py -k "rtl5i or rtl5k"` -**3.12 The deleted RTL4j ATTACH_RESUME flag is still set on every reattach.** RTL4j, deleted +**3.11 The deleted RTL4j ATTACH_RESUME flag is still set on every reattach.** RTL4j, deleted in specification 6.1.0. `_notify_state` sets `__attach_resume` on every ATTACHED and `_encode_flags` ORs it in, so every reattach carries `flags: 32` and tells the server something the server now decides for itself. `test/uts/realtime/unit/channels/channel_attach_test.py -k rtl4j` -**3.13 A decode error other than 40018 never fails the channel.** PC3. `channel.py:744-748` +**3.12 A decode error other than 40018 never fails the channel.** PC3. `channel.py:744-748` gives channel-level handling to 40018 alone; every other decode error is logged and the batch **silently skipped**, with no state change and no `error_reason`, so a vcdiff message with no decoder is simply lost. Compounding it, `message.py:302-305` checks the delta's `from` id @@ -3033,34 +3011,34 @@ decoder is simply lost. Compounding it, `message.py:302-305` checks the delta's missing-decoder branch at all. `test/uts/realtime/unit/channels/channel_delta_decoding_test.py -k pc3_no_plugin_fails` -**3.14 A failed automatic re-entry reports the NACK, not the 91004 wrapper.** RTP17e. +**3.13 A failed automatic re-entry reports the NACK, not the 91004 wrapper.** RTP17e. `_reenter_member` (`presence.py:667-674`) emits the UPDATE with `resumed=False` and the raw NACK as `reason`, where the spec requires `resumed` true and a 91004 naming the clientId with the NACK as `cause`. Local to one method. `test/uts/realtime/unit/presence/realtime_presence_reentry_test.py -k rtp17e` -**3.15 `ping()` rejects DISCONNECTED, and charges the connect wait to the caller's timeout.** +**3.14 `ping()` rejects DISCONNECTED, and charges the connect wait to the caller's timeout.** RTN13b, RTN13c, RTN13d. `ConnectionManager.ping` (`:362`) admits only CONNECTED and CONNECTING where RTN13d requires a ping from DISCONNECTED to be deferred; and `:375` enters `asyncio.wait_for` as soon as `ping()` is called, so a ping requested while CONNECTING can expire before its HEARTBEAT has gone out. `test/uts/realtime/unit/connection/connection_ping_test.py -k "rtn13b_deferred or rtn13c or rtn13d"` -**3.16 A failed RTN22 reauth leaves no trace on the connection.** RSA4c1, RSA4c3. +**3.15 A failed RTN22 reauth leaves no trace on the connection.** RSA4c1, RSA4c3. `websockettransport.py:170-175` awaits `auth.authorize()` inside a bare `except Exception` that only logs, so nothing reaches `on_error_from_authorize` and `errorReason` stays as it was. **Hold this one**: specification#466 would make the current behaviour correct, and the two UTS specs disagree about it today. `test/uts/realtime/unit/auth/connection_auth_test.py -k rsa4c3_callback_error_stays_connected` -**3.17 TokenParams passed to an authCallback carry no clientId on a realtime client.** +**3.16 TokenParams passed to an authCallback carry no clientId on a realtime client.** RSA12a, RTN2e. `Auth.__init__` nulls `client_id` for a realtime client (`rest/auth.py:36-41`) and `_ensure_valid_auth_credentials` only adds it when non-null, so an auth server never learns the configured clientId. The REST client does pass it. The same nulling is why `client.client_id` and `client.auth.client_id` disagree before CONNECTED (RTC17). `test/uts/realtime/unit/auth/connection_auth_test.py -k rtn2e` -**3.18 No 40171 log at instantiation with a non-renewable token.** RSA4a1. Requires an +**3.17 No 40171 log at instantiation with a non-renewable token.** RSA4a1. Requires an info-level record carrying 40171 and the TI5 help URL; the SDK logs at debug with no code, and `grep -rn href ably/` finds no help URLs at all. Small, and the TI5 half is a gap of its own. @@ -3753,16 +3731,16 @@ The header states how many derived tests there are, how many pass, how many are gated and how many cannot run. Those numbers are the check that the file is still true: in pytest cases, the gated count must equal the number of failures under `RUN_DEVIATIONS=1`, and gated plus unrunnable must equal the number of skips without -it. As of this writing that is 171 failures, 1569 passes and 15 skips with the variable -set, and 186 skips and 1569 passes without it, the 1569 being 1421 derived cases and 148 +it. As of this writing that is 170 failures, 1571 passes and 15 skips with the variable +set, and 185 skips and 1571 passes without it, the 1571 being 1423 derived cases and 148 harness ones — 130 under `helpers/` and 18 under `objects/helpers/`. Those figures need the `submodules` checkout CI makes: without it, three `rest/unit/encoding` tests that read the ably-common fixtures skip as well. The tiers are told apart by the junit report's `classname`, `test.uts..`, with `proxy` the package within an integration tier. The other two counts are measured from the source rather than from a run. The number of -**derived tests** is the number of `# UTS:` comments, 1480. The number of **Test IDs** is -the number of *distinct* ids in them, 1471 — not the same figure, because five ids in +**derived tests** is the number of `# UTS:` comments, 1481. The number of **Test IDs** is +the number of *distinct* ids in them, 1472 — not the same figure, because five ids in `rest/unit` are carried by more than one test function. Counting the comments and calling the result Test IDs is the easy mistake here, and it overstates the specification coverage by nine. The other is to gate or un-gate a test, record it in the body, and leave the diff --git a/test/uts/realtime/unit/channels/channels_collection_test.py b/test/uts/realtime/unit/channels/channels_collection_test.py index e455a584..8d85a2df 100644 --- a/test/uts/realtime/unit/channels/channels_collection_test.py +++ b/test/uts/realtime/unit/channels/channels_collection_test.py @@ -1,6 +1,6 @@ """Derived from uts/realtime/unit/channels/channels_collection.md in ably/specification. -Spec points: RTS1, RTS2, RTS3a, RTS4a +Spec points: RTS1, RTS2, RTS3a, RTS4c, RTS4d, RTS4e The specification's `channels.exists(name)` and `channels.names` are spelled `name in channels` and iteration over the collection in ably-python, and @@ -11,13 +11,15 @@ import asyncio +import pytest + from ably.pubsub.realtime.channel import Channels as RealtimeChannels from ably.pubsub.realtime.channel import RealtimeChannel from ably.pubsub.realtime.connection import ConnectionState from ably.pubsub.transport.websockettransport import ProtocolMessageAction from ably.pubsub.types.channelstate import ChannelState -from test.uts.helpers.client import await_connection_state, poll_until, realtime_client -from test.uts.helpers.deviations import deviation +from ably.pubsub.util.exceptions import AblyException +from test.uts.helpers.client import await_connection_state, realtime_client from test.uts.helpers.mock_websocket import ( CONNECTED_MESSAGE, MockWebSocket, @@ -119,37 +121,35 @@ async def test_rts3a_subscript_operator_channel(): assert channel1.name == channel_name -# UTS: realtime/unit/RTS4a/release-removes-channel-0 -async def test_rts4a_release_removes_channel(): - channel_name = 'test-RTS4a' +# UTS: realtime/unit/RTS4c/release-nonexistent-noop-0 +async def test_rts4c_release_nonexistent_noop(): + channel_name = 'test-RTS4c-nonexistent' client = realtime_client() - client.channels.get(channel_name) - assert (channel_name in client.channels) is True - - # `release` is synchronous here, so there is nothing to await client.channels.release(channel_name) assert (channel_name in client.channels) is False -# UTS: realtime/unit/RTS4a/release-nonexistent-noop-1 -async def test_rts4a_release_nonexistent_noop(): - channel_name = 'test-RTS4a-nonexistent' +# UTS: realtime/unit/RTS4d/release-removes-channel-0 +async def test_rts4d_release_removes_channel(): + channel_name = 'test-RTS4d' client = realtime_client() + channel = client.channels.get(channel_name) + assert channel.state == ChannelState.INITIALIZED + assert (channel_name in client.channels) is True + client.channels.release(channel_name) assert (channel_name in client.channels) is False -# UTS: realtime/unit/RTS4a/release-detaches-attached-2 -@deviation -async def test_rts4a_release_detaches_attached(): - # DEVIATION: RTS4a has release detach the channel before dropping it. - # `Channels.release` (`ably/realtime/channel.py:1012`) only deletes the entry from - # its dict, so no DETACH is sent and the channel is left attached in the Ably service. - channel_name = 'test-RTS4a-attached' +async def connected_client(answer_detach): + """A connected client whose mock answers each ATTACH, and each DETACH if `answer_detach`. + + Returns the client and the list of messages it sends. + """ messages_from_client = [] mock_ws = MockWebSocket( @@ -159,33 +159,51 @@ async def test_rts4a_release_detaches_attached(): def on_message_from_client(msg): messages_from_client.append(msg) if msg.get('action') == ProtocolMessageAction.ATTACH: - mock_ws.send_to_client(attached_message(channel_name)) - elif msg.get('action') == ProtocolMessageAction.DETACH: - mock_ws.send_to_client(detached_message(channel_name)) + mock_ws.send_to_client(attached_message(msg['channel'])) + elif msg.get('action') == ProtocolMessageAction.DETACH and answer_detach: + mock_ws.send_to_client(detached_message(msg['channel'])) mock_ws.on_message_from_client = on_message_from_client client = realtime_client(mock_ws) client.connect() await await_connection_state(client, ConnectionState.CONNECTED) + return client, messages_from_client + +# UTS: realtime/unit/RTS4d/release-after-detach-1 +async def test_rts4d_release_after_detach(): + channel_name = 'test-RTS4d-detached' + client, _ = await connected_client(answer_detach=True) channel = client.channels.get(channel_name) - await asyncio.wait_for(channel.attach(), OPERATION_TIMEOUT) - assert channel.state == ChannelState.ATTACHED - state_before_release = channel.state + await asyncio.wait_for(channel.attach(), OPERATION_TIMEOUT) + await asyncio.wait_for(channel.detach(), OPERATION_TIMEOUT) + assert channel.state == ChannelState.DETACHED client.channels.release(channel_name) - # A release which detaches sends DETACH on a task, and the channel reaches - # DETACHED only once the mock's answer has been handled, which is several - # yields away rather than one - await poll_until(lambda: channel.state == ChannelState.DETACHED, OPERATION_TIMEOUT, - 'the released channel to detach') - assert state_before_release == ChannelState.ATTACHED assert (channel_name in client.channels) is False + + +# UTS: realtime/unit/RTS4e/release-attached-fails-0 +async def test_rts4e_release_attached_fails(): + channel_name = 'test-RTS4e-attached' + client, messages_from_client = await connected_client(answer_detach=False) + channel = client.channels.get(channel_name) + + await asyncio.wait_for(channel.attach(), OPERATION_TIMEOUT) + assert channel.state == ChannelState.ATTACHED + + with pytest.raises(AblyException) as exc_info: + client.channels.release(channel_name) + + assert exc_info.value.code == 90011 + assert exc_info.value.status_code == 400 + assert channel.state == ChannelState.ATTACHED + assert (channel_name in client.channels) is True + assert client.channels.get(channel_name) is channel detach_messages = [m for m in messages_from_client if m.get('action') == ProtocolMessageAction.DETACH] - assert len(detach_messages) == 1 - assert channel.state == ChannelState.DETACHED + assert len(detach_messages) == 0 # UTS: realtime/unit/RTS3a/get-after-release-new-3 From 5b5dfb692de909ac068cd2a1a2e8e74c734a177d Mon Sep 17 00:00:00 2001 From: owenpearson Date: Mon, 26 Oct 2026 18:54:40 +0000 Subject: [PATCH 28/71] pubsub: pass the status code and the error code the right way round AblyException takes (message, status_code, code), and ten of the errors the library raises itself passed the two the other way round, so a caller reading .code got the HTTP status and .status_code got the Ably error code: detaching a FAILED channel gave status_code 90001 and code 400. The ten are the detach errors in RealtimeChannel.detach (RTL5b, RTL4h), the default errors for a failed queued message and for a NACK with no error, the vcdiff decoder errors in mixins.py and defaultvcdiffdecoder.py (PC3, RTL18), and the two ChannelOptions type checks. Each now passes the HTTP status first, which also means is_server_error is True for the two 5xx errors and False for the rest, as it should be. This is a breaking change: code that compares code or status_code against the transposed values must compare against the corrected ones, which UPDATING.md lists. The two adapted UTS tests that pinned the transposed status now assert both fields, and the deviations.md entry and candidate issue 2.9, which named six of the ten sites, are removed. Co-Authored-By: Claude Opus 5.5 --- UPDATING.md | 17 +++++++++ ably/pubsub/realtime/channel.py | 4 +- ably/pubsub/realtime/connectionmanager.py | 4 +- ably/pubsub/types/channeloptions.py | 4 +- ably/pubsub/types/mixins.py | 6 +-- ably/pubsub/vcdiff/defaultvcdiffdecoder.py | 2 +- .../realtime/realtimechannel_publish_test.py | 2 +- test/uts/deviations.md | 38 ------------------- .../unit/channels/channel_attach_test.py | 5 +-- .../unit/channels/channel_detach_test.py | 5 +-- 10 files changed, 32 insertions(+), 55 deletions(-) diff --git a/UPDATING.md b/UPDATING.md index 64f69d56..867baebd 100644 --- a/UPDATING.md +++ b/UPDATING.md @@ -124,6 +124,23 @@ await pubsub_realtime_client.channels.get('my-channel').detach() pubsub_realtime_client.channels.release('my-channel') ``` +### Some errors had their `code` and `status_code` the wrong way round + +Ten errors the library raised itself carried the Ably error code in `status_code` and the +HTTP status in `code`. They now carry them the right way round: + +| Error | 3.x `status_code`, `code` | 4.0.0 `status_code`, `code` | +|---|---|---| +| Detaching a `FAILED` realtime channel | 90001, 400 | 400, 90001 | +| A detach superseded by an attach | 90000, 409 | 409, 90000 | +| A queued message failed with no reason given | 80000, 500 | 500, 80000 | +| A NACK with no error | 50001, 500 | 500, 50001 | +| A vcdiff delta with no decoder configured | 40019, 40019 | 400, 40019 | +| A vcdiff delta that cannot be decoded | 40018, 40018 | 400, 40018 | +| `ChannelOptions` given params or options that are not a dict | 40000, 400 | 400, 40000 | + +If you branch on `code` or `status_code` for any of these, compare against the 4.0.0 values. + ## Version 2.x to 3.0.0 The 3.0.0 version of ably-python introduces several breaking changes to improve the realtime experience and align the API with the Ably specification. These include: diff --git a/ably/pubsub/realtime/channel.py b/ably/pubsub/realtime/channel.py index e484f656..17796cf2 100644 --- a/ably/pubsub/realtime/channel.py +++ b/ably/pubsub/realtime/channel.py @@ -244,7 +244,7 @@ async def detach(self) -> None: self._notify_state(ChannelState.DETACHED) return elif self.state == ChannelState.FAILED: - raise AblyException("Unable to detach; channel state = failed", 90001, 400) + raise AblyException("Unable to detach; channel state = failed", 400, 90001) else: self._request_state(ChannelState.DETACHING) @@ -258,7 +258,7 @@ async def detach(self) -> None: if new_state == ChannelState.DETACHED: return elif new_state == ChannelState.ATTACHING: - raise AblyException("Detach request superseded by a subsequent attach request", 90000, 409) + raise AblyException("Detach request superseded by a subsequent attach request", 409, 90000) else: raise state_change.reason diff --git a/ably/pubsub/realtime/connectionmanager.py b/ably/pubsub/realtime/connectionmanager.py index 407692d2..15213db8 100644 --- a/ably/pubsub/realtime/connectionmanager.py +++ b/ably/pubsub/realtime/connectionmanager.py @@ -343,7 +343,7 @@ def fail_queued_messages(self, err) -> None: f"ConnectionManager.fail_queued_messages(): discarding {len(self.queued_messages)} messages;" + f" reason = {err}" ) - error = err or AblyException("Connection failed", 80000, 500) + error = err or AblyException("Connection failed", 500, 80000) while len(self.queued_messages) > 0: pending_msg = self.queued_messages.pop() log.exception( @@ -538,7 +538,7 @@ def on_nack(self, serial: int, count: int, err: AblyException | None) -> None: err: Error information from the server """ if not err: - err = AblyException('Unable to send message; channel not responding', 50001, 500) + err = AblyException('Unable to send message; channel not responding', 500, 50001) log.error(f'ConnectionManager.on_nack(): serial={serial}, count={count}, err={err}') self.pending_message_queue.complete_messages(serial, count, None, err) diff --git a/ably/pubsub/types/channeloptions.py b/ably/pubsub/types/channeloptions.py index 9f7a99e1..71226e95 100644 --- a/ably/pubsub/types/channeloptions.py +++ b/ably/pubsub/types/channeloptions.py @@ -29,7 +29,7 @@ def __init__( self.__modes = modes # Validate params if self.__params and not isinstance(self.__params, dict): - raise AblyException("params must be a dictionary", 40000, 400) + raise AblyException("params must be a dictionary", 400, 40000) @property def cipher(self) -> CipherParams | None: @@ -77,7 +77,7 @@ def to_dict(self) -> dict[str, Any]: def from_dict(cls, options_dict: dict[str, Any]) -> ChannelOptions: """Create ChannelOptions from dictionary""" if not isinstance(options_dict, dict): - raise AblyException("options must be a dictionary", 40000, 400) + raise AblyException("options must be a dictionary", 400, 40000) return cls( cipher=options_dict.get('cipher'), diff --git a/ably/pubsub/types/mixins.py b/ably/pubsub/types/mixins.py index 976da3b1..0a352bee 100644 --- a/ably/pubsub/types/mixins.py +++ b/ably/pubsub/types/mixins.py @@ -81,11 +81,11 @@ def decode(data, encoding='', cipher=None, context=None): elif encoding == ENC_VCDIFF: if not context or not context.vcdiff_decoder: log.error('Message cannot be decoded as no VCDiff decoder available') - raise AblyException('VCDiff decoder not available', 40019, 40019) + raise AblyException('VCDiff decoder not available', 400, 40019) if not context.base_payload: log.error('VCDiff decoding requires base payload') - raise AblyException('VCDiff decode failure', 40018, 40018) + raise AblyException('VCDiff decode failure', 400, 40018) try: # Convert base payload to bytes if it's a string @@ -108,7 +108,7 @@ def decode(data, encoding='', cipher=None, context=None): except Exception as e: log.error(f'VCDiff decode failed: {e}') - raise AblyException('VCDiff decode failure', 40018, 40018) from e + raise AblyException('VCDiff decode failure', 400, 40018) from e elif encoding.startswith(f'{CipherData.ENCODING_ID}+'): if not cipher: diff --git a/ably/pubsub/vcdiff/defaultvcdiffdecoder.py b/ably/pubsub/vcdiff/defaultvcdiffdecoder.py index 07af3b9e..b5f4981c 100644 --- a/ably/pubsub/vcdiff/defaultvcdiffdecoder.py +++ b/ably/pubsub/vcdiff/defaultvcdiffdecoder.py @@ -75,7 +75,7 @@ def decode(self, delta: bytes, base: bytes) -> bytes: return result except Exception as e: log.error(f"VCDiff decode failed: {e}") - raise AblyException(f"VCDiff decode failure: {e}", 40018, 40018) from e + raise AblyException(f"VCDiff decode failure: {e}", 400, 40018) from e # Export for easy importing diff --git a/test/ably/realtime/realtimechannel_publish_test.py b/test/ably/realtime/realtimechannel_publish_test.py index a5af09f5..5afa1d27 100644 --- a/test/ably/realtime/realtimechannel_publish_test.py +++ b/test/ably/realtime/realtimechannel_publish_test.py @@ -186,7 +186,7 @@ async def check_pending(): # Force FAILED state connection_manager.notify_state( ConnectionState.FAILED, - AblyException('Test failure', 80000, 500) + AblyException('Test failure', 500, 80000) ) # Should raise exception diff --git a/test/uts/deviations.md b/test/uts/deviations.md index 28535f9f..fb6c9f07 100644 --- a/test/uts/deviations.md +++ b/test/uts/deviations.md @@ -2041,34 +2041,6 @@ asserting `clientId == 'my-client'` with the spec expectation in a comment. Adapted rather than gated because the behaviour is stable and the rest of each test — action, channel, payload — is worth running. -### `AblyException`'s status code and code are transposed at six sites - -The constructor is `AblyException(message, status_code, code)` (`util/exceptions.py:15`). -Six raises put the Ably error code in the status slot and the HTTP status in the code slot, -so the resulting exception reports each as the other: - -| Site | As written | Should be | -|---|---|---| -| `realtime/channel.py:203` | `AblyException("Unable to detach; channel state = failed", 90001, 400)` | `400, 90001` | -| `realtime/channel.py:217` | `AblyException("Detach request superseded by a subsequent attach request", 90000, 409)` | `409, 90000` | -| `realtime/connectionmanager.py:343` | `AblyException("Connection failed", 80000, 500)` | `500, 80000` | -| `types/mixins.py:84` | `AblyException('VCDiff decoder not available', 40019, 40019)` | `400, 40019` | -| `types/mixins.py:88` | `AblyException('VCDiff decode failure', 40018, 40018)` | `400, 40018` | -| `types/mixins.py:111` | `AblyException('VCDiff decode failure', 40018, 40018) from e` | `400, 40018` | - -The two neighbours that get it right — `channel.py:860` (`408, 90007`) and -`message.py:304` (`400, 40018`, for the sibling of the `mixins.py` errors) — make this a -repeated slip rather than a misunderstanding of the constructor. - -**Tests affected:** `test_rtl5b_detach_failed_errors` and `test_rtl4h_attach_while_detaching` -assert on `status_code` with a comment recording the transposition. -`test_rtn7d_fail_disconnected_no_queue`, `test_rtn7e_pending_fail_closed` and -`test_rtn7e_multiple_pending_fail` assert only what the specification asks — that a code is -present — so they pass; the wrong value is recorded here rather than asserted. - -**Status:** open bug. A mechanical fix, and a caller branching on `status_code` today gets a -five-digit number. - ### A refused connection and a connect timeout reach no failure path **Spec points:** RTN14d most directly; the same defect shapes RTN14e, RTN14f, RTN14h, @@ -2901,16 +2873,6 @@ no attribute 'key_name'`) reaches `Connection#errorReason` with no `code` at all three symptoms. `test/uts/realtime/unit/auth -k "rsa4d or rsa4f_callback_invalid_type_format"` -**2.9 `AblyException`'s status code and code are transposed at six sites.** The constructor -is `(message, status_code, code)`. `channel.py:203` (`90001, 400`), `channel.py:217` -(`90000, 409`), `connectionmanager.py:343` (`80000, 500`), `mixins.py:84` (`40019, 40019`), -`mixins.py:88` and `:111` (`40018, 40018`). `channel.py:860` (`408, 90007`) and -`message.py:304` (`400, 40018`) are right, so it is a repeated slip rather than a -misunderstanding. A caller branching on `status_code` today gets a five-digit number. -Mechanical fix. No gated test — three adapted tests assert the wrong values deliberately -(`channel_detach_test.py -k "rtl5b or rtl4h"`), so they turn red when it is fixed, which is -the signal wanted. - ### Tier 3 — silently wrong data **3.1 The clientId is sent on outgoing PresenceMessages where the spec requires it absent.** diff --git a/test/uts/realtime/unit/channels/channel_attach_test.py b/test/uts/realtime/unit/channels/channel_attach_test.py index 8f6b05d7..d59519d9 100644 --- a/test/uts/realtime/unit/channels/channel_attach_test.py +++ b/test/uts/realtime/unit/channels/channel_attach_test.py @@ -153,9 +153,8 @@ def on_message_from_client(msg): # detach and makes the pending detach fail with 90000 with pytest.raises(AblyException) as detach_error: await asyncio.wait_for(detach_future, OPERATION_TIMEOUT) - # The 90000 the library passes lands in `status_code` and the 409 in `code`, - # the two being the other way round in `AblyException(message, status_code, code)` - assert detach_error.value.status_code == 90000 + assert detach_error.value.code == 90000 + assert detach_error.value.status_code == 409 await asyncio.wait_for(attach_future, OPERATION_TIMEOUT) diff --git a/test/uts/realtime/unit/channels/channel_detach_test.py b/test/uts/realtime/unit/channels/channel_detach_test.py index e5a2a17d..f113bfb6 100644 --- a/test/uts/realtime/unit/channels/channel_detach_test.py +++ b/test/uts/realtime/unit/channels/channel_detach_test.py @@ -208,9 +208,8 @@ def on_message_from_client(msg): with pytest.raises(AblyException) as error: await asyncio.wait_for(channel.detach(), OPERATION_TIMEOUT) - # The 90001 the library passes lands in `status_code` and the 400 in `code`, - # the two being the other way round in `AblyException(message, status_code, code)` - assert error.value.status_code == 90001 + assert error.value.code == 90001 + assert error.value.status_code == 400 assert channel.state == ChannelState.FAILED From 4a07522a549695d76c0426d2e25b0165f5770bb2 Mon Sep 17 00:00:00 2001 From: owenpearson Date: Mon, 26 Oct 2026 19:02:03 +0000 Subject: [PATCH 29/71] pubsub: raise AblyException for network errors and timeouts Http.make_request re-raised whatever httpx.AsyncClient.send raised once the last host had been tried, so a request that got no response leaked httpx.ConnectError, httpx.ReadTimeout and the rest from every public method built on it: request, publish, history, presence, push admin, batch publish, auth.authorize and request_token among them. Only time() and stats() wrapped them, through catch_all, as a generic 50000. make_request now wraps an httpx.RequestError as an AblyException, 50003 with status 504 for a timeout and 80000 with status 500 for any other network error, carrying the httpx exception as cause and as __cause__ (RSC19e, TI1). The 504/50003 pair is the one the library already uses for a connect or ping that times out. A timeout or network error reaching an auth_url, which goes through its own httpx call, raises 40170 with status 401 and the same cause, as RSA4e requires. catch_all now re-raises an AblyException as it is, rather than chaining it to itself and losing the cause. On a realtime client, a token request that fails this way reaches on_error_from_authorize as an AblyException, so the connection goes DISCONNECTED with an 80019 (RSA4c) rather than recording the httpx exception as its error_reason. This is a breaking change: callers catching httpx exceptions must catch AblyException and read its code, or its cause for the httpx detail. The RSC19e UTS tests and the legacy timeout test now assert the wrapped error; the latter accepts any httpx timeout as the cause, since its connect and read budgets are both live. The RSC19e row in deviations.md is removed. Co-Authored-By: Claude Opus 5.5 --- UPDATING.md | 27 ++++++++ ably/pubsub/http/auth.py | 10 ++- ably/pubsub/request/http.py | 13 ++++ ably/pubsub/util/exceptions.py | 3 + test/ably/http/httphttp_test.py | 8 +-- test/ably/http/httprequest_test.py | 9 ++- test/unit/http_test.py | 65 ++++++++++++++++++- test/uts/deviations.md | 3 +- .../integration/proxy/rest_fallback_test.py | 2 +- test/uts/rest/unit/request_test.py | 18 ++--- 10 files changed, 135 insertions(+), 23 deletions(-) diff --git a/UPDATING.md b/UPDATING.md index 867baebd..d75089aa 100644 --- a/UPDATING.md +++ b/UPDATING.md @@ -141,6 +141,33 @@ HTTP status in `code`. They now carry them the right way round: If you branch on `code` or `status_code` for any of these, compare against the 4.0.0 values. +### Network errors and timeouts are raised as `AblyException` + +In 3.x a request that got no response, because the host could not be reached or did not +answer in time, raised the `httpx` exception itself from most methods, such as `request`, +`publish`, `history` and `auth.authorize`. In 4.0.0 every method raises an `AblyException`: +code 50003 and status 504 for a timeout, and code 80000 and status 500 for any other +network error. The `httpx` exception is its `cause`. A timeout or network error reaching +an `auth_url` is an `AblyException` with code 40170 and status 401, as for any other +`auth_url` failure. + +Example 3.x code: +```python +try: + await rest.channels.get('my-channel').publish('event', 'data') +except httpx.TimeoutException: + ... +``` + +Example 4.0.0 code: +```python +try: + await pubsub_http_client.channels.get('my-channel').publish('event', 'data') +except AblyException as e: + if e.code == 50003: + ... +``` + ## Version 2.x to 3.0.0 The 3.0.0 version of ably-python introduces several breaking changes to improve the realtime experience and align the API with the Ably specification. These include: diff --git a/ably/pubsub/http/auth.py b/ably/pubsub/http/auth.py index aa71d517..021c8d8b 100644 --- a/ably/pubsub/http/auth.py +++ b/ably/pubsub/http/auth.py @@ -6,6 +6,8 @@ from datetime import timedelta from typing import TYPE_CHECKING +import httpx + from ably.pubsub.types.options import Options if TYPE_CHECKING: @@ -410,8 +412,12 @@ async def token_request_from_auth_url(self, method: str, url: str, token_params, url = clean_url from ably.pubsub.request.http import Response - resp = await self.ably.http.request_external( - method=method, url=url, headers=headers, params=params, body=body) + try: + resp = await self.ably.http.request_external( + method=method, url=url, headers=headers, params=params, body=body) + except httpx.RequestError as e: + # RSA4e: a timeout or network error reaching the auth_url is a 40170 + raise AblyAuthException("auth_url request failed", 401, 40170, cause=e) from e response = Response(resp) AblyException.raise_for_response(response) diff --git a/ably/pubsub/request/http.py b/ably/pubsub/request/http.py index c664f1ab..cd8b54d4 100644 --- a/ably/pubsub/request/http.py +++ b/ably/pubsub/request/http.py @@ -43,6 +43,17 @@ async def wrapper(rest, *args, **kwargs): return wrapper +def wrap_request_error(host: str, e: httpx.RequestError) -> AblyException: + """Wraps the error httpx raised for a request that got no response (RSC19e). + + A timeout is 50003 and any other failure to complete the request is 80000, with + the httpx error kept as the cause. + """ + if isinstance(e, httpx.TimeoutException): + return AblyException(f'Request to {host} timed out', 504, 50003, cause=e) + return AblyException(f'Request to {host} failed with a network error', 500, 80000, cause=e) + + class Request: def __init__(self, method='GET', url='/', version=None, headers=None, body=None, skip_auth=False, raise_on_error=True): @@ -219,6 +230,8 @@ def should_stop_retrying(retry_count=retry_count): response = await self.__client.send(request) except Exception as e: if should_stop_retrying(): + if isinstance(e, httpx.RequestError): + raise wrap_request_error(host, e) from e raise e else: # RSC15l4 diff --git a/ably/pubsub/util/exceptions.py b/ably/pubsub/util/exceptions.py index 245d4a43..f961eff9 100644 --- a/ably/pubsub/util/exceptions.py +++ b/ably/pubsub/util/exceptions.py @@ -97,6 +97,9 @@ async def wrapper(*args, **kwargs): return await func(*args, **kwargs) except Exception as e: log.exception(e) + if isinstance(e, AblyException): + # Raised as it is, so that its own __cause__ is kept rather than replaced by itself + raise raise AblyException.from_exception(e) from e return wrapper diff --git a/test/ably/http/httphttp_test.py b/test/ably/http/httphttp_test.py index 7f0bfa04..91a39146 100644 --- a/test/ably/http/httphttp_test.py +++ b/test/ably/http/httphttp_test.py @@ -34,7 +34,7 @@ async def test_max_retry_attempts_and_timeouts_defaults(self): assert 'http_request_timeout' in ably.http.CONNECTION_RETRY_DEFAULTS with mock.patch('httpx.AsyncClient.send', side_effect=httpx.RequestError('')) as send_mock: - with pytest.raises(httpx.RequestError): + with pytest.raises(AblyException, match="network error"): await ably.http.make_request('GET', '/', version=Defaults.protocol_version, skip_auth=True) assert send_mock.call_count == Defaults.http_max_retry_count @@ -52,7 +52,7 @@ def sleep_and_raise(*args, **kwargs): raise httpx.TimeoutException('timeout') with mock.patch('httpx.AsyncClient.send', side_effect=sleep_and_raise) as send_mock: - with pytest.raises(httpx.TimeoutException): + with pytest.raises(AblyException, match="timed out"): await ably.http.make_request('GET', '/', skip_auth=True) assert send_mock.call_count == 1 @@ -67,7 +67,7 @@ def make_url(host): with mock.patch('httpx.Request', wraps=httpx.Request) as request_mock: with mock.patch('httpx.AsyncClient.send', side_effect=httpx.RequestError('')) as send_mock: - with pytest.raises(httpx.RequestError): + with pytest.raises(AblyException, match="network error"): await ably.http.make_request('GET', '/', skip_auth=True) assert send_mock.call_count == Defaults.http_max_retry_count @@ -93,7 +93,7 @@ async def test_no_host_fallback_nor_retries_if_custom_host(self): mock_route = respx.get("https://example.org").mock(side_effect=httpx.RequestError('')) - with pytest.raises(httpx.RequestError): + with pytest.raises(AblyException, match="network error"): await ably.http.make_request('GET', '/', skip_auth=True) assert mock_route.call_count == 1 diff --git a/test/ably/http/httprequest_test.py b/test/ably/http/httprequest_test.py index ec41c5ff..bc378e92 100644 --- a/test/ably/http/httprequest_test.py +++ b/test/ably/http/httprequest_test.py @@ -5,6 +5,7 @@ from ably.pubsub.request.paginatedresult import HttpPaginatedResponse from ably.pubsub.server import create_http_client from ably.pubsub.transport.defaults import Defaults +from ably.pubsub.util.exceptions import AblyException from test.ably.testapp import TestApp from test.ably.utils import BaseAsyncTestCase, VaryByProtocolTestsMetaclass, dont_vary_protocol @@ -102,8 +103,10 @@ async def test_timeout(self): assert ably.http.http_request_timeout == timeout # `http_open_timeout` keeps its four-second default, so the request carries a # connect budget as well as a read budget and either can be the one to expire - with pytest.raises(httpx.TimeoutException): + with pytest.raises(AblyException) as excinfo: await ably.request('GET', '/time', version=Defaults.protocol_version) + assert excinfo.value.code == 50003 + assert isinstance(excinfo.value.cause, httpx.TimeoutException) await ably.close() default_endpoint = 'https://sandbox.realtime.ably-nonprod.net/time' @@ -127,8 +130,10 @@ async def test_timeout(self): port=self.test_vars["port"], tls_port=self.test_vars["tls_port"], tls=self.test_vars["tls"]) - with pytest.raises(httpx.ConnectError): + with pytest.raises(AblyException) as excinfo: await ably.request('GET', '/time', version=Defaults.protocol_version) + assert excinfo.value.code == 80000 + assert isinstance(excinfo.value.cause, httpx.ConnectError) await ably.close() # RSC15l3 diff --git a/test/unit/http_test.py b/test/unit/http_test.py index ebae07b2..b679855b 100644 --- a/test/unit/http_test.py +++ b/test/unit/http_test.py @@ -1,9 +1,13 @@ import httpx import pytest +from ably.pubsub.realtime.connection import ConnectionState from ably.pubsub.server import create_http_client from ably.pubsub.types.testoptions import TestOptions from ably.pubsub.util.exceptions import AblyException +from test.uts.helpers.client import await_connection_state, close_open_clients, realtime_client +from test.uts.helpers.mock_http import MockHttpClient +from test.uts.helpers.mock_websocket import MockWebSocket def test_http_get_rest_hosts_works_when_fallback_realtime_host_is_set(): @@ -54,14 +58,35 @@ def refuse(request): transport = RecordingTransport(refuse) ably = create_http_client(token="foo", _test_options=TestOptions(http_transport=transport)) - with pytest.raises(AblyException): + with pytest.raises(AblyException) as excinfo: await ably.time() + # RSC19e: the httpx error is wrapped, and kept as the cause + assert excinfo.value.code == 80000 + assert excinfo.value.status_code == 500 + assert isinstance(excinfo.value.cause, httpx.ConnectError) + assert excinfo.value.__cause__ is excinfo.value.cause # Every host is tried before the error is surfaced (RSC15l) assert len(transport.requests) == len(ably.http.get_hosts()) await ably.close() +async def test_http_surfaces_transport_timeouts_as_50003(): + def time_out(request): + raise httpx.ReadTimeout("timed out", request=request) + + transport = RecordingTransport(time_out) + ably = create_http_client(key="appId.keyId:keySecret", _test_options=TestOptions(http_transport=transport)) + + with pytest.raises(AblyException) as excinfo: + await ably.channels.get('test').publish('event', 'data') + + assert excinfo.value.code == 50003 + assert excinfo.value.status_code == 504 + assert isinstance(excinfo.value.cause, httpx.TimeoutException) + await ably.close() + + def test_http_uses_a_network_transport_without_test_options(): ably = create_http_client(token="foo") assert isinstance(ably.http._Http__client._transport, httpx.AsyncHTTPTransport) @@ -86,3 +111,41 @@ def respond(request): assert 'Authorization' not in auth_requests[0].headers assert 'X-Ably-Version' not in auth_requests[0].headers await ably.close() + + +async def test_auth_url_network_errors_are_40170(): + def respond(request): + if request.url.host == 'auth.example.com': + raise httpx.ConnectError("connection refused", request=request) + return httpx.Response(200, json=[1500000000000]) + + transport = RecordingTransport(respond) + ably = create_http_client(auth_url='https://auth.example.com/token', + _test_options=TestOptions(http_transport=transport)) + + # RSA4e: a network error reaching the auth_url is a 40170, with the httpx error as the cause + with pytest.raises(AblyException) as excinfo: + await ably.auth.authorize() + + assert excinfo.value.code == 40170 + assert excinfo.value.status_code == 401 + assert isinstance(excinfo.value.cause, httpx.ConnectError) + await ably.close() + + +async def test_realtime_token_request_network_errors_reach_the_connection_wrapped(): + mock_http = MockHttpClient(on_connection_attempt=lambda conn: conn.respond_with_refused()) + mock_ws = MockWebSocket() + client = realtime_client(mock_ws, mock_http, key="appId.keyId:keySecret", use_token_auth=True, + disconnected_retry_timeout=600000) + try: + client.connect() + await await_connection_state(client, ConnectionState.DISCONNECTED) + + # RSA4c: a failed TokenRequest exchange is an auth error, so the connection is not + # left holding the httpx exception + assert isinstance(client.connection.error_reason, AblyException) + assert client.connection.error_reason.code == 80019 + assert mock_ws.connection_attempts == [] + finally: + await close_open_clients() diff --git a/test/uts/deviations.md b/test/uts/deviations.md index fb6c9f07..5ad9f7ba 100644 --- a/test/uts/deviations.md +++ b/test/uts/deviations.md @@ -2336,7 +2336,7 @@ pseudocode is not mistaken for non-compliance. | RTP19a's route | the specification models an ATTACHED without HAS_PRESENCE as `startSync()` then `endSync()`. `on_attached(has_presence=False)` calls `_synthesize_leaves(...)` then `clear()` (`presence.py:611-618`), which is the requirement itself rather than the model of it | | `hasNext` as a value | `push_admin.md` RSH1b2 writes `ASSERT result.hasNext == true`. In Python `has_next` is a bound method (`ably/http/paginatedresult.py:63`), truthy whatever the page holds, so the direct translation asserts nothing. `test_rsh1b2_list_devices_pagination` writes `result.has_next() is True`, confirmed to fail when inverted | | Push `remove` return values | the specification's remove steps assert nothing about a return value ("should not throw"). `PushDeviceRegistrations.remove` / `remove_where` and `PushChannelSubscriptions.remove` / `remove_where` return the `ably.http.http.Response` from the DELETE rather than `None` (`ably/rest/push.py:112-127`, `:176-192`), so the six derived removal tests assert `response.status_code == 204`, which is stronger than the specification asks and matches what `test/ably/rest/restpush_test.py` already asserts | -| `rest/proxy/RSC15l/unreachable-endpoint-error-0` | the specification asserts only that the error carries a non-null `status_code` or `code`, leaving the values open, so the derived test asserts exactly that and is the weakest of the eight proxy tests. What this SDK produces against a refused connection is 500 / 50000 with the message "All connection attempts failed", `catch_all` having wrapped the `httpx.ConnectError`, and that is recorded here rather than asserted, since pinning it would assert more than the specification does | +| `rest/proxy/RSC15l/unreachable-endpoint-error-0` | the specification asserts only that the error carries a non-null `status_code` or `code`, leaving the values open, so the derived test asserts exactly that and is the weakest of the eight proxy tests. What this SDK produces against a refused connection is 500 / 80000, the RSC19e wrapping of the `httpx.ConnectError`, which it keeps as the `cause`, and that is recorded here rather than asserted, since pinning it would assert more than the specification does | | `RSL1k4`'s history read | the event log is per session, so any request the client under test makes through it is recorded — including the `history()` the test verifies deduplication with. The log is therefore read **before** the history call, and the POST count assertion made against that snapshot. The history read itself is a `wall_clock_poll_until` rather than one fetch, for the same reason every other integration verification is: a published message does not reach history at once | | RSP4b1's time bounds | the specification records `time_before = now_millis()` before generating the presence events and `time_after = now_millis()` after, then asserts a `history(start=, end=)` over that window returns them. Read from the runner's clock the window is only as good as the skew against the sandbox, which decides the timestamps actually stored, so a runner running a little fast would exclude the very events the test generated. Both bounds come from `await client.time()` instead — the same instant on the clock that stamps the events. `Presence.history` passes an `int` straight through as milliseconds (`ably/types/presence.py:232-241`), which is what `client.time()` returns, so no conversion is involved | @@ -2355,7 +2355,6 @@ pseudocode is not mistaken for non-compliance. | RSH1a | Empty `recipient` or `data` rejected with code 40000, the error reaching the caller from the server | `PushAdmin.publish` validates its arguments itself and raises before touching the HTTP layer (`ably/rest/push.py:49-59`): a non-dict `recipient` or `data` raises `TypeError`, an empty one `ValueError`. So there is no request, no server error and no `code` to read. The "no HTTP request" half is satisfied, and the repository's own sandbox suite already pins the exception types (`test/ably/rest/restpush_test.py::test_admin_publish`). `test_rsh1a_push_publish_invalid_recipient` in the integration tier asserts `pytest.raises(ValueError)` alongside the unit-tier test | Open bug, minor — a stricter precondition rather than wrong behaviour. The specification's test would need a recipient the SDK will send and the server will reject, an unknown `transportType` say, to exercise the server-side path it describes | | HP6 | `errorCode` is a number | The raw header string, `'40101'` | Open bug, trivial | | HP8 | `headers` is a map | A list of `(name, value)` pairs, so the lookup the spec describes is impossible without converting, and case-insensitivity is lost | Open bug; changing the return type is breaking | -| RSC19e | An error indicated idiomatically | `httpx.ConnectError` / `ReadTimeout` reach the caller unwrapped, because `AblyRest.request` carries no `@catch_all` unlike `time()` and `stats()`. The messages do name the failure | Borderline; defensible under RSC19e | | RSC15a | Six hosts tried | Three. `Options.__get_hosts` truncates to `http_max_retry_count`, which TO3l5 sanctions | Intentional | | TI | `ErrorInfo` equality by attributes | No `__eq__`, so errors compare by identity. Python exceptions conventionally do, and the requirement appears nowhere in `features.md`. Adding `__eq__` without `__hash__` would make `AblyException` unhashable and break any caller that puts one in a set | Intentional | | TD5, RSA16a | `capability` is stringified JSON | A `Capability` object, a public convenience type used throughout `auth`. Narrowing the return type to `str` would break every caller that indexes or mutates it, so it is reserved for a future major | Intentional; a breaking change to align | diff --git a/test/uts/rest/integration/proxy/rest_fallback_test.py b/test/uts/rest/integration/proxy/rest_fallback_test.py index aed99207..afafb084 100644 --- a/test/uts/rest/integration/proxy/rest_fallback_test.py +++ b/test/uts/rest/integration/proxy/rest_fallback_test.py @@ -167,7 +167,7 @@ async def test_rsc15l_unreachable_endpoint_error(sandbox): # (the exact code/statusCode depends on the SDK's HTTP layer, but it must # be present and non-null so callers can programmatically handle it). # NOTE: the specification leaves the values open. This SDK reports 500 and - # 50000, the connection error having been wrapped by `catch_all`. + # 80000, the connection error having been wrapped per RSC19e. error = excinfo.value assert error is not None assert error.status_code is not None or error.code is not None diff --git a/test/uts/rest/unit/request_test.py b/test/uts/rest/unit/request_test.py index 3aa76313..a1bd7a47 100644 --- a/test/uts/rest/unit/request_test.py +++ b/test/uts/rest/unit/request_test.py @@ -11,6 +11,7 @@ import msgpack import pytest +from ably.pubsub.util.exceptions import AblyException from test.uts.helpers.client import rest_client from test.uts.helpers.deviations import deviation from test.uts.helpers.mock_http import MockHttpClient @@ -426,14 +427,11 @@ async def test_rsc19e_network_error_propagated(): client = rest_client(mock_http, key=SPEC_KEY, fallback_hosts=[]) - # DEVIATION: the spec expects an error carrying code 80000, or a message naming the - # network/connection failure. request() is not wrapped by @catch_all (unlike time() - # and stats()), so the transport exception reaches the caller unwrapped. Its message - # does name the connection failure, which is what RSC19e asks be indicated. - with pytest.raises(httpx.ConnectError) as excinfo: + with pytest.raises(AblyException) as excinfo: await client.request('GET', '/test', version='3') - assert 'Connection refused' in str(excinfo.value) + assert excinfo.value.code == 80000 + assert isinstance(excinfo.value.cause, httpx.ConnectError) # UTS: rest/unit/RSC19e/timeout-error-handling-1 @@ -448,13 +446,11 @@ async def test_rsc19e_timeout_error_handling(): client = rest_client(mock_http, key=SPEC_KEY, http_request_timeout=1, fallback_hosts=[]) - # DEVIATION: the spec expects an error carrying code 50003, or a message naming the - # timeout. request() is not wrapped by @catch_all, so the transport exception reaches - # the caller unwrapped; its message does name the timeout. - with pytest.raises(httpx.ReadTimeout) as excinfo: + with pytest.raises(AblyException) as excinfo: await client.request('GET', '/test', version='3') - assert 'timed out' in str(excinfo.value) + assert excinfo.value.code == 50003 + assert isinstance(excinfo.value.cause, httpx.TimeoutException) # UTS: rest/unit/RSC19e/http-error-no-fallback-2 From ab9c834b30f362653fbc22ce2304f835aa67eed1 Mon Sep 17 00:00:00 2001 From: owenpearson Date: Mon, 26 Oct 2026 19:06:58 +0000 Subject: [PATCH 30/71] pubsub: keep an AblyException raised by an auth_callback as it is Auth.request_token wrapped every exception an auth_callback raised as 401/40170, so the statusCode a callback gave was lost. RSA4d fails the connection when the auth provider answers 403, and ConnectionManager.on_error_from_authorize branches on status 403 to do so, but an auth_callback could never reach that branch: a 403 went DISCONNECTED with an 80019/401 and the client kept retrying. RSA4e only asks for the 40170 wrapping of errors that are not already an ErrorInfo. request_token now re-raises an AblyException from the callback unchanged and still wraps anything else as 40170. Auth.authorize, which also serves the RTN22 reauth a server AUTH triggers, hands a 403 to on_error_from_authorize on a realtime client, so a 403 fails the connection whether it comes while connecting, on a reauth or from an explicit authorize() (RSA4d1). Both 80019s on_error_from_authorize builds now carry the error that caused them (RSA4c1, RSA4d). The empty token string error's code, 4017, is corrected to 40170. This is a breaking change: a caller that identified its own callback's failures by code 40170 receives the callback's AblyException instead, and a realtime client whose callback raises a 403 ends FAILED rather than retrying. The four RSA4d UTS tests are un-gated, the two RSA4c tests that pinned the missing cause now assert it, and deviations.md drops the RSA4d entry and the cause half of the RSA4c adaptation, with its counts adjusted. Co-Authored-By: Claude Opus 5.5 --- UPDATING.md | 23 ++++ ably/pubsub/http/auth.py | 14 ++- ably/pubsub/realtime/connectionmanager.py | 4 +- test/unit/http_test.py | 16 +++ test/uts/deviations.md | 102 ++++++------------ .../unit/auth/auth_callback_errors_test.py | 8 +- .../unit/auth/connection_auth_test.py | 8 +- 7 files changed, 89 insertions(+), 86 deletions(-) diff --git a/UPDATING.md b/UPDATING.md index d75089aa..272ec337 100644 --- a/UPDATING.md +++ b/UPDATING.md @@ -168,6 +168,29 @@ except AblyException as e: ... ``` +### An `AblyException` raised by an `auth_callback` is no longer rewritten + +In 3.x any exception an `auth_callback` raised reached the caller as an `AblyException` with +code 40170 and status 401, the original only in its `cause`. In 4.0.0 an `AblyException` the +callback raises reaches the caller unchanged; any other exception is still wrapped as 40170. +On a realtime client, a callback error with status 403 now fails the connection, with an +80019 whose status is 403 and whose `cause` is the callback's error, whether it happens while +connecting, on a reauth or in an explicit `authorize()` call. 3.x went `DISCONNECTED` and +retried. Every 80019 the connection reports now carries the auth provider's error as its +`cause`. + +If you check for 40170 after `request_token` or `authorize` to find a failure in your own +callback, check the code your callback raised instead. Raise a status 403 only for a client +that should stop connecting: + +Example 4.0.0 code: +```python +async def auth_callback(token_params): + if not await user_may_connect(): + raise AblyException('Access revoked', 403, 40300) + return await fetch_token(token_params) +``` + ## Version 2.x to 3.0.0 The 3.0.0 version of ably-python introduces several breaking changes to improve the realtime experience and align the API with the Ably specification. These include: diff --git a/ably/pubsub/http/auth.py b/ably/pubsub/http/auth.py index 021c8d8b..aed3db26 100644 --- a/ably/pubsub/http/auth.py +++ b/ably/pubsub/http/auth.py @@ -153,7 +153,14 @@ def token_details_has_expired(self): return expires < timestamp + token_details.TOKEN_EXPIRY_BUFFER async def authorize(self, token_params: dict | None = None, auth_options=None): - return await self.__authorize_when_necessary(token_params, auth_options, force=True) + try: + return await self.__authorize_when_necessary(token_params, auth_options, force=True) + except AblyException as e: + # RSA4d, RSA4d1: a 403 from the auth provider fails the connection, even + # while the token it holds is still valid + if self.ably._is_realtime and e.status_code == 403: + self.ably.connection.connection_manager.on_error_from_authorize(e) + raise async def request_token(self, token_params: dict | None = None, # auth_options @@ -186,6 +193,9 @@ async def request_token(self, token_params: dict | None = None, log.debug("using token auth with authCallback") try: token_request = await auth_callback(token_params) + except AblyException: + # RSA4d, RSA4e: an error the callback gives as an AblyException is kept as it is + raise except Exception as e: raise AblyException("auth_callback raised an exception", 401, 40170, cause=e) from e elif auth_url: @@ -214,7 +224,7 @@ async def request_token(self, token_params: dict | None = None, raise AblyAuthException(msg, 401, 40170, cause=e) from e elif isinstance(token_request, str): if len(token_request) == 0: - raise AblyAuthException("Token string is empty", 401, 4017) + raise AblyAuthException("Token string is empty", 401, 40170) return TokenDetails(token=token_request) elif token_request is None: raise AblyAuthException("Token string was None", 401, 40170) diff --git a/ably/pubsub/realtime/connectionmanager.py b/ably/pubsub/realtime/connectionmanager.py index 15213db8..bd9a7dcd 100644 --- a/ably/pubsub/realtime/connectionmanager.py +++ b/ably/pubsub/realtime/connectionmanager.py @@ -487,11 +487,11 @@ def on_error_from_authorize(self, exception: AblyException) -> None: elif exception.status_code == 403: msg = 'Client configured authentication provider returned 403; failing the connection' log.error(f'ConnectionManager.on_error_from_authorize(): {msg}') - self.notify_state(ConnectionState.FAILED, AblyException(msg, 403, 80019)) + self.notify_state(ConnectionState.FAILED, AblyException(msg, 403, 80019, cause=exception)) else: msg = 'Client configured authentication provider request failed' log.warning(f'ConnectionManager.on_error_from_authorize: {msg}') - self.notify_state(self.__fail_state, AblyException(msg, 401, 80019)) + self.notify_state(self.__fail_state, AblyException(msg, 401, 80019, cause=exception)) async def on_closed(self) -> None: if self.transport: diff --git a/test/unit/http_test.py b/test/unit/http_test.py index b679855b..c5be5803 100644 --- a/test/unit/http_test.py +++ b/test/unit/http_test.py @@ -149,3 +149,19 @@ async def test_realtime_token_request_network_errors_reach_the_connection_wrappe assert mock_ws.connection_attempts == [] finally: await close_open_clients() + + +async def test_auth_callback_ably_exceptions_are_passed_through(): + error = AblyException('Account disabled', 403, 40300) + + async def auth_callback(params): + raise error + + ably = create_http_client(auth_callback=auth_callback) + + # RSA4e: only an error that is not already an AblyException becomes a 40170 + with pytest.raises(AblyException) as excinfo: + await ably.auth.authorize() + + assert excinfo.value is error + await ably.close() diff --git a/test/uts/deviations.md b/test/uts/deviations.md index 5ad9f7ba..24035aea 100644 --- a/test/uts/deviations.md +++ b/test/uts/deviations.md @@ -33,8 +33,8 @@ parametrized over a table of fixtures the specification gives inline; and five `objects/unit` tests are parametrized, three over a table and two over the scenarios the specification loops through. That turns 1481 derived tests into 1608 pytest cases. -Of **1472 Test IDs, derived as 1481 tests and run as 1608 pytest cases**: 1289 Test IDs -(1298 tests, 1423 cases) pass, 168 (168 tests, 170 cases) are gated behind +Of **1472 Test IDs, derived as 1481 tests and run as 1608 pytest cases**: 1293 Test IDs +(1302 tests, 1427 cases) pass, 164 (164 tests, 166 cases) are gated behind `RUN_DEVIATIONS`, and 15 (15 tests, 15 cases) cannot be run at all. The three groups are disjoint: two Test IDs have a gated part and a passing part — `TI1`, as two functions, and `TO/endpoint-affects-host`, as one parametrized test — and are counted with the gated, @@ -45,19 +45,19 @@ the Test IDs come from `uts/rest/unit` (503 tests, 536 cases), 482 from `uts/rest/integration` (84, 122), 73 from `uts/realtime/integration` (73, 95) and 17 from `uts/objects/integration` (17, 29); 8 of the REST integration ids (8, 8), 30 of the realtime ones (30, 30) and 5 of the objects ones (5, 5) come from the `proxy` package -within each. Of the gated Test IDs 79 are REST and 89 realtime, which is 80 REST cases -and 90 realtime. None of the 339 objects Test IDs is gated or unrunnable. +within each. Of the gated Test IDs 79 are REST and 85 realtime, which is 80 REST cases +and 86 realtime. None of the 339 objects Test IDs is gated or unrunnable. A further 148 pytest cases cover the harness itself and are not derived from a specification: 130 under `helpers/`, for the mocks, and 18 under `objects/helpers/`, for the objects builders and the synced-channel mock. -The 157 gated Test IDs that record SDK non-compliance — 157 tests, 159 cases — reduce to -**69 distinct root causes**, 26 on the REST side and 43 on the realtime side. Three further +The 153 gated Test IDs that record SDK non-compliance — 153 tests, 155 cases — reduce to +**68 distinct root causes**, 26 on the REST side and 42 on the realtime side. Three further defects are recorded below with no test of their own, because the specification's test cannot discriminate (RTP18a), has nothing to assert against (the timezone split on synthesized LEAVE timestamps), or is worked around in the setup of every test that would otherwise trip over it (`enterClient` on an anonymous connection), so the file -carries **72 SDK root causes** in all. The remaining 11 gated Test IDs are +carries **71 SDK root causes** in all. The remaining 11 gated Test IDs are specification faults, and reduce to 8. The objects tier records no SDK non-compliance; its specification faults are all corrected or worked around in tests that pass, and reduce to the seven candidate issues O.1–O.7, none of them filed. @@ -1735,37 +1735,14 @@ seconds rather than the default ten. token string, or leaves `_configure_client_id` alone where the fetched token carries no clientId of its own. -#### An authCallback error is always rewritten as 401/40170, so RSA4d is unreachable — 4 tests - -**Spec points:** RSA4d, RSA4d1. - -`ably/rest/auth.py:182-187` wraps **every** exception an authCallback raises as -`AblyException("auth_callback raised an exception", 401, 40170, cause=e)`, discarding the -original `statusCode`. `ConnectionManager.on_error_from_authorize` (`connectionmanager.py:479-491`) -then branches on `exception.status_code == 403` to reach FAILED, and that branch can never be -taken for an authCallback: the status is always 401, so a 403 goes to the `__fail_state` -(DISCONNECTED) with an 80019/401 instead. RSA4d requires FAILED with 80019/**403** and -`cause` set to the 403, both during the connect sequence and during an RTN22 reauth. - -| Test | Observed | -|---|---| -| `connection_auth_test.py::test_rsa4d_callback_403_causes_failed` | `Timed out waiting for connection state failed; it was disconnected` | -| `connection_auth_test.py::test_rsa4d_callback_403_reauth_causes_failed` | `Timed out waiting for connection state failed; it was connected` | -| `auth_callback_errors_test.py::test_rsa4d_callback_403_connecting_failed` | `Timed out waiting for connection state failed; it was disconnected` | -| `auth_callback_errors_test.py::test_rsa4d_callback_403_reauth_failed` | `Timed out waiting for connection state failed; it was connected` | - -**Status:** open bug. Preserving the callback error's `statusCode` — or letting an -`AblyException` from the callback through unwrapped — also restores the `cause` chain -recorded under Adapted Tests. - #### A failed RTN22 reauth leaves no trace on the connection — 1 test **Spec points:** RSA4c1, RSA4c3. `WebSocketTransport.on_protocol_message` (`websockettransport.py:170-175`) handles a server -AUTH by awaiting `auth.authorize()` inside a bare `except Exception` that only logs. Nothing -reaches `on_error_from_authorize`, so no 80019 is built and `connection.errorReason` stays -as it was. +AUTH by awaiting `auth.authorize()` inside a bare `except Exception` that only logs. Only a +403, which `authorize()` hands on for RSA4d, reaches `on_error_from_authorize`; any other +failure builds no 80019, and `connection.errorReason` stays as it was. **Tests affected:** `test_rsa4c3_callback_error_stays_connected` — `Timed out waiting for errorReason to be set`. @@ -2158,31 +2135,19 @@ either — see the UTS Spec Error above. The three want closing together. **Status:** open bug. It is the reason the fake clock exists in this suite at all; see the fake-time section at the end of this file. -### An 80019 from a failed auth carries no `cause`, and a slow callback is not attributed - -**Spec points:** RSA4c, RSA4c1, RSA4c2. - -Two adaptations with one underlying theme — an auth failure reaches the right state with the -wrong explanation. - -- `ConnectionManager.on_error_from_authorize` builds - `AblyException('Client configured authentication provider request failed', 401, 80019)` - with no `cause` argument, so the error the authCallback raised survives only in the log. - RSA4c1/RSA4c2 require `cause` to be set to the underlying error. Same root cause as the - RSA4d entry above: `request_token`'s wrapper loses the original error's shape, and - `on_error_from_authorize` then drops what is left. Adapted, not gated, because the state, - code and status are all correct and worth guarding — only the `cause` link is missing. - Tests: `test_rsa4c2_callback_error_causes_disconnected`, - `test_rsa4c2_callback_error_connecting_disconnected`. -- RSA4c treats an auth attempt that outruns `realtimeRequestTimeout` as an auth error, - giving DISCONNECTED with 80019/401. ably-python applies no timeout to the callback — - `await auth_callback(token_params)` is unbounded — and the CONNECTING transition timer - (`connectionmanager.py:699-724`) ends the attempt instead, with the generic 504/50003 it - raises for any connect that does not complete in time. Test: - `test_rsa4c2_callback_timeout_connecting_disconnected`, asserting 50003/504 on a - `FakeClock`. - -**Status:** open bugs, cosmetic in effect — the connection recovers either way, but the +### A slow authCallback is not attributed to the auth provider + +**Spec points:** RSA4c, RSA4c2. + +RSA4c treats an auth attempt that outruns `realtimeRequestTimeout` as an auth error, giving +DISCONNECTED with 80019/401. ably-python applies no timeout to the callback — +`await auth_callback(token_params)` is unbounded — and the CONNECTING transition timer +(`connectionmanager.py:699-724`) ends the attempt instead, with the generic 504/50003 it +raises for any connect that does not complete in time. Test: +`test_rsa4c2_callback_timeout_connecting_disconnected`, asserting 50003/504 on a +`FakeClock`. + +**Status:** open bug, cosmetic in effect — the connection recovers either way, but the error does not say the auth provider is at fault. ### A token error with no means to renew reports the renewal failure, not the server's error @@ -2861,16 +2826,13 @@ each request and bypasses the client's own HTTP layer entirely — no client-sco observe or stub it. Two issues in one line. The derived tests reach it only through `monkeypatch`; the three REC3 tests are skipped stubs. -**2.8 An authCallback error is always rewritten as 401/40170, so RSA4d is unreachable.** -RSA4d, RSA4d1. `rest/auth.py:182-187` wraps every callback exception as -`AblyException(…, 401, 40170, cause=e)`, discarding the original status, and -`on_error_from_authorize` branches on `status_code == 403` to reach FAILED — a branch an -authCallback can never take. A 403 from an auth provider therefore gives DISCONNECTED with -80019/401 where the spec requires FAILED with 80019/403. The same wrapper is why the 80019 -carries no `cause`, and why RSA4f's non-`AblyException` (`AttributeError: 'int' object has -no attribute 'key_name'`) reaches `Connection#errorReason` with no `code` at all. One issue, -three symptoms. -`test/uts/realtime/unit/auth -k "rsa4d or rsa4f_callback_invalid_type_format"` +**2.8 An authCallback result of the wrong type reaches `errorReason` as an `AttributeError`.** +RSA4f, RSA4c2. `Auth.request_token` has no RSA4f type check, so a callback returning `12345` +falls through to `token_request.key_name` and raises `AttributeError`, which `try_host`'s +`except AblyException` does not catch. `connect_base` then records it as +`Connection#errorReason`, with no `code` at all, where the spec requires DISCONNECTED with +80019/401. +`test/uts/realtime/unit/auth -k rsa4f_callback_invalid_type_format` ### Tier 3 — silently wrong data @@ -3692,8 +3654,8 @@ The header states how many derived tests there are, how many pass, how many are gated and how many cannot run. Those numbers are the check that the file is still true: in pytest cases, the gated count must equal the number of failures under `RUN_DEVIATIONS=1`, and gated plus unrunnable must equal the number of skips without -it. As of this writing that is 170 failures, 1571 passes and 15 skips with the variable -set, and 185 skips and 1571 passes without it, the 1571 being 1423 derived cases and 148 +it. As of this writing that is 166 failures, 1575 passes and 15 skips with the variable +set, and 181 skips and 1575 passes without it, the 1575 being 1427 derived cases and 148 harness ones — 130 under `helpers/` and 18 under `objects/helpers/`. Those figures need the `submodules` checkout CI makes: without it, three `rest/unit/encoding` tests that read the ably-common fixtures skip as well. The tiers are told apart by the junit report's diff --git a/test/uts/realtime/unit/auth/auth_callback_errors_test.py b/test/uts/realtime/unit/auth/auth_callback_errors_test.py index 7f056457..768cd870 100644 --- a/test/uts/realtime/unit/auth/auth_callback_errors_test.py +++ b/test/uts/realtime/unit/auth/auth_callback_errors_test.py @@ -73,10 +73,8 @@ async def auth_callback(params): assert client.connection.error_reason is not None assert client.connection.error_reason.code == 80019 assert client.connection.error_reason.status_code == 401 - # RSA4c2 requires `cause` to carry the underlying 50000 error. - # `ConnectionManager.on_error_from_authorize` builds the 80019 without one, - # so the callback's own error is only in the log. - assert client.connection.error_reason.cause is None + assert client.connection.error_reason.cause is not None + assert client.connection.error_reason.cause.code == 50000 disconnected_changes = [c for c in state_changes if c.current == ConnectionState.DISCONNECTED] assert len(disconnected_changes) >= 1 @@ -151,7 +149,6 @@ async def auth_callback(params): # UTS: realtime/unit/RSA4d/callback-403-connecting-failed-0 -@deviation async def test_rsa4d_callback_403_connecting_failed(): connection_attempted = False @@ -194,7 +191,6 @@ def on_connection_attempt(conn): # UTS: realtime/unit/RSA4d/callback-403-reauth-failed-1 -@deviation async def test_rsa4d_callback_403_reauth_failed(): auth_callback_count = 0 diff --git a/test/uts/realtime/unit/auth/connection_auth_test.py b/test/uts/realtime/unit/auth/connection_auth_test.py index a1c32b77..5f935696 100644 --- a/test/uts/realtime/unit/auth/connection_auth_test.py +++ b/test/uts/realtime/unit/auth/connection_auth_test.py @@ -157,10 +157,8 @@ async def auth_callback(params): assert client.connection.error_reason is not None assert client.connection.error_reason.code == 80019 assert client.connection.error_reason.status_code == 401 - # RSA4c1 requires `cause` to carry the underlying 50000 error. - # `ConnectionManager.on_error_from_authorize` builds the 80019 without one, - # so the callback's own error is only in the log. - assert client.connection.error_reason.cause is None + assert client.connection.error_reason.cause is not None + assert client.connection.error_reason.cause.code == 50000 # UTS: realtime/unit/RSA4c3/callback-error-stays-connected-0 @@ -202,7 +200,6 @@ async def auth_callback(params): # UTS: realtime/unit/RSA4d/callback-403-causes-failed-0 -@deviation async def test_rsa4d_callback_403_causes_failed(): async def auth_callback(params): raise AblyException('Account disabled', 403, 40300) @@ -221,7 +218,6 @@ async def auth_callback(params): # UTS: realtime/unit/RSA4d/callback-403-reauth-causes-failed-1 -@deviation async def test_rsa4d_callback_403_reauth_causes_failed(): auth_callback_count = 0 From 9a2e03773fec89c8554529b8c4edd42338aad068 Mon Sep 17 00:00:00 2001 From: owenpearson Date: Mon, 26 Oct 2026 19:12:19 +0000 Subject: [PATCH 31/71] pubsub: raise a 40106 AblyException for invalid client arguments RSC1b requires invalid constructor arguments, such as no key, no token and no means to create one, to give an error with code 40106. The HTTP client and Auth raised a bare ValueError instead, which has no code, for no credentials, a key alongside key_name or key_secret, a token_details that is not a TokenDetails and use_token_auth=False with no key; the realtime factory shares them. push.admin.publish likewise raised TypeError or ValueError, with no code, for a recipient or data that is empty or not a dict, where RSH1a's tests expect 40000. These now raise InvalidArgumentException, an AblyException that is also a ValueError so that `except ValueError` keeps working: 40106 with status 401 for the credential errors, and 40000 with status 400 for push publish. The REC1b and REC1c option conflicts already raised 40106 with status 400 and keep that status, now as the same subclass. The status codes follow the evidence. RSC1b and the UTS specs give the code only. ably-common's errors/codes/40106.md says 40106 "is reported with code 40106 and HTTP status 401". ably-js raises 40106/401 for no credentials (auth.ts, both sites) and 40106/400 for the REC1b and REC1c conflicts (defaults.ts), which is exactly the split adopted here. ably-java raises 400/40106 only from requestToken and 400/40000 from its constructor; ably-dotnet raises 40106 with status 404; ably-go never raises 40106 and reports a missing key as 40101/401; ably-cocoa raises an NSException with no code. So ably-js, the reference SDK, and the code's own documentation agree on 401 for missing credentials, and ably-js on 400 for the option conflicts. This is a breaking change: code catching TypeError from push.admin.publish must catch AblyException or ValueError. Code catching ValueError from the factories is unaffected, and can now read the code. The RSC1b, RTC12 and RSH1a UTS tests assert the spec's codes, and their rows in deviations.md are removed or, for RSH1a, marked compliant. Co-Authored-By: Claude Opus 5.5 --- UPDATING.md | 31 +++++++++++++++++++ ably/pubsub/http/auth.py | 14 ++++++--- ably/pubsub/http/http.py | 12 ++++--- ably/pubsub/http/push.py | 9 +++--- ably/pubsub/server/__init__.py | 14 ++++----- ably/pubsub/server/sync.py | 7 ++--- ably/pubsub/types/options.py | 8 ++--- ably/pubsub/util/exceptions.py | 8 +++++ test/ably/http/httppush_test.py | 4 +-- test/unit/options_test.py | 17 ++++++++++ test/uts/deviations.md | 3 +- .../unit/client/realtime_client_test.py | 8 ++--- test/uts/rest/integration/push_admin_test.py | 13 ++++---- test/uts/rest/unit/auth/auth_scheme_test.py | 10 +++--- .../rest/unit/push/push_admin_publish_test.py | 21 ++++--------- 15 files changed, 113 insertions(+), 66 deletions(-) diff --git a/UPDATING.md b/UPDATING.md index 272ec337..9865dc14 100644 --- a/UPDATING.md +++ b/UPDATING.md @@ -191,6 +191,37 @@ async def auth_callback(token_params): return await fetch_token(token_params) ``` +### Invalid arguments raise `AblyException` + +The factories raised a bare `ValueError` in 3.x when given no means of authenticating, a +`key` together with `key_name` or `key_secret`, a `token_details` that is not a +`TokenDetails`, or `use_token_auth=False` without a key. In 4.0.0 each raises an +`AblyException` with code 40106 and status 401. Combining `endpoint` with `environment`, +`rest_host` or `realtime_host` still raises an `AblyException` with code 40106 and status +400. `push.admin.publish` raised `TypeError` for a `recipient` or `data` that is not a dict +and `ValueError` for an empty one, and now raises an `AblyException` with code 40000 and +status 400 for all four. + +These exceptions are also a `ValueError`, so `except ValueError` still catches them, but +`except TypeError` no longer catches the `push.admin.publish` ones: + +Example 3.x code: +```python +try: + await rest.push.admin.publish(recipient, data) +except TypeError: + ... +``` + +Example 4.0.0 code: +```python +try: + await pubsub_http_client.push.admin.publish(recipient, data) +except AblyException as e: + if e.code == 40000: + ... +``` + ## Version 2.x to 3.0.0 The 3.0.0 version of ably-python introduces several breaking changes to improve the realtime experience and align the API with the Ably specification. These include: diff --git a/ably/pubsub/http/auth.py b/ably/pubsub/http/auth.py index aed3db26..f4a2a713 100644 --- a/ably/pubsub/http/auth.py +++ b/ably/pubsub/http/auth.py @@ -18,7 +18,12 @@ from ably.pubsub.types.tokendetails import TokenDetails from ably.pubsub.types.tokenrequest import TokenRequest from ably.pubsub.util.clock import select_clock -from ably.pubsub.util.exceptions import AblyAuthException, AblyException, IncompatibleClientIdException +from ably.pubsub.util.exceptions import ( + AblyAuthException, + AblyException, + IncompatibleClientIdException, + InvalidArgumentException, +) from ably.pubsub.util.helper import extract_url_params __all__ = ["Auth"] @@ -62,7 +67,7 @@ def __init__(self, ably: DefaultPubSubHttpClient | DefaultPubSubRealtimeClient, self.__basic_credentials = basic_key.decode('ascii') return elif must_not_use_token_auth and not can_use_basic_auth: - raise ValueError('If use_token_auth is False you must provide a key') + raise InvalidArgumentException('If use_token_auth is False you must provide a key', 401, 40106) # Using token auth self.__auth_mechanism = Auth.Method.TOKEN @@ -85,8 +90,9 @@ def __init__(self, ably: DefaultPubSubHttpClient | DefaultPubSubRealtimeClient, elif options.token_details: log.debug("using token auth with supplied token_details") else: - raise ValueError("Can't authenticate via token, must provide " - "auth_callback, auth_url, key, token or a TokenDetail") + # RSC1b + raise InvalidArgumentException("Can't authenticate via token, must provide " + "auth_callback, auth_url, key, token or a TokenDetail", 401, 40106) async def get_auth_transport_param(self): auth_credentials = {} diff --git a/ably/pubsub/http/http.py b/ably/pubsub/http/http.py index 7d912d0b..ca9486cf 100644 --- a/ably/pubsub/http/http.py +++ b/ably/pubsub/http/http.py @@ -22,7 +22,7 @@ from ably.pubsub.types.stats import stats_response_processor from ably.pubsub.types.tokendetails import TokenDetails from ably.pubsub.util.construction import reject_direct_construction -from ably.pubsub.util.exceptions import AblyException, catch_all +from ably.pubsub.util.exceptions import AblyException, InvalidArgumentException, catch_all log = logging.getLogger(__name__) @@ -67,20 +67,22 @@ def __init__(self, key: Optional[str] = None, token: Optional[str] = None, reject_direct_construction(type(self), 'ably.pubsub.server.create_http_client') if key is not None and ('key_name' in kwargs or 'key_secret' in kwargs): - raise ValueError("key and key_name or key_secret are mutually exclusive. " - "Provider either a key or key_name & key_secret") + raise InvalidArgumentException("key and key_name or key_secret are mutually exclusive. " + "Provider either a key or key_name & key_secret", 401, 40106) if key is not None: options = Options(key=key, **kwargs) elif token is not None: options = Options(auth_token=token, **kwargs) elif token_details is not None: if not isinstance(token_details, TokenDetails): - raise ValueError("token_details must be an instance of TokenDetails") + raise InvalidArgumentException("token_details must be an instance of TokenDetails", 401, 40106) options = Options(token_details=token_details, **kwargs) elif not ('auth_callback' in kwargs or 'auth_url' in kwargs or # and don't have both key_name and key_secret ('key_name' in kwargs and 'key_secret' in kwargs)): - raise ValueError("key is missing. Either an API key, token, or token auth method must be provided") + # RSC1b + raise InvalidArgumentException( + "key is missing. Either an API key, token, or token auth method must be provided", 401, 40106) else: options = Options(**kwargs) diff --git a/ably/pubsub/http/push.py b/ably/pubsub/http/push.py index afcea029..a127de1f 100644 --- a/ably/pubsub/http/push.py +++ b/ably/pubsub/http/push.py @@ -7,6 +7,7 @@ channels_response_processor, ) from ably.pubsub.types.device import DeviceDetails, device_details_response_processor +from ably.pubsub.util.exceptions import InvalidArgumentException class Push: @@ -47,16 +48,16 @@ async def publish(self, recipient: dict, data: dict, timeout: Optional[float] = - `data`: the data of the notification """ if not isinstance(recipient, dict): - raise TypeError(f'Unexpected {type(recipient)} recipient, expected a dict') + raise InvalidArgumentException(f'Unexpected {type(recipient)} recipient, expected a dict', 400, 40000) if not isinstance(data, dict): - raise TypeError(f'Unexpected {type(data)} data, expected a dict') + raise InvalidArgumentException(f'Unexpected {type(data)} data, expected a dict', 400, 40000) if not recipient: - raise ValueError('recipient is empty') + raise InvalidArgumentException('recipient is empty', 400, 40000) if not data: - raise ValueError('data is empty') + raise InvalidArgumentException('data is empty', 400, 40000) body = data.copy() body.update({'recipient': recipient}) diff --git a/ably/pubsub/server/__init__.py b/ably/pubsub/server/__init__.py index 15c20641..10526d5f 100644 --- a/ably/pubsub/server/__init__.py +++ b/ably/pubsub/server/__init__.py @@ -163,11 +163,10 @@ def create_http_client(**kwargs) -> PubSubHttpClient: Raises ------ - ValueError - If `key` is combined with `key_name` or `key_secret`. AblyException - If no means of authenticating is given, or if `endpoint` is combined with - `environment`, `rest_host` or `realtime_host`. + With code 40106, if no means of authenticating is given, if `key` is combined + with `key_name` or `key_secret`, or if `endpoint` is combined with + `environment`, `rest_host` or `realtime_host`. It is also a ValueError. """ with factory_construction(): return _DefaultPubSubHttpClient(**kwargs) @@ -243,11 +242,10 @@ def create_realtime_client(**kwargs) -> PubSubRealtimeClient: Raises ------ - ValueError - If `key` is combined with `key_name` or `key_secret`. AblyException - If no means of authenticating is given, or if `endpoint` is combined with - `environment`, `rest_host` or `realtime_host`. + With code 40106, if no means of authenticating is given, if `key` is combined + with `key_name` or `key_secret`, or if `endpoint` is combined with + `environment`, `rest_host` or `realtime_host`. It is also a ValueError. """ with factory_construction(): return _DefaultPubSubRealtimeClient(**kwargs) diff --git a/ably/pubsub/server/sync.py b/ably/pubsub/server/sync.py index 3fcc0eec..56357a6e 100644 --- a/ably/pubsub/server/sync.py +++ b/ably/pubsub/server/sync.py @@ -52,11 +52,10 @@ def create_http_client(**kwargs) -> PubSubHttpClient: Raises ------ - ValueError - If `key` is combined with `key_name` or `key_secret`. AblyException - If no means of authenticating is given, or if `endpoint` is combined with - `environment`, `rest_host` or `realtime_host`. + With code 40106, if no means of authenticating is given, if `key` is combined + with `key_name` or `key_secret`, or if `endpoint` is combined with + `environment`, `rest_host` or `realtime_host`. It is also a ValueError. """ with factory_construction(): return _DefaultPubSubHttpClientSync(**kwargs) diff --git a/ably/pubsub/types/options.py b/ably/pubsub/types/options.py index 0a4fc932..6518744d 100644 --- a/ably/pubsub/types/options.py +++ b/ably/pubsub/types/options.py @@ -4,7 +4,7 @@ from ably.pubsub.transport.defaults import Defaults from ably.pubsub.types.authoptions import AuthOptions -from ably.pubsub.util.exceptions import AblyException +from ably.pubsub.util.exceptions import InvalidArgumentException log = logging.getLogger(__name__) @@ -42,7 +42,7 @@ def __init__(self, client_id=None, log_level=0, tls=True, rest_host=None, realti # REC1b1: endpoint is incompatible with deprecated options if endpoint is not None: if environment is not None or rest_host is not None or realtime_host is not None: - raise AblyException( + raise InvalidArgumentException( message='endpoint is incompatible with any of environment, rest_host or realtime_host', status_code=400, code=40106, @@ -67,14 +67,14 @@ def __init__(self, client_id=None, log_level=0, tls=True, rest_host=None, realti suspended_retry_timeout = Defaults.suspended_retry_timeout if environment is not None and rest_host is not None: - raise AblyException( + raise InvalidArgumentException( message='specify rest_host or environment, not both', status_code=400, code=40106, ) if environment is not None and realtime_host is not None: - raise AblyException( + raise InvalidArgumentException( message='specify realtime_host or environment, not both', status_code=400, code=40106, diff --git a/ably/pubsub/util/exceptions.py b/ably/pubsub/util/exceptions.py index f961eff9..a1572d58 100644 --- a/ably/pubsub/util/exceptions.py +++ b/ably/pubsub/util/exceptions.py @@ -111,3 +111,11 @@ class AblyAuthException(AblyException): class IncompatibleClientIdException(AblyException): pass + + +class InvalidArgumentException(AblyException, ValueError): + """Arguments the library rejects before making any request (RSC1b, RSH1a). + + Also a ValueError, which is what 3.x raised for them, so `except ValueError` still + catches it. + """ diff --git a/test/ably/http/httppush_test.py b/test/ably/http/httppush_test.py index 9c430cf7..1e17ab18 100644 --- a/test/ably/http/httppush_test.py +++ b/test/ably/http/httppush_test.py @@ -137,9 +137,9 @@ async def test_admin_publish(self): } publish = self.ably.push.admin.publish - with pytest.raises(TypeError): + with pytest.raises(AblyException): await publish('ablyChannel', data) - with pytest.raises(TypeError): + with pytest.raises(AblyException): await publish(recipient, 25) with pytest.raises(ValueError): await publish({}, data) diff --git a/test/unit/options_test.py b/test/unit/options_test.py index 84cc963b..b58e4ce1 100644 --- a/test/unit/options_test.py +++ b/test/unit/options_test.py @@ -1,5 +1,6 @@ import pytest +from ably.pubsub.server import create_http_client from ably.pubsub.types.options import Options from ably.pubsub.util.exceptions import AblyException @@ -197,3 +198,19 @@ def test_options_localhost_no_fallback_hosts(): opts = Options(endpoint="localhost") # REC2c2: explicit hostname has empty fallback assert opts.get_fallback_hosts() == [] + + +# RSC1b: invalid arguments are a 40106, and still a ValueError as they were in 3.x +@pytest.mark.parametrize('kwargs, status_code', [ + ({}, 401), + ({'key': 'appId.keyId:keySecret', 'key_name': 'appId.keyId'}, 401), + ({'token_details': 'not-token-details'}, 401), + ({'token': 'a-token', 'use_token_auth': False}, 401), + ({'key': 'appId.keyId:keySecret', 'endpoint': 'foo', 'environment': 'foo'}, 400), +]) +def test_invalid_client_arguments_raise_40106(kwargs, status_code): + with pytest.raises(AblyException) as exinfo: + create_http_client(**kwargs) + assert exinfo.value.code == 40106 + assert exinfo.value.status_code == status_code + assert isinstance(exinfo.value, ValueError) diff --git a/test/uts/deviations.md b/test/uts/deviations.md index 24035aea..96654a0d 100644 --- a/test/uts/deviations.md +++ b/test/uts/deviations.md @@ -2312,12 +2312,11 @@ pseudocode is not mistaken for non-compliance. | RSL2 | A space in a channel name is `%20` | `+`, from `parse.quote_plus` in `Channel.__init__`. `quote_plus` is form encoding, and a `+` in a URL *path* is a literal plus, so the name reaching the server is altered | Open bug, and a genuine correctness issue | | RSL8 | `Channel#status` URI-encodes the channel id | `status()` interpolates the name with no escaping at all. `a/b` addresses the wrong resource, `a?b` truncates the name into a query string, `a#b` becomes a fragment | Open bug | | RSL2, RSL11b, RSL15b | `:` is `%3A` | Left literal, from `safe=':'`. RFC 3986 allows `:` in a path segment and Ably uses it for namespaces, so the server receives the same value | Intentional | -| RSC1b | Error code 40106 | A bare `ValueError` from `AblyRest.__init__` with an informative message, not an `AblyException`, so there is no code. The realtime constructor shares it — `test_rtc12_invalid_arguments_error` records the same behaviour | Open bug | | RTC12 / RSC1, RSC1a, RSC1c | A string constructor argument is an API key when it contains `:` and a token when it does not | `AblyRest.__init__` treats its first positional argument as a key unconditionally and hands it to `AuthOptions.set_key`, which requires exactly two colon-separated parts, so a token string raises 40101/401 "key of not len 2 parameters". A token is supplied through the separate `token` or `token_details` arguments. The empty-string case is compliant | Intentional / SDK-wide: the constructor takes credentials as distinct named arguments and has no string-sniffing path to restore | | RSC18 | The constructor rejects basic auth over HTTP | Construction succeeds; 40103 is raised from `make_request` when a request needing Basic Auth is attempted, and no request goes out. RSA1/RSC18 say only "any attempt to use" | Compliant; the UTS is stricter than its source | | REC1b1, REC1c1 | Code 40000, or a message containing "invalid" or "conflict" | 400/40106 with a specific message. The features spec mandates no code | Cosmetic | | RSAN1a3 | Code 40003 for a missing `Annotation.type` | 400/40000 | Cosmetic; worth aligning cross-SDK | -| RSH1a | Empty `recipient` or `data` rejected with code 40000, the error reaching the caller from the server | `PushAdmin.publish` validates its arguments itself and raises before touching the HTTP layer (`ably/rest/push.py:49-59`): a non-dict `recipient` or `data` raises `TypeError`, an empty one `ValueError`. So there is no request, no server error and no `code` to read. The "no HTTP request" half is satisfied, and the repository's own sandbox suite already pins the exception types (`test/ably/rest/restpush_test.py::test_admin_publish`). `test_rsh1a_push_publish_invalid_recipient` in the integration tier asserts `pytest.raises(ValueError)` alongside the unit-tier test | Open bug, minor — a stricter precondition rather than wrong behaviour. The specification's test would need a recipient the SDK will send and the server will reject, an unknown `transportType` say, to exercise the server-side path it describes | +| RSH1a | Empty `recipient` or `data` rejected with code 40000, the error reaching the caller from the server | `PushAdmin.publish` validates its arguments itself and raises before touching the HTTP layer: an empty or non-dict `recipient` or `data` raises an `AblyException` with 400/40000, the code the server gives, which is also a `ValueError`. So no request is made and there is no server error; the integration tier's `test_rsh1a_push_publish_invalid_recipient` asserts the local error's code | Compliant: RSH1a asks for empty values to be "immediately rejected". The specification's test would need a recipient the SDK will send and the server will reject, an unknown `transportType` say, to exercise the server-side path it describes | | HP6 | `errorCode` is a number | The raw header string, `'40101'` | Open bug, trivial | | HP8 | `headers` is a map | A list of `(name, value)` pairs, so the lookup the spec describes is impossible without converting, and case-insensitivity is lost | Open bug; changing the return type is breaking | | RSC15a | Six hosts tried | Three. `Options.__get_hosts` truncates to `http_max_retry_count`, which TO3l5 sanctions | Intentional | diff --git a/test/uts/realtime/unit/client/realtime_client_test.py b/test/uts/realtime/unit/client/realtime_client_test.py index b680edd5..049ef877 100644 --- a/test/uts/realtime/unit/client/realtime_client_test.py +++ b/test/uts/realtime/unit/client/realtime_client_test.py @@ -71,14 +71,10 @@ async def test_rtc12_constructor_string_detection(): async def test_rtc12_invalid_arguments_error(): mock_ws = succeeding_mock() - # DEVIATION: the spec expects error code 40106 when no valid credentials are - # provided. ably-python rejects the options in the constructor instead, with a - # plain ValueError that carries no Ably error code. This mirrors the REST suite's - # `RSC1b/no-auth-method-error-0`. - with pytest.raises(ValueError) as excinfo: + with pytest.raises(AblyException) as excinfo: realtime_client(mock_ws, key=None) - assert 'key is missing' in str(excinfo.value) + assert excinfo.value.code == 40106 assert mock_ws.connection_attempts == [] diff --git a/test/uts/rest/integration/push_admin_test.py b/test/uts/rest/integration/push_admin_test.py index 16720dd3..f997217b 100644 --- a/test/uts/rest/integration/push_admin_test.py +++ b/test/uts/rest/integration/push_admin_test.py @@ -68,15 +68,14 @@ async def test_rsh1a_push_publish_clientid(sandbox): async def test_rsh1a_push_publish_invalid_recipient(sandbox): client = sandbox_rest_client(sandbox.key(0).key_str) - # NOTE: the spec expects the server to reject the empty recipient and the - # error to carry a `code`. `PushAdmin.publish` validates the recipient - # itself (`ably/rest/push.py:57`) and raises `ValueError` before any request - # is made, so there is no server error and no code to read. See the RSH1a row - # under Adapted Tests -> REST behaviours asserted as they are, in - # test/uts/deviations.md. - with pytest.raises(ValueError): + # NOTE: the spec expects the server to reject the empty recipient. + # `PushAdmin.publish` rejects it itself, with the 40000 the server would + # give, before any request is made (RSH1a: "immediately rejected"). + with pytest.raises(AblyException) as excinfo: await client.push.admin.publish({}, {'notification': {'title': 'Test'}}) + assert excinfo.value.code is not None + # UTS: rest/integration/RSH1b3/save-and-get-device-0 async def test_rsh1b3_save_and_get_device(sandbox): diff --git a/test/uts/rest/unit/auth/auth_scheme_test.py b/test/uts/rest/unit/auth/auth_scheme_test.py index e69b732a..d51e94b2 100644 --- a/test/uts/rest/unit/auth/auth_scheme_test.py +++ b/test/uts/rest/unit/auth/auth_scheme_test.py @@ -189,13 +189,13 @@ async def test_rsc1b_no_auth_method_error(): on_request=capture_and_respond(captured_requests), ) - # DEVIATION: the spec expects an error with code 40106 from the first - # request. ably-python rejects the options in the constructor instead, with - # a plain ValueError that carries no Ably error code. - with pytest.raises(ValueError) as excinfo: + # The spec expects the error from the first request. RSC1b gives it to + # the constructor, and ably-python raises it there, so no client exists to + # make a request with. + with pytest.raises(AblyException) as excinfo: rest_client(mock_http, key=None) - assert 'key is missing' in str(excinfo.value) + assert excinfo.value.code == 40106 assert len(captured_requests) == 0 diff --git a/test/uts/rest/unit/push/push_admin_publish_test.py b/test/uts/rest/unit/push/push_admin_publish_test.py index 846baa70..f781e0c1 100644 --- a/test/uts/rest/unit/push/push_admin_publish_test.py +++ b/test/uts/rest/unit/push/push_admin_publish_test.py @@ -120,16 +120,13 @@ async def test_rsh1a_rejects_empty_recipient(): ) client = rest_client(mock_http) - # DEVIATION: the spec asserts error.code == 40000. PushAdmin.publish validates - # locally and raises a plain ValueError, which carries no Ably error code. - with pytest.raises(ValueError) as excinfo: + with pytest.raises(AblyException) as excinfo: await client.push.admin.publish( recipient={}, data={'notification': {'title': 'Test'}}, ) - assert not isinstance(excinfo.value, AblyException) - assert 'recipient' in str(excinfo.value) + assert excinfo.value.code == 40000 assert len(captured_requests) == 0 @@ -143,16 +140,13 @@ async def test_rsh1a_rejects_empty_data(): ) client = rest_client(mock_http) - # DEVIATION: the spec asserts error.code == 40000. PushAdmin.publish validates - # locally and raises a plain ValueError, which carries no Ably error code. - with pytest.raises(ValueError) as excinfo: + with pytest.raises(AblyException) as excinfo: await client.push.admin.publish( recipient={'clientId': 'user-123'}, data={}, ) - assert not isinstance(excinfo.value, AblyException) - assert 'data' in str(excinfo.value) + assert excinfo.value.code == 40000 assert len(captured_requests) == 0 @@ -166,16 +160,13 @@ async def test_rsh1a_rejects_null_recipient(): ) client = rest_client(mock_http) - # DEVIATION: the spec asserts error.code == 40000. PushAdmin.publish rejects a - # non-dict recipient with a plain TypeError, which carries no Ably error code. - with pytest.raises(TypeError) as excinfo: + with pytest.raises(AblyException) as excinfo: await client.push.admin.publish( recipient=None, data={'notification': {'title': 'Test'}}, ) - assert not isinstance(excinfo.value, AblyException) - assert 'recipient' in str(excinfo.value) + assert excinfo.value.code == 40000 assert len(captured_requests) == 0 From 174dc8470d3d4d820d6f328871fb9bc2e58d4ee3 Mon Sep 17 00:00:00 2001 From: owenpearson Date: Mon, 26 Oct 2026 18:52:24 +0000 Subject: [PATCH 32/71] pubsub: use token auth when a key comes with a means of getting a token A client given a key alongside an auth_callback, auth_url, token or token_details authenticated with basic auth: Auth.__init__ chose the scheme from use_token_auth and the presence of a key secret alone, so the callback and auth URL were never consulted, and the HTTP client's credential chain discarded a token or token_details given with a key. RSA4 selects token auth whenever use_token_auth is unspecified and any of those four is given, and RSA4a/RSA4b then treat the key as a means of renewing the token. The client now does that. The key is kept, so a token given with it can be renewed. use_token_auth=False still forces basic auth when a key is present, which RSA4 allows, since it only applies when use_token_auth is unspecified. This is a breaking change. A client configured with both now calls its auth_callback or auth_url, which reaches the customer's auth server, or presents the token it was given. Callers who relied on the key winning pass use_token_auth=False, or drop the unused option. The RSA1 token-precedence UTS test now asserts the specification's Bearer header. Co-Authored-By: Claude Opus 5.5 --- UPDATING.md | 23 +++++++++++++++++++++ ably/pubsub/http/auth.py | 7 ++++++- ably/pubsub/http/http.py | 11 +++++----- ably/pubsub/server/__init__.py | 6 ++++-- test/uts/deviations.md | 1 - test/uts/rest/unit/auth/auth_scheme_test.py | 9 ++------ 6 files changed, 41 insertions(+), 16 deletions(-) diff --git a/UPDATING.md b/UPDATING.md index 9865dc14..8de64bdc 100644 --- a/UPDATING.md +++ b/UPDATING.md @@ -222,6 +222,29 @@ except AblyException as e: ... ``` +### A key no longer overrides `auth_callback`, `auth_url` or a token + +A client given a `key` together with an `auth_callback`, `auth_url`, `token` or `token_details` +now uses token auth, and keeps the key only as a means of renewing the token. 3.x used basic +auth with the key whenever one was present, never called the `auth_callback` or `auth_url`, +and discarded a `token` or `token_details` given alongside the key. So a client configured this +way now makes requests to your auth server, or presents the token you gave it, where 3.x did +neither. + +To keep using basic auth with the key, pass `use_token_auth=False`: + +Example 3.x code: +```python +rest = AblyRest(key='xxx', auth_callback=callback) +``` + +Example 4.0.0 code: +```python +pubsub_http_client = create_http_client(key='xxx', auth_callback=callback, use_token_auth=False) +``` + +Or drop the `auth_callback`, `auth_url` or token, which 3.x never used. + ## Version 2.x to 3.0.0 The 3.0.0 version of ably-python introduces several breaking changes to improve the realtime experience and align the API with the Ably specification. These include: diff --git a/ably/pubsub/http/auth.py b/ably/pubsub/http/auth.py index f4a2a713..5707acc7 100644 --- a/ably/pubsub/http/auth.py +++ b/ably/pubsub/http/auth.py @@ -54,7 +54,12 @@ def __init__(self, ably: DefaultPubSubHttpClient | DefaultPubSubRealtimeClient, self.__token_details: TokenDetails | None = None self.__time_offset: int | None = None - must_use_token_auth = options.use_token_auth is True + # RSA4: token auth unless use_token_auth says otherwise, or nothing but a key is given + if options.use_token_auth is None: + must_use_token_auth = bool(options.auth_callback or options.auth_url + or options.auth_token or options.token_details) + else: + must_use_token_auth = options.use_token_auth must_not_use_token_auth = options.use_token_auth is False can_use_basic_auth = options.key_secret is not None if not must_use_token_auth and can_use_basic_auth: diff --git a/ably/pubsub/http/http.py b/ably/pubsub/http/http.py index ca9486cf..5c14f7bd 100644 --- a/ably/pubsub/http/http.py +++ b/ably/pubsub/http/http.py @@ -69,14 +69,15 @@ def __init__(self, key: Optional[str] = None, token: Optional[str] = None, if key is not None and ('key_name' in kwargs or 'key_secret' in kwargs): raise InvalidArgumentException("key and key_name or key_secret are mutually exclusive. " "Provider either a key or key_name & key_secret", 401, 40106) - if key is not None: - options = Options(key=key, **kwargs) - elif token is not None: - options = Options(auth_token=token, **kwargs) + # RSA4: a key given alongside a token is kept, to renew the token with + if token is not None: + options = Options(key=key, auth_token=token, **kwargs) elif token_details is not None: if not isinstance(token_details, TokenDetails): raise InvalidArgumentException("token_details must be an instance of TokenDetails", 401, 40106) - options = Options(token_details=token_details, **kwargs) + options = Options(key=key, token_details=token_details, **kwargs) + elif key is not None: + options = Options(key=key, **kwargs) elif not ('auth_callback' in kwargs or 'auth_url' in kwargs or # and don't have both key_name and key_secret ('key_name' in kwargs and 'key_secret' in kwargs)): diff --git a/ably/pubsub/server/__init__.py b/ably/pubsub/server/__init__.py index 10526d5f..e988d5d3 100644 --- a/ably/pubsub/server/__init__.py +++ b/ably/pubsub/server/__init__.py @@ -93,8 +93,10 @@ def create_http_client(**kwargs) -> PubSubHttpClient: auth_params: dict Query parameters to include in the `auth_url` request. use_token_auth: bool - Force token authentication even when an API key is available. By default the - client uses basic auth where a key is present. + Whether to use token authentication. By default the client uses token auth when + any of `auth_callback`, `auth_url`, `token` or `token_details` is given, and + basic auth when only a key is. Set it to False to authenticate with the key + regardless, or to True to request tokens with the key. query_time: bool Query the Ably service for the current time when signing token requests, rather than trusting the local clock. Defaults to False. diff --git a/test/uts/deviations.md b/test/uts/deviations.md index 96654a0d..5147427d 100644 --- a/test/uts/deviations.md +++ b/test/uts/deviations.md @@ -1895,7 +1895,6 @@ which is about a client that *does* configure `clientId: "*"`. | Spec points | Behaviour | |---|---| -| RSA4 | With a `key` present, `auth_callback` and `auth_url` are ignored when choosing the auth scheme, so Basic is selected and the callback is never called. `Auth.__init__` considers only `use_token_auth` and `key_secret`. `AblyRest.__init__`'s credential `elif` chain compounds it by discarding `token`/`token_details` when a key is given | | RSA15a, RSA15c | A mismatch between `ClientOptions.clientId` and a statically supplied `TokenDetails.clientId` is never detected. `Auth.__init__` only falls back to the token's clientId; `_configure_client_id`, which would raise, is reached only after a *fetched* token | | RSA12a | A token with a **null** clientId is rejected when `ClientOptions.clientId` is set, with 40102 "Client ID cannot be changed to 'None'". RSA15a constrains only non-wildcard token clientIds. Needs a `new_client_id is not None` guard | | RSA7, RSA16c | A clientId learned from a token is treated as immutable, so `authorize()` to a token with a different clientId raises 40102. RSA15 scopes immutability to a clientId set in `ClientOptions`. `_configure_client_id` uses `self.client_id or self.auth_options.client_id`, conflating the two. Possibly deliberate — worth a maintainer's call | diff --git a/test/uts/rest/unit/auth/auth_scheme_test.py b/test/uts/rest/unit/auth/auth_scheme_test.py index d51e94b2..ba701ca2 100644 --- a/test/uts/rest/unit/auth/auth_scheme_test.py +++ b/test/uts/rest/unit/auth/auth_scheme_test.py @@ -241,14 +241,9 @@ async def auth_callback(params): await client.request('GET', '/channels/test', version=api_version) - # DEVIATION: RSA4 selects token auth whenever use_token_auth is unspecified - # and any of auth_url, auth_callback, token or token_details is given, so - # the spec expects `bearer('callback-token')`. ably-python prefers basic auth - # whenever a key secret is present and use_token_auth is not True, and never - # calls the auth_callback. + # authCallback takes precedence, so Bearer auth is used request = captured_requests[0] - expected_basic = 'Basic ' + base64.b64encode(b'appId.keyId:keySecret').decode('ascii') - assert request.headers['Authorization'] == expected_basic + assert request.headers['Authorization'] == bearer('callback-token') # UTS: rest/unit/RSA2/basic-auth-header-format-0 From 165323cfc474392b979bc33525d80ac675bbcf89 Mon Sep 17 00:00:00 2001 From: owenpearson Date: Mon, 26 Oct 2026 18:55:33 +0000 Subject: [PATCH 33/71] pubsub: send token params to an auth_url under their wire names token_request_from_auth_url passed the SDK's internal token params dict straight to the auth URL, as the query string for GET and the form body for POST. An auth server therefore received client_id where RSA8c1a and every other SDK send clientId, a capability dict as its Python repr, a timedelta ttl as "1:00:00", and an unset param as an empty value. Issue #736 reports the client_id half: an auth server written against the docs or another SDK never sees the client ID and issues a token without one. The params are now sent under their wire names and in their wire forms: client_id becomes clientId, a capability that is not already a JSON string is serialised as canonical JSON, a timedelta ttl becomes integer milliseconds, and None values are left out. timestamp and nonce already travel under their wire names, and auth_params and any other key pass through unchanged. Token params still take precedence over auth_params on a name conflict (RSA8c2), now under the wire name. This is a breaking change for auth servers: one that reads client_id from a 3.x client must read clientId instead. There is no transition period that sends both. The names callers use in token_params are unchanged. A mocked test covers GET and POST, and the RSA12b UTS test's deviation note drops the half this fixes; it stays gated on the RSA12a clientId check. Co-Authored-By: Claude Opus 5.5 --- UPDATING.md | 25 ++++++++++++ ably/pubsub/http/auth.py | 18 +++++++++ test/ably/http/authurlparams_test.py | 47 +++++++++++++++++++++++ test/uts/deviations.md | 1 - test/uts/rest/unit/auth/client_id_test.py | 13 ++----- 5 files changed, 94 insertions(+), 10 deletions(-) create mode 100644 test/ably/http/authurlparams_test.py diff --git a/UPDATING.md b/UPDATING.md index 8de64bdc..01e049ae 100644 --- a/UPDATING.md +++ b/UPDATING.md @@ -245,6 +245,31 @@ pubsub_http_client = create_http_client(key='xxx', auth_callback=callback, use_t Or drop the `auth_callback`, `auth_url` or token, which 3.x never used. +### Token params reach an `auth_url` under their wire names + +A client with an `auth_url` now sends the token params to it under the names and in the forms +the other Ably SDKs use, in the query string for `GET` and in the form body for `POST`: + +- the client ID as `clientId`, where 3.x sent `client_id` +- a `capability` given as a dict or `Capability` as JSON, where 3.x sent its Python repr +- a `ttl` given as a `timedelta` as an integer count of milliseconds, where 3.x sent `1:00:00` + +Unset params are left out rather than sent empty. This changes what your auth server receives. +An auth server written for 3.x clients that reads `client_id` must read `clientId` instead, as it +already does for clients built with the other SDKs. There is no transition period in which both +are sent, so update the auth server before, or together with, the clients. The token params +passed to `create_token_request`, `request_token` and `authorize` keep their snake_case names. + +Example 3.x auth server code: +```python +client_id = request.args.get('client_id') +``` + +Example 4.0.0 auth server code: +```python +client_id = request.args.get('clientId') +``` + ## Version 2.x to 3.0.0 The 3.0.0 version of ably-python introduces several breaking changes to improve the realtime experience and align the API with the Ably specification. These include: diff --git a/ably/pubsub/http/auth.py b/ably/pubsub/http/auth.py index 5707acc7..82391c2c 100644 --- a/ably/pubsub/http/auth.py +++ b/ably/pubsub/http/auth.py @@ -409,11 +409,29 @@ def _timestamp(self): def _random_nonce(self): return uuid.uuid4().hex[:16] + @staticmethod + def _auth_url_token_params(token_params): + """Returns the token params under their wire names and in their wire forms (RSA8c1a)""" + params = {} + for name, value in token_params.items(): + if value is None: + continue + if name == 'client_id': + name = 'clientId' + elif name == 'ttl' and isinstance(value, timedelta): + value = int(value.total_seconds() * 1000) + elif name == 'capability' and not isinstance(value, str): + value = str(Capability(value)) + params[name] = value + return params + async def token_request_from_auth_url(self, method: str, url: str, token_params, headers, auth_params): # Extract URL parameters using utility function clean_url, url_params = extract_url_params(url) + token_params = self._auth_url_token_params(token_params) + body = None params = None if method == 'GET': diff --git a/test/ably/http/authurlparams_test.py b/test/ably/http/authurlparams_test.py new file mode 100644 index 00000000..b1f6321d --- /dev/null +++ b/test/ably/http/authurlparams_test.py @@ -0,0 +1,47 @@ +from datetime import timedelta +from urllib.parse import parse_qs + +import pytest +import respx +from httpx import Response + +from ably.pubsub.server import create_http_client + + +@pytest.fixture(scope='session', autouse=True) +def test_app_setup(): + """Avoid creating a sandbox application for these mocked requests.""" + yield + + +@pytest.mark.parametrize('method', ['GET', 'POST']) +@respx.mock +async def test_token_params_reach_auth_url_under_wire_names(method): + """RSA8c1a: send TokenParams to the auth URL as another SDK would.""" + auth_url = 'https://auth.example.test/token' + client = create_http_client(auth_url=auth_url, auth_method=method, client_id='alice', + auth_params={'clientId': 'overridden', 'extra': 'kept'}) + auth_route = respx.request(method, auth_url).mock(return_value=Response( + 200, json={'token': 'a-token', 'clientId': 'alice'})) + try: + await client.auth.authorize({ + 'ttl': timedelta(hours=1), + 'capability': {'channel': ['subscribe', 'publish']}, + 'nonce': 'a-nonce', + 'timestamp': 1234567890000, + }) + request = auth_route.calls.last.request + if method == 'GET': + sent = parse_qs(request.url.query.decode('utf-8')) + else: + sent = parse_qs(request.content.decode('utf-8')) + assert sent == { + 'clientId': ['alice'], + 'ttl': ['3600000'], + 'capability': ['{"channel": ["publish", "subscribe"]}'], + 'nonce': ['a-nonce'], + 'timestamp': ['1234567890000'], + 'extra': ['kept'], + } + finally: + await client.close() diff --git a/test/uts/deviations.md b/test/uts/deviations.md index 5147427d..541be489 100644 --- a/test/uts/deviations.md +++ b/test/uts/deviations.md @@ -1904,7 +1904,6 @@ which is about a client that *does* configure `clientId: "*"`. | RSA16c | No local expiry detection without a server time offset, which an authCallback client never obtains. See the RSA4b1 spec error above; here the specification asserts renewal *does* happen, so there is no green reading | | RSA16d | A failed renewal leaves the invalidated token in place — `_ensure_valid_auth_credentials` assigns only on success | | RSA16d | `authorize()` cannot switch a client back to basic auth: `_ensure_valid_auth_credentials` sets `Method.TOKEN` unconditionally, and `AuthOptions.replace` drops `use_token_auth`, which is stored outside the options dict | -| RSA8c1a, RSA12b | `TokenParams` reach the `auth_url` under the SDK's internal snake_case names: `Auth._ensure_valid_auth_credentials` sets `token_params['client_id']` and `token_request_from_auth_url` passes the dict straight to the query string, so an auth server sees `client_id`, not `clientId` | ### Options diff --git a/test/uts/rest/unit/auth/client_id_test.py b/test/uts/rest/unit/auth/client_id_test.py index 58a2990e..7c293d48 100644 --- a/test/uts/rest/unit/auth/client_id_test.py +++ b/test/uts/rest/unit/auth/client_id_test.py @@ -125,15 +125,10 @@ async def mock_auth_callback(params): # UTS: rest/unit/RSA12b/clientid-sent-to-authurl-0 -# DEVIATION: two departures, either of which alone fails this test. -# - The specification's auth_url response carries no clientId, and -# Auth._configure_client_id reads the absent value as an attempt to change the -# configured clientId, raising 40102 before `status()` returns. This is the -# RSA12a mechanism above. -# - RSA8c1a sends the TokenParams as query params under their wire names. -# ably-python passes its internal snake_case dict straight through -# (`token_params['client_id']` in Auth._ensure_valid_auth_credentials), so the -# auth_url receives `client_id`, not `clientId`. +# DEVIATION: the specification's auth_url response carries no clientId, and +# Auth._configure_client_id reads the absent value as an attempt to change the +# configured clientId, raising 40102 before `status()` returns. This is the RSA12a +# mechanism above. @deviation async def test_rsa12b_clientid_sent_to_authurl(): captured_requests = [] From 601eb540a710e721f0bff18bc6d05e75c392f7b8 Mon Sep 17 00:00:00 2001 From: owenpearson Date: Mon, 26 Oct 2026 19:00:12 +0000 Subject: [PATCH 34/71] pubsub: hold a realtime client's configured clientId before CONNECTED Auth.__init__ set auth.client_id to None on a realtime client, deferring it to the clientId a CONNECTED message confirms. So client.client_id and client.auth.client_id disagreed until the connection was established (RTC17), and _ensure_valid_auth_credentials, which only puts a non-null client_id into the token params, called the first auth_callback or auth_url without the configured clientId (RTN2e, RSA7d, RSA12a). The REST client did neither. A realtime client now initialises its clientId as the REST client does: the configured clientId, or failing that the clientId of a supplied TokenDetails (RSA7b1, RSA7b2). CONNECTED still validates it through _configure_client_id, which keeps a configured clientId against a wildcard and rejects a conflicting one with 40102 (RSA15a). This is a breaking change. auth.client_id is no longer None before CONNECTED on a client with a configured clientId, and the first token request now carries that clientId, so an auth_callback receives client_id and an auth server receives clientId where 3.x sent neither. An auth server that issues tokens from the params it receives now issues them for that clientId. The RTN2e UTS test is un-gated, the RTC17 one asserts the specification's equality, and three legacy realtime auth tests that pinned None before CONNECTED now expect the clientId. Co-Authored-By: Claude Opus 5.5 --- UPDATING.md | 16 +++++++ ably/pubsub/http/auth.py | 10 ++--- ably/pubsub/realtime/presence.py | 2 +- test/ably/realtime/realtimeauth_test.py | 9 ++-- test/uts/deviations.md | 44 +++++-------------- .../unit/auth/connection_auth_test.py | 5 +-- .../unit/client/realtime_client_test.py | 6 +-- 7 files changed, 39 insertions(+), 53 deletions(-) diff --git a/UPDATING.md b/UPDATING.md index 01e049ae..06907d94 100644 --- a/UPDATING.md +++ b/UPDATING.md @@ -270,6 +270,22 @@ Example 4.0.0 auth server code: client_id = request.args.get('clientId') ``` +### A realtime client knows its configured `client_id` before it connects + +A realtime client built with a `client_id` now reports it from `auth.client_id` as soon as it is +created, as an HTTP client always has. 3.x held `auth.client_id` at `None` until the connection +reached `CONNECTED`. A realtime client built with a `token_details` that carries a client ID +likewise reports that ID straight away. + +Because the client ID is known when the first token is requested, a realtime client now passes +it to its `auth_callback` as `client_id` in the token params, and to its `auth_url` as +`clientId`, which 3.x did not do. An auth server or callback that issues tokens from those params +now issues them for that client ID. Check that it does not reject or override a client ID it was +not expecting. + +Code that read `auth.client_id` before connecting to tell whether the client was connected +should check `connection.state` instead. + ## Version 2.x to 3.0.0 The 3.0.0 version of ably-python introduces several breaking changes to improve the realtime experience and align the API with the Ably specification. These include: diff --git a/ably/pubsub/http/auth.py b/ably/pubsub/http/auth.py index 82391c2c..24ebacb9 100644 --- a/ably/pubsub/http/auth.py +++ b/ably/pubsub/http/auth.py @@ -41,12 +41,10 @@ def __init__(self, ably: DefaultPubSubHttpClient | DefaultPubSubRealtimeClient, self.__auth_options = options self.__clock = select_clock(options) - if not self.ably._is_realtime: - self.__client_id = options.client_id - if not self.__client_id and options.token_details: - self.__client_id = options.token_details.client_id - else: - self.__client_id = None + # RSA7b1, RTC17: a configured clientId holds from the start, on realtime clients too + self.__client_id = options.client_id + if not self.__client_id and options.token_details: + self.__client_id = options.token_details.client_id self.__client_id_validated: bool = False self.__basic_credentials: str | None = None diff --git a/ably/pubsub/realtime/presence.py b/ably/pubsub/realtime/presence.py index 60b3b211..70e21774 100644 --- a/ably/pubsub/realtime/presence.py +++ b/ably/pubsub/realtime/presence.py @@ -27,7 +27,7 @@ def _get_client_id(presence: RealtimePresence) -> str | None: """Get the clientId for the current connection.""" - # Use auth.client_id if available (set after CONNECTED), + # Use auth.client_id if available, # otherwise fall back to auth_options.client_id return presence.channel.ably.auth.client_id or presence.channel.ably.auth.auth_options.client_id diff --git a/test/ably/realtime/realtimeauth_test.py b/test/ably/realtime/realtimeauth_test.py index da073117..2d5399fb 100644 --- a/test/ably/realtime/realtimeauth_test.py +++ b/test/ably/realtime/realtimeauth_test.py @@ -636,7 +636,8 @@ async def test_auth_client_id_mismatch(self): realtime = await TestApp.get_ably_realtime(token_details=token_details, client_id="WRONG") - assert realtime.auth.client_id is None + # RTC17: the configured clientId holds before CONNECTED + assert realtime.auth.client_id == "WRONG" state_change = await realtime.connection.once_async(ConnectionState.FAILED) @@ -655,7 +656,8 @@ async def test_auth_client_id_wildcard_token(self): realtime = await TestApp.get_ably_realtime(token_details=token_details, client_id=client_id) - assert realtime.auth.client_id is None + # RTC17: the configured clientId holds before CONNECTED + assert realtime.auth.client_id == client_id await realtime.connection.once_async(ConnectionState.CONNECTED) @@ -674,7 +676,8 @@ async def test_auth_client_id_inheritance_token(self): realtime = await TestApp.get_ably_realtime(token_details=token_details) - assert realtime.auth.client_id is None + # RSA7b2: the token's clientId holds before CONNECTED confirms it + assert realtime.auth.client_id == client_id await realtime.connection.once_async(ConnectionState.CONNECTED) diff --git a/test/uts/deviations.md b/test/uts/deviations.md index 541be489..38540f33 100644 --- a/test/uts/deviations.md +++ b/test/uts/deviations.md @@ -33,8 +33,8 @@ parametrized over a table of fixtures the specification gives inline; and five `objects/unit` tests are parametrized, three over a table and two over the scenarios the specification loops through. That turns 1481 derived tests into 1608 pytest cases. -Of **1472 Test IDs, derived as 1481 tests and run as 1608 pytest cases**: 1293 Test IDs -(1302 tests, 1427 cases) pass, 164 (164 tests, 166 cases) are gated behind +Of **1472 Test IDs, derived as 1481 tests and run as 1608 pytest cases**: 1294 Test IDs +(1303 tests, 1428 cases) pass, 163 (163 tests, 165 cases) are gated behind `RUN_DEVIATIONS`, and 15 (15 tests, 15 cases) cannot be run at all. The three groups are disjoint: two Test IDs have a gated part and a passing part — `TI1`, as two functions, and `TO/endpoint-affects-host`, as one parametrized test — and are counted with the gated, @@ -45,19 +45,19 @@ the Test IDs come from `uts/rest/unit` (503 tests, 536 cases), 482 from `uts/rest/integration` (84, 122), 73 from `uts/realtime/integration` (73, 95) and 17 from `uts/objects/integration` (17, 29); 8 of the REST integration ids (8, 8), 30 of the realtime ones (30, 30) and 5 of the objects ones (5, 5) come from the `proxy` package -within each. Of the gated Test IDs 79 are REST and 85 realtime, which is 80 REST cases -and 86 realtime. None of the 339 objects Test IDs is gated or unrunnable. +within each. Of the gated Test IDs 79 are REST and 84 realtime, which is 80 REST cases +and 85 realtime. None of the 339 objects Test IDs is gated or unrunnable. A further 148 pytest cases cover the harness itself and are not derived from a specification: 130 under `helpers/`, for the mocks, and 18 under `objects/helpers/`, for the objects builders and the synced-channel mock. -The 153 gated Test IDs that record SDK non-compliance — 153 tests, 155 cases — reduce to -**68 distinct root causes**, 26 on the REST side and 42 on the realtime side. Three further +The 152 gated Test IDs that record SDK non-compliance — 152 tests, 154 cases — reduce to +**67 distinct root causes**, 26 on the REST side and 41 on the realtime side. Three further defects are recorded below with no test of their own, because the specification's test cannot discriminate (RTP18a), has nothing to assert against (the timezone split on synthesized LEAVE timestamps), or is worked around in the setup of every test that would otherwise trip over it (`enterClient` on an anonymous connection), so the file -carries **71 SDK root causes** in all. The remaining 11 gated Test IDs are +carries **70 SDK root causes** in all. The remaining 11 gated Test IDs are specification faults, and reduce to 8. The objects tier records no SDK non-compliance; its specification faults are all corrected or worked around in tests that pass, and reduce to the seven candidate issues O.1–O.7, none of them filed. @@ -1751,22 +1751,6 @@ failure builds no 80019, and `connection.errorReason` stays as it was. ably-python's behaviour the correct one, in which case this entry closes as a spec change rather than a fix. -#### TokenParams passed to an authCallback carry no clientId on a realtime client — 1 test - -**Spec points:** RSA12a, RTN2e. - -`Auth.__init__` (`ably/rest/auth.py:36-41`) sets `self.__client_id = None` when -`ably._is_realtime`, deferring the clientId to the CONNECTED `connectionDetails`. -`_ensure_valid_auth_credentials` only adds `token_params['client_id']` when -`self.client_id is not None`, so an authCallback on a realtime client is called with the -clientId missing entirely, even though `ClientOptions.clientId` was set. The REST client does -pass it, so the SDK is inconsistent with itself. The snake_case key is idiomatic translation, -not the deviation; the absent member is. - -**Tests affected:** `test_rtn2e_callback_params_include_clientid` — `KeyError: 'client_id'`. - -**Status:** open bug. - #### RSA4f invalid-format validation is not implemented — 1 test **Spec points:** RSA4f, RSA4c2. @@ -2328,7 +2312,6 @@ pseudocode is not mistaken for non-compliance. | RTN15, RTN23 | A DISCONNECTED `ErrorInfo` needs no `statusCode` | `ConnectionManager.on_disconnected` evaluates `exception.status_code >= 500` unguarded, so a DISCONNECTED whose error omits `statusCode` raises `TypeError` in a task whose exception is only logged, and the connection silently stays CONNECTED. `DISCONNECTED_MESSAGE` supplies 400 | Open bug | | TO3l1, TO3l5 | `httpRequestTimeout` and `httpMaxRetryCount` carry their defaults on the options object | Left unset; the effective defaults are applied downstream by `Http` and by `Options.__get_hosts`. The spec's `httpRequestTimeout` is milliseconds and ably-python's `http_request_timeout` is seconds, on the value a caller passes as much as on the default — gated, with the measurement, under *`httpRequestTimeout` is seconds where the specification counts milliseconds* in Failing Tests | Intentional for where the defaults are applied; the unit is an open bug, recorded there | | RTC7 (TO3l3, TO3l4) | `client.options.httpOpenTimeout == 4000` and `httpRequestTimeout == 10000` | Both `None` on `Options`; `Http.http_open_timeout` / `http_request_timeout` fall back to `CONNECTION_RETRY_DEFAULTS`, which holds 4 and 10 — seconds, where TO3l3 and TO3l4 count milliseconds. The three realtime timeouts the same test checks are defaulted on `Options` and match | Two faults in one row: the defaults are unreadable from `options`, and the unit reaches the wire — `rest/proxy/RSC15l2/timeout-triggers-fallback-0` measures a request outliving its configured timeout by a factor of a thousand. Both open; the unit is gated under *`httpRequestTimeout` is seconds where the specification counts milliseconds* in Failing Tests | -| RTC17 (RSA7b1) | `client.clientId == client.auth.clientId` | `AblyRealtime.client_id` reads `options.client_id` and returns the configured value, while `Auth.__init__` sets `self.__client_id = None` whenever `ably._is_realtime` (`rest/auth.py:34-41`), deferring it to whatever a CONNECTED confirms. The two disagree on a client that has not connected | Open bug. RSA12b only allows the realtime clientId to be unknown while it has not been *configured* | | RTC1f | a `transportParams` boolean appears as `"true"` / `"false"` | `True` / `False`, because `WebSocketTransport.connect` builds the query string with `urllib.parse.urlencode`, which renders each value through `str()` (`websockettransport.py:89`). Integers are unaffected | Open bug. A caller can pass the strings directly, but a bool is what the spec's Stringifiable type admits | ### LiveObjects: the internal shapes the unit specifications assume — S-1 to S-6 @@ -2951,14 +2934,7 @@ was. **Hold this one**: specification#466 would make the current behaviour corre two UTS specs disagree about it today. `test/uts/realtime/unit/auth/connection_auth_test.py -k rsa4c3_callback_error_stays_connected` -**3.16 TokenParams passed to an authCallback carry no clientId on a realtime client.** -RSA12a, RTN2e. `Auth.__init__` nulls `client_id` for a realtime client (`rest/auth.py:36-41`) -and `_ensure_valid_auth_credentials` only adds it when non-null, so an auth server never -learns the configured clientId. The REST client does pass it. The same nulling is why -`client.client_id` and `client.auth.client_id` disagree before CONNECTED (RTC17). -`test/uts/realtime/unit/auth/connection_auth_test.py -k rtn2e` - -**3.17 No 40171 log at instantiation with a non-renewable token.** RSA4a1. Requires an +**3.16 No 40171 log at instantiation with a non-renewable token.** RSA4a1. Requires an info-level record carrying 40171 and the TI5 help URL; the SDK logs at debug with no code, and `grep -rn href ably/` finds no help URLs at all. Small, and the TI5 half is a gap of its own. @@ -3651,8 +3627,8 @@ The header states how many derived tests there are, how many pass, how many are gated and how many cannot run. Those numbers are the check that the file is still true: in pytest cases, the gated count must equal the number of failures under `RUN_DEVIATIONS=1`, and gated plus unrunnable must equal the number of skips without -it. As of this writing that is 166 failures, 1575 passes and 15 skips with the variable -set, and 181 skips and 1575 passes without it, the 1575 being 1427 derived cases and 148 +it. As of this writing that is 165 failures, 1576 passes and 15 skips with the variable +set, and 180 skips and 1576 passes without it, the 1576 being 1428 derived cases and 148 harness ones — 130 under `helpers/` and 18 under `objects/helpers/`. Those figures need the `submodules` checkout CI makes: without it, three `rest/unit/encoding` tests that read the ably-common fixtures skip as well. The tiers are told apart by the junit report's diff --git a/test/uts/realtime/unit/auth/connection_auth_test.py b/test/uts/realtime/unit/auth/connection_auth_test.py index 5f935696..3ce4fd42 100644 --- a/test/uts/realtime/unit/auth/connection_auth_test.py +++ b/test/uts/realtime/unit/auth/connection_auth_test.py @@ -89,7 +89,6 @@ def on_connection_attempt(conn): # UTS: realtime/unit/RTN2e/callback-params-include-clientid-2 -@deviation async def test_rtn2e_callback_params_include_clientid(): received_params = None @@ -107,9 +106,7 @@ async def auth_callback(params): assert received_params is not None # `TokenParams` is a plain dict here, and ably-python spells its members # snake_case throughout, so `client_id` is the idiomatic rendering of the - # specification's `clientId`. The deviation is that the member is absent: - # `Auth.__init__` leaves `client_id` unset on a realtime client, so - # `_ensure_valid_auth_credentials` never puts it in the token params. + # specification's `clientId`. assert received_params['client_id'] == 'my-client-id' diff --git a/test/uts/realtime/unit/client/realtime_client_test.py b/test/uts/realtime/unit/client/realtime_client_test.py index 049ef877..fd9bfa1d 100644 --- a/test/uts/realtime/unit/client/realtime_client_test.py +++ b/test/uts/realtime/unit/client/realtime_client_test.py @@ -139,11 +139,7 @@ async def test_rtc17_client_id_attribute(): assert client.client_id == 'explicit-client-id' - # DEVIATION: the spec asserts `client.clientId == client.auth.clientId`. - # `DefaultPubSubRealtimeClient.client_id` reads the client options, while `Auth.client_id` is - # held at None for a realtime client until the server confirms one in a CONNECTED - # message, so the two disagree before the connection is established. - assert client.auth.client_id is None + assert client.client_id == client.auth.client_id # UTS: realtime/unit/RTC1a/echo-messages-option-0 From 3a0cc53a70e8558fbede777ff14db4b5bf8b2b3c Mon Sep 17 00:00:00 2001 From: owenpearson Date: Mon, 26 Oct 2026 19:03:23 +0000 Subject: [PATCH 35/71] pubsub: keep an explicit clientId in authorize() and copy the params _ensure_valid_auth_credentials wrote the client's clientId into the token params whenever it had one, over any client_id the caller passed, and did so in the caller's own dict. So authorize({'client_id': 'b'}) on a client configured as 'a' silently requested a token for 'a', and the caller's dict came back changed. RSA10b and RSA10j have explicit TokenParams supersede the configured ones, and RSA10h makes Auth#clientId the value used only by default. authorize() now copies the params it is given and fills in the client's clientId only when they name none. Params taken from default_token_params still have the configured clientId written over theirs, which RSA7a4 requires. A conflicting explicit clientId now yields a token for that clientId, which _configure_client_id rejects with 40102 as for any token whose clientId conflicts with the configured one (RSA15a). This is a breaking change for a caller that passed a clientId other than the configured one: authorize() now fails with 40102 where it used to succeed with a token for the configured clientId. Such callers drop the explicit clientId or pass the configured one. The RSA10b UTS test's deviation note now names only the RSA12a check that still gates it, and two mocked tests cover the copy and the explicit clientId. Co-Authored-By: Claude Opus 5.5 --- UPDATING.md | 22 +++++++++++ ably/pubsub/http/auth.py | 7 +++- test/ably/http/authorizeparams_test.py | 45 +++++++++++++++++++++++ test/uts/deviations.md | 1 - test/uts/rest/unit/auth/authorize_test.py | 10 ++--- 5 files changed, 77 insertions(+), 8 deletions(-) create mode 100644 test/ably/http/authorizeparams_test.py diff --git a/UPDATING.md b/UPDATING.md index 06907d94..f2fb9aeb 100644 --- a/UPDATING.md +++ b/UPDATING.md @@ -286,6 +286,28 @@ not expecting. Code that read `auth.client_id` before connecting to tell whether the client was connected should check `connection.state` instead. +### `authorize()` keeps an explicit `client_id` + +A `client_id` passed in the token params to `auth.authorize()` now reaches the token request, the +`auth_callback` or the `auth_url` as given. The client's own client ID is only filled in when the +params name none. 3.x replaced an explicit `client_id` with the client's own whenever the client +had one, and wrote it into the dict the caller passed. The caller's dict is no longer modified. + +So a client configured with one client ID that authorizes for another now receives a token for +the one it asked for, and then fails with an `IncompatibleClientIdException` (code 40102), as the +client ID of a client cannot change once configured. 3.x silently issued a token for the +configured client ID instead. Drop the explicit `client_id`, or pass the configured one: + +Example 3.x code: +```python +await rest.auth.authorize({'client_id': 'someone-else', 'ttl': 3600000}) +``` + +Example 4.0.0 code: +```python +await pubsub_http_client.auth.authorize({'ttl': 3600000}) +``` + ## Version 2.x to 3.0.0 The 3.0.0 version of ably-python introduces several breaking changes to improve the realtime experience and align the API with the Ably specification. These include: diff --git a/ably/pubsub/http/auth.py b/ably/pubsub/http/auth.py index 24ebacb9..6c36c6d3 100644 --- a/ably/pubsub/http/auth.py +++ b/ably/pubsub/http/auth.py @@ -120,16 +120,21 @@ async def __authorize_when_necessary(self, token_params=None, auth_options=None, async def _ensure_valid_auth_credentials(self, token_params=None, auth_options=None, force=False): self.__auth_mechanism = Auth.Method.TOKEN + explicit_client_id = None if token_params is None: token_params = dict(self.auth_options.default_token_params) else: + token_params = dict(token_params) + explicit_client_id = token_params.get('client_id') self.auth_options.default_token_params = dict(token_params) self.auth_options.default_token_params.pop('timestamp', None) if auth_options is not None: self.auth_options.replace(auth_options) auth_options = dict(self.auth_options.auth_options) - if self.client_id is not None: + # RSA10h: Auth#clientId is only the default for an explicit clientId (RSA10b, RSA10j), + # and takes precedence over one from default_token_params (RSA7a4) + if self.client_id is not None and explicit_client_id is None: token_params['client_id'] = self.client_id token_details = self.__token_details diff --git a/test/ably/http/authorizeparams_test.py b/test/ably/http/authorizeparams_test.py new file mode 100644 index 00000000..79bbb9bf --- /dev/null +++ b/test/ably/http/authorizeparams_test.py @@ -0,0 +1,45 @@ +import pytest + +from ably.pubsub.server import IncompatibleClientIdException, TokenDetails, create_http_client + + +@pytest.fixture(scope='session', autouse=True) +def test_app_setup(): + """Avoid creating a sandbox application for these offline calls.""" + yield + + +def callback_client(received, token_client_id): + async def auth_callback(token_params): + received.append(token_params) + return TokenDetails(token='a-token', client_id=token_client_id) + + return create_http_client(auth_callback=auth_callback, client_id='configured') + + +async def test_authorize_fills_the_configured_client_id_without_mutating_params(): + """RSA10h: Auth#clientId is the default clientId, applied to a copy of the caller's params.""" + received = [] + client = callback_client(received, 'configured') + token_params = {'ttl': 60000} + try: + await client.auth.authorize(token_params) + assert received[0] == {'ttl': 60000, 'client_id': 'configured'} + assert token_params == {'ttl': 60000} + finally: + await client.close() + + +async def test_authorize_keeps_an_explicit_client_id(): + """RSA10b, RSA10j: an explicit clientId reaches the callback, and a conflicting token fails 40102.""" + received = [] + client = callback_client(received, 'explicit') + token_params = {'client_id': 'explicit'} + try: + with pytest.raises(IncompatibleClientIdException) as excinfo: + await client.auth.authorize(token_params) + assert excinfo.value.code == 40102 + assert received[0]['client_id'] == 'explicit' + assert token_params == {'client_id': 'explicit'} + finally: + await client.close() diff --git a/test/uts/deviations.md b/test/uts/deviations.md index 38540f33..83851ef2 100644 --- a/test/uts/deviations.md +++ b/test/uts/deviations.md @@ -1882,7 +1882,6 @@ which is about a client that *does* configure `clientId: "*"`. | RSA15a, RSA15c | A mismatch between `ClientOptions.clientId` and a statically supplied `TokenDetails.clientId` is never detected. `Auth.__init__` only falls back to the token's clientId; `_configure_client_id`, which would raise, is reached only after a *fetched* token | | RSA12a | A token with a **null** clientId is rejected when `ClientOptions.clientId` is set, with 40102 "Client ID cannot be changed to 'None'". RSA15a constrains only non-wildcard token clientIds. Needs a `new_client_id is not None` guard | | RSA7, RSA16c | A clientId learned from a token is treated as immutable, so `authorize()` to a token with a different clientId raises 40102. RSA15 scopes immutability to a clientId set in `ClientOptions`. `_configure_client_id` uses `self.client_id or self.auth_options.client_id`, conflating the two. Possibly deliberate — worth a maintainer's call | -| RSA10b, RSA10h, RSA10j | `authorize()` overwrites an explicit `tokenParams.clientId`. `Auth._ensure_valid_auth_credentials` assigns `self.client_id` over whatever the caller passed, whenever the client has one. RSA10h makes it the default "if not null" | | RSA5c, RSA6c | `create_token_request()` ignores `default_token_params`. The merge lives in `Auth.request_token` only, so a direct call yields `ttl=None` and `capability=None`. RSA5, RSA5b, RSA5d, RSA6, RSA6b and RSA6d all pass, so the nullability requirement itself is met | | RSA16b | `TokenDetails` built from a bare token string fabricates `expires` (now plus an hour), `issued` (0) and `capability`. RSA16b requires only `token` to be populated. The invented expiry can drive spurious renewal | | RSA16c | No local expiry detection without a server time offset, which an authCallback client never obtains. See the RSA4b1 spec error above; here the specification asserts renewal *does* happen, so there is no green reading | diff --git a/test/uts/rest/unit/auth/authorize_test.py b/test/uts/rest/unit/auth/authorize_test.py index 5d1f50d3..e7aff225 100644 --- a/test/uts/rest/unit/auth/authorize_test.py +++ b/test/uts/rest/unit/auth/authorize_test.py @@ -99,12 +99,10 @@ async def mock_auth_callback(params): ) client = rest_client(mock_http, auth_callback=mock_auth_callback, client_id='default-client') - # DEVIATION: `Auth._ensure_valid_auth_credentials` overwrites the caller's - # `client_id` with `Auth.client_id` unconditionally, so the callback is handed - # 'default-client'. RSA10h has `Auth#clientId` supply a default only, and RSA10j - # has explicit `TokenParams` supersede configured ones. The token that comes - # back then carries no clientId, which `Auth._configure_client_id` rejects with - # IncompatibleClientIdException 40102 before the assertions are reached. + # DEVIATION: the token that comes back carries no clientId, which + # `Auth._configure_client_id` reads as an attempt to change the configured + # clientId and rejects with IncompatibleClientIdException 40102 before the + # assertions are reached. This is the RSA12a mechanism in client_id_test.py. await client.auth.authorize({'client_id': 'override-client', 'ttl': 7200000}) params = callback_params[0] From 1a931d53ea9aba33deeb1e791799104e5fb4b9f0 Mon Sep 17 00:00:00 2001 From: owenpearson Date: Mon, 26 Oct 2026 19:07:25 +0000 Subject: [PATCH 36/71] pubsub: reject a token_details whose clientId conflicts with client_id A client built with a client_id and a token_details naming a different, non-wildcard clientId was never checked: Auth.__init__ only fell back to the token's clientId when none was configured, and _configure_client_id, which raises on a conflict, runs only after a fetched token. An HTTP client then made requests under a token for the other clientId. RSA15a requires the configured clientId to match any non-wildcard clientId in the TokenDetails, and RSA15c requires an error response. Auth.__init__ now raises IncompatibleClientIdException with 40102, the code _configure_client_id already uses for the same conflict, when token auth is selected and the token's clientId is known and conflicts. The check runs at construction, which the RSA7 UTS test explicitly permits. A wildcard or absent token clientId passes (RSA15b). A bare token string carries no clientId the client can read, so it is checked where RSA15a says: against the CONNECTED connectionDetails on a realtime client, which already fails the connection with 40102. A REST client has nothing to check it against. This is a breaking change: such a client now fails to construct, and a realtime one no longer gets as far as connecting. Callers make the two clientIds agree or drop client_id. The RSA15a UTS test is un-gated, the RSA7 mismatch and RSA15c ones expect the error from the constructor, and the legacy realtime mismatch test expects it there too. Co-Authored-By: Claude Opus 5.5 --- UPDATING.md | 22 ++++++++++++++ ably/pubsub/http/auth.py | 6 ++++ ably/pubsub/server/__init__.py | 2 ++ test/ably/realtime/realtimeauth_test.py | 18 +++++------ test/uts/deviations.md | 17 +++++------ test/uts/rest/unit/auth/client_id_test.py | 37 ++++++++++------------- 6 files changed, 61 insertions(+), 41 deletions(-) diff --git a/UPDATING.md b/UPDATING.md index f2fb9aeb..786206b7 100644 --- a/UPDATING.md +++ b/UPDATING.md @@ -308,6 +308,28 @@ Example 4.0.0 code: await pubsub_http_client.auth.authorize({'ttl': 3600000}) ``` +### A `token_details` whose client ID conflicts with `client_id` is rejected + +Creating a client with both a `client_id` and a `token_details` whose own client ID is a different +one, and is not the wildcard `*`, now raises an `IncompatibleClientIdException` (code 40102) from +the factory. 3.x built the client: an HTTP client then sent requests under a token for the other +client ID, and a realtime client failed only once it connected. Make the two agree, or leave +`client_id` out and let the client take the token's. + +Example 3.x code: +```python +rest = AblyRest(token_details=token_details_for_bob, client_id='alice') +``` + +Example 4.0.0 code: +```python +pubsub_http_client = create_http_client(token_details=token_details_for_bob) +``` + +A bare `token` string carries no client ID the client can read, so it cannot be checked when the +client is created. A realtime client still checks it against the client ID the server reports +when the connection opens, and fails the connection with 40102 if they conflict. + ## Version 2.x to 3.0.0 The 3.0.0 version of ably-python introduces several breaking changes to improve the realtime experience and align the API with the Ably specification. These include: diff --git a/ably/pubsub/http/auth.py b/ably/pubsub/http/auth.py index 6c36c6d3..7a3b662b 100644 --- a/ably/pubsub/http/auth.py +++ b/ably/pubsub/http/auth.py @@ -77,6 +77,12 @@ def __init__(self, ably: DefaultPubSubHttpClient | DefaultPubSubRealtimeClient, if options.token_details: self.__token_details = options.token_details + # RSA15a, RSA15c: the token's clientId is known before any request, so check it now + token_client_id = options.token_details.client_id + if options.client_id is not None and token_client_id not in (None, '*', options.client_id): + raise IncompatibleClientIdException( + f"clientId '{options.client_id}' in the client options does not match the " + f"token's clientId '{token_client_id}'", 400, 40102) elif options.auth_token: self.__token_details = TokenDetails(token=options.auth_token) else: diff --git a/ably/pubsub/server/__init__.py b/ably/pubsub/server/__init__.py index e988d5d3..cb75bba7 100644 --- a/ably/pubsub/server/__init__.py +++ b/ably/pubsub/server/__init__.py @@ -169,6 +169,8 @@ def create_http_client(**kwargs) -> PubSubHttpClient: With code 40106, if no means of authenticating is given, if `key` is combined with `key_name` or `key_secret`, or if `endpoint` is combined with `environment`, `rest_host` or `realtime_host`. It is also a ValueError. + IncompatibleClientIdException + If `client_id` conflicts with the client ID of `token_details`. """ with factory_construction(): return _DefaultPubSubHttpClient(**kwargs) diff --git a/test/ably/realtime/realtimeauth_test.py b/test/ably/realtime/realtimeauth_test.py index 2d5399fb..781e379b 100644 --- a/test/ably/realtime/realtimeauth_test.py +++ b/test/ably/realtime/realtimeauth_test.py @@ -10,6 +10,7 @@ from ably.pubsub.types.channelstate import ChannelState from ably.pubsub.types.connectionstate import ConnectionEvent from ably.pubsub.types.tokendetails import TokenDetails +from ably.pubsub.util.exceptions import IncompatibleClientIdException from test.ably.testapp import TestApp from test.ably.utils import BaseAsyncTestCase, random_string @@ -625,25 +626,20 @@ async def auth_callback(_): await realtime.close() await rest.close() - # Rest token generation with client_id, then connecting with a - # different client_id, should fail with a library-generated message - # (RSA15a, RSA15c) + # Rest token generation with client_id, then constructing a client with a + # different client_id, should fail with a library-generated error before + # connecting (RSA15a, RSA15c) async def test_auth_client_id_mismatch(self): rest = await TestApp.get_ably_rest() client_id = 'test_client_id' token_details = await rest.auth.request_token({"client_id": client_id}) - realtime = await TestApp.get_ably_realtime(token_details=token_details, client_id="WRONG") + with pytest.raises(IncompatibleClientIdException) as excinfo: + await TestApp.get_ably_realtime(token_details=token_details, client_id="WRONG") - # RTC17: the configured clientId holds before CONNECTED - assert realtime.auth.client_id == "WRONG" - - state_change = await realtime.connection.once_async(ConnectionState.FAILED) + assert excinfo.value.code == 40102 - assert state_change.reason.code == 40102 - - await realtime.close() await rest.close() # Rest token generation with clientId '*', then connecting with just the diff --git a/test/uts/deviations.md b/test/uts/deviations.md index 83851ef2..265604bd 100644 --- a/test/uts/deviations.md +++ b/test/uts/deviations.md @@ -33,8 +33,8 @@ parametrized over a table of fixtures the specification gives inline; and five `objects/unit` tests are parametrized, three over a table and two over the scenarios the specification loops through. That turns 1481 derived tests into 1608 pytest cases. -Of **1472 Test IDs, derived as 1481 tests and run as 1608 pytest cases**: 1294 Test IDs -(1303 tests, 1428 cases) pass, 163 (163 tests, 165 cases) are gated behind +Of **1472 Test IDs, derived as 1481 tests and run as 1608 pytest cases**: 1297 Test IDs +(1306 tests, 1431 cases) pass, 160 (160 tests, 162 cases) are gated behind `RUN_DEVIATIONS`, and 15 (15 tests, 15 cases) cannot be run at all. The three groups are disjoint: two Test IDs have a gated part and a passing part — `TI1`, as two functions, and `TO/endpoint-affects-host`, as one parametrized test — and are counted with the gated, @@ -45,19 +45,19 @@ the Test IDs come from `uts/rest/unit` (503 tests, 536 cases), 482 from `uts/rest/integration` (84, 122), 73 from `uts/realtime/integration` (73, 95) and 17 from `uts/objects/integration` (17, 29); 8 of the REST integration ids (8, 8), 30 of the realtime ones (30, 30) and 5 of the objects ones (5, 5) come from the `proxy` package -within each. Of the gated Test IDs 79 are REST and 84 realtime, which is 80 REST cases +within each. Of the gated Test IDs 76 are REST and 84 realtime, which is 77 REST cases and 85 realtime. None of the 339 objects Test IDs is gated or unrunnable. A further 148 pytest cases cover the harness itself and are not derived from a specification: 130 under `helpers/`, for the mocks, and 18 under `objects/helpers/`, for the objects builders and the synced-channel mock. -The 152 gated Test IDs that record SDK non-compliance — 152 tests, 154 cases — reduce to -**67 distinct root causes**, 26 on the REST side and 41 on the realtime side. Three further +The 149 gated Test IDs that record SDK non-compliance — 149 tests, 151 cases — reduce to +**66 distinct root causes**, 25 on the REST side and 41 on the realtime side. Three further defects are recorded below with no test of their own, because the specification's test cannot discriminate (RTP18a), has nothing to assert against (the timezone split on synthesized LEAVE timestamps), or is worked around in the setup of every test that would otherwise trip over it (`enterClient` on an anonymous connection), so the file -carries **70 SDK root causes** in all. The remaining 11 gated Test IDs are +carries **69 SDK root causes** in all. The remaining 11 gated Test IDs are specification faults, and reduce to 8. The objects tier records no SDK non-compliance; its specification faults are all corrected or worked around in tests that pass, and reduce to the seven candidate issues O.1–O.7, none of them filed. @@ -1879,7 +1879,6 @@ which is about a client that *does* configure `clientId: "*"`. | Spec points | Behaviour | |---|---| -| RSA15a, RSA15c | A mismatch between `ClientOptions.clientId` and a statically supplied `TokenDetails.clientId` is never detected. `Auth.__init__` only falls back to the token's clientId; `_configure_client_id`, which would raise, is reached only after a *fetched* token | | RSA12a | A token with a **null** clientId is rejected when `ClientOptions.clientId` is set, with 40102 "Client ID cannot be changed to 'None'". RSA15a constrains only non-wildcard token clientIds. Needs a `new_client_id is not None` guard | | RSA7, RSA16c | A clientId learned from a token is treated as immutable, so `authorize()` to a token with a different clientId raises 40102. RSA15 scopes immutability to a clientId set in `ClientOptions`. `_configure_client_id` uses `self.client_id or self.auth_options.client_id`, conflating the two. Possibly deliberate — worth a maintainer's call | | RSA5c, RSA6c | `create_token_request()` ignores `default_token_params`. The merge lives in `Auth.request_token` only, so a direct call yields `ttl=None` and `capability=None`. RSA5, RSA5b, RSA5d, RSA6, RSA6b and RSA6d all pass, so the nullability requirement itself is met | @@ -3626,8 +3625,8 @@ The header states how many derived tests there are, how many pass, how many are gated and how many cannot run. Those numbers are the check that the file is still true: in pytest cases, the gated count must equal the number of failures under `RUN_DEVIATIONS=1`, and gated plus unrunnable must equal the number of skips without -it. As of this writing that is 165 failures, 1576 passes and 15 skips with the variable -set, and 180 skips and 1576 passes without it, the 1576 being 1428 derived cases and 148 +it. As of this writing that is 162 failures, 1579 passes and 15 skips with the variable +set, and 177 skips and 1579 passes without it, the 1579 being 1431 derived cases and 148 harness ones — 130 under `helpers/` and 18 under `objects/helpers/`. Those figures need the `submodules` checkout CI makes: without it, three `rest/unit/encoding` tests that read the ably-common fixtures skip as well. The tiers are told apart by the junit report's diff --git a/test/uts/rest/unit/auth/client_id_test.py b/test/uts/rest/unit/auth/client_id_test.py index 7c293d48..2a9a072c 100644 --- a/test/uts/rest/unit/auth/client_id_test.py +++ b/test/uts/rest/unit/auth/client_id_test.py @@ -201,19 +201,18 @@ async def test_rsa12_wildcard_clientid(): # UTS: rest/unit/RSA7/clientid-mismatch-error-1 -@deviation async def test_rsa7_clientid_mismatch_error(): # Case 2 of the spec's table: ClientOptions 'client-a' against a token - # naming 'client-b'. ably-python never compares the two for a statically - # supplied TokenDetails, so no error is reported at construction or at use. - client = rest_client(channel_details_mock(), client_id='client-a', - token_details=TokenDetails( - token='mismatched-token', - expires=now() + 3600000, - client_id='client-b', - )) - + # naming 'client-b'. The specification leaves the timing of the detection to + # the implementation, constructor or first use; ably-python detects it at + # construction, so the constructor is what fails. with pytest.raises(AblyException) as excinfo: + client = rest_client(channel_details_mock(), client_id='client-a', + token_details=TokenDetails( + token='mismatched-token', + expires=now() + 3600000, + client_id='client-b', + )) await client.channels.get('test').status() message = excinfo.value.message or '' @@ -221,7 +220,6 @@ async def test_rsa7_clientid_mismatch_error(): # UTS: rest/unit/RSA15a/token-clientid-must-match-0 -@deviation async def test_rsa15a_token_clientid_must_match(): client_match = rest_client(channel_details_mock(), client_id='my-client', token_details=TokenDetails( @@ -232,7 +230,6 @@ async def test_rsa15a_token_clientid_must_match(): assert client_match.auth.client_id == 'my-client' - # ably-python constructs this client without complaint. with pytest.raises(AblyException) as excinfo: rest_client(channel_details_mock(), client_id='my-client', token_details=TokenDetails( @@ -257,19 +254,17 @@ async def test_rsa15b_wildcard_token_permits_any(): # UTS: rest/unit/RSA15c/incompatible-clientid-error-0 -@deviation async def test_rsa15c_incompatible_clientid_error(): # The spec defers to the RSA15a mismatch case: a REST operation under a # TokenDetails whose clientId is incompatible must fail with code 40102. - # ably-python performs the operation instead. - client = rest_client(channel_details_mock(), client_id='client-a', - token_details=TokenDetails( - token='incompatible-token', - expires=now() + 3600000, - client_id='client-b', - )) - + # ably-python detects the mismatch at construction, as in RSA7 above. with pytest.raises(AblyException) as excinfo: + client = rest_client(channel_details_mock(), client_id='client-a', + token_details=TokenDetails( + token='incompatible-token', + expires=now() + 3600000, + client_id='client-b', + )) await client.channels.get('test').status() assert excinfo.value.code == 40102 From 60ddc183175833f1c23974b22443b2a6df92753c Mon Sep 17 00:00:00 2001 From: owenpearson Date: Mon, 26 Oct 2026 19:10:12 +0000 Subject: [PATCH 37/71] pubsub: make TokenDetails.capability the server's JSON string TokenDetails parsed its capability into a Capability mapping, so a token's capability could not be read as the JSON string TD5 and RSA16a describe, and a missing one became an empty Capability rather than None. Separately, Capability.add_operation_to_resource always raised AttributeError: Capability stores each resource's operations as a set, and the method called .append on it. TokenDetails.capability is now a str holding the capability exactly as the server sent it, or None. A dict or Capability passed to the constructor is serialised as canonical JSON, and to_dict() returns the string. Capability stays the builder for the capability token param and is still canonicalised before a TokenRequest is signed (RSA9f). add_operation_to_resource now adds the operation to the set. Why as received rather than canonicalised: TD5 says the attribute "contains the capability JSON stringified" and TC2 that "a string representation of the JSON will suffice", and nothing in features.md asks a client to canonicalise what the server returns; c14n appears only for TokenRequest signing. The UTS specs assert the literal server string (rest/unit/RSA16a/reflects-capability-1, rest/unit/TD/ token-details-from-json-0), which json.dumps canonicalisation would fail on its separators alone. ably-js (TokenDetails.capability: string, the response passed through JSON.parse), ably-java (a String deserialised by gson, Capability.c14n applied only to outgoing params), ably-cocoa (NSString via artString) and ably-go (a string struct field) all store it as received. Only ably-dotnet parses and re-serialises it. This is a breaking change: code that indexed or compared token_details.capability as a Capability wraps it in Capability(...), and to_dict() now gives a string, so an auth server returning to_dict() from an auth_url sends the capability in the wire form rather than as a JSON object. The TD and RSA16a UTS tests now assert the specification's strings, and the legacy capability and token tests wrap the attribute. Co-Authored-By: Claude Opus 5.5 --- UPDATING.md | 27 +++++++++++++++++++ ably/pubsub/types/capability.py | 2 +- ably/pubsub/types/tokendetails.py | 7 +++-- test/ably/http/httpcapability_test.py | 18 ++++++------- test/ably/http/httptoken_test.py | 8 +++--- test/uts/deviations.md | 1 - test/uts/rest/unit/auth/token_details_test.py | 13 ++------- test/uts/rest/unit/types/token_types_test.py | 6 ++--- 8 files changed, 50 insertions(+), 32 deletions(-) diff --git a/UPDATING.md b/UPDATING.md index 786206b7..21402048 100644 --- a/UPDATING.md +++ b/UPDATING.md @@ -330,6 +330,33 @@ A bare `token` string carries no client ID the client can read, so it cannot be client is created. A realtime client still checks it against the client ID the server reports when the connection opens, and fails the connection with 40102 if they conflict. +### `TokenDetails.capability` is a JSON string + +`TokenDetails.capability` now holds the capability as the JSON string the Ably service returned, +exactly as it was sent, or `None` where none is known. 3.x parsed it into a `Capability`. Wrap +the string in `Capability` where you indexed or compared it: + +Example 3.x code: +```python +token_details = await rest.auth.request_token() +operations = token_details.capability['my-channel'] +``` + +Example 4.0.0 code: +```python +from ably.pubsub.server import Capability + +token_details = await pubsub_http_client.auth.request_token() +operations = Capability(token_details.capability)['my-channel'] +``` + +`Capability` is unchanged as the way to build a `capability` token param, and its +`add_operation_to_resource` method now works where 3.x raised `AttributeError`. A `TokenDetails` +you construct yourself with a dict or `Capability` stores it as canonical JSON, and +`TokenDetails.to_dict()` now gives the capability as that string rather than as a dict. So an +auth server that returns `to_dict()` to its clients from an `auth_url` now sends the capability +as a JSON string, the form the Ably service itself returns, rather than as a JSON object. + ## Version 2.x to 3.0.0 The 3.0.0 version of ably-python introduces several breaking changes to improve the realtime experience and align the API with the Ably specification. These include: diff --git a/ably/pubsub/types/capability.py b/ably/pubsub/types/capability.py index 4f931466..1a7afacc 100644 --- a/ably/pubsub/types/capability.py +++ b/ably/pubsub/types/capability.py @@ -76,7 +76,7 @@ def add_resource(self, resource, operations=None): self[resource] = list(operations) def add_operation_to_resource(self, operation, resource): - self.setdefault(resource, []).append(operation) + self[resource] = self.get(resource, set()) | {operation} def __str__(self): return Capability.c14n(self) diff --git a/ably/pubsub/types/tokendetails.py b/ably/pubsub/types/tokendetails.py index 29c7dd24..18ea8cea 100644 --- a/ably/pubsub/types/tokendetails.py +++ b/ably/pubsub/types/tokendetails.py @@ -20,7 +20,10 @@ def __init__(self, token=None, expires=None, issued=0, self.__expires = expires self.__token = token self.__issued = issued - self.__capability = Capability(capability or {}) + # TD5, RSA16a: the capability JSON as the server sent it, not a canonical form + if capability is not None and not isinstance(capability, str): + capability = str(Capability(capability)) + self.__capability = capability self.__client_id = client_id @property @@ -48,7 +51,7 @@ def to_dict(self): 'expires': self.expires, 'token': self.token, 'issued': self.issued, - 'capability': self.capability.to_dict(), + 'capability': self.capability, 'clientId': self.client_id, } diff --git a/test/ably/http/httpcapability_test.py b/test/ably/http/httpcapability_test.py index 7e96a57d..c40b7ecc 100644 --- a/test/ably/http/httpcapability_test.py +++ b/test/ably/http/httpcapability_test.py @@ -23,7 +23,7 @@ async def test_blanket_intersection_with_key(self): token_details = await self.ably.auth.request_token(key_name=key['key_name'], key_secret=key['key_secret']) expected_capability = Capability(key["capability"]) assert token_details.token is not None, "Expected token" - assert expected_capability == token_details.capability, "Unexpected capability." + assert expected_capability == Capability(token_details.capability), "Unexpected capability." async def test_equal_intersection_with_key(self): key = self.test_vars['keys'][1] @@ -36,7 +36,7 @@ async def test_equal_intersection_with_key(self): expected_capability = Capability(key["capability"]) assert token_details.token is not None, "Expected token" - assert expected_capability == token_details.capability, "Unexpected capability" + assert expected_capability == Capability(token_details.capability), "Unexpected capability" @dont_vary_protocol async def test_empty_ops_intersection(self): @@ -74,7 +74,7 @@ async def test_non_empty_ops_intersection(self): token_details = await self.ably.auth.request_token(token_params, **kwargs) assert token_details.token is not None, "Expected token" - assert expected_capability == token_details.capability, "Unexpected capability" + assert expected_capability == Capability(token_details.capability), "Unexpected capability" async def test_non_empty_paths_intersection(self): key = self.test_vars['keys'][4] @@ -97,7 +97,7 @@ async def test_non_empty_paths_intersection(self): token_details = await self.ably.auth.request_token(token_params, **kwargs) assert token_details.token is not None, "Expected token" - assert expected_capability == token_details.capability, "Unexpected capability" + assert expected_capability == Capability(token_details.capability), "Unexpected capability" async def test_wildcard_ops_intersection(self): key = self.test_vars['keys'][4] @@ -119,7 +119,7 @@ async def test_wildcard_ops_intersection(self): token_details = await self.ably.auth.request_token(token_params, **kwargs) assert token_details.token is not None, "Expected token" - assert expected_capability == token_details.capability, "Unexpected capability" + assert expected_capability == Capability(token_details.capability), "Unexpected capability" async def test_wildcard_ops_intersection_2(self): key = self.test_vars['keys'][4] @@ -141,7 +141,7 @@ async def test_wildcard_ops_intersection_2(self): token_details = await self.ably.auth.request_token(token_params, **kwargs) assert token_details.token is not None, "Expected token" - assert expected_capability == token_details.capability, "Unexpected capability" + assert expected_capability == Capability(token_details.capability), "Unexpected capability" async def test_wildcard_resources_intersection(self): key = self.test_vars['keys'][2] @@ -163,7 +163,7 @@ async def test_wildcard_resources_intersection(self): token_details = await self.ably.auth.request_token(token_params, **kwargs) assert token_details.token is not None, "Expected token" - assert expected_capability == token_details.capability, "Unexpected capability" + assert expected_capability == Capability(token_details.capability), "Unexpected capability" async def test_wildcard_resources_intersection_2(self): key = self.test_vars['keys'][2] @@ -185,7 +185,7 @@ async def test_wildcard_resources_intersection_2(self): token_details = await self.ably.auth.request_token(token_params, **kwargs) assert token_details.token is not None, "Expected token" - assert expected_capability == token_details.capability, "Unexpected capability" + assert expected_capability == Capability(token_details.capability), "Unexpected capability" async def test_wildcard_resources_intersection_3(self): key = self.test_vars['keys'][2] @@ -208,7 +208,7 @@ async def test_wildcard_resources_intersection_3(self): token_details = await self.ably.auth.request_token(token_params, **kwargs) assert token_details.token is not None, "Expected token" - assert expected_capability == token_details.capability, "Unexpected capability" + assert expected_capability == Capability(token_details.capability), "Unexpected capability" @dont_vary_protocol async def test_invalid_capabilities(self): diff --git a/test/ably/http/httptoken_test.py b/test/ably/http/httptoken_test.py index 2406f120..1e9f0440 100644 --- a/test/ably/http/httptoken_test.py +++ b/test/ably/http/httptoken_test.py @@ -38,7 +38,7 @@ async def test_request_token_null_params(self): assert token_details.token is not None, "Expected token" assert token_details.issued + 300 >= pre_time, "Unexpected issued time" assert token_details.issued <= post_time + 500, "Unexpected issued time" - assert self.permit_all == str(token_details.capability), "Unexpected capability" + assert self.permit_all == str(Capability(token_details.capability)), "Unexpected capability" async def test_request_token_explicit_timestamp(self): pre_time = await self.server_time() @@ -87,7 +87,7 @@ async def test_request_token_with_capability_that_subsets_key_capability(self): assert token_details is not None assert token_details.token is not None - assert capability == token_details.capability, "Unexpected capability" + assert capability == Capability(token_details.capability), "Unexpected capability" async def test_request_token_with_specified_key(self): test_vars = await TestApp.get_test_vars() @@ -95,7 +95,7 @@ async def test_request_token_with_specified_key(self): token_details = await self.ably.auth.request_token( key_name=key["key_name"], key_secret=key["key_secret"]) assert token_details.token is not None, "Expected token" - assert key.get("capability") == token_details.capability, "Unexpected capability" + assert key.get("capability") == Capability(token_details.capability), "Unexpected capability" @dont_vary_protocol async def test_request_token_with_invalid_mac(self): @@ -309,7 +309,7 @@ async def auth_callback(token_params): token = await ably.auth.authorize() - assert str(token.capability) == str(capability) + assert str(Capability(token.capability)) == str(capability) await ably.close() @dont_vary_protocol diff --git a/test/uts/deviations.md b/test/uts/deviations.md index 265604bd..ec305c98 100644 --- a/test/uts/deviations.md +++ b/test/uts/deviations.md @@ -2301,7 +2301,6 @@ pseudocode is not mistaken for non-compliance. | HP8 | `headers` is a map | A list of `(name, value)` pairs, so the lookup the spec describes is impossible without converting, and case-insensitivity is lost | Open bug; changing the return type is breaking | | RSC15a | Six hosts tried | Three. `Options.__get_hosts` truncates to `http_max_retry_count`, which TO3l5 sanctions | Intentional | | TI | `ErrorInfo` equality by attributes | No `__eq__`, so errors compare by identity. Python exceptions conventionally do, and the requirement appears nowhere in `features.md`. Adding `__eq__` without `__hash__` would make `AblyException` unhashable and break any caller that puts one in a set | Intentional | -| TD5, RSA16a | `capability` is stringified JSON | A `Capability` object, a public convenience type used throughout `auth`. Narrowing the return type to `str` would break every caller that indexes or mutates it, so it is reserved for a future major | Intentional; a breaking change to align | | RSA6b, RSA6d | The capability literal as passed | Canonicalised by `Capability.c14n`, which RSA9f requires | Compliant; rendering only | | RSA8d | `error.message` contains the callback's text | Wrapped as 40170 with the original in `cause`; `__str__` renders both | Rendering | | CHM2 | Missing metrics default to 0 | `ChannelMetrics.from_dict` uses a bare `obj.get(name)`, so any omitted metric parses as `None` | Open bug, broader than CHM2g/h | diff --git a/test/uts/rest/unit/auth/token_details_test.py b/test/uts/rest/unit/auth/token_details_test.py index d4031743..a37cb852 100644 --- a/test/uts/rest/unit/auth/token_details_test.py +++ b/test/uts/rest/unit/auth/token_details_test.py @@ -8,7 +8,6 @@ import pytest from ably.pubsub.types.authoptions import AuthOptions -from ably.pubsub.types.capability import Capability from ably.pubsub.types.tokendetails import TokenDetails from ably.pubsub.util.exceptions import AblyException from test.uts.helpers.client import rest_client @@ -428,13 +427,5 @@ async def auth_callback(params): assert client.auth.token_details is not None - # DEVIATION: the spec asserts the stringified JSON - # '{"channel1":["publish","subscribe"],"channel2":["subscribe"]}'. - # `TokenDetails.capability` is a parsed `Capability` throughout ably-python, an - # established SDK-wide choice, so the same capability is asserted in that form. - capability = client.auth.token_details.capability - assert isinstance(capability, Capability) - assert capability.to_dict() == { - 'channel1': ['publish', 'subscribe'], - 'channel2': ['subscribe'], - } + assert client.auth.token_details.capability == ( + '{"channel1":["publish","subscribe"],"channel2":["subscribe"]}') diff --git a/test/uts/rest/unit/types/token_types_test.py b/test/uts/rest/unit/types/token_types_test.py index 243291b3..0cbb5e3a 100644 --- a/test/uts/rest/unit/types/token_types_test.py +++ b/test/uts/rest/unit/types/token_types_test.py @@ -50,9 +50,7 @@ def test_td1_token_details_attributes(): expires=1234567890000, capability='{"*":["*"]}', ) - # NOTE: the spec asserts capability IS the JSON string. ably-python parses it into a - # Capability mapping, which stringifies back to the same JSON. - assert json.loads(str(token_with_capability.capability)) == {'*': ['*']} + assert token_with_capability.capability == '{"*":["*"]}' # TD5 - clientId attribute token_with_client = TokenDetails( @@ -79,7 +77,7 @@ def test_td_token_details_from_json(): assert token_details.token == 'deserialized-token' assert token_details.expires == 1234567890000 assert token_details.issued == 1234567800000 - assert json.loads(str(token_details.capability)) == {'channel-1': ['publish']} + assert token_details.capability == '{"channel-1":["publish"]}' assert token_details.client_id == 'json-client' From 522c31e6aa79cd0884796b1f48cb4e51118ae744 Mon Sep 17 00:00:00 2001 From: owenpearson Date: Mon, 26 Oct 2026 19:12:26 +0000 Subject: [PATCH 38/71] pubsub: stop inventing expires and issued for a bare token string TokenDetails defaulted expires to the local time plus an hour, read from time.time() rather than the clock, and issued to 0. So a TokenDetails known only by its token string, whether passed as token, returned as a bare string or JWT by an auth_callback or auth_url, or constructed directly, reported an expiry nobody had stated. RSA16b requires only the token attribute to be populated in that case. expires and issued now default to None, as capability already does since the previous commit. Every reader of expires handles None: token_details_has_expired returns False for it, as it does with no server time offset, which is the only check RSA4b1 permits and the one reauth_if_expired relies on; and the cached-token debug log now formats it with %s rather than %d, which would have raised inside logging. A bare token with a known time offset is therefore no longer renewed an hour after it was wrapped; the server's token error drives its renewal (RSA4b). TokenDetails no longer reads the time at all. This is a breaking change for code that read expires or issued from such a TokenDetails, which must handle None. Nothing on the wire changes. The two RSA16b UTS tests are un-gated. Co-Authored-By: Claude Opus 5.5 --- UPDATING.md | 20 +++++++++++++++++++ ably/pubsub/http/auth.py | 2 +- ably/pubsub/types/tokendetails.py | 9 +++------ test/uts/deviations.md | 17 ++++++++-------- test/uts/rest/unit/auth/token_details_test.py | 7 ------- 5 files changed, 32 insertions(+), 23 deletions(-) diff --git a/UPDATING.md b/UPDATING.md index 21402048..f00fe5f4 100644 --- a/UPDATING.md +++ b/UPDATING.md @@ -357,6 +357,26 @@ you construct yourself with a dict or `Capability` stores it as canonical JSON, auth server that returns `to_dict()` to its clients from an `auth_url` now sends the capability as a JSON string, the form the Ably service itself returns, rather than as a JSON object. +### A `TokenDetails` known only by its token string no longer invents its details + +A `TokenDetails` built from a token string alone, whether you construct it yourself, pass a +`token`, or have an `auth_callback` or `auth_url` return a bare token string or JWT, now has +`expires`, `issued` and `capability` set to `None`. 3.x invented an expiry an hour after the +`TokenDetails` was created, an `issued` of `0` and an empty capability. Code that read those +values must handle `None`; the client itself does, and leaves it to the Ably service to report +when such a token has expired. + +Example 3.x code: +```python +seconds_left = (rest.auth.token_details.expires - now_ms) / 1000 +``` + +Example 4.0.0 code: +```python +expires = pubsub_http_client.auth.token_details.expires +seconds_left = None if expires is None else (expires - now_ms) / 1000 +``` + ## Version 2.x to 3.0.0 The 3.0.0 version of ably-python introduces several breaking changes to improve the realtime experience and align the API with the Ably specification. These include: diff --git a/ably/pubsub/http/auth.py b/ably/pubsub/http/auth.py index 7a3b662b..04792a11 100644 --- a/ably/pubsub/http/auth.py +++ b/ably/pubsub/http/auth.py @@ -145,7 +145,7 @@ async def _ensure_valid_auth_credentials(self, token_params=None, auth_options=N token_details = self.__token_details if not force and not self.token_details_has_expired(): - log.debug("using cached token; expires = %d", + log.debug("using cached token; expires = %s", token_details.expires) return token_details diff --git a/ably/pubsub/types/tokendetails.py b/ably/pubsub/types/tokendetails.py index 18ea8cea..1baea8be 100644 --- a/ably/pubsub/types/tokendetails.py +++ b/ably/pubsub/types/tokendetails.py @@ -1,5 +1,4 @@ import json -import time from ably.pubsub.types.capability import Capability @@ -12,12 +11,10 @@ class TokenDetails: # new requests 9000ms before it expires TOKEN_EXPIRY_BUFFER = 15 * 1000 - def __init__(self, token=None, expires=None, issued=0, + def __init__(self, token=None, expires=None, issued=None, capability=None, client_id=None): - if expires is None: - self.__expires = time.time() * 1000 + TokenDetails.DEFAULTS['ttl'] - else: - self.__expires = expires + # RSA16b: nothing is invented for a token whose details are unknown + self.__expires = expires self.__token = token self.__issued = issued # TD5, RSA16a: the capability JSON as the server sent it, not a canonical form diff --git a/test/uts/deviations.md b/test/uts/deviations.md index ec305c98..ebe95250 100644 --- a/test/uts/deviations.md +++ b/test/uts/deviations.md @@ -33,8 +33,8 @@ parametrized over a table of fixtures the specification gives inline; and five `objects/unit` tests are parametrized, three over a table and two over the scenarios the specification loops through. That turns 1481 derived tests into 1608 pytest cases. -Of **1472 Test IDs, derived as 1481 tests and run as 1608 pytest cases**: 1297 Test IDs -(1306 tests, 1431 cases) pass, 160 (160 tests, 162 cases) are gated behind +Of **1472 Test IDs, derived as 1481 tests and run as 1608 pytest cases**: 1299 Test IDs +(1308 tests, 1433 cases) pass, 158 (158 tests, 160 cases) are gated behind `RUN_DEVIATIONS`, and 15 (15 tests, 15 cases) cannot be run at all. The three groups are disjoint: two Test IDs have a gated part and a passing part — `TI1`, as two functions, and `TO/endpoint-affects-host`, as one parametrized test — and are counted with the gated, @@ -45,19 +45,19 @@ the Test IDs come from `uts/rest/unit` (503 tests, 536 cases), 482 from `uts/rest/integration` (84, 122), 73 from `uts/realtime/integration` (73, 95) and 17 from `uts/objects/integration` (17, 29); 8 of the REST integration ids (8, 8), 30 of the realtime ones (30, 30) and 5 of the objects ones (5, 5) come from the `proxy` package -within each. Of the gated Test IDs 76 are REST and 84 realtime, which is 77 REST cases +within each. Of the gated Test IDs 74 are REST and 84 realtime, which is 75 REST cases and 85 realtime. None of the 339 objects Test IDs is gated or unrunnable. A further 148 pytest cases cover the harness itself and are not derived from a specification: 130 under `helpers/`, for the mocks, and 18 under `objects/helpers/`, for the objects builders and the synced-channel mock. -The 149 gated Test IDs that record SDK non-compliance — 149 tests, 151 cases — reduce to -**66 distinct root causes**, 25 on the REST side and 41 on the realtime side. Three further +The 147 gated Test IDs that record SDK non-compliance — 147 tests, 149 cases — reduce to +**65 distinct root causes**, 24 on the REST side and 41 on the realtime side. Three further defects are recorded below with no test of their own, because the specification's test cannot discriminate (RTP18a), has nothing to assert against (the timezone split on synthesized LEAVE timestamps), or is worked around in the setup of every test that would otherwise trip over it (`enterClient` on an anonymous connection), so the file -carries **69 SDK root causes** in all. The remaining 11 gated Test IDs are +carries **68 SDK root causes** in all. The remaining 11 gated Test IDs are specification faults, and reduce to 8. The objects tier records no SDK non-compliance; its specification faults are all corrected or worked around in tests that pass, and reduce to the seven candidate issues O.1–O.7, none of them filed. @@ -1882,7 +1882,6 @@ which is about a client that *does* configure `clientId: "*"`. | RSA12a | A token with a **null** clientId is rejected when `ClientOptions.clientId` is set, with 40102 "Client ID cannot be changed to 'None'". RSA15a constrains only non-wildcard token clientIds. Needs a `new_client_id is not None` guard | | RSA7, RSA16c | A clientId learned from a token is treated as immutable, so `authorize()` to a token with a different clientId raises 40102. RSA15 scopes immutability to a clientId set in `ClientOptions`. `_configure_client_id` uses `self.client_id or self.auth_options.client_id`, conflating the two. Possibly deliberate — worth a maintainer's call | | RSA5c, RSA6c | `create_token_request()` ignores `default_token_params`. The merge lives in `Auth.request_token` only, so a direct call yields `ttl=None` and `capability=None`. RSA5, RSA5b, RSA5d, RSA6, RSA6b and RSA6d all pass, so the nullability requirement itself is met | -| RSA16b | `TokenDetails` built from a bare token string fabricates `expires` (now plus an hour), `issued` (0) and `capability`. RSA16b requires only `token` to be populated. The invented expiry can drive spurious renewal | | RSA16c | No local expiry detection without a server time offset, which an authCallback client never obtains. See the RSA4b1 spec error above; here the specification asserts renewal *does* happen, so there is no green reading | | RSA16d | A failed renewal leaves the invalidated token in place — `_ensure_valid_auth_credentials` assigns only on success | | RSA16d | `authorize()` cannot switch a client back to basic auth: `_ensure_valid_auth_credentials` sets `Method.TOKEN` unconditionally, and `AuthOptions.replace` drops `use_token_auth`, which is stored outside the options dict | @@ -3624,8 +3623,8 @@ The header states how many derived tests there are, how many pass, how many are gated and how many cannot run. Those numbers are the check that the file is still true: in pytest cases, the gated count must equal the number of failures under `RUN_DEVIATIONS=1`, and gated plus unrunnable must equal the number of skips without -it. As of this writing that is 162 failures, 1579 passes and 15 skips with the variable -set, and 177 skips and 1579 passes without it, the 1579 being 1431 derived cases and 148 +it. As of this writing that is 160 failures, 1581 passes and 15 skips with the variable +set, and 175 skips and 1581 passes without it, the 1581 being 1433 derived cases and 148 harness ones — 130 under `helpers/` and 18 under `objects/helpers/`. Those figures need the `submodules` checkout CI makes: without it, three `rest/unit/encoding` tests that read the ably-common fixtures skip as well. The tiers are told apart by the junit report's diff --git a/test/uts/rest/unit/auth/token_details_test.py b/test/uts/rest/unit/auth/token_details_test.py index a37cb852..65db8c68 100644 --- a/test/uts/rest/unit/auth/token_details_test.py +++ b/test/uts/rest/unit/auth/token_details_test.py @@ -104,11 +104,7 @@ def on_request(request): # UTS: rest/unit/RSA16b/token-string-in-options-0 -@deviation async def test_rsa16b_token_string_in_options(): - # DEVIATION: RSA16b requires that only `token` is populated. `TokenDetails.__init__` - # fabricates `expires` from a default one-hour TTL, defaults `issued` to 0 and turns - # a missing capability into an empty `Capability`. mock_http = MockHttpClient( on_connection_attempt=lambda conn: conn.respond_with_success(), on_request=respond_with_channel_details, @@ -126,10 +122,7 @@ async def test_rsa16b_token_string_in_options(): # UTS: rest/unit/RSA16b/token-string-from-callback-1 -@deviation async def test_rsa16b_token_string_from_callback(): - # DEVIATION: as above. `Auth.request_token` wraps a token string in a bare - # `TokenDetails`, which fabricates `expires` and defaults `issued` to 0. async def auth_callback(params): return 'just-a-token-string' From 70a15195d823972068dda582ac4137fe3e55d37a Mon Sep 17 00:00:00 2001 From: owenpearson Date: Mon, 26 Oct 2026 19:15:02 +0000 Subject: [PATCH 39/71] pubsub: apply default_token_params in create_token_request() create_token_request() read only the token params handed to it, so a direct call ignored default_token_params entirely and signed a request with no ttl or capability (RSA5c, RSA6c). request_token(), meanwhile, merged explicit params over the defaults key by key. Both now take the same rule: with no token params the defaults apply, and given params, even an empty dict, replace the defaults wholesale. That is what RSA8e and RSA9h say ("the values of each attribute are not merged with the configured client library defaults, but rather are used instead of the stored values (even when null)"), what RSA10j says for authorize(), and what authorize() here already did. Replace rather than merge on the evidence of the other SDKs too: ably-js (`tokenParams || Utils.copy(this.tokenParams)` in both _requestTokenImpl and _createTokenRequestImpl), ably-java (`params == null ? this.tokenParams : params.copy()` in requestToken and createTokenRequest), ably-cocoa (`tokenParams ? : [_tokenParams copy]`, commented "they're used instead of stored, don't merge them") and ably-dotnet (`tokenParams ?? CurrentTokenParams ?? ...`) all replace. ably-go replaces in requestToken and never reads DefaultTokenParams in createTokenRequest. None merges key by key. request_token() also copies the params it passes to an auth_callback, as the merge used to. This is a breaking change in both directions. An auth server whose client sets default_token_params and signs with create_token_request() now issues tokens carrying those defaults, typically a shorter ttl or a narrower capability. A request_token() call that names some params no longer inherits the rest from the defaults, so a default capability that restricted its tokens no longer applies. Callers pass every param they want. The RSA5c and RSA6c UTS tests are un-gated, and a mocked test covers both methods. Co-Authored-By: Claude Opus 5.5 --- UPDATING.md | 28 +++++++++++++++++++ ably/pubsub/http/auth.py | 11 +++++--- test/ably/http/authorizeparams_test.py | 26 +++++++++++++++++ test/uts/deviations.md | 17 ++++++----- .../unit/auth/token_request_params_test.py | 9 +----- 5 files changed, 70 insertions(+), 21 deletions(-) diff --git a/UPDATING.md b/UPDATING.md index f00fe5f4..94f4182d 100644 --- a/UPDATING.md +++ b/UPDATING.md @@ -377,6 +377,34 @@ expires = pubsub_http_client.auth.token_details.expires seconds_left = None if expires is None else (expires - now_ms) / 1000 ``` +### `default_token_params` apply to `create_token_request()`, and given params replace them + +`auth.create_token_request()` called without token params now applies the client's +`default_token_params`, as `request_token()` and `authorize()` already did. 3.x ignored them. So +an auth server whose client is configured with `default_token_params` and which signs token +requests with `create_token_request()` now issues tokens with those defaults, typically a shorter +`ttl` or a narrower `capability` than before. Check that the defaults are what you want your +clients' tokens to carry, or remove them. + +Token params passed to `create_token_request()` or `request_token()` now replace the defaults +entirely, as they already did for `authorize()` and as every other Ably SDK does. 3.x merged them +over the defaults key by key in `request_token()`. Take care where a default `capability` or +`ttl` restricted the tokens: a `request_token()` call that names other params no longer carries +them, so its token gets the key's full capability and the service's default `ttl`. Pass every +param you want, including the defaults you still want: + +Example 3.x code: +```python +rest = AblyRest(key='xxx', default_token_params={'capability': capability}) +await rest.auth.request_token({'ttl': 600000}) # carried the default capability +``` + +Example 4.0.0 code: +```python +pubsub_http_client = create_http_client(key='xxx', default_token_params={'capability': capability}) +await pubsub_http_client.auth.request_token({'ttl': 600000, 'capability': capability}) +``` + ## Version 2.x to 3.0.0 The 3.0.0 version of ably-python introduces several breaking changes to improve the realtime experience and align the API with the Ably specification. These include: diff --git a/ably/pubsub/http/auth.py b/ably/pubsub/http/auth.py index 04792a11..a6f12342 100644 --- a/ably/pubsub/http/auth.py +++ b/ably/pubsub/http/auth.py @@ -188,9 +188,10 @@ async def request_token(self, token_params: dict | None = None, auth_url: str | None = None, auth_method: str | None = None, auth_headers: dict | None = None, auth_params: dict | None = None, query_time=None): - token_params = token_params or {} - token_params = dict(self.auth_options.default_token_params, - **token_params) + # RSA8e: given params replace the defaults rather than merging with them + if token_params is None: + token_params = self.auth_options.default_token_params + token_params = dict(token_params) key_name = key_name or self.auth_options.key_name key_secret = key_secret or self.auth_options.key_secret @@ -264,7 +265,9 @@ async def request_token(self, token_params: dict | None = None, async def create_token_request(self, token_params: dict | str | None = None, key_name: str | None = None, key_secret: str | None = None, query_time=None): - token_params = token_params or {} + # RSA5c, RSA6c, RSA9h: the defaults apply when no params are given, and given params replace them + if token_params is None: + token_params = dict(self.auth_options.default_token_params) token_request = {} key_name = key_name or self.auth_options.key_name diff --git a/test/ably/http/authorizeparams_test.py b/test/ably/http/authorizeparams_test.py index 79bbb9bf..1c27c289 100644 --- a/test/ably/http/authorizeparams_test.py +++ b/test/ably/http/authorizeparams_test.py @@ -43,3 +43,29 @@ async def test_authorize_keeps_an_explicit_client_id(): assert token_params == {'client_id': 'explicit'} finally: await client.close() + + +async def test_given_token_params_replace_the_defaults(): + """RSA8e, RSA9h: default_token_params apply only when no params are given.""" + received = [] + + async def auth_callback(token_params): + received.append(token_params) + return 'a-token' + + defaults = {'ttl': 1800000, 'capability': '{"*":["subscribe"]}'} + key_client = create_http_client(key='app.key:secret', default_token_params=dict(defaults)) + callback_client = create_http_client(auth_callback=auth_callback, default_token_params=dict(defaults)) + try: + token_request = await key_client.auth.create_token_request() + assert (token_request.ttl, token_request.capability) == (1800000, '{"*": ["subscribe"]}') + + token_request = await key_client.auth.create_token_request({'ttl': 600000}) + assert (token_request.ttl, token_request.capability) == (600000, None) + + await callback_client.auth.request_token() + await callback_client.auth.request_token({'ttl': 600000}) + assert received == [defaults, {'ttl': 600000}] + finally: + await key_client.close() + await callback_client.close() diff --git a/test/uts/deviations.md b/test/uts/deviations.md index ebe95250..9b45b3d5 100644 --- a/test/uts/deviations.md +++ b/test/uts/deviations.md @@ -33,8 +33,8 @@ parametrized over a table of fixtures the specification gives inline; and five `objects/unit` tests are parametrized, three over a table and two over the scenarios the specification loops through. That turns 1481 derived tests into 1608 pytest cases. -Of **1472 Test IDs, derived as 1481 tests and run as 1608 pytest cases**: 1299 Test IDs -(1308 tests, 1433 cases) pass, 158 (158 tests, 160 cases) are gated behind +Of **1472 Test IDs, derived as 1481 tests and run as 1608 pytest cases**: 1301 Test IDs +(1310 tests, 1435 cases) pass, 156 (156 tests, 158 cases) are gated behind `RUN_DEVIATIONS`, and 15 (15 tests, 15 cases) cannot be run at all. The three groups are disjoint: two Test IDs have a gated part and a passing part — `TI1`, as two functions, and `TO/endpoint-affects-host`, as one parametrized test — and are counted with the gated, @@ -45,19 +45,19 @@ the Test IDs come from `uts/rest/unit` (503 tests, 536 cases), 482 from `uts/rest/integration` (84, 122), 73 from `uts/realtime/integration` (73, 95) and 17 from `uts/objects/integration` (17, 29); 8 of the REST integration ids (8, 8), 30 of the realtime ones (30, 30) and 5 of the objects ones (5, 5) come from the `proxy` package -within each. Of the gated Test IDs 74 are REST and 84 realtime, which is 75 REST cases +within each. Of the gated Test IDs 72 are REST and 84 realtime, which is 73 REST cases and 85 realtime. None of the 339 objects Test IDs is gated or unrunnable. A further 148 pytest cases cover the harness itself and are not derived from a specification: 130 under `helpers/`, for the mocks, and 18 under `objects/helpers/`, for the objects builders and the synced-channel mock. -The 147 gated Test IDs that record SDK non-compliance — 147 tests, 149 cases — reduce to -**65 distinct root causes**, 24 on the REST side and 41 on the realtime side. Three further +The 145 gated Test IDs that record SDK non-compliance — 145 tests, 147 cases — reduce to +**64 distinct root causes**, 23 on the REST side and 41 on the realtime side. Three further defects are recorded below with no test of their own, because the specification's test cannot discriminate (RTP18a), has nothing to assert against (the timezone split on synthesized LEAVE timestamps), or is worked around in the setup of every test that would otherwise trip over it (`enterClient` on an anonymous connection), so the file -carries **68 SDK root causes** in all. The remaining 11 gated Test IDs are +carries **67 SDK root causes** in all. The remaining 11 gated Test IDs are specification faults, and reduce to 8. The objects tier records no SDK non-compliance; its specification faults are all corrected or worked around in tests that pass, and reduce to the seven candidate issues O.1–O.7, none of them filed. @@ -1881,7 +1881,6 @@ which is about a client that *does* configure `clientId: "*"`. |---|---| | RSA12a | A token with a **null** clientId is rejected when `ClientOptions.clientId` is set, with 40102 "Client ID cannot be changed to 'None'". RSA15a constrains only non-wildcard token clientIds. Needs a `new_client_id is not None` guard | | RSA7, RSA16c | A clientId learned from a token is treated as immutable, so `authorize()` to a token with a different clientId raises 40102. RSA15 scopes immutability to a clientId set in `ClientOptions`. `_configure_client_id` uses `self.client_id or self.auth_options.client_id`, conflating the two. Possibly deliberate — worth a maintainer's call | -| RSA5c, RSA6c | `create_token_request()` ignores `default_token_params`. The merge lives in `Auth.request_token` only, so a direct call yields `ttl=None` and `capability=None`. RSA5, RSA5b, RSA5d, RSA6, RSA6b and RSA6d all pass, so the nullability requirement itself is met | | RSA16c | No local expiry detection without a server time offset, which an authCallback client never obtains. See the RSA4b1 spec error above; here the specification asserts renewal *does* happen, so there is no green reading | | RSA16d | A failed renewal leaves the invalidated token in place — `_ensure_valid_auth_credentials` assigns only on success | | RSA16d | `authorize()` cannot switch a client back to basic auth: `_ensure_valid_auth_credentials` sets `Method.TOKEN` unconditionally, and `AuthOptions.replace` drops `use_token_auth`, which is stored outside the options dict | @@ -3623,8 +3622,8 @@ The header states how many derived tests there are, how many pass, how many are gated and how many cannot run. Those numbers are the check that the file is still true: in pytest cases, the gated count must equal the number of failures under `RUN_DEVIATIONS=1`, and gated plus unrunnable must equal the number of skips without -it. As of this writing that is 160 failures, 1581 passes and 15 skips with the variable -set, and 175 skips and 1581 passes without it, the 1581 being 1433 derived cases and 148 +it. As of this writing that is 158 failures, 1583 passes and 15 skips with the variable +set, and 173 skips and 1583 passes without it, the 1583 being 1435 derived cases and 148 harness ones — 130 under `helpers/` and 18 under `objects/helpers/`. Those figures need the `submodules` checkout CI makes: without it, three `rest/unit/encoding` tests that read the ably-common fixtures skip as well. The tiers are told apart by the junit report's diff --git a/test/uts/rest/unit/auth/token_request_params_test.py b/test/uts/rest/unit/auth/token_request_params_test.py index 1a341833..cf5e7577 100644 --- a/test/uts/rest/unit/auth/token_request_params_test.py +++ b/test/uts/rest/unit/auth/token_request_params_test.py @@ -4,7 +4,6 @@ """ from test.uts.helpers.client import rest_client -from test.uts.helpers.deviations import deviation from test.uts.helpers.mock_http import MockHttpClient KEY = 'appId.keyId:keySecret' @@ -34,15 +33,11 @@ async def test_rsa5b_explicit_ttl_preserved(): # UTS: rest/unit/RSA5c/ttl-from-default-params-0 -@deviation async def test_rsa5c_ttl_from_default_params(): client = rest_client(unreachable_mock(), key=KEY, default_token_params={'ttl': 1800000}) token_request = await client.auth.create_token_request() - # DEVIATION: `Auth.create_token_request` reads only the token params handed - # to it (`token_params = token_params or {}`); the defaults are merged in - # `Auth.request_token`, one level up, so a direct call sees none of them. assert token_request.ttl == 1800000 @@ -79,15 +74,13 @@ async def test_rsa6b_explicit_capability_preserved(): # UTS: rest/unit/RSA6c/capability-from-default-params-0 -@deviation async def test_rsa6c_capability_from_default_params(): client = rest_client(unreachable_mock(), key=KEY, default_token_params={'capability': '{"*":["subscribe"]}'}) token_request = await client.auth.create_token_request() - # DEVIATION: as for RSA5c, `Auth.create_token_request` does not merge - # `default_token_params`. + # NOTE: canonical rendering, as in RSA6b. assert token_request.capability == '{"*": ["subscribe"]}' From ed2a857f7cf01c488196e0b20590b6032537fe4f Mon Sep 17 00:00:00 2001 From: owenpearson Date: Mon, 26 Oct 2026 19:19:43 +0000 Subject: [PATCH 40/71] pubsub: accept a token that names no clientId on a configured client _configure_client_id ran after every token fetch and read a token with no clientId as an attempt to change the configured one, raising 40102. A bare token string or JWT from an auth_callback or auth_url is wrapped without being parsed, so its clientId is always None: a client configured with clientId whose auth server returns a JWT for that very clientId failed its first attempt, and a realtime client then waited out a full disconnected_retry_timeout, fifteen seconds, before the retry took the cached-token branch and connected (deviations.md I.4, RSA7). A REST client failed the request outright. RSA15a constrains only a non-wildcard token clientId, and the RSA7 UTS case table has a configured clientId with a null token clientId succeed. _configure_client_id now leaves a configured clientId as it is when the token names none. The server still validates the token against the clientId: a realtime client sends it on connect (RSA7e1) and fails on a CONNECTED clientId that conflicts (RSA15a), and a mismatched JWT now reaches that failure on its first attempt. This is not a breaking change. The RSA12b, RSA10b and realtime integration RSA7 matching-clientId UTS tests are un-gated, the RSA12a test asserts the specification's success, and the mismatched-clientId integration test no longer needs a twenty-second wait. The header's REST root-cause count also drops the RSA10b cause, which closed with the explicit-clientId fix but whose test stayed gated on this one. Co-Authored-By: Claude Opus 5.5 --- ably/pubsub/http/auth.py | 5 ++ test/uts/deviations.md | 84 +++++----------------- test/uts/realtime/integration/auth_test.py | 9 +-- test/uts/rest/unit/auth/authorize_test.py | 7 +- test/uts/rest/unit/auth/client_id_test.py | 17 +---- 5 files changed, 24 insertions(+), 98 deletions(-) diff --git a/ably/pubsub/http/auth.py b/ably/pubsub/http/auth.py index a6f12342..3098a378 100644 --- a/ably/pubsub/http/auth.py +++ b/ably/pubsub/http/auth.py @@ -371,6 +371,11 @@ def _configure_client_id(self, new_client_id): log.debug("Auth._configure_client_id(): new client_id = %s", new_client_id) original_client_id = self.client_id or self.auth_options.client_id + # RSA12a, RSA15a: a token that names no clientId, such as a bare token string or a JWT + # the library does not parse, leaves a configured clientId as it is + if new_client_id is None and self.auth_options.client_id is not None: + return + # If new client ID from Ably is a wildcard, but preconfigured clientId is set, # then keep the existing clientId if original_client_id != '*' and new_client_id == '*': diff --git a/test/uts/deviations.md b/test/uts/deviations.md index 9b45b3d5..42d567dc 100644 --- a/test/uts/deviations.md +++ b/test/uts/deviations.md @@ -33,8 +33,8 @@ parametrized over a table of fixtures the specification gives inline; and five `objects/unit` tests are parametrized, three over a table and two over the scenarios the specification loops through. That turns 1481 derived tests into 1608 pytest cases. -Of **1472 Test IDs, derived as 1481 tests and run as 1608 pytest cases**: 1301 Test IDs -(1310 tests, 1435 cases) pass, 156 (156 tests, 158 cases) are gated behind +Of **1472 Test IDs, derived as 1481 tests and run as 1608 pytest cases**: 1304 Test IDs +(1313 tests, 1438 cases) pass, 153 (153 tests, 155 cases) are gated behind `RUN_DEVIATIONS`, and 15 (15 tests, 15 cases) cannot be run at all. The three groups are disjoint: two Test IDs have a gated part and a passing part — `TI1`, as two functions, and `TO/endpoint-affects-host`, as one parametrized test — and are counted with the gated, @@ -45,19 +45,19 @@ the Test IDs come from `uts/rest/unit` (503 tests, 536 cases), 482 from `uts/rest/integration` (84, 122), 73 from `uts/realtime/integration` (73, 95) and 17 from `uts/objects/integration` (17, 29); 8 of the REST integration ids (8, 8), 30 of the realtime ones (30, 30) and 5 of the objects ones (5, 5) come from the `proxy` package -within each. Of the gated Test IDs 72 are REST and 84 realtime, which is 73 REST cases -and 85 realtime. None of the 339 objects Test IDs is gated or unrunnable. +within each. Of the gated Test IDs 70 are REST and 83 realtime, which is 71 REST cases +and 84 realtime. None of the 339 objects Test IDs is gated or unrunnable. A further 148 pytest cases cover the harness itself and are not derived from a specification: 130 under `helpers/`, for the mocks, and 18 under `objects/helpers/`, for the objects builders and the synced-channel mock. -The 145 gated Test IDs that record SDK non-compliance — 145 tests, 147 cases — reduce to -**64 distinct root causes**, 23 on the REST side and 41 on the realtime side. Three further +The 142 gated Test IDs that record SDK non-compliance — 142 tests, 144 cases — reduce to +**61 distinct root causes**, 21 on the REST side and 40 on the realtime side. Three further defects are recorded below with no test of their own, because the specification's test cannot discriminate (RTP18a), has nothing to assert against (the timezone split on synthesized LEAVE timestamps), or is worked around in the setup of every test that would otherwise trip over it (`enterClient` on an anonymous connection), so the file -carries **67 SDK root causes** in all. The remaining 11 gated Test IDs are +carries **64 SDK root causes** in all. The remaining 11 gated Test IDs are specification faults, and reduce to 8. The objects tier records no SDK non-compliance; its specification faults are all corrected or worked around in tests that pass, and reduce to the seven candidate issues O.1–O.7, none of them filed. @@ -679,11 +679,8 @@ immediately below it says "the key assertion is that the connection enters FAILE with error code 40102". Both cannot hold: a constructor has no token to compare a clientId against, so nothing is knowable until the server answers. Measured: construction does not raise — the client comes back INITIALIZED — and the connection reaches FAILED with -40102/401 "invalid clientId for credentials" about fifteen seconds later. The derived test -drops the throw and makes the specification's own named assertion. The fifteen seconds are -not the server's: they are the spurious `disconnected_retry_timeout` recorded under -`#### A JWT string with a matching clientId is rejected 40102` below, which is why the -test waits twenty seconds rather than the default ten. +40102/401 "invalid clientId for credentials" on its first attempt. The derived test drops +the throw and makes the specification's own named assertion. Filed as [#551](https://github.com/ably/specification/issues/551). @@ -1698,43 +1695,6 @@ two aware `datetime.now(timezone.utc)` readings and passes. ### Auth -#### A JWT string with a matching clientId is rejected 40102 — 1 test - -**Spec point:** RSA7, `realtime/integration/RSA7/matching-clientid-succeeds-0`. - -RSA7 requires that a token whose `clientId` equals the client's configured `clientId` is -accepted. `Auth._ensure_valid_auth_credentials` calls -`self._configure_client_id(self.__token_details.client_id)` after every token fetch -(`ably/rest/auth.py:126`). Where an `auth_callback` returns a **JWT string**, -`request_token` wraps it as `TokenDetails(token=)` (`auth.py:213`) without parsing the -JWT, so `token_details.client_id` is `None`. `_configure_client_id(None)` then finds -`None != 'test-client-…'` and raises `IncompatibleClientIdException(…, 400, 40102)` -(`auth.py:345`), although the JWT's own `x-ably-clientId` claim is exactly the configured -id. Measured against the sandbox, one auth-callback invocation: - -``` -initialized→connecting None -connecting→disconnected AblyAuthException 80019/401 'Client configured authentication provider request failed' -disconnected→connecting None -connecting→connected None -``` - -The connection does arrive in the end, because `__token_details` is assigned before the -raise: the retry takes the cached-token branch and skips `_configure_client_id` altogether. -The observable cost is a spurious failed attempt and a full `disconnected_retry_timeout` — -fifteen seconds — on a clientId that matched. - -**Tests affected:** -`realtime/integration/auth_test.py::test_rsa7_matching_clientid_succeeds` — -`AssertionError: Timed out waiting for connection state connected; it was disconnected`. -The sibling `test_rsa7_mismatched_clientid_fails` is not gated and pays the same fifteen -seconds before the server's genuine rejection arrives, which is why it waits twenty -seconds rather than the default ten. - -**Status:** open bug. A fix either reads the `x-ably-clientId` claim when wrapping a bare -token string, or leaves `_configure_client_id` alone where the fetched token carries no -clientId of its own. - #### A failed RTN22 reauth leaves no trace on the connection — 1 test **Spec points:** RSA4c1, RSA4c3. @@ -1879,7 +1839,6 @@ which is about a client that *does* configure `clientId: "*"`. | Spec points | Behaviour | |---|---| -| RSA12a | A token with a **null** clientId is rejected when `ClientOptions.clientId` is set, with 40102 "Client ID cannot be changed to 'None'". RSA15a constrains only non-wildcard token clientIds. Needs a `new_client_id is not None` guard | | RSA7, RSA16c | A clientId learned from a token is treated as immutable, so `authorize()` to a token with a different clientId raises 40102. RSA15 scopes immutability to a clientId set in `ClientOptions`. `_configure_client_id` uses `self.client_id or self.auth_options.client_id`, conflating the two. Possibly deliberate — worth a maintainer's call | | RSA16c | No local expiry detection without a server time offset, which an authCallback client never obtains. See the RSA4b1 spec error above; here the specification asserts renewal *does* happen, so there is no green reading | | RSA16d | A failed renewal leaves the invalidated token in place — `_ensure_valid_auth_credentials` assigns only on success | @@ -2974,11 +2933,11 @@ none of these shows up as a failure — which is why they are easy to lose. The five tiers above classify the realtime unit derivation; the REST unit derivation's candidates went upstream as [#709](https://github.com/ably/ably-python/issues/709)–[#712](https://github.com/ably/ably-python/issues/712). -Six further defects came out of the two integration tiers, and none of them is filed. I.1 -and I.2 are tier 2 by the ranking above — an error where there should be none — and I.3 is -tier 1 for a client that configures the option it concerns, since the call does not come -back when the caller asked for it to. I.4 and I.6 are tier 2 as well; I.5 is tier 3, a -presence member silently leaving the set with nothing to report it. +Five further defects came out of the two integration tiers, and none of them is filed; each +keeps the label it was first given. I.1 and I.2 are tier 2 by the ranking above — an error +where there should be none — and I.3 is tier 1 for a client that configures the option it +concerns, since the call does not come back when the caller asked for it to. I.6 is tier 2 +as well; I.5 is tier 3, a presence member silently leaving the set with nothing to report it. The realtime integration tier also put four defects the unit tier had already recorded in front of the real server, and those extend the entries above rather than opening issues of @@ -3020,17 +2979,6 @@ no timeout and no fallback at all. Distinct from bounding a single socket read rather than the request; a fix wants to settle both. `test/uts/rest/integration/proxy/rest_fallback_test.py -k rsc15l2` -**I.4 A JWT string whose clientId matches the configured one is rejected 40102.** RSA7. -An `auth_callback` returning an Ably JWT gets wrapped as `TokenDetails(token=)` -without the JWT being parsed (`ably/rest/auth.py:213`), so `token_details.client_id` is -`None`; `_configure_client_id(None)` (`:126`, raising at `:345`) then reads that as a -clientId change and raises 40102 although the JWT's `x-ably-clientId` claim is the -configured id exactly. The connection recovers on the retry, which takes the cached-token -branch, so the cost is a spurious failed attempt and a full `disconnected_retry_timeout` — -fifteen seconds — on every such client. Affects anyone whose auth provider returns a JWT -and who also configures `clientId`, which is the ordinary shape of a JWT deployment. -`test/uts/realtime/integration/auth_test.py -k rsa7_matching` - **I.5 RTP17i re-entry never runs on a channel that is already ATTACHED.** RTP17i, RTP17g. `RealtimePresence.on_attached()` is reached only from `RealtimeChannel._notify_state()`, and an ATTACHED arriving on an already-attached channel takes the RTL12 branch @@ -3622,8 +3570,8 @@ The header states how many derived tests there are, how many pass, how many are gated and how many cannot run. Those numbers are the check that the file is still true: in pytest cases, the gated count must equal the number of failures under `RUN_DEVIATIONS=1`, and gated plus unrunnable must equal the number of skips without -it. As of this writing that is 158 failures, 1583 passes and 15 skips with the variable -set, and 173 skips and 1583 passes without it, the 1583 being 1435 derived cases and 148 +it. As of this writing that is 155 failures, 1586 passes and 15 skips with the variable +set, and 170 skips and 1586 passes without it, the 1586 being 1438 derived cases and 148 harness ones — 130 under `helpers/` and 18 under `objects/helpers/`. Those figures need the `submodules` checkout CI makes: without it, three `rest/unit/encoding` tests that read the ably-common fixtures skip as well. The tiers are told apart by the junit report's diff --git a/test/uts/realtime/integration/auth_test.py b/test/uts/realtime/integration/auth_test.py index 3bb60ae2..f69ce40d 100644 --- a/test/uts/realtime/integration/auth_test.py +++ b/test/uts/realtime/integration/auth_test.py @@ -17,17 +17,11 @@ from ably.pubsub.realtime.connection import ConnectionState from test.uts.helpers.client import await_connection_state, sandbox_realtime_client -from test.uts.helpers.deviations import deviation from test.uts.helpers.sandbox import extract_key_name, extract_key_secret, generate_jwt, random_id # The lifetime every JWT here is issued with, as the specification's `ttl: 3600000`. TOKEN_TTL = 3600000 -# The wait for the mismatched-clientId connection to be failed by the server. The SDK spends -# its `disconnected_retry_timeout` — 15 seconds by default — between the first attempt and -# the one the server rejects, so the specification's unstated wait has to clear that. -FAIL_TIMEOUT = 20.0 - def connection_id(client): """The specification's `connection.id`, held on the connection manager.""" @@ -105,7 +99,6 @@ async def test_rsa8_token_auth_connect(realtime_sandbox): # UTS: realtime/integration/RSA7/matching-clientid-succeeds-0 -@deviation async def test_rsa7_matching_clientid_succeeds(realtime_sandbox): test_client_id = 'test-client-' + random_id() @@ -132,7 +125,7 @@ async def test_rsa7_mismatched_clientid_fails(realtime_sandbox): client_id='wrong-client-id', auto_connect=False) client.connect() - await await_connection_state(client, ConnectionState.FAILED, FAIL_TIMEOUT) + await await_connection_state(client, ConnectionState.FAILED) assert client.connection.state is ConnectionState.FAILED assert client.connection.error_reason.code == 40102 diff --git a/test/uts/rest/unit/auth/authorize_test.py b/test/uts/rest/unit/auth/authorize_test.py index e7aff225..c1c0a3dd 100644 --- a/test/uts/rest/unit/auth/authorize_test.py +++ b/test/uts/rest/unit/auth/authorize_test.py @@ -13,7 +13,7 @@ from ably.pubsub.types.tokendetails import TokenDetails from ably.pubsub.util.exceptions import AblyException from test.uts.helpers.client import rest_client -from test.uts.helpers.deviations import deviation, spec_error +from test.uts.helpers.deviations import spec_error from test.uts.helpers.mock_http import MockHttpClient KEY = 'appId.keyId:keySecret' @@ -85,7 +85,6 @@ def on_request(request): # UTS: rest/unit/RSA10b/authorize-explicit-params-0 -@deviation async def test_rsa10b_authorize_explicit_params(): callback_params = [] @@ -99,10 +98,6 @@ async def mock_auth_callback(params): ) client = rest_client(mock_http, auth_callback=mock_auth_callback, client_id='default-client') - # DEVIATION: the token that comes back carries no clientId, which - # `Auth._configure_client_id` reads as an attempt to change the configured - # clientId and rejects with IncompatibleClientIdException 40102 before the - # assertions are reached. This is the RSA12a mechanism in client_id_test.py. await client.auth.authorize({'client_id': 'override-client', 'ttl': 7200000}) params = callback_params[0] diff --git a/test/uts/rest/unit/auth/client_id_test.py b/test/uts/rest/unit/auth/client_id_test.py index 2a9a072c..7feea69d 100644 --- a/test/uts/rest/unit/auth/client_id_test.py +++ b/test/uts/rest/unit/auth/client_id_test.py @@ -11,7 +11,6 @@ from ably.pubsub.types.tokendetails import TokenDetails from ably.pubsub.util.exceptions import AblyException, IncompatibleClientIdException from test.uts.helpers.client import rest_client -from test.uts.helpers.deviations import deviation from test.uts.helpers.mock_http import MockHttpClient # `channels.get(name).status()` walks status.occupancy.metrics unguarded, so the @@ -108,28 +107,14 @@ async def mock_auth_callback(params): client = rest_client(channel_details_mock(), auth_callback=mock_auth_callback, client_id='library-client-id') - # DEVIATION: the spec expects the operation to succeed. The returned token - # carries no clientId, which RSA15a leaves unconstrained (only a non-wildcard - # token clientId has to match ClientOptions), but Auth._configure_client_id - # reads the absent value as an attempt to change the configured clientId and - # raises 40102. - with pytest.raises(IncompatibleClientIdException) as excinfo: - await client.channels.get('test').status() - - assert excinfo.value.code == 40102 + await client.channels.get('test').status() - # The spec's own assertions, which hold: the clientId reached the callback. # ably-python passes TokenParams as a dict with snake_case keys. assert len(received_params) >= 1 assert received_params[0]['client_id'] == 'library-client-id' # UTS: rest/unit/RSA12b/clientid-sent-to-authurl-0 -# DEVIATION: the specification's auth_url response carries no clientId, and -# Auth._configure_client_id reads the absent value as an attempt to change the -# configured clientId, raising 40102 before `status()` returns. This is the RSA12a -# mechanism above. -@deviation async def test_rsa12b_clientid_sent_to_authurl(): captured_requests = [] From f893550eb44d9d2aecb7ecf82df0d9175802d7db Mon Sep 17 00:00:00 2001 From: owenpearson Date: Mon, 26 Oct 2026 19:24:54 +0000 Subject: [PATCH 41/71] pubsub: let a basic-auth realtime connection enter presence for others A realtime client built from a key alone is told clientId "*" in its CONNECTED message. _configure_client_id opens with a branch that keeps a configured clientId against a server wildcard, and with no clientId configured that branch still fired, recording the clientId as validated and None. can_assume_client_id then refused every clientId, so presence.enter_client could never succeed on such a connection (deviations.md I.2; RSA7b4, RTP14, RTP15). The UTS integration presence tests and the repository's own presence suite worked around it by configuring client_id='*'. The branch now applies only when a clientId is configured, so with none the client takes the wildcard as its clientId, which RSA7b4 asks for. Because auth.client_id can now be "*", the Basic auth X-Ably-ClientId header is taken from the configured clientId, as RSA7e2 specifies, rather than from auth.client_id, so a wildcard the server reported is never sent back to it as a clientId. A later authorize() on such a client requests a wildcard token, which RSA10h asks for and the key already permits. This is not a breaking change. The three batch presence tests and the two realtime integration presence tests that configured client_id='*' as a workaround now build their clients as the specifications do. Co-Authored-By: Claude Opus 5.5 --- ably/pubsub/http/auth.py | 11 +-- test/uts/deviations.md | 68 ++----------------- .../presence/presence_sync_test.py | 6 +- .../integration/presence_lifecycle_test.py | 15 ++-- .../rest/integration/batch_presence_test.py | 12 +--- 5 files changed, 21 insertions(+), 91 deletions(-) diff --git a/ably/pubsub/http/auth.py b/ably/pubsub/http/auth.py index 3098a378..106e10c8 100644 --- a/ably/pubsub/http/auth.py +++ b/ably/pubsub/http/auth.py @@ -377,8 +377,8 @@ def _configure_client_id(self, new_client_id): return # If new client ID from Ably is a wildcard, but preconfigured clientId is set, - # then keep the existing clientId - if original_client_id != '*' and new_client_id == '*': + # then keep the existing clientId. With none set, the wildcard is the clientId (RSA7b4) + if original_client_id is not None and original_client_id != '*' and new_client_id == '*': self.__client_id_validated = True self.__client_id = original_client_id return @@ -404,11 +404,12 @@ def can_assume_client_id(self, assumed_client_id): async def _get_auth_headers(self): if self.__auth_mechanism == Auth.Method.BASIC: - # RSA7e2 - if self.client_id: + # RSA7e2: the configured clientId, not a wildcard the server reported + client_id = self.auth_options.client_id + if client_id: return { 'Authorization': f'Basic {self.basic_credentials}', - 'X-Ably-ClientId': base64.b64encode(self.client_id.encode('utf-8')) + 'X-Ably-ClientId': base64.b64encode(client_id.encode('utf-8')) } return { 'Authorization': f'Basic {self.basic_credentials}', diff --git a/test/uts/deviations.md b/test/uts/deviations.md index 42d567dc..bcd2df0f 100644 --- a/test/uts/deviations.md +++ b/test/uts/deviations.md @@ -52,12 +52,10 @@ specification: 130 under `helpers/`, for the mocks, and 18 under `objects/helper the objects builders and the synced-channel mock. The 142 gated Test IDs that record SDK non-compliance — 142 tests, 144 cases — reduce to -**61 distinct root causes**, 21 on the REST side and 40 on the realtime side. Three further +**61 distinct root causes**, 21 on the REST side and 40 on the realtime side. Two further defects are recorded below with no test of their own, because the specification's test -cannot discriminate (RTP18a), has nothing to assert against (the timezone split on -synthesized LEAVE timestamps), or is worked around in the setup of every test that -would otherwise trip over it (`enterClient` on an anonymous connection), so the file -carries **64 SDK root causes** in all. The remaining 11 gated Test IDs are +cannot discriminate (RTP18a) or has nothing to assert against (the timezone split on +synthesized LEAVE timestamps), so the file carries **63 SDK root causes** in all. The remaining 11 gated Test IDs are specification faults, and reduce to 8. The objects tier records no SDK non-compliance; its specification faults are all corrected or worked around in tests that pass, and reduce to the seven candidate issues O.1–O.7, none of them filed. @@ -1790,51 +1788,6 @@ JWT, fall with it. renew and retry on a token error whatever the flag says, and only then decide whether the final response is raised or returned. -#### A basic-auth connection records its clientId as validated and `None`, so `enterClient` can never succeed — no test - -**Spec points:** RSA7b4, RTP14, RTP15. - -A client built from a full-access key alone and asked to `enter_client("user-1", …)` -raises `AblyException: 40012 400 Unable to enter presence channel with clientId user-1 as -it does not match the current clientId None`. - -The server tells such a connection `clientId: "*"` in CONNECTED, and -`Auth._configure_client_id` (`ably/rest/auth.py:335-353`) opens with a branch that exists -to stop a server wildcard overwriting a clientId the caller configured: - -```python -if original_client_id != '*' and new_client_id == '*': - self.__client_id_validated = True - self.__client_id = original_client_id - return -``` - -With no configured clientId, `original_client_id` is `None`, `None != '*'` holds, and the -client id is recorded as **validated and `None`**. `can_assume_client_id` -(`auth.py:356-363`) then takes the validated path and answers `None == '*' or None == -assumed`, which is `False` for every clientId, so `enter_client` cannot succeed for any of -them. The `original_client_id is None` escape in the unvalidated branch is unreachable once -CONNECTED has arrived. The branch should apply only where a clientId was configured, and -`__client_id` should become `'*'` otherwise, which is what RSA7b4 asks for. - -**Tests affected:** none, which is why this is recorded here rather than gated. The three -`batch_presence.md` tests build their realtime client with `client_id='*'` instead, in -`entering_client()`, with the diagnosis in its docstring — the repository's own presence -suite carries the same workaround for the same reason, commented "Use wildcard auth for -enterClient" (`test/ably/realtime/realtimepresence_test.py:394-396`). It is setup rather -than subject: those tests are about `batchPresence`, and nothing they assert depends on how -the members got there. A fix would let the setup halves be written exactly as the -specification writes them. Both `realtime/integration` presence specifications carry the -same workaround for the same reason — `presence_lifecycle_test.py::test_rtp4_bulk_enter_observed` -and `presence/presence_sync_test.py::test_rtp2_sync_multiple_members` each build the client -that enters members on behalf of others with `client_id='*'` — which makes five tests -across two tiers that would otherwise be written as their specifications write them. - -This is distinct from the wildcard-clientId contradiction recorded under UTS Spec Errors, -which is about a client that *does* configure `clientId: "*"`. - -**Status:** open bug, one line from a fix. - #### Auth behaviour on the REST client | Spec points | Behaviour | @@ -2933,9 +2886,9 @@ none of these shows up as a failure — which is why they are easy to lose. The five tiers above classify the realtime unit derivation; the REST unit derivation's candidates went upstream as [#709](https://github.com/ably/ably-python/issues/709)–[#712](https://github.com/ably/ably-python/issues/712). -Five further defects came out of the two integration tiers, and none of them is filed; each -keeps the label it was first given. I.1 and I.2 are tier 2 by the ranking above — an error -where there should be none — and I.3 is tier 1 for a client that configures the option it +Four further defects came out of the two integration tiers, and none of them is filed; each +keeps the label it was first given. I.1 is tier 2 by the ranking above — an error where +there should be none — and I.3 is tier 1 for a client that configures the option it concerns, since the call does not come back when the caller asked for it to. I.6 is tier 2 as well; I.5 is tier 3, a presence member silently leaving the set with nothing to report it. @@ -2956,15 +2909,6 @@ offset. The same expired token renews correctly through `publish()`. A fix has t the two meanings `raise_on_error` carries. `test/uts/rest/integration/auth_test.py -k rsc10` -**I.2 `enterClient` cannot succeed on a basic-auth connection.** RSA7b4, RTP14, RTP15. A -connection built from a key alone is told `clientId: "*"`, and the branch of -`Auth._configure_client_id` (`ably/rest/auth.py:335-353`) that guards a configured clientId -against a server wildcard fires when there is no configured clientId, recording it as -validated and `None`. `can_assume_client_id` then refuses every clientId. One line from a -fix, and `enter_client` is unusable without one. No test gates on it — the three tests that -would hit it pass `client_id='*'` in setup instead, as the repository's own presence suite -does — so it will not show up as a failure. - **I.3 `httpRequestTimeout` is applied as seconds where the specification counts milliseconds.** TO3l4, TO3l3, RSC15l2. `ably/http/http.py:193` hands `(http_open_timeout, http_request_timeout)` to `httpx`, which reads seconds, so a client diff --git a/test/uts/realtime/integration/presence/presence_sync_test.py b/test/uts/realtime/integration/presence/presence_sync_test.py index 20a46ed6..c8101e78 100644 --- a/test/uts/realtime/integration/presence/presence_sync_test.py +++ b/test/uts/realtime/integration/presence/presence_sync_test.py @@ -11,10 +11,6 @@ The specification is json only — it has no `## Protocol Variants` section — so these two take the JSON default `sandbox_realtime_client` applies rather than the `use_binary_protocol` fixture. - -`enter_client` on a connection authenticated by key alone is refused 40012, a root cause -recorded in [deviations.md](../../../deviations.md), so the client entering members on -behalf of others is built with `client_id='*'`. """ from ably.pubsub.realtime.connection import ConnectionState @@ -61,7 +57,7 @@ async def test_rtp2_sync_multiple_members(realtime_sandbox): api_key = realtime_sandbox.key_str channel_name = 'presence-sync-multi-' + random_id() - client_a = sandbox_realtime_client(api_key, auto_connect=False, client_id='*') + client_a = sandbox_realtime_client(api_key, auto_connect=False) client_b = sandbox_realtime_client(api_key, auto_connect=False) client_a.connect() diff --git a/test/uts/realtime/integration/presence_lifecycle_test.py b/test/uts/realtime/integration/presence_lifecycle_test.py index 0a8a7c70..1883bb79 100644 --- a/test/uts/realtime/integration/presence_lifecycle_test.py +++ b/test/uts/realtime/integration/presence_lifecycle_test.py @@ -10,14 +10,10 @@ attach, and a presence action is named by its lowercase wire name — `'enter'`, `'present'`, `'update'`, `'leave'` — which is what `set_presence` emits. -Two adaptations, both of them root causes already recorded in -[deviations.md](../../deviations.md): - -- Subscribing to several actions at once (RTP6b) raises `TypeError: unhashable type: - 'list'`, because the list reaches pyee as a dict key. The one listener is registered - once per action instead, which is what the array form means. -- `enter_client` on a connection authenticated by key alone is refused 40012, so the - client entering members on behalf of others is built with `client_id='*'`. +One adaptation, a root cause already recorded in [deviations.md](../../deviations.md): +subscribing to several actions at once (RTP6b) raises `TypeError: unhashable type: +'list'`, because the list reaches pyee as a dict key. The one listener is registered once +per action instead, which is what the array form means. """ import asyncio @@ -45,8 +41,7 @@ async def test_rtp4_bulk_enter_observed(realtime_sandbox, use_binary_protocol): api_key = realtime_sandbox.key_str channel_name = 'presence-bulk-' + random_id() - client_a = sandbox_realtime_client( - api_key, use_binary_protocol=use_binary_protocol, client_id='*') + client_a = sandbox_realtime_client(api_key, use_binary_protocol=use_binary_protocol) client_b = sandbox_realtime_client(api_key, use_binary_protocol=use_binary_protocol) client_a.connect() diff --git a/test/uts/rest/integration/batch_presence_test.py b/test/uts/rest/integration/batch_presence_test.py index ecf2d93c..9feb49fb 100644 --- a/test/uts/rest/integration/batch_presence_test.py +++ b/test/uts/rest/integration/batch_presence_test.py @@ -44,19 +44,13 @@ def member_for(entry, client_id): async def entering_client(key, use_binary_protocol): """A CONNECTED realtime client that may enter presence for any clientId. - The specifications build this with the full-access key alone and call - `enterClient` straight away. ably-python needs `client_id='*'` on top: a basic-auth - connection is told `clientId: "*"` by the server, and `Auth._configure_client_id` - answers a wildcard from the server by marking the client id validated while leaving - it `None` (`ably/rest/auth.py:335`), after which `can_assume_client_id` refuses every - id and `enter_client` raises 40012. The repository's own presence suite carries the - same `client_id='*'` for the same reason - (`test/ably/realtime/realtimepresence_test.py:396`). + The specifications build this with the full-access key alone, which the server + answers with the wildcard clientId `"*"` (RSA7b4). The connection is awaited to CONNECTED before any enter, since a presence enter on a connection that is still CONNECTING is queued rather than sent. """ - client = sandbox_realtime_client(key, client_id='*', use_binary_protocol=use_binary_protocol) + client = sandbox_realtime_client(key, use_binary_protocol=use_binary_protocol) await await_connection_state(client, ConnectionState.CONNECTED) return client From 2e32cd1bc155157cdb6c488544bc9e3858c8842e Mon Sep 17 00:00:00 2001 From: owenpearson Date: Mon, 26 Oct 2026 18:17:18 +0000 Subject: [PATCH 42/71] pubsub: read the effective http defaults back on the client options RTC7 has a client use the timeouts configured in its ClientOptions, falling back to the library defaults, and the UTS test for it reads those defaults back from client.options. The realtime timeouts were already resolved on Options, but http_open_timeout, http_request_timeout, http_max_retry_count and http_max_retry_duration were left as None there and defaulted later, by Http from its CONNECTION_RETRY_DEFAULTS and by the host list from Defaults.http_max_retry_count, so client.options could not say what a client was going to use. The four defaults now live on Defaults and are resolved by Options, the same way as the realtime ones, and Http reads them from the options. CONNECTION_RETRY_DEFAULTS is gone. The values are unchanged and stay in seconds, so the TO3l3 and TO3l4 unit mismatch remains recorded as a deviation in test/uts/deviations.md. This is a breaking change for code that read these options and took None to mean unset: they now read 4, 10, 3 and 15 by default. UPDATING.md says so. Co-Authored-By: Claude Opus 5.5 --- UPDATING.md | 8 +++++++ ably/pubsub/request/http.py | 23 ++++--------------- ably/pubsub/transport/defaults.py | 4 ++++ ably/pubsub/types/options.py | 15 ++++++++++-- test/ably/http/httphttp_test.py | 6 ++--- test/uts/deviations.md | 14 +++++------ .../unit/client/realtime_timeouts_test.py | 11 ++++----- .../uts/rest/unit/types/options_types_test.py | 12 +++++----- 8 files changed, 49 insertions(+), 44 deletions(-) diff --git a/UPDATING.md b/UPDATING.md index 94f4182d..dd8c67f2 100644 --- a/UPDATING.md +++ b/UPDATING.md @@ -405,6 +405,14 @@ pubsub_http_client = create_http_client(key='xxx', default_token_params={'capabi await pubsub_http_client.auth.request_token({'ttl': 600000, 'capability': capability}) ``` +### The HTTP options read back their defaults + +`http_open_timeout`, `http_request_timeout`, `http_max_retry_count` and +`http_max_retry_duration` on `client.options` used to read `None` unless you set them, while +the client applied defaults of its own. They now read back the values in effect: 4, 10, 3 and +15 by default. The timeouts and the retry duration are still in seconds. Code that tested an +option for `None`, to tell whether it had been set, will no longer find it so. + ## Version 2.x to 3.0.0 The 3.0.0 version of ably-python introduces several breaking changes to improve the realtime experience and align the API with the Ably specification. These include: diff --git a/ably/pubsub/request/http.py b/ably/pubsub/request/http.py index cd8b54d4..5b69c336 100644 --- a/ably/pubsub/request/http.py +++ b/ably/pubsub/request/http.py @@ -126,13 +126,6 @@ def __getattr__(self, attr): class Http: - CONNECTION_RETRY_DEFAULTS = { - 'http_open_timeout': 4, - 'http_request_timeout': 10, - 'http_max_retry_count': 3, - 'http_max_retry_duration': 15, - } - def __init__(self, ably, options): options = options or {} self.__ably = ably @@ -318,24 +311,16 @@ def preferred_scheme(self): @property def http_open_timeout(self): - if self.options.http_open_timeout is not None: - return self.options.http_open_timeout - return self.CONNECTION_RETRY_DEFAULTS['http_open_timeout'] + return self.options.http_open_timeout @property def http_request_timeout(self): - if self.options.http_request_timeout is not None: - return self.options.http_request_timeout - return self.CONNECTION_RETRY_DEFAULTS['http_request_timeout'] + return self.options.http_request_timeout @property def http_max_retry_count(self): - if self.options.http_max_retry_count is not None: - return self.options.http_max_retry_count - return self.CONNECTION_RETRY_DEFAULTS['http_max_retry_count'] + return self.options.http_max_retry_count @property def http_max_retry_duration(self): - if self.options.http_max_retry_duration is not None: - return self.options.http_max_retry_duration - return self.CONNECTION_RETRY_DEFAULTS['http_max_retry_duration'] + return self.options.http_max_retry_duration diff --git a/ably/pubsub/transport/defaults.py b/ably/pubsub/transport/defaults.py index ae5310e6..d239656a 100644 --- a/ably/pubsub/transport/defaults.py +++ b/ably/pubsub/transport/defaults.py @@ -19,7 +19,11 @@ class Defaults: transports = [] # ["web_socket", "comet"] + # The HTTP timeouts are in seconds, unlike the realtime ones above + http_open_timeout = 4 + http_request_timeout = 10 http_max_retry_count = 3 + http_max_retry_duration = 15 fallback_retry_timeout = 600000 # 10min diff --git a/ably/pubsub/types/options.py b/ably/pubsub/types/options.py index 6518744d..76b385b7 100644 --- a/ably/pubsub/types/options.py +++ b/ably/pubsub/types/options.py @@ -55,6 +55,19 @@ def __init__(self, client_id=None, log_level=0, tls=True, rest_host=None, realti if realtime_request_timeout is None: realtime_request_timeout = Defaults.realtime_request_timeout + # RTC7: the HTTP defaults are resolved here too, so they read back from the options + if http_open_timeout is None: + http_open_timeout = Defaults.http_open_timeout + + if http_request_timeout is None: + http_request_timeout = Defaults.http_request_timeout + + if http_max_retry_count is None: + http_max_retry_count = Defaults.http_max_retry_count + + if http_max_retry_duration is None: + http_max_retry_duration = Defaults.http_max_retry_duration + if disconnected_retry_timeout is None: disconnected_retry_timeout = Defaults.disconnected_retry_timeout @@ -329,8 +342,6 @@ def __get_hosts(self): fallback_hosts = self.get_fallback_hosts() http_max_retry_count = self.http_max_retry_count - if http_max_retry_count is None: - http_max_retry_count = Defaults.http_max_retry_count # Shuffle fallback_hosts = list(fallback_hosts) diff --git a/test/ably/http/httphttp_test.py b/test/ably/http/httphttp_test.py index 91a39146..d2167c3b 100644 --- a/test/ably/http/httphttp_test.py +++ b/test/ably/http/httphttp_test.py @@ -30,8 +30,8 @@ async def test_host_for_url_brackets_only_ipv6_literals(self): async def test_max_retry_attempts_and_timeouts_defaults(self): ably = create_http_client(token="foo") - assert 'http_open_timeout' in ably.http.CONNECTION_RETRY_DEFAULTS - assert 'http_request_timeout' in ably.http.CONNECTION_RETRY_DEFAULTS + assert ably.options.http_open_timeout == Defaults.http_open_timeout + assert ably.options.http_request_timeout == Defaults.http_request_timeout with mock.patch('httpx.AsyncClient.send', side_effect=httpx.RequestError('')) as send_mock: with pytest.raises(AblyException, match="network error"): @@ -43,7 +43,7 @@ async def test_max_retry_attempts_and_timeouts_defaults(self): async def test_cumulative_timeout(self): ably = create_http_client(token="foo") - assert 'http_max_retry_duration' in ably.http.CONNECTION_RETRY_DEFAULTS + assert ably.options.http_max_retry_duration == Defaults.http_max_retry_duration ably.options.http_max_retry_duration = 0.5 diff --git a/test/uts/deviations.md b/test/uts/deviations.md index bcd2df0f..703f16b0 100644 --- a/test/uts/deviations.md +++ b/test/uts/deviations.md @@ -1810,7 +1810,7 @@ final response is raised or returned. `ably/http/http.py:193` builds `timeout = (self.http_open_timeout, self.http_request_timeout)` and hands it to `httpx`, which reads both as **seconds**. TO3l4's `httpRequestTimeout` is milliseconds, default 10000, and -`CONNECTION_RETRY_DEFAULTS` holds `10`. So a caller passing the value the published +`Defaults.http_request_timeout` holds `10`. So a caller passing the value the published specification describes gets a deadline a thousand times longer than the one asked for: `http_request_timeout=3000` is three thousand seconds. The **defaults** come out right by coincidence — 10 seconds is TO3l4's 10000 ms, and `http_open_timeout`'s 4 is TO3l3's @@ -1825,9 +1825,9 @@ Passing `3` in its place makes the same test pass in 3.1 seconds: the timeout fi retry goes to the fallback host and succeeds. So RSC15l2's fallback path is compliant and the unit is the whole of the defect. -The same mismatch is recorded twice under *Adapted Tests*, at `TO3l1, TO3l5` and at -`RTC7 (TO3l3, TO3l4)` in *REST behaviours asserted as they are*, where it is what makes -the effective defaults unreadable from `options`. This is that defect seen from outside: +The same mismatch is recorded twice under *Adapted Tests*, at `TO3l4, TO3l5` and at +`RTC7 (TO3l3, TO3l4)` in *REST behaviours asserted as they are*, where it is why the +defaults read back from `options` as 4 and 10. This is that defect seen from outside: the same line of `http.py`, reached through a public client option rather than through an attribute, so a caller is affected whether or not they ever read `options`. It is counted as one root cause, here, because it is the only gated test on it; neither Adapted Tests @@ -1839,7 +1839,7 @@ row is counted again. fixed. **Status:** open bug. The fix converts at the boundary — `Http` dividing the option by -1000 before it reaches `httpx`, with `CONNECTION_RETRY_DEFAULTS` restated in +1000 before it reaches `httpx`, with the `Defaults` restated in milliseconds — and has to move `http_open_timeout` (TO3l3) with it, since both halves of the tuple are read the same way. That same line carries a second, filed defect: [#709](https://github.com/ably/ably-python/issues/709) is that the value, in whatever @@ -2217,8 +2217,8 @@ pseudocode is not mistaken for non-compliance. | CHM2g, CHM2h | `objectPublishers` and `objectSubscribers` on `ChannelMetrics` | Neither is modelled, so both are dropped on parsing. The test asserts their absence, and turns red once they are added | Open bug | | TO3l8 | `maxMessageSize` is a client option, default 65536 | Rejected by `Options.__init__`. `ably/realtime/channel.py:422` reads it with `getattr(..., 65536)`, so the default holds but cannot be configured, nor overridden by `connectionDetails` (CD2c) | Open bug | | RTN15, RTN23 | A DISCONNECTED `ErrorInfo` needs no `statusCode` | `ConnectionManager.on_disconnected` evaluates `exception.status_code >= 500` unguarded, so a DISCONNECTED whose error omits `statusCode` raises `TypeError` in a task whose exception is only logged, and the connection silently stays CONNECTED. `DISCONNECTED_MESSAGE` supplies 400 | Open bug | -| TO3l1, TO3l5 | `httpRequestTimeout` and `httpMaxRetryCount` carry their defaults on the options object | Left unset; the effective defaults are applied downstream by `Http` and by `Options.__get_hosts`. The spec's `httpRequestTimeout` is milliseconds and ably-python's `http_request_timeout` is seconds, on the value a caller passes as much as on the default — gated, with the measurement, under *`httpRequestTimeout` is seconds where the specification counts milliseconds* in Failing Tests | Intentional for where the defaults are applied; the unit is an open bug, recorded there | -| RTC7 (TO3l3, TO3l4) | `client.options.httpOpenTimeout == 4000` and `httpRequestTimeout == 10000` | Both `None` on `Options`; `Http.http_open_timeout` / `http_request_timeout` fall back to `CONNECTION_RETRY_DEFAULTS`, which holds 4 and 10 — seconds, where TO3l3 and TO3l4 count milliseconds. The three realtime timeouts the same test checks are defaulted on `Options` and match | Two faults in one row: the defaults are unreadable from `options`, and the unit reaches the wire — `rest/proxy/RSC15l2/timeout-triggers-fallback-0` measures a request outliving its configured timeout by a factor of a thousand. Both open; the unit is gated under *`httpRequestTimeout` is seconds where the specification counts milliseconds* in Failing Tests | +| TO3l4, TO3l5 | `httpRequestTimeout` defaults to 10000 and `httpMaxRetryCount` to 3 on the options object | Both are defaulted on `Options`. `http_max_retry_count` reads back as 3 and matches; `http_request_timeout` reads back as 10, because ably-python's is seconds where the spec's is milliseconds, on the value a caller passes as much as on the default — gated, with the measurement, under *`httpRequestTimeout` is seconds where the specification counts milliseconds* in Failing Tests | The unit is an open bug, recorded there | +| RTC7 (TO3l3, TO3l4) | `client.options.httpOpenTimeout == 4000` and `httpRequestTimeout == 10000` | 4 and 10: both are defaulted on `Options`, in seconds where TO3l3 and TO3l4 count milliseconds. The three realtime timeouts the same test checks are milliseconds and match | Open bug, and the unit reaches the wire — `rest/proxy/RSC15l2/timeout-triggers-fallback-0` measures a request outliving its configured timeout by a factor of a thousand. Gated under *`httpRequestTimeout` is seconds where the specification counts milliseconds* in Failing Tests. Moving the HTTP durations to milliseconds is breaking, and is held back from 4.0.0 pending a decision on units across the options | | RTC1f | a `transportParams` boolean appears as `"true"` / `"false"` | `True` / `False`, because `WebSocketTransport.connect` builds the query string with `urllib.parse.urlencode`, which renders each value through `str()` (`websockettransport.py:89`). Integers are unaffected | Open bug. A caller can pass the strings directly, but a bool is what the spec's Stringifiable type admits | ### LiveObjects: the internal shapes the unit specifications assume — S-1 to S-6 diff --git a/test/uts/realtime/unit/client/realtime_timeouts_test.py b/test/uts/realtime/unit/client/realtime_timeouts_test.py index 3a9c21d9..09da81d4 100644 --- a/test/uts/realtime/unit/client/realtime_timeouts_test.py +++ b/test/uts/realtime/unit/client/realtime_timeouts_test.py @@ -173,10 +173,7 @@ async def test_rtc7_default_timeouts_applied(): assert client.options.suspended_retry_timeout == 30000 # DEVIATION: the spec asserts httpOpenTimeout == 4000 and httpRequestTimeout == - # 10000 on the options. ably-python leaves both unset on the options and holds the - # defaults on the HTTP layer, in seconds rather than milliseconds. See - # deviations.md. - assert client.options.http_open_timeout is None - assert client.options.http_request_timeout is None - assert client.http.http_open_timeout == 4 - assert client.http.http_request_timeout == 10 + # 10000. ably-python's HTTP timeouts are in seconds rather than milliseconds, so the + # same defaults read back as 4 and 10. See deviations.md. + assert client.options.http_open_timeout == 4 + assert client.options.http_request_timeout == 10 diff --git a/test/uts/rest/unit/types/options_types_test.py b/test/uts/rest/unit/types/options_types_test.py index bc5848cd..36fe2e4a 100644 --- a/test/uts/rest/unit/types/options_types_test.py +++ b/test/uts/rest/unit/types/options_types_test.py @@ -37,11 +37,11 @@ async def test_to3_client_options_attributes(): assert options.auth_method == 'GET' assert options.tls is True - # DEVIATION: httpRequestTimeout and httpMaxRetryCount have no default on the options - # object; both are left unset and the effective defaults are applied downstream, the - # first by Http and the second by Options.__get_hosts via Defaults.http_max_retry_count. - assert options.http_request_timeout is None - assert options.http_max_retry_count is None + # DEVIATION: the spec asserts httpRequestTimeout == 10000. ably-python's + # http_request_timeout is in seconds rather than milliseconds, so the same default + # reads back as 10. See deviations.md. + assert options.http_request_timeout == 10 + assert options.http_max_retry_count == 3 assert options.use_binary_protocol is True assert options.idempotent_rest_publishing is True @@ -54,7 +54,7 @@ async def test_to3_client_options_attributes(): with pytest.raises(TypeError): Options(max_message_size=65536) - # The effective defaults the options leave unset: 10 seconds, and 3 hosts to try. + # The HTTP layer applies the defaults the options hold: 10 seconds, and 3 hosts to try. client = rest_client(mock_http_client()) assert client.http.http_request_timeout == 10 assert len(client.options.get_hosts()) == 3 From e35b7569e5c7bf43b5636843e8abf6168d42650f Mon Sep 17 00:00:00 2001 From: owenpearson Date: Mon, 26 Oct 2026 18:19:32 +0000 Subject: [PATCH 43/71] pubsub: remove the per-call timeout argument from requests time(), stats(), the REST channel's publish variants, get_message() on the REST and realtime channels and push.admin.publish() each accepted a timeout keyword argument and passed it down to Http.make_request, which replaced it with the client's (http_open_timeout, http_request_timeout) before building the request. So the argument has never had any effect. stats() also took a paginated argument that nothing read. Both arguments are removed, from the clients, the prototypes and the Http get, post, put, patch and delete helpers and make_request, which no caller needs them for. RTC7 has the timeouts come from the ClientOptions, which is where they were always taken from. This is a breaking change: passing timeout or paginated now raises TypeError. Callers should drop the argument and set http_request_timeout or http_open_timeout on the client instead, as UPDATING.md describes. Co-Authored-By: Claude Opus 5.5 --- UPDATING.md | 20 ++++++++++++++++++++ ably/pubsub/http/channel.py | 16 ++++++++-------- ably/pubsub/http/http.py | 6 +++--- ably/pubsub/http/push.py | 6 ++---- ably/pubsub/prototypes.py | 4 ++-- ably/pubsub/realtime/channel.py | 6 ++---- ably/pubsub/request/http.py | 22 +++++++++++----------- 7 files changed, 48 insertions(+), 32 deletions(-) diff --git a/UPDATING.md b/UPDATING.md index dd8c67f2..58798e18 100644 --- a/UPDATING.md +++ b/UPDATING.md @@ -413,6 +413,26 @@ the client applied defaults of its own. They now read back the values in effect: 15 by default. The timeouts and the retry duration are still in seconds. Code that tested an option for `None`, to tell whether it had been set, will no longer find it so. +### Requests no longer take a `timeout` argument + +`time()`, `stats()`, a REST channel's `publish()`, `publish_message()`, `publish_messages()` +and `publish_name_data()`, `get_message()` on REST and realtime channels, and +`push.admin.publish()` accepted a `timeout` keyword argument that was never applied: every +request used the client's `http_open_timeout` and `http_request_timeout` whatever was passed. +The argument is gone, and passing it raises `TypeError`. `stats()` also loses its unused +`paginated` argument. Set the timeouts on the client instead: + +Example 3.x code: +```python +await rest.time(timeout=5) +``` + +Example 4.0.0 code: +```python +pubsub_http_client = create_http_client(key='xxx', http_request_timeout=5) +await pubsub_http_client.time() +``` + ## Version 2.x to 3.0.0 The 3.0.0 version of ably-python introduces several breaking changes to improve the realtime experience and align the API with the Ably specification. These include: diff --git a/ably/pubsub/http/channel.py b/ably/pubsub/http/channel.py index 321ea275..e2bd13a0 100644 --- a/ably/pubsub/http/channel.py +++ b/ably/pubsub/http/channel.py @@ -95,10 +95,10 @@ async def _publish(self, arg, *args, **kwargs): else: raise TypeError(f'Unexpected type {type(arg)}') - async def publish_message(self, message, params=None, timeout=None): - return await self.publish_messages([message], params, timeout=timeout) + async def publish_message(self, message, params=None): + return await self.publish_messages([message], params) - async def publish_messages(self, messages, params=None, timeout=None): + async def publish_messages(self, messages, params=None): request_body = self.__publish_request_body(messages) if not self.ably.options.use_binary_protocol: request_body = json.dumps(request_body, separators=(',', ':')) @@ -109,7 +109,7 @@ async def publish_messages(self, messages, params=None, timeout=None): if params: params = {k: str(v).lower() if isinstance(v, bool) else v for k, v in params.items()} path += '?' + parse.urlencode(params) - response = await self.ably.http.post(path, body=request_body, timeout=timeout) + response = await self.ably.http.post(path, body=request_body) # Parse response to extract serials result_data = response.to_native() @@ -117,9 +117,9 @@ async def publish_messages(self, messages, params=None, timeout=None): return PublishResult.from_dict(result_data) return PublishResult() - async def publish_name_data(self, name, data, timeout=None): + async def publish_name_data(self, name, data): messages = [Message(name, data)] - return await self.publish_messages(messages, timeout=timeout) + return await self.publish_messages(messages) async def publish(self, *args, **kwargs): """Publishes a message on this channel. @@ -259,7 +259,7 @@ async def append_message(self, message: Message, operation: MessageOperation = N """ return await self._send_update(message, MessageAction.MESSAGE_APPEND, operation, params) - async def get_message(self, serial_or_message, timeout=None): + async def get_message(self, serial_or_message): """Retrieves a single message by its serial. Parameters: @@ -291,7 +291,7 @@ async def get_message(self, serial_or_message, timeout=None): path = self.__base_path + 'messages/' + parse.quote_plus(serial, safe=':') # Make the request - response = await self.ably.http.get(path, timeout=timeout) + response = await self.ably.http.get(path) # Create Message from the response message_handler = make_single_message_response_handler(self.__cipher) diff --git a/ably/pubsub/http/http.py b/ably/pubsub/http/http.py index 5c14f7bd..dd738c1a 100644 --- a/ably/pubsub/http/http.py +++ b/ably/pubsub/http/http.py @@ -103,7 +103,7 @@ async def __aenter__(self): @catch_all async def stats(self, direction: Optional[str] = None, start=None, end=None, params: Optional[dict] = None, - limit: Optional[int] = None, paginated=None, unit=None, timeout=None): + limit: Optional[int] = None, unit=None): """Returns the stats for this application""" formatted_params = format_params(params, direction=direction, start=start, end=end, limit=limit, unit=unit) url = '/stats' + formatted_params @@ -111,9 +111,9 @@ async def stats(self, direction: Optional[str] = None, start=None, end=None, par self.http, url=url, response_processor=stats_response_processor) @catch_all - async def time(self, timeout: Optional[float] = None) -> float: + async def time(self) -> float: """Returns the current server time in ms since the unix epoch""" - r = await self.http.get('/time', skip_auth=True, timeout=timeout) + r = await self.http.get('/time', skip_auth=True) AblyException.raise_for_response(r) return r.to_native()[0] diff --git a/ably/pubsub/http/push.py b/ably/pubsub/http/push.py index a127de1f..1fc6ea98 100644 --- a/ably/pubsub/http/push.py +++ b/ably/pubsub/http/push.py @@ -1,5 +1,3 @@ -from typing import Optional - from ably.pubsub.request.paginatedresult import PaginatedResult, format_params from ably.pubsub.types.channelsubscription import ( PushChannelSubscription, @@ -40,7 +38,7 @@ def device_registrations(self): def channel_subscriptions(self): return self.__channel_subscriptions - async def publish(self, recipient: dict, data: dict, timeout: Optional[float] = None): + async def publish(self, recipient: dict, data: dict): """Publish a push notification to a single device. :Parameters: @@ -61,7 +59,7 @@ async def publish(self, recipient: dict, data: dict, timeout: Optional[float] = body = data.copy() body.update({'recipient': recipient}) - await self.ably.http.post('/push/publish', body=body, timeout=timeout) + await self.ably.http.post('/push/publish', body=body) class PushDeviceRegistrations: diff --git a/ably/pubsub/prototypes.py b/ably/pubsub/prototypes.py index 4fc2613e..3ec07b9d 100644 --- a/ably/pubsub/prototypes.py +++ b/ably/pubsub/prototypes.py @@ -65,11 +65,11 @@ def push(self) -> Push: ... async def stats(self, direction: str | None = None, start=None, end=None, params: dict | None = None, - limit: int | None = None, paginated=None, unit=None, timeout=None) -> PaginatedResult: + limit: int | None = None, unit=None) -> PaginatedResult: """Return the stats for this application.""" ... - async def time(self, timeout: float | None = None) -> float: + async def time(self) -> float: """Return the current server time in ms since the unix epoch.""" ... diff --git a/ably/pubsub/realtime/channel.py b/ably/pubsub/realtime/channel.py index 17796cf2..7da4ed1f 100644 --- a/ably/pubsub/realtime/channel.py +++ b/ably/pubsub/realtime/channel.py @@ -689,15 +689,13 @@ async def append_message( """ return await self._send_update(message, MessageAction.MESSAGE_APPEND, operation, params) - async def get_message(self, serial_or_message, timeout=None): + async def get_message(self, serial_or_message): """Retrieves a single message by its serial using the REST API. Parameters ---------- serial_or_message : str or Message Either a string serial or a Message object with a serial field. - timeout : float, optional - Timeout for the request. Returns ------- @@ -710,7 +708,7 @@ async def get_message(self, serial_or_message, timeout=None): If the serial is missing or the message cannot be retrieved. """ # Delegate to parent Channel (REST) implementation - return await Channel.get_message(self, serial_or_message, timeout=timeout) + return await Channel.get_message(self, serial_or_message) async def get_message_versions(self, serial_or_message, params=None): """Retrieves version history for a message using the REST API. diff --git a/ably/pubsub/request/http.py b/ably/pubsub/request/http.py index 5b69c336..8118d659 100644 --- a/ably/pubsub/request/http.py +++ b/ably/pubsub/request/http.py @@ -172,7 +172,7 @@ def get_hosts(self): @reauth_if_expired async def make_request(self, method, path, version=None, headers=None, body=None, - skip_auth=False, timeout=None, raise_on_error=True): + skip_auth=False, raise_on_error=True): if body is not None and type(body) not in (bytes, str): body = self.dump_body(body) @@ -260,29 +260,29 @@ async def request_external(self, method, url, headers=None, params=None, body=No """ return await self.__client.request(method=method, url=url, headers=headers, params=params, data=body) - async def delete(self, url, headers=None, skip_auth=False, timeout=None): + async def delete(self, url, headers=None, skip_auth=False): result = await self.make_request('DELETE', url, headers=headers, - skip_auth=skip_auth, timeout=timeout) + skip_auth=skip_auth) return result - async def get(self, url, headers=None, skip_auth=False, timeout=None): + async def get(self, url, headers=None, skip_auth=False): result = await self.make_request('GET', url, headers=headers, - skip_auth=skip_auth, timeout=timeout) + skip_auth=skip_auth) return result - async def patch(self, url, headers=None, body=None, skip_auth=False, timeout=None): + async def patch(self, url, headers=None, body=None, skip_auth=False): result = await self.make_request('PATCH', url, headers=headers, body=body, - skip_auth=skip_auth, timeout=timeout) + skip_auth=skip_auth) return result - async def post(self, url, headers=None, body=None, skip_auth=False, timeout=None): + async def post(self, url, headers=None, body=None, skip_auth=False): result = await self.make_request('POST', url, headers=headers, body=body, - skip_auth=skip_auth, timeout=timeout) + skip_auth=skip_auth) return result - async def put(self, url, headers=None, body=None, skip_auth=False, timeout=None): + async def put(self, url, headers=None, body=None, skip_auth=False): result = await self.make_request('PUT', url, headers=headers, body=body, - skip_auth=skip_auth, timeout=timeout) + skip_auth=skip_auth) return result @property From 5b8be73e25961aa8b905dd3ccf52c3c7a4b9419b Mon Sep 17 00:00:00 2001 From: owenpearson Date: Mon, 26 Oct 2026 18:22:07 +0000 Subject: [PATCH 44/71] pubsub: bound the write and pool phases of an http request Http.make_request passed httpx the tuple (http_open_timeout, http_request_timeout), which httpx expands to a connect and a read timeout and leaves write and pool as None. Writing a request body to a wedged connection, or waiting for a free pooled connection, therefore never timed out, whatever the client's options said. This is one of the three parts of issue #709. The timeout is now an explicit httpx.Timeout: connect takes http_open_timeout (TO3l3), and read, write and pool take http_request_timeout (TO3l4). The other parts of #709, that the read timeout applies per socket read and that the RSC15 retry budget is not consulted while an attempt is in flight, are left as they are, as is the unit of both options, which stays seconds. This is a breaking change in behaviour: a request that used to hang in either phase now raises httpx.TimeoutException after http_request_timeout and is retried against a fallback host. Callers who send large bodies over slow links may need to raise http_request_timeout. Co-Authored-By: Claude Opus 5.5 --- UPDATING.md | 13 +++++++++++++ ably/pubsub/request/http.py | 3 ++- test/unit/http_test.py | 12 ++++++++++++ test/uts/deviations.md | 21 ++++++++++++--------- 4 files changed, 39 insertions(+), 10 deletions(-) diff --git a/UPDATING.md b/UPDATING.md index 58798e18..9db3867e 100644 --- a/UPDATING.md +++ b/UPDATING.md @@ -433,6 +433,19 @@ pubsub_http_client = create_http_client(key='xxx', http_request_timeout=5) await pubsub_http_client.time() ``` +### `http_request_timeout` bounds writes and connection-pool waits + +In 3.x, `http_open_timeout` bounded connecting and `http_request_timeout` bounded each read +of a response, while writing a request body and waiting for a free pooled connection had no +timeout at all. `http_request_timeout` now bounds those too, so a request stuck writing to, +or waiting for, a wedged connection raises `httpx.TimeoutException` and is retried against a +fallback host instead of hanging. If you publish large bodies over a slow link, raise +`http_request_timeout` to fit: + +```python +pubsub_http_client = create_http_client(key='xxx', http_request_timeout=30) +``` + ## Version 2.x to 3.0.0 The 3.0.0 version of ably-python introduces several breaking changes to improve the realtime experience and align the API with the Ably specification. These include: diff --git a/ably/pubsub/request/http.py b/ably/pubsub/request/http.py index 8118d659..9e19e6e0 100644 --- a/ably/pubsub/request/http.py +++ b/ably/pubsub/request/http.py @@ -195,7 +195,8 @@ async def make_request(self, method, path, version=None, headers=None, body=None if headers: all_headers.update(headers) - timeout = (self.http_open_timeout, self.http_request_timeout) + # TO3l3 bounds opening the connection, and TO3l4 each read, write and wait for a pooled connection + timeout = httpx.Timeout(self.http_request_timeout, connect=self.http_open_timeout) http_max_retry_duration = self.http_max_retry_duration requested_at = self.__clock.now_ms() diff --git a/test/unit/http_test.py b/test/unit/http_test.py index c5be5803..70608c81 100644 --- a/test/unit/http_test.py +++ b/test/unit/http_test.py @@ -51,6 +51,18 @@ async def test_http_sends_requests_through_an_injected_transport(): await ably.close() +async def test_http_bounds_every_phase_of_a_request(): + transport = RecordingTransport(lambda request: httpx.Response(200, json=[1500000000000])) + ably = create_http_client(token="foo", http_open_timeout=2, http_request_timeout=7, + _test_options=TestOptions(http_transport=transport)) + + await ably.time() + + # TO3l3 bounds the connect, and TO3l4 the read, the write and the wait for a pooled connection + assert transport.requests[0].extensions['timeout'] == {'connect': 2, 'read': 7, 'write': 7, 'pool': 7} + await ably.close() + + async def test_http_surfaces_transport_connection_errors(): def refuse(request): raise httpx.ConnectError("connection refused", request=request) diff --git a/test/uts/deviations.md b/test/uts/deviations.md index 703f16b0..ab73ca78 100644 --- a/test/uts/deviations.md +++ b/test/uts/deviations.md @@ -1807,8 +1807,9 @@ final response is raised or returned. **Spec points:** TO3l4, RSC15l2, `rest/proxy/RSC15l2/timeout-triggers-fallback-0`. -`ably/http/http.py:193` builds `timeout = (self.http_open_timeout, -self.http_request_timeout)` and hands it to `httpx`, which reads both as **seconds**. +`Http.make_request` (`ably/pubsub/request/http.py`) builds `timeout = +httpx.Timeout(self.http_request_timeout, connect=self.http_open_timeout)` and hands it +to `httpx`, which reads both as **seconds**. TO3l4's `httpRequestTimeout` is milliseconds, default 10000, and `Defaults.http_request_timeout` holds `10`. So a caller passing the value the published specification describes gets a deadline a thousand times longer than the one asked for: @@ -1840,13 +1841,15 @@ fixed. **Status:** open bug. The fix converts at the boundary — `Http` dividing the option by 1000 before it reaches `httpx`, with the `Defaults` restated in -milliseconds — and has to move `http_open_timeout` (TO3l3) with it, since both halves of -the tuple are read the same way. That same line carries a second, filed defect: +milliseconds — and has to move `http_open_timeout` (TO3l3) with it, since both are +read the same way. Moving the unit is breaking, and is held back from 4.0.0 pending a +decision on units across the options. That same line carries a second, filed defect: [#709](https://github.com/ably/ably-python/issues/709) is that the value, in whatever unit, bounds one socket read rather than the request, so a connection that keeps -producing frames is never timed out and `write` and `pool` are left unbounded. The two -want fixing together, since both change what one attempt may spend of the RSC15 retry -budget. +producing frames is never timed out. Of #709's three parts, only the unbounded `write` +and `pool` timeouts are fixed, both now taking `http_request_timeout`. The rest of it and +the unit want fixing together, since both change what one attempt may spend of the +RSC15 retry budget. ### Requests @@ -2910,8 +2913,8 @@ the two meanings `raise_on_error` carries. `test/uts/rest/integration/auth_test.py -k rsc10` **I.3 `httpRequestTimeout` is applied as seconds where the specification counts -milliseconds.** TO3l4, TO3l3, RSC15l2. `ably/http/http.py:193` hands -`(http_open_timeout, http_request_timeout)` to `httpx`, which reads seconds, so a client +milliseconds.** TO3l4, TO3l3, RSC15l2. `Http.make_request` hands `http_open_timeout` +and `http_request_timeout` to `httpx`, which reads seconds, so a client built with the specification's `httpRequestTimeout: 3000` waits three thousand seconds. Measured through `uts-proxy` against a session delaying `/time` by 20 seconds: the request sat out the whole delay, succeeded on the primary host and attempted no From 42d83c2b966b18723cd260ecad72d7ad1cbee9df Mon Sep 17 00:00:00 2001 From: owenpearson Date: Mon, 26 Oct 2026 18:26:32 +0000 Subject: [PATCH 45/71] pubsub: count only fallback hosts against http_max_retry_count TO3l5 defines httpMaxRetryCount as the maximum number of fallback hosts to use when a request to the primary host fails. Options built the host list by truncating the primary plus the fallbacks to http_max_retry_count, so it counted the primary too: the default of 3 tried the primary and two fallbacks, one host fewer than ably-js, and http_max_retry_count=0 left the list empty, so every request failed with an AttributeError wrapped as a 50000. The host list is now the primary followed by at most http_max_retry_count of the shuffled fallback hosts, so the default tries four hosts and 0 tries the primary alone. RSC15a's row in test/uts/deviations.md called the old truncation intentional; it now records the four hosts as compliant. This is a breaking change: a client tries one more host per request for the same setting. Callers who want the 3.x number of attempts should pass one less, as UPDATING.md shows. Co-Authored-By: Claude Opus 5.5 --- UPDATING.md | 20 +++++++++++++++++++ ably/pubsub/server/__init__.py | 3 ++- ably/pubsub/types/options.py | 10 +++------- test/ably/http/httphttp_test.py | 7 ++++--- test/unit/http_test.py | 18 +++++++++++++++++ test/uts/deviations.md | 2 +- test/uts/rest/unit/fallback_test.py | 8 ++++---- test/uts/rest/unit/request_test.py | 4 ++-- .../uts/rest/unit/types/options_types_test.py | 5 +++-- 9 files changed, 57 insertions(+), 20 deletions(-) diff --git a/UPDATING.md b/UPDATING.md index 9db3867e..444a265f 100644 --- a/UPDATING.md +++ b/UPDATING.md @@ -446,6 +446,26 @@ fallback host instead of hanging. If you publish large bodies over a slow link, pubsub_http_client = create_http_client(key='xxx', http_request_timeout=30) ``` +### `http_max_retry_count` counts fallback hosts only + +In 3.x, `http_max_retry_count` capped the total number of hosts a request was tried +against, the primary endpoint included, so the default of 3 meant the primary and two +fallbacks. It now counts fallback hosts only, as in the Ably specification and the other +Ably SDKs: a request goes to the primary endpoint and then to at most +`http_max_retry_count` fallback hosts. The default of 3 therefore tries one more host than +before, and `http_max_retry_count=0`, which used to fail every request, now tries the +primary endpoint alone. To keep the 3.x number of attempts, subtract one: + +Example 3.x code: +```python +rest = AblyRest(key='xxx', http_max_retry_count=3) +``` + +Example 4.0.0 code: +```python +pubsub_http_client = create_http_client(key='xxx', http_max_retry_count=2) +``` + ## Version 2.x to 3.0.0 The 3.0.0 version of ably-python introduces several breaking changes to improve the realtime experience and align the API with the Ably specification. These include: diff --git a/ably/pubsub/server/__init__.py b/ably/pubsub/server/__init__.py index cb75bba7..8d2da1f0 100644 --- a/ably/pubsub/server/__init__.py +++ b/ably/pubsub/server/__init__.py @@ -136,7 +136,8 @@ def create_http_client(**kwargs) -> PubSubHttpClient: How long (in seconds) the client keeps retrying a request across hosts before giving up. The default is 15 seconds. http_max_retry_count: int - The most hosts the client will try for a single request. + The most fallback hosts the client will try for a single request, after the + primary endpoint. The default is 3; 0 tries the primary endpoint only. **Protocol** diff --git a/ably/pubsub/types/options.py b/ably/pubsub/types/options.py index 76b385b7..ae448e49 100644 --- a/ably/pubsub/types/options.py +++ b/ably/pubsub/types/options.py @@ -335,23 +335,19 @@ def __get_hosts(self): """ Return the list of hosts as they should be tried. First comes the main host. Then the fallback hosts in random order. - The returned list will have a length of up to http_max_retry_count. + The returned list will hold up to http_max_retry_count fallback hosts. """ host = Defaults.get_hostname(self.endpoint) # REC2: Determine fallback hosts fallback_hosts = self.get_fallback_hosts() - http_max_retry_count = self.http_max_retry_count - # Shuffle fallback_hosts = list(fallback_hosts) random.shuffle(fallback_hosts) self.__fallback_hosts = fallback_hosts - # First main host - hosts = [host] + fallback_hosts - hosts = hosts[:http_max_retry_count] - return hosts + # First main host, then TO3l5: http_max_retry_count counts the fallback hosts only + return [host] + fallback_hosts[:self.http_max_retry_count] def get_hosts(self): return self.__hosts diff --git a/test/ably/http/httphttp_test.py b/test/ably/http/httphttp_test.py index d2167c3b..0336d0ef 100644 --- a/test/ably/http/httphttp_test.py +++ b/test/ably/http/httphttp_test.py @@ -37,7 +37,8 @@ async def test_max_retry_attempts_and_timeouts_defaults(self): with pytest.raises(AblyException, match="network error"): await ably.http.make_request('GET', '/', version=Defaults.protocol_version, skip_auth=True) - assert send_mock.call_count == Defaults.http_max_retry_count + # TO3l5: the primary host, then http_max_retry_count fallbacks + assert send_mock.call_count == Defaults.http_max_retry_count + 1 assert send_mock.call_args == mock.call(mock.ANY) await ably.close() @@ -70,7 +71,7 @@ def make_url(host): with pytest.raises(AblyException, match="network error"): await ably.http.make_request('GET', '/', skip_auth=True) - assert send_mock.call_count == Defaults.http_max_retry_count + assert send_mock.call_count == Defaults.http_max_retry_count + 1 expected_urls_set = { make_url(host) @@ -169,7 +170,7 @@ async def test_500_errors(self): with pytest.raises(AblyException): await ably.http.make_request('GET', '/', skip_auth=True) - assert mock_request.call_count == 3 + assert mock_request.call_count == 4 await ably.close() diff --git a/test/unit/http_test.py b/test/unit/http_test.py index 70608c81..91960630 100644 --- a/test/unit/http_test.py +++ b/test/unit/http_test.py @@ -99,6 +99,24 @@ def time_out(request): await ably.close() +async def test_http_tries_the_primary_and_up_to_max_retry_count_fallbacks(): + def refuse(request): + raise httpx.ConnectError("connection refused", request=request) + + # TO3l5: http_max_retry_count counts the fallback hosts, after the primary + for retry_count, attempts in [(0, 1), (2, 3), (3, 4), (10, 6)]: + transport = RecordingTransport(refuse) + ably = create_http_client(token="foo", http_max_retry_count=retry_count, + _test_options=TestOptions(http_transport=transport)) + + with pytest.raises(AblyException): + await ably.time() + + assert len(transport.requests) == attempts, retry_count + assert transport.requests[0].url.host == ably.options.get_host() + await ably.close() + + def test_http_uses_a_network_transport_without_test_options(): ably = create_http_client(token="foo") assert isinstance(ably.http._Http__client._transport, httpx.AsyncHTTPTransport) diff --git a/test/uts/deviations.md b/test/uts/deviations.md index ab73ca78..f0412b45 100644 --- a/test/uts/deviations.md +++ b/test/uts/deviations.md @@ -2212,7 +2212,7 @@ pseudocode is not mistaken for non-compliance. | RSH1a | Empty `recipient` or `data` rejected with code 40000, the error reaching the caller from the server | `PushAdmin.publish` validates its arguments itself and raises before touching the HTTP layer: an empty or non-dict `recipient` or `data` raises an `AblyException` with 400/40000, the code the server gives, which is also a `ValueError`. So no request is made and there is no server error; the integration tier's `test_rsh1a_push_publish_invalid_recipient` asserts the local error's code | Compliant: RSH1a asks for empty values to be "immediately rejected". The specification's test would need a recipient the SDK will send and the server will reject, an unknown `transportType` say, to exercise the server-side path it describes | | HP6 | `errorCode` is a number | The raw header string, `'40101'` | Open bug, trivial | | HP8 | `headers` is a map | A list of `(name, value)` pairs, so the lookup the spec describes is impossible without converting, and case-insensitivity is lost | Open bug; changing the return type is breaking | -| RSC15a | Six hosts tried | Three. `Options.__get_hosts` truncates to `http_max_retry_count`, which TO3l5 sanctions | Intentional | +| RSC15a | Six hosts tried | Four: the primary and three fallbacks, because TO3l5's `httpMaxRetryCount`, default 3, caps the fallback hosts tried. The specification's six is the number of responses its setup queues, with a comment counting the primary and all five fallbacks, and no assertion reads it | Compliant; the setup's comment overlooks TO3l5 | | TI | `ErrorInfo` equality by attributes | No `__eq__`, so errors compare by identity. Python exceptions conventionally do, and the requirement appears nowhere in `features.md`. Adding `__eq__` without `__hash__` would make `AblyException` unhashable and break any caller that puts one in a set | Intentional | | RSA6b, RSA6d | The capability literal as passed | Canonicalised by `Capability.c14n`, which RSA9f requires | Compliant; rendering only | | RSA8d | `error.message` contains the callback's text | Wrapped as 40170 with the original in `cause`; `__str__` renders both | Rendering | diff --git a/test/uts/rest/unit/fallback_test.py b/test/uts/rest/unit/fallback_test.py index 3cc8eafd..d5f77a38 100644 --- a/test/uts/rest/unit/fallback_test.py +++ b/test/uts/rest/unit/fallback_test.py @@ -82,10 +82,10 @@ async def test_rsc15a_fallback_random_order(): assert all(host in DEFAULT_FALLBACK_HOSTS for host in fallback_hosts_used) assert len(set(fallback_hosts_used)) == len(fallback_hosts_used) - # NOTE: the spec queues six responses, for the primary and all five fallbacks. - # ably-python truncates the host list to http_max_retry_count (3 by default), so - # the primary and two fallbacks are tried. - assert len(requests) == 3 + # NOTE: the spec queues six responses, for the primary and all five fallbacks, and + # asserts no count. TO3l5's http_max_retry_count (3 by default) caps the fallbacks + # tried, so the primary and three fallbacks are. + assert len(requests) == 4 # The spec leaves verifying randomness to the implementation. Ordering is drawn once # per client, so several clients are built and their host orders compared. diff --git a/test/uts/rest/unit/request_test.py b/test/uts/rest/unit/request_test.py index a1bd7a47..20132fa7 100644 --- a/test/uts/rest/unit/request_test.py +++ b/test/uts/rest/unit/request_test.py @@ -19,8 +19,8 @@ SPEC_KEY = 'appId.keyId:keySecret' # The number of hosts a single request is tried against: the primary domain plus -# fallbacks, capped at Defaults.http_max_retry_count -HOST_ATTEMPTS = 3 +# up to Defaults.http_max_retry_count fallbacks (TO3l5) +HOST_ATTEMPTS = 4 def succeed(conn): diff --git a/test/uts/rest/unit/types/options_types_test.py b/test/uts/rest/unit/types/options_types_test.py index 36fe2e4a..471b940a 100644 --- a/test/uts/rest/unit/types/options_types_test.py +++ b/test/uts/rest/unit/types/options_types_test.py @@ -54,10 +54,11 @@ async def test_to3_client_options_attributes(): with pytest.raises(TypeError): Options(max_message_size=65536) - # The HTTP layer applies the defaults the options hold: 10 seconds, and 3 hosts to try. + # The HTTP layer applies the defaults the options hold: 10 seconds, and the primary + # host plus 3 fallbacks to try (TO3l5). client = rest_client(mock_http_client()) assert client.http.http_request_timeout == 10 - assert len(client.options.get_hosts()) == 3 + assert len(client.options.get_hosts()) == 4 options = Options( key='appId.keyId:keySecret', From 4aca1bdb2945e0543e2cbc0fc0319eb1c542004f Mon Sep 17 00:00:00 2001 From: owenpearson Date: Mon, 26 Oct 2026 18:29:07 +0000 Subject: [PATCH 46/71] pubsub: remove the log_level client option TO3b leaves the implementation of logLevel to the platform. ably-python accepted a log_level option, stored it on Options and gave it a property and a setter, but nothing read it, so setting it changed nothing. The library has always logged through the standard logging module, under the ably.pubsub logger, which is where Python applications expect to set a level or attach a handler. The option is removed, and the factory docstrings now say how to configure the ably.pubsub logger instead. No log handler option is added. The RSC2, RSC3, RSC4, TO3b, TO3c and TO3c2 tests stay gated, and their row in test/uts/deviations.md now records that neither option exists. This is a breaking change: passing log_level now raises TypeError. Callers should drop it and configure logging.getLogger('ably.pubsub') instead, as UPDATING.md shows. Co-Authored-By: Claude Opus 5.5 --- UPDATING.md | 23 +++++++++++++++++++++++ ably/pubsub/server/__init__.py | 8 ++++++-- ably/pubsub/server/sync.py | 1 + ably/pubsub/types/options.py | 11 +---------- test/uts/deviations.md | 2 +- test/uts/rest/unit/logging_test.py | 18 ++++++++++-------- 6 files changed, 42 insertions(+), 21 deletions(-) diff --git a/UPDATING.md b/UPDATING.md index 444a265f..827c087b 100644 --- a/UPDATING.md +++ b/UPDATING.md @@ -466,6 +466,29 @@ Example 4.0.0 code: pubsub_http_client = create_http_client(key='xxx', http_max_retry_count=2) ``` +### The `log_level` option is removed + +`log_level` was stored on the client options and never read, so it changed nothing. It is +gone, and passing it raises `TypeError`. The library logs through Python's standard +`logging` module, to the `ably.pubsub` logger and the loggers beneath it, so configure that +logger instead to set the level or to choose where records go: + +Example 3.x code: +```python +rest = AblyRest(key='xxx', log_level=logging.DEBUG) +``` + +Example 4.0.0 code: +```python +import logging + +logger = logging.getLogger('ably.pubsub') +logger.setLevel(logging.DEBUG) +logger.addHandler(logging.StreamHandler()) + +pubsub_http_client = create_http_client(key='xxx') +``` + ## Version 2.x to 3.0.0 The 3.0.0 version of ably-python introduces several breaking changes to improve the realtime experience and align the API with the Ably specification. These include: diff --git a/ably/pubsub/server/__init__.py b/ably/pubsub/server/__init__.py index 8d2da1f0..55a1cf9f 100644 --- a/ably/pubsub/server/__init__.py +++ b/ably/pubsub/server/__init__.py @@ -60,6 +60,11 @@ def create_http_client(**kwargs) -> PubSubHttpClient: of `key`, `token` or `token_details` to authenticate, or configure token auth through `auth_callback` or `auth_url`. + The client logs through the standard :mod:`logging` module, to the `ably.pubsub` logger + and the loggers beneath it, and adds no handler of its own beyond a `NullHandler`. + Configure that logger to choose the level and where records go, for example + ``logging.getLogger('ably.pubsub').setLevel(logging.DEBUG)``. + Parameters ---------- **kwargs: client options @@ -149,8 +154,6 @@ def create_http_client(**kwargs) -> PubSubHttpClient: add_request_ids: bool Add a `request_id` query parameter to every request, which makes a request traceable in Ably's logs. Defaults to False. - log_level: int - Verbosity of the library's own logging. Defaults to 0. **Deprecated** @@ -183,6 +186,7 @@ def create_realtime_client(**kwargs) -> PubSubRealtimeClient: Everything the HTTP client does, plus subscribing to channels and entering presence. Accepts every option :func:`create_http_client` does — see there for credentials, token authentication, connection and HTTP behaviour — and the following in addition. + It logs to the `ably.pubsub` logger, as :func:`create_http_client` describes. Parameters ---------- diff --git a/ably/pubsub/server/sync.py b/ably/pubsub/server/sync.py index 56357a6e..c5d8cbe1 100644 --- a/ably/pubsub/server/sync.py +++ b/ably/pubsub/server/sync.py @@ -44,6 +44,7 @@ def create_http_client(**kwargs) -> PubSubHttpClient: Accepts every option :func:`ably.pubsub.server.create_http_client` does — see there for credentials, token authentication, connection and HTTP behaviour — and behaves identically, without the event loop. The realtime-only options have no effect here. + It logs to the `ably.pubsub` logger, as that function describes. Returns ------- diff --git a/ably/pubsub/types/options.py b/ably/pubsub/types/options.py index ae448e49..e33bbb97 100644 --- a/ably/pubsub/types/options.py +++ b/ably/pubsub/types/options.py @@ -26,7 +26,7 @@ def decode(self, delta: bytes, base: bytes) -> bytes: class Options(AuthOptions): - def __init__(self, client_id=None, log_level=0, tls=True, rest_host=None, realtime_host=None, port=0, + def __init__(self, client_id=None, tls=True, rest_host=None, realtime_host=None, port=0, tls_port=0, use_binary_protocol=True, queue_messages=True, recover=False, endpoint=None, environment=None, http_open_timeout=None, http_request_timeout=None, realtime_request_timeout=None, http_max_retry_count=None, http_max_retry_duration=None, @@ -115,7 +115,6 @@ def __init__(self, client_id=None, log_level=0, tls=True, rest_host=None, realti endpoint = Defaults.endpoint self.__client_id = client_id - self.__log_level = log_level self.__tls = tls self.__port = port self.__tls_port = tls_port @@ -155,14 +154,6 @@ def client_id(self): def client_id(self, value): self.__client_id = value - @property - def log_level(self): - return self.__log_level - - @log_level.setter - def log_level(self, value): - self.__log_level = value - @property def tls(self): return self.__tls diff --git a/test/uts/deviations.md b/test/uts/deviations.md index f0412b45..a607c6d3 100644 --- a/test/uts/deviations.md +++ b/test/uts/deviations.md @@ -1044,7 +1044,7 @@ on the `recover` parameter the connection never sends. | RTS5, RTS5a, RTS5a1, RTS5a2, DO2a | Derived channels: `DeriveOptions` and `Channels.getDerived`. `grep -r derive ably/` is empty. Each test imports `DeriveOptions` inside its body so the module still loads | 5 | | RTB1, RTB1a, RTB1b | Retry backoff, jitter and `retryIn`. Retry timers schedule the flat configured timeout (`connectionmanager.py:753`, `channel.py:866-871`); `grep` for jitter/backoff/retry_in returns nothing, and neither `ConnectionStateChange` nor `ChannelStateChange` carries `retryIn` | 4 | | RTL25, RTL25a, RTL25b | `RealtimeChannel#whenState`. `Connection._when_state` exists (private, awaitable), so this is a gap on the channel rather than a house style; the tests are written against a `channel.when_state(state)` matching the shape the connection already has | 4 | -| RSC2, RSC3, RSC4, TO3b, TO3c, TO3c2 | `log_handler` as a client option, and any use of `log_level` — it is stored on `Options` and read by nothing | 4 | +| RSC2, RSC3, RSC4, TO3b, TO3c, TO3c2 | `log_level` and `log_handler` as client options. The library logs through the standard `logging` module under the `ably.pubsub` logger, whose level and handlers are the idiomatic way to do both; `log_level`, which was stored on `Options` and read by nothing, was removed in 4.0.0, so passing either raises `TypeError` | 4 | | RSP3a2, RSP3a3 | `clientId` and `connectionId` filters on `RestPresence#get`. `Presence.get` is `get(self, limit=None)` (`ably/types/presence.py:216`), while `Presence.history` does take its documented params. `presence.get(client_id=...)` raises `TypeError: get() got an unexpected keyword argument 'client_id'`. The absence also forces the RSP5 decoding adaptation below | 4 (5 cases) | | TP3a, TP3d, TP3g | Presence attributes defaulted from the encapsulating ProtocolMessage. There is no ProtocolMessage type; `ably/realtime/channel.py:751-761` passes the presence array through without context. Matters for synthesized-leave detection and `memberKey` | 3 | | TB4, RTL7h, RTP6e | `attachOnSubscribe`. `ChannelOptions.__init__` (`channeloptions.py:22-26`) takes only `cipher`, `params` and `modes`, and `subscribe()` on the channel, on presence and on annotations all end unconditionally with `await attach()`. This absence also forces the largest single adaptation in the suite, below | 3 | diff --git a/test/uts/rest/unit/logging_test.py b/test/uts/rest/unit/logging_test.py index 21984752..47b8d3c7 100644 --- a/test/uts/rest/unit/logging_test.py +++ b/test/uts/rest/unit/logging_test.py @@ -2,10 +2,11 @@ Spec points: RSC2, RSC2b, RSC3, RSC4, TO3b, TO3c, TO3c2 -ably-python has no log_handler client option, and logs through the standard library -`logging` module: a handler attached to the `ably` logger is the idiomatic rendering of -the specification's `logHandler`. The `log_level` option exists but is never applied to -any logger, so the tests that turn on verbosity or silence it depart from the spec. +ably-python has neither a log_level nor a log_handler client option, and logs through the +standard library `logging` module, under the `ably.pubsub` logger: setting that logger's +level, or attaching a handler to it, is the idiomatic rendering of the specification's +`logLevel` and `logHandler`. The tests that set either as a client option depart from the +spec. """ import logging @@ -79,8 +80,8 @@ async def test_rsc2_default_log_level_warn(): # UTS: rest/unit/TO3b/log-level-changeable-0 @deviation async def test_to3b_log_level_changeable(): - # DEVIATION: log_level is accepted and stored, but never applied to a logger, so raising - # it to verbose emits nothing. The SDK also has no "HTTP request" log event. + # DEVIATION: there is no log_level client option, so this raises TypeError on + # construction. The SDK also has no "HTTP request" log event. captured_logs = [] mock_http = MockHttpClient(on_connection_attempt=connect_successfully, on_request=time_responder) @@ -143,8 +144,9 @@ def handler(level, message, context): # UTS: rest/unit/RSC2b/log-level-none-suppresses-all-0 @deviation async def test_rsc2b_log_level_none_suppresses_all(): - # DEVIATION: log_level is never applied, so it silences nothing. The SDK's logger is - # turned up to debug first, so that suppression is observable at all. + # DEVIATION: there is no log_level client option, so this raises TypeError on + # construction. The SDK's logger is turned up to debug first, so that suppression + # would be observable at all. captured_logs = [] mock_http = MockHttpClient(on_connection_attempt=connect_successfully, on_request=time_responder) From 50f7c334f0d0806da87964abfd4953216ec96c9a Mon Sep 17 00:00:00 2001 From: owenpearson Date: Mon, 26 Oct 2026 18:33:38 +0000 Subject: [PATCH 47/71] pubsub: remove the deprecated and duplicate client options environment, rest_host and realtime_host were deprecated in favour of endpoint in 3.0.0, and TO3k1, TO3k2 and TO3k3 let a library with no backwards-compatibility requirement omit them. Options still accepted all three, mapped them onto endpoint with a warning, and carried the REC1b1 and REC1c1 conflict checks between them. The client options also took the token under two names, token (TO3j2), which the factories document, and auth_token, which the http client used internally and passed through from callers unchecked. And Options had a connection_state_ttl property and setter that copied Defaults.connection_state_ttl and that nothing read; the realtime client's docstring documented a connection_state_ttl keyword the constructor has never accepted. All of these are removed, along with the docstrings that mentioned them. endpoint is the one way to name a host, and the token option is token, read back as options.token. The UTS tests for REC1b1, REC1c, REC1d, REC2c5 and REC2c6 that exist only to exercise the removed host options are skipped as not applicable, and the four that used one only to name a host use endpoint instead; test/uts/deviations.md records both and updates its counts. This is a breaking change: passing environment, rest_host, realtime_host or auth_token now raises TypeError, and options.auth_token and options.connection_state_ttl are gone. Callers should pass endpoint for the first three and token for the fourth, as UPDATING.md shows. Co-Authored-By: Claude Opus 5.5 --- UPDATING.md | 37 ++++++++-- ably/pubsub/http/auth.py | 8 +- ably/pubsub/http/http.py | 9 +-- ably/pubsub/realtime/realtime.py | 10 --- ably/pubsub/server/__init__.py | 25 ++----- ably/pubsub/server/sync.py | 5 +- ably/pubsub/types/authoptions.py | 15 ++-- ably/pubsub/types/options.py | 55 +------------- test/ably/http/httpinit_test.py | 12 +-- test/unit/options_test.py | 74 ++----------------- test/unit/websockettransport_test.py | 2 +- test/uts/deviations.md | 36 ++++++--- .../unit/connection/fallback_hosts_test.py | 5 +- test/uts/rest/unit/fallback_test.py | 19 +++++ .../uts/rest/unit/types/options_types_test.py | 11 +-- .../rest/unit/types/paginated_result_test.py | 3 +- 16 files changed, 127 insertions(+), 199 deletions(-) diff --git a/UPDATING.md b/UPDATING.md index 827c087b..9b609914 100644 --- a/UPDATING.md +++ b/UPDATING.md @@ -196,11 +196,9 @@ async def auth_callback(token_params): The factories raised a bare `ValueError` in 3.x when given no means of authenticating, a `key` together with `key_name` or `key_secret`, a `token_details` that is not a `TokenDetails`, or `use_token_auth=False` without a key. In 4.0.0 each raises an -`AblyException` with code 40106 and status 401. Combining `endpoint` with `environment`, -`rest_host` or `realtime_host` still raises an `AblyException` with code 40106 and status -400. `push.admin.publish` raised `TypeError` for a `recipient` or `data` that is not a dict -and `ValueError` for an empty one, and now raises an `AblyException` with code 40000 and -status 400 for all four. +`AblyException` with code 40106 and status 401. `push.admin.publish` raised `TypeError` +for a `recipient` or `data` that is not a dict and `ValueError` for an empty one, and now +raises an `AblyException` with code 40000 and status 400 for all four. These exceptions are also a `ValueError`, so `except ValueError` still catches them, but `except TypeError` no longer catches the `push.admin.publish` ones: @@ -489,6 +487,35 @@ logger.addHandler(logging.StreamHandler()) pubsub_http_client = create_http_client(key='xxx') ``` +### Deprecated and duplicate options are removed + +These options were deprecated in 3.x, or duplicated another option, and are gone in 4.0.0. +Passing any of them raises `TypeError`: + +- `environment`: pass the same value as `endpoint`, so `environment='sandbox'` becomes + `endpoint='sandbox'`. +- `rest_host` and `realtime_host`: pass the host as `endpoint`, which one host now serves + for both REST and realtime. +- `auth_token`: pass the token as `token`, the name the factories document. The token is now + read back as `client.options.token` rather than `client.options.auth_token`. +- `connection_state_ttl`: this was never accepted as an option, and the + `client.options.connection_state_ttl` property, which nothing read, is removed. The value + the server sends is on `connection.connection_details.connection_state_ttl`. + +Example 3.x code: +```python +rest = AblyRest(key='xxx', environment='sandbox') +realtime = AblyRealtime(key='xxx', realtime_host='custom.ably.example.com') +rest = AblyRest(auth_token='xxx') +``` + +Example 4.0.0 code: +```python +pubsub_http_client = create_http_client(key='xxx', endpoint='sandbox') +pubsub_realtime_client = create_realtime_client(key='xxx', endpoint='custom.ably.example.com') +pubsub_http_client = create_http_client(token='xxx') +``` + ## Version 2.x to 3.0.0 The 3.0.0 version of ably-python introduces several breaking changes to improve the realtime experience and align the API with the Ably specification. These include: diff --git a/ably/pubsub/http/auth.py b/ably/pubsub/http/auth.py index 106e10c8..3f0399e7 100644 --- a/ably/pubsub/http/auth.py +++ b/ably/pubsub/http/auth.py @@ -55,7 +55,7 @@ def __init__(self, ably: DefaultPubSubHttpClient | DefaultPubSubRealtimeClient, # RSA4: token auth unless use_token_auth says otherwise, or nothing but a key is given if options.use_token_auth is None: must_use_token_auth = bool(options.auth_callback or options.auth_url - or options.auth_token or options.token_details) + or options.token or options.token_details) else: must_use_token_auth = options.use_token_auth must_not_use_token_auth = options.use_token_auth is False @@ -83,8 +83,8 @@ def __init__(self, ably: DefaultPubSubHttpClient | DefaultPubSubRealtimeClient, raise IncompatibleClientIdException( f"clientId '{options.client_id}' in the client options does not match the " f"token's clientId '{token_client_id}'", 400, 40102) - elif options.auth_token: - self.__token_details = TokenDetails(token=options.auth_token) + elif options.token: + self.__token_details = TokenDetails(token=options.token) else: self.__token_details = None @@ -94,7 +94,7 @@ def __init__(self, ably: DefaultPubSubHttpClient | DefaultPubSubRealtimeClient, log.debug("using token auth with auth_url") elif options.key_secret: log.debug("using token auth with client-side signing") - elif options.auth_token: + elif options.token: log.debug("using token auth with supplied token only") elif options.token_details: log.debug("using token auth with supplied token_details") diff --git a/ably/pubsub/http/http.py b/ably/pubsub/http/http.py index dd738c1a..a426a4e8 100644 --- a/ably/pubsub/http/http.py +++ b/ably/pubsub/http/http.py @@ -50,17 +50,10 @@ def __init__(self, key: Optional[str] = None, token: Optional[str] = None, - `client_id`: Undocumented - `endpoint`: Endpoint specifies either a routing policy name or fully qualified domain name to connect to Ably. - - `rest_host`: Deprecated: this property is deprecated and will - be removed in a future version. The host to connect to. - Defaults to rest.ably.io - - `environment`: Deprecated: this property is deprecated and - will be removed in a future version. The environment to use. - Defaults to 'production' - `port`: The port to connect to. Defaults to 80 - `tls_port`: The tls_port to connect to. Defaults to 443 - `tls`: Specifies whether the client should use TLS. Defaults to True - - `auth_token`: Undocumented - `auth_callback`: Undocumented - `auth_url`: Undocumented """ @@ -71,7 +64,7 @@ def __init__(self, key: Optional[str] = None, token: Optional[str] = None, "Provider either a key or key_name & key_secret", 401, 40106) # RSA4: a key given alongside a token is kept, to renew the token with if token is not None: - options = Options(key=key, auth_token=token, **kwargs) + options = Options(key=key, token=token, **kwargs) elif token_details is not None: if not isinstance(token_details, TokenDetails): raise InvalidArgumentException("token_details must be an instance of TokenDetails", 401, 40106) diff --git a/ably/pubsub/realtime/realtime.py b/ably/pubsub/realtime/realtime.py index 36b1118d..291c6331 100644 --- a/ably/pubsub/realtime/realtime.py +++ b/ably/pubsub/realtime/realtime.py @@ -55,13 +55,6 @@ def __init__(self, key: Optional[str] = None, loop: Optional[asyncio.AbstractEve **kwargs: client options endpoint: str Endpoint specifies either a routing policy name or fully qualified domain name to connect to Ably. - realtime_host: str - Deprecated: this property is deprecated and will be removed in a future version. - Enables a non-default Ably host to be specified for realtime connections. - For development environments only. The default value is realtime.ably.io. - environment: str - Deprecated: this property is deprecated and will be removed in a future version. - Enables a custom environment to be used with the Ably service. Defaults to `production` realtime_request_timeout: float Timeout (in milliseconds) for the wait of acknowledgement for operations performed via a realtime connection. Operations include establishing a connection with Ably, or sending a HEARTBEAT, @@ -77,9 +70,6 @@ def __init__(self, key: Optional[str] = None, loop: Optional[asyncio.AbstractEve An array of fallback hosts to be used in the case of an error necessitating the use of an alternative host. If you have been provided a set of custom fallback hosts by Ably, please specify them here. - connection_state_ttl: float - The duration that Ably will persist the connection state for when a Realtime client is abruptly - disconnected. suspended_retry_timeout: float When the connection enters the SUSPENDED state, after this delay, if the state is still SUSPENDED, the client library attempts to reconnect automatically. The default is 30 seconds. diff --git a/ably/pubsub/server/__init__.py b/ably/pubsub/server/__init__.py index 55a1cf9f..d672ed38 100644 --- a/ably/pubsub/server/__init__.py +++ b/ably/pubsub/server/__init__.py @@ -115,8 +115,7 @@ def create_http_client(**kwargs) -> PubSubHttpClient: cannot be changed once the client has authenticated. endpoint: str Either a routing policy name or a fully qualified domain name to connect to. - Defaults to `main`. Incompatible with `environment`, `rest_host` and - `realtime_host`. + Defaults to `main`. tls: bool Whether the client should use TLS. Defaults to True. port: int @@ -155,13 +154,6 @@ def create_http_client(**kwargs) -> PubSubHttpClient: Add a `request_id` query parameter to every request, which makes a request traceable in Ably's logs. Defaults to False. - **Deprecated** - - rest_host: str - Deprecated: this option will be removed in a future version. Use `endpoint`. - environment: str - Deprecated: this option will be removed in a future version. Use `endpoint`. - Returns ------- PubSubHttpClient @@ -170,9 +162,8 @@ def create_http_client(**kwargs) -> PubSubHttpClient: Raises ------ AblyException - With code 40106, if no means of authenticating is given, if `key` is combined - with `key_name` or `key_secret`, or if `endpoint` is combined with - `environment`, `rest_host` or `realtime_host`. It is also a ValueError. + With code 40106, if no means of authenticating is given, or if `key` is + combined with `key_name` or `key_secret`. It is also a ValueError. IncompatibleClientIdException If `client_id` conflicts with the client ID of `token_details`. """ @@ -239,11 +230,6 @@ def create_realtime_client(**kwargs) -> PubSubRealtimeClient: The decoder used to apply delta-encoded messages on channels configured with `params={"delta": "vcdiff"}`. Pass an :class:`AblyVCDiffDecoder`. - **Deprecated** - - realtime_host: str - Deprecated: this option will be removed in a future version. Use `endpoint`. - Returns ------- PubSubRealtimeClient @@ -252,9 +238,8 @@ def create_realtime_client(**kwargs) -> PubSubRealtimeClient: Raises ------ AblyException - With code 40106, if no means of authenticating is given, if `key` is combined - with `key_name` or `key_secret`, or if `endpoint` is combined with - `environment`, `rest_host` or `realtime_host`. It is also a ValueError. + With code 40106, if no means of authenticating is given, or if `key` is + combined with `key_name` or `key_secret`. It is also a ValueError. """ with factory_construction(): return _DefaultPubSubRealtimeClient(**kwargs) diff --git a/ably/pubsub/server/sync.py b/ably/pubsub/server/sync.py index c5d8cbe1..75039039 100644 --- a/ably/pubsub/server/sync.py +++ b/ably/pubsub/server/sync.py @@ -54,9 +54,8 @@ def create_http_client(**kwargs) -> PubSubHttpClient: Raises ------ AblyException - With code 40106, if no means of authenticating is given, if `key` is combined - with `key_name` or `key_secret`, or if `endpoint` is combined with - `environment`, `rest_host` or `realtime_host`. It is also a ValueError. + With code 40106, if no means of authenticating is given, or if `key` is + combined with `key_name` or `key_secret`. It is also a ValueError. """ with factory_construction(): return _DefaultPubSubHttpClientSync(**kwargs) diff --git a/ably/pubsub/types/authoptions.py b/ably/pubsub/types/authoptions.py index 18f0fb30..36315c54 100644 --- a/ably/pubsub/types/authoptions.py +++ b/ably/pubsub/types/authoptions.py @@ -3,7 +3,7 @@ class AuthOptions: def __init__(self, auth_callback=None, auth_url=None, auth_method='GET', - auth_token=None, auth_headers=None, auth_params=None, + token=None, auth_headers=None, auth_params=None, key_name=None, key_secret=None, key=None, query_time=False, token_details=None, use_token_auth=None, default_token_params=None): @@ -18,7 +18,7 @@ def __init__(self, auth_callback=None, auth_url=None, auth_method='GET', self.auth_options['key_secret'] = key_secret self.set_key(key) - self.__auth_token = auth_token + self.__token = token self.__token_details = token_details self.__use_token_auth = use_token_auth default_token_params = default_token_params or {} @@ -97,13 +97,14 @@ def key_secret(self): def key_secret(self, value): self.auth_options['key_secret'] = value + # TO3j2 @property - def auth_token(self): - return self.__auth_token + def token(self): + return self.__token - @auth_token.setter - def auth_token(self, value): - self.__auth_token = value + @token.setter + def token(self, value): + self.__token = value @property def auth_headers(self): diff --git a/ably/pubsub/types/options.py b/ably/pubsub/types/options.py index e33bbb97..84551f46 100644 --- a/ably/pubsub/types/options.py +++ b/ably/pubsub/types/options.py @@ -4,7 +4,6 @@ from ably.pubsub.transport.defaults import Defaults from ably.pubsub.types.authoptions import AuthOptions -from ably.pubsub.util.exceptions import InvalidArgumentException log = logging.getLogger(__name__) @@ -26,9 +25,9 @@ def decode(self, delta: bytes, base: bytes) -> bytes: class Options(AuthOptions): - def __init__(self, client_id=None, tls=True, rest_host=None, realtime_host=None, port=0, + def __init__(self, client_id=None, tls=True, port=0, tls_port=0, use_binary_protocol=True, queue_messages=True, recover=False, endpoint=None, - environment=None, http_open_timeout=None, http_request_timeout=None, + http_open_timeout=None, http_request_timeout=None, realtime_request_timeout=None, http_max_retry_count=None, http_max_retry_duration=None, fallback_hosts=None, fallback_retry_timeout=None, disconnected_retry_timeout=None, idempotent_rest_publishing=None, loop=None, auto_connect=True, @@ -39,15 +38,6 @@ def __init__(self, client_id=None, tls=True, rest_host=None, realtime_host=None, super().__init__(**kwargs) - # REC1b1: endpoint is incompatible with deprecated options - if endpoint is not None: - if environment is not None or rest_host is not None or realtime_host is not None: - raise InvalidArgumentException( - message='endpoint is incompatible with any of environment, rest_host or realtime_host', - status_code=400, - code=40106, - ) - # TODO check these defaults if fallback_retry_timeout is None: fallback_retry_timeout = Defaults.fallback_retry_timeout @@ -74,43 +64,15 @@ def __init__(self, client_id=None, tls=True, rest_host=None, realtime_host=None, if connectivity_check_url is None: connectivity_check_url = Defaults.connectivity_check_url - connection_state_ttl = Defaults.connection_state_ttl - if suspended_retry_timeout is None: suspended_retry_timeout = Defaults.suspended_retry_timeout - if environment is not None and rest_host is not None: - raise InvalidArgumentException( - message='specify rest_host or environment, not both', - status_code=400, - code=40106, - ) - - if environment is not None and realtime_host is not None: - raise InvalidArgumentException( - message='specify realtime_host or environment, not both', - status_code=400, - code=40106, - ) - if idempotent_rest_publishing is None: from ably.pubsub.version import api_version idempotent_rest_publishing = api_version >= '1.2' - if environment is not None and endpoint is None: - log.warning("environment client option is deprecated, please use endpoint instead") - endpoint = environment - - # REC1d: restHost or realtimeHost option - # REC1d1: restHost takes precedence over realtimeHost - if rest_host is not None and endpoint is None: - log.warning("rest_host client option is deprecated, please use endpoint instead") - endpoint = rest_host - elif realtime_host is not None and endpoint is None: - # REC1d2: realtimeHost if restHost not specified - log.warning("realtime_host client option is deprecated, please use endpoint instead") - endpoint = realtime_host - + # REC1a: the default routing policy. ably-python omits the deprecated environment, + # rest_host and realtime_host options (TO3k1-3), so REC1c and REC1d do not apply if endpoint is None: endpoint = Defaults.endpoint @@ -137,7 +99,6 @@ def __init__(self, client_id=None, tls=True, rest_host=None, realtime_host=None, self.__idempotent_rest_publishing = idempotent_rest_publishing self.__loop = loop self.__auto_connect = auto_connect - self.__connection_state_ttl = connection_state_ttl self.__suspended_retry_timeout = suspended_retry_timeout self.__connectivity_check_url = connectivity_check_url self.__add_request_ids = add_request_ids @@ -276,14 +237,6 @@ def loop(self): def auto_connect(self): return self.__auto_connect - @property - def connection_state_ttl(self): - return self.__connection_state_ttl - - @connection_state_ttl.setter - def connection_state_ttl(self, value): - self.__connection_state_ttl = value - @property def suspended_retry_timeout(self): return self.__suspended_retry_timeout diff --git a/test/ably/http/httpinit_test.py b/test/ably/http/httpinit_test.py index b8202a3b..702cafac 100644 --- a/test/ably/http/httpinit_test.py +++ b/test/ably/http/httpinit_test.py @@ -28,7 +28,7 @@ def per_protocol_setup(self, use_binary_protocol): @dont_vary_protocol def test_with_token(self): ably = create_http_client(token="foo") - assert ably.options.auth_token == "foo", "Token not set at options" + assert ably.options.token == "foo", "Token not set at options" @dont_vary_protocol def test_with_token_details(self): @@ -68,7 +68,7 @@ def test_with_options_auth_url(self): # RSC11 @dont_vary_protocol - def test_rest_host_and_environment(self): + def test_endpoint(self): # endpoint host ably = create_http_client(token='foo', endpoint="some.other.host") assert "some.other.host" == ably.options.get_host(), "Unexpected host mismatch" @@ -83,10 +83,10 @@ def test_rest_host_and_environment(self): host = ably.options.get_host() assert "sandbox.realtime.ably-nonprod.net" == host, f"Unexpected host mismatch {host}" - # both, as per #TO3k2 - with pytest.raises(AblyException): - ably = create_http_client(token='foo', rest_host="some.other.host", - endpoint="some.other.environment") + # The deprecated host options are omitted (TO3k1-3) + for option in ('environment', 'rest_host', 'realtime_host'): + with pytest.raises(TypeError): + create_http_client(token='foo', **{option: 'some.other.host'}) # RSC15 @dont_vary_protocol diff --git a/test/unit/options_test.py b/test/unit/options_test.py index b58e4ce1..fb68159f 100644 --- a/test/unit/options_test.py +++ b/test/unit/options_test.py @@ -5,22 +5,13 @@ from ably.pubsub.util.exceptions import AblyException -# REC1b1: endpoint is incompatible with deprecated options -def test_options_should_fail_early_with_incompatible_client_options(): - # REC1b1: endpoint with environment - with pytest.raises(AblyException) as exinfo: - Options(endpoint="foo", environment="foo") - assert exinfo.value.code == 40106 - - # REC1b1: endpoint with rest_host - with pytest.raises(AblyException) as exinfo: - Options(endpoint="foo", rest_host="foo") - assert exinfo.value.code == 40106 - - # REC1b1: endpoint with realtime_host - with pytest.raises(AblyException) as exinfo: - Options(endpoint="foo", realtime_host="foo") - assert exinfo.value.code == 40106 +# TO3k1-3: the deprecated environment, rest_host and realtime_host options are omitted, +# and with them the duplicate auth_token and the unused connection_state_ttl +@pytest.mark.parametrize('option', ['environment', 'rest_host', 'realtime_host', 'auth_token', + 'connection_state_ttl']) +def test_options_should_reject_removed_options(option): + with pytest.raises(TypeError): + Options(**{option: 'foo'}) # REC1a @@ -71,56 +62,6 @@ def test_options_should_return_localhost(): assert not opts.get_fallback_hosts() -# REC1c1: environment with rest_host or realtime_host is invalid -def test_options_should_fail_with_environment_and_rest_or_realtime_host(): - # REC1c1: environment with rest_host - with pytest.raises(AblyException) as exinfo: - Options(environment="foo", rest_host="bar") - assert exinfo.value.code == 40106 - - # REC1c1: environment with realtime_host - with pytest.raises(AblyException) as exinfo: - Options(environment="foo", realtime_host="bar") - assert exinfo.value.code == 40106 - - -# REC1c2: environment defines production routing policy ID -def test_options_with_environment_should_return_routing_policy_hostnames(): - opts = Options(environment="foo") - # REC1c2: primary domain is [id].realtime.ably.net - assert opts.get_host() == "foo.realtime.ably.net" - # REC2c5: fallback domains for production routing policy ID via environment - assert "foo.a.fallback.ably-realtime.com" in opts.get_fallback_hosts() - assert "foo.e.fallback.ably-realtime.com" in opts.get_fallback_hosts() - - -# REC1d1: rest_host takes precedence for primary domain -def test_options_with_rest_host_should_return_rest_host(): - opts = Options(rest_host="custom.example.com") - # REC1d1: primary domain is the value of the restHost option - assert opts.get_host() == "custom.example.com" - # REC2c6: fallback domains for restHost is empty - assert not opts.get_fallback_hosts() - - -# REC1d2: realtime_host if rest_host not specified -def test_options_with_realtime_host_should_return_realtime_host(): - opts = Options(realtime_host="custom.example.com") - # REC1d2: primary domain is the value of the realtimeHost option - assert opts.get_host() == "custom.example.com" - # REC2c6: fallback domains for realtimeHost is empty - assert not opts.get_fallback_hosts() - - -# REC1d1: rest_host takes precedence over realtime_host -def test_options_with_rest_host_takes_precedence_over_realtime_host(): - opts = Options(rest_host="rest.example.com", realtime_host="realtime.example.com") - # REC1d1: restHost takes precedence - assert opts.get_host() == "rest.example.com" - # REC2c6: fallback domains is empty - assert not opts.get_fallback_hosts() - - # REC2a2: fallback_hosts value is used when specified def test_options_with_fallback_hosts_should_use_specified_hosts(): custom_fallbacks = ["fallback1.example.com", "fallback2.example.com"] @@ -206,7 +147,6 @@ def test_options_localhost_no_fallback_hosts(): ({'key': 'appId.keyId:keySecret', 'key_name': 'appId.keyId'}, 401), ({'token_details': 'not-token-details'}, 401), ({'token': 'a-token', 'use_token_auth': False}, 401), - ({'key': 'appId.keyId:keySecret', 'endpoint': 'foo', 'environment': 'foo'}, 400), ]) def test_invalid_client_arguments_raise_40106(kwargs, status_code): with pytest.raises(AblyException) as exinfo: diff --git a/test/unit/websockettransport_test.py b/test/unit/websockettransport_test.py index 3cf1fffb..1c0e3015 100644 --- a/test/unit/websockettransport_test.py +++ b/test/unit/websockettransport_test.py @@ -6,7 +6,7 @@ def _connect_url(host='example.com', **option_kwargs): connection_manager = MagicMock() - connection_manager.options = Options(auth_token='foo', **option_kwargs) + connection_manager.options = Options(token='foo', **option_kwargs) transport = WebSocketTransport(connection_manager, host, {'format': 'json'}) with patch.object(transport, 'ws_connect', MagicMock()) as mock_ws_connect: with patch('ably.pubsub.transport.websockettransport.asyncio.create_task') as mock_create_task: diff --git a/test/uts/deviations.md b/test/uts/deviations.md index a607c6d3..22eeebc1 100644 --- a/test/uts/deviations.md +++ b/test/uts/deviations.md @@ -33,9 +33,9 @@ parametrized over a table of fixtures the specification gives inline; and five `objects/unit` tests are parametrized, three over a table and two over the scenarios the specification loops through. That turns 1481 derived tests into 1608 pytest cases. -Of **1472 Test IDs, derived as 1481 tests and run as 1608 pytest cases**: 1304 Test IDs -(1313 tests, 1438 cases) pass, 153 (153 tests, 155 cases) are gated behind -`RUN_DEVIATIONS`, and 15 (15 tests, 15 cases) cannot be run at all. The three groups are +Of **1472 Test IDs, derived as 1481 tests and run as 1608 pytest cases**: 1292 Test IDs +(1301 tests, 1426 cases) pass, 153 (153 tests, 155 cases) are gated behind +`RUN_DEVIATIONS`, and 27 (27 tests, 27 cases) cannot be run at all. The three groups are disjoint: two Test IDs have a gated part and a passing part — `TI1`, as two functions, and `TO/endpoint-affects-host`, as one parametrized test — and are counted with the gated, while their passing parts count among the passing tests and cases. Every gated test has @@ -2003,8 +2003,9 @@ through their setups. `ConnectionDetails.from_dict` parses `connectionStateTtl` (`types/connectiondetails.py:19`) and nothing ever reads it. `ConnectionManager.start_suspend_timer` (`connectionmanager.py:745`) uses `Defaults.connection_state_ttl` — 120000 — directly, and -no client option overrides it (`types/options.py:64` accepts the keyword and then discards -it). A server that shortens or lengthens the TTL is ignored. +no client option overrides it (TO3 defines none, and the `Options.connection_state_ttl` +property that copied the default and was read by nothing was removed in 4.0.0). A server +that shortens or lengthens the TTL is ignored. **Tests affected:** every test that has to reach a SUSPENDED connection — nine across the connection, channel and presence suites — sends the specification's `connectionStateTtl` and @@ -2207,7 +2208,6 @@ pseudocode is not mistaken for non-compliance. | RSL2, RSL11b, RSL15b | `:` is `%3A` | Left literal, from `safe=':'`. RFC 3986 allows `:` in a path segment and Ably uses it for namespaces, so the server receives the same value | Intentional | | RTC12 / RSC1, RSC1a, RSC1c | A string constructor argument is an API key when it contains `:` and a token when it does not | `AblyRest.__init__` treats its first positional argument as a key unconditionally and hands it to `AuthOptions.set_key`, which requires exactly two colon-separated parts, so a token string raises 40101/401 "key of not len 2 parameters". A token is supplied through the separate `token` or `token_details` arguments. The empty-string case is compliant | Intentional / SDK-wide: the constructor takes credentials as distinct named arguments and has no string-sniffing path to restore | | RSC18 | The constructor rejects basic auth over HTTP | Construction succeeds; 40103 is raised from `make_request` when a request needing Basic Auth is attempted, and no request goes out. RSA1/RSC18 say only "any attempt to use" | Compliant; the UTS is stricter than its source | -| REC1b1, REC1c1 | Code 40000, or a message containing "invalid" or "conflict" | 400/40106 with a specific message. The features spec mandates no code | Cosmetic | | RSAN1a3 | Code 40003 for a missing `Annotation.type` | 400/40000 | Cosmetic; worth aligning cross-SDK | | RSH1a | Empty `recipient` or `data` rejected with code 40000, the error reaching the caller from the server | `PushAdmin.publish` validates its arguments itself and raises before touching the HTTP layer: an empty or non-dict `recipient` or `data` raises an `AblyException` with 400/40000, the code the server gives, which is also a `ValueError`. So no request is made and there is no server error; the integration tier's `test_rsh1a_push_publish_invalid_recipient` asserts the local error's code | Compliant: RSH1a asks for empty values to be "immediately rejected". The specification's test would need a recipient the SDK will send and the server will reject, an unknown `transportType` say, to exercise the server-side path it describes | | HP6 | `errorCode` is a number | The raw header string, `'40101'` | Open bug, trivial | @@ -2273,7 +2273,7 @@ largest, the typed views, has its own section at the end of this file. ## Mock Infrastructure Limitations Tests that cannot be implemented as written, kept as skipped stubs carrying their Test -IDs so the specification's coverage is still accounted for. Fifteen in total, none of +IDs so the specification's coverage is still accounted for. Twenty-seven in total, none of them from `uts/objects`. Two of the entries are caused by the SDK rather than by the mock, but they land here because the effect is the same: no test can observe the behaviour. @@ -2351,6 +2351,24 @@ Optional per TO3k7, and `REC1b1` and `REC2a1` scope their checks to libraries th support it, so these are skipped as not applicable rather than recorded as deviations. +### `environment`, `restHost` and `realtimeHost` are omitted — 12 tests + +**Spec points:** REC1b1 (`endpoint-conflicts-environment-0`, `-resthost-1`, +`-realtimehost-2`), REC1c1 (two), REC1c2, REC1d, REC1d1, REC1d2, REC2c5, and REC2c6 (two), +all in `fallback_test.py`. + +TO3k1, TO3k2 and TO3k3 mark the three options optional and deprecated, and say a library +with no backwards-compatibility requirement may omit them. ably-python 4.0.0 is a breaking +release and omits them, so `endpoint` is the only way to name a host, passing any of the +three raises `TypeError`, and the REC1 and REC2 clauses that only they select do not apply. +The tests are skipped as not applicable, as the `fallbackHostsUseDefault` ones are. + +Four tests elsewhere use one of the three only to name a host, and pass with `endpoint` in +its place, which REC1b2 resolves to the same host: `TG/pagination-relative-urls-5`, +`TO3/client-options-custom-hosts-1` and `RTN17g/empty-fallback-set-error-0` each carry a +NOTE saying so, and `TO/conflicting-options-validation-1` asserts the `TypeError` for its +restHost-and-endpoint case, there being no conflict left to construct. + ## Investigated and not defects Claims raised during derivation, investigated, and found not to be SDK faults. They are @@ -3517,8 +3535,8 @@ The header states how many derived tests there are, how many pass, how many are gated and how many cannot run. Those numbers are the check that the file is still true: in pytest cases, the gated count must equal the number of failures under `RUN_DEVIATIONS=1`, and gated plus unrunnable must equal the number of skips without -it. As of this writing that is 155 failures, 1586 passes and 15 skips with the variable -set, and 170 skips and 1586 passes without it, the 1586 being 1438 derived cases and 148 +it. As of this writing that is 155 failures, 1574 passes and 27 skips with the variable +set, and 182 skips and 1574 passes without it, the 1574 being 1426 derived cases and 148 harness ones — 130 under `helpers/` and 18 under `objects/helpers/`. Those figures need the `submodules` checkout CI makes: without it, three `rest/unit/encoding` tests that read the ably-common fixtures skip as well. The tiers are told apart by the junit report's diff --git a/test/uts/realtime/unit/connection/fallback_hosts_test.py b/test/uts/realtime/unit/connection/fallback_hosts_test.py index 69294ba8..b24dfb68 100644 --- a/test/uts/realtime/unit/connection/fallback_hosts_test.py +++ b/test/uts/realtime/unit/connection/fallback_hosts_test.py @@ -208,9 +208,10 @@ def on_connection_attempt(conn): conn.respond_with_refused() mock_ws = MockWebSocket(on_connection_attempt=on_connection_attempt) - # REC2c2: a custom domain has no fallback set of its own + # REC2c2: a custom domain has no fallback set of its own. The spec sets realtimeHost, + # which ably-python omits (TO3k3); endpoint names the same host client = fallback_client( - mock_ws, realtime_host='custom.example.com', realtime_request_timeout=300, + mock_ws, endpoint='custom.example.com', realtime_request_timeout=300, disconnected_retry_timeout=60000) client.connect() diff --git a/test/uts/rest/unit/fallback_test.py b/test/uts/rest/unit/fallback_test.py index d5f77a38..477d79f0 100644 --- a/test/uts/rest/unit/fallback_test.py +++ b/test/uts/rest/unit/fallback_test.py @@ -43,6 +43,13 @@ def connect_successfully(conn): conn.respond_with_success() +# TO3k1, TO3k2 and TO3k3 let a library with no backwards-compatibility requirement omit +# the deprecated environment, restHost and realtimeHost options, and ably-python 4 does. +OMITTED_HOST_OPTIONS_SKIP = ( + 'environment, restHost and realtimeHost are optional and deprecated (features.md TO3k1-3), ' + 'and ably-python omits them, so the REC1 and REC2 behaviour they select does not apply.') + + def new_mock(): return MockHttpClient(on_connection_attempt=connect_successfully) @@ -529,6 +536,7 @@ async def test_rec1b4_production_routing_policy(): # UTS: rest/unit/REC1b1/endpoint-conflicts-environment-0 +@pytest.mark.skip(reason=OMITTED_HOST_OPTIONS_SKIP) async def test_rec1b1_endpoint_conflicts_environment(): with pytest.raises(AblyException) as excinfo: rest_client(new_mock(), endpoint='test', environment='production') @@ -541,6 +549,7 @@ async def test_rec1b1_endpoint_conflicts_environment(): # UTS: rest/unit/REC1b1/endpoint-conflicts-resthost-1 +@pytest.mark.skip(reason=OMITTED_HOST_OPTIONS_SKIP) async def test_rec1b1_endpoint_conflicts_resthost(): with pytest.raises(AblyException) as excinfo: rest_client(new_mock(), endpoint='test', rest_host='custom.host.com') @@ -552,6 +561,7 @@ async def test_rec1b1_endpoint_conflicts_resthost(): # UTS: rest/unit/REC1b1/endpoint-conflicts-realtimehost-2 +@pytest.mark.skip(reason=OMITTED_HOST_OPTIONS_SKIP) async def test_rec1b1_endpoint_conflicts_realtimehost(): with pytest.raises(AblyException) as excinfo: rest_client(new_mock(), endpoint='test', realtime_host='custom.realtime.com') @@ -572,6 +582,7 @@ async def test_rec1b1_endpoint_conflicts_fallback_default(): # UTS: rest/unit/REC1c2/environment-sets-primary-domain-0 +@pytest.mark.skip(reason=OMITTED_HOST_OPTIONS_SKIP) async def test_rec1c2_environment_sets_primary_domain(): mock_http = new_mock() mock_http.queue_response(200, TIME_BODY) @@ -584,6 +595,7 @@ async def test_rec1c2_environment_sets_primary_domain(): # UTS: rest/unit/REC1c1/environment-conflicts-resthost-0 +@pytest.mark.skip(reason=OMITTED_HOST_OPTIONS_SKIP) async def test_rec1c1_environment_conflicts_resthost(): with pytest.raises(AblyException) as excinfo: rest_client(new_mock(), environment='sandbox', rest_host='custom.host.com') @@ -595,6 +607,7 @@ async def test_rec1c1_environment_conflicts_resthost(): # UTS: rest/unit/REC1c1/environment-conflicts-realtimehost-1 +@pytest.mark.skip(reason=OMITTED_HOST_OPTIONS_SKIP) async def test_rec1c1_environment_conflicts_realtimehost(): with pytest.raises(AblyException) as excinfo: rest_client(new_mock(), environment='sandbox', realtime_host='custom.realtime.com') @@ -606,6 +619,7 @@ async def test_rec1c1_environment_conflicts_realtimehost(): # UTS: rest/unit/REC1d1/resthost-sets-primary-domain-0 +@pytest.mark.skip(reason=OMITTED_HOST_OPTIONS_SKIP) async def test_rec1d1_resthost_sets_primary_domain(): mock_http = new_mock() mock_http.queue_response(200, TIME_BODY) @@ -618,6 +632,7 @@ async def test_rec1d1_resthost_sets_primary_domain(): # UTS: rest/unit/REC1d2/realtimehost-sets-primary-domain-0 +@pytest.mark.skip(reason=OMITTED_HOST_OPTIONS_SKIP) async def test_rec1d2_realtimehost_sets_primary_domain(): mock_http = new_mock() mock_http.queue_response(200, TIME_BODY) @@ -630,6 +645,7 @@ async def test_rec1d2_realtimehost_sets_primary_domain(): # UTS: rest/unit/REC1d/resthost-precedence-over-realtimehost-0 +@pytest.mark.skip(reason=OMITTED_HOST_OPTIONS_SKIP) async def test_rec1d_resthost_precedence_over_realtimehost(): mock_http = new_mock() mock_http.queue_response(200, TIME_BODY) @@ -750,6 +766,7 @@ async def test_rec2c4_production_endpoint_fallback_domains(): # UTS: rest/unit/REC2c5/production-environment-fallback-domains-0 +@pytest.mark.skip(reason=OMITTED_HOST_OPTIONS_SKIP) async def test_rec2c5_production_environment_fallback_domains(): mock_http = new_mock() mock_http.queue_response(500, error_body(500)) @@ -765,6 +782,7 @@ async def test_rec2c5_production_environment_fallback_domains(): # UTS: rest/unit/REC2c6/custom-resthost-no-fallbacks-0 +@pytest.mark.skip(reason=OMITTED_HOST_OPTIONS_SKIP) async def test_rec2c6_custom_resthost_no_fallbacks(): mock_http = new_mock() mock_http.queue_response(500, error_body(500)) @@ -780,6 +798,7 @@ async def test_rec2c6_custom_resthost_no_fallbacks(): # UTS: rest/unit/REC2c6/custom-realtimehost-no-fallbacks-1 +@pytest.mark.skip(reason=OMITTED_HOST_OPTIONS_SKIP) async def test_rec2c6_custom_realtimehost_no_fallbacks(): mock_http = new_mock() mock_http.queue_response(500, error_body(500)) diff --git a/test/uts/rest/unit/types/options_types_test.py b/test/uts/rest/unit/types/options_types_test.py index 471b940a..a0a46c33 100644 --- a/test/uts/rest/unit/types/options_types_test.py +++ b/test/uts/rest/unit/types/options_types_test.py @@ -9,7 +9,6 @@ from ably.pubsub.types.authoptions import AuthOptions from ably.pubsub.types.options import Options from ably.pubsub.types.tokendetails import TokenDetails -from ably.pubsub.util.exceptions import AblyException from test.uts.helpers.client import rest_client from test.uts.helpers.deviations import deviation from test.uts.helpers.mock_http import MockHttpClient @@ -86,11 +85,12 @@ async def test_to3_client_options_attributes(): def test_to3_client_options_custom_hosts(): options = Options( key='appId.keyId:keySecret', - rest_host='custom.ably.example.com', + # NOTE: the spec sets restHost, which ably-python omits (TO3k2); endpoint names the + # same host + endpoint='custom.ably.example.com', fallback_hosts=['fallback1.example.com', 'fallback2.example.com'], ) - # rest_host is deprecated in favour of endpoint (REC1d), which is where it is held assert options.endpoint == 'custom.ably.example.com' # The fallbacks are kept as given, in the random order RSC15a tries them in assert sorted(options.fallback_hosts) == ['fallback1.example.com', 'fallback2.example.com'] @@ -184,9 +184,10 @@ async def auth_callback(params): assert client.auth.auth_options.auth_callback is auth_callback # Case 2: restHost + endpoint conflict - with pytest.raises(AblyException) as excinfo: + # NOTE: ably-python omits restHost (TO3k2), so there is no conflict to construct; the + # option itself is rejected + with pytest.raises(TypeError): Options(key='appId.keyId:keySecret', rest_host='custom.host.com', endpoint='test') - assert 'rest_host' in excinfo.value.message or 'endpoint' in excinfo.value.message # Case 3: no auth options # The missing-credentials check is a constructor argument check, so it raises ValueError diff --git a/test/uts/rest/unit/types/paginated_result_test.py b/test/uts/rest/unit/types/paginated_result_test.py index 22f0f1bb..15661d3a 100644 --- a/test/uts/rest/unit/types/paginated_result_test.py +++ b/test/uts/rest/unit/types/paginated_result_test.py @@ -265,7 +265,8 @@ async def test_tg_pagination_relative_urls(): ([{'id': 'item2'}], {}), ]), ) - client = rest_client(mock_http, rest_host='rest.ably.io') + # NOTE: the spec sets restHost, which ably-python omits (TO3k2); endpoint names the same host + client = rest_client(mock_http, endpoint='rest.ably.io') channel = client.channels.get('test') page1 = await channel.history() From 1a97b10d29fb12fb49cc48c2121d9b153bc1a0b2 Mon Sep 17 00:00:00 2001 From: owenpearson Date: Mon, 26 Oct 2026 18:37:59 +0000 Subject: [PATCH 48/71] pubsub: check publishes against a configurable max_message_size TO3l8 makes maxMessageSize a client option, default 65536, and RSL1i has a publish whose messages exceed it between them rejected with 40009 before it is sent. ably-python had no such option, and REST publishes were never checked. The realtime publish read the limit with getattr(options, 'max_message_size', 65536), which always fell back to the default, and measured the whole encoded message, msgpack or JSON with ASCII escaping, rather than the TM6 size, so it counted the id, the encoding and every other field the specification leaves out. max_message_size is now a client option, defaulting to Defaults.max_message_size. message_size measures a message per TM6: the lengths of its name, clientId and string data, the bytes of binary or encrypted data, and the length of the JSON of object data and extras. The REST and realtime publishes both check their messages with it, after encryption and before encoding. CD2c, the connectionDetails override for realtime, is not implemented, since ably-python does not parse that field. The RSL1i UTS test is un-gated. Its at-limit case is five bytes over once TM6 counts the name, so the derived test corrects the fixture, which test/uts/deviations.md records as a spec error along with the updated counts. This is a breaking change: a REST publish over 64 KiB now raises an AblyException with code 40009 instead of being sent, and some realtime publishes that were refused are now sent. Callers whose account allows larger messages can raise max_message_size. Co-Authored-By: Claude Opus 5.5 --- UPDATING.md | 18 +++++++++ ably/pubsub/http/channel.py | 4 ++ ably/pubsub/realtime/channel.py | 7 ++-- ably/pubsub/server/__init__.py | 4 ++ ably/pubsub/transport/defaults.py | 2 + ably/pubsub/types/message.py | 36 ++++++++++++++++++ ably/pubsub/types/options.py | 11 +++++- ably/pubsub/util/helper.py | 29 -------------- test/unit/message_test.py | 38 +++++++++++++++++++ test/uts/deviations.md | 26 +++++++------ test/uts/rest/unit/channel/publish_test.py | 8 ++-- .../uts/rest/unit/types/options_types_test.py | 6 +-- 12 files changed, 136 insertions(+), 53 deletions(-) diff --git a/UPDATING.md b/UPDATING.md index 9b609914..801db6dc 100644 --- a/UPDATING.md +++ b/UPDATING.md @@ -516,6 +516,24 @@ pubsub_realtime_client = create_realtime_client(key='xxx', endpoint='custom.ably pubsub_http_client = create_http_client(token='xxx') ``` +### REST publishes are checked against `max_message_size` + +`max_message_size` is now a client option, defaulting to 65536 bytes (64 KiB). A publish +whose messages exceed it between them raises an `AblyException` with code 40009 and is not +sent. In 3.x the realtime client checked a fixed 64 KiB limit and REST publishes were not +checked at all, so an oversized REST publish now fails before it reaches Ably rather than +being rejected by the server. + +Both clients now measure a message the way the Ably specification does: the length of its +name, client ID and string data, the byte length of binary data, and the length of the JSON +of object data and of extras. 3.x measured the realtime publish as the whole encoded +message, so some realtime publishes that 3.x refused are now sent. Raise the limit only if +your Ably account allows larger messages: + +```python +pubsub_http_client = create_http_client(key='xxx', max_message_size=131072) +``` + ## Version 2.x to 3.0.0 The 3.0.0 version of ably-python introduces several breaking changes to improve the realtime experience and align the API with the Ably specification. These include: diff --git a/ably/pubsub/http/channel.py b/ably/pubsub/http/channel.py index e2bd13a0..134de023 100644 --- a/ably/pubsub/http/channel.py +++ b/ably/pubsub/http/channel.py @@ -16,6 +16,7 @@ assign_idempotent_ids, make_message_response_handler, make_single_message_response_handler, + validate_message_size, ) from ably.pubsub.types.operations import MessageOperation, PublishResult, UpdateDeleteResult from ably.pubsub.types.presence import Presence @@ -76,6 +77,9 @@ def __publish_request_body(self, messages): request_body_list.append(m) + # RSL1i + validate_message_size(request_body_list, self.ably.options.max_message_size) + request_body = [ message.as_dict(binary=self.ably.options.use_binary_protocol) for message in request_body_list] diff --git a/ably/pubsub/realtime/channel.py b/ably/pubsub/realtime/channel.py index 7da4ed1f..c4421fbe 100644 --- a/ably/pubsub/realtime/channel.py +++ b/ably/pubsub/realtime/channel.py @@ -17,14 +17,14 @@ from ably.pubsub.types.channeloptions import ChannelOptions from ably.pubsub.types.channelstate import ChannelState, ChannelStateChange from ably.pubsub.types.flags import Flag, has_flag -from ably.pubsub.types.message import Message, MessageAction, MessageVersion +from ably.pubsub.types.message import Message, MessageAction, MessageVersion, validate_message_size from ably.pubsub.types.mixins import DecodingContext from ably.pubsub.types.operations import MessageOperation, PublishResult, UpdateDeleteResult from ably.pubsub.types.presence import PresenceMessage from ably.pubsub.util.clock import select_clock from ably.pubsub.util.eventemitter import EventEmitter from ably.pubsub.util.exceptions import AblyException, IncompatibleClientIdException -from ably.pubsub.util.helper import Timer, is_callable_or_coroutine, validate_message_size +from ably.pubsub.util.helper import Timer, is_callable_or_coroutine if TYPE_CHECKING: from ably.pubsub.realtime.realtime import DefaultPubSubRealtimeClient @@ -464,8 +464,7 @@ async def publish(self, *args, **kwargs) -> PublishResult: encoded_messages.append(msg_dict) # RSL1i: Check message size limit - max_message_size = getattr(self.ably.options, 'max_message_size', 65536) # 64KB default - validate_message_size(encoded_messages, self.ably.options.use_binary_protocol, max_message_size) + validate_message_size(messages, self.ably.options.max_message_size) # RTL6c: Check connection and channel state self._throw_if_unpublishable_state() diff --git a/ably/pubsub/server/__init__.py b/ably/pubsub/server/__init__.py index d672ed38..d203ae8d 100644 --- a/ably/pubsub/server/__init__.py +++ b/ably/pubsub/server/__init__.py @@ -153,6 +153,10 @@ def create_http_client(**kwargs) -> PubSubHttpClient: add_request_ids: bool Add a `request_id` query parameter to every request, which makes a request traceable in Ably's logs. Defaults to False. + max_message_size: int + The largest total size of the messages one publish may carry, counting each + message's name, data, client ID and extras. A larger publish raises an + AblyException with code 40009 and is not sent. The default is 65536 (64 KiB). Returns ------- diff --git a/ably/pubsub/transport/defaults.py b/ably/pubsub/transport/defaults.py index d239656a..cdb93d73 100644 --- a/ably/pubsub/transport/defaults.py +++ b/ably/pubsub/transport/defaults.py @@ -27,6 +27,8 @@ class Defaults: fallback_retry_timeout = 600000 # 10min + max_message_size = 65536 # 64KiB + @staticmethod def get_port(options): if options.tls: diff --git a/ably/pubsub/types/message.py b/ably/pubsub/types/message.py index ad574154..aaf1b4d6 100644 --- a/ably/pubsub/types/message.py +++ b/ably/pubsub/types/message.py @@ -1,4 +1,5 @@ import base64 +import json import logging import os from enum import IntEnum @@ -393,6 +394,41 @@ def update_inner_message_fields(proto_msg: dict): msg_index = msg_index + 1 +def message_size(message): + """The size of a message as TM6 defines it, for checking against maxMessageSize (TO3l8).""" + def json_length(value): + return len(json.dumps(value, separators=(',', ':'), ensure_ascii=False)) + + # TM6a: the sum of the name, data, clientId and extras sizes; TM6e: an absent one is zero + size = len(message.name or '') + len(message.client_id or '') + if message.extras is not None: + # TM6d + size += json_length(message.extras) + + data = message.data + if isinstance(data, CipherData): + data = data.buffer + if isinstance(data, (bytes, bytearray, str)): + # TM6c: the bytes themselves, whichever protocol carries them; TM6f: a string's length + size += len(data) + elif isinstance(data, (dict, list)): + # TM6b + size += json_length(data) + return size + + +def validate_message_size(messages, max_message_size): + """Raises 40009 if the messages exceed max_message_size between them (RSL1i, TO3l8f).""" + size = sum(message_size(message) for message in messages) + if size > max_message_size: + raise AblyException( + f"Maximum size of messages that can be published at once exceeded " + f"(was {size} bytes; limit is {max_message_size} bytes)", + 400, + 40009, + ) + + def assign_idempotent_ids(messages): """Gives every message a library-generated id, unless one of them already has an id (RSL1k1).""" if all(message.id is None for message in messages): diff --git a/ably/pubsub/types/options.py b/ably/pubsub/types/options.py index 84551f46..36597162 100644 --- a/ably/pubsub/types/options.py +++ b/ably/pubsub/types/options.py @@ -34,7 +34,7 @@ def __init__(self, client_id=None, tls=True, port=0, suspended_retry_timeout=None, connectivity_check_url=None, channel_retry_timeout=Defaults.channel_retry_timeout, add_request_ids=False, vcdiff_decoder: VCDiffDecoder = None, transport_params=None, echo_messages=True, - _test_options=None, **kwargs): + max_message_size=None, _test_options=None, **kwargs): super().__init__(**kwargs) @@ -67,6 +67,10 @@ def __init__(self, client_id=None, tls=True, port=0, if suspended_retry_timeout is None: suspended_retry_timeout = Defaults.suspended_retry_timeout + # TO3l8 + if max_message_size is None: + max_message_size = Defaults.max_message_size + if idempotent_rest_publishing is None: from ably.pubsub.version import api_version idempotent_rest_publishing = api_version >= '1.2' @@ -104,6 +108,7 @@ def __init__(self, client_id=None, tls=True, port=0, self.__add_request_ids = add_request_ids self.__vcdiff_decoder = vcdiff_decoder self.__transport_params = transport_params or {} + self.__max_message_size = max_message_size self.__test_options = _test_options self.__hosts = self.__get_hosts() @@ -271,6 +276,10 @@ def vcdiff_decoder(self): def transport_params(self): return self.__transport_params + @property + def max_message_size(self): + return self.__max_message_size + @property def _test_options(self): return self.__test_options diff --git a/ably/pubsub/util/helper.py b/ably/pubsub/util/helper.py index 934a1af1..f3bcf351 100644 --- a/ably/pubsub/util/helper.py +++ b/ably/pubsub/util/helper.py @@ -1,16 +1,11 @@ import asyncio import inspect -import json import random import string import time from typing import Callable, Dict, Tuple from urllib.parse import parse_qs, urlparse -import msgpack - -from ably.pubsub.util.exceptions import AblyException - def get_random_id(): # get random string of letters and digits @@ -76,30 +71,6 @@ def cancel(self): self._task.cancel() -def validate_message_size(encoded_messages: list, use_binary_protocol: bool, max_message_size: int) -> None: - """Validate that encoded messages don't exceed the maximum size limit. - - Args: - encoded_messages: List of encoded message dictionaries - use_binary_protocol: Whether to use binary (msgpack) or JSON encoding - max_message_size: Maximum allowed size in bytes - - Raises: - AblyException: If the encoded messages exceed the maximum size - """ - if use_binary_protocol: - size = len(msgpack.packb(encoded_messages, use_bin_type=True)) - else: - size = len(json.dumps(encoded_messages, separators=(',', ':')).encode('utf-8')) - - if size > max_message_size: - raise AblyException( - f"Maximum size of messages that can be published at once exceeded " - f"(was {size} bytes; limit is {max_message_size} bytes)", - 400, - 40009, - ) - def to_text(value): if value is None: return value diff --git a/test/unit/message_test.py b/test/unit/message_test.py index f49e9bf9..22e0b53a 100644 --- a/test/unit/message_test.py +++ b/test/unit/message_test.py @@ -1,4 +1,8 @@ +import pytest + import ably.pubsub.types.message +from ably.pubsub.types.message import Message, message_size, validate_message_size +from ably.pubsub.util.exceptions import AblyException # TM2a, TM2c, TM2f @@ -57,3 +61,37 @@ def test_invalid_base64_data_is_delivered_with_remaining_encoding(): assert message.data == 'encrypted-data-here' assert message.encoding == 'custom-encryption/base64' + + +# TM6a, TM6e +def test_message_size_sums_name_data_client_id_and_extras(): + assert message_size(Message()) == 0 + assert message_size(Message(name='event', data='hello', client_id='me')) == 12 + # TM6d: extras count as their JSON + assert message_size(Message(extras={'push': {'a': 1}})) == len('{"push":{"a":1}}') + # An id is not counted + assert message_size(Message(id='abc:0', data='x')) == 1 + + +# TM6b, TM6c, TM6f +def test_message_size_of_each_kind_of_data(): + # TM6f: a string is its length, and a non-ASCII one is not counted as its JSON escape + assert message_size(Message(data='h\u00e9llo')) == 5 + # TM6c: binary is its byte length, not the length of its base64 + assert message_size(Message(data=b'\x00' * 10)) == 10 + assert message_size(Message(data=bytearray(10))) == 10 + # TM6b: an object or array is the length of its JSON + assert message_size(Message(data={'k': 'v'})) == len('{"k":"v"}') + assert message_size(Message(data=[1, '\u00e9'])) == len('[1,"\u00e9"]') + + +# RSL1i, TO3l8f: the limit applies to the messages published together +def test_validate_message_size_raises_40009_above_the_limit(): + messages = [Message(name='a', data='x' * 4), Message(name='b', data='x' * 4)] + + validate_message_size(messages, 10) + with pytest.raises(AblyException) as excinfo: + validate_message_size(messages, 9) + + assert excinfo.value.code == 40009 + assert excinfo.value.status_code == 400 diff --git a/test/uts/deviations.md b/test/uts/deviations.md index 22eeebc1..8ef1f73e 100644 --- a/test/uts/deviations.md +++ b/test/uts/deviations.md @@ -33,8 +33,8 @@ parametrized over a table of fixtures the specification gives inline; and five `objects/unit` tests are parametrized, three over a table and two over the scenarios the specification loops through. That turns 1481 derived tests into 1608 pytest cases. -Of **1472 Test IDs, derived as 1481 tests and run as 1608 pytest cases**: 1292 Test IDs -(1301 tests, 1426 cases) pass, 153 (153 tests, 155 cases) are gated behind +Of **1472 Test IDs, derived as 1481 tests and run as 1608 pytest cases**: 1293 Test IDs +(1302 tests, 1427 cases) pass, 152 (152 tests, 154 cases) are gated behind `RUN_DEVIATIONS`, and 27 (27 tests, 27 cases) cannot be run at all. The three groups are disjoint: two Test IDs have a gated part and a passing part — `TI1`, as two functions, and `TO/endpoint-affects-host`, as one parametrized test — and are counted with the gated, @@ -45,17 +45,17 @@ the Test IDs come from `uts/rest/unit` (503 tests, 536 cases), 482 from `uts/rest/integration` (84, 122), 73 from `uts/realtime/integration` (73, 95) and 17 from `uts/objects/integration` (17, 29); 8 of the REST integration ids (8, 8), 30 of the realtime ones (30, 30) and 5 of the objects ones (5, 5) come from the `proxy` package -within each. Of the gated Test IDs 70 are REST and 83 realtime, which is 71 REST cases +within each. Of the gated Test IDs 69 are REST and 83 realtime, which is 70 REST cases and 84 realtime. None of the 339 objects Test IDs is gated or unrunnable. A further 148 pytest cases cover the harness itself and are not derived from a specification: 130 under `helpers/`, for the mocks, and 18 under `objects/helpers/`, for the objects builders and the synced-channel mock. -The 142 gated Test IDs that record SDK non-compliance — 142 tests, 144 cases — reduce to -**61 distinct root causes**, 21 on the REST side and 40 on the realtime side. Two further +The 141 gated Test IDs that record SDK non-compliance — 141 tests, 143 cases — reduce to +**60 distinct root causes**, 20 on the REST side and 40 on the realtime side. Two further defects are recorded below with no test of their own, because the specification's test cannot discriminate (RTP18a) or has nothing to assert against (the timezone split on -synthesized LEAVE timestamps), so the file carries **63 SDK root causes** in all. The remaining 11 gated Test IDs are +synthesized LEAVE timestamps), so the file carries **62 SDK root causes** in all. The remaining 11 gated Test IDs are specification faults, and reduce to 8. The objects tier records no SDK non-compliance; its specification faults are all corrected or worked around in tests that pass, and reduce to the seven candidate issues O.1–O.7, none of them filed. @@ -252,6 +252,11 @@ rather than merging." `RSA10k`'s setup depends on the same premise and cannot re - `RSL4/encoding-fixtures-ably-common-0` loads `encoding.json` from ably-common. No such file exists; RSL6a1 names `messages-encoding.json`, which has a different schema and runs in the decode direction. +- `RSL1i/message-size-limit-0` publishes `name: "event"` with `"x" * size` as the data + and expects the 1024-byte case to pass a 1024-byte limit, but TM6a counts the name as + well as the data, so that message is 1029 bytes. The derived test shortens the data by + the name's five bytes, so each message is the size its case states, and passes. Not + filed. ### Presence actions are written as strings @@ -1058,8 +1063,7 @@ on the `recover` parameter the connection never sends. | TB3 | `ChannelOptions.withCipherKey`. The nearest equivalent, `ably.util.crypto.get_default_params({'key': key})`, is not reachable from `ChannelOptions` | 1 | | RTL2i, TH6 | `hasBacklog` on `ChannelStateChange`. `Flag.HAS_BACKLOG` is defined (`types/flags.py:7`) but `_on_message` reads only RESUMED and HAS_PRESENCE (`channel.py:715-721`). See the UTS Spec Error above: the features spec makes this optional | 1 | | RTP6b | Subscribing to an *array* of presence actions. The list reaches `EventEmitter.on` and pyee uses the event as a dict key, so `presence.subscribe([ENTER, LEAVE], listener)` raises `TypeError: unhashable type: 'list'`. A fix has to fan the list out into one registration per action, and `unsubscribe` with it. `realtime/integration/presence_lifecycle_test.py::test_rtp4_bulk_enter_observed` adapts instead of gating: it registers the one listener once for `'enter'` and once for `'present'`, which is what the array form means, and the counted set is identical | 1 | -| RSL1i | REST publish never calls `validate_message_size`. The helper exists and is correct, but only `ably/realtime/channel.py:423` calls it, so an oversized REST publish goes out | 1 | -| TP5 | `size` on `PresenceMessage`. The related `maxMessageSize` gap is adapted rather than gated, below; `features.md` TM6 has no UTS test | 1 | +| TP5 | `size` on `PresenceMessage`. Published messages are measured per TM6 and checked against `max_message_size` (RSL1i, TO3l8), but presence messages are measured by nothing, and neither type exposes its size | 1 | ### Connection @@ -2218,7 +2222,7 @@ pseudocode is not mistaken for non-compliance. | RSA8d | `error.message` contains the callback's text | Wrapped as 40170 with the original in `cause`; `__str__` renders both | Rendering | | CHM2 | Missing metrics default to 0 | `ChannelMetrics.from_dict` uses a bare `obj.get(name)`, so any omitted metric parses as `None` | Open bug, broader than CHM2g/h | | CHM2g, CHM2h | `objectPublishers` and `objectSubscribers` on `ChannelMetrics` | Neither is modelled, so both are dropped on parsing. The test asserts their absence, and turns red once they are added | Open bug | -| TO3l8 | `maxMessageSize` is a client option, default 65536 | Rejected by `Options.__init__`. `ably/realtime/channel.py:422` reads it with `getattr(..., 65536)`, so the default holds but cannot be configured, nor overridden by `connectionDetails` (CD2c) | Open bug | +| TO3l8, CD2c | `maxMessageSize` is a client option, default 65536, which `connectionDetails.maxMessageSize` overrides for realtime publishes | The option and its default match, and REST and realtime publishes both measure per TM6 and raise 40009. `ConnectionDetails.from_dict` does not parse `maxMessageSize`, so a realtime publish is checked against the client option alone. No test asserts CD2c; `test_rtn24_connection_details_override` sends the field and reads back only the details ably-python parses | Open bug, CD2c only | | RTN15, RTN23 | A DISCONNECTED `ErrorInfo` needs no `statusCode` | `ConnectionManager.on_disconnected` evaluates `exception.status_code >= 500` unguarded, so a DISCONNECTED whose error omits `statusCode` raises `TypeError` in a task whose exception is only logged, and the connection silently stays CONNECTED. `DISCONNECTED_MESSAGE` supplies 400 | Open bug | | TO3l4, TO3l5 | `httpRequestTimeout` defaults to 10000 and `httpMaxRetryCount` to 3 on the options object | Both are defaulted on `Options`. `http_max_retry_count` reads back as 3 and matches; `http_request_timeout` reads back as 10, because ably-python's is seconds where the spec's is milliseconds, on the value a caller passes as much as on the default — gated, with the measurement, under *`httpRequestTimeout` is seconds where the specification counts milliseconds* in Failing Tests | The unit is an open bug, recorded there | | RTC7 (TO3l3, TO3l4) | `client.options.httpOpenTimeout == 4000` and `httpRequestTimeout == 10000` | 4 and 10: both are defaulted on `Options`, in seconds where TO3l3 and TO3l4 count milliseconds. The three realtime timeouts the same test checks are milliseconds and match | Open bug, and the unit reaches the wire — `rest/proxy/RSC15l2/timeout-triggers-fallback-0` measures a request outliving its configured timeout by a factor of a thousand. Gated under *`httpRequestTimeout` is seconds where the specification counts milliseconds* in Failing Tests. Moving the HTTP durations to milliseconds is breaking, and is held back from 4.0.0 pending a decision on units across the options | @@ -3535,8 +3539,8 @@ The header states how many derived tests there are, how many pass, how many are gated and how many cannot run. Those numbers are the check that the file is still true: in pytest cases, the gated count must equal the number of failures under `RUN_DEVIATIONS=1`, and gated plus unrunnable must equal the number of skips without -it. As of this writing that is 155 failures, 1574 passes and 27 skips with the variable -set, and 182 skips and 1574 passes without it, the 1574 being 1426 derived cases and 148 +it. As of this writing that is 154 failures, 1575 passes and 27 skips with the variable +set, and 181 skips and 1575 passes without it, the 1575 being 1427 derived cases and 148 harness ones — 130 under `helpers/` and 18 under `objects/helpers/`. Those figures need the `submodules` checkout CI makes: without it, three `rest/unit/encoding` tests that read the ably-common fixtures skip as well. The tiers are told apart by the junit report's diff --git a/test/uts/rest/unit/channel/publish_test.py b/test/uts/rest/unit/channel/publish_test.py index 7ebef9d4..2cca9168 100644 --- a/test/uts/rest/unit/channel/publish_test.py +++ b/test/uts/rest/unit/channel/publish_test.py @@ -11,7 +11,7 @@ from ably.pubsub.types.message import Message from ably.pubsub.util.exceptions import AblyException from test.uts.helpers.client import rest_client -from test.uts.helpers.deviations import deviation, spec_error +from test.uts.helpers.deviations import spec_error from test.uts.helpers.mock_http import MockHttpClient @@ -173,7 +173,6 @@ def on_request(request): # UTS: rest/unit/RSL1i/message-size-limit-0 -@deviation async def test_rsl1i_message_size_limit(): channel_name = f'test-RSL1i-{random_id()}' captured_requests = [] @@ -203,7 +202,10 @@ def on_request(request): captured_requests.clear() request_count = 0 - large_data = 'x' * size + # UTS SPEC ERROR: the spec publishes `"x" * size` as the data, but TM6a counts the + # name too, so its at-limit case is five bytes over. The data is shortened by the + # name's length so that the message is the size each case states. See deviations.md. + large_data = 'x' * (size - len('event')) if expect_success: await channel.publish(name='event', data=large_data) diff --git a/test/uts/rest/unit/types/options_types_test.py b/test/uts/rest/unit/types/options_types_test.py index a0a46c33..93a9c520 100644 --- a/test/uts/rest/unit/types/options_types_test.py +++ b/test/uts/rest/unit/types/options_types_test.py @@ -47,11 +47,7 @@ async def test_to3_client_options_attributes(): assert options.add_request_ids is False assert options.query_time is False - # DEVIATION: TO3l8 `maxMessageSize` (default 65536) is not a client option here. Realtime - # publishes read it with getattr(options, 'max_message_size', 65536), so the 64KiB default - # holds, but it cannot be configured and passing it is rejected. - with pytest.raises(TypeError): - Options(max_message_size=65536) + assert options.max_message_size == 65536 # The HTTP layer applies the defaults the options hold: 10 seconds, and the primary # host plus 3 fallbacks to try (TO3l5). From d6835beee69f863fe855278a8ac692f9dd8dfb62 Mon Sep 17 00:00:00 2001 From: owenpearson Date: Mon, 26 Oct 2026 18:41:11 +0000 Subject: [PATCH 49/71] pubsub: remove the recover client option TO3i's recover option names a recovery key, from which RTN16 has a client inherit the state of an earlier connection. ably-python does not implement connection recovery: it accepted recover, documented it on the realtime factory, and stored it with a property and a setter, but never sent it, so a client given a recovery key silently started a fresh connection. The option, its property and its docstring are removed. The RTN16 and RTC1c tests stay gated, and the RTN16f1 malformed-key test, which passed only because the key was ignored, is gated with them; test/uts/deviations.md records the change and updates its counts. This is a breaking change: passing recover now raises TypeError. Callers should drop it. Connection resumption (RTN15) is unaffected, and connection recovery may return, with the option, in a minor release, as UPDATING.md notes. Co-Authored-By: Claude Opus 5.5 --- UPDATING.md | 19 ++++++++++++++ ably/pubsub/server/__init__.py | 3 --- ably/pubsub/types/options.py | 11 +------- test/uts/deviations.md | 26 ++++++++++--------- .../unit/client/realtime_client_test.py | 4 +-- .../connection/connection_recovery_test.py | 15 ++++++----- 6 files changed, 44 insertions(+), 34 deletions(-) diff --git a/UPDATING.md b/UPDATING.md index 801db6dc..3e9085fd 100644 --- a/UPDATING.md +++ b/UPDATING.md @@ -534,6 +534,25 @@ your Ably account allows larger messages: pubsub_http_client = create_http_client(key='xxx', max_message_size=131072) ``` +### The `recover` option is removed + +The realtime client accepted a `recover` option, documented as resuming an earlier +connection from its recovery key, but connection recovery is not implemented: the option was +stored and never sent, so a client given one always started a fresh connection. It is gone, +and passing it raises `TypeError`. Remove it, and rely on the client's own resumption of a +dropped connection, which is implemented. Connection recovery may return in a minor release, +along with the `recover` option. + +Example 3.x code: +```python +realtime = AblyRealtime(key='xxx', recover=recovery_key) +``` + +Example 4.0.0 code: +```python +pubsub_realtime_client = create_realtime_client(key='xxx') +``` + ## Version 2.x to 3.0.0 The 3.0.0 version of ably-python introduces several breaking changes to improve the realtime experience and align the API with the Ably specification. These include: diff --git a/ably/pubsub/server/__init__.py b/ably/pubsub/server/__init__.py index d203ae8d..0582dcf4 100644 --- a/ably/pubsub/server/__init__.py +++ b/ably/pubsub/server/__init__.py @@ -194,9 +194,6 @@ def create_realtime_client(**kwargs) -> PubSubRealtimeClient: auto_connect: bool Connect as soon as the client is created. Set this to False to connect later with `connect()`. Defaults to True. - recover: str - A recovery key from a previous connection, used to resume that connection's - state and message continuity rather than starting fresh. queue_messages: bool Hold messages published while the connection is not yet established and send them once it is, rather than failing them. Defaults to True. diff --git a/ably/pubsub/types/options.py b/ably/pubsub/types/options.py index 36597162..4ff30c99 100644 --- a/ably/pubsub/types/options.py +++ b/ably/pubsub/types/options.py @@ -26,7 +26,7 @@ def decode(self, delta: bytes, base: bytes) -> bytes: class Options(AuthOptions): def __init__(self, client_id=None, tls=True, port=0, - tls_port=0, use_binary_protocol=True, queue_messages=True, recover=False, endpoint=None, + tls_port=0, use_binary_protocol=True, queue_messages=True, endpoint=None, http_open_timeout=None, http_request_timeout=None, realtime_request_timeout=None, http_max_retry_count=None, http_max_retry_duration=None, fallback_hosts=None, fallback_retry_timeout=None, disconnected_retry_timeout=None, @@ -87,7 +87,6 @@ def __init__(self, client_id=None, tls=True, port=0, self.__use_binary_protocol = use_binary_protocol self.__queue_messages = queue_messages self.__echo_messages = echo_messages - self.__recover = recover self.__endpoint = endpoint self.__http_open_timeout = http_open_timeout self.__http_request_timeout = http_request_timeout @@ -165,14 +164,6 @@ def queue_messages(self, value): def echo_messages(self): return self.__echo_messages - @property - def recover(self): - return self.__recover - - @recover.setter - def recover(self, value): - self.__recover = value - @property def endpoint(self): return self.__endpoint diff --git a/test/uts/deviations.md b/test/uts/deviations.md index 8ef1f73e..b42aa6b0 100644 --- a/test/uts/deviations.md +++ b/test/uts/deviations.md @@ -33,8 +33,8 @@ parametrized over a table of fixtures the specification gives inline; and five `objects/unit` tests are parametrized, three over a table and two over the scenarios the specification loops through. That turns 1481 derived tests into 1608 pytest cases. -Of **1472 Test IDs, derived as 1481 tests and run as 1608 pytest cases**: 1293 Test IDs -(1302 tests, 1427 cases) pass, 152 (152 tests, 154 cases) are gated behind +Of **1472 Test IDs, derived as 1481 tests and run as 1608 pytest cases**: 1292 Test IDs +(1301 tests, 1426 cases) pass, 153 (153 tests, 155 cases) are gated behind `RUN_DEVIATIONS`, and 27 (27 tests, 27 cases) cannot be run at all. The three groups are disjoint: two Test IDs have a gated part and a passing part — `TI1`, as two functions, and `TO/endpoint-affects-host`, as one parametrized test — and are counted with the gated, @@ -45,13 +45,13 @@ the Test IDs come from `uts/rest/unit` (503 tests, 536 cases), 482 from `uts/rest/integration` (84, 122), 73 from `uts/realtime/integration` (73, 95) and 17 from `uts/objects/integration` (17, 29); 8 of the REST integration ids (8, 8), 30 of the realtime ones (30, 30) and 5 of the objects ones (5, 5) come from the `proxy` package -within each. Of the gated Test IDs 69 are REST and 83 realtime, which is 70 REST cases -and 84 realtime. None of the 339 objects Test IDs is gated or unrunnable. +within each. Of the gated Test IDs 69 are REST and 84 realtime, which is 70 REST cases +and 85 realtime. None of the 339 objects Test IDs is gated or unrunnable. A further 148 pytest cases cover the harness itself and are not derived from a specification: 130 under `helpers/`, for the mocks, and 18 under `objects/helpers/`, for the objects builders and the synced-channel mock. -The 141 gated Test IDs that record SDK non-compliance — 141 tests, 143 cases — reduce to +The 142 gated Test IDs that record SDK non-compliance — 142 tests, 144 cases — reduce to **60 distinct root causes**, 20 on the REST side and 40 on the realtime side. Two further defects are recorded below with no test of their own, because the specification's test cannot discriminate (RTP18a) or has nothing to assert against (the timezone split on @@ -1036,15 +1036,17 @@ issued token are all real, and each test reaches the missing call before it fail assertions either side of it are known to hold against real server responses. The `push_channels` file was additionally run against a throwaway shim — a `PushChannel` posting and deleting `/push/channelSubscriptions` with `X-Ably-DeviceToken` — and passes in -full against it. The two recovery tests are a different shape: there is no missing call for -them to reach, so each runs end to end against the sandbox through `uts-proxy` and fails -on the `recover` parameter the connection never sends. +full against it. The two recovery tests reach the missing option rather than a missing +call: each fails building the client that passes `recover`, which raises `TypeError` now the +option is gone, after RTN16d has done its first phase against the sandbox through +`uts-proxy`. Before 4.0.0 removed the option, both ran end to end and failed only on the +`recover` parameter the connection never sent. | Spec points | Missing | Test IDs | |---|---|---| | RSA17, RSA17b–g, BAR2, TRS2, TRF2 | `Auth#revokeTokens`, `TokenRevocationTargetSpecifier` and the two token revocation result types; the `BatchResult` they would arrive in is the batch API's. Gated against `auth.revoke_tokens(targets, issued_before=, allow_reauth_margin=)` returning `success_count` / `failure_count` / `results`, with `target` / `issued_before` / `applies_at` / `error` per result. RSA17d is the one case that needs no server at all — a token-authenticated client must refuse locally with 40162/401 — so it can be satisfied before any of the wire work | 21 | | RSH7, RSH7a–e, RSH6, RSH8 | `PushChannel`: `channel.push`, `client.device`, `LocalDevice`. The push *admin* surface (RSH1) does exist | 12 | -| RTN16, RTN16f–k, RTC1c (TO3i) | Connection recovery, entire. `recover` is in the `Options` signature, stored, and given a property and a setter (`options.py:30,111,193,196`), and read nowhere. No `Connection#createRecoveryKey`, no `recover` connect parameter, no recovery-key decoding. Measured through the proxy: a client built with a valid `recover=` opened a `ws_connect` whose query parameters were `{'accessToken': …, 'v': '5'}` — no `recover` — and was given a fresh `connectionId`. RTN16l is otherwise fully compliant, taking the proxy's `recovery-failed-new-id`, `recovery-failed-new-key` and error 80008 and staying CONNECTED; only the absent parameter fails it | 8 | +| RTN16, RTN16f–k, RTC1c (TO3i) | Connection recovery, entire. No `recover` client option — 3.x accepted and stored one and read it nowhere, and 4.0.0 removed it, so passing it raises `TypeError` — no `Connection#createRecoveryKey`, no `recover` connect parameter, no recovery-key decoding. Measured through the proxy before the option was removed: a client built with a valid `recover=` opened a `ws_connect` whose query parameters were `{'accessToken': …, 'v': '5'}` — no `recover` — and was given a fresh `connectionId`. RTN16l was otherwise fully compliant, taking the proxy's `recovery-failed-new-id`, `recovery-failed-new-key` and error 80008 and staying CONNECTED; only the absent parameter failed it. RTN16f1's malformed key passed while the option was ignored, and is gated with the rest now that it cannot be passed | 9 | | RTL22, RTL22a–d, MFI1, MFI2a–e | `MessageFilter`. `RealtimeChannel.subscribe` (`channel.py:262-273`) accepts only a `str` or a callable, and there is no filter type of any shape to spell. Each test builds its filter through the module's `message_filter()` helper, which is the one place to repoint when the type lands | 5 | | RTS5, RTS5a, RTS5a1, RTS5a2, DO2a | Derived channels: `DeriveOptions` and `Channels.getDerived`. `grep -r derive ably/` is empty. Each test imports `DeriveOptions` inside its body so the module still loads | 5 | | RTB1, RTB1a, RTB1b | Retry backoff, jitter and `retryIn`. Retry timers schedule the flat configured timeout (`connectionmanager.py:753`, `channel.py:866-871`); `grep` for jitter/backoff/retry_in returns nothing, and neither `ConnectionStateChange` nor `ChannelStateChange` carries `retryIn` | 4 | @@ -2878,7 +2880,7 @@ Each row is one feature and one issue. None is a bug in existing code. | Feature | Spec points | Tests | Reproduction (`-k` against `test/uts/realtime/unit/`) | |---|---|---|---| -| Connection recovery, entire — `createRecoveryKey`, the `recover` connect parameter, recovery-key decoding. `recover` is stored on `Options` and read nowhere. Measured through the proxy: a client given a valid `recover=` sends no `recover` query parameter and is issued a fresh `connectionId` | RTN16, RTN16f–k, RTC1c | 8 | `connection/connection_recovery_test.py`, `client/realtime_client_test.py -k rtc1c`, and `test/uts/realtime/integration/proxy/connection_resume_test.py -k "rtn16d or rtn16l"` | +| Connection recovery, entire — the `recover` client option, `createRecoveryKey`, the `recover` connect parameter, recovery-key decoding. 3.x stored `recover` and read it nowhere, and 4.0.0 removed it. Measured through the proxy before then: a client given a valid `recover=` sent no `recover` query parameter and was issued a fresh `connectionId` | RTN16, RTN16f–k, RTC1c | 9 | `connection/connection_recovery_test.py`, `client/realtime_client_test.py -k rtc1c`, and `test/uts/realtime/integration/proxy/connection_resume_test.py -k "rtn16d or rtn16l"` | | `MessageFilter` and filtered subscriptions | RTL22, RTL22a–d, MFI1, MFI2a–e | 5 | `channels/channel_subscribe_test.py -k rtl22` | | Derived channels — `DeriveOptions`, `Channels.getDerived` | RTS5, RTS5a, RTS5a1, RTS5a2, DO2a | 5 | `channels/channel_options_test.py -k "rts5 or do2a"` | | Retry backoff, jitter and `retryIn` on both state-change types | RTB1, RTB1a, RTB1b | 4 | `connection/backoff_jitter_test.py` | @@ -3539,8 +3541,8 @@ The header states how many derived tests there are, how many pass, how many are gated and how many cannot run. Those numbers are the check that the file is still true: in pytest cases, the gated count must equal the number of failures under `RUN_DEVIATIONS=1`, and gated plus unrunnable must equal the number of skips without -it. As of this writing that is 154 failures, 1575 passes and 27 skips with the variable -set, and 181 skips and 1575 passes without it, the 1575 being 1427 derived cases and 148 +it. As of this writing that is 155 failures, 1574 passes and 27 skips with the variable +set, and 182 skips and 1574 passes without it, the 1574 being 1426 derived cases and 148 harness ones — 130 under `helpers/` and 18 under `objects/helpers/`. Those figures need the `submodules` checkout CI makes: without it, three `rest/unit/encoding` tests that read the ably-common fixtures skip as well. The tiers are told apart by the junit report's diff --git a/test/uts/realtime/unit/client/realtime_client_test.py b/test/uts/realtime/unit/client/realtime_client_test.py index fd9bfa1d..a5f2a005 100644 --- a/test/uts/realtime/unit/client/realtime_client_test.py +++ b/test/uts/realtime/unit/client/realtime_client_test.py @@ -195,8 +195,8 @@ async def test_rtc1b_auto_connect_option(): # UTS: realtime/unit/RTC1c/recover-option-0 -# DEVIATION: the `recover` option is stored and never read, so no `recover` query -# parameter is ever sent. See deviations.md. +# DEVIATION: there is no `recover` client option, so this raises TypeError on +# construction. See deviations.md. @deviation async def test_rtc1c_recover_option(): recovery_key = encode_recovery_key('previous-connection-key', 5, {'channel1': 'serial1'}) diff --git a/test/uts/realtime/unit/connection/connection_recovery_test.py b/test/uts/realtime/unit/connection/connection_recovery_test.py index 9684513c..c862c14b 100644 --- a/test/uts/realtime/unit/connection/connection_recovery_test.py +++ b/test/uts/realtime/unit/connection/connection_recovery_test.py @@ -3,11 +3,10 @@ Spec points: RTN16d, RTN16f, RTN16f1, RTN16g, RTN16g1, RTN16g2, RTN16g3, RTN16i, RTN16j, RTN16k, RTN16l -RTN16 connection recovery is absent from ably-python: `recover` is a client -option with a property and a setter but is read nowhere in the library, there is -no `recover` connect parameter and no recovery key to create or decode. Every -test here but the malformed-key one is therefore gated; see -deviations.md. +RTN16 connection recovery is absent from ably-python: there is no `recover` +client option, no `recover` connect parameter and no recovery key to create or +decode, so passing `recover` raises TypeError. Every test here is therefore +gated; see deviations.md. """ import asyncio @@ -240,6 +239,9 @@ async def test_rtn16f_recover_initializes_msgserial(): # UTS: realtime/unit/RTN16f1/malformed-recovery-key-0 +# DEVIATION: there is no `recover` client option, so this raises TypeError on +# construction. See deviations.md. +@deviation async def test_rtn16f1_malformed_recovery_key(): attempts = [] @@ -259,8 +261,7 @@ def on_connection_attempt(conn): assert client.connection.connection_manager.connection_id == 'fresh-conn' assert client.connection.connection_details.connection_key == 'fresh-key' - # The malformed key reaches the connection parameters no more than a valid - # one would: `recover` is stored and never read + # RTN16f1: a malformed key is dropped, and the connection proceeds without it assert 'recover' not in attempts[0].url.query_params assert 'resume' not in attempts[0].url.query_params assert len(attempts) == 1 From 2e5c6d4291b337742e1de71bfdcf964532444b1f Mon Sep 17 00:00:00 2001 From: owenpearson Date: Mon, 26 Oct 2026 20:38:46 +0000 Subject: [PATCH 50/71] pubsub: make PresenceMessage.timestamp milliseconds since the epoch PresenceMessage.timestamp was decoded into a naive datetime through the deprecated datetime.utcfromtimestamp, while the LEAVE messages the client synthesizes (RTP19, RTP19a) were stamped with an aware datetime.now(timezone.utc), so the two could not be compared, and the synthesized ones bypassed the client's clock. TP3g gives the timestamp as milliseconds since the epoch, which is what Message.timestamp already is. The timestamp is now the int from the wire, sent back unchanged by to_encoded, and a synthesized LEAVE takes its timestamp from the clock's now_ms(). The RTP2b1 newness check compares timestamps numerically, and the realtime channel still fills a missing timestamp from the ProtocolMessage's before decoding (TP3g). Presence history's start and end now also accept an aware datetime, as channel history does, where before an aware one raised TypeError; a naive one is still read as UTC. This is a breaking change, and a silent one: comparing a timestamp with == against a datetime now returns False. Callers that need a datetime convert with datetime.fromtimestamp(timestamp / 1000, tz=timezone.utc). The UTS tests drop their datetime workarounds and assert the specification's millisecond values, which closes the timezone-split deviation. Co-Authored-By: Claude Opus 5.5 --- UPDATING.md | 21 +++++++++ ably/pubsub/realtime/presence.py | 7 +-- ably/pubsub/types/presence.py | 35 +++++++-------- test/ably/http/httppresence_test.py | 4 +- test/ably/realtime/presencemap_test.py | 15 +++---- test/uts/deviations.md | 44 ++++--------------- test/uts/helpers/presence.py | 7 ++- .../unit/presence/presence_sync_test.py | 12 ++--- .../rest/unit/presence/rest_presence_test.py | 9 ++-- .../unit/types/presence_message_types_test.py | 10 +---- 10 files changed, 73 insertions(+), 91 deletions(-) diff --git a/UPDATING.md b/UPDATING.md index 3e9085fd..19b0245d 100644 --- a/UPDATING.md +++ b/UPDATING.md @@ -553,6 +553,27 @@ Example 4.0.0 code: pubsub_realtime_client = create_realtime_client(key='xxx') ``` +### `PresenceMessage.timestamp` is milliseconds since the epoch + +**This change is silent: comparing a timestamp with `==` against a `datetime` now returns +`False` rather than raising.** `PresenceMessage.timestamp` is an `int` of milliseconds since +the unix epoch, as `Message.timestamp` already is. 3.x returned a naive UTC `datetime` for a +presence message received from Ably, and an aware one for a LEAVE the library synthesized. +An ordering comparison such as `<` against a `datetime` raises `TypeError`. Convert the +timestamp where you need a `datetime`: + +Example 3.x code: +```python +received_at = member.timestamp +``` + +Example 4.0.0 code: +```python +from datetime import datetime, timezone + +received_at = datetime.fromtimestamp(member.timestamp / 1000, tz=timezone.utc) +``` + ## Version 2.x to 3.0.0 The 3.0.0 version of ably-python introduces several breaking changes to improve the realtime experience and align the API with the Ably specification. These include: diff --git a/ably/pubsub/realtime/presence.py b/ably/pubsub/realtime/presence.py index 70e21774..2c87f50e 100644 --- a/ably/pubsub/realtime/presence.py +++ b/ably/pubsub/realtime/presence.py @@ -9,13 +9,13 @@ import asyncio import logging -from datetime import datetime, timezone from typing import TYPE_CHECKING, Any from ably.pubsub.realtime.connection import ConnectionState from ably.pubsub.realtime.presencemap import PresenceMap from ably.pubsub.types.channelstate import ChannelState, ChannelStateChange from ably.pubsub.types.presence import PresenceAction, PresenceMessage +from ably.pubsub.util.clock import select_clock from ably.pubsub.util.eventemitter import EventEmitter from ably.pubsub.util.exceptions import AblyException @@ -69,6 +69,7 @@ def __init__(self, channel: RealtimeChannel): super().__init__() self.channel = channel self.sync_complete = False + self._clock = select_clock(channel.ably.options) # RTP2: Main presence map keyed by memberKey (connectionId:clientId) self.members = PresenceMap( @@ -583,7 +584,7 @@ def set_presence( connection_id=member.connection_id, data=member.data, encoding=member.encoding, - timestamp=datetime.now(timezone.utc) + timestamp=self._clock.now_ms() ) broadcast_messages.append(synthesized_leave) @@ -717,7 +718,7 @@ def _synthesize_leaves(self, members: list[PresenceMessage]) -> None: connection_id=member.connection_id, data=member.data, encoding=member.encoding, - timestamp=datetime.now(timezone.utc) + timestamp=self._clock.now_ms() ) self._subscriptions._emit('leave', synthesized_leave) diff --git a/ably/pubsub/types/presence.py b/ably/pubsub/types/presence.py index 66cbdea7..0043f459 100644 --- a/ably/pubsub/types/presence.py +++ b/ably/pubsub/types/presence.py @@ -1,4 +1,4 @@ -from datetime import datetime, timedelta +from datetime import datetime, timedelta, timezone from urllib import parse from ably.pubsub.request.paginatedresult import PaginatedResult @@ -7,16 +7,20 @@ from ably.pubsub.util.crypto import CipherData from ably.pubsub.util.encoding import encode_data +_EPOCH = datetime(1970, 1, 1, tzinfo=timezone.utc) -def _ms_since_epoch(dt): - epoch = datetime.utcfromtimestamp(0) - delta = dt - epoch - return int(delta.total_seconds() * 1000) +def _ms_since_epoch(t): + """A history bound as milliseconds since the epoch, as channel history accepts one. -def _dt_from_ms_epoch(ms): - epoch = datetime.utcfromtimestamp(0) - return epoch + timedelta(milliseconds=ms) + An int is already milliseconds. A datetime may be aware or naive, and a naive one is + read as UTC. + """ + if isinstance(t, int): + return t + if t.tzinfo is None: + t = t.replace(tzinfo=timezone.utc) + return (t - _EPOCH) // timedelta(milliseconds=1) class PresenceAction: @@ -164,7 +168,7 @@ def to_encoded(self, binary=False): if self.extras: result['extras'] = self.extras if self.timestamp: - result['timestamp'] = _ms_since_epoch(self.timestamp) + result['timestamp'] = self.timestamp return result @@ -180,9 +184,6 @@ def from_encoded(obj, cipher=None, context=None): # member_key = obj.get('memberKey', None) extras = obj.get('extras', None) - if timestamp is not None: - timestamp = _dt_from_ms_epoch(timestamp) - decoded_data = PresenceMessage.decode(data, encoding, cipher) return PresenceMessage( @@ -237,15 +238,9 @@ async def history(self, limit=None, direction=None, start=None, end=None): if direction: qs['direction'] = direction if start: - if isinstance(start, int): - qs['start'] = start - else: - qs['start'] = _ms_since_epoch(start) + qs['start'] = _ms_since_epoch(start) if end: - if isinstance(end, int): - qs['end'] = end - else: - qs['end'] = _ms_since_epoch(end) + qs['end'] = _ms_since_epoch(end) if 'start' in qs and 'end' in qs and qs['start'] > qs['end']: raise ValueError("'end' parameter has to be greater than or equal to 'start'") diff --git a/test/ably/http/httppresence_test.py b/test/ably/http/httppresence_test.py index 1fa52d42..187c3ab4 100644 --- a/test/ably/http/httppresence_test.py +++ b/test/ably/http/httppresence_test.py @@ -60,10 +60,10 @@ async def test_presence_get_encoded(self): assert presence_history.items[-4].data == '{ "test": "This is a JSONObject clientData payload"}' assert presence_history.items[-5].data == {"example": {"json": "Object"}} - async def test_timestamp_is_datetime(self): + async def test_timestamp_is_ms_since_epoch(self): presence_page = await self.channel.presence.get() member = presence_page.items[0] - assert isinstance(member.timestamp, datetime) + assert isinstance(member.timestamp, int) async def test_presence_message_has_correct_member_key(self): presence_page = await self.channel.presence.get() diff --git a/test/ably/realtime/presencemap_test.py b/test/ably/realtime/presencemap_test.py index de6002bf..53600295 100644 --- a/test/ably/realtime/presencemap_test.py +++ b/test/ably/realtime/presencemap_test.py @@ -4,7 +4,6 @@ Tests RTP2 specification requirements for presence map operations. """ -from datetime import datetime import pytest @@ -117,7 +116,7 @@ def test_to_encoded(self): client_id='client1', action=PresenceAction.ENTER, data='test data', - timestamp=datetime(2024, 1, 1, 12, 0, 0) + timestamp=1704110400000 ) encoded = msg.to_encoded() assert encoded['action'] == PresenceAction.ENTER @@ -125,7 +124,7 @@ def test_to_encoded(self): assert encoded['connectionId'] == 'connection123' assert encoded['clientId'] == 'client1' assert encoded['data'] == 'test data' - assert 'timestamp' in encoded + assert encoded['timestamp'] == 1704110400000 def test_to_encoded_with_dict_data(self): """Test converting message with dict data (should be JSON serialized).""" @@ -246,21 +245,21 @@ def test_synthesized_message_newer_by_timestamp(self): connection_id='connection123', client_id='client1', action=PresenceAction.LEAVE, - timestamp=datetime(2024, 1, 1, 12, 0, 0) + timestamp=1704110400000 ) newer = PresenceMessage( id='connection123:5:0', connection_id='connection123', client_id='client1', action=PresenceAction.PRESENT, - timestamp=datetime(2024, 1, 1, 12, 0, 1) + timestamp=1704110401000 ) assert _is_newer(newer, older) assert not _is_newer(older, newer) def test_synthesized_equal_timestamp_incoming_wins(self): """Test RTP2b1a: equal timestamps, incoming is newer.""" - timestamp = datetime(2024, 1, 1, 12, 0, 0) + timestamp = 1704110400000 existing = PresenceMessage( id='different:0:0', connection_id='connection123', @@ -285,14 +284,14 @@ def test_normal_message_newer_by_msg_serial(self): connection_id='connection123', client_id='client1', action=PresenceAction.PRESENT, - timestamp=datetime(2024, 1, 1, 12, 0, 0) + timestamp=1704110400000 ) newer = PresenceMessage( id='connection123:10:0', connection_id='connection123', client_id='client1', action=PresenceAction.PRESENT, - timestamp=datetime(2024, 1, 1, 11, 0, 0) # Earlier timestamp doesn't matter + timestamp=1704106800000 # Earlier timestamp doesn't matter ) assert _is_newer(newer, older) assert not _is_newer(older, newer) diff --git a/test/uts/deviations.md b/test/uts/deviations.md index b42aa6b0..c5143bbd 100644 --- a/test/uts/deviations.md +++ b/test/uts/deviations.md @@ -52,13 +52,13 @@ specification: 130 under `helpers/`, for the mocks, and 18 under `objects/helper the objects builders and the synced-channel mock. The 142 gated Test IDs that record SDK non-compliance — 142 tests, 144 cases — reduce to -**60 distinct root causes**, 20 on the REST side and 40 on the realtime side. Two further -defects are recorded below with no test of their own, because the specification's test -cannot discriminate (RTP18a) or has nothing to assert against (the timezone split on -synthesized LEAVE timestamps), so the file carries **62 SDK root causes** in all. The remaining 11 gated Test IDs are -specification faults, and reduce to 8. The objects tier records no SDK non-compliance; -its specification faults are all corrected or worked around in tests that pass, and -reduce to the seven candidate issues O.1–O.7, none of them filed. +**60 distinct root causes**, 20 on the REST side and 40 on the realtime side. One further +defect is recorded below with no test of its own, because the specification's test +cannot discriminate (RTP18a), so the file carries **61 SDK root causes** in all. The +remaining 11 gated Test IDs are specification faults, and reduce to 8. The objects tier +records no SDK non-compliance; its specification faults are all corrected or worked +around in tests that pass, and reduce to the seven candidate issues O.1–O.7, none of +them filed. Entries closed by a fix are removed rather than kept as history; `git log` holds that. @@ -1677,26 +1677,6 @@ correcting the test does not read as a new discovery. **Status:** open bug, untested. -#### A synthesized LEAVE's timestamp is timezone-aware; every other one is naive — no test - -**Spec points:** RTP19, RTP19a, against TP3g. - -`_synthesize_leaves` and `set_presence` build the LEAVE with `datetime.now(timezone.utc)` -(`presence.py:586`, `:720`), while every wire-derived presence message gets a **naive** -`datetime` from `_dt_from_ms_epoch`, which is built on `datetime.utcfromtimestamp(0)` -(`types/presence.py:12-19`, `:182-184`). Comparing the two raises -`TypeError: can't compare offset-naive and offset-aware datetimes`, verified directly. - -Nothing inside the library compares them, because synthesized leaves are emitted rather than -stored, so this bites only application code — but any subscriber that sorts or compares the -timestamps of the messages it receives will hit it. Related to the deprecation work in -PR #656. - -**Tests affected:** none. `test_rtp19_synth_leave_null_id_timestamp` brackets the LEAVE with -two aware `datetime.now(timezone.utc)` readings and passes. - -**Status:** open bug, untested. - ### Auth #### A failed RTN22 reauth leaves no trace on the connection — 1 test @@ -2599,7 +2579,7 @@ is missing accessors over correct behaviour. Within a tier the order is blast ra |---|---|---| | [#706](https://github.com/ably/ably-python/issues/706) | the fabricated `"None:0"` message id | 3.3 | | [#658](https://github.com/ably/ably-python/issues/658) | presence messages sent on reconnection before reattach | adjacent to 3.13, which is the other half of RTP17 automatic re-entry | -| [#656](https://github.com/ably/ably-python/issues/656) | `utcfromtimestamp` deprecation | adjacent to 2.4 | +| [#656](https://github.com/ably/ably-python/issues/656) | `utcfromtimestamp` deprecation | none: `PresenceMessage.timestamp` is now milliseconds since the epoch, the synthesized LEAVE's included, and no `utcfromtimestamp` remains | | [#709](https://github.com/ably/ably-python/issues/709)–[#712](https://github.com/ably/ably-python/issues/712) | REST request timeout, token-request nonce reuse, single-host retry, `dispose()` teardown | adjacent to I.3, which is a second defect on the line #709 is about and is not covered by it; otherwise none, these having been filed from the REST derivation | Every reproduction below is prefixed by: @@ -2705,14 +2685,6 @@ hasattr(client.channels, 'foo') # True len(client.channels) # 1 ``` -**2.4 Synthesized LEAVE timestamps are timezone-aware; every other one is naive.** RTP19, -RTP19a, TP3g. `presence.py:586` and `:720` use `datetime.now(timezone.utc)`, while every -wire-derived `PresenceMessage.timestamp` comes from `_dt_from_ms_epoch`, built on -`datetime.utcfromtimestamp(0)` (`types/presence.py:12-19`). Comparing them raises -`TypeError: can't compare offset-naive and offset-aware datetimes`. Nothing inside the -library compares them, so it bites only application code — but any subscriber that sorts the -timestamps of the messages it receives will hit it. Related to #656. No gated test. - **2.5 `Channels._on_channel_message` subscripts before it checks.** `channel.py:1038-1045` does `channel = self.__all[channel_name]` and only then `if not channel:`, so a message for an unknown channel raises `KeyError` inside the protocol task — swallowed and logged — and diff --git a/test/uts/helpers/presence.py b/test/uts/helpers/presence.py index cec057a9..5218e902 100644 --- a/test/uts/helpers/presence.py +++ b/test/uts/helpers/presence.py @@ -12,6 +12,7 @@ from ably.pubsub.realtime.presencemap import PresenceMap from ably.pubsub.transport.websockettransport import ProtocolMessageAction from ably.pubsub.types.presence import PresenceAction, PresenceMessage +from ably.pubsub.types.testoptions import TestOptions #: The lowercase action names `RealtimePresence` emits its events under, which come #: from `PresenceAction._action_name` and exist only once `presence.py` is imported. @@ -35,16 +36,18 @@ def presence_message(action, client_id, connection_id, id, timestamp, data=None) ) -def subscribed_presence(connection_id='conn-1', name='presence-test'): +def subscribed_presence(connection_id='conn-1', name='presence-test', clock=None): """A `RealtimePresence` over a stub channel, with every presence event recorded. Returns the presence object and the list of `(event_name, message)` pairs its subscribers receive. One listener is registered per event name because - `EventEmitter` keys its wrappers on the listener alone. + `EventEmitter` keys its wrappers on the listener alone. `clock` is the time the + presence object reads when it synthesizes a LEAVE. """ channel = SimpleNamespace( name=name, ably=SimpleNamespace( + options=SimpleNamespace(_test_options=TestOptions(clock=clock)), connection=SimpleNamespace( connection_manager=SimpleNamespace(connection_id=connection_id), ), diff --git a/test/uts/realtime/unit/presence/presence_sync_test.py b/test/uts/realtime/unit/presence/presence_sync_test.py index 0d228862..3f75205d 100644 --- a/test/uts/realtime/unit/presence/presence_sync_test.py +++ b/test/uts/realtime/unit/presence/presence_sync_test.py @@ -10,9 +10,8 @@ messages. See test/uts/deviations.md. """ -from datetime import datetime, timezone - from ably.pubsub.types.presence import PresenceAction +from test.uts.helpers.clock import FakeClock from test.uts.helpers.deviations import deviation from test.uts.helpers.presence import presence_map, presence_message, subscribed_presence @@ -84,19 +83,22 @@ def test_rtp19_stale_members_leave_after_sync(): # UTS: realtime/unit/RTP19/synth-leave-null-id-timestamp-1 def test_rtp19_synth_leave_null_id_timestamp(): - presence, events = subscribed_presence() + # A fake clock holds still between the two readings, so the bounds pin the value the + # LEAVE must take. + clock = FakeClock() + presence, events = subscribed_presence(clock=clock) presence.set_presence([ presence_message(PresenceAction.ENTER, 'bob', 'c2', 'c2:0:0', 100, data='bob-data'), ], is_sync=False) - before_time = datetime.now(timezone.utc) + before_time = clock.now_ms() events.clear() # A sync carrying no members for bob presence.set_presence([], is_sync=True) - after_time = datetime.now(timezone.utc) + after_time = clock.now_ms() leave_events = leaves(events) assert len(leave_events) == 1 diff --git a/test/uts/rest/unit/presence/rest_presence_test.py b/test/uts/rest/unit/presence/rest_presence_test.py index 584949c8..fa97703c 100644 --- a/test/uts/rest/unit/presence/rest_presence_test.py +++ b/test/uts/rest/unit/presence/rest_presence_test.py @@ -6,7 +6,7 @@ import base64 import uuid -from datetime import datetime +from datetime import datetime, timezone import msgpack import pytest @@ -126,9 +126,7 @@ async def test_rsp3b_get_returns_presence_messages(): assert message.client_id == 'user123' assert message.connection_id == 'conn456' assert message.data == 'status data' - # NOTE: the spec asserts the raw millisecond value. PresenceMessage.timestamp is a - # datetime, which is the "Date/Time object where appropriate to the language" form. - assert message.timestamp == datetime.utcfromtimestamp(1234567890000 / 1000) + assert message.timestamp == 1234567890000 # UTS: rest/unit/RSP3c/get-empty-members-0 @@ -319,8 +317,7 @@ async def test_rsp4b1_history_datetime_objects(): captured_requests = [] mock_http = capturing_mock(captured_requests, []) client = rest_client(mock_http) - # ably-python treats a naive datetime as UTC when converting to epoch milliseconds - start_datetime = datetime(2021, 1, 1, 0, 0, 0) + start_datetime = datetime(2021, 1, 1, 0, 0, 0, tzinfo=timezone.utc) await client.channels.get(channel_name).presence.history(start=start_datetime) diff --git a/test/uts/rest/unit/types/presence_message_types_test.py b/test/uts/rest/unit/types/presence_message_types_test.py index 7c6b2b7c..34375ec7 100644 --- a/test/uts/rest/unit/types/presence_message_types_test.py +++ b/test/uts/rest/unit/types/presence_message_types_test.py @@ -3,17 +3,10 @@ Spec points: TP1, TP2, TP3, TP3a, TP3b, TP3c, TP3d, TP3e, TP3f, TP3g, TP3h, TP3i, TP4, TP5 """ -from datetime import datetime, timedelta - from ably.pubsub.types.presence import PresenceAction, PresenceMessage from test.uts.helpers.deviations import deviation, spec_error -def datetime_from_ms(ms): - """The datetime ably-python decodes a wire timestamp into.""" - return datetime.utcfromtimestamp(0) + timedelta(milliseconds=ms) - - def decode_protocol_message_presence(protocol_message): """The `presence` array of a ProtocolMessage, decoded the way the client decodes one. @@ -158,8 +151,7 @@ def test_tp3_presence_from_json(): assert msg.client_id == 'user-1' assert msg.connection_id == 'conn-1' assert msg.data == 'hello' - # NOTE: features.md types TP3g as `Time`, so a datetime is the idiomatic rendering. - assert msg.timestamp == datetime_from_ms(1234567890000) + assert msg.timestamp == 1234567890000 assert msg.extras['headers']['x-key'] == 'x-value' From d8c4fff714cde215de5710b55f785709125b71fd Mon Sep 17 00:00:00 2001 From: owenpearson Date: Mon, 26 Oct 2026 20:43:23 +0000 Subject: [PATCH 51/71] pubsub: decode the TP3 ProtocolMessage tests the way the channel does The TP3a, TP3d and TP3g UTS tests were gated as unimplemented, on the reading that the realtime channel decodes a presence array with no context from its ProtocolMessage. It does not: RealtimeChannel._on_message calls Message.update_inner_message_fields first, which fills each presence entry's missing id, connectionId and timestamp from the ProtocolMessage. The tests' decoding helper skipped that call. The helper now makes it, so the three tests pass and are un-gated. TP3g's asserts the raw millisecond value, which it can only do now that PresenceMessage.timestamp is milliseconds. No SDK code changes, and nothing breaks. deviations.md drops the row, records the refuted claim under Investigated and not defects, and recounts. Co-Authored-By: Claude Opus 5.5 --- test/uts/deviations.md | 26 ++++++++++++------- .../unit/types/presence_message_types_test.py | 11 ++++---- 2 files changed, 23 insertions(+), 14 deletions(-) diff --git a/test/uts/deviations.md b/test/uts/deviations.md index c5143bbd..006052d9 100644 --- a/test/uts/deviations.md +++ b/test/uts/deviations.md @@ -33,8 +33,8 @@ parametrized over a table of fixtures the specification gives inline; and five `objects/unit` tests are parametrized, three over a table and two over the scenarios the specification loops through. That turns 1481 derived tests into 1608 pytest cases. -Of **1472 Test IDs, derived as 1481 tests and run as 1608 pytest cases**: 1292 Test IDs -(1301 tests, 1426 cases) pass, 153 (153 tests, 155 cases) are gated behind +Of **1472 Test IDs, derived as 1481 tests and run as 1608 pytest cases**: 1295 Test IDs +(1304 tests, 1429 cases) pass, 150 (150 tests, 152 cases) are gated behind `RUN_DEVIATIONS`, and 27 (27 tests, 27 cases) cannot be run at all. The three groups are disjoint: two Test IDs have a gated part and a passing part — `TI1`, as two functions, and `TO/endpoint-affects-host`, as one parametrized test — and are counted with the gated, @@ -45,16 +45,16 @@ the Test IDs come from `uts/rest/unit` (503 tests, 536 cases), 482 from `uts/rest/integration` (84, 122), 73 from `uts/realtime/integration` (73, 95) and 17 from `uts/objects/integration` (17, 29); 8 of the REST integration ids (8, 8), 30 of the realtime ones (30, 30) and 5 of the objects ones (5, 5) come from the `proxy` package -within each. Of the gated Test IDs 69 are REST and 84 realtime, which is 70 REST cases +within each. Of the gated Test IDs 66 are REST and 84 realtime, which is 67 REST cases and 85 realtime. None of the 339 objects Test IDs is gated or unrunnable. A further 148 pytest cases cover the harness itself and are not derived from a specification: 130 under `helpers/`, for the mocks, and 18 under `objects/helpers/`, for the objects builders and the synced-channel mock. -The 142 gated Test IDs that record SDK non-compliance — 142 tests, 144 cases — reduce to -**60 distinct root causes**, 20 on the REST side and 40 on the realtime side. One further +The 139 gated Test IDs that record SDK non-compliance — 139 tests, 141 cases — reduce to +**59 distinct root causes**, 19 on the REST side and 40 on the realtime side. One further defect is recorded below with no test of its own, because the specification's test -cannot discriminate (RTP18a), so the file carries **61 SDK root causes** in all. The +cannot discriminate (RTP18a), so the file carries **60 SDK root causes** in all. The remaining 11 gated Test IDs are specification faults, and reduce to 8. The objects tier records no SDK non-compliance; its specification faults are all corrected or worked around in tests that pass, and reduce to the seven candidate issues O.1–O.7, none of @@ -1053,7 +1053,6 @@ option is gone, after RTN16d has done its first phase against the sandbox throug | RTL25, RTL25a, RTL25b | `RealtimeChannel#whenState`. `Connection._when_state` exists (private, awaitable), so this is a gap on the channel rather than a house style; the tests are written against a `channel.when_state(state)` matching the shape the connection already has | 4 | | RSC2, RSC3, RSC4, TO3b, TO3c, TO3c2 | `log_level` and `log_handler` as client options. The library logs through the standard `logging` module under the `ably.pubsub` logger, whose level and handlers are the idiomatic way to do both; `log_level`, which was stored on `Options` and read by nothing, was removed in 4.0.0, so passing either raises `TypeError` | 4 | | RSP3a2, RSP3a3 | `clientId` and `connectionId` filters on `RestPresence#get`. `Presence.get` is `get(self, limit=None)` (`ably/types/presence.py:216`), while `Presence.history` does take its documented params. `presence.get(client_id=...)` raises `TypeError: get() got an unexpected keyword argument 'client_id'`. The absence also forces the RSP5 decoding adaptation below | 4 (5 cases) | -| TP3a, TP3d, TP3g | Presence attributes defaulted from the encapsulating ProtocolMessage. There is no ProtocolMessage type; `ably/realtime/channel.py:751-761` passes the presence array through without context. Matters for synthesized-leave detection and `memberKey` | 3 | | TB4, RTL7h, RTP6e | `attachOnSubscribe`. `ChannelOptions.__init__` (`channeloptions.py:22-26`) takes only `cipher`, `params` and `modes`, and `subscribe()` on the channel, on presence and on annotations all end unconditionally with `await attach()`. This absence also forces the largest single adaptation in the suite, below | 3 | | RSL7 | `RestChannel#setOptions`. The realtime channel implements it; the REST `options` setter expects the kwargs dict `Channels.get` collected, so a `ChannelOptions` raises `TypeError` | 2 | | RTL7f | `echoMessages` in its client-filter form. The `echo_messages` option sends `echo=false` (RTC1a), which the server honours, but the client does not itself drop a message carrying its own `connectionId`, and that is the only form a mock-backed test can observe | 1 | @@ -2392,6 +2391,15 @@ method, or at the calling level". ably-python uses the calling level (`presencemap.py:140-142`), which is what RTP19 needs. The three RTP19 cases the specification distinguishes all come out right. +### A presence message does take its missing fields from the ProtocolMessage + +TP3a, TP3d and TP3g were gated as unimplemented, on the reading that the realtime channel +hands the `presence` array to `from_encoded_array` with no context. It does not: +`RealtimeChannel._on_message` first calls `Message.update_inner_message_fields`, which +fills each presence entry's missing `id`, `connectionId` and `timestamp` from the +ProtocolMessage, and `test/unit/message_test.py` covers it. The derived tests' decoding +helper skipped that call; it now makes it, and the three tests pass. + ### RTL18 delta recovery matches the specification exactly Verified end to end: the recovery ATTACH carries the `channelSerial` of the last batch that @@ -3513,8 +3521,8 @@ The header states how many derived tests there are, how many pass, how many are gated and how many cannot run. Those numbers are the check that the file is still true: in pytest cases, the gated count must equal the number of failures under `RUN_DEVIATIONS=1`, and gated plus unrunnable must equal the number of skips without -it. As of this writing that is 155 failures, 1574 passes and 27 skips with the variable -set, and 182 skips and 1574 passes without it, the 1574 being 1426 derived cases and 148 +it. As of this writing that is 152 failures, 1577 passes and 27 skips with the variable +set, and 179 skips and 1577 passes without it, the 1577 being 1429 derived cases and 148 harness ones — 130 under `helpers/` and 18 under `objects/helpers/`. Those figures need the `submodules` checkout CI makes: without it, three `rest/unit/encoding` tests that read the ably-common fixtures skip as well. The tiers are told apart by the junit report's diff --git a/test/uts/rest/unit/types/presence_message_types_test.py b/test/uts/rest/unit/types/presence_message_types_test.py index 34375ec7..12cd7277 100644 --- a/test/uts/rest/unit/types/presence_message_types_test.py +++ b/test/uts/rest/unit/types/presence_message_types_test.py @@ -3,6 +3,7 @@ Spec points: TP1, TP2, TP3, TP3a, TP3b, TP3c, TP3d, TP3e, TP3f, TP3g, TP3h, TP3i, TP4, TP5 """ +from ably.pubsub.types.message import Message from ably.pubsub.types.presence import PresenceAction, PresenceMessage from test.uts.helpers.deviations import deviation, spec_error @@ -10,9 +11,12 @@ def decode_protocol_message_presence(protocol_message): """The `presence` array of a ProtocolMessage, decoded the way the client decodes one. - ably-python has no ProtocolMessage type; `RealtimeChannel._on_message` hands the raw - `presence` array of an incoming PRESENCE or SYNC message to `from_encoded_array`. + ably-python has no ProtocolMessage type; `RealtimeChannel._on_message` fills each + entry's missing `id`, `connectionId` and `timestamp` from the raw ProtocolMessage + through `Message.update_inner_message_fields`, then hands the `presence` array of an + incoming PRESENCE or SYNC message to `from_encoded_array`. """ + Message.update_inner_message_fields(protocol_message) return PresenceMessage.from_encoded_array(protocol_message['presence']) @@ -77,7 +81,6 @@ def test_tp3h_member_key_combines_ids(): # UTS: rest/unit/TP3d/connectionid-from-protocol-message-0 -@deviation def test_tp3d_connectionid_from_protocol_message(): protocol_msg = { 'action': 14, # PRESENCE @@ -94,7 +97,6 @@ def test_tp3d_connectionid_from_protocol_message(): # UTS: rest/unit/TP3a/id-from-protocol-message-1 -@deviation def test_tp3a_id_from_protocol_message(): protocol_msg = { 'action': 14, # PRESENCE @@ -113,7 +115,6 @@ def test_tp3a_id_from_protocol_message(): # UTS: rest/unit/TP3g/timestamp-from-protocol-message-0 -@deviation def test_tp3g_timestamp_from_protocol_message(): protocol_msg = { 'action': 14, # PRESENCE From 14eaa71fcffd64cd848ac84867e57ec2bce9bb7a Mon Sep 17 00:00:00 2001 From: owenpearson Date: Mon, 26 Oct 2026 20:48:03 +0000 Subject: [PATCH 52/71] pubsub: make Stats.interval_time a timezone-aware UTC datetime Stats.interval_time was parsed from the intervalId with strptime and left naive, although an intervalId is always a UTC time. TS12p asks for a language-idiomatic Time, and for Python that is an aware datetime: a naive one says nothing about its zone and could not be compared with the aware datetimes the rest of an application holds. interval_time is now the same instant with tzinfo set to UTC. This is a breaking change: an ordering comparison against a naive datetime now raises TypeError, and == against one quietly returns False. Callers must compare it with aware datetimes, for example datetime(..., tzinfo= timezone.utc). With PresenceMessage.timestamp no longer decoded through it, nothing under ably/ calls datetime.utcfromtimestamp. Co-Authored-By: Claude Opus 5.5 --- UPDATING.md | 18 ++++++++++++++++++ ably/pubsub/types/stats.py | 5 +++-- test/ably/http/httpstats_test.py | 4 ++-- test/unit/stats_test.py | 23 +++++++++++++++++++++++ 4 files changed, 46 insertions(+), 4 deletions(-) create mode 100644 test/unit/stats_test.py diff --git a/UPDATING.md b/UPDATING.md index 19b0245d..b2d8aa9a 100644 --- a/UPDATING.md +++ b/UPDATING.md @@ -574,6 +574,24 @@ from datetime import datetime, timezone received_at = datetime.fromtimestamp(member.timestamp / 1000, tz=timezone.utc) ``` +### `Stats.interval_time` is timezone-aware + +`Stats.interval_time` is now an aware `datetime` in UTC, where 3.x returned a naive one. An +ordering comparison such as `<` against a naive `datetime` now raises `TypeError`, and `==` +against one quietly returns `False`. Compare it with aware datetimes: + +Example 3.x code: +```python +if stat.interval_time >= datetime(2024, 1, 1): + ... +``` + +Example 4.0.0 code: +```python +if stat.interval_time >= datetime(2024, 1, 1, tzinfo=timezone.utc): + ... +``` + ## Version 2.x to 3.0.0 The 3.0.0 version of ably-python introduces several breaking changes to improve the realtime experience and align the API with the Ably specification. These include: diff --git a/ably/pubsub/types/stats.py b/ably/pubsub/types/stats.py index ead5e548..ea88f993 100644 --- a/ably/pubsub/types/stats.py +++ b/ably/pubsub/types/stats.py @@ -1,5 +1,5 @@ import logging -from datetime import datetime +from datetime import datetime, timezone log = logging.getLogger(__name__) @@ -64,4 +64,5 @@ def granularity_from_interval_id(interval_id): def interval_from_interval_id(interval_id): granularity = granularity_from_interval_id(interval_id) - return datetime.strptime(interval_id, INTERVALS_FMT[granularity]) + # TS12p: the intervalId is a UTC time + return datetime.strptime(interval_id, INTERVALS_FMT[granularity]).replace(tzinfo=timezone.utc) diff --git a/test/ably/http/httpstats_test.py b/test/ably/http/httpstats_test.py index 5f039058..dedb8149 100644 --- a/test/ably/http/httpstats_test.py +++ b/test/ably/http/httpstats_test.py @@ -1,5 +1,5 @@ import logging -from datetime import datetime, timedelta +from datetime import datetime, timedelta, timezone import pytest @@ -304,4 +304,4 @@ async def test_interval(self): stat = stats[0] assert stat.unit == 'minute' assert stat.interval_id == self.last_interval.strftime('%Y-%m-%d:%H:%M') - assert stat.interval_time == self.last_interval + assert stat.interval_time == self.last_interval.replace(tzinfo=timezone.utc) diff --git a/test/unit/stats_test.py b/test/unit/stats_test.py new file mode 100644 index 00000000..b2e85d1b --- /dev/null +++ b/test/unit/stats_test.py @@ -0,0 +1,23 @@ +"""Unit tests for the Stats type. + +Tests cover: +- TS12p: intervalTime parsed from the intervalId as a UTC time +""" + +from datetime import datetime, timezone + +import pytest + +from ably.pubsub.types.stats import Stats + + +@pytest.mark.parametrize('interval_id, expected', [ + ('2024-01-02:03:04', datetime(2024, 1, 2, 3, 4, tzinfo=timezone.utc)), + ('2024-01-02:03', datetime(2024, 1, 2, 3, tzinfo=timezone.utc)), + ('2024-01-02', datetime(2024, 1, 2, tzinfo=timezone.utc)), + ('2024-01', datetime(2024, 1, 1, tzinfo=timezone.utc)), +]) +def test_interval_time_is_utc(interval_id, expected): + stats = Stats.from_dict({'intervalId': interval_id}) + assert stats.interval_time == expected + assert stats.interval_time.tzinfo is timezone.utc From f0b1f8f22f3ef3255fdc7ee0064eade08dbb22dd Mon Sep 17 00:00:00 2001 From: owenpearson Date: Mon, 26 Oct 2026 20:55:46 +0000 Subject: [PATCH 53/71] pubsub: make HttpPaginatedResponse.error_code an int HttpPaginatedResponse.error_code returned the X-Ably-Errorcode header as the server sent it, a string such as '40101', where HP6 makes errorCode an integer, and every other error code the library exposes, on AblyException and ErrorInfo, is an int. error_code is now the header parsed as an int, and still None when the header is absent. This is a breaking change, and a silent one: a caller comparing error_code with a string, as in == '40101', now gets False. Callers must compare it with an int. The UTS tests drop their string assertions for the specification's integer ones, which closes the HP6 deviation. Co-Authored-By: Claude Opus 5.5 --- UPDATING.md | 19 +++++++++++++++++++ ably/pubsub/request/paginatedresult.py | 3 ++- test/ably/http/httprequest_test.py | 2 +- test/uts/deviations.md | 1 - test/uts/rest/integration/auth_test.py | 3 +-- test/uts/rest/unit/request_test.py | 7 ++----- 6 files changed, 25 insertions(+), 10 deletions(-) diff --git a/UPDATING.md b/UPDATING.md index b2d8aa9a..54e182c0 100644 --- a/UPDATING.md +++ b/UPDATING.md @@ -592,6 +592,25 @@ if stat.interval_time >= datetime(2024, 1, 1, tzinfo=timezone.utc): ... ``` +### `HttpPaginatedResponse.error_code` is an `int` + +**This change is silent: comparing `error_code` with a string, as in `== '40101'`, now +returns `False` rather than raising.** The `error_code` of the response that `request()` +returns is now the `int` parsed from the `X-Ably-Errorcode` header, where 3.x returned the +header's string. It is still `None` when the header is absent. Compare it with an `int`: + +Example 3.x code: +```python +if response.error_code == '40101': + ... +``` + +Example 4.0.0 code: +```python +if response.error_code == 40101: + ... +``` + ## Version 2.x to 3.0.0 The 3.0.0 version of ably-python introduces several breaking changes to improve the realtime experience and align the API with the Ably specification. These include: diff --git a/ably/pubsub/request/paginatedresult.py b/ably/pubsub/request/paginatedresult.py index d3361ee1..3d20e36a 100644 --- a/ably/pubsub/request/paginatedresult.py +++ b/ably/pubsub/request/paginatedresult.py @@ -124,7 +124,8 @@ def success(self): @property def error_code(self): - return self.response.headers.get('X-Ably-Errorcode') + error_code = self.response.headers.get('X-Ably-Errorcode') + return int(error_code) if error_code is not None else None @property def error_message(self): diff --git a/test/ably/http/httprequest_test.py b/test/ably/http/httprequest_test.py index bc378e92..36d146b2 100644 --- a/test/ably/http/httprequest_test.py +++ b/test/ably/http/httprequest_test.py @@ -85,7 +85,7 @@ async def test_error(self): assert isinstance(result, HttpPaginatedResponse) # RSC19d assert result.status_code == 400 # HP4 assert not result.success - assert result.error_code + assert isinstance(result.error_code, int) # HP6 assert result.error_message async def test_headers(self): diff --git a/test/uts/deviations.md b/test/uts/deviations.md index 006052d9..f4122b07 100644 --- a/test/uts/deviations.md +++ b/test/uts/deviations.md @@ -2195,7 +2195,6 @@ pseudocode is not mistaken for non-compliance. | RSC18 | The constructor rejects basic auth over HTTP | Construction succeeds; 40103 is raised from `make_request` when a request needing Basic Auth is attempted, and no request goes out. RSA1/RSC18 say only "any attempt to use" | Compliant; the UTS is stricter than its source | | RSAN1a3 | Code 40003 for a missing `Annotation.type` | 400/40000 | Cosmetic; worth aligning cross-SDK | | RSH1a | Empty `recipient` or `data` rejected with code 40000, the error reaching the caller from the server | `PushAdmin.publish` validates its arguments itself and raises before touching the HTTP layer: an empty or non-dict `recipient` or `data` raises an `AblyException` with 400/40000, the code the server gives, which is also a `ValueError`. So no request is made and there is no server error; the integration tier's `test_rsh1a_push_publish_invalid_recipient` asserts the local error's code | Compliant: RSH1a asks for empty values to be "immediately rejected". The specification's test would need a recipient the SDK will send and the server will reject, an unknown `transportType` say, to exercise the server-side path it describes | -| HP6 | `errorCode` is a number | The raw header string, `'40101'` | Open bug, trivial | | HP8 | `headers` is a map | A list of `(name, value)` pairs, so the lookup the spec describes is impossible without converting, and case-insensitivity is lost | Open bug; changing the return type is breaking | | RSC15a | Six hosts tried | Four: the primary and three fallbacks, because TO3l5's `httpMaxRetryCount`, default 3, caps the fallback hosts tried. The specification's six is the number of responses its setup queues, with a comment counting the primary and all five fallbacks, and no assertion reads it | Compliant; the setup's comment overlooks TO3l5 | | TI | `ErrorInfo` equality by attributes | No `__eq__`, so errors compare by identity. Python exceptions conventionally do, and the requirement appears nowhere in `features.md`. Adding `__eq__` without `__hash__` would make `AblyException` unhashable and break any caller that puts one in a set | Intentional | diff --git a/test/uts/rest/integration/auth_test.py b/test/uts/rest/integration/auth_test.py index 9d7c253d..d83d9511 100644 --- a/test/uts/rest/integration/auth_test.py +++ b/test/uts/rest/integration/auth_test.py @@ -140,8 +140,7 @@ async def test_rsa4_invalid_credentials_rejected(sandbox): result = await channel_status(client, channel_name) assert result.status_code == 401 - # `error_code` is the raw `X-Ably-Errorcode` header, so a string. - assert result.error_code == '40400' + assert result.error_code == 40400 # UTS: rest/integration/RSC10/token-renewal-expired-jwt-0 diff --git a/test/uts/rest/unit/request_test.py b/test/uts/rest/unit/request_test.py index 20132fa7..4cca30d4 100644 --- a/test/uts/rest/unit/request_test.py +++ b/test/uts/rest/unit/request_test.py @@ -313,9 +313,7 @@ async def test_rsc19d_response_error_code_header(): response = await client.request('GET', '/test', version='3') - # DEVIATION: the spec asserts errorCode == 40101, and HP6 types it as a number. - # error_code returns the raw X-Ably-Errorcode header value, so it is a string. - assert response.error_code == '40101' + assert response.error_code == 40101 # UTS: rest/unit/RSC19d/response-error-message-header-3 @@ -470,8 +468,7 @@ async def test_rsc19e_http_error_no_fallback(): # Should return the error response, not retry to fallback assert response.status_code == 400 assert response.success is False - # DEVIATION: the spec asserts errorCode == 40000; error_code returns the raw header - assert response.error_code == '40000' + assert response.error_code == 40000 # Only one request should have been made (no fallback) assert len(mock_http.captured_requests) == 1 From ee1fc93e6b0c63c20c51f0cfa022372beff34fac Mon Sep 17 00:00:00 2001 From: owenpearson Date: Mon, 26 Oct 2026 21:00:23 +0000 Subject: [PATCH 54/71] pubsub: make HttpPaginatedResponse.headers a read-only mapping HttpPaginatedResponse.headers returned a list of (name, value) pairs, so a caller could not look a header up by name without converting the list, and lost the case-insensitivity HTTP header names have. HP8's prose calls headers an array of key value pairs, but the IDL types it Dict, and the UTS looks headers up by name in their original case, as in headers["Content-Type"]. headers is now a ResponseHeaders, a read-only Mapping looked up by name case-insensitively. As before, names are lower-cased and a repeated header's values are joined with ', '. This is a breaking change: iterating the mapping yields only the names, so `for name, value in headers` must become `for name, value in headers.items()`, and code relying on list methods or indexing by position must look headers up by name. The UTS test asserts the specification's lookups directly, which closes the HP8 deviation. Co-Authored-By: Claude Opus 5.5 --- UPDATING.md | 22 ++++++++++++++ ably/pubsub/request/paginatedresult.py | 28 ++++++++++++++++- test/ably/http/httprequest_test.py | 5 ++- test/unit/paginatedresult_test.py | 42 ++++++++++++++++++++++++++ test/uts/deviations.md | 1 - test/uts/rest/unit/request_test.py | 10 +++--- 6 files changed, 99 insertions(+), 9 deletions(-) create mode 100644 test/unit/paginatedresult_test.py diff --git a/UPDATING.md b/UPDATING.md index 54e182c0..43a6b690 100644 --- a/UPDATING.md +++ b/UPDATING.md @@ -611,6 +611,28 @@ if response.error_code == 40101: ... ``` +### `HttpPaginatedResponse.headers` is a read-only mapping + +The `headers` of the response that `request()` returns is now a read-only mapping from header +name to value, looked up case-insensitively, where 3.x returned a list of `(name, value)` +pairs. As before, names are lower-cased and a header sent more than once has its values +joined with `', '`. Iterating over the mapping itself now yields only the names, so look a +header up by name, or iterate over `.items()`: + +Example 3.x code: +```python +for name, value in response.headers: + ... +``` + +Example 4.0.0 code: +```python +request_id = response.headers.get('X-Request-Id') + +for name, value in response.headers.items(): + ... +``` + ## Version 2.x to 3.0.0 The 3.0.0 version of ably-python introduces several breaking changes to improve the realtime experience and align the API with the Ably specification. These include: diff --git a/ably/pubsub/request/paginatedresult.py b/ably/pubsub/request/paginatedresult.py index 3d20e36a..8be1ef50 100644 --- a/ably/pubsub/request/paginatedresult.py +++ b/ably/pubsub/request/paginatedresult.py @@ -1,5 +1,6 @@ import calendar import logging +from collections.abc import Mapping from urllib.parse import urlencode from ably.pubsub.request.http import Request @@ -42,6 +43,31 @@ def format_params(params=None, direction=None, start=None, end=None, limit=None, return '?' + urlencode(params) if params else '' +class ResponseHeaders(Mapping): + """The headers of a response, read-only and looked up by name case-insensitively (HP8). + + Names iterate lower-cased, and a header sent more than once reads as its values joined + with ', '. + """ + + def __init__(self, headers): + self.__headers = {name.lower(): value for name, value in headers.items()} + + def __getitem__(self, name): + if not isinstance(name, str): + raise KeyError(name) + return self.__headers[name.lower()] + + def __iter__(self): + return iter(self.__headers) + + def __len__(self): + return len(self.__headers) + + def __repr__(self): + return f'{type(self).__name__}({self.__headers!r})' + + class PaginatedResult: def __init__(self, http, items, content_type, rel_first, rel_next, response_processor, response): @@ -133,4 +159,4 @@ def error_message(self): @property def headers(self): - return list(self.response.headers.items()) + return ResponseHeaders(self.response.headers) diff --git a/test/ably/http/httprequest_test.py b/test/ably/http/httprequest_test.py index 36d146b2..9de29f59 100644 --- a/test/ably/http/httprequest_test.py +++ b/test/ably/http/httprequest_test.py @@ -1,3 +1,5 @@ +from collections.abc import Mapping + import httpx import pytest import respx @@ -69,7 +71,8 @@ async def test_get(self): assert result.success is True # HP5 assert result.error_code is None # HP6 assert result.error_message is None # HP7 - assert isinstance(result.headers, list) # HP7 + assert isinstance(result.headers, Mapping) # HP8 + assert result.headers['Content-Type'] == result.headers['content-type'] @dont_vary_protocol async def test_not_found(self): diff --git a/test/unit/paginatedresult_test.py b/test/unit/paginatedresult_test.py new file mode 100644 index 00000000..5308c5fb --- /dev/null +++ b/test/unit/paginatedresult_test.py @@ -0,0 +1,42 @@ +"""Unit tests for the headers of an HttpPaginatedResponse. + +Tests cover: +- HP8: headers as a read-only mapping, looked up by name case-insensitively +""" + +import httpx +import pytest + +from ably.pubsub.request.paginatedresult import ResponseHeaders + + +def test_headers_look_up_case_insensitively(): + headers = ResponseHeaders(httpx.Headers({'Content-Type': 'application/json'})) + assert headers['content-type'] == 'application/json' + assert headers['CONTENT-TYPE'] == 'application/json' + assert headers.get('Content-Type') == 'application/json' + assert 'Content-Type' in headers + assert 'X-Missing' not in headers + assert headers.get('X-Missing') is None + assert 1 not in headers + + +def test_headers_iterate_as_lower_cased_names(): + headers = ResponseHeaders(httpx.Headers([('X-One', '1'), ('X-Two', '2')])) + assert list(headers) == ['x-one', 'x-two'] + assert dict(headers.items()) == {'x-one': '1', 'x-two': '2'} + assert len(headers) == 2 + + +def test_repeated_header_joins_its_values(): + headers = ResponseHeaders(httpx.Headers([('Link', ''), ('Link', '')])) + assert headers['Link'] == ', ' + assert len(headers) == 1 + + +def test_headers_are_read_only(): + headers = ResponseHeaders(httpx.Headers({'X-One': '1'})) + with pytest.raises(TypeError): + headers['X-One'] = '2' + with pytest.raises(TypeError): + del headers['X-One'] diff --git a/test/uts/deviations.md b/test/uts/deviations.md index f4122b07..844e5fff 100644 --- a/test/uts/deviations.md +++ b/test/uts/deviations.md @@ -2195,7 +2195,6 @@ pseudocode is not mistaken for non-compliance. | RSC18 | The constructor rejects basic auth over HTTP | Construction succeeds; 40103 is raised from `make_request` when a request needing Basic Auth is attempted, and no request goes out. RSA1/RSC18 say only "any attempt to use" | Compliant; the UTS is stricter than its source | | RSAN1a3 | Code 40003 for a missing `Annotation.type` | 400/40000 | Cosmetic; worth aligning cross-SDK | | RSH1a | Empty `recipient` or `data` rejected with code 40000, the error reaching the caller from the server | `PushAdmin.publish` validates its arguments itself and raises before touching the HTTP layer: an empty or non-dict `recipient` or `data` raises an `AblyException` with 400/40000, the code the server gives, which is also a `ValueError`. So no request is made and there is no server error; the integration tier's `test_rsh1a_push_publish_invalid_recipient` asserts the local error's code | Compliant: RSH1a asks for empty values to be "immediately rejected". The specification's test would need a recipient the SDK will send and the server will reject, an unknown `transportType` say, to exercise the server-side path it describes | -| HP8 | `headers` is a map | A list of `(name, value)` pairs, so the lookup the spec describes is impossible without converting, and case-insensitivity is lost | Open bug; changing the return type is breaking | | RSC15a | Six hosts tried | Four: the primary and three fallbacks, because TO3l5's `httpMaxRetryCount`, default 3, caps the fallback hosts tried. The specification's six is the number of responses its setup queues, with a comment counting the primary and all five fallbacks, and no assertion reads it | Compliant; the setup's comment overlooks TO3l5 | | TI | `ErrorInfo` equality by attributes | No `__eq__`, so errors compare by identity. Python exceptions conventionally do, and the requirement appears nowhere in `features.md`. Adding `__eq__` without `__hash__` would make `AblyException` unhashable and break any caller that puts one in a set | Intentional | | RSA6b, RSA6d | The capability literal as passed | Canonicalised by `Capability.c14n`, which RSA9f requires | Compliant; rendering only | diff --git a/test/uts/rest/unit/request_test.py b/test/uts/rest/unit/request_test.py index 4cca30d4..612dfe8c 100644 --- a/test/uts/rest/unit/request_test.py +++ b/test/uts/rest/unit/request_test.py @@ -349,12 +349,10 @@ async def test_rsc19d_response_headers_accessible(): response = await client.request('GET', '/test', version='3') - # DEVIATION: the spec (and HP8) expects a map keyed by header name. `headers` is a - # list of (name, value) pairs, with the names lower-cased by httpx. - headers = dict(response.headers) - assert headers['content-type'] == 'application/json' - assert headers['x-request-id'] == 'req-123' - assert headers['x-custom-header'] == 'custom-value' + headers = response.headers + assert headers['Content-Type'] == 'application/json' + assert headers['X-Request-Id'] == 'req-123' + assert headers['X-Custom-Header'] == 'custom-value' # UTS: rest/unit/RSC19d/response-items-decoded-5 From 6e344d085e5a83fcb8b19ca8290beb789c0faa11 Mon Sep 17 00:00:00 2001 From: owenpearson Date: Mon, 26 Oct 2026 21:06:17 +0000 Subject: [PATCH 55/71] pubsub: return nothing from push removals and hide the paginated response remove and remove_where on push.admin.device_registrations and push.admin.channel_subscriptions returned the HTTP response of the DELETE, and every PaginatedResult carried the HTTP response it was built from as a public response attribute, so the library's public surface handed out the underlying httpx.Response. RSH1b4, RSH1b5, RSH1c4 and RSH1c5 give these removals no return value, and nothing in TG or HP asks for the response. The four removals now return None, and a failed one still raises AblyException. PaginatedResult no longer keeps the response; HttpPaginatedResponse reads its status code and headers from it once, at construction, so status_code, success, error_code, error_message and headers work as before. This is a breaking change: callers that checked remove()'s status_code must stop, since success is the absence of an exception, and callers that read result.response must read the dedicated HttpPaginatedResponse attributes instead. The UTS tests assert that the removals return None, which closes the push remove return-value note. Co-Authored-By: Claude Opus 5.5 --- UPDATING.md | 21 +++++++++++++ ably/pubsub/http/push.py | 8 ++--- ably/pubsub/request/paginatedresult.py | 16 +++++++--- test/ably/http/httppush_test.py | 33 +++++++------------- test/ably/http/httprequest_test.py | 16 +++++++--- test/uts/deviations.md | 1 - test/uts/rest/integration/push_admin_test.py | 25 +++++---------- 7 files changed, 67 insertions(+), 53 deletions(-) diff --git a/UPDATING.md b/UPDATING.md index 43a6b690..ae848f2f 100644 --- a/UPDATING.md +++ b/UPDATING.md @@ -633,6 +633,27 @@ for name, value in response.headers.items(): ... ``` +### Push removals return nothing, and results no longer expose the HTTP response + +`remove` and `remove_where` on `push.admin.device_registrations` and +`push.admin.channel_subscriptions` now return `None`, where 3.x returned the underlying HTTP +response. A removal that fails raises an `AblyException`, so there is no status code to check: + +Example 3.x code: +```python +response = await rest.push.admin.device_registrations.remove(device_id) +assert response.status_code == 204 +``` + +Example 4.0.0 code: +```python +await pubsub_http_client.push.admin.device_registrations.remove(device_id) +``` + +For the same reason a `PaginatedResult` no longer has a `response` attribute. The +`HttpPaginatedResponse` that `request()` returns keeps `status_code`, `success`, +`error_code`, `error_message` and `headers`, so read those instead of `response`. + ## Version 2.x to 3.0.0 The 3.0.0 version of ably-python introduces several breaking changes to improve the realtime experience and align the API with the Ably specification. These include: diff --git a/ably/pubsub/http/push.py b/ably/pubsub/http/push.py index 1fc6ea98..d349d72b 100644 --- a/ably/pubsub/http/push.py +++ b/ably/pubsub/http/push.py @@ -115,7 +115,7 @@ async def remove(self, device_id: str): - `device_id`: the id of the device """ path = f'/push/deviceRegistrations/{device_id}' - return await self.ably.http.delete(path) + await self.ably.http.delete(path) async def remove_where(self, **params): """Deletes the registered devices identified by the given parameters. @@ -124,7 +124,7 @@ async def remove_where(self, **params): - `**params`: the parameters that identify the devices to remove """ path = '/push/deviceRegistrations' + format_params(params) - return await self.ably.http.delete(path) + await self.ably.http.delete(path) class PushChannelSubscriptions: @@ -180,7 +180,7 @@ async def remove(self, subscription: dict): """ subscription = PushChannelSubscription.factory(subscription) params = subscription.as_dict() - return await self.remove_where(**params) + await self.remove_where(**params) async def remove_where(self, **params): """Deletes the subscriptions identified by the given parameters. @@ -189,4 +189,4 @@ async def remove_where(self, **params): - `**params`: the parameters that identify the subscriptions to remove """ path = '/push/channelSubscriptions' + format_params(**params) - return await self.ably.http.delete(path) + await self.ably.http.delete(path) diff --git a/ably/pubsub/request/paginatedresult.py b/ably/pubsub/request/paginatedresult.py index 8be1ef50..a94ceeaa 100644 --- a/ably/pubsub/request/paginatedresult.py +++ b/ably/pubsub/request/paginatedresult.py @@ -77,7 +77,6 @@ def __init__(self, http, items, content_type, rel_first, rel_next, self.__rel_first = rel_first self.__rel_next = rel_next self.__response_processor = response_processor - self.response = response @property def items(self): @@ -139,9 +138,16 @@ async def paginated_query_with_request(cls, http, request, response_processor, class HttpPaginatedResponse(PaginatedResult): + def __init__(self, http, items, content_type, rel_first, rel_next, + response_processor, response): + super().__init__(http, items, content_type, rel_first, rel_next, + response_processor, response) + self.__status_code = response.status_code + self.__headers = ResponseHeaders(response.headers) + @property def status_code(self): - return self.response.status_code + return self.__status_code @property def success(self): @@ -150,13 +156,13 @@ def success(self): @property def error_code(self): - error_code = self.response.headers.get('X-Ably-Errorcode') + error_code = self.__headers.get('X-Ably-Errorcode') return int(error_code) if error_code is not None else None @property def error_message(self): - return self.response.headers.get('X-Ably-Errormessage') + return self.__headers.get('X-Ably-Errormessage') @property def headers(self): - return ResponseHeaders(self.response.headers) + return self.__headers diff --git a/test/ably/http/httppush_test.py b/test/ably/http/httppush_test.py index 1e17ab18..2a82300f 100644 --- a/test/ably/http/httppush_test.py +++ b/test/ably/http/httppush_test.py @@ -226,14 +226,12 @@ async def test_admin_device_registrations_remove(self): # Remove get_response = await get(device.id) assert get_response.id == device.id # Exists - remove_device_response = await self.remove_device(device.id) - assert remove_device_response.status_code == 204 + assert await self.remove_device(device.id) is None with pytest.raises(AblyException): # Doesn't exist await get(device.id) # Remove again, it doesn't fail - remove_device_response = await self.remove_device(device.id) - assert remove_device_response.status_code == 204 + assert await self.remove_device(device.id) is None # RSH1b5 async def test_admin_device_registrations_remove_where(self): @@ -243,8 +241,7 @@ async def test_admin_device_registrations_remove_where(self): device = self.get_device() foo_device = await get(device.id) assert foo_device.id == device.id # Exists - remove_foo_device_response = await self.remove_device_where(deviceId=device.id) - assert remove_foo_device_response.status_code == 204 + assert await self.remove_device_where(deviceId=device.id) is None with pytest.raises(AblyException): # Doesn't exist await get(device.id) @@ -252,8 +249,7 @@ async def test_admin_device_registrations_remove_where(self): device = self.get_device() boo_device = await get(device.id) assert boo_device.id == device.id # Exists - remove_boo_device_response = await self.remove_device_where(clientId=device.client_id) - assert remove_boo_device_response.status_code == 204 + assert await self.remove_device_where(clientId=device.client_id) is None # Doesn't exist (Deletion is async: wait up to a few seconds before giving up) with pytest.raises(AblyException): for _ in range(5): @@ -261,8 +257,7 @@ async def test_admin_device_registrations_remove_where(self): await get(device.id) # Remove with no matching params - remove_boo_device_response = await self.remove_device_where(clientId=device.client_id) - assert remove_boo_device_response.status_code == 204 + assert await self.remove_device_where(clientId=device.client_id) is None # # RSH1c1 async def test_admin_channel_subscriptions_list(self): @@ -351,8 +346,7 @@ async def test_admin_channel_subscriptions_remove(self): subscription = await save(PushChannelSubscription(channel, device_id=device.id)) list_response = await list_(channel=channel) assert device.id in (x.device_id for x in list_response.items) - remove_response = await remove(subscription) - assert remove_response.status_code == 204 + assert await remove(subscription) is None list_response = await list_(channel=channel) assert device.id not in (x.device_id for x in list_response.items) @@ -361,14 +355,12 @@ async def test_admin_channel_subscriptions_remove(self): subscription = await save(PushChannelSubscription(channel, client_id=client_id)) list_response = await list_(channel=channel) assert client_id in (x.client_id for x in list_response.items) - remove_response = await remove(subscription) - assert remove_response.status_code == 204 + assert await remove(subscription) is None list_response = await list_(channel=channel) assert client_id not in (x.client_id for x in list_response.items) # Remove again, it doesn't fail - remove_response = await remove(subscription) - assert remove_response.status_code == 204 + assert await remove(subscription) is None # RSH1c5 async def test_admin_channel_subscriptions_remove_where(self): @@ -383,8 +375,7 @@ async def test_admin_channel_subscriptions_remove_where(self): await save(PushChannelSubscription(channel, device_id=device.id)) list_response = await list_(channel=channel) assert device.id in (x.device_id for x in list_response.items) - remove_response = await remove(channel=channel, device_id=device.id) - assert remove_response.status_code == 204 + assert await remove(channel=channel, device_id=device.id) is None list_response = await list_(channel=channel) assert device.id not in (x.device_id for x in list_response.items) @@ -393,11 +384,9 @@ async def test_admin_channel_subscriptions_remove_where(self): await save(PushChannelSubscription(channel, client_id=client_id)) list_response = await list_(channel=channel) assert client_id in (x.client_id for x in list_response.items) - remove_response = await remove(channel=channel, client_id=client_id) - assert remove_response.status_code == 204 + assert await remove(channel=channel, client_id=client_id) is None list_response = await list_(channel=channel) assert client_id not in (x.client_id for x in list_response.items) # Remove again, it doesn't fail - remove_response = await remove(channel=channel, client_id=client_id) - assert remove_response.status_code == 204 + assert await remove(channel=channel, client_id=client_id) is None diff --git a/test/ably/http/httprequest_test.py b/test/ably/http/httprequest_test.py index 9de29f59..107ecae7 100644 --- a/test/ably/http/httprequest_test.py +++ b/test/ably/http/httprequest_test.py @@ -91,11 +91,15 @@ async def test_error(self): assert isinstance(result.error_code, int) # HP6 assert result.error_message + @dont_vary_protocol + @respx.mock async def test_headers(self): key = 'X-Test' value = 'lorem ipsum' - result = await self.ably.request('GET', '/time', headers={key: value}, version=Defaults.protocol_version) - assert result.response.request.headers[key] == value + route = respx.get('https://sandbox.realtime.ably-nonprod.net/time') + route.return_value = httpx.Response(200, headers={'Content-Type': 'application/json'}, text='[123]') + await self.ably.request('GET', '/time', headers={key: value}, version=Defaults.protocol_version) + assert route.calls.last.request.headers[key] == value # RSC19e @dont_vary_protocol @@ -233,7 +237,11 @@ async def test_400_cloudfront_fallback(self): assert result.items[0] == 123 await ably.close() + @dont_vary_protocol + @respx.mock async def test_version(self): version = "150" # chosen arbitrarily - result = await self.ably.request('GET', '/time', "150") - assert result.response.request.headers["X-Ably-Version"] == version + route = respx.get('https://sandbox.realtime.ably-nonprod.net/time') + route.return_value = httpx.Response(200, headers={'Content-Type': 'application/json'}, text='[123]') + await self.ably.request('GET', '/time', "150") + assert route.calls.last.request.headers["X-Ably-Version"] == version diff --git a/test/uts/deviations.md b/test/uts/deviations.md index 844e5fff..f6eaebf3 100644 --- a/test/uts/deviations.md +++ b/test/uts/deviations.md @@ -2179,7 +2179,6 @@ pseudocode is not mistaken for non-compliance. | RTP17b's synthesized-LEAVE filter | RTP17b's own implementation note allows the check to live "either inside the presence map's `remove()` method, or at the calling level". ably-python uses the calling level (`presence.py:557-558`). Compliant | | RTP19a's route | the specification models an ATTACHED without HAS_PRESENCE as `startSync()` then `endSync()`. `on_attached(has_presence=False)` calls `_synthesize_leaves(...)` then `clear()` (`presence.py:611-618`), which is the requirement itself rather than the model of it | | `hasNext` as a value | `push_admin.md` RSH1b2 writes `ASSERT result.hasNext == true`. In Python `has_next` is a bound method (`ably/http/paginatedresult.py:63`), truthy whatever the page holds, so the direct translation asserts nothing. `test_rsh1b2_list_devices_pagination` writes `result.has_next() is True`, confirmed to fail when inverted | -| Push `remove` return values | the specification's remove steps assert nothing about a return value ("should not throw"). `PushDeviceRegistrations.remove` / `remove_where` and `PushChannelSubscriptions.remove` / `remove_where` return the `ably.http.http.Response` from the DELETE rather than `None` (`ably/rest/push.py:112-127`, `:176-192`), so the six derived removal tests assert `response.status_code == 204`, which is stronger than the specification asks and matches what `test/ably/rest/restpush_test.py` already asserts | | `rest/proxy/RSC15l/unreachable-endpoint-error-0` | the specification asserts only that the error carries a non-null `status_code` or `code`, leaving the values open, so the derived test asserts exactly that and is the weakest of the eight proxy tests. What this SDK produces against a refused connection is 500 / 80000, the RSC19e wrapping of the `httpx.ConnectError`, which it keeps as the `cause`, and that is recorded here rather than asserted, since pinning it would assert more than the specification does | | `RSL1k4`'s history read | the event log is per session, so any request the client under test makes through it is recorded — including the `history()` the test verifies deduplication with. The log is therefore read **before** the history call, and the POST count assertion made against that snapshot. The history read itself is a `wall_clock_poll_until` rather than one fetch, for the same reason every other integration verification is: a published message does not reach history at once | | RSP4b1's time bounds | the specification records `time_before = now_millis()` before generating the presence events and `time_after = now_millis()` after, then asserts a `history(start=, end=)` over that window returns them. Read from the runner's clock the window is only as good as the skew against the sandbox, which decides the timestamps actually stored, so a runner running a little fast would exclude the very events the test generated. Both bounds come from `await client.time()` instead — the same instant on the clock that stamps the events. `Presence.history` passes an `int` straight through as milliseconds (`ably/types/presence.py:232-241`), which is what `client.time()` returns, so no conversion is involved | diff --git a/test/uts/rest/integration/push_admin_test.py b/test/uts/rest/integration/push_admin_test.py index f997217b..48d54a4e 100644 --- a/test/uts/rest/integration/push_admin_test.py +++ b/test/uts/rest/integration/push_admin_test.py @@ -195,9 +195,7 @@ async def test_rsh1b4_remove_device(sandbox): await registrations.save(apns_device(device_id)) - response = await registrations.remove(device_id) - # `remove` answers with the raw response rather than nothing. - assert response.status_code == 204 + assert await registrations.remove(device_id) is None # Removal is asynchronous, so an immediate `get` can still find the registration. async def removed(): @@ -216,10 +214,8 @@ async def removed(): async def test_rsh1b4_remove_nonexistent_device(sandbox): client = sandbox_rest_client(sandbox.key(0).key_str) - response = await client.push.admin.device_registrations.remove( - f'nonexistent-device-{random_id()}') - - assert response.status_code == 204 + assert await client.push.admin.device_registrations.remove( + f'nonexistent-device-{random_id()}') is None # UTS: rest/integration/RSH1b5/remove-where-clientid-0 @@ -238,8 +234,7 @@ async def test_rsh1b5_remove_where_clientid(sandbox): # A registration under a different clientId, which removeWhere must leave alone. await registrations.save(apns_device(survivor_id, client_id=survivor_client_id)) try: - response = await registrations.remove_where(clientId=client_id) - assert response.status_code == 204 + assert await registrations.remove_where(clientId=client_id) is None # Deletion is asynchronous on the server side. async def removed(): @@ -349,8 +344,7 @@ async def test_rsh1c4_remove_channel_subscription(sandbox): # The filter has to narrow for the assertion below to mean anything. assert len((await subscriptions.list(channel=channel_name, clientId=client_id)).items) == 1 - response = await subscriptions.remove(subscription) - assert response.status_code == 204 + assert await subscriptions.remove(subscription) is None async def removed(): result = await subscriptions.list(channel=channel_name, clientId=client_id) @@ -365,10 +359,8 @@ async def removed(): async def test_rsh1c4_remove_nonexistent_subscription(sandbox): client = sandbox_rest_client(sandbox.key(0).key_str) - response = await client.push.admin.channel_subscriptions.remove(PushChannelSubscription( - f'{PUSH_NAMESPACE}:nonexistent-{random_id()}', client_id='nonexistent-client')) - - assert response.status_code == 204 + assert await client.push.admin.channel_subscriptions.remove(PushChannelSubscription( + f'{PUSH_NAMESPACE}:nonexistent-{random_id()}', client_id='nonexistent-client')) is None # UTS: rest/integration/RSH1c5/remove-where-subscriptions-0 @@ -389,8 +381,7 @@ async def test_rsh1c5_remove_where_subscriptions(sandbox): survivor = PushChannelSubscription(survivor_channel, client_id=survivor_client_id) await subscriptions.save(survivor) try: - response = await subscriptions.remove_where(clientId=client_id) - assert response.status_code == 204 + assert await subscriptions.remove_where(clientId=client_id) is None async def removed(): result = await subscriptions.list(clientId=client_id) From c381f93f644f923f93cd1914dc1344fa849e9f66 Mon Sep 17 00:00:00 2001 From: owenpearson Date: Mon, 26 Oct 2026 21:14:07 +0000 Subject: [PATCH 56/71] pubsub: add DevicePushDetails and ignore unknown push fields DeviceDetails.push was the dict the server sent, where PCD7 types it as a DevicePushDetails with the recipient, state and errorReason of PCP2-PCP4. And DeviceDetails.from_dict and PushChannelSubscription.from_dict passed every key of the decoded object to the constructor with cls(**obj), so a field the server added that the class did not know raised TypeError and failed the whole get, list or save. DeviceDetails.push is now a DevicePushDetails, built from a dict or taken as one, which validates the recipient's transportType as DeviceDetails used to and decodes errorReason into an AblyException. Both from_dict methods, and DevicePushDetails', now ignore fields they do not model. DevicePushDetails lives beside DeviceDetails in ably.pubsub.types.device and is not added to ably.pubsub.server, whose exports are still to be decided. This is a breaking change: device.push['recipient'] now raises TypeError, and callers must read device.push.recipient, .state and .error_reason. A device can still be built or saved with push given as a dict. A misspelled key in a dict passed to save() is now ignored rather than rejected. The UTS tests build devices with DevicePushDetails as the specification does, which closes the DevicePushDetails missing-accessor row. Co-Authored-By: Claude Opus 5.5 --- UPDATING.md | 22 +++++ ably/pubsub/types/channelsubscription.py | 9 +- ably/pubsub/types/device.py | 83 ++++++++++++++++--- test/unit/push_types_test.py | 76 +++++++++++++++++ test/uts/deviations.md | 1 - test/uts/rest/integration/push_admin_test.py | 17 ++-- .../rest/integration/push_channels_test.py | 7 +- .../push/push_device_registrations_test.py | 23 +++-- 8 files changed, 196 insertions(+), 42 deletions(-) create mode 100644 test/unit/push_types_test.py diff --git a/UPDATING.md b/UPDATING.md index ae848f2f..0359db06 100644 --- a/UPDATING.md +++ b/UPDATING.md @@ -654,6 +654,28 @@ For the same reason a `PaginatedResult` no longer has a `response` attribute. Th `HttpPaginatedResponse` that `request()` returns keeps `status_code`, `success`, `error_code`, `error_message` and `headers`, so read those instead of `response`. +### `DeviceDetails.push` is a `DevicePushDetails` + +`DeviceDetails.push` is now a `DevicePushDetails`, with `recipient`, `state` and +`error_reason` attributes, where 3.x returned the dict the server sent. `error_reason` is an +`AblyException`. Indexing it, as in `device.push['recipient']`, now raises `TypeError`. +Read the attributes instead. A device can still be built or saved with `push` given as a +dict: + +Example 3.x code: +```python +token = device.push['recipient']['deviceToken'] +``` + +Example 4.0.0 code: +```python +token = device.push.recipient['deviceToken'] +``` + +`DeviceDetails` and `PushChannelSubscription` now also ignore fields they do not know when +decoding a dict, where 3.x raised `TypeError` on any field the server added. A misspelled key +in a dict you pass to `save()` is therefore ignored rather than rejected. + ## Version 2.x to 3.0.0 The 3.0.0 version of ably-python introduces several breaking changes to improve the realtime experience and align the API with the Ably specification. These include: diff --git a/ably/pubsub/types/channelsubscription.py b/ably/pubsub/types/channelsubscription.py index 9cc386c9..2c4ee693 100644 --- a/ably/pubsub/types/channelsubscription.py +++ b/ably/pubsub/types/channelsubscription.py @@ -3,6 +3,8 @@ class PushChannelSubscription: + _fields = ['channel', 'device_id', 'client_id', 'app_id'] + def __init__(self, channel, device_id=None, client_id=None, app_id=None): if not device_id and not client_id: raise ValueError('missing expected device or client id') @@ -32,10 +34,8 @@ def app_id(self): return self.__app_id def as_dict(self): - keys = ['channel', 'device_id', 'client_id', 'app_id'] - obj = {} - for key in keys: + for key in self._fields: value = getattr(self, key) if value is not None: key = case.snake_to_camel(key) @@ -46,7 +46,8 @@ def as_dict(self): @classmethod def from_dict(cls, obj): obj = {case.camel_to_snake(key): value for key, value in obj.items()} - return cls(**obj) + # Fields this version does not model are ignored, so the server can add them + return cls(**{key: value for key, value in obj.items() if key in cls._fields}) @classmethod def from_array(cls, array): diff --git a/ably/pubsub/types/device.py b/ably/pubsub/types/device.py index 3948b751..421fe250 100644 --- a/ably/pubsub/types/device.py +++ b/ably/pubsub/types/device.py @@ -1,22 +1,83 @@ from ably.pubsub.util import case +from ably.pubsub.util.exceptions import AblyException DevicePushTransportType = {'fcm', 'gcm', 'apns', 'web'} DevicePlatform = {'android', 'ios', 'browser'} DeviceFormFactor = {'phone', 'tablet', 'desktop', 'tv', 'watch', 'car', 'embedded', 'other'} +class DevicePushDetails: + """The push registration of a device (PCP1). + + `recipient` is a dict naming the push transport and address, `state` is the state as + the server sends it, such as 'Active', and `error_reason` is an AblyException. + """ + + def __init__(self, recipient=None, state=None, error_reason=None): + if recipient: + transport_type = recipient.get('transportType') + if transport_type is not None and transport_type not in DevicePushTransportType: + raise ValueError(f'unexpected transport type {transport_type}') + + self.__recipient = recipient + self.__state = state + self.__error_reason = error_reason + + @property + def recipient(self): + return self.__recipient + + @property + def state(self): + return self.__state + + @property + def error_reason(self): + return self.__error_reason + + def as_dict(self): + obj = {} + if self.recipient is not None: + obj['recipient'] = self.recipient + if self.state is not None: + obj['state'] = self.state + if self.error_reason is not None: + obj['errorReason'] = { + 'message': self.error_reason.message, + 'statusCode': self.error_reason.status_code, + 'code': self.error_reason.code, + } + + return obj + + @classmethod + def from_dict(cls, obj): + error_reason = obj.get('errorReason') + if isinstance(error_reason, dict): + error_reason = AblyException.from_dict(error_reason) + + # Fields this version does not model are ignored, so the server can add them + return cls(recipient=obj.get('recipient'), state=obj.get('state'), error_reason=error_reason) + + @classmethod + def factory(cls, push): + if isinstance(push, cls): + return push + + return cls.from_dict(push) + + class DeviceDetails: + _fields = ['id', 'client_id', 'form_factor', 'metadata', 'platform', + 'push', 'update_token', 'app_id', 'device_identity_token', 'modified', 'device_secret'] + def __init__(self, id, client_id=None, form_factor=None, metadata=None, platform=None, push=None, update_token=None, app_id=None, device_identity_token=None, modified=None, device_secret=None): - if push: - recipient = push.get('recipient') - if recipient: - transport_type = recipient.get('transportType') - if transport_type is not None and transport_type not in DevicePushTransportType: - raise ValueError(f'unexpected transport type {transport_type}') + if push is not None: + push = DevicePushDetails.factory(push) if platform is not None and platform not in DevicePlatform: raise ValueError(f'unexpected platform {platform}') @@ -81,13 +142,12 @@ def device_secret(self): return self.__device_secret def as_dict(self): - keys = ['id', 'client_id', 'form_factor', 'metadata', 'platform', - 'push', 'update_token', 'app_id', 'device_identity_token', 'modified', 'device_secret'] - obj = {} - for key in keys: + for key in self._fields: value = getattr(self, key) if value is not None: + if key == 'push': + value = value.as_dict() key = case.snake_to_camel(key) obj[key] = value @@ -96,7 +156,8 @@ def as_dict(self): @classmethod def from_dict(cls, obj): obj = {case.camel_to_snake(key): value for key, value in obj.items()} - return cls(**obj) + # Fields this version does not model are ignored, so the server can add them + return cls(**{key: value for key, value in obj.items() if key in cls._fields}) @classmethod def from_array(cls, array): diff --git a/test/unit/push_types_test.py b/test/unit/push_types_test.py new file mode 100644 index 00000000..9208b7c1 --- /dev/null +++ b/test/unit/push_types_test.py @@ -0,0 +1,76 @@ +"""Unit tests for the push admin types. + +Tests cover: +- PCD7, PCP1-PCP4: DeviceDetails.push as a DevicePushDetails +- fields the server adds that the types do not model are ignored when decoding +""" + +import pytest + +from ably.pubsub.types.channelsubscription import PushChannelSubscription +from ably.pubsub.types.device import DeviceDetails, DevicePushDetails +from ably.pubsub.util.exceptions import AblyException + +DEVICE = { + 'id': 'device-001', + 'clientId': 'client-abc', + 'platform': 'ios', + 'formFactor': 'phone', + 'push': { + 'recipient': {'transportType': 'apns', 'deviceToken': 'token-123'}, + 'state': 'Failing', + 'errorReason': {'message': 'Bad token', 'statusCode': 400, 'code': 40000}, + }, +} + + +def test_device_push_is_device_push_details(): + device = DeviceDetails.from_dict(DEVICE) + assert isinstance(device.push, DevicePushDetails) + assert device.push.recipient == {'transportType': 'apns', 'deviceToken': 'token-123'} + assert device.push.state == 'Failing' + assert isinstance(device.push.error_reason, AblyException) + assert device.push.error_reason.code == 40000 + assert device.push.error_reason.status_code == 400 + assert device.push.error_reason.message == 'Bad token' + + +def test_device_as_dict_round_trips(): + assert DeviceDetails.from_dict(DEVICE).as_dict() == DEVICE + + +def test_device_accepts_push_as_dict_or_details(): + recipient = {'transportType': 'apns', 'deviceToken': 'token-123'} + from_dict = DeviceDetails('device-001', push={'recipient': recipient}) + from_details = DeviceDetails('device-001', push=DevicePushDetails(recipient=recipient)) + assert from_dict.push.recipient == recipient + assert from_details.push.recipient == recipient + assert from_dict.as_dict() == from_details.as_dict() == {'id': 'device-001', 'push': {'recipient': recipient}} + + +def test_push_rejects_unknown_transport_type(): + with pytest.raises(ValueError): + DevicePushDetails(recipient={'transportType': 'xyz'}) + with pytest.raises(ValueError): + DeviceDetails('device-001', push={'recipient': {'transportType': 'xyz'}}) + + +def test_device_ignores_unknown_fields(): + device = DeviceDetails.from_dict({ + **DEVICE, + 'someNewField': 'value', + 'push': {**DEVICE['push'], 'someNewPushField': 'value'}, + }) + assert device.id == 'device-001' + assert device.as_dict() == DEVICE + + +def test_channel_subscription_ignores_unknown_fields(): + subscription = PushChannelSubscription.from_dict({ + 'channel': 'pushenabled:test', + 'clientId': 'client-abc', + 'someNewField': 'value', + }) + assert subscription.channel == 'pushenabled:test' + assert subscription.client_id == 'client-abc' + assert subscription.as_dict() == {'channel': 'pushenabled:test', 'clientId': 'client-abc'} diff --git a/test/uts/deviations.md b/test/uts/deviations.md index f6eaebf3..cc20db47 100644 --- a/test/uts/deviations.md +++ b/test/uts/deviations.md @@ -1871,7 +1871,6 @@ through an internal object to get at a value the specification makes public. | RTP19, and the `Interface Under Test` block of `presence_sync.md` | `endSync() -> List`, the synthesized LEAVEs | `end_sync()` returns `(residual, absent)` of the *stored* members; the synthesis lives one level up in `RealtimePresence.set_presence` (`presence.py:575-587`). Tests reading only counts and clientIds concatenate the two lists exactly as `set_presence` does; tests reading the LEAVE itself drive a `RealtimePresence` and assert on what its subscribers receive | 4 in `presence_sync_test.py` | | TB2, RTS3b, RTS3c, RTS3c1, RTL16 | `channel.options` as a `ChannelOptions` | a dict keyed by wire names, because `RealtimeChannel` passes `ChannelOptions.to_dict()` to the REST `Channel` constructor (`channel.py:84`). Assertions read `channel.options['params']['rewind']`. On `ChannelOptions` itself the cipher attribute is spelled `cipher`, not `cipherParams`. `set_options_without_reattach` replaces the stored mapping wholesale rather than merging, which `test_rts3c_options_updated_existing` pins | 5 | | RTS2, RTS4c, RTS4d, RTS4e | `channels.exists(name)` and `channels.names` | `name in client.channels` (`Channels.__contains__`); the collection iterates over its channels rather than their names. Genuinely idiomatic spelling rather than an absence — recorded only because of the `__getattr__` hazard noted below | 6 | -| RSH1b1, RSH1b2, RSH1b3, RSH1b4, RSH1b5, RSH1c3 | `DevicePushDetails`. The specification builds every device as `DeviceDetails(…, push: DevicePushDetails(recipient: {…}))`; ably-python has no such type | `DeviceDetails.__init__` takes `push` as a plain dict and stores it unchanged (`ably/types/device.py:10-40`), and `DeviceDetails.push` hands that dict back, so the tests read `{'recipient': {…}}` directly. The recipient's `transportType` is still validated against `DevicePushTransportType` in the constructor, which is the only part of `DevicePushDetails` carrying behaviour | the 7 in `push_admin_test.py` that register a device, through its `apns_device()` helper | **Status:** open bugs of the missing-API kind, not of the wrong-behaviour kind. Adding the accessors would leave every assertion above unchanged; only the spelling would move. diff --git a/test/uts/rest/integration/push_admin_test.py b/test/uts/rest/integration/push_admin_test.py index 48d54a4e..e6ea2b97 100644 --- a/test/uts/rest/integration/push_admin_test.py +++ b/test/uts/rest/integration/push_admin_test.py @@ -8,6 +8,7 @@ from ably.pubsub.request.paginatedresult import PaginatedResult from ably.pubsub.server import AblyException, DeviceDetails, PushChannelSubscription +from ably.pubsub.types.device import DevicePushDetails from test.uts.helpers.client import sandbox_rest_client, wall_clock_poll_until from test.uts.helpers.sandbox import random_id @@ -23,17 +24,13 @@ def apns_device(device_id, client_id=None, device_token=DEVICE_TOKEN, platform='ios', form_factor='phone'): - """The specifications' `DeviceDetails(... push: DevicePushDetails(recipient: ...))`. - - There is no `DevicePushDetails` type in ably-python; `DeviceDetails.push` is - the plain dict the wire carries, so the recipient goes in directly. - """ + """The specifications' `DeviceDetails(... push: DevicePushDetails(recipient: ...))`.""" return DeviceDetails( id=device_id, client_id=client_id, platform=platform, form_factor=form_factor, - push={'recipient': {'transportType': 'apns', 'deviceToken': device_token}}, + push=DevicePushDetails(recipient={'transportType': 'apns', 'deviceToken': device_token}), ) @@ -90,7 +87,7 @@ async def test_rsh1b3_save_and_get_device(sandbox): assert saved.id == device_id assert saved.platform == 'ios' assert saved.form_factor == 'phone' - assert saved.push['recipient']['transportType'] == 'apns' + assert saved.push.recipient['transportType'] == 'apns' retrieved = await registrations.get(device_id) assert isinstance(retrieved, DeviceDetails) @@ -113,10 +110,10 @@ async def test_rsh1b3_update_device_registration(sandbox): updated = await registrations.save(apns_device(device_id, device_token=token_v2)) assert updated.id == device_id - assert updated.push['recipient']['deviceToken'] == token_v2 + assert updated.push.recipient['deviceToken'] == token_v2 retrieved = await registrations.get(device_id) - assert retrieved.push['recipient']['deviceToken'] == token_v2 + assert retrieved.push.recipient['deviceToken'] == token_v2 finally: await remove_device_quietly(client, device_id) @@ -144,7 +141,7 @@ async def test_rsh1b2_list_devices_filtered(sandbox): id=device_id, platform='android', form_factor='tablet', - push={'recipient': {'transportType': 'gcm', 'registrationToken': f'token-{random_id()}'}}, + push=DevicePushDetails(recipient={'transportType': 'gcm', 'registrationToken': f'token-{random_id()}'}), )) await registrations.save(apns_device(decoy_id)) try: diff --git a/test/uts/rest/integration/push_channels_test.py b/test/uts/rest/integration/push_channels_test.py index 79b60a65..4c031e4f 100644 --- a/test/uts/rest/integration/push_channels_test.py +++ b/test/uts/rest/integration/push_channels_test.py @@ -4,7 +4,7 @@ DEVIATION: ably-python implements neither the PushChannel interface (RSH7, the `push` field on a channel) nor LocalDevice (RSH8). `ably/rest/channel.py` gives a channel no -`push`, `DefaultPubSubHttpClient` no `device`, and `ably/types/device.py` defines only `DeviceDetails`. +`push`, `DefaultPubSubHttpClient` no `device`, and `ably/types/device.py` defines no `LocalDevice`. Both tests here therefore depart from the specification and are gated behind RUN_DEVIATIONS, against the same spelling [test/uts/rest/unit/push/push_channels_test.py](../unit/push/push_channels_test.py) @@ -32,6 +32,7 @@ """ from ably.pubsub.server import AblyException, DeviceDetails +from ably.pubsub.types.device import DevicePushDetails from test.uts.helpers.client import sandbox_rest_client, wall_clock_poll_until from test.uts.helpers.deviations import deviation from test.uts.helpers.sandbox import random_id @@ -93,13 +94,11 @@ async def test_rsh7a_subscribe_unsubscribe_device(sandbox): device_token = 'test-apns-token-' + random_id() # A device has to be registered before a device channel subscription will take. - # There is no `DevicePushDetails` type in ably-python; `DeviceDetails.push` is the - # plain dict the wire carries, so the recipient goes in directly. registration = await client.push.admin.device_registrations.save(DeviceDetails( id=device_id, platform='ios', form_factor='phone', - push={'recipient': {'transportType': 'apns', 'deviceToken': device_token}}, + push=DevicePushDetails(recipient={'transportType': 'apns', 'deviceToken': device_token}), )) try: # SPEC FAULT: the specification's own comment says the deviceIdentityToken is diff --git a/test/uts/rest/unit/push/push_device_registrations_test.py b/test/uts/rest/unit/push/push_device_registrations_test.py index 495ecc64..78dbee34 100644 --- a/test/uts/rest/unit/push/push_device_registrations_test.py +++ b/test/uts/rest/unit/push/push_device_registrations_test.py @@ -9,7 +9,7 @@ import pytest from ably.pubsub.request.paginatedresult import PaginatedResult -from ably.pubsub.types.device import DeviceDetails +from ably.pubsub.types.device import DeviceDetails, DevicePushDetails from ably.pubsub.util.exceptions import AblyException from test.uts.helpers.client import rest_client from test.uts.helpers.deviations import deviation @@ -61,10 +61,9 @@ async def test_rsh1b1_get_device_details(): assert device.form_factor == 'phone' assert device.platform == 'ios' assert device.metadata['model'] == 'iPhone 14' - # NOTE: the spec models push as a DevicePushDetails object. DeviceDetails.push is a - # plain dict here, so the same values are read by key. - assert device.push['recipient']['transportType'] == 'apns' - assert device.push['state'] == 'Active' + assert isinstance(device.push, DevicePushDetails) + assert device.push.recipient['transportType'] == 'apns' + assert device.push.state == 'Active' # UTS: rest/unit/RSH1b1/get-unknown-device-error-1 @@ -205,7 +204,7 @@ async def test_rsh1b3_save_put_device_details(): client_id='client-abc', platform='ios', form_factor='phone', - push={'recipient': {'transportType': 'apns', 'deviceToken': 'token-123'}}, + push=DevicePushDetails(recipient={'transportType': 'apns', 'deviceToken': 'token-123'}), ) result = await client.push.admin.device_registrations.save(device) @@ -225,7 +224,7 @@ async def test_rsh1b3_save_put_device_details(): assert isinstance(result, DeviceDetails) assert result.id == 'device-001' - assert result.push['state'] == 'Active' + assert result.push.state == 'Active' # UTS: rest/unit/RSH1b3/save-updates-existing-1 @@ -263,7 +262,7 @@ def on_request(request): id='device-001', platform='ios', form_factor='phone', - push={'recipient': {'transportType': 'apns', 'deviceToken': 'token-old'}}, + push=DevicePushDetails(recipient={'transportType': 'apns', 'deviceToken': 'token-old'}), ) result1 = await client.push.admin.device_registrations.save(device) @@ -272,13 +271,13 @@ def on_request(request): id='device-001', platform='ios', form_factor='phone', - push={'recipient': {'transportType': 'apns', 'deviceToken': 'token-new'}}, + push=DevicePushDetails(recipient={'transportType': 'apns', 'deviceToken': 'token-new'}), ) result2 = await client.push.admin.device_registrations.save(updated_device) - assert result1.push['recipient']['deviceToken'] == 'token-old' - assert result2.push['recipient']['deviceToken'] == 'token-new' + assert result1.push.recipient['deviceToken'] == 'token-old' + assert result2.push.recipient['deviceToken'] == 'token-new' assert request_count == 2 @@ -297,7 +296,7 @@ async def test_rsh1b3_save_error_propagated(): id='device-001', platform='ios', form_factor='phone', - push={'recipient': {}}, + push=DevicePushDetails(recipient={}), ) with pytest.raises(AblyException) as excinfo: From 0d84c6ccb48fba9bd5cf8b2bbdb5734b81299f52 Mon Sep 17 00:00:00 2001 From: owenpearson Date: Mon, 26 Oct 2026 20:55:44 +0000 Subject: [PATCH 57/71] pubsub: make channel options a ChannelOptions that set_options replaces channel.options was the dict ChannelOptions.to_dict() produced, and the cipher a channel encrypted with lived beside it in a separate __cipher that the REST options setter only ever overwrote when the dict had a 'cipher' key. to_dict() leaves a None cipher out, so the cipher could never be removed: set_options(ChannelOptions(params=...)) on an encrypted channel, or a repeat channels.get with options that left the cipher out, kept encrypting while channel.options showed no cipher. REST Presence also captured the channel's cipher once, when it was built. channel.options is now the ChannelOptions last set (TB2, RTS3), and the cipher is derived from it. Setting options replaces them whole, as ably-js, java, cocoa, go, dotnet and ruby do, and rebuilds the cached cipher, so options without a cipher turn encryption off (RTL16, RSL7, RTS3c, RSN3c). Every reader, REST Presence and the LiveObjects RTO2b mode check included, now reads the current options. The REST channel gains set_options (RSL7), and channels.get(name, ChannelOptions(...)) works on REST, where it raised TypeError. The kwargs and dict forms channels.get already took keep working. A reattach compared the whole options, cipher included. Per RTL16a and RTS3c1 only params or modes that differ from the current ones now make an attaching or attached channel reattach, so set_options reattaches, and channels.get raises 40000, only for those. A change of cipher alone no longer raises, and repeating the same params is still accepted, as test_channel_get_channel_with_same_params expects. The spec's "provided" is read as provided and different, as ably-js reads it. This is a breaking change. Read channel.options.params rather than channel.options['params'], and pass the cipher again whenever setting the options of an encrypted channel. The two RSL7 UTS tests are un-gated. Co-Authored-By: Claude Opus 5.5 --- UPDATING.md | 39 ++++++++ ably/pubsub/http/channel.py | 66 +++++++++----- ably/pubsub/liveobjects/realtimeobject.py | 2 +- ably/pubsub/realtime/channel.py | 68 +++++++------- ably/pubsub/types/presence.py | 6 +- test/unit/channel_options_test.py | 88 +++++++++++++++++++ test/uts/deviations.md | 18 ++-- .../unit/channels/channel_options_test.py | 23 +++-- .../channel/rest_channel_attributes_test.py | 13 +-- 9 files changed, 233 insertions(+), 90 deletions(-) create mode 100644 test/unit/channel_options_test.py diff --git a/UPDATING.md b/UPDATING.md index 0359db06..adb91294 100644 --- a/UPDATING.md +++ b/UPDATING.md @@ -676,6 +676,45 @@ token = device.push.recipient['deviceToken'] decoding a dict, where 3.x raised `TypeError` on any field the server added. A misspelled key in a dict you pass to `save()` is therefore ignored rather than rejected. +### Setting channel options replaces them, cipher included + +**`set_options(ChannelOptions(params=...))` on an encrypted channel now turns encryption off.** +`channel.set_options(options)`, and `channels.get(name, options)` on a channel that already +exists, replace the channel's options whole rather than merging them, so options without a +`cipher` leave the channel unencrypted. 3.x kept the old cipher whenever the new options left +it out, and on a realtime channel even when they gave `cipher=None`. Pass the cipher again +whenever you set the options of an encrypted channel: + +Example 3.x code: +```python +channel = realtime.channels.get('my-channel', ChannelOptions(cipher=cipher_params)) +await channel.set_options(ChannelOptions(params={'rewind': '1'})) +``` + +Example 4.0.0 code: +```python +channel = pubsub_realtime_client.channels.get('my-channel', ChannelOptions(cipher=cipher_params)) +await channel.set_options(ChannelOptions(cipher=cipher_params, params={'rewind': '1'})) +``` + +`channel.options` is now the `ChannelOptions` that was last set, rather than a dict, so read +its attributes rather than its keys: + +Example 3.x code: +```python +rewind = channel.options['params']['rewind'] +``` + +Example 4.0.0 code: +```python +rewind = channel.options.params['rewind'] +``` + +HTTP channels now take a `ChannelOptions` too, through `channels.get(name, options)` or +`await channel.set_options(options)`. `channels.get` raises when given options that would make +an attaching or attached realtime channel reattach, which only a change to `params` or `modes` +does; a change to the cipher alone no longer raises. + ## Version 2.x to 3.0.0 The 3.0.0 version of ably-python introduces several breaking changes to improve the realtime experience and align the API with the Ably specification. These include: diff --git a/ably/pubsub/http/channel.py b/ably/pubsub/http/channel.py index 134de023..27200f63 100644 --- a/ably/pubsub/http/channel.py +++ b/ably/pubsub/http/channel.py @@ -9,6 +9,7 @@ from ably.pubsub.http.annotations import HttpAnnotations from ably.pubsub.request.paginatedresult import PaginatedResult, format_params from ably.pubsub.types.channeldetails import ChannelDetails +from ably.pubsub.types.channeloptions import ChannelOptions from ably.pubsub.types.message import ( Message, MessageAction, @@ -33,12 +34,11 @@ class Channel: __annotations: HttpAnnotations - def __init__(self, ably, name, options): + def __init__(self, ably, name, options: Optional[ChannelOptions] = None): self.__ably = ably self.__name = name self.__base_path = '/channels/{}/'.format(parse.quote_plus(name, safe=':')) - self.__cipher = None - self.options = options + self._set_options(options or ChannelOptions()) self.__presence = Presence(self) self.__annotations = HttpAnnotations(self) @@ -48,7 +48,7 @@ async def history(self, direction=None, limit: int = None, start=None, end=None) params = format_params({}, direction=direction, start=start, end=end, limit=limit) path = self.__base_path + 'messages' + params - message_handler = make_message_response_handler(self.__cipher) + message_handler = make_message_response_handler(self.cipher) return await PaginatedResult.paginated_query( self.ably.http, url=path, response_processor=message_handler) @@ -73,7 +73,7 @@ def __publish_request_body(self, messages): 400, 40012) if self.cipher: - m.encrypt(self.__cipher) + m.encrypt(self.cipher) request_body_list.append(m) @@ -199,7 +199,7 @@ async def _send_update( # Encrypt if needed if self.cipher: - update_message.encrypt(self.__cipher) + update_message.encrypt(self.cipher) # Serialize the message request_body = update_message.as_dict(binary=self.ably.options.use_binary_protocol) @@ -298,7 +298,7 @@ async def get_message(self, serial_or_message): response = await self.ably.http.get(path) # Create Message from the response - message_handler = make_single_message_response_handler(self.__cipher) + message_handler = make_single_message_response_handler(self.cipher) return message_handler(response) async def get_message_versions(self, serial_or_message, params=None): @@ -335,7 +335,7 @@ async def get_message_versions(self, serial_or_message, params=None): path = self.__base_path + 'messages/' + parse.quote_plus(serial, safe=':') + '/versions' + params_str # Create message handler for decoding - message_handler = make_message_response_handler(self.__cipher) + message_handler = make_message_response_handler(self.cipher) # Return paginated result return await PaginatedResult.paginated_query( @@ -358,10 +358,13 @@ def base_path(self): @property def cipher(self): + """The cipher built from `options.cipher`, or None when the channel is not encrypted""" return self.__cipher + # TB2 @property - def options(self): + def options(self) -> ChannelOptions: + """The channel's options, as last set""" return self.__options @property @@ -372,15 +375,20 @@ def presence(self): def annotations(self) -> HttpAnnotations: return self.__annotations - @options.setter - def options(self, options): - self.__options = options + # RSL7 + async def set_options(self, options: ChannelOptions) -> None: + """Replaces the channel's options. - if options and 'cipher' in options: - cipher = options.get('cipher') - if cipher is not None: - cipher = get_cipher(cipher) - self.__cipher = cipher + The options are replaced whole rather than merged, so options without a cipher turn + encryption off. + """ + self._set_options(options) + + def _set_options(self, options: ChannelOptions) -> None: + self.__options = options + # The cipher carries the CBC chaining state from one message to the next, so it is built + # once per set of options rather than on every read + self.__cipher = get_cipher(options.cipher) if options.cipher is not None else None class Channels: @@ -388,16 +396,32 @@ def __init__(self, rest): self.__ably = rest self.__all: dict = OrderedDict() - def get(self, name, **kwargs): + @staticmethod + def _channel_options(options, kwargs) -> Optional[ChannelOptions]: + """What `get` was given - a ChannelOptions, a dict of its fields or keyword arguments - as a + ChannelOptions, or None if it was given no options""" + if options is None and kwargs: + return ChannelOptions(**kwargs) + if isinstance(options, dict): + return ChannelOptions.from_dict(options) + return options + + # RSN3 + def get(self, name, options: Optional[ChannelOptions] = None, **kwargs): + """Returns the channel called `name`, creating it if need be. + + Options given for an existing channel replace its options (RSN3c). + """ if isinstance(name, bytes): name = name.decode('ascii') + options = self._channel_options(options, kwargs) if name not in self.__all: - result = self.__all[name] = Channel(self.__ably, name, kwargs) + result = self.__all[name] = Channel(self.__ably, name, options) else: result = self.__all[name] - if len(kwargs) != 0: - result.options = kwargs + if options is not None: + result._set_options(options) return result diff --git a/ably/pubsub/liveobjects/realtimeobject.py b/ably/pubsub/liveobjects/realtimeobject.py index 62c4792d..da28c893 100644 --- a/ably/pubsub/liveobjects/realtimeobject.py +++ b/ably/pubsub/liveobjects/realtimeobject.py @@ -570,7 +570,7 @@ def _throw_if_missing_channel_mode(self, mode: ChannelMode) -> None: modes = self._channel.modes if self._channel.state == ChannelState.ATTACHED else None # RTO2a if not modes: - modes = (self._channel.options or {}).get('modes') or [] # RTO2b + modes = self._channel.options.modes or [] # RTO2b if mode not in modes: # RTO2a2, RTO2b2 raise AblyException(f'"{mode.name}" channel mode must be set for this operation', 400, 40024) diff --git a/ably/pubsub/realtime/channel.py b/ably/pubsub/realtime/channel.py index c4421fbe..dd15298b 100644 --- a/ably/pubsub/realtime/channel.py +++ b/ably/pubsub/realtime/channel.py @@ -71,7 +71,6 @@ def __init__(self, realtime: DefaultPubSubRealtimeClient, name: str, self.__channel_serial: str | None = None self.__retry_timer: Timer | None = None self.__error_reason: AblyException | None = None - self.__channel_options = channel_options or ChannelOptions() self.__params: dict[str, str] | None = None self.__modes: list[ChannelMode] = [] # Channel mode flags from ATTACHED message @@ -84,8 +83,7 @@ def __init__(self, realtime: DefaultPubSubRealtimeClient, name: str, # will be disrupted if the user called .off() to remove all listeners self.__internal_state_emitter = EventEmitter() - # Pass channel options as dictionary to parent Channel class - Channel.__init__(self, realtime, name, self.__channel_options.to_dict()) + Channel.__init__(self, realtime, name, channel_options) # Initialize presence for this channel @@ -97,10 +95,15 @@ def __init__(self, realtime: DefaultPubSubRealtimeClient, name: str, # RTL27: the LiveObjects entry point for this channel self.__object = RealtimeObject(self) + # RTL16 async def set_options(self, channel_options: ChannelOptions) -> None: - """Set channel options""" + """Replaces the channel's options, reattaching if their params or modes change. + + The options are replaced whole rather than merged, so options without a cipher turn + encryption off. + """ should_reattach = self.should_reattach_to_set_options(channel_options) - self.set_options_without_reattach(channel_options) + self._set_options(channel_options) if should_reattach: self._attach_impl() @@ -108,12 +111,6 @@ async def set_options(self, channel_options: ChannelOptions) -> None: if state_change.current in (ChannelState.SUSPENDED, ChannelState.FAILED): raise state_change.reason - def set_options_without_reattach(self, channel_options: ChannelOptions) -> None: - """Internal method""" - self.__channel_options = channel_options - # Update parent class options - self.options = channel_options.to_dict() - # RTL4 async def attach(self) -> None: """Attach to channel @@ -172,7 +169,7 @@ def _attach_impl(self): # RTL4c attach_msg = { "action": ProtocolMessageAction.ATTACH, - "params": self.__channel_options.params, + "params": self.options.params, "channel": self.name, } @@ -946,9 +943,13 @@ def __on_retry_timer_expire(self) -> None: def should_reattach_to_set_options(self, new_options: ChannelOptions) -> bool: """Internal method""" + # RTL16a: only params or modes that differ from the current ones call for a reattach, and + # only while attaching or attached; the cipher never reaches the server if self.state != ChannelState.ATTACHING and self.state != ChannelState.ATTACHED: return False - return self.__channel_options != new_options + if new_options.params is not None and new_options.params != (self.options.params or {}): + return True + return new_options.modes is not None and set(new_options.modes) != set(self.options.modes or []) # RTL23 @property @@ -1016,7 +1017,7 @@ def _start_decode_failure_recovery(self, error: AblyException) -> None: self._check_pending_state() def _encode_flags(self) -> int | None: - if not self.__channel_options.modes and not self.__attach_resume: + if not self.options.modes and not self.__attach_resume: return None flags = 0 @@ -1024,8 +1025,8 @@ def _encode_flags(self) -> int | None: if self.__attach_resume: flags |= Flag.ATTACH_RESUME - if self.__channel_options.modes: - flags |= encode_channel_mode(self.__channel_options.modes) + if self.options.modes: + flags |= encode_channel_mode(self.options.modes) return flags @@ -1051,30 +1052,33 @@ def get(self, name: str, options: ChannelOptions | None = None, **kwargs) -> Rea name: str Channel name options: ChannelOptions or dict, optional - Channel options for the channel + Channel options for the channel. For an existing channel they replace its options (RTS3c) **kwargs: Additional keyword arguments to create ChannelOptions (e.g., cipher, params) + + Raises + ------ + AblyException + With code 40000 if the options for an existing channel would make it reattach (RTS3c1). + Use RealtimeChannel.set_options() for those. """ - # Convert kwargs to ChannelOptions if provided - if kwargs and not options: - options = ChannelOptions(**kwargs) - elif options and isinstance(options, dict): - options = ChannelOptions.from_dict(options) + options = self._channel_options(options, kwargs) if name not in self.__all: channel = self.__all[name] = RealtimeChannel(self.__ably, name, options) else: channel = self.__all[name] - # Update options if channel is not attached or currently attaching - if options and channel.should_reattach_to_set_options(options): - raise AblyException( - 'Channels.get() cannot be used to set channel options that would cause the channel to ' - 'reattach. Please, use RealtimeChannel.setOptions() instead.', - 400, - 40000 - ) - elif options: - channel.set_options_without_reattach(options) + if options is not None: + # RTS3c1 + if channel.should_reattach_to_set_options(options): + raise AblyException( + 'Channels.get() cannot be used to set channel options that would cause the channel to ' + 'reattach. Please, use RealtimeChannel.set_options() instead.', + 400, + 40000 + ) + # RTS3c + channel._set_options(options) return channel # RTS4 diff --git a/ably/pubsub/types/presence.py b/ably/pubsub/types/presence.py index 0043f459..6e8adf2c 100644 --- a/ably/pubsub/types/presence.py +++ b/ably/pubsub/types/presence.py @@ -209,7 +209,7 @@ def __init__(self, channel): self.__base_path = f'/channels/{parse.quote_plus(channel.name)}/' self.__binary = channel.ably.options.use_binary_protocol self.__http = channel.ably.http - self.__cipher = channel.cipher + self.__channel = channel def _path_with_qs(self, rel_path, qs=None): path = rel_path @@ -225,7 +225,7 @@ async def get(self, limit=None): qs['limit'] = limit path = self._path_with_qs(self.__base_path + 'presence', qs) - presence_handler = make_presence_response_handler(self.__cipher) + presence_handler = make_presence_response_handler(self.__channel.cipher) return await PaginatedResult.paginated_query( self.__http, url=path, response_processor=presence_handler) @@ -247,7 +247,7 @@ async def history(self, limit=None, direction=None, start=None, end=None): path = self._path_with_qs(self.__base_path + 'presence/history', qs) - presence_handler = make_presence_response_handler(self.__cipher) + presence_handler = make_presence_response_handler(self.__channel.cipher) return await PaginatedResult.paginated_query( self.__http, url=path, response_processor=presence_handler) diff --git a/test/unit/channel_options_test.py b/test/unit/channel_options_test.py new file mode 100644 index 00000000..097ea5ea --- /dev/null +++ b/test/unit/channel_options_test.py @@ -0,0 +1,88 @@ +from ably.pubsub.server import ( + ChannelMode, + ChannelOptions, + create_http_client, + create_realtime_client, + generate_random_key, + get_default_params, +) +from ably.pubsub.types.message import Message + + +def cipher_options(**kwargs): + return ChannelOptions(cipher=get_default_params({'key': generate_random_key()}), **kwargs) + + +def is_encrypted(channel): + message = Message('name', 'data') + if channel.cipher: + message.encrypt(channel.cipher) + return 'cipher' in message.as_dict().get('encoding', '') + + +async def test_http_set_options_without_a_cipher_turns_encryption_off(): + ably = create_http_client(token='foo') + channel = ably.channels.get('test', cipher_options()) + assert is_encrypted(channel) + + await channel.set_options(ChannelOptions()) + + assert channel.options.cipher is None + assert channel.cipher is None + assert not is_encrypted(channel) + await ably.close() + + +async def test_http_get_with_options_replaces_the_cipher(): + ably = create_http_client(token='foo') + channel = ably.channels.get('test', cipher_options()) + + assert ably.channels.get('test', ChannelOptions()) is channel + + assert channel.cipher is None + await ably.close() + + +async def test_http_get_without_options_keeps_the_cipher(): + ably = create_http_client(token='foo') + options = cipher_options() + channel = ably.channels.get('test', options) + + assert ably.channels.get('test') is channel + + assert channel.options is options + assert is_encrypted(channel) + await ably.close() + + +async def test_realtime_set_options_without_a_cipher_turns_encryption_off(): + ably = create_realtime_client(key='fake.key:secret', auto_connect=False) + channel = ably.channels.get('test', cipher_options(params={'rewind': '1'})) + + await channel.set_options(ChannelOptions(params={'rewind': '1'})) + + assert channel.options.params == {'rewind': '1'} + assert channel.options.cipher is None + assert not is_encrypted(channel) + await ably.close() + + +async def test_realtime_set_options_replaces_rather_than_merges(): + ably = create_realtime_client(key='fake.key:secret', auto_connect=False) + channel = ably.channels.get('test', ChannelOptions(params={'rewind': '1'}, modes=[ChannelMode.SUBSCRIBE])) + + await channel.set_options(ChannelOptions(params={'delta': 'vcdiff'})) + + assert channel.options.params == {'delta': 'vcdiff'} + assert channel.options.modes is None + await ably.close() + + +async def test_realtime_get_with_options_replaces_the_cipher(): + ably = create_realtime_client(key='fake.key:secret', auto_connect=False) + channel = ably.channels.get('test', cipher_options()) + + assert ably.channels.get('test', ChannelOptions()) is channel + + assert not is_encrypted(channel) + await ably.close() diff --git a/test/uts/deviations.md b/test/uts/deviations.md index cc20db47..8bd771d4 100644 --- a/test/uts/deviations.md +++ b/test/uts/deviations.md @@ -33,8 +33,8 @@ parametrized over a table of fixtures the specification gives inline; and five `objects/unit` tests are parametrized, three over a table and two over the scenarios the specification loops through. That turns 1481 derived tests into 1608 pytest cases. -Of **1472 Test IDs, derived as 1481 tests and run as 1608 pytest cases**: 1295 Test IDs -(1304 tests, 1429 cases) pass, 150 (150 tests, 152 cases) are gated behind +Of **1472 Test IDs, derived as 1481 tests and run as 1608 pytest cases**: 1297 Test IDs +(1306 tests, 1431 cases) pass, 148 (148 tests, 150 cases) are gated behind `RUN_DEVIATIONS`, and 27 (27 tests, 27 cases) cannot be run at all. The three groups are disjoint: two Test IDs have a gated part and a passing part — `TI1`, as two functions, and `TO/endpoint-affects-host`, as one parametrized test — and are counted with the gated, @@ -45,16 +45,16 @@ the Test IDs come from `uts/rest/unit` (503 tests, 536 cases), 482 from `uts/rest/integration` (84, 122), 73 from `uts/realtime/integration` (73, 95) and 17 from `uts/objects/integration` (17, 29); 8 of the REST integration ids (8, 8), 30 of the realtime ones (30, 30) and 5 of the objects ones (5, 5) come from the `proxy` package -within each. Of the gated Test IDs 66 are REST and 84 realtime, which is 67 REST cases +within each. Of the gated Test IDs 64 are REST and 84 realtime, which is 65 REST cases and 85 realtime. None of the 339 objects Test IDs is gated or unrunnable. A further 148 pytest cases cover the harness itself and are not derived from a specification: 130 under `helpers/`, for the mocks, and 18 under `objects/helpers/`, for the objects builders and the synced-channel mock. -The 139 gated Test IDs that record SDK non-compliance — 139 tests, 141 cases — reduce to -**59 distinct root causes**, 19 on the REST side and 40 on the realtime side. One further +The 137 gated Test IDs that record SDK non-compliance — 137 tests, 139 cases — reduce to +**58 distinct root causes**, 18 on the REST side and 40 on the realtime side. One further defect is recorded below with no test of its own, because the specification's test -cannot discriminate (RTP18a), so the file carries **60 SDK root causes** in all. The +cannot discriminate (RTP18a), so the file carries **59 SDK root causes** in all. The remaining 11 gated Test IDs are specification faults, and reduce to 8. The objects tier records no SDK non-compliance; its specification faults are all corrected or worked around in tests that pass, and reduce to the seven candidate issues O.1–O.7, none of @@ -1054,7 +1054,6 @@ option is gone, after RTN16d has done its first phase against the sandbox throug | RSC2, RSC3, RSC4, TO3b, TO3c, TO3c2 | `log_level` and `log_handler` as client options. The library logs through the standard `logging` module under the `ably.pubsub` logger, whose level and handlers are the idiomatic way to do both; `log_level`, which was stored on `Options` and read by nothing, was removed in 4.0.0, so passing either raises `TypeError` | 4 | | RSP3a2, RSP3a3 | `clientId` and `connectionId` filters on `RestPresence#get`. `Presence.get` is `get(self, limit=None)` (`ably/types/presence.py:216`), while `Presence.history` does take its documented params. `presence.get(client_id=...)` raises `TypeError: get() got an unexpected keyword argument 'client_id'`. The absence also forces the RSP5 decoding adaptation below | 4 (5 cases) | | TB4, RTL7h, RTP6e | `attachOnSubscribe`. `ChannelOptions.__init__` (`channeloptions.py:22-26`) takes only `cipher`, `params` and `modes`, and `subscribe()` on the channel, on presence and on annotations all end unconditionally with `await attach()`. This absence also forces the largest single adaptation in the suite, below | 3 | -| RSL7 | `RestChannel#setOptions`. The realtime channel implements it; the REST `options` setter expects the kwargs dict `Channels.get` collected, so a `ChannelOptions` raises `TypeError` | 2 | | RTL7f | `echoMessages` in its client-filter form. The `echo_messages` option sends `echo=false` (RTC1a), which the server honours, but the client does not itself drop a message carrying its own `connectionId`, and that is the only form a mock-backed test can observe | 1 | | RTP12, RTP12a, RTP12c | `RealtimePresence#history`. The realtime *channel* does delegate `history` to the REST implementation; only the presence object is missing it | 2 | | RTN23c1, RTN23c2 | PING/PONG. `ProtocolMessageAction` stops at `ANNOTATION` (21), so PING (22) and PONG (23) are not modelled and action 22 matches no branch of `on_protocol_message` (`websockettransport.py:37-59`, `:143-199`). The message is counted as activity and discarded | 2 | @@ -1869,7 +1868,6 @@ through an internal object to get at a value the specification makes public. | RTP17, RTP17h | a distinct `LocalPresenceMap` type keyed by `clientId` | `RealtimePresence._my_members`, the same `PresenceMap` class built with `member_key_fn=lambda msg: msg.client_id` (`presence.py:79-81`). The keying requirement is met; the newness check is not, and is gated above | all of `local_presence_map_test.py` | | RTP2d1, RTP2h1a, and the `Interface Under Test` blocks of all three presence-map specs | `put(message) -> PresenceMessage?` and `remove(message) -> PresenceMessage?` | both return `bool` (`presencemap.py:111`, `:159`); the message to emit is the caller's own, which `set_presence` appends to `broadcast_messages` when the return is true. `IS NOT null` is read as `is True`. `put` stores a *copy* with the action rewritten to PRESENT and leaves the caller's message untouched, so RTP2d1's "emit the original action" falls out for free | all of `presence_map_test.py` | | RTP19, and the `Interface Under Test` block of `presence_sync.md` | `endSync() -> List`, the synthesized LEAVEs | `end_sync()` returns `(residual, absent)` of the *stored* members; the synthesis lives one level up in `RealtimePresence.set_presence` (`presence.py:575-587`). Tests reading only counts and clientIds concatenate the two lists exactly as `set_presence` does; tests reading the LEAVE itself drive a `RealtimePresence` and assert on what its subscribers receive | 4 in `presence_sync_test.py` | -| TB2, RTS3b, RTS3c, RTS3c1, RTL16 | `channel.options` as a `ChannelOptions` | a dict keyed by wire names, because `RealtimeChannel` passes `ChannelOptions.to_dict()` to the REST `Channel` constructor (`channel.py:84`). Assertions read `channel.options['params']['rewind']`. On `ChannelOptions` itself the cipher attribute is spelled `cipher`, not `cipherParams`. `set_options_without_reattach` replaces the stored mapping wholesale rather than merging, which `test_rts3c_options_updated_existing` pins | 5 | | RTS2, RTS4c, RTS4d, RTS4e | `channels.exists(name)` and `channels.names` | `name in client.channels` (`Channels.__contains__`); the collection iterates over its channels rather than their names. Genuinely idiomatic spelling rather than an absence — recorded only because of the `__getattr__` hazard noted below | 6 | **Status:** open bugs of the missing-API kind, not of the wrong-behaviour kind. Adding the @@ -3517,8 +3515,8 @@ The header states how many derived tests there are, how many pass, how many are gated and how many cannot run. Those numbers are the check that the file is still true: in pytest cases, the gated count must equal the number of failures under `RUN_DEVIATIONS=1`, and gated plus unrunnable must equal the number of skips without -it. As of this writing that is 152 failures, 1577 passes and 27 skips with the variable -set, and 179 skips and 1577 passes without it, the 1577 being 1429 derived cases and 148 +it. As of this writing that is 150 failures, 1579 passes and 27 skips with the variable +set, and 177 skips and 1579 passes without it, the 1579 being 1431 derived cases and 148 harness ones — 130 under `helpers/` and 18 under `objects/helpers/`. Those figures need the `submodules` checkout CI makes: without it, three `rest/unit/encoding` tests that read the ably-common fixtures skip as well. The tiers are told apart by the junit report's diff --git a/test/uts/realtime/unit/channels/channel_options_test.py b/test/uts/realtime/unit/channels/channel_options_test.py index 3da5dd78..dce75329 100644 --- a/test/uts/realtime/unit/channels/channel_options_test.py +++ b/test/uts/realtime/unit/channels/channel_options_test.py @@ -3,10 +3,7 @@ Spec points: DO2a, RTL16, RTL16a, RTS3b, RTS3c, RTS3c1, RTS5, RTS5a, RTS5a1, RTS5a2, TB2, TB2c, TB2d, TB3, TB4 -`ChannelOptions.cipher` is the specification's `cipherParams`, and the options a -channel carries are the mapping `ChannelOptions.to_dict()` produces rather than -a `ChannelOptions` object, so `channel.options['params']` stands in for the -specification's `channel.options.params`. +`ChannelOptions.cipher` is the specification's `cipherParams`. """ import asyncio @@ -138,8 +135,8 @@ async def test_rts3b_options_set_on_new(): channel = client.channels.get(channel_name, channel_options) - assert channel.options['params']['rewind'] == '1' - assert ChannelMode.SUBSCRIBE in channel.options['modes'] + assert channel.options.params['rewind'] == '1' + assert ChannelMode.SUBSCRIBE in channel.options.modes # UTS: realtime/unit/RTS3c/options-updated-existing-0 @@ -157,8 +154,8 @@ async def test_rts3c_options_updated_existing(): same_channel = client.channels.get(channel_name, new_options) assert same_channel is channel - assert channel.options['cipher'] is not None - assert 'modes' not in channel.options + assert channel.options.cipher is not None + assert channel.options.modes is None # UTS: realtime/unit/RTS3c1/error-reattach-params-0 @@ -179,7 +176,7 @@ async def test_rts3c1_error_reattach_params(): client.channels.get(channel_name, new_options) assert error.value.code == 40000 - assert channel.options.get('params') is None + assert channel.options.params is None # UTS: realtime/unit/RTS3c1/error-reattach-modes-1 @@ -213,7 +210,7 @@ async def test_rtl16_set_options_updates(): new_options = ChannelOptions(params={'delta': 'vcdiff'}) await asyncio.wait_for(channel.set_options(new_options), OPERATION_TIMEOUT) - assert channel.options['params']['delta'] == 'vcdiff' + assert channel.options.params['delta'] == 'vcdiff' # UTS: realtime/unit/RTL16a/triggers-reattach-0 @@ -251,7 +248,7 @@ def record(change): assert any(change.current == ChannelState.ATTACHING for change in state_changes) assert channel.state == ChannelState.ATTACHED - assert channel.options['params']['rewind'] == '1' + assert channel.options.params['rewind'] == '1' # UTS: realtime/unit/RTS5a/creates-derived-channel-0 @@ -333,8 +330,8 @@ async def test_rts5_get_derived_with_options(): channel = client.channels.get_derived(base_channel_name, derive_options, channel_options) - assert ChannelMode.SUBSCRIBE in channel.options['modes'] - assert channel.options['attachOnSubscribe'] is False + assert ChannelMode.SUBSCRIBE in channel.options.modes + assert channel.options.attach_on_subscribe is False # UTS: realtime/unit/DO2a/filter-attribute-0 diff --git a/test/uts/rest/unit/channel/rest_channel_attributes_test.py b/test/uts/rest/unit/channel/rest_channel_attributes_test.py index 479ebd90..c1559187 100644 --- a/test/uts/rest/unit/channel/rest_channel_attributes_test.py +++ b/test/uts/rest/unit/channel/rest_channel_attributes_test.py @@ -7,7 +7,6 @@ from ably.pubsub.types.channeldetails import ChannelDetails from ably.pubsub.types.channeloptions import ChannelOptions from test.uts.helpers.client import rest_client -from test.uts.helpers.deviations import deviation from test.uts.helpers.mock_http import MockHttpClient @@ -58,12 +57,7 @@ async def test_rsl9_channel_name_attribute(): # UTS: rest/unit/RSL7/setoptions-updates-options-0 -@deviation async def test_rsl7_setoptions_updates_options(): - # DEVIATION: the REST channel has no `set_options`. `ably/rest/channel.py` exposes only an - # `options` property whose setter expects the mapping of keyword arguments that - # `Channels.get` collected, so a `ChannelOptions` raises `TypeError` from - # `'cipher' in options`. `ably/realtime/channel.py` does implement `set_options`. mock_http = MockHttpClient( on_connection_attempt=lambda conn: conn.respond_with_success(), on_request=lambda request: request.respond_with(200, []), @@ -78,9 +72,7 @@ async def test_rsl7_setoptions_updates_options(): # UTS: rest/unit/RSL7/setoptions-stores-options-1 -@deviation async def test_rsl7_setoptions_stores_options(): - # DEVIATION: as above, `set_options` is absent from the REST channel. mock_http = MockHttpClient( on_connection_attempt=lambda conn: conn.respond_with_success(), on_request=lambda request: request.respond_with(200, []), @@ -91,9 +83,10 @@ async def test_rsl7_setoptions_stores_options(): # Set options - the effect of channel options is primarily on encryption (RSL5), # which is not exercised here; the call completing is what the spec checks. - await channel.set_options(ChannelOptions()) + options = ChannelOptions() + await channel.set_options(options) - assert channel.options is not None + assert channel.options is options # UTS: rest/unit/RSL8/status-get-correct-endpoint-0 From bd1402e2411c2b60b94c2a57a96f1a358aaf3b3a Mon Sep 17 00:00:00 2001 From: owenpearson Date: Mon, 26 Oct 2026 20:56:18 +0000 Subject: [PATCH 58/71] pubsub: stop creating a channel for any unknown attribute of channels Channels.__getattr__ answered every attribute the collection did not define with self.get(name), and the realtime Channels inherited it. So hasattr(client.channels, 'x') was always true and created a channel called x, a typo became a phantom channel rather than an AttributeError, and client.channels.get_derived(...) failed with "'RealtimeChannel' object is not callable". It is removed, so an unknown attribute raises AttributeError on both clients. This is a breaking change: client.channels.my_channel no longer returns a channel; use client.channels.get('my_channel'), or client.channels['my_channel'], instead. The legacy tests that relied on it, and the deviations.md notes about it, are updated. Co-Authored-By: Claude Opus 5.5 --- UPDATING.md | 17 +++++++++++++++++ ably/pubsub/http/channel.py | 3 --- test/ably/http/httpauth_test.py | 8 ++++---- test/uts/deviations.md | 18 +----------------- .../unit/channels/channel_options_test.py | 4 +--- .../unit/channels/channels_collection_test.py | 4 +--- 6 files changed, 24 insertions(+), 30 deletions(-) diff --git a/UPDATING.md b/UPDATING.md index adb91294..1743960c 100644 --- a/UPDATING.md +++ b/UPDATING.md @@ -715,6 +715,23 @@ HTTP channels now take a `ChannelOptions` too, through `channels.get(name, optio an attaching or attached realtime channel reattach, which only a change to `params` or `modes` does; a change to the cipher alone no longer raises. +### Channels are no longer attributes of the channels collection + +`client.channels.my_channel` used to get or create the channel called `my_channel`, because the +collection answered any attribute it did not define with a channel of that name. That made a +typo, or even `hasattr(client.channels, ...)`, create a channel. In 4.0.0 an unknown attribute +raises `AttributeError`, so get the channel by name instead: + +Example 3.x code: +```python +channel = rest.channels.my_channel +``` + +Example 4.0.0 code: +```python +channel = pubsub_http_client.channels.get('my_channel') +``` + ## Version 2.x to 3.0.0 The 3.0.0 version of ably-python introduces several breaking changes to improve the realtime experience and align the API with the Ably specification. These include: diff --git a/ably/pubsub/http/channel.py b/ably/pubsub/http/channel.py index 27200f63..fb7a98fe 100644 --- a/ably/pubsub/http/channel.py +++ b/ably/pubsub/http/channel.py @@ -428,9 +428,6 @@ def get(self, name, options: Optional[ChannelOptions] = None, **kwargs): def __getitem__(self, key): return self.get(key) - def __getattr__(self, name): - return self.get(name) - def __contains__(self, item): if isinstance(item, Channel): name = item.name diff --git a/test/ably/http/httpauth_test.py b/test/ably/http/httpauth_test.py index 3f793565..5857536b 100644 --- a/test/ably/http/httpauth_test.py +++ b/test/ably/http/httpauth_test.py @@ -186,11 +186,11 @@ async def test_if_authorize_changes_auth_mechanism_to_token(self): @dont_vary_protocol async def test_authorize_always_creates_new_token(self): await self.ably.auth.authorize({'capability': {'test': ['publish']}}) - await self.ably.channels.test.publish('event', 'data') + await self.ably.channels.get('test').publish('event', 'data') await self.ably.auth.authorize({'capability': {'test': ['subscribe']}}) with pytest.raises(AblyAuthException): - await self.ably.channels.test.publish('event', 'data') + await self.ably.channels.get('test').publish('event', 'data') async def test_authorize_create_new_token_if_expired(self): token = await self.ably.auth.authorize() @@ -223,7 +223,7 @@ async def test_with_token_str_https(self): token = token.token ably = await TestApp.get_ably_rest(key=None, token=token, tls=True, use_binary_protocol=self.use_binary_protocol) - await ably.channels.test_auth_with_token_str.publish('event', 'foo_bar') + await ably.channels.get('test_auth_with_token_str').publish('event', 'foo_bar') await ably.close() async def test_with_token_str_http(self): @@ -231,7 +231,7 @@ async def test_with_token_str_http(self): token = token.token ably = await TestApp.get_ably_rest(key=None, token=token, tls=False, use_binary_protocol=self.use_binary_protocol) - await ably.channels.test_auth_with_token_str.publish('event', 'foo_bar') + await ably.channels.get('test_auth_with_token_str').publish('event', 'foo_bar') await ably.close() async def test_if_default_client_id_is_used(self): diff --git a/test/uts/deviations.md b/test/uts/deviations.md index 8bd771d4..c09fbcd5 100644 --- a/test/uts/deviations.md +++ b/test/uts/deviations.md @@ -1868,7 +1868,6 @@ through an internal object to get at a value the specification makes public. | RTP17, RTP17h | a distinct `LocalPresenceMap` type keyed by `clientId` | `RealtimePresence._my_members`, the same `PresenceMap` class built with `member_key_fn=lambda msg: msg.client_id` (`presence.py:79-81`). The keying requirement is met; the newness check is not, and is gated above | all of `local_presence_map_test.py` | | RTP2d1, RTP2h1a, and the `Interface Under Test` blocks of all three presence-map specs | `put(message) -> PresenceMessage?` and `remove(message) -> PresenceMessage?` | both return `bool` (`presencemap.py:111`, `:159`); the message to emit is the caller's own, which `set_presence` appends to `broadcast_messages` when the return is true. `IS NOT null` is read as `is True`. `put` stores a *copy* with the action rewritten to PRESENT and leaves the caller's message untouched, so RTP2d1's "emit the original action" falls out for free | all of `presence_map_test.py` | | RTP19, and the `Interface Under Test` block of `presence_sync.md` | `endSync() -> List`, the synthesized LEAVEs | `end_sync()` returns `(residual, absent)` of the *stored* members; the synthesis lives one level up in `RealtimePresence.set_presence` (`presence.py:575-587`). Tests reading only counts and clientIds concatenate the two lists exactly as `set_presence` does; tests reading the LEAVE itself drive a `RealtimePresence` and assert on what its subscribers receive | 4 in `presence_sync_test.py` | -| RTS2, RTS4c, RTS4d, RTS4e | `channels.exists(name)` and `channels.names` | `name in client.channels` (`Channels.__contains__`); the collection iterates over its channels rather than their names. Genuinely idiomatic spelling rather than an absence — recorded only because of the `__getattr__` hazard noted below | 6 | **Status:** open bugs of the missing-API kind, not of the wrong-behaviour kind. Adding the accessors would leave every assertion above unchanged; only the spelling would move. @@ -2167,7 +2166,7 @@ pseudocode is not mistaken for non-compliance. | Subject | Note | |---|---| | DISCONNECTED as a restable state | RTN15a retries a drop from CONNECTED through `loop.call_soon` (`connectionmanager.py:668`) with no time passing, so DISCONNECTED is left within the same turn of the event loop and a listener registered afterwards never sees it. The heartbeat specification says so itself, under "Verifying Transient States"; tests record the whole sequence with `connection.on(...)` and assert `CONTAINS_IN_ORDER`. Where a test needs the connection to *rest* in DISCONNECTED it fails the immediate retry and raises `disconnected_retry_timeout`. Correct RTN15a behaviour | -| `Channels.__getattr__` | `ably/rest/channel.py:408` answers **any** unknown attribute with `self.get(name)`, so reading an attribute that does not exist creates a channel named after it and never raises — `hasattr(channels, 'get_derived')` is `True`, and mutates the collection. Consequences: a typo becomes a phantom channel, and `client.channels.get_derived(...)` fails with `TypeError: 'RealtimeChannel' object is not callable` rather than `AttributeError`. No test loses coverage to it, because no derived test names an attribute the collection does not define; it is recorded because it silently changes what a probing test measures. `Channels.__iter__` is annotated `Iterator[str]` but yields `Channel` objects, which is a second, smaller instance | +| `Channels.__iter__` | annotated `Iterator[str]`, but yields the `Channel` objects rather than their names | | `RealtimeChannel.publish()` | positional-only (`*args`, `channel.py:342`). The keyword form the specifications write raises `ValueError`, although `RestChannel.publish()` accepts it. Every test uses the positional form | | ACK `res` | every ACK in the specifications is written `res: { "serials": [...] }`, a single object. `WebSocketTransport` (`websockettransport.py:191-193`) reads `res` as a list, one entry per acknowledged ProtocolMessage, which matches the protocol definition. The specification's single object is shorthand for the one-message case | | `subscribe` before `connect` | all eight `message_field_population.md` tests subscribe in their setup, before `client.connect()`. `subscribe` awaits `attach()`, which raises 90001 unless the connection is CONNECTING, CONNECTED or DISCONNECTED, so the derived tests connect first. Nothing they assert depends on the order | @@ -2672,21 +2671,6 @@ The registry needs an `(event, listener)` key holding a list per key. `test/uts/realtime/unit/channels/channel_subscribe_test.py -k rtl8b` `test/uts/realtime/unit/presence/realtime_presence_subscribe_test.py -k rtp7b` -**2.3 `Channels.__getattr__` answers any unknown attribute by creating a channel.** -`ably/rest/channel.py:408` is `return self.get(name)`, so `hasattr(client.channels, 'x')` -is always `True` **and creates a channel called `x`**. A typo becomes a phantom channel, -`client.channels.get_derived(...)` raises `TypeError: 'RealtimeChannel' object is not -callable` rather than `AttributeError`, and any code that probes the collection mutates it. -No derived test loses coverage to it, because none names an attribute the collection does -not define — that constraint is itself the evidence. `Channels.__iter__` is annotated -`Iterator[str]` and yields `Channel` objects, which is the same carelessness one size down. -No gated test; reproduce with: -```python -len(client.channels) # 0 -hasattr(client.channels, 'foo') # True -len(client.channels) # 1 -``` - **2.5 `Channels._on_channel_message` subscripts before it checks.** `channel.py:1038-1045` does `channel = self.__all[channel_name]` and only then `if not channel:`, so a message for an unknown channel raises `KeyError` inside the protocol task — swallowed and logged — and diff --git a/test/uts/realtime/unit/channels/channel_options_test.py b/test/uts/realtime/unit/channels/channel_options_test.py index dce75329..40b85f1b 100644 --- a/test/uts/realtime/unit/channels/channel_options_test.py +++ b/test/uts/realtime/unit/channels/channel_options_test.py @@ -255,9 +255,7 @@ def record(change): @deviation async def test_rts5a_creates_derived_channel(): # DEVIATION: `Channels.get_derived` is absent (`ably/realtime/channel.py:964`), as is - # `DeriveOptions`. `Channels.__getattr__` (`ably/rest/channel.py:408`) answers any - # unknown attribute with a channel of that name, so the call raises - # `TypeError: 'RealtimeChannel' object is not callable` rather than AttributeError. + # `DeriveOptions`, so the import fails before the call would raise AttributeError. from ably.pubsub.server import DeriveOptions base_channel_name = 'test-RTS5a' diff --git a/test/uts/realtime/unit/channels/channels_collection_test.py b/test/uts/realtime/unit/channels/channels_collection_test.py index 8d85a2df..ef64317a 100644 --- a/test/uts/realtime/unit/channels/channels_collection_test.py +++ b/test/uts/realtime/unit/channels/channels_collection_test.py @@ -4,9 +4,7 @@ The specification's `channels.exists(name)` and `channels.names` are spelled `name in channels` and iteration over the collection in ably-python, and -`channels.release(name)` is synchronous. Reading an attribute the collection -does not define creates a channel of that name (`Channels.__getattr__` in -`ably/rest/channel.py`), so these tests never name one that is not there. +`channels.release(name)` is synchronous. """ import asyncio From 4a761b66288a264c94975a841dc726e63844db7c Mon Sep 17 00:00:00 2001 From: owenpearson Date: Mon, 26 Oct 2026 20:57:24 +0000 Subject: [PATCH 59/71] pubsub: encode a space in a channel name as %20 in HTTP paths The HTTP channel built its path with parse.quote_plus, which is form encoding and writes a space as +. In a URL path the Ably service reads + as a literal plus. Measured against a throwaway sandbox app, a publish to /channels/rsl2+probe+/messages was stored in a channel named "rsl2+probe+", which history read back through either + or %2B, while /channels/rsl2%20probe%20/messages reached "rsl2 probe ", the channel a realtime client attaches to. So every HTTP operation on a channel whose name has a space went to another channel. ably-js (encodeURIComponent) and ably-java (HttpUtils.encodeURIComponent) both send %20. Channel.status interpolated the name with no escaping at all, so a name with "/", "?" or "#" addressed the wrong resource. Channel names are now percent-encoded with parse.quote (RSL2), status() escapes the name the same way (RSL8), REST presence and annotations build on the channel's own base path rather than encoding the name again, and message serials use parse.quote too. ":" is still left literal, as RFC 3986 allows in a path segment. This is a breaking change for HTTP data stored under a name with a space: 3.x stored it in the "+" channel, which is still reachable as channels.get('my+channel'). The two deviations.md rows for RSL2's space and RSL8's escaping are removed and the RSL2 UTS test asserts %20. REST presence used to escape ":" as %3A too, so the legacy presence tests now mock the path with the literal ":" that is sent. Co-Authored-By: Claude Opus 5.5 --- UPDATING.md | 18 ++++++++++++++++++ ably/pubsub/http/annotations.py | 3 +-- ably/pubsub/http/channel.py | 12 +++++++----- ably/pubsub/types/presence.py | 2 +- test/ably/http/httppresence_test.py | 4 ++-- test/uts/deviations.md | 4 +--- test/uts/rest/unit/channel/get_message_test.py | 2 +- test/uts/rest/unit/channel/history_test.py | 7 ++----- .../channel/rest_channel_attributes_test.py | 8 ++------ .../unit/channel/update_delete_message_test.py | 2 +- 10 files changed, 36 insertions(+), 26 deletions(-) diff --git a/UPDATING.md b/UPDATING.md index 1743960c..041f3ebe 100644 --- a/UPDATING.md +++ b/UPDATING.md @@ -732,6 +732,24 @@ Example 4.0.0 code: channel = pubsub_http_client.channels.get('my_channel') ``` +### A space in a channel name reaches the channel it names over HTTP + +3.x wrote a space in a channel name as `+` in HTTP request paths, which the Ably service reads +as a literal plus, so HTTP publishes, history, presence, annotations, message reads and +`status()` for `my channel` all went to a different channel, `my+channel`, from the one +realtime clients attach to. 4.0.0 writes the space as `%20`, so they reach `my channel`. +Messages that 3.x stored over HTTP for such a name are in the `+` channel, and stay reachable +by that name: + +Example 4.0.0 code: +```python +stored_by_3x = pubsub_http_client.channels.get('my+channel') +history = await stored_by_3x.history() +``` + +`status()` now escapes the channel name too, so a name containing `/`, `?` or `#` no longer +addresses the wrong resource. + ## Version 2.x to 3.0.0 The 3.0.0 version of ably-python introduces several breaking changes to improve the realtime experience and align the API with the Ably specification. These include: diff --git a/ably/pubsub/http/annotations.py b/ably/pubsub/http/annotations.py index 88332352..051fa488 100644 --- a/ably/pubsub/http/annotations.py +++ b/ably/pubsub/http/annotations.py @@ -116,8 +116,7 @@ def __base_path_for_serial(self, serial): Returns: str: The API path """ - channel_path = '/channels/{}/'.format(parse.quote_plus(self.__channel.name, safe=':')) - return channel_path + 'messages/' + parse.quote_plus(serial, safe=':') + '/annotations' + return self.__channel.base_path + 'messages/' + parse.quote(serial, safe=':') + '/annotations' async def __send_annotation(self, annotation: Annotation, params: dict | None = None): """ diff --git a/ably/pubsub/http/channel.py b/ably/pubsub/http/channel.py index fb7a98fe..3fa079c2 100644 --- a/ably/pubsub/http/channel.py +++ b/ably/pubsub/http/channel.py @@ -37,7 +37,8 @@ class Channel: def __init__(self, ably, name, options: Optional[ChannelOptions] = None): self.__ably = ably self.__name = name - self.__base_path = '/channels/{}/'.format(parse.quote_plus(name, safe=':')) + # RSL2: quote_plus is form encoding, and its `+` for a space is a literal plus in a path + self.__base_path = '/channels/{}/'.format(parse.quote(name, safe=':')) self._set_options(options or ChannelOptions()) self.__presence = Presence(self) self.__annotations = HttpAnnotations(self) @@ -156,7 +157,8 @@ async def publish(self, *args, **kwargs): async def status(self): """Retrieves current channel active status with no. of publishers, subscribers, presence_members etc""" - path = f'/channels/{self.name}' + # RSL8 + path = '/channels/{}'.format(parse.quote(self.name, safe=':')) response = await self.ably.http.get(path) obj = response.to_native() return ChannelDetails.from_dict(obj) @@ -210,7 +212,7 @@ async def _send_update( request_body = msgpack.packb(request_body, use_bin_type=True) # Build path with params - path = self.__base_path + 'messages/{}'.format(parse.quote_plus(message.serial, safe=':')) + path = self.__base_path + 'messages/{}'.format(parse.quote(message.serial, safe=':')) if params: params = {k: str(v).lower() if isinstance(v, bool) else v for k, v in params.items()} path += '?' + parse.urlencode(params) @@ -292,7 +294,7 @@ async def get_message(self, serial_or_message): ) # Build the path - path = self.__base_path + 'messages/' + parse.quote_plus(serial, safe=':') + path = self.__base_path + 'messages/' + parse.quote(serial, safe=':') # Make the request response = await self.ably.http.get(path) @@ -332,7 +334,7 @@ async def get_message_versions(self, serial_or_message, params=None): # Build the path params_str = format_params({}, **params) if params else '' - path = self.__base_path + 'messages/' + parse.quote_plus(serial, safe=':') + '/versions' + params_str + path = self.__base_path + 'messages/' + parse.quote(serial, safe=':') + '/versions' + params_str # Create message handler for decoding message_handler = make_message_response_handler(self.cipher) diff --git a/ably/pubsub/types/presence.py b/ably/pubsub/types/presence.py index 6e8adf2c..5084ff0e 100644 --- a/ably/pubsub/types/presence.py +++ b/ably/pubsub/types/presence.py @@ -206,7 +206,7 @@ def from_encoded_array(encoded_array, cipher=None, context=None): class Presence: def __init__(self, channel): - self.__base_path = f'/channels/{parse.quote_plus(channel.name)}/' + self.__base_path = channel.base_path self.__binary = channel.ably.options.use_binary_protocol self.__http = channel.ably.http self.__channel = channel diff --git a/test/ably/http/httppresence_test.py b/test/ably/http/httppresence_test.py index 187c3ab4..4ba6cf75 100644 --- a/test/ably/http/httppresence_test.py +++ b/test/ably/http/httppresence_test.py @@ -81,7 +81,7 @@ def presence_mock_url(self): kwargs['port_sufix'] = '' else: kwargs['port_sufix'] = ':' + str(port) - url = '{scheme}://{host}{port_sufix}/channels/persisted%3Apresence_fixtures/presence' + url = '{scheme}://{host}{port_sufix}/channels/persisted:presence_fixtures/presence' return url.format(**kwargs) def history_mock_url(self): @@ -94,7 +94,7 @@ def history_mock_url(self): kwargs['port_sufix'] = '' else: kwargs['port_sufix'] = ':' + str(port) - url = '{scheme}://{host}{port_sufix}/channels/persisted%3Apresence_fixtures/presence/history' + url = '{scheme}://{host}{port_sufix}/channels/persisted:presence_fixtures/presence/history' return url.format(**kwargs) @dont_vary_protocol diff --git a/test/uts/deviations.md b/test/uts/deviations.md index c09fbcd5..3f652778 100644 --- a/test/uts/deviations.md +++ b/test/uts/deviations.md @@ -2183,9 +2183,7 @@ pseudocode is not mistaken for non-compliance. | Spec points | Specification | ably-python | Status | |---|---|---|---| -| RSL2 | A space in a channel name is `%20` | `+`, from `parse.quote_plus` in `Channel.__init__`. `quote_plus` is form encoding, and a `+` in a URL *path* is a literal plus, so the name reaching the server is altered | Open bug, and a genuine correctness issue | -| RSL8 | `Channel#status` URI-encodes the channel id | `status()` interpolates the name with no escaping at all. `a/b` addresses the wrong resource, `a?b` truncates the name into a query string, `a#b` becomes a fragment | Open bug | -| RSL2, RSL11b, RSL15b | `:` is `%3A` | Left literal, from `safe=':'`. RFC 3986 allows `:` in a path segment and Ably uses it for namespaces, so the server receives the same value | Intentional | +| RSL2, RSL8, RSL11b, RSL15b | `:` is `%3A` | Left literal, from `safe=':'`. RFC 3986 allows `:` in a path segment and Ably uses it for namespaces, so the server receives the same value | Intentional | | RTC12 / RSC1, RSC1a, RSC1c | A string constructor argument is an API key when it contains `:` and a token when it does not | `AblyRest.__init__` treats its first positional argument as a key unconditionally and hands it to `AuthOptions.set_key`, which requires exactly two colon-separated parts, so a token string raises 40101/401 "key of not len 2 parameters". A token is supplied through the separate `token` or `token_details` arguments. The empty-string case is compliant | Intentional / SDK-wide: the constructor takes credentials as distinct named arguments and has no string-sniffing path to restore | | RSC18 | The constructor rejects basic auth over HTTP | Construction succeeds; 40103 is raised from `make_request` when a request needing Basic Auth is attempted, and no request goes out. RSA1/RSC18 say only "any attempt to use" | Compliant; the UTS is stricter than its source | | RSAN1a3 | Code 40003 for a missing `Annotation.type` | 400/40000 | Cosmetic; worth aligning cross-SDK | diff --git a/test/uts/rest/unit/channel/get_message_test.py b/test/uts/rest/unit/channel/get_message_test.py index 8f3f1571..11fb2c1d 100644 --- a/test/uts/rest/unit/channel/get_message_test.py +++ b/test/uts/rest/unit/channel/get_message_test.py @@ -125,7 +125,7 @@ def on_request(request): request = captured_requests[0] # DEVIATION RSL11b: the spec asserts the serial is encoded as encodeURIComponent # would, giving "serial%2Fwith%3Aspecial%2Bchars". ably-python encodes the serial - # with `quote_plus(serial, safe=':')`, so the colon travels literally. A colon is a + # with `quote(serial, safe=':')`, so the colon travels literally. A colon is a # legal path character, so the server still receives the same serial; the slash and # the plus, which would otherwise change the serial, are both encoded. assert wire_path(request) == ( diff --git a/test/uts/rest/unit/channel/history_test.py b/test/uts/rest/unit/channel/history_test.py index cfae1d4b..c7d43679 100644 --- a/test/uts/rest/unit/channel/history_test.py +++ b/test/uts/rest/unit/channel/history_test.py @@ -177,14 +177,11 @@ def on_request(request): (simple, f'/channels/{simple}/messages'), # DEVIATION RSL2: the spec asserts the colon is percent-encoded, as # encodeURIComponent would ("test-RSL2-with%3Acolon-..."). ably-python builds the - # path with `quote_plus(name, safe=':')`, so a colon travels literally. It is a + # path with `quote(name, safe=':')`, so a colon travels literally. It is a # legal path character, so the server still receives the same channel name. (with_colon, f'/channels/{with_colon}/messages'), (with_slash, f"/channels/{with_slash.replace('/', '%2F')}/messages"), - # DEVIATION RSL2: the spec asserts a space is percent-encoded as "%20". - # `quote_plus` renders it as "+", which in a URL *path* is a literal plus rather - # than a space, so the channel name arrives at the server altered. - (with_space, f"/channels/{with_space.replace(' ', '+')}/messages"), + (with_space, f"/channels/{with_space.replace(' ', '%20')}/messages"), ] for channel_name, expected_path in test_cases: diff --git a/test/uts/rest/unit/channel/rest_channel_attributes_test.py b/test/uts/rest/unit/channel/rest_channel_attributes_test.py index c1559187..f585da01 100644 --- a/test/uts/rest/unit/channel/rest_channel_attributes_test.py +++ b/test/uts/rest/unit/channel/rest_channel_attributes_test.py @@ -128,12 +128,8 @@ async def test_rsl8_status_special_chars_encoded(): assert request.method == 'GET' # DEVIATION: the spec asserts the path ends with "/channels/" + encode_uri_component(name). - # `Channel.status` in `ably/rest/channel.py` interpolates the raw name into - # `f'/channels/{self.name}'` and encodes nothing; the escaping visible below is httpx - # normalising the URL on the way out. For this name the wire form still reaches the right - # channel, differing from encodeURIComponent only in leaving ":" unescaped, which RFC 3986 - # permits in a path segment. Names containing "/", "?" or "#" are genuinely mis-routed -- - # see the report accompanying this suite. + # `Channel.status` leaves ":" unescaped, as every REST path does, which RFC 3986 permits in + # a path segment, so the name still reaches the right channel. assert str(request.url).endswith('/channels/namespace:my%20channel') # request.url.path is the percent-decoded path, so it carries the name verbatim assert request.url.path == f'/channels/{channel_name}' diff --git a/test/uts/rest/unit/channel/update_delete_message_test.py b/test/uts/rest/unit/channel/update_delete_message_test.py index cb4de9c7..c01474f3 100644 --- a/test/uts/rest/unit/channel/update_delete_message_test.py +++ b/test/uts/rest/unit/channel/update_delete_message_test.py @@ -305,7 +305,7 @@ async def test_rsl15b_serial_url_encoded_path(): request = captured_requests[0] # DEVIATION: the spec asserts encode_uri_component(serial), which escapes ':' as %3A. - # Channel._send_update escapes the serial with `parse.quote_plus(serial, safe=':')`, so + # Channel._send_update escapes the serial with `parse.quote(serial, safe=':')`, so # '/' becomes %2F but ':' is left as-is. ':' is a legal path character and Ably serials # carry it, so both forms name the same serial once the server decodes them. assert raw_path(request) == f'/channels/{channel_name}/messages/serial%2Fspecial:chars' From eb50756c56dbc23977271ddf6ec01bfb8034a600 Mon Sep 17 00:00:00 2001 From: owenpearson Date: Mon, 26 Oct 2026 21:00:43 +0000 Subject: [PATCH 60/71] pubsub: raise from attach() when the channel does not attach attach() raised only when the attach ended in SUSPENDED or FAILED. When the connection closed during the attach, RTL3b moved the channel to DETACHED and attach() returned None, as if it had attached; so did an attach overtaken by a detach, which returned on the DETACHING state change. RTL4d fails the attach in every case but ATTACHED. A SUSPENDED or FAILED change with no reason also made it raise None, which Python reports as a TypeError. attach() now raises unless the channel attaches: the reason for the state it moved to, else an AblyException with code 90001, as _ensure_active (RTL33b1) already did, which now just calls attach(). The two attaches nobody awaits, the reattach of a SUSPENDED channel when the connection reconnects and the implicit attach of a presence enter or update (RTP8d), go through _attach_implicitly, which logs a failure rather than leaving an exception unretrieved on the task. subscribe and the presence and annotation subscribes await their attach, so they raise in the same cases. This is a breaking change: code that closes the client, or detaches, while an attach is pending must expect attach() and subscribe() to raise. The RTL3b test asserts the failure the specification has, closing its deviation, and the RTL5i test that overtakes an attach with a detach now asserts the rejection the specification also allows. Co-Authored-By: Claude Opus 5.5 --- UPDATING.md | 19 ++++++++ ably/pubsub/realtime/channel.py | 48 +++++++++---------- ably/pubsub/realtime/presence.py | 2 +- test/uts/deviations.md | 44 +++++------------ .../unit/channels/channel_attributes_test.py | 7 +-- .../channels/channel_connection_state_test.py | 6 +-- .../unit/channels/channel_detach_test.py | 5 +- .../channel_server_initiated_detach_test.py | 3 +- .../presence/realtime_presence_enter_test.py | 4 +- 9 files changed, 67 insertions(+), 71 deletions(-) diff --git a/UPDATING.md b/UPDATING.md index 041f3ebe..d364970e 100644 --- a/UPDATING.md +++ b/UPDATING.md @@ -750,6 +750,25 @@ history = await stored_by_3x.history() `status()` now escapes the channel name too, so a name containing `/`, `?` or `#` no longer addresses the wrong resource. +### `attach()` raises when the channel does not attach + +`await channel.attach()` raised only when the channel became `SUSPENDED` or `FAILED`. When the +connection closed while the channel was attaching, the channel became `DETACHED` and `attach()` +returned as if it had succeeded; so did an `attach()` overtaken by a `detach()`. In 4.0.0 it +raises an `AblyException` whenever the channel moves to any state but `ATTACHED`, carrying the +reason for that state, or code 90001 when there is none. `subscribe()`, which attaches, raises +in the same cases. Code that closes the client, or detaches, while an attach may be pending +should expect the exception: + +Example 4.0.0 code: +```python +try: + await channel.attach() +except AblyException: + # the channel did not attach, for instance because the client was closed meanwhile + ... +``` + ## Version 2.x to 3.0.0 The 3.0.0 version of ably-python introduces several breaking changes to improve the realtime experience and align the API with the Ably specification. These include: diff --git a/ably/pubsub/realtime/channel.py b/ably/pubsub/realtime/channel.py index dd15298b..eed2fc5d 100644 --- a/ably/pubsub/realtime/channel.py +++ b/ably/pubsub/realtime/channel.py @@ -121,23 +121,15 @@ async def attach(self) -> None: Raises ------ AblyException - If unable to attach channel - """ - await self._attach() - - async def _attach(self) -> ChannelStateChange | None: - """The attach procedure `attach` runs, returning the state change it ended with, or None if - the channel was already ATTACHED. - - Raises where `attach` does. An attach that ends in a state other than ATTACHED, - SUSPENDED or FAILED, such as DETACHED when the connection closes first, returns that - state change. + If unable to attach channel, including when the channel moves to DETACHED instead, as + it does when the connection closes first (RTL4d). It carries the reason for the state + the channel moved to, else code 90001 """ log.info(f'RealtimeChannel.attach() called, channel = {self.name}') # RTL4a - if channel is attached do nothing if self.state == ChannelState.ATTACHED: - return None + return self.__error_reason = None @@ -158,10 +150,23 @@ async def _attach(self) -> ChannelStateChange | None: self._request_state(ChannelState.ATTACHING) state_change = await self.__internal_state_emitter.once_async() + self.__raise_unless_attached(state_change) - if state_change.current in (ChannelState.SUSPENDED, ChannelState.FAILED): - raise state_change.reason - return state_change + def __raise_unless_attached(self, state_change: ChannelStateChange) -> None: + # RTL4d: any state but ATTACHED fails the attach, DETACHED included (RTL3b) + if state_change.current != ChannelState.ATTACHED: + raise state_change.reason or AblyException( + f'Unable to attach channel; channel state = {state_change.current.value}, ' + f'connection state = {self.__realtime.connection.state.value}', 400, 90001) + + def _attach_implicitly(self) -> None: + """Starts an attach nobody awaits, logging its failure rather than leaving it unretrieved.""" + def on_done(task: asyncio.Task) -> None: + if not task.cancelled() and task.exception() is not None: + log.warning(f'RealtimeChannel implicit attach failed; channel = {self.name}, ' + f'error = {task.exception()}') + + asyncio.create_task(self.attach()).add_done_callback(on_done) def _attach_impl(self): log.debug("RealtimeChannel.attach_impl(): sending ATTACH protocol message") @@ -200,14 +205,9 @@ async def _ensure_active(self) -> None: if self.state == ChannelState.FAILED: raise AblyException(f"Channel operation failed as channel state is {self.state.value}", 400, 90001) - # RTL33b - state_change = await self._attach() - if state_change is not None and state_change.current != ChannelState.ATTACHED: - # RTL33b1, RTL4d: the attach ended without the channel attaching, as when the connection - # closes or `detach` is called first - raise state_change.reason or AblyException( - f'Unable to attach channel; channel state = {state_change.current.value}, ' - f'connection state = {self.__realtime.connection.state.value}', 400, 90001) + # RTL33b, RTL33b1: `attach` raises unless the channel attaches, as when the connection closes + # or `detach` is called first + await self.attach() # RTL5 async def detach(self) -> None: @@ -1164,7 +1164,7 @@ def _on_connected(self) -> None: if channel.state == ChannelState.ATTACHING or channel.state == ChannelState.DETACHING: channel._check_pending_state() elif channel.state == ChannelState.SUSPENDED: - asyncio.create_task(channel.attach()) + channel._attach_implicitly() elif channel.state == ChannelState.ATTACHED: channel._request_state(ChannelState.ATTACHING) diff --git a/ably/pubsub/realtime/presence.py b/ably/pubsub/realtime/presence.py index 2c87f50e..73edfa4f 100644 --- a/ably/pubsub/realtime/presence.py +++ b/ably/pubsub/realtime/presence.py @@ -260,7 +260,7 @@ async def _enter_or_update_client( return await self._send_presence([wire_msg]) elif channel.state in [ChannelState.INITIALIZED, ChannelState.DETACHED]: # RTP8d: Implicitly attach - asyncio.create_task(channel.attach()) + channel._attach_implicitly() # Queue the message return await self._queue_presence(wire_msg) elif channel.state == ChannelState.ATTACHING: diff --git a/test/uts/deviations.md b/test/uts/deviations.md index 3f652778..0cf683e2 100644 --- a/test/uts/deviations.md +++ b/test/uts/deviations.md @@ -1329,7 +1329,7 @@ stored, and `asyncio.wait_for(channel.set_options(...), 1.0)` raises `TimeoutErr **Status:** open bug. A hang, not a wrong value. -#### A server-initiated DETACHED discards its error, and the pending call then raises `TypeError` — 3 tests +#### A server-initiated DETACHED discards its error, which `set_options` and `detach` then raise as `TypeError` — 3 tests **Spec points:** RTL24, RTL4c, with RTL13a and RTL13b. @@ -1340,23 +1340,23 @@ An attach rejected by a DETACHED carrying an `ErrorInfo` must fail with that err `self._notify_state(ChannelState.SUSPENDED)` (`channel.py:735`), passing **no reason**, so the message's error is dropped. The same shape applies to RTL13a's `_request_state(ATTACHING)` and RTL13b's `_notify_state(SUSPENDED)`. -2. `attach()` then reaches `raise state_change.reason` (`channel.py:150`) with `reason` - `None`, which Python reports as `TypeError: exceptions must derive from BaseException` - rather than as an `AblyException`. The same unguarded `raise` is at `:102` in - `set_options` and `:219` in `detach`. +2. `attach()` then raises its 90001 fallback, having no reason to raise, but + `set_options` and `detach` reach `raise state_change.reason` with `reason` `None`, which + Python reports as `TypeError: exceptions must derive from BaseException` rather than as + an `AblyException`. **Tests affected:** `test_rtl24_error_reason_attach_failure` and -`test_rtl4c_error_cleared_on_attach`, both `TypeError: exceptions must derive from -BaseException` at `channel.py:150`, and `test_rtl13b_attaching_detached_to_suspended`, +`test_rtl4c_error_cleared_on_attach`, both failing on `error_reason` being `None` once the +attach has raised, and `test_rtl13b_attaching_detached_to_suspended`, which reads the dropped error off the SUSPENDED state change and so falls one step earlier. The clearing half of RTL4c is still covered by `test_rtl4c_error_cleared_preserved_detach`, which sets the error with an ERROR message instead and passes. One adapted test — `test_rtl13a_attached_reattach_triggered` — asserts `reason is None`, and will need revisiting once the reason is carried through. -**Status:** open bug, in two parts. The `raise None` is worth fixing on its own even before -the reason is plumbed through: a missing reason should give an `AblyException`, not a -`TypeError`. +**Status:** open bug, in two parts. The `raise None` is worth fixing in `set_options` and +`detach` on its own, as it is in `attach`, even before the reason is plumbed through: a +missing reason should give an `AblyException`, not a `TypeError`. #### An ATTACHED received while DETACHING or DETACHED is ignored — 2 tests @@ -2069,24 +2069,6 @@ POSTed to `/keys/{keyName}/revokeTokens` directly, reproduced three times. RSA4a does not say which state a token error observed mid-connection should be reported in, and other SDKs may report DISCONNECTED first and fail afterwards. -### A pending `attach()` resolves, rather than failing, when the connection closes - -**Spec points:** RTL3b, RTL4d. - -RTL3b moves an ATTACHING channel to DETACHED when the connection closes, and RTL4d has the -attach's callback invoked with an `ErrorInfo` "in all other cases" than ATTACHED. -`channel_connection_state.md` spells it `AWAIT attach_future FAILS WITH error`. The RTL3b -transition is correct; the pending `attach()` then returns `None`, because `attach()` ends -with `if state_change.current in (SUSPENDED, FAILED): raise state_change.reason` -(`channel.py:148-150`) and DETACHED is in neither, so the coroutine falls through as a -success. - -**Tests affected:** `test_rtl3b_closed_attaching_to_detached` asserts `await attach_future is -None`, and makes every other assertion the specification does. It fails if the SDK starts -raising, so it does guard the behaviour. - -**Status:** open bug. - ### An attach requested while DETACHING pre-empts the detach **Spec point:** RTL4h. @@ -2151,7 +2133,7 @@ adapted. | Spec points | Specification | ably-python | Tests | |---|---|---|---| | RTL13a | the ATTACHING state change triggered by a server-initiated DETACHED carries that message's `error` as its `reason` | `_on_message` discards the error and calls `_request_state(ATTACHING)` with no reason, so `reason` is null. Gated in its own right under RTL24/RTL4c above; this test asserts `reason is None` | `test_rtl13a_attached_reattach_triggered` | -| RTP16c | answering an ATTACH with a DETACHED puts the channel in DETACHED, and a presence operation from there errors | the channel lands in SUSPENDED (`channel.py:735`) and `attach()` raises `TypeError` on the null reason. RTP16c is about the presence error, which does arrive with 90001, so the test accepts either channel state and either attach error and turns on the presence operation alone | `test_rtp16c_presence_errors_other_states` | +| RTP16c | answering an ATTACH with a DETACHED puts the channel in DETACHED, and a presence operation from there errors | the channel lands in SUSPENDED (`channel.py:735`) and `attach()` raises its 90001 fallback for want of a reason. RTP16c is about the presence error, which does arrive with 90001, so the test accepts either channel state and either attach error and turns on the presence operation alone | `test_rtp16c_presence_errors_other_states` | | RTL7g | `channel.subscribe(listener)` registers the listener even when the implicit attach is rejected | the listener *is* registered (`channel.py:279-282`), and then `subscribe` awaits `attach()`, which re-raises the failure (`:150`). So `await channel.subscribe(...)` raises where the specification's fire-and-forget call returns. The RTL7g requirement itself holds | `test_rtl7g_listener_registered_attach_fails`, `test_rtl7g_no_attach_when_attaching` | | RTL10b | an `AblyException` when `untilAttach` is used on an unattached channel | an `AblyException`, but for the wrong reason: `history` takes no `until_attach`, and `catch_all` wraps the `TypeError` as `50000 500 Unexpected exception` whatever the channel's state. The test also asserts that no HTTP request was made | `test_rtl10b_errors_when_not_attached` | | RTL19b, RTL19c, RTL20, RTL21, PC3 | a vcdiff-decoded payload equals a string literal | a `bytearray`. `EncodeDataMixin.decode` (`mixins.py:106`) leaves the vcdiff result binary because no further encoding step turns it back into text, which is correct — the specification's own `RTL19b/json-wire-form-base-1`, which does carry `utf-8/vcdiff`, receives a string. Five tests assert `b'second message'` where the specification writes the string | 5 in `channel_delta_decoding_test.py` | @@ -2643,8 +2625,8 @@ have the grouping, so the SDK is inconsistent with itself. ### Tier 2 — the wrong error, or an error where there should be none **2.1 `raise state_change.reason` raises `TypeError` when the reason is `None`.** RTL24, -RTL4c, RTL13b, RTP16c. Three sites — `channel.py:102` (`set_options`), `:150` (`attach`), -`:219` (`detach`) — raise whatever the state change carries, and the state change often +RTL4c, RTL13b, RTP16c. Two sites — `set_options` and `detach`; `attach` falls back to a +90001 — raise whatever the state change carries, and the state change often carries nothing, because a server-initiated DETACHED's error is discarded at `:735`, `_request_state(ATTACHING)` and `_notify_state(SUSPENDED)` being called with no reason. The caller gets `TypeError: exceptions must derive from BaseException` instead of an diff --git a/test/uts/realtime/unit/channels/channel_attributes_test.py b/test/uts/realtime/unit/channels/channel_attributes_test.py index aeeeaff2..30be9ac8 100644 --- a/test/uts/realtime/unit/channels/channel_attributes_test.py +++ b/test/uts/realtime/unit/channels/channel_attributes_test.py @@ -95,9 +95,7 @@ async def test_rtl24_error_reason_attach_failure(): # DEVIATION: an attach rejected with a DETACHED carrying an error leaves `error_reason` # unset. `_on_message` (`ably/realtime/channel.py:735`) answers a DETACHED received while # ATTACHING with `_notify_state(ChannelState.SUSPENDED)` and no reason, discarding the - # error. `attach()` then reaches `raise state_change.reason` (`:150`) with `reason` None, - # which raises `TypeError: exceptions must derive from BaseException` rather than the - # AblyException RTL24 describes. + # error, so `attach()` raises its 90001 fallback rather than the error RTL24 describes. channel_name = 'test-RTL24-attach-fail' mock_ws = MockWebSocket( @@ -125,8 +123,7 @@ def on_message_from_client(msg): @deviation async def test_rtl4c_error_cleared_on_attach(): # DEVIATION: as above, the error on the DETACHED which rejects the first attach is - # discarded, so `error_reason` is never set and the attach raises TypeError rather than - # an AblyException. The clearing half of RTL4c is covered by + # discarded, so `error_reason` is never set. The clearing half of RTL4c is covered by # `test_rtl4c_error_cleared_preserved_detach`, which sets the error with an ERROR message. channel_name = 'test-RTL24-clear-attach' attach_count = 0 diff --git a/test/uts/realtime/unit/channels/channel_connection_state_test.py b/test/uts/realtime/unit/channels/channel_connection_state_test.py index d1b2a947..d700fce5 100644 --- a/test/uts/realtime/unit/channels/channel_connection_state_test.py +++ b/test/uts/realtime/unit/channels/channel_connection_state_test.py @@ -309,10 +309,8 @@ async def test_rtl3b_closed_attaching_to_detached(): await asyncio.wait_for(client.close(), OPERATION_TIMEOUT) assert client.connection.state == ConnectionState.CLOSED - # The specification has the pending attach fail. ably-python's `attach()` - # raises only for SUSPENDED and FAILED, so the DETACHED which RTL3b brings - # about resolves it instead. See deviations.md - assert await asyncio.wait_for(attach_future, OPERATION_TIMEOUT) is None + with pytest.raises(AblyException): + await asyncio.wait_for(attach_future, OPERATION_TIMEOUT) assert channel.state == ChannelState.DETACHED diff --git a/test/uts/realtime/unit/channels/channel_detach_test.py b/test/uts/realtime/unit/channels/channel_detach_test.py index f113bfb6..013f43ba 100644 --- a/test/uts/realtime/unit/channels/channel_detach_test.py +++ b/test/uts/realtime/unit/channels/channel_detach_test.py @@ -171,8 +171,9 @@ def on_message_from_client(msg): mock_ws.send_to_client(attached_message(channel_name)) # The specification allows the superseded attach either to resolve or to be - # rejected; ably-python resolves it on the DETACHING state change - await asyncio.wait_for(attach_future, OPERATION_TIMEOUT) + # rejected; ably-python rejects it on the DETACHING state change + with pytest.raises(AblyException): + await asyncio.wait_for(attach_future, OPERATION_TIMEOUT) await asyncio.wait_for(detach_future, OPERATION_TIMEOUT) assert channel.state == ChannelState.DETACHED diff --git a/test/uts/realtime/unit/channels/channel_server_initiated_detach_test.py b/test/uts/realtime/unit/channels/channel_server_initiated_detach_test.py index fc681546..5b7e10c5 100644 --- a/test/uts/realtime/unit/channels/channel_server_initiated_detach_test.py +++ b/test/uts/realtime/unit/channels/channel_server_initiated_detach_test.py @@ -215,8 +215,7 @@ async def test_rtl13b_attaching_detached_to_suspended(): # DEVIATION: RTL13b carries the DETACHED message's error onto the SUSPENDED state # change. `RealtimeChannel._on_message` answers a DETACHED received while ATTACHING # with `_notify_state(ChannelState.SUSPENDED)` and no reason, so the error is - # discarded, and the pending `attach()` reaches `raise state_change.reason` with - # `reason` None, raising `TypeError` rather than an AblyException. The state cycle + # discarded, and the pending `attach()` raises its 90001 fallback instead. The state cycle # this drives is covered ungated by # `test_rtl13b_failed_reattach_suspended_retry` above. channel_name = 'test-RTL13b-attaching' diff --git a/test/uts/realtime/unit/presence/realtime_presence_enter_test.py b/test/uts/realtime/unit/presence/realtime_presence_enter_test.py index 9168d22a..d917f69e 100644 --- a/test/uts/realtime/unit/presence/realtime_presence_enter_test.py +++ b/test/uts/realtime/unit/presence/realtime_presence_enter_test.py @@ -502,9 +502,9 @@ def on_message_from_client(msg): # RTP16c is about the presence error below, so the attach is only asked to leave the # channel somewhere other than ATTACHED. A DETACHED received while ATTACHING moves # the channel to SUSPENDED rather than the DETACHED the specification expects, with - # no reason attached, so `attach()` raises `None` as a TypeError. Both are recorded + # no reason attached, so `attach()` raises its 90001 fallback. Both are recorded # against the channel specifications; see test/uts/deviations.md. - with pytest.raises((AblyException, TypeError)): + with pytest.raises(AblyException): await channel.attach() assert channel.state in (ChannelState.DETACHED, ChannelState.SUSPENDED) From 7ff41eb8ea16e930ab1dd47fdb09a7d9630b6b0a Mon Sep 17 00:00:00 2001 From: owenpearson Date: Mon, 26 Oct 2026 21:02:30 +0000 Subject: [PATCH 61/71] pubsub: detach at once when the connection is not connected detach() requested DETACHING whatever the connection's state, and _check_pending_state only sends the DETACH once the connection is CONNECTED, so from CONNECTING, DISCONNECTED or any other state but CONNECTED it awaited a state change nothing would produce, and never returned. It also raised 90001 first whenever the connection was CLOSING or FAILED, which are the old RTL5g, and from CONNECTING called connect(), which is the old RTL5h; specification 5.0.0 replaced both with RTL5l. detach() now follows RTL5 in order: nothing to do from INITIALIZED or DETACHED (RTL5a), 90001 from FAILED (RTL5b), straight to DETACHED from SUSPENDED (RTL5j), and, when the connection is in any state but CONNECTED, straight to DETACHED with no DETACH sent (RTL5l). Only a CONNECTED connection sends the DETACH and waits for the DETACHED (RTL5d). A CLOSING or CLOSED connection has already moved its channels to DETACHED (RTL3b) and a FAILED one to FAILED (RTL3a), so those cases now land on RTL5a and RTL5b rather than the 90001 RTL5g raised. This matters more now that release() requires a detached channel. It is not a breaking change: a call that hung now returns, and one that raised for a CLOSING connection now returns for its already DETACHED channel. The two RTL5l UTS tests are un-gated. Co-Authored-By: Claude Opus 5.5 --- ably/pubsub/realtime/channel.py | 26 +++++------ test/uts/deviations.md | 45 ++++--------------- .../unit/channels/channel_detach_test.py | 7 +-- 3 files changed, 24 insertions(+), 54 deletions(-) diff --git a/ably/pubsub/realtime/channel.py b/ably/pubsub/realtime/channel.py index eed2fc5d..e115d89e 100644 --- a/ably/pubsub/realtime/channel.py +++ b/ably/pubsub/realtime/channel.py @@ -225,29 +225,25 @@ async def detach(self) -> None: log.info(f'RealtimeChannel.detach() called, channel = {self.name}') - # RTL5g, RTL5b - raise exception if state invalid - if self.__realtime.connection.state in [ConnectionState.CLOSING, ConnectionState.FAILED]: - raise AblyException( - message=f"Unable to detach; channel state = {self.state}", - code=90001, - status_code=400 - ) - # RTL5a - if channel already detached do nothing if self.state in [ChannelState.INITIALIZED, ChannelState.DETACHED]: return + # RTL5b + if self.state == ChannelState.FAILED: + raise AblyException("Unable to detach; channel state = failed", 400, 90001) + + # RTL5j if self.state == ChannelState.SUSPENDED: self._notify_state(ChannelState.DETACHED) return - elif self.state == ChannelState.FAILED: - raise AblyException("Unable to detach; channel state = failed", 400, 90001) - else: - self._request_state(ChannelState.DETACHING) - # RTL5h - wait for pending connection - if self.__realtime.connection.state == ConnectionState.CONNECTING: - self.__realtime.connect() + # RTL5l: with no connection to send a DETACH on, the channel detaches at once + if self.__realtime.connection.state != ConnectionState.CONNECTED: + self._notify_state(ChannelState.DETACHED) + return + + self._request_state(ChannelState.DETACHING) state_change = await self.__internal_state_emitter.once_async() new_state = state_change.current diff --git a/test/uts/deviations.md b/test/uts/deviations.md index 0cf683e2..d07ab8ed 100644 --- a/test/uts/deviations.md +++ b/test/uts/deviations.md @@ -33,8 +33,8 @@ parametrized over a table of fixtures the specification gives inline; and five `objects/unit` tests are parametrized, three over a table and two over the scenarios the specification loops through. That turns 1481 derived tests into 1608 pytest cases. -Of **1472 Test IDs, derived as 1481 tests and run as 1608 pytest cases**: 1297 Test IDs -(1306 tests, 1431 cases) pass, 148 (148 tests, 150 cases) are gated behind +Of **1472 Test IDs, derived as 1481 tests and run as 1608 pytest cases**: 1299 Test IDs +(1308 tests, 1433 cases) pass, 146 (146 tests, 148 cases) are gated behind `RUN_DEVIATIONS`, and 27 (27 tests, 27 cases) cannot be run at all. The three groups are disjoint: two Test IDs have a gated part and a passing part — `TI1`, as two functions, and `TO/endpoint-affects-host`, as one parametrized test — and are counted with the gated, @@ -45,16 +45,16 @@ the Test IDs come from `uts/rest/unit` (503 tests, 536 cases), 482 from `uts/rest/integration` (84, 122), 73 from `uts/realtime/integration` (73, 95) and 17 from `uts/objects/integration` (17, 29); 8 of the REST integration ids (8, 8), 30 of the realtime ones (30, 30) and 5 of the objects ones (5, 5) come from the `proxy` package -within each. Of the gated Test IDs 64 are REST and 84 realtime, which is 65 REST cases -and 85 realtime. None of the 339 objects Test IDs is gated or unrunnable. +within each. Of the gated Test IDs 64 are REST and 82 realtime, which is 65 REST cases +and 83 realtime. None of the 339 objects Test IDs is gated or unrunnable. A further 148 pytest cases cover the harness itself and are not derived from a specification: 130 under `helpers/`, for the mocks, and 18 under `objects/helpers/`, for the objects builders and the synced-channel mock. -The 137 gated Test IDs that record SDK non-compliance — 137 tests, 139 cases — reduce to -**58 distinct root causes**, 18 on the REST side and 40 on the realtime side. One further +The 135 gated Test IDs that record SDK non-compliance — 135 tests, 137 cases — reduce to +**57 distinct root causes**, 18 on the REST side and 39 on the realtime side. One further defect is recorded below with no test of its own, because the specification's test -cannot discriminate (RTP18a), so the file carries **59 SDK root causes** in all. The +cannot discriminate (RTP18a), so the file carries **58 SDK root causes** in all. The remaining 11 gated Test IDs are specification faults, and reduce to 8. The objects tier records no SDK non-compliance; its specification faults are all corrected or worked around in tests that pass, and reduce to the seven candidate issues O.1–O.7, none of @@ -1283,26 +1283,6 @@ for the connection"; `test_rtn13c_deferred_ping_timeout` fails with ### Channels -#### `detach()` never returns when the connection is not CONNECTED — 2 tests - -**Spec point:** RTL5l. - -When the connection is in any state other than CONNECTED and no earlier channel-state -condition applies, the channel must transition immediately to DETACHED. Instead `detach()` -requests DETACHING, `_check_pending_state()` returns without sending anything because the -connection is not CONNECTED, and `detach()` then awaits the internal state emitter -(`channel.py:212`) for a transition nothing will produce. **The coroutine never returns** -and the channel is left in DETACHING. - -**Tests affected:** `test_rtl5l_detach_not_connected_immediate` and -`test_rtl5l_detach_attached_when_disconnected`, each with a one-second `asyncio.wait_for` -so the hang is reported as a failure rather than a stuck run. Both fail with -`asyncio.exceptions.TimeoutError`. In the second, the assertions that set the scene — the -connection settling in DISCONNECTED with the channel still ATTACHED — pass first, so the -failure is unambiguously the detach. - -**Status:** open bug. A hang, not a wrong value. - #### `set_options()` never returns for an already-ATTACHED channel — 1 test **Spec point:** RTL16a. @@ -2599,13 +2579,6 @@ connection fails. `test/uts/realtime/unit/channels/channel_publish_pending_test.py -k rtn7e` `test/uts/realtime/integration/proxy/connection_resume_test.py -k rtn15j` -**1.3 `detach()` never returns when the connection is not CONNECTED.** RTL5l. -RTL5l requires an immediate transition to DETACHED. `detach()` requests DETACHING, -`_check_pending_state()` sends nothing because the connection is not CONNECTED, and -`detach()` then awaits the internal state emitter (`channel.py:212`) for a transition -nothing will produce. The coroutine never returns and the channel is stuck in DETACHING. -`test/uts/realtime/unit/channels/channel_detach_test.py -k rtl5l` - **1.4 `set_options()` never returns for an already-ATTACHED channel.** RTL16a. `_attach_impl()` (`channel.py:99`) sends the ATTACH without going through `_request_state(ATTACHING)`, so the channel is still ATTACHED when the server's ATTACHED @@ -3479,8 +3452,8 @@ The header states how many derived tests there are, how many pass, how many are gated and how many cannot run. Those numbers are the check that the file is still true: in pytest cases, the gated count must equal the number of failures under `RUN_DEVIATIONS=1`, and gated plus unrunnable must equal the number of skips without -it. As of this writing that is 150 failures, 1579 passes and 27 skips with the variable -set, and 177 skips and 1579 passes without it, the 1579 being 1431 derived cases and 148 +it. As of this writing that is 148 failures, 1581 passes and 27 skips with the variable +set, and 175 skips and 1581 passes without it, the 1581 being 1433 derived cases and 148 harness ones — 130 under `helpers/` and 18 under `objects/helpers/`. Those figures need the `submodules` checkout CI makes: without it, three `rest/unit/encoding` tests that read the ably-common fixtures skip as well. The tiers are told apart by the junit report's diff --git a/test/uts/realtime/unit/channels/channel_detach_test.py b/test/uts/realtime/unit/channels/channel_detach_test.py index 013f43ba..fe176595 100644 --- a/test/uts/realtime/unit/channels/channel_detach_test.py +++ b/test/uts/realtime/unit/channels/channel_detach_test.py @@ -251,7 +251,6 @@ def on_message_from_client(msg): # UTS: realtime/unit/RTL5l/detach-not-connected-immediate-0 -@deviation async def test_rtl5l_detach_not_connected_immediate(): channel_name = 'test-RTL5l' detach_messages = [] @@ -278,11 +277,13 @@ def on_message_from_client(msg): assert channel.state == ChannelState.DETACHED assert detach_messages == [] - attach_future.cancel() + # The detach leaves the pending attach to fail (RTL4d), which is retrieved so it is not left + # unhandled + with pytest.raises(AblyException): + await asyncio.wait_for(attach_future, OPERATION_TIMEOUT) # UTS: realtime/unit/RTL5l/detach-attached-when-disconnected-1 -@deviation async def test_rtl5l_detach_attached_when_disconnected(): channel_name = 'test-channel' messages_sent = [] From b3271efd2c8baab1596fe0973ce29469a084c55c Mon Sep 17 00:00:00 2001 From: owenpearson Date: Mon, 26 Oct 2026 21:04:06 +0000 Subject: [PATCH 62/71] pubsub: defer a ping from DISCONNECTED and time it from the heartbeat ConnectionManager.ping admitted only CONNECTED and CONNECTING and raised 40000 at once from DISCONNECTED, where RTN13d defers a ping requested while CONNECTING or DISCONNECTED until the connection is CONNECTED, and RTN13b fails it only from INITIALIZED, SUSPENDED, CLOSING, CLOSED or FAILED. It also entered asyncio.wait_for as soon as ping() was called, so the whole connecting period was charged to realtimeRequestTimeout and a deferred ping could time out before its HEARTBEAT went out, where RTN13c with RTN13d counts the timeout from the HEARTBEAT. ping() now admits DISCONNECTED too. A deferred ping waits until its heartbeat is sent, which happens on CONNECTED as before, or until a state that fails pending pings (SUSPENDED, CLOSING, CLOSED, FAILED) fails it, and only then starts the realtimeRequestTimeout wait. This is a behaviour change rather than a new API: a ping from DISCONNECTED that raised 40000 at once now waits for the connection, up to the connectionStateTtl after which it suspends and the ping fails. Wrap the call in asyncio.wait_for to bound it. The three gated RTN13 UTS tests now pass and are un-gated. Co-Authored-By: Claude Opus 5.5 --- UPDATING.md | 13 +++++ ably/pubsub/realtime/connectionmanager.py | 18 +++++-- test/uts/deviations.md | 53 ++++--------------- .../unit/connection/connection_ping_test.py | 13 ++--- 4 files changed, 40 insertions(+), 57 deletions(-) diff --git a/UPDATING.md b/UPDATING.md index d364970e..9595b77f 100644 --- a/UPDATING.md +++ b/UPDATING.md @@ -769,6 +769,19 @@ except AblyException: ... ``` +### `connection.ping()` waits for a disconnected connection + +`await connection.ping()` raised an `AblyException` with code 40000 at once when the connection +was `DISCONNECTED`. In 4.0.0 it waits for the connection to reconnect, as it already did while +`CONNECTING`, and then times out `realtime_request_timeout` after its heartbeat goes out. It +still fails if the connection becomes `SUSPENDED`, `CLOSED` or `FAILED` first, which can be up to +two minutes later. Bound the wait yourself if you need an answer sooner: + +Example 4.0.0 code: +```python +response_time_ms = await asyncio.wait_for(pubsub_realtime_client.connection.ping(), timeout=5) +``` + ## Version 2.x to 3.0.0 The 3.0.0 version of ably-python introduces several breaking changes to improve the realtime experience and align the API with the Ably specification. These include: diff --git a/ably/pubsub/realtime/connectionmanager.py b/ably/pubsub/realtime/connectionmanager.py index bd9a7dcd..cf72e389 100644 --- a/ably/pubsub/realtime/connectionmanager.py +++ b/ably/pubsub/realtime/connectionmanager.py @@ -127,6 +127,8 @@ def __init__(self, id: str, clock: Clock): self.id = id self.clock = clock self.future: asyncio.Future[None] = asyncio.get_running_loop().create_future() + # done once the heartbeat has gone out, which is when the ping's timeout starts (RTN13c) + self.sent: asyncio.Future[None] = asyncio.get_running_loop().create_future() self.start_time: float = clock.monotonic_ms() @property @@ -364,17 +366,25 @@ def fail_queued_messages(self, err) -> None: async def ping(self) -> float: # RTN13b - if self.__state not in (ConnectionState.CONNECTED, ConnectionState.CONNECTING): + if self.__state not in ( + ConnectionState.CONNECTED, + ConnectionState.CONNECTING, + ConnectionState.DISCONNECTED, + ): raise AblyException("Cannot send ping request. Calling ping in invalid state", 400, 40000) # RTN13e: the id tells this ping's echo apart from server heartbeats and other pings pending_ping = PendingPing(get_random_id(), self.clock) self.__pending_pings[pending_ping.id] = pending_ping try: - # RTN13d: while connecting, the heartbeat goes out once the connection is established if self.__state == ConnectionState.CONNECTED: await self.__send_heartbeat(pending_ping) - # RTN13c + else: + # RTN13d: the heartbeat goes out once the connection is established, unless a state + # that fails the ping (RTN13b) comes first + await asyncio.wait( + (pending_ping.sent, pending_ping.future), return_when=asyncio.FIRST_COMPLETED) + # RTN13c: counted from the heartbeat going out, not from the call await asyncio.wait_for(pending_ping.future, self.__timeout_in_secs) except asyncio.TimeoutError: raise AblyException("Timeout waiting for ping response", 504, 50003) from None @@ -384,6 +394,8 @@ async def ping(self) -> float: async def __send_heartbeat(self, pending_ping: PendingPing) -> None: pending_ping.start_time = self.clock.monotonic_ms() + if not pending_ping.sent.done(): + pending_ping.sent.set_result(None) try: await self.send_protocol_message(pending_ping.message) except Exception as error: diff --git a/test/uts/deviations.md b/test/uts/deviations.md index d07ab8ed..8f901f15 100644 --- a/test/uts/deviations.md +++ b/test/uts/deviations.md @@ -33,8 +33,8 @@ parametrized over a table of fixtures the specification gives inline; and five `objects/unit` tests are parametrized, three over a table and two over the scenarios the specification loops through. That turns 1481 derived tests into 1608 pytest cases. -Of **1472 Test IDs, derived as 1481 tests and run as 1608 pytest cases**: 1299 Test IDs -(1308 tests, 1433 cases) pass, 146 (146 tests, 148 cases) are gated behind +Of **1472 Test IDs, derived as 1481 tests and run as 1608 pytest cases**: 1302 Test IDs +(1311 tests, 1436 cases) pass, 143 (143 tests, 145 cases) are gated behind `RUN_DEVIATIONS`, and 27 (27 tests, 27 cases) cannot be run at all. The three groups are disjoint: two Test IDs have a gated part and a passing part — `TI1`, as two functions, and `TO/endpoint-affects-host`, as one parametrized test — and are counted with the gated, @@ -45,16 +45,16 @@ the Test IDs come from `uts/rest/unit` (503 tests, 536 cases), 482 from `uts/rest/integration` (84, 122), 73 from `uts/realtime/integration` (73, 95) and 17 from `uts/objects/integration` (17, 29); 8 of the REST integration ids (8, 8), 30 of the realtime ones (30, 30) and 5 of the objects ones (5, 5) come from the `proxy` package -within each. Of the gated Test IDs 64 are REST and 82 realtime, which is 65 REST cases -and 83 realtime. None of the 339 objects Test IDs is gated or unrunnable. +within each. Of the gated Test IDs 64 are REST and 79 realtime, which is 65 REST cases +and 80 realtime. None of the 339 objects Test IDs is gated or unrunnable. A further 148 pytest cases cover the harness itself and are not derived from a specification: 130 under `helpers/`, for the mocks, and 18 under `objects/helpers/`, for the objects builders and the synced-channel mock. -The 135 gated Test IDs that record SDK non-compliance — 135 tests, 137 cases — reduce to -**57 distinct root causes**, 18 on the REST side and 39 on the realtime side. One further +The 132 gated Test IDs that record SDK non-compliance — 132 tests, 134 cases — reduce to +**56 distinct root causes**, 18 on the REST side and 38 on the realtime side. One further defect is recorded below with no test of its own, because the specification's test -cannot discriminate (RTP18a), so the file carries **58 SDK root causes** in all. The +cannot discriminate (RTP18a), so the file carries **57 SDK root causes** in all. The remaining 11 gated Test IDs are specification faults, and reduce to 8. The objects tier records no SDK non-compliance; its specification faults are all corrected or worked around in tests that pass, and reduce to the seven candidate issues O.1–O.7, none of @@ -1253,34 +1253,6 @@ that the first merely permitted. the adapted RTN25 unit test round to the specification's option A, so that test's adaptation goes with the fix. -#### `ping()` rejects DISCONNECTED instead of deferring, and charges the wait to the caller — 3 tests - -**Spec points:** RTN13b, RTN13c, RTN13d. - -Two defects in the same method. - -- RTN13b errors only for INITIALIZED, SUSPENDED, CLOSING, CLOSED and FAILED, and RTN13d - defers a ping requested while CONNECTING *or DISCONNECTED* until the connection is - CONNECTED. `ConnectionManager.ping` (`connectionmanager.py:362`) admits only CONNECTED - and CONNECTING and raises `AblyException("Cannot send ping request. Calling ping in - invalid state", 400, 40000)` for DISCONNECTED. Note that - `deferred-ping-error-suspended-5` would otherwise pass for the wrong reason: its only - stated assertion is that an error arrives, and one does — just immediately, rather than - when the connection suspends. -- RTN13c with RTN13d requires a deferred ping's timeout to run "from when the HEARTBEAT is - actually sent, not when `ping()` is called". `ping()` enters - `asyncio.wait_for(pending_ping.future, self.__timeout_in_secs)` (`:375`) as soon as it is - called, so the whole CONNECTING period is charged against it and a ping requested while - connecting can expire before its HEARTBEAT has gone out. - -**Tests affected:** `test_rtn13d_ping_deferred_disconnected` and -`test_rtn13b_deferred_ping_error_suspended` fail with "the ping errored instead of waiting -for the connection"; `test_rtn13c_deferred_ping_timeout` fails with -`assert 0.096… >= (0.4 * 0.9)` — the error arrived 96 ms after CONNECTED where a 400 ms -`realtimeRequestTimeout` should have run from that point. - -**Status:** open bug. - ### Channels #### `set_options()` never returns for an already-ATTACHED channel — 1 test @@ -2765,13 +2737,6 @@ NACK as `reason`, where the spec requires `resumed` true and a 91004 naming the the NACK as `cause`. Local to one method. `test/uts/realtime/unit/presence/realtime_presence_reentry_test.py -k rtp17e` -**3.14 `ping()` rejects DISCONNECTED, and charges the connect wait to the caller's timeout.** -RTN13b, RTN13c, RTN13d. `ConnectionManager.ping` (`:362`) admits only CONNECTED and -CONNECTING where RTN13d requires a ping from DISCONNECTED to be deferred; and `:375` enters -`asyncio.wait_for` as soon as `ping()` is called, so a ping requested while CONNECTING can -expire before its HEARTBEAT has gone out. -`test/uts/realtime/unit/connection/connection_ping_test.py -k "rtn13b_deferred or rtn13c or rtn13d"` - **3.15 A failed RTN22 reauth leaves no trace on the connection.** RSA4c1, RSA4c3. `websockettransport.py:170-175` awaits `auth.authorize()` inside a bare `except Exception` that only logs, so nothing reaches `on_error_from_authorize` and `errorReason` stays as it @@ -3452,8 +3417,8 @@ The header states how many derived tests there are, how many pass, how many are gated and how many cannot run. Those numbers are the check that the file is still true: in pytest cases, the gated count must equal the number of failures under `RUN_DEVIATIONS=1`, and gated plus unrunnable must equal the number of skips without -it. As of this writing that is 148 failures, 1581 passes and 27 skips with the variable -set, and 175 skips and 1581 passes without it, the 1581 being 1433 derived cases and 148 +it. As of this writing that is 145 failures, 1584 passes and 27 skips with the variable +set, and 172 skips and 1584 passes without it, the 1584 being 1436 derived cases and 148 harness ones — 130 under `helpers/` and 18 under `objects/helpers/`. Those figures need the `submodules` checkout CI makes: without it, three `rest/unit/encoding` tests that read the ably-common fixtures skip as well. The tiers are told apart by the junit report's diff --git a/test/uts/realtime/unit/connection/connection_ping_test.py b/test/uts/realtime/unit/connection/connection_ping_test.py index dc3abd62..e7a503cd 100644 --- a/test/uts/realtime/unit/connection/connection_ping_test.py +++ b/test/uts/realtime/unit/connection/connection_ping_test.py @@ -20,7 +20,6 @@ realtime_client, ) from test.uts.helpers.clock import FakeClock, settle -from test.uts.helpers.deviations import deviation from test.uts.helpers.mock_websocket import ( ERROR_MESSAGE, MockWebSocket, @@ -284,7 +283,6 @@ async def test_rtn13d_ping_deferred_connecting(): # UTS: realtime/unit/RTN13d/ping-deferred-disconnected-1 -@deviation async def test_rtn13d_ping_deferred_disconnected(): clock = FakeClock() attempts = [] @@ -310,8 +308,7 @@ def on_connection_attempt(conn): ping_task = asyncio.ensure_future(client.connection.ping()) await settle() - # RTN13d: a ping requested while DISCONNECTED waits for the connection. - # ably-python admits only CONNECTED and CONNECTING and errors at once + # RTN13d: a ping requested while DISCONNECTED waits for the connection deferred = not ping_task.done() await clock.advance(600) @@ -348,7 +345,6 @@ async def test_rtn13b_deferred_ping_error_failed(): # UTS: realtime/unit/RTN13b/deferred-ping-error-suspended-5 -@deviation async def test_rtn13b_deferred_ping_error_suspended(): clock = FakeClock() mock_ws = MockWebSocket(on_connection_attempt=lambda conn: conn.respond_with_dns_error()) @@ -362,8 +358,7 @@ async def test_rtn13b_deferred_ping_error_suspended(): ping_task = asyncio.ensure_future(client.connection.ping()) await settle() - # RTN13d: the ping waits for the connection rather than failing where the - # specification expects it to be deferred + # RTN13d: the ping waits for the connection rather than failing at once deferred = not ping_task.done() await clock.advance(CONNECTION_STATE_TTL + 1000) @@ -376,7 +371,6 @@ async def test_rtn13b_deferred_ping_error_suspended(): # UTS: realtime/unit/RTN13c/deferred-ping-timeout-1 -@deviation async def test_rtn13c_deferred_ping_timeout(): clock = FakeClock() attempts = [] @@ -404,6 +398,5 @@ async def test_rtn13c_deferred_ping_timeout(): assert error is not None assert 'timeout' in str(error).lower() # RTN13c with RTN13d: a deferred ping's timeout runs from the HEARTBEAT going - # out, not from the call. ably-python starts `wait_for` when `ping()` is - # called, so the ping expires while the connection is still establishing + # out, not from the call assert failed_after >= timeout_in_secs * 0.9 From 72d00f80c09ce26d743bafbde9276f02a70efd03 Mon Sep 17 00:00:00 2001 From: owenpearson Date: Mon, 26 Oct 2026 21:05:38 +0000 Subject: [PATCH 63/71] pubsub: return from set_options once an attached channel reattaches set_options on an ATTACHED channel whose params or modes changed sent the ATTACH through _attach_impl() and then awaited the internal state emitter. The channel never left ATTACHED, so the server's ATTACHED took the RTL12 branch of _on_message, which emits "update" on the public emitter only, and set_options never returned. It raised the state change's reason, None included, for SUSPENDED and FAILED, and returned for any other state. The reattach now goes through _request_state(ATTACHING), so the channel passes through ATTACHING and the ATTACHED that answers ends the wait, which then raises as attach() does unless the channel attached (RTL16a, RTL4d). On an ATTACHING channel the new ATTACH goes out and the wait ends with the pending attach. Not a breaking change; a call that hung now returns. The RTL16a UTS test is un-gated, and the two attach tests that ran set_options as a task and cancelled it now await it. Co-Authored-By: Claude Opus 5.5 --- ably/pubsub/realtime/channel.py | 6 +- test/uts/deviations.md | 67 +++++-------------- .../unit/channels/channel_attach_test.py | 4 +- .../unit/channels/channel_options_test.py | 11 --- 4 files changed, 21 insertions(+), 67 deletions(-) diff --git a/ably/pubsub/realtime/channel.py b/ably/pubsub/realtime/channel.py index e115d89e..1fd9e29c 100644 --- a/ably/pubsub/realtime/channel.py +++ b/ably/pubsub/realtime/channel.py @@ -106,10 +106,10 @@ async def set_options(self, channel_options: ChannelOptions) -> None: self._set_options(channel_options) if should_reattach: - self._attach_impl() + # RTL16a: through ATTACHING, so that the ATTACHED answering the ATTACH ends the wait + self._request_state(ChannelState.ATTACHING) state_change = await self.__internal_state_emitter.once_async() - if state_change.current in (ChannelState.SUSPENDED, ChannelState.FAILED): - raise state_change.reason + self.__raise_unless_attached(state_change) # RTL4 async def attach(self) -> None: diff --git a/test/uts/deviations.md b/test/uts/deviations.md index 8f901f15..936d3b54 100644 --- a/test/uts/deviations.md +++ b/test/uts/deviations.md @@ -33,8 +33,8 @@ parametrized over a table of fixtures the specification gives inline; and five `objects/unit` tests are parametrized, three over a table and two over the scenarios the specification loops through. That turns 1481 derived tests into 1608 pytest cases. -Of **1472 Test IDs, derived as 1481 tests and run as 1608 pytest cases**: 1302 Test IDs -(1311 tests, 1436 cases) pass, 143 (143 tests, 145 cases) are gated behind +Of **1472 Test IDs, derived as 1481 tests and run as 1608 pytest cases**: 1303 Test IDs +(1312 tests, 1437 cases) pass, 142 (142 tests, 144 cases) are gated behind `RUN_DEVIATIONS`, and 27 (27 tests, 27 cases) cannot be run at all. The three groups are disjoint: two Test IDs have a gated part and a passing part — `TI1`, as two functions, and `TO/endpoint-affects-host`, as one parametrized test — and are counted with the gated, @@ -45,16 +45,16 @@ the Test IDs come from `uts/rest/unit` (503 tests, 536 cases), 482 from `uts/rest/integration` (84, 122), 73 from `uts/realtime/integration` (73, 95) and 17 from `uts/objects/integration` (17, 29); 8 of the REST integration ids (8, 8), 30 of the realtime ones (30, 30) and 5 of the objects ones (5, 5) come from the `proxy` package -within each. Of the gated Test IDs 64 are REST and 79 realtime, which is 65 REST cases -and 80 realtime. None of the 339 objects Test IDs is gated or unrunnable. +within each. Of the gated Test IDs 64 are REST and 78 realtime, which is 65 REST cases +and 79 realtime. None of the 339 objects Test IDs is gated or unrunnable. A further 148 pytest cases cover the harness itself and are not derived from a specification: 130 under `helpers/`, for the mocks, and 18 under `objects/helpers/`, for the objects builders and the synced-channel mock. -The 132 gated Test IDs that record SDK non-compliance — 132 tests, 134 cases — reduce to -**56 distinct root causes**, 18 on the REST side and 38 on the realtime side. One further +The 131 gated Test IDs that record SDK non-compliance — 131 tests, 133 cases — reduce to +**55 distinct root causes**, 18 on the REST side and 37 on the realtime side. One further defect is recorded below with no test of its own, because the specification's test -cannot discriminate (RTP18a), so the file carries **57 SDK root causes** in all. The +cannot discriminate (RTP18a), so the file carries **56 SDK root causes** in all. The remaining 11 gated Test IDs are specification faults, and reduce to 8. The objects tier records no SDK non-compliance; its specification faults are all corrected or worked around in tests that pass, and reduce to the seven candidate issues O.1–O.7, none of @@ -1255,33 +1255,7 @@ adaptation goes with the fix. ### Channels -#### `set_options()` never returns for an already-ATTACHED channel — 1 test - -**Spec point:** RTL16a. - -When `params` or `modes` are supplied to `setOptions` on an attached channel, the channel -must reattach, pass through ATTACHING, return to ATTACHED, and `setOptions` must resolve. -Instead it hangs, and the root cause is a single missing call. - -`set_options` (`channel.py:93-102`) calls `_attach_impl()` and then awaits -`self.__internal_state_emitter.once_async()`. `_attach_impl()` sends the ATTACH **without -going through `_request_state(ChannelState.ATTACHING)`**, so the channel is still ATTACHED -when the server's ATTACHED arrives. `_on_message` therefore takes the RTL12 branch -(`:722-726`), which emits `update` on the *public* emitter and returns. The internal state -emitter is written only by `_notify_state` (`:821`), which that branch never reaches, so -the await has nothing to wake it. Both halves of the defect — no ATTACHING transition, and -no internal event — follow from the one missing `_request_state`. - -Measured: the second ATTACH is sent, the server's ATTACHED is received, the options *are* -stored, and `asyncio.wait_for(channel.set_options(...), 1.0)` raises `TimeoutError`. - -**Tests affected:** `test_rtl16a_triggers_reattach`. It also constrains -`test_rtl4c1_includes_channel_serial` and `test_rtl4j_attach_resume_flag_not_set`, which run -`set_options` as a task and cancel it — recorded under Adapted Tests. - -**Status:** open bug. A hang, not a wrong value. - -#### A server-initiated DETACHED discards its error, which `set_options` and `detach` then raise as `TypeError` — 3 tests +#### A server-initiated DETACHED discards its error, which `detach` then raises as `TypeError` — 3 tests **Spec points:** RTL24, RTL4c, with RTL13a and RTL13b. @@ -1292,8 +1266,8 @@ An attach rejected by a DETACHED carrying an `ErrorInfo` must fail with that err `self._notify_state(ChannelState.SUSPENDED)` (`channel.py:735`), passing **no reason**, so the message's error is dropped. The same shape applies to RTL13a's `_request_state(ATTACHING)` and RTL13b's `_notify_state(SUSPENDED)`. -2. `attach()` then raises its 90001 fallback, having no reason to raise, but - `set_options` and `detach` reach `raise state_change.reason` with `reason` `None`, which +2. `attach()` and `set_options` then raise their 90001 fallback, having no reason to + raise, but `detach` reaches `raise state_change.reason` with `reason` `None`, which Python reports as `TypeError: exceptions must derive from BaseException` rather than as an `AblyException`. @@ -1306,8 +1280,8 @@ earlier. The clearing half of RTL4c is still covered by instead and passes. One adapted test — `test_rtl13a_attached_reattach_triggered` — asserts `reason is None`, and will need revisiting once the reason is carried through. -**Status:** open bug, in two parts. The `raise None` is worth fixing in `set_options` and -`detach` on its own, as it is in `attach`, even before the reason is plumbed through: a +**Status:** open bug, in two parts. The `raise None` is worth fixing in `detach` on its +own, as it is in `attach` and `set_options`, even before the reason is plumbed through: a missing reason should give an `AblyException`, not a `TypeError`. #### An ATTACHED received while DETACHING or DETACHED is ignored — 2 tests @@ -2551,15 +2525,6 @@ connection fails. `test/uts/realtime/unit/channels/channel_publish_pending_test.py -k rtn7e` `test/uts/realtime/integration/proxy/connection_resume_test.py -k rtn15j` -**1.4 `set_options()` never returns for an already-ATTACHED channel.** RTL16a. -`_attach_impl()` (`channel.py:99`) sends the ATTACH without going through -`_request_state(ATTACHING)`, so the channel is still ATTACHED when the server's ATTACHED -arrives; `_on_message` takes the RTL12 branch (`:722-726`), which emits `update` on the -*public* emitter, while `set_options` awaits the *internal* one, written only by -`_notify_state`. The options are stored; the call never resolves. Adding the -`_request_state` fixes both halves. -`test/uts/realtime/unit/channels/channel_options_test.py -k rtl16a` - **1.5 A presence action on a DETACHED channel re-attaches and then hangs.** RTL11, RTP8g. Both require an immediate error with nothing sent. `_enter_or_update_client` groups DETACHED with INITIALIZED (`presence.py:258-264`), starts an implicit attach and queues the message, @@ -2570,8 +2535,8 @@ have the grouping, so the SDK is inconsistent with itself. ### Tier 2 — the wrong error, or an error where there should be none **2.1 `raise state_change.reason` raises `TypeError` when the reason is `None`.** RTL24, -RTL4c, RTL13b, RTP16c. Two sites — `set_options` and `detach`; `attach` falls back to a -90001 — raise whatever the state change carries, and the state change often +RTL4c, RTL13b, RTP16c. `detach` raises whatever the state change carries, where `attach` +and `set_options` fall back to a 90001, and the state change often carries nothing, because a server-initiated DETACHED's error is discarded at `:735`, `_request_state(ATTACHING)` and `_notify_state(SUSPENDED)` being called with no reason. The caller gets `TypeError: exceptions must derive from BaseException` instead of an @@ -3417,8 +3382,8 @@ The header states how many derived tests there are, how many pass, how many are gated and how many cannot run. Those numbers are the check that the file is still true: in pytest cases, the gated count must equal the number of failures under `RUN_DEVIATIONS=1`, and gated plus unrunnable must equal the number of skips without -it. As of this writing that is 145 failures, 1584 passes and 27 skips with the variable -set, and 172 skips and 1584 passes without it, the 1584 being 1436 derived cases and 148 +it. As of this writing that is 144 failures, 1585 passes and 27 skips with the variable +set, and 171 skips and 1585 passes without it, the 1585 being 1437 derived cases and 148 harness ones — 130 under `helpers/` and 18 under `objects/helpers/`. Those figures need the `submodules` checkout CI makes: without it, three `rest/unit/encoding` tests that read the ably-common fixtures skip as well. The tiers are told apart by the junit report's diff --git a/test/uts/realtime/unit/channels/channel_attach_test.py b/test/uts/realtime/unit/channels/channel_attach_test.py index d59519d9..4401c58d 100644 --- a/test/uts/realtime/unit/channels/channel_attach_test.py +++ b/test/uts/realtime/unit/channels/channel_attach_test.py @@ -427,7 +427,7 @@ def on_message_from_client(msg): assert len(attach_messages) == 2 assert attach_messages[0].get('channelSerial') is None assert attach_messages[1].get('channelSerial') == 'serial-from-server-1' - set_options.cancel() + await asyncio.wait_for(set_options, OPERATION_TIMEOUT) # UTS: realtime/unit/RTL4f/timeout-to-suspended-0 @@ -561,4 +561,4 @@ def on_message_from_client(msg): assert len(attach_messages) == 2 assert not attach_messages[0].get('flags', 0) & Flag.ATTACH_RESUME assert not attach_messages[1].get('flags', 0) & Flag.ATTACH_RESUME - set_options.cancel() + await asyncio.wait_for(set_options, OPERATION_TIMEOUT) diff --git a/test/uts/realtime/unit/channels/channel_options_test.py b/test/uts/realtime/unit/channels/channel_options_test.py index 40b85f1b..88a72b33 100644 --- a/test/uts/realtime/unit/channels/channel_options_test.py +++ b/test/uts/realtime/unit/channels/channel_options_test.py @@ -214,16 +214,7 @@ async def test_rtl16_set_options_updates(): # UTS: realtime/unit/RTL16a/triggers-reattach-0 -@deviation async def test_rtl16a_triggers_reattach(): - # DEVIATION: `set_options` never returns for an attached channel. It calls - # `_attach_impl()` directly and then awaits the internal state emitter - # (`ably/realtime/channel.py:99-102`), but the server's ATTACHED arrives while the - # channel is still ATTACHED, so `_on_message` takes the RTL12 branch (`:722-726`) and - # emits `update` on the public emitter alone. Nothing ever reaches the internal - # emitter, so the await hangs. Because `_attach_impl()` is called in place of - # `_request_state(ATTACHING)`, no ATTACHING state change is emitted either. The - # options themselves are stored, before the reattach is requested. channel_name = 'test-RTL16a' attach_messages = [] @@ -242,8 +233,6 @@ def record(change): channel.on(record) new_options = ChannelOptions(params={'rewind': '1'}) - # The specification awaits this without a deadline; a deadline is what turns the - # hang into a failure rather than a stuck test run await asyncio.wait_for(channel.set_options(new_options), OPERATION_TIMEOUT) assert any(change.current == ChannelState.ATTACHING for change in state_changes) From 9ec2366870e7f4b49319dcab754f2cef2af8629c Mon Sep 17 00:00:00 2001 From: owenpearson Date: Wed, 28 Oct 2026 13:20:32 +0000 Subject: [PATCH 64/71] pubsub: take the millisecond timing options in seconds, as *_seconds TO3l1, TO3l2, TO3l7, TO3l10 and TO3l11 count disconnectedRetryTimeout, suspendedRetryTimeout, channelRetryTimeout, fallbackRetryTimeout and realtimeRequestTimeout in milliseconds, and ably-python took realtime_request_timeout, disconnected_retry_timeout, suspended_retry_timeout, channel_retry_timeout and fallback_retry_timeout in milliseconds too. Its three HTTP timing options took seconds, so a client's options came in two units with nothing in their names to say which. The five options now take seconds, as an int or a float, and are renamed realtime_request_timeout_seconds, disconnected_retry_timeout_seconds, suspended_retry_timeout_seconds, channel_retry_timeout_seconds and fallback_retry_timeout_seconds, defaulting to 10, 15, 30, 15 and 600. They read back from Options under the new names. The clock still counts milliseconds, so Options converts each one in a single place, seconds_to_ms, behind a private _