Skip to content
25 changes: 15 additions & 10 deletions aws_advanced_python_wrapper/aio/host_list_provider.py
Original file line number Diff line number Diff line change
Expand Up @@ -172,12 +172,14 @@ async def stop(self) -> None:
class AsyncAuroraHostListProvider:
"""Aurora topology discovery over an async driver connection.

Runs an Aurora-topology query (e.g., ``SELECT server_id,
session_id = 'MASTER_SESSION_ID' AS is_writer, ... FROM
aurora_replica_status``) and returns a tuple of :class:`HostInfo`
objects. The exact SQL and parsing live on the shared sync
Runs an Aurora-topology query (e.g., ``SELECT server_id, session_id
OPERATOR(pg_catalog.=) 'MASTER_SESSION_ID' AS is_writer, ... FROM
pg_catalog.aurora_replica_status()``) and returns a tuple of
:class:`HostInfo` objects. The exact SQL and parsing live on the shared sync
:class:`TopologyAwareDatabaseDialect` classes (reused rather than
duplicated).
duplicated). Note the schema qualification: every function and operator in a
PostgreSQL query the driver issues internally is pinned to ``pg_catalog`` so
the session's ``search_path`` cannot redirect it.

Refresh flow (N.1b, matches sync RdsHostListProvider -> ClusterTopologyMonitor):

Expand All @@ -199,18 +201,21 @@ def __init__(
props: Properties,
driver_dialect: AsyncDriverDialect,
topology_query: str = (
"SELECT SERVER_ID, SESSION_ID = 'MASTER_SESSION_ID' AS IS_WRITER "
"FROM aurora_replica_status() "
"WHERE EXTRACT(EPOCH FROM (NOW() - LAST_UPDATE_TIMESTAMP)) <= 300 "
"OR SESSION_ID = 'MASTER_SESSION_ID' "
"SELECT SERVER_ID, "
"SESSION_ID OPERATOR(pg_catalog.=) 'MASTER_SESSION_ID' AS IS_WRITER "
"FROM pg_catalog.aurora_replica_status() "
"WHERE EXTRACT(EPOCH FROM (pg_catalog.NOW() "
"OPERATOR(pg_catalog.-) LAST_UPDATE_TIMESTAMP)) "
"OPERATOR(pg_catalog.<=) 300 "
"OR SESSION_ID OPERATOR(pg_catalog.=) 'MASTER_SESSION_ID' "
# A reader whose replica status hasn't reported yet has a NULL
# LAST_UPDATE_TIMESTAMP -- the freshness check above is NULL
# (not TRUE) for it, so without this clause every such reader is
# dropped and the topology collapses to writer-only. The failover
# host list is then stranded at one host and a writer outage
# yields FailoverFailedError (fail_from_reader_to_writer,
# failover_with_iam, failover_with_secrets_manager). Mirrors the
# sync Aurora-PG topology query (database_dialect.py:505-511).
# sync Aurora-PG topology query.
"OR LAST_UPDATE_TIMESTAMP IS NULL"
),
cluster_id: Optional[str] = None,
Expand Down
12 changes: 10 additions & 2 deletions aws_advanced_python_wrapper/utils/properties.py
Original file line number Diff line number Diff line change
Expand Up @@ -913,11 +913,19 @@ def log_properties(props: Properties):

return f"\n{props}"

_SECRET_NAME_MARKERS = ("password", "passwd")

@staticmethod
def _is_secret_property(key: str) -> bool:
lowered = key.lower()
return any(marker in lowered for marker in PropertiesUtils._SECRET_NAME_MARKERS)

@staticmethod
def mask_properties(props: Properties) -> Properties:
masked_properties = Properties(props.copy())
if WrapperProperties.PASSWORD.name in masked_properties:
masked_properties[WrapperProperties.PASSWORD.name] = "***"
for key in masked_properties:
if PropertiesUtils._is_secret_property(key):
masked_properties[key] = "***"

return masked_properties

Expand Down
24 changes: 24 additions & 0 deletions docs/GettingStarted.md
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,30 @@ Some features pull in additional packages when used with the asyncio API:
- The async Federated Authentication and Okta Authentication plugins use [aiohttp](https://docs.aiohttp.org/) for their HTTP flows: `pip install aiohttp`.
- Async SQLAlchemy (`create_async_engine`) requires SQLAlchemy's asyncio support, which includes `greenlet`: `pip install "sqlalchemy[asyncio]"`.

### Asyncio on Windows

Python's default event loop on Windows is `ProactorEventLoop`, and Psycopg's async
connection refuses to run on it:

```
psycopg.InterfaceError: Psycopg cannot use the 'ProactorEventLoop' to run in async mode
```

Every async PostgreSQL connection fails at connect until a selector event loop is
selected. Do this once, before any async code runs:

```python
import asyncio
import sys

if sys.platform == "win32":
asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy())
```

This is a requirement of the underlying driver rather than of the wrapper, and it does
not apply on Linux or macOS, where the default loop is already a selector loop. See
[Psycopg's asynchronous operations documentation](https://www.psycopg.org/psycopg3/docs/advanced/async.html#async).

## Using the AWS Advanced Python Wrapper

To start using the wrapper with Psycopg, you need to pass Psycopg's connect function to the `AwsWrapperConnection#connect` method as shown in the following example:
Expand Down
15 changes: 14 additions & 1 deletion docs/using-the-python-wrapper/GlobalDatabases.md
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,14 @@ Use the global cluster endpoint:

> **Note:** Add additional plugins as needed for your use case.

> **Warning:** The plugin lists above include `host_monitoring_v2`, which requires a driver that
> supports aborting a connection from a separate thread. The MySQL Connector/Python driver does not,
> so **omit `host_monitoring_v2` when connecting to Aurora MySQL with `mysql-connector-python`** —
> loading it raises `Aborting connections from a separate thread is not supported for the detected
> driver dialect` and the connection never opens. The asynchronous MySQL driver (`aiomysql`) does
> support it. See [Host Monitoring Plugin](./using-plugins/UsingTheHostMonitoringPlugin.md) and the
> [plugin compatibility matrix](./PluginChainCompatibility.md).

### Reader Connections

**Connection String:**
Expand All @@ -53,6 +61,9 @@ Use the cluster reader endpoint:

> **Note:** Add additional plugins as needed for your use case.

> **Warning:** As with writer connections, omit `host_monitoring_v2` when connecting to Aurora MySQL
> with `mysql-connector-python`. See the warning under [Writer Connections](#writer-connections).

## Example Configuration

### PostgreSQL Example
Expand Down Expand Up @@ -101,7 +112,9 @@ from mysql.connector import Connect
with AwsWrapperConnection.connect(
Connect,
"host=my-global-db.global-xyz.global.rds.amazonaws.com database=mydb user=admin password=pwd",
plugins="initial_connection,failover_v2,host_monitoring_v2",
# host_monitoring_v2 is intentionally absent: mysql-connector-python cannot abort a
# connection from a separate thread, so the plugin refuses to load on this driver.
plugins="initial_connection,failover_v2",
wrapper_dialect="global-aurora-mysql",
cluster_id="1",
global_cluster_instance_host_patterns="us-east-1:?.abc123.us-east-1.rds.amazonaws.com,us-west-2:?.def456.us-west-2.rds.amazonaws.com",
Expand Down
2 changes: 1 addition & 1 deletion docs/using-the-python-wrapper/PluginChainCompatibility.md
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ a chain you haven't used before.
|------------------------------|------------------------------------------------|-----|
| `host_monitoring` (EFM v1) | sync MySQL (`mysql-connector-python`) | EFM requires a thread-based connection abort that `mysql-connector-python` doesn't expose. Symptoms: monitor threads hang on shutdown, "Python hangs on exit" when a host is unreachable. Use the chain **without EFM**, or switch to the async driver (`aiomysql`). |
| `host_monitoring_v2` (EFM v2) | sync MySQL (`mysql-connector-python`) | Same root cause as v1 — EFM v2 still depends on thread-based abort. |
| `iam_authentication` | MySQL `use_pure=True` (pure-Python connector) | The pure-Python connector truncates passwords at 255 chars; IAM tokens are typically longer. Expect `int1store requires 0 <= i <= 255` or `struct.error: ubyte format requires 0 <= number <= 255`. See README "Known Limitations". |
| `iam` | MySQL `use_pure=True` (pure-Python connector) | The pure-Python connector truncates passwords at 255 chars; IAM tokens are typically longer. Expect `int1store requires 0 <= i <= 255` or `struct.error: ubyte format requires 0 <= number <= 255`. See README "Known Limitations". |

## Recommended canonical chains

Expand Down
6 changes: 6 additions & 0 deletions docs/using-the-python-wrapper/SqlAlchemySupport.md
Original file line number Diff line number Diff line change
Expand Up @@ -172,6 +172,12 @@ Read/write splitting is **not** supported with SQLAlchemy: the plugins switch in
> [!NOTE]\
> Async engines require SQLAlchemy's asyncio support (`pip install "sqlalchemy[asyncio]"`, which includes `greenlet`), plus the async driver: Psycopg for PostgreSQL, [aiomysql](https://github.com/aio-libs/aiomysql) for MySQL.

> [!IMPORTANT]\
> **On Windows**, select a selector event loop before running any async code, or every
> async PostgreSQL connection fails with `Psycopg cannot use the 'ProactorEventLoop' to
> run in async mode` — `ProactorEventLoop` is Python's default there. See
> [Asyncio on Windows](../GettingStarted.md#asyncio-on-windows).

The wrapper exposes a native async path. `create_async_engine` works with the URL-based dialect:

```python
Expand Down
2 changes: 1 addition & 1 deletion docs/using-the-python-wrapper/UsingThePythonWrapper.md
Original file line number Diff line number Diff line change
Expand Up @@ -98,7 +98,7 @@ The AWS Advanced Python Wrapper has several built-in plugins that are available
| [AWS Secrets Manager Plugin](./using-plugins/UsingTheAwsSecretsManagerPlugin.md) | `aws_secrets_manager` | Any database | Enables fetching database credentials from the AWS Secrets Manager service. | [Boto3 - AWS SDK for Python](https://aws.amazon.com/sdk-for-python/) and [valid AWS credentials](./AwsCredentials.md) |
| [Federated Authentication Plugin](./using-plugins/UsingTheFederatedAuthenticationPlugin.md) | `federated_auth` | Any database | Enables users to authenticate via Federated Identity and then database access via IAM. | [Boto3 - AWS SDK for Python](https://aws.amazon.com/sdk-for-python/) and [valid AWS credentials](./AwsCredentials.md) |
| [Okta Authentication Plugin](./using-plugins/UsingTheOktaAuthenticationPlugin.md) | `okta` | Aurora, RDS Multi-AZ DB Cluster | Enables users to authenticate using Federated Identity and then connect to their Amazon Aurora Cluster using AWS Identity and Access Management (IAM). | [Boto3 - AWS SDK for Python](https://aws.amazon.com/sdk-for-python/) and [valid AWS credentials](./AwsCredentials.md) |
| [Custom Endpoint Plugin](./using-plugins/UsingTheCustomEndpointPlugin.md) | `customEndpoint` | Aurora, <br>RDS Instance | Enables custom endpoint support. | [Boto3 - AWS SDK for Python](https://aws.amazon.com/sdk-for-python/) and [valid AWS credentials](./AwsCredentials.md) |
| [Custom Endpoint Plugin](./using-plugins/UsingTheCustomEndpointPlugin.md) | `custom_endpoint` | Aurora, <br>RDS Instance | Enables custom endpoint support. | [Boto3 - AWS SDK for Python](https://aws.amazon.com/sdk-for-python/) and [valid AWS credentials](./AwsCredentials.md) |
| Aurora Stale DNS Plugin | `stale_dns` | Aurora | Prevents incorrectly opening a new connection to an old writer host when DNS records have not yet updated after a recent failover event. <br><br> :warning:**Note:** Contrary to `failover` plugin, `stale_dns` plugin doesn't implement failover support itself. It helps to eliminate opening wrong connections to an old writer host after cluster failover is completed. <br><br> :warning:**Note:** This logic is already included in `failover` plugin so you can omit using both plugins at the same time. | None |
| [Aurora Connection Tracker Plugin](./using-plugins/UsingTheAuroraConnectionTrackerPlugin.md) | `aurora_connection_tracker` | Aurora | Tracks all the opened connections. In the event of a cluster failover, the plugin will close all the impacted connections to the host. This plugin is enabled by default. | None |
| [Read Write Splitting Plugin](./using-plugins/UsingTheReadWriteSplittingPlugin.md) | `read_write_splitting` | Aurora | Enables read write splitting functionality where users can switch between database reader and writer instances. | None |
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,9 @@

When using [Aurora Global Databases](../GlobalDatabases.md), an application may only be able to reach a subset of the regions the global cluster spans (for example, because of network routing, VPC peering, or security constraints). The `gdb_accessible_regions` property restricts the AWS Advanced Python Wrapper to a set of reachable AWS regions, excluding hosts in all other regions from host selection.

> [!IMPORTANT]\
> **Currently supported in the synchronous wrapper only.**

## `gdb_accessible_regions`

| Property | Value | Default |
Expand All @@ -25,7 +28,7 @@ The accessible-regions filter is applied **before** all other selection logic

### Fail-loud, not silent fallback

Consistent with the [AWS Advanced Python Wrapper](https://github.com/aws/aws-advanced-python-wrapper), the filter is a **hard restriction**. When the writer is in an inaccessible region, the wrapper raises an error rather than silently connecting to an unreachable or unintended host. When reader filtering leaves no candidates in accessible regions, the wrapper does **not** fall back to the unfiltered host list.
The filter is a **hard restriction**. When the writer is in an inaccessible region, the wrapper raises an error rather than silently connecting to an unreachable or unintended host. When reader filtering leaves no candidates in accessible regions, the wrapper does **not** fall back to the unfiltered host list.

## Example

Expand All @@ -36,7 +39,7 @@ from psycopg import Connection
with AwsWrapperConnection.connect(
Connection.connect,
"host=my-global-db.global-xyz.global.rds.amazonaws.com dbname=mydb user=admin password=pwd",
plugins="initial_connection,failover2,efm2",
plugins="initial_connection,failover_v2,host_monitoring_v2",
wrapper_dialect="global-aurora-pg",
cluster_id="1",
global_cluster_instance_host_patterns="us-east-1:?.abc123.us-east-1.rds.amazonaws.com,us-west-2:?.def456.us-west-2.rds.amazonaws.com",
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -69,7 +69,7 @@ from psycopg import Connection
with AwsWrapperConnection.connect(
Connection.connect,
"host=my-cluster.cluster-xyz.us-east-1.rds.amazonaws.com dbname=mydb user=admin password=pwd",
plugins="failover2,efm2",
plugins="failover_v2,host_monitoring_v2",
monitoring_connection_priority="writer-or-reader",
autocommit=True
) as awsconn:
Expand All @@ -87,7 +87,7 @@ from psycopg import Connection
with AwsWrapperConnection.connect(
Connection.connect,
"host=my-global-db.global-xyz.global.rds.amazonaws.com dbname=mydb user=admin password=pwd",
plugins="initial_connection,gdb_failover,efm2",
plugins="initial_connection,gdb_failover,host_monitoring_v2",
wrapper_dialect="global-aurora-pg",
failover_home_region="us-west-2",
global_cluster_instance_host_patterns="us-east-1:?.abc123.us-east-1.rds.amazonaws.com,us-west-2:?.def456.us-west-2.rds.amazonaws.com",
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@ from aws_advanced_python_wrapper.pep249 import Error

params = {
"plugins": "dev",
"dialect": "aurora-pg"
"wrapper_dialect": "aurora-pg"
}
exception: Error = Error("test")
ExceptionSimulatorManager.raise_exception_on_next_connect(exception)
Expand All @@ -45,7 +45,7 @@ from aws_advanced_python_wrapper import AwsWrapperConnection

params = {
"plugins": "dev",
"dialect": "aurora-pg"
"wrapper_dialect": "aurora-pg"
}
exception: RuntimeError = RuntimeError("test")
ExceptionSimulatorManager.raise_exception_on_next_method("Connection.cursor", exception)
Expand Down
Loading