From df5cfc191d7d1a45b9c0a21814563e240d414b6e Mon Sep 17 00:00:00 2001 From: glasstiger Date: Thu, 8 Oct 2026 13:49:25 +0100 Subject: [PATCH 01/29] docs(oidc): document fallback claim lists for acl.oidc.sub.claim and acl.oidc.groups.claim acl.oidc.sub.claim and acl.oidc.groups.claim accept a comma-separated list of claim names in priority order (questdb/questdb-enterprise#1267). - OIDC guide: new "User and group claims" section covering fallback claim lists, the rules for missing, null and empty claims, the startup validation and the "Failed to find required claims" log message. - Entra ID guide: accepting managed identity and service principal (app-only) tokens alongside user logins. - Configuration reference: claim list syntax for both settings; acl.oidc.groups.claim has no default and is required with OIDC enabled. - Configuration reference: rename acl.oidc.pkce.enabled to acl.oidc.pkce.required, the name the server reads. - Changelog: October 2026 entries. --- documentation/changelog.mdx | 11 ++ documentation/configuration/oidc.md | 53 ++++++-- documentation/security/oidc.mdx | 187 ++++++++++++++++++++++++++-- 3 files changed, 234 insertions(+), 17 deletions(-) diff --git a/documentation/changelog.mdx b/documentation/changelog.mdx index 833739bcce..82ca0d16a7 100644 --- a/documentation/changelog.mdx +++ b/documentation/changelog.mdx @@ -8,6 +8,17 @@ description: Recent updates and improvements to the QuestDB documentation. This page tracks significant updates to the QuestDB documentation. +## October 2026 + +### Reference + +- [`acl.oidc.groups.claim`](/docs/configuration/oidc/#acloidcgroupsclaim) and [`acl.oidc.sub.claim`](/docs/configuration/oidc/#acloidcsubclaim) - Documented the comma-separated fallback claim lists in priority order and the startup validation. `acl.oidc.groups.claim` is now listed with no default, as it must be set when OIDC is enabled +- [`acl.oidc.pkce.required`](/docs/configuration/oidc/#acloidcpkcerequired) - Corrected the property name, previously documented as `acl.oidc.pkce.enabled`, which QuestDB ignores + +### Updated + +- [OpenID Connect (OIDC)](/docs/security/oidc/#user-and-group-claims) - New section on how QuestDB reads the principal and the groups from the user information: fallback claim lists, the rules for missing, `null`, and empty claims, the startup checks, and the `Failed to find required claims` log message. The Microsoft Entra ID guide now covers [accepting managed identity and service principal tokens](/docs/security/oidc/#accept-managed-identity-and-service-principal-tokens) next to user logins + ## September 2026 ### QuestDB Enterprise Releases diff --git a/documentation/configuration/oidc.md b/documentation/configuration/oidc.md index e0c5932bdc..9a266c1754 100644 --- a/documentation/configuration/oidc.md +++ b/documentation/configuration/oidc.md @@ -100,13 +100,17 @@ When enabled, the PGWire endpoint supports OIDC authentication. The OAuth2 token should be sent in the password field, while the username field should contain the string `_sso`, or left empty if that is an option. -### acl.oidc.pkce.enabled +### acl.oidc.pkce.required - **Default**: `true` - **Reloadable**: no -Enables or disables PKCE for the Authorization Code Flow. This should always -be enabled in production. The Web Console is not fully secure without it. +Whether the Web Console uses PKCE (Proof Key for Code Exchange) in the +Authorization Code Flow. This should always be enabled in production. The Web +Console is not fully secure without it. + +Earlier versions of this page listed the property as `acl.oidc.pkce.enabled`. +QuestDB ignores that name, so a value set under it has no effect. ### acl.oidc.ropc.flow.enabled @@ -196,6 +200,25 @@ which it connects. ## User and group claims +QuestDB reads the principal and the group memberships of an external user from +the user information: the User Info endpoint's response, or the ID token when +`acl.oidc.groups.encoded.in.token` is `true`. `acl.oidc.sub.claim` and +`acl.oidc.groups.claim` each take a single claim name, or a comma-separated +list of claim names in priority order. QuestDB uses the first claim on the list +that carries a value, so one configuration can accept tokens that carry the +same information under different claim names. For the resolution rules and an +example, see +[User and group claims](/docs/security/oidc/#user-and-group-claims) in the +OIDC guide. + +Claim lists are available since QuestDB Enterprise 4.0.2. Earlier versions +read the whole value as a single claim name, so a list makes every OIDC login +fail. + +With OIDC enabled, QuestDB refuses to start when either setting is empty, when +a setting lists the same claim twice, or when both settings list the same +claim. + ### acl.oidc.cache.ttl - **Default**: `30000` @@ -207,11 +230,17 @@ is validated and user info refreshed. ### acl.oidc.groups.claim -- **Default**: `groups` +- **Default**: none - **Reloadable**: no -The name of the custom claim in the user information that contains the -group memberships of the user. +The claim in the user information that contains the group memberships of the +user, as an array of group names or as a single group name. Required when OIDC +is enabled. + +Accepts a comma-separated list of claims in priority order, such as +`groups,roles`. QuestDB takes the groups from the first claim on the list that +holds at least one group name, and does not combine groups from several +claims. A login is rejected when none of the listed claims holds a group name. ### acl.oidc.groups.encoded.in.token @@ -227,6 +256,12 @@ group memberships directly into the token. - **Default**: `sub` - **Reloadable**: no -The name of the claim in the user information that contains the user's name. -Could be a username, full name, or email. Displayed in the Web Console and -logged for audit purposes. +The claim in the user information that contains the user's principal. Could be +a username, full name, email, or object ID. Displayed in the Web Console, +returned by `current_user()`, and logged for audit purposes. + +Accepts a comma-separated list of claims in priority order, such as +`name,oid,sub`. QuestDB takes the principal from the first claim on the list +that holds a non-empty value. A login is rejected when none of the listed +claims holds one. An empty value does not fall back to the default `sub`: with +OIDC enabled, QuestDB refuses to start. diff --git a/documentation/security/oidc.mdx b/documentation/security/oidc.mdx index 51e5d01ab5..e30a6e576d 100644 --- a/documentation/security/oidc.mdx +++ b/documentation/security/oidc.mdx @@ -580,6 +580,12 @@ with Sender.from_conf(conf) as sender: sender.dataframe(df, table_name="foo", at="ts") ``` +Tokens that the OIDC provider issues to an application, rather than to a user, +often carry the principal and the groups under different claim names. To accept +both kinds of token with one configuration, list the claims of both in +`acl.oidc.sub.claim` and `acl.oidc.groups.claim`, as described in +[User and group claims](#user-and-group-claims). + ## OIDC for the PGWire endpoint If the @@ -666,12 +672,15 @@ QuestDB works the list of external groups out from the User Info response message. If we take the example used earlier, we will see that the message contains a -claim called `groups`. This name is configurable in QuestDB. +claim called `groups`. This name is configurable in QuestDB, and QuestDB can +fall back to other claims when a token does not carry it. See +[User and group claims](#user-and-group-claims). -If the groups claim is missing or it is an empty list, the user cannot access -the database. +If the groups claim is missing or holds no group name, QuestDB rejects the +login. -Although the user is authenticated, they have no permissions at all. +If none of the user's groups is mapped to a QuestDB group, the user is +authenticated, but has no permissions at all. The user has to have at least the `HTTP` permission to be able to successfully login via the [Web Console](/docs/getting-started/web-console/overview/). @@ -709,11 +718,122 @@ be synchronized with the Identity Providers. It should be set accordingly to your organization's policies. -Other parts of the user information, such as the `sub` and the `name` also used -by QuestDB. +## User and group claims + +QuestDB reads two values from the user information of an external user: the +principal, which identifies the user, and the groups the user belongs to. The +user information is the response of the User Info endpoint, or the payload of +the token itself, such as an ID token, when `acl.oidc.groups.encoded.in.token` +is `true`. + +- [`acl.oidc.sub.claim`](/docs/configuration/oidc/#acloidcsubclaim) names the + claim that carries the principal, `sub` by default. The principal identifies + the user in the [Web Console](/docs/getting-started/web-console/overview/), + in the server logs, and in the result of `current_user()`. +- [`acl.oidc.groups.claim`](/docs/configuration/oidc/#acloidcgroupsclaim) + names the claim that carries the groups, as an array of group names or as a + single group name. It has no default and must be set when OIDC is enabled. + +QuestDB rejects the login when the user information carries no principal or no +group. + +### Fallback claim lists + +Both settings accept a comma-separated list of claim names in priority order, +so one configuration can accept tokens that carry the principal and the groups +under different claim names. Tokens issued to users and tokens issued to +applications often differ this way: + +```ini title="server.conf" +acl.oidc.sub.claim=name,oid,sub +acl.oidc.groups.claim=groups,roles +``` + +QuestDB takes the principal from the first claim on the list that carries a +non-empty value, and the groups from the first claim on the list that carries +at least one group name. With the configuration above, QuestDB reads these two +token payloads, shortened to the relevant claims, as follows: + +```json title="User token payload" +{ + "sub": "AAAAAAAAAAAAAAAAAAAAAIkzqFVrSaSaFHy782bbtaQ", + "oid": "6f2d8c4e-31a7-4b5e-9c0d-8e1f2a3b4c5d", + "name": "Jane Smith", + "groups": ["87654321-1234-1234-1234-123456789abc"] +} +``` + +```json title="Application token payload" +{ + "oid": "4c9a4e3b-0d6f-4f43-9a8e-27d2b1f0c5a1", + "roles": ["QuestDB.Ingest"] +} +``` + +| Token | Principal | Groups | +| ----------- | -------------------------- | ------------------------------ | +| User | `Jane Smith`, from `name` | `87654321-...`, from `groups` | +| Application | `4c9a4e3b-...`, from `oid` | `QuestDB.Ingest`, from `roles` | + +The following rules decide which claim QuestDB uses. They apply to a single +claim name as well: + +- The order of the list decides, not the order of the claims in the token. A + token that carries both `groups` and `roles` gets its groups from `groups` + alone. QuestDB does not combine groups from several claims. +- A claim that is missing, `null`, or empty falls through to the next claim on + the list. Empty means an empty string, or an empty array for a groups claim. + The string `"null"` counts as missing too. +- Group names that are `null` or empty are skipped. A groups claim that holds + nothing else falls through to the next claim. +- Only top-level claims count. A claim nested in another claim, such as `name` + inside `address`, is ignored. +- Claim names are case-sensitive. +- A listed claim must have the expected shape: a single value, such as a + string, for the principal, and a single value or a flat array of values for + the groups. A listed claim of any other shape, such as an array in a + principal claim or an object in a groups claim, fails the login, even when a + claim earlier on the list carries a value. +- Claims that are not listed are ignored, whatever their shape. + +:::note + +Claim lists are available since QuestDB Enterprise 4.0.2. Earlier versions read +the whole value as a single claim name, so a list makes every OIDC login fail. + +::: + +### Startup validation + +With OIDC enabled, QuestDB checks both settings at startup. Spaces around claim +names are trimmed and empty entries are skipped, so `name, oid,,sub` reads as +`name,oid,sub`. QuestDB refuses to start when: + +- `acl.oidc.groups.claim` is not set, or either setting is empty: + `required property is not set: acl.oidc.groups.claim`, or the same message + for `acl.oidc.sub.claim`. An empty `acl.oidc.sub.claim` does not fall back + to the default `sub`. +- A setting lists the same claim twice: + `claim is listed more than once in acl.oidc.sub.claim [claim=name]`. +- Both settings list the same claim: + `claim cannot be listed in both acl.oidc.sub.claim and acl.oidc.groups.claim [claim=oid]`. + +### Rejected logins + +When the user information carries none of the listed principal claims, or none +of the listed groups claims, QuestDB rejects the login. HTTP clients receive +`401 Unauthorized`, and PGWire clients receive `invalid username/password`. The +server log names both lists, followed by the user information QuestDB +received: + +```text +Failed to find required claims [subClaims=[name,oid,sub], groupsClaims=[groups,roles], userInfo={...}] +``` -They could be displayed in the [Web Console](/docs/getting-started/web-console/overview/), or appear in -the logs, for example. +Compare the claims in `userInfo` with the two lists to find the claim names +that your provider uses. A listed claim of an unexpected shape, such as an +object in a groups claim, is logged as +`Request failed [error=unexpected input format ...]` instead. ## Configuration options @@ -1339,4 +1459,55 @@ To test, head to the Web Console and login. If all has been wired up well, then login will succeed, and the user will have the access granted to them. +#### Accept managed identity and service principal tokens + +Services that authenticate as themselves, with a managed identity or a service +principal, receive app-only access tokens from Entra ID. These tokens carry no +`name` or `groups` claim. They identify the caller by the object ID of its +service principal in the `oid` claim, and list the +[app roles](https://learn.microsoft.com/en-us/entra/identity-platform/howto-add-app-roles-in-apps) +assigned to it in the `roles` claim. + +To accept app-only tokens alongside user logins, replace the two claim settings +of the [QuestDB configuration](#questdb-configuration) with fallback lists: + +```shell +# user tokens carry name, app-only tokens carry oid +acl.oidc.sub.claim=name,oid,sub + +# user tokens carry groups, app-only tokens carry roles +acl.oidc.groups.claim=groups,roles +``` + +User logins still take the principal from `name` and the groups from `groups`. +App-only tokens take the principal from `oid` and the groups from `roles`. See +[User and group claims](#user-and-group-claims) for how QuestDB picks a claim +from each list. + +App-only tokens have further requirements: + +- Keep `acl.oidc.groups.encoded.in.token=true`. QuestDB then validates + app-only tokens against the signing keys of Entra ID, the same way as ID + tokens. +- Make the QuestDB application issue v2.0 access tokens by setting + `requestedAccessTokenVersion` to `2` in its manifest. QuestDB accepts a + token only when its `aud` claim matches `acl.oidc.audience`, which defaults + to the application ID. A v2.0 token always carries the application ID there. + A v1.0 token, the default for applications that accept organizational + accounts only, can carry the Application ID URI instead, such as + `api://...`. +- Assign at least one app role of the QuestDB application to each managed + identity or service principal, and define the roles with the _Applications_ + member type. QuestDB rejects a token that carries no role. + +Map each app role to a QuestDB group by the role's value, the same way as an +Entra ID group: + +```questdb-sql title="Map an app role to a QuestDB group" +CREATE GROUP ingestApps WITH EXTERNAL ALIAS 'QuestDB.Ingest'; +``` + +Then grant the group the permissions the service needs, as described in +[Map groups and grant permissions](#map-groups-and-grant-permissions). +
From 4a5cba0a5559ab24a6dab4e7cce31e2797b5887d Mon Sep 17 00:00:00 2001 From: glasstiger Date: Thu, 8 Oct 2026 15:38:03 +0100 Subject: [PATCH 02/29] docs(oidc): version-gate claim lists and complete the Entra app-only setup - Require a single-tenant QuestDB app registration for app-role mapping: token mode checks the signature, aud and exp, but not the issuer or tenant - Add 4.0.2 warnings to the claims section, the Entra ID subsection and the configuration reference, including the pre-4.0.2 requirement for a groups array in token mode - Link the non-interactive clients section and the claim examples to the Entra ID subsection, and label the example payloads as Entra ID tokens - Cover app role assignment, managed identity token caching, the HTTP and INSERT grants, the token scope, and token refresh for services - Document token-mode validation (kid, aud, exp, non-empty sub) and its log messages under Rejected logins, and add sub to the app-only payload - Describe token mode as reading the JWT the client presents, not only the ID token --- documentation/configuration/oidc.md | 50 ++++++---- documentation/security/oidc.mdx | 138 +++++++++++++++++++++++----- 2 files changed, 148 insertions(+), 40 deletions(-) diff --git a/documentation/configuration/oidc.md b/documentation/configuration/oidc.md index 9a266c1754..586d1346cf 100644 --- a/documentation/configuration/oidc.md +++ b/documentation/configuration/oidc.md @@ -201,19 +201,26 @@ which it connects. ## User and group claims QuestDB reads the principal and the group memberships of an external user from -the user information: the User Info endpoint's response, or the ID token when -`acl.oidc.groups.encoded.in.token` is `true`. `acl.oidc.sub.claim` and -`acl.oidc.groups.claim` each take a single claim name, or a comma-separated -list of claim names in priority order. QuestDB uses the first claim on the list -that carries a value, so one configuration can accept tokens that carry the -same information under different claim names. For the resolution rules and an -example, see +the user information: the User Info endpoint's response, or the payload of the +JWT that the client presents, such as an ID token or an Entra ID app-only +access token, when `acl.oidc.groups.encoded.in.token` is `true`. +`acl.oidc.sub.claim` and `acl.oidc.groups.claim` each take a single claim name, +or a comma-separated list of claim names in priority order. QuestDB uses the +first claim on the list that carries a value, so one configuration can accept +tokens that carry the same information under different claim names. For the +resolution rules and an example, see [User and group claims](/docs/security/oidc/#user-and-group-claims) in the OIDC guide. +:::warning Requires QuestDB Enterprise 4.0.2 + Claim lists are available since QuestDB Enterprise 4.0.2. Earlier versions read the whole value as a single claim name, so a list makes every OIDC login -fail. +fail. With `acl.oidc.groups.encoded.in.token=true`, earlier versions also +reject any token that does not carry a `groups` array, whatever +`acl.oidc.groups.claim` names. + +::: With OIDC enabled, QuestDB refuses to start when either setting is empty, when a setting lists the same claim twice, or when both settings list the same @@ -237,19 +244,23 @@ The claim in the user information that contains the group memberships of the user, as an array of group names or as a single group name. Required when OIDC is enabled. -Accepts a comma-separated list of claims in priority order, such as -`groups,roles`. QuestDB takes the groups from the first claim on the list that -holds at least one group name, and does not combine groups from several -claims. A login is rejected when none of the listed claims holds a group name. +Since QuestDB Enterprise 4.0.2, accepts a comma-separated list of claims in +priority order, such as `groups,roles`. QuestDB takes the groups from the first +claim on the list that holds at least one group name, and does not combine +groups from several claims. A login is rejected when none of the listed claims +holds a group name. On earlier versions, a list makes every OIDC login fail. ### acl.oidc.groups.encoded.in.token - **Default**: `false` - **Reloadable**: no -When `true`, QuestDB looks for group memberships in the ID token instead of -calling the User Info endpoint. Set to `true` if the OIDC Provider encodes -group memberships directly into the token. +When `true`, QuestDB reads the principal and the group memberships from the +JWT that the client presents, such as an ID token or an Entra ID app-only +access token, instead of calling the User Info endpoint. QuestDB validates the +token itself, as described in +[Rejected logins](/docs/security/oidc/#rejected-logins). Set to `true` if the +OIDC Provider encodes group memberships directly into the token. ### acl.oidc.sub.claim @@ -260,8 +271,9 @@ The claim in the user information that contains the user's principal. Could be a username, full name, email, or object ID. Displayed in the Web Console, returned by `current_user()`, and logged for audit purposes. -Accepts a comma-separated list of claims in priority order, such as -`name,oid,sub`. QuestDB takes the principal from the first claim on the list -that holds a non-empty value. A login is rejected when none of the listed -claims holds one. An empty value does not fall back to the default `sub`: with +Since QuestDB Enterprise 4.0.2, accepts a comma-separated list of claims in +priority order, such as `name,oid,sub`. QuestDB takes the principal from the +first claim on the list that holds a non-empty value. A login is rejected when +none of the listed claims holds one. On earlier versions, a list makes every +OIDC login fail. An empty value does not fall back to the default `sub`: with OIDC enabled, QuestDB refuses to start. diff --git a/documentation/security/oidc.mdx b/documentation/security/oidc.mdx index e30a6e576d..b9f021f8a2 100644 --- a/documentation/security/oidc.mdx +++ b/documentation/security/oidc.mdx @@ -584,7 +584,10 @@ Tokens that the OIDC provider issues to an application, rather than to a user, often carry the principal and the groups under different claim names. To accept both kinds of token with one configuration, list the claims of both in `acl.oidc.sub.claim` and `acl.oidc.groups.claim`, as described in -[User and group claims](#user-and-group-claims). +[User and group claims](#user-and-group-claims). For Microsoft Entra ID managed +identities and service principals, which also need v2.0 access tokens and an +app role, see +[Accept managed identity and service principal tokens](#accept-managed-identity-and-service-principal-tokens). ## OIDC for the PGWire endpoint @@ -737,12 +740,25 @@ is `true`. QuestDB rejects the login when the user information carries no principal or no group. +:::warning Requires QuestDB Enterprise 4.0.2 + +This section describes QuestDB Enterprise 4.0.2 and later. Earlier versions +read the whole value of each setting as a single claim name, so a claim list +makes every OIDC login fail. With `acl.oidc.groups.encoded.in.token=true`, +earlier versions also reject any token that does not carry a `groups` array, +whatever `acl.oidc.groups.claim` names. + +::: + ### Fallback claim lists Both settings accept a comma-separated list of claim names in priority order, so one configuration can accept tokens that carry the principal and the groups under different claim names. Tokens issued to users and tokens issued to -applications often differ this way: +applications often differ this way. For example, Microsoft Entra ID user tokens +carry `name` and `groups`, while its +[app-only tokens](#accept-managed-identity-and-service-principal-tokens) carry +`oid` and `roles`: ```ini title="server.conf" acl.oidc.sub.claim=name,oid,sub @@ -754,7 +770,7 @@ non-empty value, and the groups from the first claim on the list that carries at least one group name. With the configuration above, QuestDB reads these two token payloads, shortened to the relevant claims, as follows: -```json title="User token payload" +```json title="Entra ID user token payload" { "sub": "AAAAAAAAAAAAAAAAAAAAAIkzqFVrSaSaFHy782bbtaQ", "oid": "6f2d8c4e-31a7-4b5e-9c0d-8e1f2a3b4c5d", @@ -763,17 +779,18 @@ token payloads, shortened to the relevant claims, as follows: } ``` -```json title="Application token payload" +```json title="Entra ID app-only token payload" { + "sub": "4c9a4e3b-0d6f-4f43-9a8e-27d2b1f0c5a1", "oid": "4c9a4e3b-0d6f-4f43-9a8e-27d2b1f0c5a1", "roles": ["QuestDB.Ingest"] } ``` -| Token | Principal | Groups | -| ----------- | -------------------------- | ------------------------------ | -| User | `Jane Smith`, from `name` | `87654321-...`, from `groups` | -| Application | `4c9a4e3b-...`, from `oid` | `QuestDB.Ingest`, from `roles` | +| Token | Principal | Groups | +| -------- | -------------------------- | ------------------------------ | +| User | `Jane Smith`, from `name` | `87654321-...`, from `groups` | +| App-only | `4c9a4e3b-...`, from `oid` | `QuestDB.Ingest`, from `roles` | The following rules decide which claim QuestDB uses. They apply to a single claim name as well: @@ -796,12 +813,10 @@ claim name as well: claim earlier on the list carries a value. - Claims that are not listed are ignored, whatever their shape. -:::note - -Claim lists are available since QuestDB Enterprise 4.0.2. Earlier versions read -the whole value as a single claim name, so a list makes every OIDC login fail. - -::: +With `acl.oidc.groups.encoded.in.token=true`, QuestDB validates the token before +it applies these rules, and the token must carry a non-empty `sub` claim even +when another claim supplies the principal. See +[Rejected logins](#rejected-logins). ### Startup validation @@ -835,6 +850,25 @@ that your provider uses. A listed claim of an unexpected shape, such as an object in a groups claim, is logged as `Request failed [error=unexpected input format ...]` instead. +With `acl.oidc.groups.encoded.in.token=true`, QuestDB validates the token +before it reads any claim. The token must be a JWT whose `kid` header names one +of the signing keys that the provider publishes. It must also carry an `aud` +claim that matches +[`acl.oidc.audience`](/docs/configuration/oidc/#acloidcaudience), an `exp` +claim that has not passed, and a non-empty `sub` claim, whatever claims +`acl.oidc.sub.claim` lists. A token that fails these checks is rejected the +same way, but the server log gives the reason instead of the claim lists, for +example: + +```text +Request failed [errorMsg=Failed to decode JWT token Error(InvalidAudience)] +Request failed [errorMsg=Failed to decode JWT token Error(ExpiredSignature)] +``` + +`InvalidAudience` usually means that the token was requested for another API, +or that Entra ID issued a v1.0 access token. See +[Accept managed identity and service principal tokens](#accept-managed-identity-and-service-principal-tokens). + ## Configuration options For all OIDC-related configuration options of QuestDB, see @@ -1468,6 +1502,14 @@ service principal in the `oid` claim, and list the [app roles](https://learn.microsoft.com/en-us/entra/identity-platform/howto-add-app-roles-in-apps) assigned to it in the `roles` claim. +:::warning Requires QuestDB Enterprise 4.0.2 + +Accepting app-only tokens requires QuestDB Enterprise 4.0.2 or later. On +earlier versions, the configuration below rejects every OIDC login, user logins +included. + +::: + To accept app-only tokens alongside user logins, replace the two claim settings of the [QuestDB configuration](#questdb-configuration) with fallback lists: @@ -1489,6 +1531,13 @@ App-only tokens have further requirements: - Keep `acl.oidc.groups.encoded.in.token=true`. QuestDB then validates app-only tokens against the signing keys of Entra ID, the same way as ID tokens. +- Keep the QuestDB application single-tenant: its supported account type must + be _Accounts in this organizational directory only_. QuestDB checks the + signature, `aud`, and `exp` of a token, but not its issuer or tenant, and an + app role has the same value in every tenant that uses the application. With + a multitenant registration, another tenant can assign the same role to its + own service principals, and QuestDB maps their tokens to the same QuestDB + groups. - Make the QuestDB application issue v2.0 access tokens by setting `requestedAccessTokenVersion` to `2` in its manifest. QuestDB accepts a token only when its `aud` claim matches `acl.oidc.audience`, which defaults @@ -1496,18 +1545,65 @@ App-only tokens have further requirements: A v1.0 token, the default for applications that accept organizational accounts only, can carry the Application ID URI instead, such as `api://...`. -- Assign at least one app role of the QuestDB application to each managed - identity or service principal, and define the roles with the _Applications_ - member type. QuestDB rejects a token that carries no role. +- Define the app roles with the _Applications_ member type, and assign at + least one of them to each managed identity or service principal. QuestDB + rejects a token that carries no role. +- Assign the role to the identity itself, because Entra ID does not add a role + assigned to a group to the tokens of the service principals in that group. + For the service principal of an app registration, add the role as an + application permission under the registration's _API permissions_, and grant + admin consent. For a managed identity, assign the role through Microsoft + Graph, as described in + [Assign a managed identity to an application role](https://learn.microsoft.com/en-us/entra/identity/managed-identities-azure-resources/how-to-assign-app-role-managed-identity). + +QuestDB reads the roles from the token, so a change to the roles of a service +takes effect when the service presents a new token. Azure caches managed +identity tokens for around 24 hours, so a role assigned to, or removed from, a +managed identity can take hours to reach QuestDB. See the +[managed identity best practices](https://learn.microsoft.com/en-us/entra/identity/managed-identities-azure-resources/managed-identity-best-practice-recommendations). Map each app role to a QuestDB group by the role's value, the same way as an -Entra ID group: +Entra ID group, and grant the group what the service needs. A service that +ingests over HTTP needs the `HTTP` endpoint permission and `INSERT` on its +tables: -```questdb-sql title="Map an app role to a QuestDB group" +```questdb-sql title="Map an app role to a group that ingests into fx_trades" CREATE GROUP ingestApps WITH EXTERNAL ALIAS 'QuestDB.Ingest'; +GRANT HTTP TO ingestApps; +GRANT INSERT ON fx_trades TO ingestApps; +``` + +If the service sends columns that the table does not have yet, also grant the +group `ADD COLUMN` on the table. See +[Endpoint permissions](/docs/security/rbac/#endpoint-permissions) for the other +endpoints. + +The service requests its token for the QuestDB application. Under _Expose an +API_, give the QuestDB application an Application ID URI, which defaults to +`api://`. The service then requests the scope +`/.default`. A token for any other API, such as Microsoft +Graph, carries that API in `aud`, and QuestDB rejects it. With the +[Azure Identity library](https://learn.microsoft.com/en-us/python/api/overview/azure/identity-readme) +for Python, `DefaultAzureCredential` requests the token as the managed identity +of the Azure resource that the service runs on, or as a service principal set +up in environment variables: + +```python title="Request an app-only token for QuestDB" +from azure.identity import DefaultAzureCredential + +# Application ID URI of the QuestDB application, followed by /.default +scope = "api://8de84b90-1ea5-4e41-9e84-dba860aa01a6/.default" +token = DefaultAzureCredential().get_token(scope) + +conf = f"http::addr=localhost:9000;token={token.token};" ``` -Then grant the group the permissions the service needs, as described in -[Map groups and grant permissions](#map-groups-and-grant-permissions). +Use `conf` with the `Sender`, as in +[Non-interactive clients](#non-interactive-clients). For the +[PGWire endpoint](#oidc-for-the-pgwire-endpoint), send the token as the +password of the `_sso` user instead. The token expires at `token.expires_on`, a +Unix timestamp, and QuestDB rejects an expired token, so a long-running service +requests a new token before then, and creates a new sender or connection with +it.
From fc79dd60cd9e2802301b42de1de8202f42095f95 Mon Sep 17 00:00:00 2001 From: glasstiger Date: Thu, 8 Oct 2026 22:15:46 +0100 Subject: [PATCH 03/29] docs(oidc): harden the claim list and Entra app-only guidance - App-only tokens can carry a groups claim, so list roles before groups (acl.oidc.groups.claim=roles,groups) and reject only tokens with neither a role nor a group - State that the principal must be unique and use preferred_username for Entra ID users instead of the display name - List every behavior that is new in QuestDB Enterprise 4.0.2, including exp validation, and gate the config page statements - Add a Token validation section and extend Rejected logins - Reorder the Entra app-only setup into steps, select a single-tenant account type at registration, and rewrite the Python example with the current client over TLS - Correct the acl.oidc.pkce.enabled note, file the new sections under New in the changelog, and link the managed identity section from the Entra intro and the Azure deployment page --- documentation/changelog.mdx | 12 +- documentation/configuration/oidc.md | 70 +++--- documentation/deployment/azure.md | 3 + documentation/security/oidc.mdx | 356 +++++++++++++++++++--------- 4 files changed, 285 insertions(+), 156 deletions(-) diff --git a/documentation/changelog.mdx b/documentation/changelog.mdx index 82ca0d16a7..48388af82c 100644 --- a/documentation/changelog.mdx +++ b/documentation/changelog.mdx @@ -10,14 +10,20 @@ This page tracks significant updates to the QuestDB documentation. ## October 2026 +### New + +- [User and group claims](/docs/security/oidc/#user-and-group-claims) - How QuestDB Enterprise 4.0.2 reads the principal and the groups from the user information: fallback claim lists, the rules for missing, `null`, and empty claims, why the principal must be unique, the startup checks, token validation, and the log messages of rejected logins +- [Accept managed identity and service principal tokens](/docs/security/oidc/#accept-managed-identity-and-service-principal-tokens) - Let services that run as a Microsoft Entra ID managed identity or service principal authenticate to QuestDB Enterprise 4.0.2 with app-only tokens, next to user logins: the Entra ID setup, the claim lists, the app role mapping, and a Python example + ### Reference -- [`acl.oidc.groups.claim`](/docs/configuration/oidc/#acloidcgroupsclaim) and [`acl.oidc.sub.claim`](/docs/configuration/oidc/#acloidcsubclaim) - Documented the comma-separated fallback claim lists in priority order and the startup validation. `acl.oidc.groups.claim` is now listed with no default, as it must be set when OIDC is enabled -- [`acl.oidc.pkce.required`](/docs/configuration/oidc/#acloidcpkcerequired) - Corrected the property name, previously documented as `acl.oidc.pkce.enabled`, which QuestDB ignores +- [`acl.oidc.groups.claim`](/docs/configuration/oidc/#acloidcgroupsclaim) and [`acl.oidc.sub.claim`](/docs/configuration/oidc/#acloidcsubclaim) - Documented the comma-separated fallback claim lists of QuestDB Enterprise 4.0.2 and the startup validation. `acl.oidc.groups.claim` is now listed with no default, as it must be set when OIDC is enabled +- [`acl.oidc.pkce.required`](/docs/configuration/oidc/#acloidcpkcerequired) - Corrected the property name, previously documented as `acl.oidc.pkce.enabled`, which QuestDB does not recognize ### Updated -- [OpenID Connect (OIDC)](/docs/security/oidc/#user-and-group-claims) - New section on how QuestDB reads the principal and the groups from the user information: fallback claim lists, the rules for missing, `null`, and empty claims, the startup checks, and the `Failed to find required claims` log message. The Microsoft Entra ID guide now covers [accepting managed identity and service principal tokens](/docs/security/oidc/#accept-managed-identity-and-service-principal-tokens) next to user logins +- [Mapping user permissions](/docs/security/oidc/#mapping-user-permissions) - Corrected what happens when the groups claim is missing or empty: QuestDB rejects the login. A user is authenticated without permissions only when none of their groups is mapped to a QuestDB group +- [Microsoft Entra ID guide](/docs/security/oidc/#microsoft-entraid) - Uses `preferred_username` as the principal claim instead of the display name, and registers the application as single-tenant ## September 2026 diff --git a/documentation/configuration/oidc.md b/documentation/configuration/oidc.md index 586d1346cf..16ec75ee8e 100644 --- a/documentation/configuration/oidc.md +++ b/documentation/configuration/oidc.md @@ -109,8 +109,10 @@ Whether the Web Console uses PKCE (Proof Key for Code Exchange) in the Authorization Code Flow. This should always be enabled in production. The Web Console is not fully secure without it. -Earlier versions of this page listed the property as `acl.oidc.pkce.enabled`. -QuestDB ignores that name, so a value set under it has no effect. +`acl.oidc.pkce.enabled` is not a QuestDB setting. QuestDB reports it as an +invalid setting at startup and does not apply its value, and with +[`config.validation.strict=true`](/docs/configuration/overview/#configvalidationstrict) +it refuses to start. Use `acl.oidc.pkce.required` instead. ### acl.oidc.ropc.flow.enabled @@ -201,31 +203,15 @@ which it connects. ## User and group claims QuestDB reads the principal and the group memberships of an external user from -the user information: the User Info endpoint's response, or the payload of the -JWT that the client presents, such as an ID token or an Entra ID app-only -access token, when `acl.oidc.groups.encoded.in.token` is `true`. -`acl.oidc.sub.claim` and `acl.oidc.groups.claim` each take a single claim name, -or a comma-separated list of claim names in priority order. QuestDB uses the -first claim on the list that carries a value, so one configuration can accept -tokens that carry the same information under different claim names. For the -resolution rules and an example, see +the user information: the User Info endpoint's response or, when +`acl.oidc.groups.encoded.in.token` is `true`, the payload of the JWT that the +client presents. `acl.oidc.sub.claim` and `acl.oidc.groups.claim` each take a +single claim name or, since QuestDB Enterprise 4.0.2, a comma-separated list of +claim names in priority order. For how QuestDB picks the claims, with examples, +and for how versions before 4.0.2 differ, see [User and group claims](/docs/security/oidc/#user-and-group-claims) in the OIDC guide. -:::warning Requires QuestDB Enterprise 4.0.2 - -Claim lists are available since QuestDB Enterprise 4.0.2. Earlier versions -read the whole value as a single claim name, so a list makes every OIDC login -fail. With `acl.oidc.groups.encoded.in.token=true`, earlier versions also -reject any token that does not carry a `groups` array, whatever -`acl.oidc.groups.claim` names. - -::: - -With OIDC enabled, QuestDB refuses to start when either setting is empty, when -a setting lists the same claim twice, or when both settings list the same -claim. - ### acl.oidc.cache.ttl - **Default**: `30000` @@ -245,10 +231,15 @@ user, as an array of group names or as a single group name. Required when OIDC is enabled. Since QuestDB Enterprise 4.0.2, accepts a comma-separated list of claims in -priority order, such as `groups,roles`. QuestDB takes the groups from the first +priority order, such as `roles,groups`. QuestDB takes the groups from the first claim on the list that holds at least one group name, and does not combine groups from several claims. A login is rejected when none of the listed claims -holds a group name. On earlier versions, a list makes every OIDC login fail. +holds a group name. With OIDC enabled, QuestDB refuses to start when the list +names a claim twice, or names a claim that `acl.oidc.sub.claim` also lists. + +On earlier versions, a list makes every OIDC login fail. With +`acl.oidc.groups.encoded.in.token=true`, earlier versions also require a +`groups` array in the token, whatever this setting names. ### acl.oidc.groups.encoded.in.token @@ -259,21 +250,30 @@ When `true`, QuestDB reads the principal and the group memberships from the JWT that the client presents, such as an ID token or an Entra ID app-only access token, instead of calling the User Info endpoint. QuestDB validates the token itself, as described in -[Rejected logins](/docs/security/oidc/#rejected-logins). Set to `true` if the +[Token validation](/docs/security/oidc/#token-validation). Set to `true` if the OIDC Provider encodes group memberships directly into the token. +Since QuestDB Enterprise 4.0.2, QuestDB also rejects expired tokens. Earlier +versions do not check the `exp` claim of the token. + ### acl.oidc.sub.claim - **Default**: `sub` - **Reloadable**: no -The claim in the user information that contains the user's principal. Could be -a username, full name, email, or object ID. Displayed in the Web Console, -returned by `current_user()`, and logged for audit purposes. +The claim in the user information that contains the user's principal, such as +a username, an email address, or an object ID. Displayed in the Web Console, +returned by `current_user()`, and logged for audit purposes. The principal must +be unique for each user: QuestDB keeps one external user per principal and +replaces its groups at every login, so users who share a principal share +permissions. Avoid display names, such as the `name` claim. Since QuestDB Enterprise 4.0.2, accepts a comma-separated list of claims in -priority order, such as `name,oid,sub`. QuestDB takes the principal from the -first claim on the list that holds a non-empty value. A login is rejected when -none of the listed claims holds one. On earlier versions, a list makes every -OIDC login fail. An empty value does not fall back to the default `sub`: with -OIDC enabled, QuestDB refuses to start. +priority order, such as `preferred_username,oid`. QuestDB takes the principal +from the first claim on the list that holds a non-empty value. A login is +rejected when none of the listed claims holds one. With OIDC enabled, QuestDB +refuses to start when the value is empty, when the list names a claim twice, or +when it names a claim that `acl.oidc.groups.claim` also lists. An empty value +does not fall back to the default `sub`. + +On earlier versions, a list makes every OIDC login fail. diff --git a/documentation/deployment/azure.md b/documentation/deployment/azure.md index edd2c8b496..4c9ee2babf 100644 --- a/documentation/deployment/azure.md +++ b/documentation/deployment/azure.md @@ -346,6 +346,9 @@ QuestDB Enterprise adds production features for Azure: For EntraID integration, see the [Microsoft EntraID OIDC guide](/docs/security/oidc/#microsoft-entraid). +Azure services that run as a managed identity or a service principal can also +authenticate to QuestDB with Entra ID app-only tokens. See +[Accept managed identity and service principal tokens](/docs/security/oidc/#accept-managed-identity-and-service-principal-tokens). See [Enterprise Quick Start](/docs/getting-started/enterprise-quick-start/) for setup. diff --git a/documentation/security/oidc.mdx b/documentation/security/oidc.mdx index b9f021f8a2..437a289227 100644 --- a/documentation/security/oidc.mdx +++ b/documentation/security/oidc.mdx @@ -1,6 +1,6 @@ --- title: OpenID Connect (OIDC) Integration -description: Configure OpenID Connect (OIDC) integration with external Identity Providers for SSO authentication in QuestDB Enterprise Web Console. +description: Configure OpenID Connect (OIDC) in QuestDB Enterprise for Web Console SSO, token authentication for clients and services, claim mapping, and Entra ID managed identities. --- import Screenshot from "@theme/Screenshot"; @@ -13,8 +13,10 @@ import { EnterpriseNote } from "@site/src/components/EnterpriseNote" OpenID Connect (OIDC) integrates with Identity Providers (IdP) external to QuestDB. -It is a convenient way to integrate QuestDB into your enterprise environment, -and it provides SSO (Single Sign-On) for the [Web Console](/docs/getting-started/web-console/overview/). +It is a convenient way to integrate QuestDB into your enterprise environment. +It provides SSO (Single Sign-On) for the [Web Console](/docs/getting-started/web-console/overview/), +and token authentication for applications and services that connect to +QuestDB. Microsoft Active Directory and Azure AD, for example, can be turned into an Identity Provider. @@ -259,6 +261,11 @@ https://oidc.provider:443/idp/userinfo.openid Authorization: Bearer gslpJtzmmi6RwaPSx0dYGD4tEkom ``` +With `acl.oidc.groups.encoded.in.token=true`, QuestDB skips this call. It +validates the token itself, as described in +[Token validation](#token-validation), and reads the user information from the +token's payload. + ### 9. Receive user information If the access token is valid, QuestDB receives the required user information @@ -297,7 +304,7 @@ and then sends the results back: "type": "STRING" } ], - "dataset": [["External User"]], + "dataset": [["externalUser"]], "count": 1, "timestamp": -1 } @@ -584,9 +591,15 @@ Tokens that the OIDC provider issues to an application, rather than to a user, often carry the principal and the groups under different claim names. To accept both kinds of token with one configuration, list the claims of both in `acl.oidc.sub.claim` and `acl.oidc.groups.claim`, as described in -[User and group claims](#user-and-group-claims). For Microsoft Entra ID managed -identities and service principals, which also need v2.0 access tokens and an -app role, see +[User and group claims](#user-and-group-claims). + +QuestDB must also be able to validate the application's token. With +`acl.oidc.groups.encoded.in.token=true`, the token must be a JWT whose `aud` +claim matches `acl.oidc.audience`, as described in +[Token validation](#token-validation). Otherwise, QuestDB sends the token to the +provider's User Info endpoint, which some providers refuse for tokens issued to +applications. For Microsoft Entra ID managed identities and service principals, +see [Accept managed identity and service principal tokens](#accept-managed-identity-and-service-principal-tokens). ## OIDC for the PGWire endpoint @@ -725,9 +738,9 @@ It should be set accordingly to your organization's policies. QuestDB reads two values from the user information of an external user: the principal, which identifies the user, and the groups the user belongs to. The -user information is the response of the User Info endpoint, or the payload of -the token itself, such as an ID token, when `acl.oidc.groups.encoded.in.token` -is `true`. +user information is the response of the User Info endpoint or, when +`acl.oidc.groups.encoded.in.token` is `true`, the payload of the JWT that the +client presents, such as an ID token or an Entra ID app-only access token. - [`acl.oidc.sub.claim`](/docs/configuration/oidc/#acloidcsubclaim) names the claim that carries the principal, `sub` by default. The principal identifies @@ -740,13 +753,32 @@ is `true`. QuestDB rejects the login when the user information carries no principal or no group. +The principal must be unique for each user and service. QuestDB keeps one +external user for each principal, and every login replaces the groups of that +user with the groups of the login. Two people who resolve to the same principal +share one QuestDB user, and a session that is already open, such as a PGWire +connection, can run with the groups of whoever logged in last. Pick a claim +that the provider keeps unique, such as a username or an object ID, not a +display name. Changing the claim on an existing deployment gives every user a +new principal. Permissions stay the same, because they come from the groups. + :::warning Requires QuestDB Enterprise 4.0.2 This section describes QuestDB Enterprise 4.0.2 and later. Earlier versions -read the whole value of each setting as a single claim name, so a claim list -makes every OIDC login fail. With `acl.oidc.groups.encoded.in.token=true`, -earlier versions also reject any token that does not carry a `groups` array, -whatever `acl.oidc.groups.claim` names. +differ as follows: + +- They read the whole value of each setting as a single claim name, so a claim + list makes every OIDC login fail. +- With `acl.oidc.groups.encoded.in.token=true`, they reject any token that does + not carry a `groups` array, whatever `acl.oidc.groups.claim` names. They do + not check the `exp` claim, so they accept expired tokens. +- They read a `null` claim as the value `null`. Every account whose principal + claim is `null` logs in as the same principal, `null`, and `null` and empty + group names count as groups. +- They reject a login when a claim that is not listed holds an array of + objects or nested arrays. +- They start with an empty `acl.oidc.sub.claim`, or with the same claim in both + settings, and then reject every OIDC login. ::: @@ -755,14 +787,15 @@ whatever `acl.oidc.groups.claim` names. Both settings accept a comma-separated list of claim names in priority order, so one configuration can accept tokens that carry the principal and the groups under different claim names. Tokens issued to users and tokens issued to -applications often differ this way. For example, Microsoft Entra ID user tokens -carry `name` and `groups`, while its +applications often differ this way. For example, Microsoft Entra ID user +tokens, configured as in the [Entra ID guide](#microsoft-entraid), carry +`preferred_username` and `groups`, while its [app-only tokens](#accept-managed-identity-and-service-principal-tokens) carry `oid` and `roles`: ```ini title="server.conf" -acl.oidc.sub.claim=name,oid,sub -acl.oidc.groups.claim=groups,roles +acl.oidc.sub.claim=preferred_username,oid +acl.oidc.groups.claim=roles,groups ``` QuestDB takes the principal from the first claim on the list that carries a @@ -774,7 +807,7 @@ token payloads, shortened to the relevant claims, as follows: { "sub": "AAAAAAAAAAAAAAAAAAAAAIkzqFVrSaSaFHy782bbtaQ", "oid": "6f2d8c4e-31a7-4b5e-9c0d-8e1f2a3b4c5d", - "name": "Jane Smith", + "preferred_username": "jane.smith@example.com", "groups": ["87654321-1234-1234-1234-123456789abc"] } ``` @@ -787,24 +820,31 @@ token payloads, shortened to the relevant claims, as follows: } ``` -| Token | Principal | Groups | -| -------- | -------------------------- | ------------------------------ | -| User | `Jane Smith`, from `name` | `87654321-...`, from `groups` | -| App-only | `4c9a4e3b-...`, from `oid` | `QuestDB.Ingest`, from `roles` | +| Token | Principal | Groups | +| -------- | --------------------------------------------------- | ------------------------------ | +| User | `jane.smith@example.com`, from `preferred_username` | `87654321-...`, from `groups` | +| App-only | `4c9a4e3b-...`, from `oid` | `QuestDB.Ingest`, from `roles` | + +`roles` comes before `groups` because an app-only token can also carry a +`groups` claim, and QuestDB takes the groups from the first claim on the list +that holds one. See +[Accept managed identity and service principal tokens](#accept-managed-identity-and-service-principal-tokens). The following rules decide which claim QuestDB uses. They apply to a single claim name as well: -- The order of the list decides, not the order of the claims in the token. A - token that carries both `groups` and `roles` gets its groups from `groups` - alone. QuestDB does not combine groups from several claims. +- The order of the list decides, not the order of the claims in the token. + With the configuration above, a token that carries both `roles` and `groups` + gets its groups from `roles` alone. QuestDB does not combine groups from + several claims. - A claim that is missing, `null`, or empty falls through to the next claim on the list. Empty means an empty string, or an empty array for a groups claim. The string `"null"` counts as missing too. - Group names that are `null` or empty are skipped. A groups claim that holds nothing else falls through to the next claim. -- Only top-level claims count. A claim nested in another claim, such as `name` - inside `address`, is ignored. +- Only top-level claims count. A claim nested in another claim is ignored, + such as the `roles` that Keycloak puts inside its `realm_access` claim. To + use such values, configure the provider to emit them as a top-level claim. - Claim names are case-sensitive. - A listed claim must have the expected shape: a single value, such as a string, for the principal, and a single value or a flat array of values for @@ -814,50 +854,77 @@ claim name as well: - Claims that are not listed are ignored, whatever their shape. With `acl.oidc.groups.encoded.in.token=true`, QuestDB validates the token before -it applies these rules, and the token must carry a non-empty `sub` claim even -when another claim supplies the principal. See -[Rejected logins](#rejected-logins). +it applies these rules. See [Token validation](#token-validation). ### Startup validation With OIDC enabled, QuestDB checks both settings at startup. Spaces around claim -names are trimmed and empty entries are skipped, so `name, oid,,sub` reads as -`name,oid,sub`. QuestDB refuses to start when: +names are trimmed and empty entries are skipped, so +`preferred_username, oid,,sub` reads as `preferred_username,oid,sub`. QuestDB +refuses to start when: - `acl.oidc.groups.claim` is not set, or either setting is empty: `required property is not set: acl.oidc.groups.claim`, or the same message for `acl.oidc.sub.claim`. An empty `acl.oidc.sub.claim` does not fall back to the default `sub`. - A setting lists the same claim twice: - `claim is listed more than once in acl.oidc.sub.claim [claim=name]`. + `claim is listed more than once in acl.oidc.sub.claim [claim=oid]`. - Both settings list the same claim: `claim cannot be listed in both acl.oidc.sub.claim and acl.oidc.groups.claim [claim=oid]`. +### Token validation + +With `acl.oidc.groups.encoded.in.token=true`, QuestDB validates the token +itself before it reads any claim: + +- The token must be a JWT whose `kid` header names one of the signing keys that + the provider publishes. +- Its `aud` claim must match + [`acl.oidc.audience`](/docs/configuration/oidc/#acloidcaudience), which + defaults to the client ID. QuestDB accepts a single audience. +- Its `exp` claim must not have passed. QuestDB allows 60 seconds of clock + skew. +- It must carry a non-empty `sub` claim, whatever claims `acl.oidc.sub.claim` + lists. + +QuestDB does not check the issuer (`iss`) or the tenant of the token. It +accepts any token that the provider's keys sign for the expected audience, so +make sure that no one outside your organization can obtain tokens for the +QuestDB application. With Microsoft Entra ID, register the application as +single-tenant, as described in +[Set up the client application in Entra ID](#set-up-the-client-application-in-entra-id). + +QuestDB checks a token when a request or a connection authenticates with it, +and caches the result for up to +[`acl.oidc.cache.ttl`](/docs/configuration/oidc/#acloidccachettl). An open +PGWire or WebSocket connection stays open when its token expires. + +With `acl.oidc.groups.encoded.in.token=false`, QuestDB sends the token to the +provider's User Info endpoint, which validates it. See +[Find user information](#8-find-user-information). + ### Rejected logins When the user information carries none of the listed principal claims, or none of the listed groups claims, QuestDB rejects the login. HTTP clients receive `401 Unauthorized`, and PGWire clients receive `invalid username/password`. The -server log names both lists, followed by the user information QuestDB -received: +server log contains `Failed to find required claims`, followed by both lists +and the user information QuestDB received: ```text -Failed to find required claims [subClaims=[name,oid,sub], groupsClaims=[groups,roles], userInfo={...}] +Failed to find required claims [subClaims=[preferred_username,oid], groupsClaims=[roles,groups], userInfo={...}] ``` Compare the claims in `userInfo` with the two lists to find the claim names that your provider uses. A listed claim of an unexpected shape, such as an object in a groups claim, is logged as -`Request failed [error=unexpected input format ...]` instead. +`Request failed [error=unexpected input format ...]` instead. With Microsoft +Entra ID, a user who belongs to more than 200 groups gets no `groups` claim in +the token, so QuestDB rejects the login unless another listed claim carries +groups. -With `acl.oidc.groups.encoded.in.token=true`, QuestDB validates the token -before it reads any claim. The token must be a JWT whose `kid` header names one -of the signing keys that the provider publishes. It must also carry an `aud` -claim that matches -[`acl.oidc.audience`](/docs/configuration/oidc/#acloidcaudience), an `exp` -claim that has not passed, and a non-empty `sub` claim, whatever claims -`acl.oidc.sub.claim` lists. A token that fails these checks is rejected the -same way, but the server log gives the reason instead of the claim lists, for +A token that fails [token validation](#token-validation) is rejected the same +way, but the server log gives the reason instead of the claim lists, for example: ```text @@ -868,6 +935,17 @@ Request failed [errorMsg=Failed to decode JWT token Error(ExpiredSignature)] `InvalidAudience` usually means that the token was requested for another API, or that Entra ID issued a v1.0 access token. See [Accept managed identity and service principal tokens](#accept-managed-identity-and-service-principal-tokens). +Don't set `acl.oidc.audience` to the Application ID URI to work around it: +QuestDB accepts a single audience, and the ID tokens of Web Console users carry +the application ID, so their logins would fail. + +With `acl.oidc.groups.encoded.in.token=false`, the provider's User Info +endpoint rejects a token that is invalid or expired, and QuestDB rejects the +login. + +If the login succeeds but every request is denied, check that at least one of +the user's groups is mapped to a QuestDB group that has the endpoint +permission. See [Mapping user permissions](#mapping-user-permissions). ## Configuration options @@ -1231,6 +1309,10 @@ If all has been wired up well, then login will succeed. This document sets up SSO authentication for the [QuestDB Web Console](/docs/getting-started/web-console/overview/) in [Microsoft EntraID](https://www.microsoft.com/en-gb/security/business/identity-access/microsoft-entra-id), formerly known as Azure AD. +It also shows how to let services that run as a managed identity or a service +principal call QuestDB with app-only tokens. See +[Accept managed identity and service principal tokens](#accept-managed-identity-and-service-principal-tokens). + :::tip @@ -1245,6 +1327,11 @@ First thing first, let's pick a name for the client! Then head to _Microsoft Entra Admin Center_, and register the application under _Identity - App registrations - New registration_. +Under _Supported account types_, select _Accounts in this organizational +directory only_. QuestDB does not check which tenant issued a token, so the +application must not accept accounts from other tenants. See +[Token validation](#token-validation). + `. Services request their tokens for the scope + `/.default`. A token requested for any other API, such + as Microsoft Graph, carries that API in `aud`, and QuestDB rejects it. +4. Under _App roles_, define a role for each kind of access, with the + _Applications_ member type. The examples below use the value + `QuestDB.Ingest`. +5. Assign at least one role to each managed identity or service principal. + Assign the role to the identity itself: Entra ID does not add a role + assigned to a group to the tokens of the service principals in that group. + + - For the service principal of an app registration, open that registration, + not the QuestDB one. Under _API permissions_, select _Add a permission_, + _APIs my organization uses_, the QuestDB application, and + _Application permissions_. Add the role, and grant admin consent. + - For a managed identity, assign the role through Microsoft Graph, as + described in + [Assign a managed identity to an application role](https://learn.microsoft.com/en-us/entra/identity/managed-identities-azure-resources/how-to-assign-app-role-managed-identity). + +Then replace the two claim settings of the +[QuestDB configuration](#questdb-configuration) with fallback lists. Keep +`acl.oidc.groups.encoded.in.token=true`, so that QuestDB validates app-only +tokens against the signing keys of Entra ID, the same way as ID tokens: -```shell -# user tokens carry name, app-only tokens carry oid -acl.oidc.sub.claim=name,oid,sub +```ini title="server.conf" +# user tokens carry preferred_username, app-only tokens carry oid +acl.oidc.sub.claim=preferred_username,oid -# user tokens carry groups, app-only tokens carry roles -acl.oidc.groups.claim=groups,roles +# app-only tokens carry roles, user tokens carry groups +acl.oidc.groups.claim=roles,groups ``` -User logins still take the principal from `name` and the groups from `groups`. -App-only tokens take the principal from `oid` and the groups from `roles`. See -[User and group claims](#user-and-group-claims) for how QuestDB picks a claim -from each list. - -App-only tokens have further requirements: - -- Keep `acl.oidc.groups.encoded.in.token=true`. QuestDB then validates - app-only tokens against the signing keys of Entra ID, the same way as ID - tokens. -- Keep the QuestDB application single-tenant: its supported account type must - be _Accounts in this organizational directory only_. QuestDB checks the - signature, `aud`, and `exp` of a token, but not its issuer or tenant, and an - app role has the same value in every tenant that uses the application. With - a multitenant registration, another tenant can assign the same role to its - own service principals, and QuestDB maps their tokens to the same QuestDB - groups. -- Make the QuestDB application issue v2.0 access tokens by setting - `requestedAccessTokenVersion` to `2` in its manifest. QuestDB accepts a - token only when its `aud` claim matches `acl.oidc.audience`, which defaults - to the application ID. A v2.0 token always carries the application ID there. - A v1.0 token, the default for applications that accept organizational - accounts only, can carry the Application ID URI instead, such as - `api://...`. -- Define the app roles with the _Applications_ member type, and assign at - least one of them to each managed identity or service principal. QuestDB - rejects a token that carries no role. -- Assign the role to the identity itself, because Entra ID does not add a role - assigned to a group to the tokens of the service principals in that group. - For the service principal of an app registration, add the role as an - application permission under the registration's _API permissions_, and grant - admin consent. For a managed identity, assign the role through Microsoft - Graph, as described in - [Assign a managed identity to an application role](https://learn.microsoft.com/en-us/entra/identity/managed-identities-azure-resources/how-to-assign-app-role-managed-identity). - -QuestDB reads the roles from the token, so a change to the roles of a service -takes effect when the service presents a new token. Azure caches managed -identity tokens for around 24 hours, so a role assigned to, or removed from, a -managed identity can take hours to reach QuestDB. See the -[managed identity best practices](https://learn.microsoft.com/en-us/entra/identity/managed-identities-azure-resources/managed-identity-best-practice-recommendations). +User logins take the principal from `preferred_username` and the groups from +`groups`, as before. Their tokens carry no `roles` claim, as long as the app +roles of the QuestDB application admit applications only. App-only tokens take +the principal from `oid` and the groups from `roles`, even when they also carry +a `groups` claim. QuestDB rejects a token that carries neither a role nor a +group. See [User and group claims](#user-and-group-claims) for how QuestDB +picks a claim from each list. Map each app role to a QuestDB group by the role's value, the same way as an Entra ID group, and grant the group what the service needs. A service that -ingests over HTTP needs the `HTTP` endpoint permission and `INSERT` on its -tables: +ingests over HTTP or QWP needs the `HTTP` endpoint permission and `INSERT` on +its tables: ```questdb-sql title="Map an app role to a group that ingests into fx_trades" CREATE GROUP ingestApps WITH EXTERNAL ALIAS 'QuestDB.Ingest'; @@ -1573,37 +1671,59 @@ GRANT HTTP TO ingestApps; GRANT INSERT ON fx_trades TO ingestApps; ``` -If the service sends columns that the table does not have yet, also grant the -group `ADD COLUMN` on the table. See +If the service creates tables, also grant the group `CREATE TABLE`. If it sends +columns that a table does not have yet, grant `ADD COLUMN` on the table. See [Endpoint permissions](/docs/security/rbac/#endpoint-permissions) for the other endpoints. -The service requests its token for the QuestDB application. Under _Expose an -API_, give the QuestDB application an Application ID URI, which defaults to -`api://`. The service then requests the scope -`/.default`. A token for any other API, such as Microsoft -Graph, carries that API in `aud`, and QuestDB rejects it. With the +Finally, the service requests its token for the QuestDB application. With the [Azure Identity library](https://learn.microsoft.com/en-us/python/api/overview/azure/identity-readme) for Python, `DefaultAzureCredential` requests the token as the managed identity of the Azure resource that the service runs on, or as a service principal set -up in environment variables: +up in environment variables. For a user-assigned managed identity, set the +`AZURE_CLIENT_ID` environment variable to the client ID of the identity. +Otherwise, the credential uses the system-assigned identity. Install the +libraries with `python3 -m pip install -U questdb azure-identity`: -```python title="Request an app-only token for QuestDB" +```python title="Ingest into fx_trades with an app-only token" +import questdb from azure.identity import DefaultAzureCredential +from questdb import TimestampNanos # Application ID URI of the QuestDB application, followed by /.default scope = "api://8de84b90-1ea5-4e41-9e84-dba860aa01a6/.default" -token = DefaultAzureCredential().get_token(scope) - -conf = f"http::addr=localhost:9000;token={token.token};" +credential = DefaultAzureCredential() +token = credential.get_token(scope) + +conf = f"wss::addr=questdb.example.com:9000;token={token.token};" +with questdb.connect(conf) as db: + with db.sender() as sender: + sender.row( + "fx_trades", + symbols={"symbol": "EURUSD", "side": "buy"}, + columns={"price": 1.0842, "quantity": 100000.0}, + at=TimestampNanos.now(), + ) + sender.flush(wait=True) ``` -Use `conf` with the `Sender`, as in -[Non-interactive clients](#non-interactive-clients). For the +The client presents the same token on every connection that it opens, and +QuestDB rejects an expired token. Before `token.expires_on`, a Unix timestamp, +a long-running service closes the handle and connects again with a new token. +The credential caches the token in memory and requests a new one when the +cached token nears expiry, so calling `get_token()` before each reconnect is +cheap. See +[Authentication and TLS](/docs/connect/clients/python/#authentication-and-tls) +for the client options. For the [PGWire endpoint](#oidc-for-the-pgwire-endpoint), send the token as the -password of the `_sso` user instead. The token expires at `token.expires_on`, a -Unix timestamp, and QuestDB rejects an expired token, so a long-running service -requests a new token before then, and creates a new sender or connection with -it. +password of the `_sso` user instead. + +QuestDB reads the roles from the token, so a change to the roles of a service +takes effect when the service presents a new token. Azure caches managed +identity tokens for around 24 hours, so a role assigned to, or removed from, a +managed identity can take hours to reach QuestDB. See the +[managed identity best practices](https://learn.microsoft.com/en-us/entra/identity/managed-identities-azure-resources/managed-identity-best-practice-recommendations). +To cut a service off without waiting for its token to expire, revoke the +permissions of its QuestDB group.
From 583f26c224f37e01cc2ef0b14eee9681f6006288 Mon Sep 17 00:00:00 2001 From: glasstiger Date: Thu, 8 Oct 2026 23:56:16 +0100 Subject: [PATCH 04/29] docs(oidc): correct app-only access and table creation, restructure the Entra app-only guide - An app-only token without a role falls back to the groups claim. Document it, and how to keep app roles the only way in: keep services out of mapped security groups, or enable Assignment required. - QuestDB cannot create a table on ingestion for an external user. Replace the CREATE TABLE advice with creating tables in advance, and link OWNED BY. - Promote the managed identity guide to an H3 with step H4s, keeping its anchor. Add prerequisites, the PGWIRE requirements, rejection handling, a token renewal example for long-running services, and a verify step. - With acl.oidc.groups.encoded.in.token=true, the Web Console and ROPC use the ID token. Correct steps 7-8, non-interactive clients and the configuration reference. - Move the claim rules and the pre-4.0.2 differences into their own sections. Document that exp is required, and that a token can be accepted for up to 60 seconds plus acl.oidc.cache.ttl after it expires. - Configuration reference: single audience, what acl.oidc.cache.ttl caches, and links to the claim rules. - RBAC: QWP uses the HTTP endpoint permission, and OIDC authenticates external users and services. - Spell Entra ID consistently, and update the changelog. --- documentation/changelog.mdx | 10 +- documentation/configuration/oidc.md | 46 ++-- documentation/deployment/azure.md | 15 +- documentation/security/oidc.mdx | 337 ++++++++++++++++++++-------- documentation/security/rbac.md | 17 +- 5 files changed, 297 insertions(+), 128 deletions(-) diff --git a/documentation/changelog.mdx b/documentation/changelog.mdx index 48388af82c..07d95a6fda 100644 --- a/documentation/changelog.mdx +++ b/documentation/changelog.mdx @@ -12,17 +12,21 @@ This page tracks significant updates to the QuestDB documentation. ### New -- [User and group claims](/docs/security/oidc/#user-and-group-claims) - How QuestDB Enterprise 4.0.2 reads the principal and the groups from the user information: fallback claim lists, the rules for missing, `null`, and empty claims, why the principal must be unique, the startup checks, token validation, and the log messages of rejected logins -- [Accept managed identity and service principal tokens](/docs/security/oidc/#accept-managed-identity-and-service-principal-tokens) - Let services that run as a Microsoft Entra ID managed identity or service principal authenticate to QuestDB Enterprise 4.0.2 with app-only tokens, next to user logins: the Entra ID setup, the claim lists, the app role mapping, and a Python example +- [User and group claims](/docs/security/oidc/#user-and-group-claims) - How QuestDB Enterprise 4.0.2 reads the principal and the groups from the user information: fallback claim lists, the rules for missing, `null`, and empty claims, why the principal must be unique, the startup checks, token validation, the log messages of rejected logins, and how earlier versions differ +- [Microsoft Entra ID managed identities and service principals](/docs/security/oidc/#accept-managed-identity-and-service-principal-tokens) - Let services that run as a Microsoft Entra ID managed identity or service principal authenticate to QuestDB Enterprise 4.0.2 with app-only tokens, next to user logins: the Entra ID setup, the claim lists, the app role mapping, how to keep app roles the only way in, and Python examples that ingest and renew the token ### Reference - [`acl.oidc.groups.claim`](/docs/configuration/oidc/#acloidcgroupsclaim) and [`acl.oidc.sub.claim`](/docs/configuration/oidc/#acloidcsubclaim) - Documented the comma-separated fallback claim lists of QuestDB Enterprise 4.0.2 and the startup validation. `acl.oidc.groups.claim` is now listed with no default, as it must be set when OIDC is enabled - [`acl.oidc.pkce.required`](/docs/configuration/oidc/#acloidcpkcerequired) - Corrected the property name, previously documented as `acl.oidc.pkce.enabled`, which QuestDB does not recognize +- [`acl.oidc.groups.encoded.in.token`](/docs/configuration/oidc/#acloidcgroupsencodedintoken) - Documented that QuestDB Enterprise 4.0.2 rejects expired tokens and tokens without an `exp` claim, which earlier versions accept, and which token QuestDB reads the principal and the groups from +- [`acl.oidc.audience`](/docs/configuration/oidc/#acloidcaudience) and [`acl.oidc.cache.ttl`](/docs/configuration/oidc/#acloidccachettl) - Documented that QuestDB accepts a single audience, and what the cache holds when the groups are encoded in the token ### Updated -- [Mapping user permissions](/docs/security/oidc/#mapping-user-permissions) - Corrected what happens when the groups claim is missing or empty: QuestDB rejects the login. A user is authenticated without permissions only when none of their groups is mapped to a QuestDB group +- [Mapping user permissions](/docs/security/oidc/#mapping-user-permissions) - Corrected what happens when the groups claim is missing or empty: QuestDB rejects the login. A user is authenticated without permissions only when none of their groups is mapped to a QuestDB group. With the groups encoded in the token, a change to them takes effect when the client presents a new token +- [Authentication and authorization flow](/docs/security/oidc/#8-find-user-information) and [non-interactive clients](/docs/security/oidc/#non-interactive-clients) - Corrected which token reaches QuestDB with `acl.oidc.groups.encoded.in.token=true`: the ID token, not the access token +- [Endpoint permissions](/docs/security/rbac/#endpoint-permissions) - The `HTTP` permission also covers QWP over WebSocket, and the authentication methods point to OIDC for external users and services - [Microsoft Entra ID guide](/docs/security/oidc/#microsoft-entraid) - Uses `preferred_username` as the principal claim instead of the display name, and registers the application as single-tenant ## September 2026 diff --git a/documentation/configuration/oidc.md b/documentation/configuration/oidc.md index 16ec75ee8e..b6361c3435 100644 --- a/documentation/configuration/oidc.md +++ b/documentation/configuration/oidc.md @@ -26,6 +26,12 @@ For detailed information about OIDC, see the OAuth2 audience as set on the tokens issued by the OIDC Provider. Defaults to the client ID if not set. +With `acl.oidc.groups.encoded.in.token=true`, QuestDB accepts a token only when +its `aud` claim matches this value, and accepts a single audience. Keep the +default: the ID tokens that the Web Console sends carry the client ID, so +another value makes Web Console logins fail. See +[Token validation](/docs/security/oidc/#token-validation). + ### acl.oidc.client.id - **Default**: none @@ -204,22 +210,26 @@ which it connects. QuestDB reads the principal and the group memberships of an external user from the user information: the User Info endpoint's response or, when -`acl.oidc.groups.encoded.in.token` is `true`, the payload of the JWT that the -client presents. `acl.oidc.sub.claim` and `acl.oidc.groups.claim` each take a -single claim name or, since QuestDB Enterprise 4.0.2, a comma-separated list of -claim names in priority order. For how QuestDB picks the claims, with examples, -and for how versions before 4.0.2 differ, see +`acl.oidc.groups.encoded.in.token` is `true`, the payload of a JWT, such as an +ID token. `acl.oidc.sub.claim` and `acl.oidc.groups.claim` each take a single +claim name or, since QuestDB Enterprise 4.0.2, a comma-separated list of claim +names in priority order. For how QuestDB picks the claims, with examples, see [User and group claims](/docs/security/oidc/#user-and-group-claims) in the -OIDC guide. +OIDC guide. For how versions before 4.0.2 differ, see +[Versions before 4.0.2](/docs/security/oidc/#versions-before-402). ### acl.oidc.cache.ttl - **Default**: `30000` - **Reloadable**: no -User info cache entry TTL in milliseconds. QuestDB caches user info responses -for each valid access token. This setting controls how often the access token -is validated and user info refreshed. +How long QuestDB caches the user information of a valid token, in +milliseconds: the User Info endpoint's response or, when +`acl.oidc.groups.encoded.in.token` is `true`, the result of validating the +token. This setting controls how often a token is validated again and the user +information refreshed. With `acl.oidc.groups.encoded.in.token=true`, the user +information comes from the token itself, so a change to it takes effect when +the client presents a new token. ### acl.oidc.groups.claim @@ -241,20 +251,25 @@ On earlier versions, a list makes every OIDC login fail. With `acl.oidc.groups.encoded.in.token=true`, earlier versions also require a `groups` array in the token, whatever this setting names. +See [How QuestDB picks a claim](/docs/security/oidc/#how-questdb-picks-a-claim) +and [Startup validation](/docs/security/oidc/#startup-validation). + ### acl.oidc.groups.encoded.in.token - **Default**: `false` - **Reloadable**: no -When `true`, QuestDB reads the principal and the group memberships from the -JWT that the client presents, such as an ID token or an Entra ID app-only -access token, instead of calling the User Info endpoint. QuestDB validates the +When `true`, QuestDB reads the principal and the group memberships from a JWT +instead of calling the User Info endpoint: the ID token, which the Web Console +sends and QuestDB obtains itself in the ROPC flow, or a token that a client +presents, such as an Entra ID app-only access token. QuestDB validates the token itself, as described in [Token validation](/docs/security/oidc/#token-validation). Set to `true` if the OIDC Provider encodes group memberships directly into the token. -Since QuestDB Enterprise 4.0.2, QuestDB also rejects expired tokens. Earlier -versions do not check the `exp` claim of the token. +Since QuestDB Enterprise 4.0.2, QuestDB also rejects expired tokens and tokens +without an `exp` claim. Earlier versions do not check the `exp` claim of the +token. ### acl.oidc.sub.claim @@ -277,3 +292,6 @@ when it names a claim that `acl.oidc.groups.claim` also lists. An empty value does not fall back to the default `sub`. On earlier versions, a list makes every OIDC login fail. + +See [How QuestDB picks a claim](/docs/security/oidc/#how-questdb-picks-a-claim) +and [Startup validation](/docs/security/oidc/#startup-validation). diff --git a/documentation/deployment/azure.md b/documentation/deployment/azure.md index 4c9ee2babf..747e611533 100644 --- a/documentation/deployment/azure.md +++ b/documentation/deployment/azure.md @@ -342,13 +342,14 @@ QuestDB Enterprise adds production features for Azure: - **Blob Storage replication** - Continuous backup for durability - **Cold storage** - Move old partitions to Blob Storage, query on-demand - **High availability** - Automatic failover across instances -- **EntraID SSO** - Single sign-on with Microsoft Entra ID - -For EntraID integration, see the -[Microsoft EntraID OIDC guide](/docs/security/oidc/#microsoft-entraid). -Azure services that run as a managed identity or a service principal can also -authenticate to QuestDB with Entra ID app-only tokens. See -[Accept managed identity and service principal tokens](/docs/security/oidc/#accept-managed-identity-and-service-principal-tokens). +- **Entra ID SSO** - Single sign-on with Microsoft Entra ID + +For Entra ID integration, see the +[Microsoft Entra ID OIDC guide](/docs/security/oidc/#microsoft-entraid). +Since QuestDB Enterprise 4.0.2, Azure services that run as a managed identity +or a service principal can also authenticate to QuestDB with Entra ID app-only +tokens. See +[Microsoft Entra ID managed identities and service principals](/docs/security/oidc/#accept-managed-identity-and-service-principal-tokens). See [Enterprise Quick Start](/docs/getting-started/enterprise-quick-start/) for setup. diff --git a/documentation/security/oidc.mdx b/documentation/security/oidc.mdx index 437a289227..ff5bf65216 100644 --- a/documentation/security/oidc.mdx +++ b/documentation/security/oidc.mdx @@ -227,7 +227,9 @@ The validity of the tokens are configurable inside the OIDC Provider. With the tokens, the Web Console can interact with the database. -The access token is in the header of every request sent to QuestDB. +The access token is in the header of every request sent to QuestDB. With +`acl.oidc.groups.encoded.in.token=true`, the Web Console sends the ID token +instead, as described in step 8. > **Worried about exposing the token?** It is rather opaque and does not contain > user details. @@ -261,10 +263,10 @@ https://oidc.provider:443/idp/userinfo.openid Authorization: Bearer gslpJtzmmi6RwaPSx0dYGD4tEkom ``` -With `acl.oidc.groups.encoded.in.token=true`, QuestDB skips this call. It -validates the token itself, as described in -[Token validation](#token-validation), and reads the user information from the -token's payload. +With `acl.oidc.groups.encoded.in.token=true`, the Web Console sends the ID +token instead of the access token, and QuestDB skips this call. It validates +the ID token itself, as described in [Token validation](#token-validation), and +reads the user information from the token's payload. ### 9. Receive user information @@ -587,20 +589,20 @@ with Sender.from_conf(conf) as sender: sender.dataframe(df, table_name="foo", at="ts") ``` -Tokens that the OIDC provider issues to an application, rather than to a user, -often carry the principal and the groups under different claim names. To accept -both kinds of token with one configuration, list the claims of both in -`acl.oidc.sub.claim` and `acl.oidc.groups.claim`, as described in -[User and group claims](#user-and-group-claims). - -QuestDB must also be able to validate the application's token. With -`acl.oidc.groups.encoded.in.token=true`, the token must be a JWT whose `aud` -claim matches `acl.oidc.audience`, as described in -[Token validation](#token-validation). Otherwise, QuestDB sends the token to the +With `acl.oidc.groups.encoded.in.token=true`, QuestDB validates the token itself +and reads the user information from it, so the token must be a JWT that passes +[Token validation](#token-validation). In the first example above, send the ID +token, `tokens["id_token"]`, instead of the access token, which the provider +often issues for another audience. With +`acl.oidc.groups.encoded.in.token=false`, QuestDB sends the token to the provider's User Info endpoint, which some providers refuse for tokens issued to -applications. For Microsoft Entra ID managed identities and service principals, -see -[Accept managed identity and service principal tokens](#accept-managed-identity-and-service-principal-tokens). +applications. + +Tokens that the provider issues to an application, rather than to a user, often +carry the principal and the groups under different claim names. To accept both +kinds of token with one configuration, list the claims of both, as described in +[User and group claims](#user-and-group-claims). For Microsoft Entra ID, see +[Microsoft Entra ID managed identities and service principals](#accept-managed-identity-and-service-principal-tokens). ## OIDC for the PGWire endpoint @@ -635,8 +637,10 @@ access list. As a reminder, the access list is the list of permissions that determines what the user can and cannot do. -QuestDB itself does not store external users, nor their passwords or any other -authentication related detail. +QuestDB itself does not persist external users, nor their passwords or any +other authentication related detail. It keeps the groups of each external +user's latest login in memory only, as described in +[User and group claims](#user-and-group-claims). External users and their authentication methods are managed by the Identity Provider. @@ -684,8 +688,9 @@ External users cannot be given a query memory limit directly; mapped QuestDB group instead; see [memory limits](/docs/security/rbac/#memory-limits). -QuestDB works the list of external groups out from the User Info response -message. +QuestDB works the list of external groups out from the user information: the +User Info response or, when `acl.oidc.groups.encoded.in.token` is `true`, the +token's payload. If we take the example used earlier, we will see that the message contains a claim called `groups`. This name is configurable in QuestDB, and QuestDB can @@ -734,13 +739,20 @@ be synchronized with the Identity Providers. It should be set accordingly to your organization's policies. +With `acl.oidc.groups.encoded.in.token=true`, QuestDB reads the groups from the +token instead of asking the Identity Provider, so refreshing the cache does not +pick up changes. A change to the user's groups or status takes effect when the +client presents a new token. A token that QuestDB has already accepted stays +valid until it expires, as described in [Token validation](#token-validation). + ## User and group claims QuestDB reads two values from the user information of an external user: the principal, which identifies the user, and the groups the user belongs to. The user information is the response of the User Info endpoint or, when -`acl.oidc.groups.encoded.in.token` is `true`, the payload of the JWT that the -client presents, such as an ID token or an Entra ID app-only access token. +`acl.oidc.groups.encoded.in.token` is `true`, the payload of a JWT: the ID +token, which the Web Console sends and QuestDB obtains itself in the ROPC flow, +or a token that a client presents, such as an Entra ID app-only access token. - [`acl.oidc.sub.claim`](/docs/configuration/oidc/#acloidcsubclaim) names the claim that carries the principal, `sub` by default. The principal identifies @@ -754,8 +766,8 @@ QuestDB rejects the login when the user information carries no principal or no group. The principal must be unique for each user and service. QuestDB keeps one -external user for each principal, and every login replaces the groups of that -user with the groups of the login. Two people who resolve to the same principal +external user for each principal, in memory, and every login replaces the +groups of that user with the groups of the login. Two people who resolve to the same principal share one QuestDB user, and a session that is already open, such as a PGWire connection, can run with the groups of whoever logged in last. Pick a claim that the provider keeps unique, such as a username or an object ID, not a @@ -764,21 +776,9 @@ new principal. Permissions stay the same, because they come from the groups. :::warning Requires QuestDB Enterprise 4.0.2 -This section describes QuestDB Enterprise 4.0.2 and later. Earlier versions -differ as follows: - -- They read the whole value of each setting as a single claim name, so a claim - list makes every OIDC login fail. -- With `acl.oidc.groups.encoded.in.token=true`, they reject any token that does - not carry a `groups` array, whatever `acl.oidc.groups.claim` names. They do - not check the `exp` claim, so they accept expired tokens. -- They read a `null` claim as the value `null`. Every account whose principal - claim is `null` logs in as the same principal, `null`, and `null` and empty - group names count as groups. -- They reject a login when a claim that is not listed holds an array of - objects or nested arrays. -- They start with an empty `acl.oidc.sub.claim`, or with the same claim in both - settings, and then reject every OIDC login. +This section describes QuestDB Enterprise 4.0.2 and later. Claim lists require +4.0.2: on earlier versions, a claim list makes every OIDC login fail. For the +other differences, see [Versions before 4.0.2](#versions-before-402). ::: @@ -828,14 +828,17 @@ token payloads, shortened to the relevant claims, as follows: `roles` comes before `groups` because an app-only token can also carry a `groups` claim, and QuestDB takes the groups from the first claim on the list that holds one. See -[Accept managed identity and service principal tokens](#accept-managed-identity-and-service-principal-tokens). +[Microsoft Entra ID managed identities and service principals](#accept-managed-identity-and-service-principal-tokens). -The following rules decide which claim QuestDB uses. They apply to a single -claim name as well: +### How QuestDB picks a claim + +The following rules decide which claim QuestDB uses, for a single claim name +and for a list: - The order of the list decides, not the order of the claims in the token. - With the configuration above, a token that carries both `roles` and `groups` - gets its groups from `roles` alone. QuestDB does not combine groups from + With the configuration in [Fallback claim lists](#fallback-claim-lists), a + token that carries both `roles` and `groups` gets its groups from `roles` + alone. QuestDB does not combine groups from several claims. - A claim that is missing, `null`, or empty falls through to the next claim on the list. Empty means an empty string, or an empty array for a groups claim. @@ -882,8 +885,8 @@ itself before it reads any claim: - Its `aud` claim must match [`acl.oidc.audience`](/docs/configuration/oidc/#acloidcaudience), which defaults to the client ID. QuestDB accepts a single audience. -- Its `exp` claim must not have passed. QuestDB allows 60 seconds of clock - skew. +- It must carry an `exp` claim that has not passed. QuestDB allows 60 seconds + of clock skew. - It must carry a non-empty `sub` claim, whatever claims `acl.oidc.sub.claim` lists. @@ -896,8 +899,10 @@ single-tenant, as described in QuestDB checks a token when a request or a connection authenticates with it, and caches the result for up to -[`acl.oidc.cache.ttl`](/docs/configuration/oidc/#acloidccachettl). An open -PGWire or WebSocket connection stays open when its token expires. +[`acl.oidc.cache.ttl`](/docs/configuration/oidc/#acloidccachettl). A token can +therefore be accepted for up to 60 seconds plus `acl.oidc.cache.ttl` after it +expires. An open PGWire or WebSocket connection stays open when its token +expires, but QuestDB rejects a new connection that presents an expired token. With `acl.oidc.groups.encoded.in.token=false`, QuestDB sends the token to the provider's User Info endpoint, which validates it. See @@ -934,7 +939,7 @@ Request failed [errorMsg=Failed to decode JWT token Error(ExpiredSignature)] `InvalidAudience` usually means that the token was requested for another API, or that Entra ID issued a v1.0 access token. See -[Accept managed identity and service principal tokens](#accept-managed-identity-and-service-principal-tokens). +[Microsoft Entra ID managed identities and service principals](#accept-managed-identity-and-service-principal-tokens). Don't set `acl.oidc.audience` to the Application ID URI to work around it: QuestDB accepts a single audience, and the ID tokens of Web Console users carry the application ID, so their logins would fail. @@ -947,6 +952,24 @@ If the login succeeds but every request is denied, check that at least one of the user's groups is mapped to a QuestDB group that has the endpoint permission. See [Mapping user permissions](#mapping-user-permissions). +### Versions before 4.0.2 + +QuestDB Enterprise versions before 4.0.2 differ as follows: + +- They read the whole value of each setting as a single claim name, so a claim + list makes every OIDC login fail. +- With `acl.oidc.groups.encoded.in.token=true`, they reject any token that does + not carry a `groups` array, whatever `acl.oidc.groups.claim` names. They do + not check the `exp` claim, so they accept expired tokens and tokens without + an `exp` claim. +- They read a `null` claim as the value `null`. Every account whose principal + claim is `null` logs in as the same principal, `null`, and `null` and empty + group names count as groups. +- They reject a login when a claim that is not listed holds an array of + objects or nested arrays. +- They start with an empty `acl.oidc.sub.claim`, or with the same claim in both + settings, and then reject every OIDC login. + ## Configuration options For all OIDC-related configuration options of QuestDB, see @@ -956,7 +979,8 @@ For all OIDC-related configuration options of QuestDB, see ## Active Directory -The following sections are guides for setting up single sign-on (SSO) with various OAuth2 providers. +The following sections are guides for setting up single sign-on (SSO) with various OAuth2 providers, +and for authenticating services with app-only tokens from Microsoft Entra ID. ### PingFederate @@ -1304,14 +1328,14 @@ If all has been wired up well, then login will succeed.
-### Microsoft EntraId +### Microsoft Entra ID {#microsoft-entraid} This document sets up SSO authentication for the [QuestDB Web Console](/docs/getting-started/web-console/overview/) in -[Microsoft EntraID](https://www.microsoft.com/en-gb/security/business/identity-access/microsoft-entra-id), formerly known as Azure AD. +[Microsoft Entra ID](https://www.microsoft.com/en-gb/security/business/identity-access/microsoft-entra-id), formerly known as Azure AD. -It also shows how to let services that run as a managed identity or a service -principal call QuestDB with app-only tokens. See -[Accept managed identity and service principal tokens](#accept-managed-identity-and-service-principal-tokens). +To also let services that run as a managed identity or a service principal +call QuestDB with app-only tokens, complete this setup, then follow +[Microsoft Entra ID managed identities and service principals](#accept-managed-identity-and-service-principal-tokens). :::tip @@ -1381,7 +1405,7 @@ secret (e.g. a password) could not be kept safely. As single page applications r the browser, they fall into this category. The redirect URIs are also added to the _CORS_ (Cross-Origin Resource Sharing) policy -of EntraID. CORS is a mechanism to allow a web page, such as the Web Console, to access +of Entra ID. CORS is a mechanism to allow a web page, such as the Web Console, to access resources from a different domain than the one that served the page. In this context this means that we let the Web Console to access Entra ID, while its origin is the HTTP endpoint of QuestDB. @@ -1580,7 +1604,7 @@ To test, head to the Web Console and login. If all has been wired up well, then login will succeed, and the user will have the access granted to them. -#### Accept managed identity and service principal tokens +### Microsoft Entra ID managed identities and service principals {#accept-managed-identity-and-service-principal-tokens} Services that authenticate as themselves, with a managed identity or a service principal, receive app-only access tokens from Entra ID. These tokens carry no @@ -1589,12 +1613,29 @@ of its service principal in the `oid` claim, and list the [app roles](https://learn.microsoft.com/en-us/entra/identity-platform/howto-add-app-roles-in-apps) assigned to it in the `roles` claim. They can also carry a `groups` claim, when the service principal is a member of a security group and the QuestDB -application emits group claims, as set up under _Token configuration_ above. +application emits group claims, as set up under _Token configuration_ in +[Set up the client application in Entra ID](#set-up-the-client-application-in-entra-id). QuestDB treats the service as an external user, not as a QuestDB [service account](/docs/security/rbac/#users-and-service-accounts). Like any -external user, it gets its permissions only from the QuestDB groups that its -app roles map to, and permissions cannot be granted to it directly. +external user, it gets its permissions from the QuestDB groups that its claims +map to, and permissions cannot be granted to it directly. With the +configuration below, these are the groups that its app roles map to. When its +token carries no role but carries a `groups` claim, QuestDB takes the groups +from `groups` instead, and the service gets the permissions of its Entra ID +security groups. + +Unless _Assignment required_ is enabled for the QuestDB application, Entra ID +issues tokens for the application to any service principal in the tenant, and +the token of a service principal without a role carries no `roles` claim. To +make an app role the only way for a service to get permissions, do one of the +following: + +- Keep managed identities and service principals out of the security groups + that are mapped to QuestDB groups. +- Enable _Assignment required_ in the _Properties_ of the QuestDB application, + under _Enterprise applications_. Users then need an assignment too, directly + or through a group, to log in to the Web Console. :::warning Requires QuestDB Enterprise 4.0.2 @@ -1604,15 +1645,22 @@ included. ::: -First, configure the QuestDB application in Entra ID: +This setup builds on the Web Console setup above: the application registered in +[Set up the client application in Entra ID](#set-up-the-client-application-in-entra-id), +and the [QuestDB configuration](#questdb-configuration). + +#### Configure the application in Entra ID -1. Make sure that the application is single-tenant: its supported account type - must be _Accounts in this organizational directory only_. QuestDB does not - check the issuer or the tenant of a token, and an app role has the same - value in every tenant that uses the application. With a multitenant - registration, another tenant can assign the same role to its own service - principals, and QuestDB maps their tokens to the same QuestDB groups. -2. Make the application issue v2.0 access tokens: in its _Manifest_, set +1. Make sure that the application is single-tenant, as described in + [Token validation](#token-validation). An app role has the same value in + every tenant that uses the application, so with a multitenant registration, + another tenant could assign the role to its own service principals, and + QuestDB would map their tokens to the same QuestDB groups. +2. Under _Expose an API_, set the Application ID URI, which defaults to + `api://`. Services request their tokens for the scope + `/.default`. A token requested for any other API, such + as Microsoft Graph, carries that API in `aud`, and QuestDB rejects it. +3. Make the application issue v2.0 access tokens: in its _Manifest_, set `api.requestedAccessTokenVersion` to `2`. The legacy manifest format calls this property `accessTokenAcceptedVersion`. QuestDB accepts a token only when its `aud` claim matches `acl.oidc.audience`, which defaults to the @@ -1620,15 +1668,11 @@ First, configure the QuestDB application in Entra ID: v1.0 token, the default for applications that accept organizational accounts only, can carry the Application ID URI instead. Web Console logins are not affected, because the Web Console presents ID tokens. -3. Under _Expose an API_, set the Application ID URI, which defaults to - `api://`. Services request their tokens for the scope - `/.default`. A token requested for any other API, such - as Microsoft Graph, carries that API in `aud`, and QuestDB rejects it. 4. Under _App roles_, define a role for each kind of access, with the _Applications_ member type. The examples below use the value `QuestDB.Ingest`. -5. Assign at least one role to each managed identity or service principal. - Assign the role to the identity itself: Entra ID does not add a role +5. Assign a role to each managed identity or service principal that needs + access. Assign the role to the identity itself: Entra ID does not add a role assigned to a group to the tokens of the service principals in that group. - For the service principal of an app registration, open that registration, @@ -1639,10 +1683,13 @@ First, configure the QuestDB application in Entra ID: described in [Assign a managed identity to an application role](https://learn.microsoft.com/en-us/entra/identity/managed-identities-azure-resources/how-to-assign-app-role-managed-identity). -Then replace the two claim settings of the -[QuestDB configuration](#questdb-configuration) with fallback lists. Keep -`acl.oidc.groups.encoded.in.token=true`, so that QuestDB validates app-only -tokens against the signing keys of Entra ID, the same way as ID tokens: +#### Configure QuestDB + +Replace the two claim settings of the +[QuestDB configuration](#questdb-configuration) with fallback lists, and +restart QuestDB. Keep `acl.oidc.groups.encoded.in.token=true`, so that QuestDB +validates app-only tokens against the signing keys of Entra ID, the same way as +ID tokens: ```ini title="server.conf" # user tokens carry preferred_username, app-only tokens carry oid @@ -1657,13 +1704,14 @@ User logins take the principal from `preferred_username` and the groups from roles of the QuestDB application admit applications only. App-only tokens take the principal from `oid` and the groups from `roles`, even when they also carry a `groups` claim. QuestDB rejects a token that carries neither a role nor a -group. See [User and group claims](#user-and-group-claims) for how QuestDB -picks a claim from each list. +group. See [How QuestDB picks a claim](#how-questdb-picks-a-claim). + +#### Map app roles to QuestDB groups Map each app role to a QuestDB group by the role's value, the same way as an Entra ID group, and grant the group what the service needs. A service that -ingests over HTTP or QWP needs the `HTTP` endpoint permission and `INSERT` on -its tables: +ingests over HTTP or [QWP](/docs/connect/wire-protocols/qwp-ingress-websocket/) +needs the `HTTP` endpoint permission and `INSERT` on its tables: ```questdb-sql title="Map an app role to a group that ingests into fx_trades" CREATE GROUP ingestApps WITH EXTERNAL ALIAS 'QuestDB.Ingest'; @@ -1671,19 +1719,29 @@ GRANT HTTP TO ingestApps; GRANT INSERT ON fx_trades TO ingestApps; ``` -If the service creates tables, also grant the group `CREATE TABLE`. If it sends -columns that a table does not have yet, grant `ADD COLUMN` on the table. See +QuestDB cannot create a table on ingestion for an external user, because it +cannot make an external user the owner of the new table. Create the tables +that the service writes to in advance, with the columns that it sends. An +external user that creates a table with SQL must name one of its groups as the +owner, with [`OWNED BY`](/docs/query/sql/create-table/#owned-by). + +A service that connects to the [PGWire endpoint](#oidc-for-the-pgwire-endpoint) +also needs the `PGWIRE` endpoint permission. See [Endpoint permissions](/docs/security/rbac/#endpoint-permissions) for the other endpoints. -Finally, the service requests its token for the QuestDB application. With the +#### Request a token in the service + +The service requests its token for the QuestDB application. With the [Azure Identity library](https://learn.microsoft.com/en-us/python/api/overview/azure/identity-readme) for Python, `DefaultAzureCredential` requests the token as the managed identity of the Azure resource that the service runs on, or as a service principal set up in environment variables. For a user-assigned managed identity, set the `AZURE_CLIENT_ID` environment variable to the client ID of the identity. -Otherwise, the credential uses the system-assigned identity. Install the -libraries with `python3 -m pip install -U questdb azure-identity`: +Otherwise, the credential uses the system-assigned identity. + +The examples use the QWP API of the `questdb` package, version 5.0 or later. +Install the libraries with `python3 -m pip install -U questdb azure-identity`: ```python title="Ingest into fx_trades with an app-only token" import questdb @@ -1707,16 +1765,97 @@ with questdb.connect(conf) as db: sender.flush(wait=True) ``` -The client presents the same token on every connection that it opens, and -QuestDB rejects an expired token. Before `token.expires_on`, a Unix timestamp, -a long-running service closes the handle and connects again with a new token. -The credential caches the token in memory and requests a new one when the -cached token nears expiry, so calling `get_token()` before each reconnect is -cheap. See -[Authentication and TLS](/docs/connect/clients/python/#authentication-and-tls) -for the client options. For the -[PGWire endpoint](#oidc-for-the-pgwire-endpoint), send the token as the -password of the `_sso` user instead. +`flush(wait=True)` returns when the server has accepted the rows. Rows that the +server rejects, for example because the group lacks `INSERT` on the table, go +to the error handler of the connection, which logs them by default. See +[Server rejections](/docs/connect/clients/python/#server-rejections). + +The client presents the same token on every connection that it opens: the +first one, the additional connections of its pool, and the connections that +replace a failed one. An open connection stays open when its token expires, but +QuestDB rejects a new connection that presents an expired token. A long-running +service therefore closes `db`, the handle that `questdb.connect()` returns, +before `token.expires_on`, a Unix timestamp, and connects again with a new +token. The credential caches the token in memory and requests a new one when +the cached token nears expiry, so calling `get_token()` often is cheap: + +```python title="Keep ingesting across token expiry" +import time + +import questdb +from azure.identity import DefaultAzureCredential +from questdb import TimestampNanos + +SCOPE = "api://8de84b90-1ea5-4e41-9e84-dba860aa01a6/.default" +ADDR = "questdb.example.com:9000" +# connect again when the token expires within this many seconds +REFRESH_MARGIN_SECONDS = 600 + +credential = DefaultAzureCredential() + + +def on_rejection(error): + # rows that the server rejected, for example for a missing grant + print("rejected:", error.category.tag, error.message) + + +def connect(token): + conf = f"wss::addr={ADDR};token={token.token};" + return questdb.connect(conf, error_handler=on_rejection) + + +def next_trades(): + # replace with your source of trades + time.sleep(1) + return [("EURUSD", "buy", 1.0842, 100000.0)] + + +token = credential.get_token(SCOPE) +db = connect(token) +try: + while True: + trades = next_trades() + if time.time() > token.expires_on - REFRESH_MARGIN_SECONDS: + new_token = credential.get_token(SCOPE) + # the credential returns the cached token until it renews it + if new_token.token != token.token: + db.close() + token = new_token + db = connect(token) + with db.sender() as sender: + for symbol, side, price, quantity in trades: + sender.row( + "fx_trades", + symbols={"symbol": symbol, "side": side}, + columns={"price": price, "quantity": quantity}, + at=TimestampNanos.now(), + ) + sender.flush(wait=True) +finally: + db.close() +``` + +See [Authentication and TLS](/docs/connect/clients/python/#authentication-and-tls) +for the client options. For the [PGWire endpoint](#oidc-for-the-pgwire-endpoint), +enable `acl.oidc.pg.token.as.password.enabled`, and send the token as the +password of the `_sso` user. + +#### Verify the setup + +`current_user()` returns the principal that QuestDB sees for the service: the +object ID of its service principal, from the `oid` claim. Run it from the +service while it is connected, with pandas installed for `to_pandas()`: + +```python title="Check the principal of the service" +with db.query("SELECT current_user()") as result: + print(result.to_pandas()) +``` + +If the service cannot connect, the server log gives the reason, as described in +[Rejected logins](#rejected-logins). For a token that carries neither a role +nor a group, the log lists the claims that the token carries. + +#### Change or revoke access QuestDB reads the roles from the token, so a change to the roles of a service takes effect when the service presents a new token. Azure caches managed diff --git a/documentation/security/rbac.md b/documentation/security/rbac.md index 62dd034475..7d5eb4edc0 100644 --- a/documentation/security/rbac.md +++ b/documentation/security/rbac.md @@ -332,6 +332,13 @@ QuestDB supports three authentication methods: | **JWK Token** | ILP ingestion | InfluxDB Line Protocol | | **REST API Token** | Programmatic REST access | REST API | +QuestDB Enterprise can also authenticate external users and services with +tokens from an OpenID Connect provider, such as Microsoft Entra ID. They get +their permissions from the QuestDB groups that their external groups map to. +See [OpenID Connect (OIDC)](/docs/security/oidc/#user-permissions), and for +Azure services, +[Microsoft Entra ID managed identities and service principals](/docs/security/oidc/#accept-managed-identity-and-service-principal-tokens). + Users can have multiple authentication methods enabled simultaneously: ```questdb-sql @@ -360,11 +367,11 @@ to protect credentials in transit. Before a user can connect, they need endpoint permissions: -| Permission | Allows access to | -| ---------- | -------------------------------------- | -| `HTTP` | REST API, Web Console, ILP over HTTP | -| `PGWIRE` | PostgreSQL Wire Protocol (port 8812) | -| `ILP` | InfluxDB Line Protocol TCP (port 9009) | +| Permission | Allows access to | +| ---------- | -------------------------------------------------------- | +| `HTTP` | REST API, Web Console, ILP over HTTP, QWP over WebSocket | +| `PGWIRE` | PostgreSQL Wire Protocol (port 8812) | +| `ILP` | InfluxDB Line Protocol TCP (port 9009) | ```questdb-sql -- Typical setup for an interactive user From 61ad2e0abb6f973854b36c7b0a72306e6c19bdde Mon Sep 17 00:00:00 2001 From: glasstiger Date: Thu, 8 Oct 2026 23:56:16 +0100 Subject: [PATCH 05/29] docs(clients): link the OIDC non-interactive and Entra ID sections Point the OIDC links of the Python, .NET, C and C++, Go and QWP pages at the non-interactive clients section of the OIDC guide, and the Python page at the Entra ID managed identity example. --- documentation/connect/clients/c-and-cpp.md | 2 +- documentation/connect/clients/dotnet.md | 2 +- documentation/connect/clients/go.md | 4 ++-- documentation/connect/clients/python.md | 2 +- documentation/connect/wire-protocols/qwp-ingress-websocket.md | 2 +- 5 files changed, 6 insertions(+), 6 deletions(-) diff --git a/documentation/connect/clients/c-and-cpp.md b/documentation/connect/clients/c-and-cpp.md index 9b82a307ca..22f3c450fb 100644 --- a/documentation/connect/clients/c-and-cpp.md +++ b/documentation/connect/clients/c-and-cpp.md @@ -332,7 +332,7 @@ following are **not** supported: | Path | Status | Workaround | |---|---|---| -| OIDC token acquisition or in-band refresh | Not supported. The client does not negotiate with an identity provider and has no callback to refresh a token mid-session. | QuestDB itself supports OIDC; see [OpenID Connect](/docs/security/oidc/). Acquire an access token out-of-band from your IdP, pass it via `token=...`, and rebuild the pool when the token nears expiry. | +| OIDC token acquisition or in-band refresh | Not supported. The client does not negotiate with an identity provider and has no callback to refresh a token mid-session. | QuestDB itself supports OIDC; see [OpenID Connect](/docs/security/oidc/#non-interactive-clients). Acquire an access token out-of-band from your IdP, pass it via `token=...`, and rebuild the pool when the token nears expiry. | | Mutual TLS (client certificates) | Not supported. The QuestDB server does not negotiate client certificates regardless of client. | Use bearer-token auth over `wss`. See the connect-string reference's [TLS section](/docs/connect/clients/connect-string/#tls). | | Token rotation mid-session | Not supported. Credentials are presented once during the WebSocket upgrade and are not re-sent. | On token expiry, close the pool and build a fresh one with the new token. | diff --git a/documentation/connect/clients/dotnet.md b/documentation/connect/clients/dotnet.md index 684ea97cd3..3e3136de84 100644 --- a/documentation/connect/clients/dotnet.md +++ b/documentation/connect/clients/dotnet.md @@ -228,7 +228,7 @@ that purpose; left unset, it inherits `auth_timeout_ms`. | Path | Status | Workaround | |---|---|---| -| OIDC token acquisition or in-band refresh | Not supported by this client. It does not negotiate with an identity provider and has no callback to refresh a token mid-session. | QuestDB itself supports OIDC — see [OpenID Connect](/docs/security/oidc/). Acquire an access token out-of-band from your IdP, pass it via `token=...` above, and rebuild the sender / query client when the token nears expiry. | +| OIDC token acquisition or in-band refresh | Not supported by this client. It does not negotiate with an identity provider and has no callback to refresh a token mid-session. | QuestDB itself supports OIDC; see [OpenID Connect](/docs/security/oidc/#non-interactive-clients). Acquire an access token out-of-band from your IdP, pass it via `token=...` above, and rebuild the sender / query client when the token nears expiry. | | Mutual TLS (client certificates) | Not supported. The QuestDB server does not negotiate client certificates regardless of client. | Use bearer-token auth over `wss://`. See the connect-string reference for the canonical statement. | | Token rotation mid-session | Not supported. Credentials are presented once during the WebSocket upgrade and are not re-sent. | On token expiry, `await sender.DisposeAsync()` and build a fresh sender with the new token. The same applies to `QueryClient`. | diff --git a/documentation/connect/clients/go.md b/documentation/connect/clients/go.md index 32c9ac9c08..cd3c375680 100644 --- a/documentation/connect/clients/go.md +++ b/documentation/connect/clients/go.md @@ -360,8 +360,8 @@ db, err := qdb.Connect(ctx, The token is sent as an `Authorization: Bearer YOUR_BEARER_TOKEN` header on both the ingress and egress WebSocket upgrades. It is a **static credential**: the client sends exactly the string you pass and never refreshes or renews it. -Acquire it out of band — QuestDB Enterprise issues bearer tokens through its -[OpenID Connect flow](/docs/security/oidc/) — and manage its lifetime yourself. +Acquire it out of band, for example from the identity provider of the QuestDB Enterprise +[OpenID Connect flow](/docs/security/oidc/#non-interactive-clients), and manage its lifetime yourself. When the token expires or is rotated, construct a new handle with the new token. An expired or rejected token surfaces as an authentication failure (see [Connection-level errors](#connection-level-errors)). It is mutually exclusive diff --git a/documentation/connect/clients/python.md b/documentation/connect/clients/python.md index be8d79aa24..3ad5b1e93b 100644 --- a/documentation/connect/clients/python.md +++ b/documentation/connect/clients/python.md @@ -171,7 +171,7 @@ following are **not** supported: | Path | Status | Workaround | | --- | --- | --- | -| OIDC token acquisition or in-band refresh | Not supported. The client does not negotiate with an identity provider and cannot refresh a token mid-session. | QuestDB itself supports OIDC; see [OpenID Connect](/docs/security/oidc/). Acquire an access token out-of-band from your IdP, pass it via `token=...`, and rebuild the handle when the token nears expiry. | +| OIDC token acquisition or in-band refresh | Not supported. The client does not negotiate with an identity provider and cannot refresh a token mid-session. | QuestDB itself supports OIDC; see [OpenID Connect](/docs/security/oidc/#non-interactive-clients). Acquire an access token out-of-band from your IdP, pass it via `token=...`, and rebuild the handle when the token nears expiry. For an example with an Entra ID managed identity, see [Microsoft Entra ID managed identities and service principals](/docs/security/oidc/#accept-managed-identity-and-service-principal-tokens). | | Mutual TLS (client certificates) | Not supported. The QuestDB server does not negotiate client certificates regardless of client. | Use bearer-token auth over `wss`. | | Token rotation mid-session | Not supported. The handle keeps the credentials it was built with and presents them on every connection it opens — including reconnects and failover, so an expired token also breaks mid-session reconnection. | On token expiry, close the handle and build a fresh one with the new token. | diff --git a/documentation/connect/wire-protocols/qwp-ingress-websocket.md b/documentation/connect/wire-protocols/qwp-ingress-websocket.md index e27c7ea5eb..bced386f5b 100644 --- a/documentation/connect/wire-protocols/qwp-ingress-websocket.md +++ b/documentation/connect/wire-protocols/qwp-ingress-websocket.md @@ -143,7 +143,7 @@ Supported methods: [HTTP basic authentication](/docs/connect/compatibility/rest-api/#http-basic-authentication). - **Token-based auth** (Enterprise only): see [Authentication via token in QuestDB Enterprise](/docs/connect/compatibility/rest-api/#authentication-via-token-in-questdb-enterprise). -- **OIDC** (Enterprise only): see [OpenID Connect](/docs/security/oidc/). +- **OIDC** (Enterprise only): see [OpenID Connect](/docs/security/oidc/#non-interactive-clients). A failed authentication results in a `401` or `403` HTTP response before the WebSocket connection is established. No QWP-level auth handshake exists. From fd4f5a3d8f1e6a331faeb0fad0ef0196eea103e3 Mon Sep 17 00:00:00 2001 From: glasstiger Date: Fri, 9 Oct 2026 00:57:36 +0100 Subject: [PATCH 06/29] docs(oidc): address review findings on claims, token validation and the Entra app-only guide - Principal claim: add "Choose the principal claim" with the oid versus preferred_username trade-off. Entra ID documents preferred_username as mutable and reusable, so the Entra guide and the configuration reference no longer call it unique, and say when to use oid. - External users cannot create a table or add a column on ingestion. State it for every OIDC user in "Tables created by external users", note that the non-interactive examples need the table to exist, and create fx_trades in the managed identity guide. - Step 7: in token mode the Web Console sends the ID token, a readable JWT, so replace the "opaque token" reassurance with a TLS note. - Managed identity guide: open with what and why, the 4.0.2 warning and the prerequisites. Move Assignment required into its own step, and require tenant-wide admin consent with it. - Make Token validation, Troubleshooting OIDC logins and Versions before 4.0.2 their own sections, add an upgrade checklist, rename "Active Directory" to "Identity provider guides" with its anchor kept, and link the managed identity guide from the intro. - Make the token expiry wording consistent, document that the Web Console's HTTP session is not tied to the token, and state that a token whose aud list contains acl.oidc.audience is accepted. - Renewal example: connect with the new token before closing the old handle, and keep both when Entra ID or QuestDB cannot be reached. - Client pages, failover and RBAC: say which token to send, add QWP to the second endpoint table, and describe external users. --- documentation/changelog.mdx | 12 +- documentation/configuration/oidc.md | 16 +- documentation/connect/clients/c-and-cpp.md | 2 +- documentation/connect/clients/dotnet.md | 2 +- documentation/connect/clients/python.md | 2 +- documentation/high-availability/failover.md | 5 +- documentation/security/oidc.mdx | 396 ++++++++++++++------ documentation/security/rbac.md | 17 +- 8 files changed, 305 insertions(+), 147 deletions(-) diff --git a/documentation/changelog.mdx b/documentation/changelog.mdx index 07d95a6fda..dd77df67d9 100644 --- a/documentation/changelog.mdx +++ b/documentation/changelog.mdx @@ -12,7 +12,9 @@ This page tracks significant updates to the QuestDB documentation. ### New -- [User and group claims](/docs/security/oidc/#user-and-group-claims) - How QuestDB Enterprise 4.0.2 reads the principal and the groups from the user information: fallback claim lists, the rules for missing, `null`, and empty claims, why the principal must be unique, the startup checks, token validation, the log messages of rejected logins, and how earlier versions differ +- [User and group claims](/docs/security/oidc/#user-and-group-claims) - How QuestDB Enterprise 4.0.2 reads the principal and the groups from the user information: fallback claim lists, the rules for missing, `null`, and empty claims, how to choose the principal claim, and the startup checks +- [Token validation](/docs/security/oidc/#token-validation), [Troubleshooting OIDC logins](/docs/security/oidc/#troubleshooting-oidc-logins), and [Versions before 4.0.2](/docs/security/oidc/#versions-before-402) - What QuestDB checks in a token and how long it accepts one after it expires, the log messages of rejected logins, how earlier versions differ, and what to check when you upgrade +- [Tables created by external users](/docs/security/oidc/#tables-created-by-external-users) - Ingestion cannot create a table or add a column for a user who logs in through OIDC, so create the tables in advance, or create them with SQL and `OWNED BY` - [Microsoft Entra ID managed identities and service principals](/docs/security/oidc/#accept-managed-identity-and-service-principal-tokens) - Let services that run as a Microsoft Entra ID managed identity or service principal authenticate to QuestDB Enterprise 4.0.2 with app-only tokens, next to user logins: the Entra ID setup, the claim lists, the app role mapping, how to keep app roles the only way in, and Python examples that ingest and renew the token ### Reference @@ -20,14 +22,14 @@ This page tracks significant updates to the QuestDB documentation. - [`acl.oidc.groups.claim`](/docs/configuration/oidc/#acloidcgroupsclaim) and [`acl.oidc.sub.claim`](/docs/configuration/oidc/#acloidcsubclaim) - Documented the comma-separated fallback claim lists of QuestDB Enterprise 4.0.2 and the startup validation. `acl.oidc.groups.claim` is now listed with no default, as it must be set when OIDC is enabled - [`acl.oidc.pkce.required`](/docs/configuration/oidc/#acloidcpkcerequired) - Corrected the property name, previously documented as `acl.oidc.pkce.enabled`, which QuestDB does not recognize - [`acl.oidc.groups.encoded.in.token`](/docs/configuration/oidc/#acloidcgroupsencodedintoken) - Documented that QuestDB Enterprise 4.0.2 rejects expired tokens and tokens without an `exp` claim, which earlier versions accept, and which token QuestDB reads the principal and the groups from -- [`acl.oidc.audience`](/docs/configuration/oidc/#acloidcaudience) and [`acl.oidc.cache.ttl`](/docs/configuration/oidc/#acloidccachettl) - Documented that QuestDB accepts a single audience, and what the cache holds when the groups are encoded in the token +- [`acl.oidc.audience`](/docs/configuration/oidc/#acloidcaudience) and [`acl.oidc.cache.ttl`](/docs/configuration/oidc/#acloidccachettl) - Documented that `acl.oidc.audience` takes a single value, which the `aud` claim of a token must be or contain, and what the cache holds when the groups are encoded in the token ### Updated -- [Mapping user permissions](/docs/security/oidc/#mapping-user-permissions) - Corrected what happens when the groups claim is missing or empty: QuestDB rejects the login. A user is authenticated without permissions only when none of their groups is mapped to a QuestDB group. With the groups encoded in the token, a change to them takes effect when the client presents a new token +- [Mapping user permissions](/docs/security/oidc/#mapping-user-permissions) - Corrected what happens when the groups claim is missing or empty: QuestDB rejects the login. A user is authenticated without permissions only when none of their groups is mapped to a QuestDB group. With the groups encoded in the token, a change to them takes effect when the client presents a new token. The HTTP session of the Web Console is not tied to the token - [Authentication and authorization flow](/docs/security/oidc/#8-find-user-information) and [non-interactive clients](/docs/security/oidc/#non-interactive-clients) - Corrected which token reaches QuestDB with `acl.oidc.groups.encoded.in.token=true`: the ID token, not the access token -- [Endpoint permissions](/docs/security/rbac/#endpoint-permissions) - The `HTTP` permission also covers QWP over WebSocket, and the authentication methods point to OIDC for external users and services -- [Microsoft Entra ID guide](/docs/security/oidc/#microsoft-entraid) - Uses `preferred_username` as the principal claim instead of the display name, and registers the application as single-tenant +- [Endpoint permissions](/docs/security/rbac/#endpoint-permissions) - The `HTTP` permission also covers QWP over WebSocket. The [authentication methods](/docs/security/rbac/#authentication) and [users and service accounts](/docs/security/rbac/#users-and-service-accounts) describe external users, who log in through OIDC +- [Microsoft Entra ID guide](/docs/security/oidc/#microsoft-entraid) - Uses `preferred_username` as the principal claim instead of the display name, explains when to use `oid` instead, and registers the application as single-tenant ## September 2026 diff --git a/documentation/configuration/oidc.md b/documentation/configuration/oidc.md index b6361c3435..6f837754d3 100644 --- a/documentation/configuration/oidc.md +++ b/documentation/configuration/oidc.md @@ -27,9 +27,9 @@ OAuth2 audience as set on the tokens issued by the OIDC Provider. Defaults to the client ID if not set. With `acl.oidc.groups.encoded.in.token=true`, QuestDB accepts a token only when -its `aud` claim matches this value, and accepts a single audience. Keep the -default: the ID tokens that the Web Console sends carry the client ID, so -another value makes Web Console logins fail. See +its `aud` claim is this value, or a list that contains it. The setting takes a +single value. Keep the default: the ID tokens that the Web Console sends carry +the client ID, so another value makes Web Console logins fail. See [Token validation](/docs/security/oidc/#token-validation). ### acl.oidc.client.id @@ -278,10 +278,12 @@ token. The claim in the user information that contains the user's principal, such as a username, an email address, or an object ID. Displayed in the Web Console, -returned by `current_user()`, and logged for audit purposes. The principal must -be unique for each user: QuestDB keeps one external user per principal and -replaces its groups at every login, so users who share a principal share -permissions. Avoid display names, such as the `name` claim. +returned by `current_user()`, and logged for audit purposes. Pick a claim that +identifies one user or service, and that the provider never gives to anyone +else: QuestDB keeps one external user per principal and replaces its groups at +every login, so users who share a principal share permissions. Avoid display +names, such as the `name` claim. See +[Choose the principal claim](/docs/security/oidc/#choose-the-principal-claim). Since QuestDB Enterprise 4.0.2, accepts a comma-separated list of claims in priority order, such as `preferred_username,oid`. QuestDB takes the principal diff --git a/documentation/connect/clients/c-and-cpp.md b/documentation/connect/clients/c-and-cpp.md index 22f3c450fb..511bff592a 100644 --- a/documentation/connect/clients/c-and-cpp.md +++ b/documentation/connect/clients/c-and-cpp.md @@ -332,7 +332,7 @@ following are **not** supported: | Path | Status | Workaround | |---|---|---| -| OIDC token acquisition or in-band refresh | Not supported. The client does not negotiate with an identity provider and has no callback to refresh a token mid-session. | QuestDB itself supports OIDC; see [OpenID Connect](/docs/security/oidc/#non-interactive-clients). Acquire an access token out-of-band from your IdP, pass it via `token=...`, and rebuild the pool when the token nears expiry. | +| OIDC token acquisition or in-band refresh | Not supported. The client does not negotiate with an identity provider and has no callback to refresh a token mid-session. | QuestDB itself supports OIDC; see [OpenID Connect](/docs/security/oidc/#non-interactive-clients) for which token to send. Acquire the token out-of-band from your IdP, pass it via `token=...`, and rebuild the pool when the token nears expiry. | | Mutual TLS (client certificates) | Not supported. The QuestDB server does not negotiate client certificates regardless of client. | Use bearer-token auth over `wss`. See the connect-string reference's [TLS section](/docs/connect/clients/connect-string/#tls). | | Token rotation mid-session | Not supported. Credentials are presented once during the WebSocket upgrade and are not re-sent. | On token expiry, close the pool and build a fresh one with the new token. | diff --git a/documentation/connect/clients/dotnet.md b/documentation/connect/clients/dotnet.md index 3e3136de84..d1c9516b5e 100644 --- a/documentation/connect/clients/dotnet.md +++ b/documentation/connect/clients/dotnet.md @@ -228,7 +228,7 @@ that purpose; left unset, it inherits `auth_timeout_ms`. | Path | Status | Workaround | |---|---|---| -| OIDC token acquisition or in-band refresh | Not supported by this client. It does not negotiate with an identity provider and has no callback to refresh a token mid-session. | QuestDB itself supports OIDC; see [OpenID Connect](/docs/security/oidc/#non-interactive-clients). Acquire an access token out-of-band from your IdP, pass it via `token=...` above, and rebuild the sender / query client when the token nears expiry. | +| OIDC token acquisition or in-band refresh | Not supported by this client. It does not negotiate with an identity provider and has no callback to refresh a token mid-session. | QuestDB itself supports OIDC; see [OpenID Connect](/docs/security/oidc/#non-interactive-clients) for which token to send. Acquire the token out-of-band from your IdP, pass it via `token=...` above, and rebuild the sender / query client when the token nears expiry. | | Mutual TLS (client certificates) | Not supported. The QuestDB server does not negotiate client certificates regardless of client. | Use bearer-token auth over `wss://`. See the connect-string reference for the canonical statement. | | Token rotation mid-session | Not supported. Credentials are presented once during the WebSocket upgrade and are not re-sent. | On token expiry, `await sender.DisposeAsync()` and build a fresh sender with the new token. The same applies to `QueryClient`. | diff --git a/documentation/connect/clients/python.md b/documentation/connect/clients/python.md index 3ad5b1e93b..f16125ae58 100644 --- a/documentation/connect/clients/python.md +++ b/documentation/connect/clients/python.md @@ -171,7 +171,7 @@ following are **not** supported: | Path | Status | Workaround | | --- | --- | --- | -| OIDC token acquisition or in-band refresh | Not supported. The client does not negotiate with an identity provider and cannot refresh a token mid-session. | QuestDB itself supports OIDC; see [OpenID Connect](/docs/security/oidc/#non-interactive-clients). Acquire an access token out-of-band from your IdP, pass it via `token=...`, and rebuild the handle when the token nears expiry. For an example with an Entra ID managed identity, see [Microsoft Entra ID managed identities and service principals](/docs/security/oidc/#accept-managed-identity-and-service-principal-tokens). | +| OIDC token acquisition or in-band refresh | Not supported. The client does not negotiate with an identity provider and cannot refresh a token mid-session. | QuestDB itself supports OIDC; see [OpenID Connect](/docs/security/oidc/#non-interactive-clients) for which token to send. Acquire the token out-of-band from your IdP, pass it via `token=...`, and rebuild the handle when the token nears expiry. For an example with an Entra ID managed identity, see [Microsoft Entra ID managed identities and service principals](/docs/security/oidc/#accept-managed-identity-and-service-principal-tokens). | | Mutual TLS (client certificates) | Not supported. The QuestDB server does not negotiate client certificates regardless of client. | Use bearer-token auth over `wss`. | | Token rotation mid-session | Not supported. The handle keeps the credentials it was built with and presents them on every connection it opens — including reconnects and failover, so an expired token also breaks mid-session reconnection. | On token expiry, close the handle and build a fresh one with the new token. | diff --git a/documentation/high-availability/failover.md b/documentation/high-availability/failover.md index 3b153faaa8..5ebca4c0fe 100644 --- a/documentation/high-availability/failover.md +++ b/documentation/high-availability/failover.md @@ -284,8 +284,9 @@ The [minimal HTTP server](/docs/operations/logging-metrics/#minimal-http-server) on port 9003 exposes the same switch to external coordinators. It accepts the same credentials as the main HTTP server: HTTP basic authentication, a [REST token](/docs/connect/compatibility/rest-api/#authentication-via-token-in-questdb-enterprise) -as `Authorization: Bearer`, or an OIDC access token. The principal needs the -`HTTP` endpoint permission and, for the switch, `SWITCH ROLE`. With access +as `Authorization: Bearer`, or an +[OIDC token](/docs/security/oidc/#non-interactive-clients). The principal needs +the `HTTP` endpoint permission and, for the switch, `SWITCH ROLE`. With access control disabled no credentials are needed. TLS for this port is configured with the `http.min.tls.*` settings on the [TLS](/docs/configuration/tls/) page. diff --git a/documentation/security/oidc.mdx b/documentation/security/oidc.mdx index ff5bf65216..4dbb53ea6f 100644 --- a/documentation/security/oidc.mdx +++ b/documentation/security/oidc.mdx @@ -1,13 +1,13 @@ --- title: OpenID Connect (OIDC) Integration -description: Configure OpenID Connect (OIDC) in QuestDB Enterprise for Web Console SSO, token authentication for clients and services, claim mapping, and Entra ID managed identities. +description: "Configure OIDC in QuestDB Enterprise: Web Console SSO, token authentication for clients and services, claim mapping, and Entra ID managed identities." --- import Screenshot from "@theme/Screenshot"; import { EnterpriseNote } from "@site/src/components/EnterpriseNote" - OpenID Connect (OIDC) enables SSO authentication with external Identity Providers. + OpenID Connect (OIDC) enables SSO and token authentication with external Identity Providers. OpenID Connect (OIDC) integrates with Identity Providers (IdP) external to @@ -16,7 +16,9 @@ QuestDB. It is a convenient way to integrate QuestDB into your enterprise environment. It provides SSO (Single Sign-On) for the [Web Console](/docs/getting-started/web-console/overview/), and token authentication for applications and services that connect to -QuestDB. +QuestDB. For Azure services that run as a managed identity or a service +principal, see +[Microsoft Entra ID managed identities and service principals](#accept-managed-identity-and-service-principal-tokens). Microsoft Active Directory and Azure AD, for example, can be turned into an Identity Provider. @@ -231,8 +233,14 @@ The access token is in the header of every request sent to QuestDB. With `acl.oidc.groups.encoded.in.token=true`, the Web Console sends the ID token instead, as described in step 8. -> **Worried about exposing the token?** It is rather opaque and does not contain -> user details. +:::note + +Treat the token as a credential, and serve QuestDB over +[TLS](/docs/security/tls/). An access token can be opaque, but an ID token is a +signed JWT that anyone who holds it can decode, including the user's name and +groups. + +::: To carry out permission checks, the database has to know more about the user. @@ -389,7 +397,10 @@ with request.urlopen(req) as f: access_token = resp["access_token"] ``` -This token can be used to authenticate with QuestDB: +This token can be used to authenticate with QuestDB. With +`acl.oidc.groups.encoded.in.token=true`, use the ID token, `resp["id_token"]`, +instead. [Non-interactive clients](#non-interactive-clients) explains which +token to send: ```python query = parse.urlencode({ @@ -589,19 +600,34 @@ with Sender.from_conf(conf) as sender: sender.dataframe(df, table_name="foo", at="ts") ``` -With `acl.oidc.groups.encoded.in.token=true`, QuestDB validates the token itself -and reads the user information from it, so the token must be a JWT that passes -[Token validation](#token-validation). In the first example above, send the ID -token, `tokens["id_token"]`, instead of the access token, which the provider -often issues for another audience. With -`acl.oidc.groups.encoded.in.token=false`, QuestDB sends the token to the -provider's User Info endpoint, which some providers refuse for tokens issued to -applications. +Which token to send depends on where QuestDB reads the user information from: + +- With `acl.oidc.groups.encoded.in.token=false`, QuestDB sends the token to the + provider's User Info endpoint, so send the access token. Some providers + refuse tokens issued to applications at that endpoint. +- With `acl.oidc.groups.encoded.in.token=true`, QuestDB validates the token + itself, so send a JWT that passes [Token validation](#token-validation). For + a user, that is the ID token, because the provider often issues the access + token for another audience. The `aud` claim of an ID token is the client + that requested it, so request the token with QuestDB's client ID, the value + of `acl.oidc.client.id`. The examples above assume that it is `testclient`. + +With `acl.oidc.groups.encoded.in.token=true`, the first example above sends the +ID token instead of the access token: + +```python +conf = f"http::addr=localhost:9000;token={tokens['id_token']};" +``` + +QuestDB cannot create a table or add a column on ingestion for an external +user, so the examples above need the table `foo` to exist, with the columns of +`data.csv`. See +[Tables created by external users](#tables-created-by-external-users). Tokens that the provider issues to an application, rather than to a user, often carry the principal and the groups under different claim names. To accept both -kinds of token with one configuration, list the claims of both, as described in -[User and group claims](#user-and-group-claims). For Microsoft Entra ID, see +kinds of token with one configuration, list the claims of both. See +[Fallback claim lists](#fallback-claim-lists), and for Microsoft Entra ID, [Microsoft Entra ID managed identities and service principals](#accept-managed-identity-and-service-principal-tokens). ## OIDC for the PGWire endpoint @@ -688,9 +714,8 @@ External users cannot be given a query memory limit directly; mapped QuestDB group instead; see [memory limits](/docs/security/rbac/#memory-limits). -QuestDB works the list of external groups out from the user information: the -User Info response or, when `acl.oidc.groups.encoded.in.token` is `true`, the -token's payload. +QuestDB works the list of external groups out from the +[user information](#user-and-group-claims). If we take the example used earlier, we will see that the message contains a claim called `groups`. This name is configurable in QuestDB, and QuestDB can @@ -698,7 +723,7 @@ fall back to other claims when a token does not carry it. See [User and group claims](#user-and-group-claims). If the groups claim is missing or holds no group name, QuestDB rejects the -login. +login. See [Troubleshooting OIDC logins](#troubleshooting-oidc-logins). If none of the user's groups is mapped to a QuestDB group, the user is authenticated, but has no permissions at all. @@ -742,17 +767,47 @@ It should be set accordingly to your organization's policies. With `acl.oidc.groups.encoded.in.token=true`, QuestDB reads the groups from the token instead of asking the Identity Provider, so refreshing the cache does not pick up changes. A change to the user's groups or status takes effect when the -client presents a new token. A token that QuestDB has already accepted stays -valid until it expires, as described in [Token validation](#token-validation). +client presents a new token. Until then, QuestDB accepts the old token for up +to 60 seconds plus `acl.oidc.cache.ttl` after it expires, and connections that +are already open stay open. See [Token validation](#token-validation). + +The HTTP session that the Web Console opens at login is not tied to the token. +QuestDB accepts requests in the session, with the groups of the user's latest +login, until the user logs out or the session has been idle for +`http.session.timeout`, 30 minutes by default. + +### Tables created by external users + +When a user creates a table or adds a column, QuestDB grants that user all +permissions on it. Permissions cannot be granted to external users, so: + +- Ingestion, over ILP/HTTP or QWP, cannot create a table or add a column for an + external user. Create the tables in advance, with every column that the + client sends. +- An external user that creates a table or adds a column with SQL must name + one of its groups as the owner, with `OWNED BY`. See + [`CREATE TABLE`](/docs/query/sql/create-table/#owned-by) and + [`ALTER TABLE ADD COLUMN`](/docs/query/sql/alter-table-add-column/#owned-by). ## User and group claims +:::warning Requires QuestDB Enterprise 4.0.2 + +This section describes QuestDB Enterprise 4.0.2 and later. For earlier +versions, see [Versions before 4.0.2](#versions-before-402). + +::: + QuestDB reads two values from the user information of an external user: the principal, which identifies the user, and the groups the user belongs to. The -user information is the response of the User Info endpoint or, when -`acl.oidc.groups.encoded.in.token` is `true`, the payload of a JWT: the ID -token, which the Web Console sends and QuestDB obtains itself in the ROPC flow, -or a token that a client presents, such as an Entra ID app-only access token. +user information is: + +- By default, the response of the provider's User Info endpoint. +- With `acl.oidc.groups.encoded.in.token=true`, the payload of a JWT: the ID + token, for Web Console logins and in the ROPC flow, or the token that a + client presents, such as an Entra ID app-only access token. + +Two settings name the claims that QuestDB reads: - [`acl.oidc.sub.claim`](/docs/configuration/oidc/#acloidcsubclaim) names the claim that carries the principal, `sub` by default. The principal identifies @@ -765,23 +820,28 @@ or a token that a client presents, such as an Entra ID app-only access token. QuestDB rejects the login when the user information carries no principal or no group. -The principal must be unique for each user and service. QuestDB keeps one -external user for each principal, in memory, and every login replaces the -groups of that user with the groups of the login. Two people who resolve to the same principal -share one QuestDB user, and a session that is already open, such as a PGWire -connection, can run with the groups of whoever logged in last. Pick a claim -that the provider keeps unique, such as a username or an object ID, not a -display name. Changing the claim on an existing deployment gives every user a +### Choose the principal claim + +QuestDB keeps one in-memory external user for each principal, and every login +replaces the groups of that user. Two people who resolve to the same principal +therefore share one QuestDB user, and a session that is already open, such as +a PGWire connection, can run with the groups of whoever logged in last. Pick a +claim that identifies one user or service, and that the provider never gives +to anyone else: + +- An object ID, such as `oid` in Microsoft Entra ID, never changes and is never + reused, but the Web Console, the logs, and `current_user()` then show an + opaque ID. +- A username or an email address, such as `preferred_username` in Microsoft + Entra ID, is readable, but the provider can change it, and can give the name + of a former user to a new one. Microsoft + [documents `preferred_username` as mutable](https://learn.microsoft.com/en-us/entra/identity-platform/id-token-claims-reference). + Use such a claim only if your organization never reuses names. + +Never use a display name, such as the `name` claim, because different people +can share one. Changing the claim on an existing deployment gives every user a new principal. Permissions stay the same, because they come from the groups. -:::warning Requires QuestDB Enterprise 4.0.2 - -This section describes QuestDB Enterprise 4.0.2 and later. Claim lists require -4.0.2: on earlier versions, a claim list makes every OIDC login fail. For the -other differences, see [Versions before 4.0.2](#versions-before-402). - -::: - ### Fallback claim lists Both settings accept a comma-separated list of claim names in priority order, @@ -838,8 +898,7 @@ and for a list: - The order of the list decides, not the order of the claims in the token. With the configuration in [Fallback claim lists](#fallback-claim-lists), a token that carries both `roles` and `groups` gets its groups from `roles` - alone. QuestDB does not combine groups from - several claims. + alone. QuestDB does not combine groups from several claims. - A claim that is missing, `null`, or empty falls through to the next claim on the list. Empty means an empty string, or an empty array for a groups claim. The string `"null"` counts as missing too. @@ -875,7 +934,7 @@ refuses to start when: - Both settings list the same claim: `claim cannot be listed in both acl.oidc.sub.claim and acl.oidc.groups.claim [claim=oid]`. -### Token validation +## Token validation With `acl.oidc.groups.encoded.in.token=true`, QuestDB validates the token itself before it reads any claim: @@ -884,9 +943,11 @@ itself before it reads any claim: the provider publishes. - Its `aud` claim must match [`acl.oidc.audience`](/docs/configuration/oidc/#acloidcaudience), which - defaults to the client ID. QuestDB accepts a single audience. + defaults to the client ID. The setting takes a single value. A token whose + `aud` claim is a list is accepted when the list contains that value. - It must carry an `exp` claim that has not passed. QuestDB allows 60 seconds - of clock skew. + of clock skew. Versions before 4.0.2 do not check `exp`. See + [Versions before 4.0.2](#versions-before-402). - It must carry a non-empty `sub` claim, whatever claims `acl.oidc.sub.claim` lists. @@ -899,16 +960,18 @@ single-tenant, as described in QuestDB checks a token when a request or a connection authenticates with it, and caches the result for up to -[`acl.oidc.cache.ttl`](/docs/configuration/oidc/#acloidccachettl). A token can -therefore be accepted for up to 60 seconds plus `acl.oidc.cache.ttl` after it -expires. An open PGWire or WebSocket connection stays open when its token -expires, but QuestDB rejects a new connection that presents an expired token. +[`acl.oidc.cache.ttl`](/docs/configuration/oidc/#acloidccachettl). Because of +the clock skew and the cache, QuestDB can accept a token, on new connections +too, for up to 60 seconds plus `acl.oidc.cache.ttl` after it expires. After +that, it rejects every request and new connection that presents the token. A +PGWire or WebSocket connection that is already open stays open when its token +expires. With `acl.oidc.groups.encoded.in.token=false`, QuestDB sends the token to the provider's User Info endpoint, which validates it. See [Find user information](#8-find-user-information). -### Rejected logins +## Troubleshooting OIDC logins When the user information carries none of the listed principal claims, or none of the listed groups claims, QuestDB rejects the login. HTTP clients receive @@ -923,10 +986,16 @@ Failed to find required claims [subClaims=[preferred_username,oid], groupsClaims Compare the claims in `userInfo` with the two lists to find the claim names that your provider uses. A listed claim of an unexpected shape, such as an object in a groups claim, is logged as -`Request failed [error=unexpected input format ...]` instead. With Microsoft -Entra ID, a user who belongs to more than 200 groups gets no `groups` claim in -the token, so QuestDB rejects the login unless another listed claim carries -groups. +`Request failed [error=unexpected input format ...]` instead. Versions before +4.0.2 log `Failed to find required claims [claims=sub,groups userInfo=...]`, +with the two configured claim names. + +With Microsoft Entra ID, a user who belongs to more than 200 groups, nested +groups included, gets no `groups` claim in the token, so QuestDB rejects the +login unless another listed claim carries groups. To avoid this, emit only the +groups that are assigned to the application: in the _Token configuration_ of +the QuestDB application, select _Groups assigned to the application_, then +assign the mapped groups to the application under _Enterprise applications_. A token that fails [token validation](#token-validation) is rejected the same way, but the server log gives the reason instead of the claim lists, for @@ -935,14 +1004,21 @@ example: ```text Request failed [errorMsg=Failed to decode JWT token Error(InvalidAudience)] Request failed [errorMsg=Failed to decode JWT token Error(ExpiredSignature)] +Request failed [errorMsg=Unable to find public key with key id [...]] ``` `InvalidAudience` usually means that the token was requested for another API, or that Entra ID issued a v1.0 access token. See [Microsoft Entra ID managed identities and service principals](#accept-managed-identity-and-service-principal-tokens). -Don't set `acl.oidc.audience` to the Application ID URI to work around it: -QuestDB accepts a single audience, and the ID tokens of Web Console users carry -the application ID, so their logins would fail. +Don't set `acl.oidc.audience` to the Application ID URI to work around it: the +setting takes a single value, and the ID tokens of Web Console users carry the +application ID, so their logins would fail. + +`Unable to find public key with key id` means that none of the signing keys +that the provider publishes matches the token. QuestDB downloads the keys again +when it meets a key ID that it does not know, so either another provider issued +the token, or QuestDB cannot reach the provider's keys, which it logs as +`Unable to download public keys from OIDC provider`. With `acl.oidc.groups.encoded.in.token=false`, the provider's User Info endpoint rejects a token that is invalid or expired, and QuestDB rejects the @@ -952,7 +1028,7 @@ If the login succeeds but every request is denied, check that at least one of the user's groups is mapped to a QuestDB group that has the endpoint permission. See [Mapping user permissions](#mapping-user-permissions). -### Versions before 4.0.2 +## Versions before 4.0.2 QuestDB Enterprise versions before 4.0.2 differ as follows: @@ -970,6 +1046,22 @@ QuestDB Enterprise versions before 4.0.2 differ as follows: - They start with an empty `acl.oidc.sub.claim`, or with the same claim in both settings, and then reject every OIDC login. +When you upgrade from an earlier version: + +- With `acl.oidc.groups.encoded.in.token=true`, QuestDB rejects expired tokens + and tokens without an `exp` claim. A client that keeps using a token after it + expires, such as a client configured with a fixed token, must request a new + token before the old one expires. +- QuestDB refuses to start with an empty `acl.oidc.sub.claim`, or with a claim + that is listed twice or in both settings. See + [Startup validation](#startup-validation). +- QuestDB rejects logins whose principal claim is `null`, or whose groups claim + holds only `null` or empty group names, unless another listed claim carries a + value. Earlier versions let them log in. +- In a replicated cluster, upgrade every node before you change a setting to a + claim list. Earlier versions reject every OIDC login when a setting lists + more than one claim. + ## Configuration options For all OIDC-related configuration options of QuestDB, see @@ -977,7 +1069,7 @@ For all OIDC-related configuration options of QuestDB, see
-## Active Directory +## Identity provider guides {#active-directory} The following sections are guides for setting up single sign-on (SSO) with various OAuth2 providers, and for authenticating services with app-only tokens from Microsoft Entra ID. @@ -1525,7 +1617,7 @@ The below should be set in QuestDB's `server.conf`: # enable OIDC acl.oidc.enabled=true -# the claim contains the user's unique username +# the claim contains the user's sign-in name acl.oidc.sub.claim=preferred_username # the claim contains the user's group memberships @@ -1551,6 +1643,12 @@ acl.oidc.scope=openid profile offline_access acl.oidc.ropc.flow.enabled=true ``` +`preferred_username` shows the user's sign-in name in the Web Console and in +the logs. Entra ID can change a sign-in name, and can give it to a new user +later. To identify users by an object ID that never changes, set +`acl.oidc.sub.claim=oid`. See +[Choose the principal claim](#choose-the-principal-claim). + The application ID and the OIDC configuration endpoint's URL can be found in the Overview of the application in Entra ID. @@ -1606,36 +1704,10 @@ will have the access granted to them. ### Microsoft Entra ID managed identities and service principals {#accept-managed-identity-and-service-principal-tokens} -Services that authenticate as themselves, with a managed identity or a service -principal, receive app-only access tokens from Entra ID. These tokens carry no -`name` or `preferred_username` claim. They identify the caller by the object ID -of its service principal in the `oid` claim, and list the -[app roles](https://learn.microsoft.com/en-us/entra/identity-platform/howto-add-app-roles-in-apps) -assigned to it in the `roles` claim. They can also carry a `groups` claim, when -the service principal is a member of a security group and the QuestDB -application emits group claims, as set up under _Token configuration_ in -[Set up the client application in Entra ID](#set-up-the-client-application-in-entra-id). - -QuestDB treats the service as an external user, not as a QuestDB -[service account](/docs/security/rbac/#users-and-service-accounts). Like any -external user, it gets its permissions from the QuestDB groups that its claims -map to, and permissions cannot be granted to it directly. With the -configuration below, these are the groups that its app roles map to. When its -token carries no role but carries a `groups` claim, QuestDB takes the groups -from `groups` instead, and the service gets the permissions of its Entra ID -security groups. - -Unless _Assignment required_ is enabled for the QuestDB application, Entra ID -issues tokens for the application to any service principal in the tenant, and -the token of a service principal without a role carries no `roles` claim. To -make an app role the only way for a service to get permissions, do one of the -following: - -- Keep managed identities and service principals out of the security groups - that are mapped to QuestDB groups. -- Enable _Assignment required_ in the _Properties_ of the QuestDB application, - under _Enterprise applications_. Users then need an assignment too, directly - or through a group, to log in to the Web Console. +Azure services that run as a managed identity or a service principal can +authenticate to QuestDB with app-only access tokens from Entra ID, next to the +users who log in to the Web Console. Each service gets the permissions of the +QuestDB groups that its app roles map to. :::warning Requires QuestDB Enterprise 4.0.2 @@ -1649,6 +1721,22 @@ This setup builds on the Web Console setup above: the application registered in [Set up the client application in Entra ID](#set-up-the-client-application-in-entra-id), and the [QuestDB configuration](#questdb-configuration). +App-only tokens identify the caller by the object ID of its service principal, +in the `oid` claim, and list the +[app roles](https://learn.microsoft.com/en-us/entra/identity-platform/howto-add-app-roles-in-apps) +assigned to it in the `roles` claim. They carry no `name` or +`preferred_username` claim. If the service principal is a member of a security +group, its token also carries a `groups` claim, because the QuestDB application +emits group claims, as set up under _Token configuration_ in +[Set up the client application in Entra ID](#set-up-the-client-application-in-entra-id). + +QuestDB treats the service as an external user, not as a QuestDB +[service account](/docs/security/rbac/#users-and-service-accounts), so +permissions cannot be granted to it directly. With the configuration below, +when its token carries no role but carries a `groups` claim, the service gets +the permissions of its Entra ID security groups instead. See +[Restrict access to app roles](#restrict-access-to-app-roles). + #### Configure the application in Entra ID 1. Make sure that the application is single-tenant, as described in @@ -1662,12 +1750,12 @@ and the [QuestDB configuration](#questdb-configuration). as Microsoft Graph, carries that API in `aud`, and QuestDB rejects it. 3. Make the application issue v2.0 access tokens: in its _Manifest_, set `api.requestedAccessTokenVersion` to `2`. The legacy manifest format calls - this property `accessTokenAcceptedVersion`. QuestDB accepts a token only - when its `aud` claim matches `acl.oidc.audience`, which defaults to the - application ID. A v2.0 token always carries the application ID there. A - v1.0 token, the default for applications that accept organizational - accounts only, can carry the Application ID URI instead. Web Console logins - are not affected, because the Web Console presents ID tokens. + this property `accessTokenAcceptedVersion`. The `aud` claim of a v2.0 token + always carries the application ID, which + [Token validation](#token-validation) checks. A v1.0 token, the default for + applications that accept organizational accounts only, can carry the + Application ID URI instead. Web Console logins are not affected, because the + Web Console presents ID tokens. 4. Under _App roles_, define a role for each kind of access, with the _Applications_ member type. The examples below use the value `QuestDB.Ingest`. @@ -1706,8 +1794,28 @@ the principal from `oid` and the groups from `roles`, even when they also carry a `groups` claim. QuestDB rejects a token that carries neither a role nor a group. See [How QuestDB picks a claim](#how-questdb-picks-a-claim). +If your QuestDB configuration sets `acl.oidc.sub.claim=oid`, as described in +[Choose the principal claim](#choose-the-principal-claim), keep it: app-only +tokens carry `oid` too, so only `acl.oidc.groups.claim` needs a list. + #### Map app roles to QuestDB groups +QuestDB cannot create a table or add a column on ingestion for the service, +because the service is an external user. See +[Tables created by external users](#tables-created-by-external-users). As the +admin, create the tables that the service writes to, with every column that it +sends: + +```questdb-sql title="Create fx_trades before the service writes to it" +CREATE TABLE fx_trades ( + timestamp TIMESTAMP_NS, + symbol SYMBOL, + side SYMBOL, + price DOUBLE, + quantity DOUBLE +) TIMESTAMP(timestamp) PARTITION BY DAY; +``` + Map each app role to a QuestDB group by the role's value, the same way as an Entra ID group, and grant the group what the service needs. A service that ingests over HTTP or [QWP](/docs/connect/wire-protocols/qwp-ingress-websocket/) @@ -1719,17 +1827,30 @@ GRANT HTTP TO ingestApps; GRANT INSERT ON fx_trades TO ingestApps; ``` -QuestDB cannot create a table on ingestion for an external user, because it -cannot make an external user the owner of the new table. Create the tables -that the service writes to in advance, with the columns that it sends. An -external user that creates a table with SQL must name one of its groups as the -owner, with [`OWNED BY`](/docs/query/sql/create-table/#owned-by). - -A service that connects to the [PGWire endpoint](#oidc-for-the-pgwire-endpoint) -also needs the `PGWIRE` endpoint permission. See +A service that queries its tables also needs `SELECT` on them, and a service +that connects to the [PGWire endpoint](#oidc-for-the-pgwire-endpoint) also +needs the `PGWIRE` endpoint permission. See [Endpoint permissions](/docs/security/rbac/#endpoint-permissions) for the other endpoints. +#### Restrict access to app roles + +Unless _Assignment required_ is enabled for the QuestDB application, any +service principal in the tenant can get a token for QuestDB. A token without a +role falls back to the `groups` claim, so a service principal in a mapped +security group gets the permissions of that group without an app role. To make +an app role the only way for a service to get permissions, do one of the +following: + +- Keep managed identities and service principals out of the security groups + that are mapped to QuestDB groups. +- Enable _Assignment required_ in the _Properties_ of the QuestDB application, + under _Enterprise applications_. Users then need an assignment too, directly + or through a group, to log in to the Web Console. Entra ID also stops asking + users for consent, so + [grant tenant-wide admin consent](https://learn.microsoft.com/en-us/entra/identity/enterprise-apps/grant-admin-consent) + to the QuestDB application, or users cannot log in. + #### Request a token in the service The service requests its token for the QuestDB application. With the @@ -1741,7 +1862,9 @@ up in environment variables. For a user-assigned managed identity, set the Otherwise, the credential uses the system-assigned identity. The examples use the QWP API of the `questdb` package, version 5.0 or later. -Install the libraries with `python3 -m pip install -U questdb azure-identity`: +Install the libraries with +`python3 -m pip install -U questdb azure-identity pandas`. pandas is only +needed for `to_pandas()` in [Verify the setup](#verify-the-setup): ```python title="Ingest into fx_trades with an app-only token" import questdb @@ -1772,23 +1895,30 @@ to the error handler of the connection, which logs them by default. See The client presents the same token on every connection that it opens: the first one, the additional connections of its pool, and the connections that -replace a failed one. An open connection stays open when its token expires, but -QuestDB rejects a new connection that presents an expired token. A long-running -service therefore closes `db`, the handle that `questdb.connect()` returns, -before `token.expires_on`, a Unix timestamp, and connects again with a new -token. The credential caches the token in memory and requests a new one when -the cached token nears expiry, so calling `get_token()` often is cheap: +replace a failed one. Shortly after the token expires, QuestDB rejects these +new connections, as described in [Token validation](#token-validation). A +long-running service must therefore switch to a new token before the old one +expires: it connects with the new token, then closes the old handle. +`token.expires_on` is a Unix timestamp. + +The credential caches the token in memory, and requests a new one when the +cached token expires within 5 minutes, or earlier when the token's refresh +time has passed, so calling `get_token()` often is cheap. The following example +looks for a new token in the last 10 minutes of the old one. When it cannot get +a new token or connect with it, it keeps the old token and handle, and tries +again with the next batch: ```python title="Keep ingesting across token expiry" import time import questdb +from azure.core.exceptions import AzureError from azure.identity import DefaultAzureCredential -from questdb import TimestampNanos +from questdb import QuestDBError, TimestampNanos SCOPE = "api://8de84b90-1ea5-4e41-9e84-dba860aa01a6/.default" ADDR = "questdb.example.com:9000" -# connect again when the token expires within this many seconds +# look for a new token when the old one expires within this many seconds REFRESH_MARGIN_SECONDS = 600 credential = DefaultAzureCredential() @@ -1804,6 +1934,22 @@ def connect(token): return questdb.connect(conf, error_handler=on_rejection) +def renew(token, db): + # returns the token and the handle to use from now on + try: + new_token = credential.get_token(SCOPE) + # the credential returns the cached token until it renews it + if new_token.token == token.token: + return token, db + new_db = connect(new_token) + except (AzureError, QuestDBError) as e: + # keep the old token and handle, and try again with the next batch + print("token renewal failed:", e) + return token, db + db.close() + return new_token, new_db + + def next_trades(): # replace with your source of trades time.sleep(1) @@ -1816,12 +1962,7 @@ try: while True: trades = next_trades() if time.time() > token.expires_on - REFRESH_MARGIN_SECONDS: - new_token = credential.get_token(SCOPE) - # the credential returns the cached token until it renews it - if new_token.token != token.token: - db.close() - token = new_token - db = connect(token) + token, db = renew(token, db) with db.sender() as sender: for symbol, side, price, quantity in trades: sender.row( @@ -1843,8 +1984,10 @@ password of the `_sso` user. #### Verify the setup `current_user()` returns the principal that QuestDB sees for the service: the -object ID of its service principal, from the `oid` claim. Run it from the -service while it is connected, with pandas installed for `to_pandas()`: +object ID of its service principal, from the `oid` claim. For a managed +identity, that is the _Object (principal) ID_ that the Azure portal shows for +the identity, not its client ID. Run the query from the service while it is +connected: ```python title="Check the principal of the service" with db.query("SELECT current_user()") as result: @@ -1852,8 +1995,9 @@ with db.query("SELECT current_user()") as result: ``` If the service cannot connect, the server log gives the reason, as described in -[Rejected logins](#rejected-logins). For a token that carries neither a role -nor a group, the log lists the claims that the token carries. +[Troubleshooting OIDC logins](#troubleshooting-oidc-logins). For a token that +carries neither a role nor a group, the log lists the claims that the token +carries. #### Change or revoke access @@ -1863,6 +2007,8 @@ identity tokens for around 24 hours, so a role assigned to, or removed from, a managed identity can take hours to reach QuestDB. See the [managed identity best practices](https://learn.microsoft.com/en-us/entra/identity/managed-identities-azure-resources/managed-identity-best-practice-recommendations). To cut a service off without waiting for its token to expire, revoke the -permissions of its QuestDB group. +permissions of its QuestDB group. This cuts off every service that holds the +app role, so to cut off one service at a time, give each service an app role, +and a QuestDB group, of its own.
diff --git a/documentation/security/rbac.md b/documentation/security/rbac.md index 7d5eb4edc0..1daee46209 100644 --- a/documentation/security/rbac.md +++ b/documentation/security/rbac.md @@ -57,11 +57,11 @@ Control *what data* users can access: Control *how* users can connect: -| Permission | Protocol | Use case | -| ---------- | ------------------------------- | ------------------------------------------ | -| `HTTP` | REST API, Web Console, ILP/HTTP | Interactive users, web applications | -| `PGWIRE` | PostgreSQL Wire Protocol | SQL clients, BI tools, programmatic access | -| `ILP` | InfluxDB Line Protocol (TCP) | High-throughput data ingestion | +| Permission | Protocol | Use case | +| ---------- | --------------------------------------------------- | ------------------------------------------ | +| `HTTP` | REST API, Web Console, ILP/HTTP, QWP over WebSocket | Interactive users, web applications | +| `PGWIRE` | PostgreSQL Wire Protocol | SQL clients, BI tools, programmatic access | +| `ILP` | InfluxDB Line Protocol (TCP) | High-throughput data ingestion | ```questdb-sql -- User can connect via PostgreSQL protocol only (not web console) @@ -263,6 +263,13 @@ CREATE SERVICE ACCOUNT app_account WITH PASSWORD 'pwd'; Names must be unique across all users, service accounts, and groups. +QuestDB Enterprise also accepts external users: people and services that log +in through an [OpenID Connect provider](/docs/security/oidc/), such as Azure +services that run as a Microsoft Entra ID managed identity. QuestDB does not +store external users, and permissions cannot be granted to them directly. They +get the permissions of the QuestDB groups that their external groups map to. +See [User permissions](/docs/security/oidc/#user-permissions). + #### Why service accounts? Service accounts provide **clean, testable application access**: From 368e65923a6102be3d4ac732f6d2ba0256ebeb23 Mon Sep 17 00:00:00 2001 From: glasstiger Date: Fri, 9 Oct 2026 10:38:58 +0100 Subject: [PATCH 07/29] docs(oidc): move the identity provider guides to their own pages Split the Microsoft Entra ID and PingFederate guides out of the OIDC page into security/oidc-entra-id and security/oidc-pingfederate, grouped under OpenID Connect (OIDC) in the sidebar. Update links from the client, configuration, deployment and SQL reference pages, document acl.oidc.public.keys.expiry and acl.oidc.state.required, and correct when the Python client raises on bad credentials. Co-Authored-By: Claude Opus 5.5 --- documentation/changelog.mdx | 14 +- documentation/configuration/oidc.md | 42 +- documentation/connect/clients/c-and-cpp.md | 2 +- documentation/connect/clients/dotnet.md | 2 +- documentation/connect/clients/go.md | 2 + documentation/connect/clients/python.md | 35 +- .../connect/compatibility/ilp/overview.md | 5 + documentation/deployment/azure.md | 4 +- .../query/sql/alter-table-add-column.md | 5 +- documentation/query/sql/create-table.md | 5 +- documentation/security/oidc-entra-id.md | 629 +++++++++ documentation/security/oidc-pingfederate.md | 359 +++++ documentation/security/oidc.mdx | 1186 +++-------------- documentation/security/rbac.md | 11 +- documentation/sidebars.js | 25 +- 15 files changed, 1252 insertions(+), 1074 deletions(-) create mode 100644 documentation/security/oidc-entra-id.md create mode 100644 documentation/security/oidc-pingfederate.md diff --git a/documentation/changelog.mdx b/documentation/changelog.mdx index dd77df67d9..56e2be415e 100644 --- a/documentation/changelog.mdx +++ b/documentation/changelog.mdx @@ -12,10 +12,10 @@ This page tracks significant updates to the QuestDB documentation. ### New +- [Microsoft Entra ID](/docs/security/oidc-entra-id/) - New page for Microsoft Entra ID: the Web Console sign-in guide, moved from the OIDC page, and app-only tokens for Azure managed identities and service principals in QuestDB Enterprise 4.0.2, with the app role mapping, how to keep app roles the only way in, and Python examples that ingest and renew the token - [User and group claims](/docs/security/oidc/#user-and-group-claims) - How QuestDB Enterprise 4.0.2 reads the principal and the groups from the user information: fallback claim lists, the rules for missing, `null`, and empty claims, how to choose the principal claim, and the startup checks - [Token validation](/docs/security/oidc/#token-validation), [Troubleshooting OIDC logins](/docs/security/oidc/#troubleshooting-oidc-logins), and [Versions before 4.0.2](/docs/security/oidc/#versions-before-402) - What QuestDB checks in a token and how long it accepts one after it expires, the log messages of rejected logins, how earlier versions differ, and what to check when you upgrade -- [Tables created by external users](/docs/security/oidc/#tables-created-by-external-users) - Ingestion cannot create a table or add a column for a user who logs in through OIDC, so create the tables in advance, or create them with SQL and `OWNED BY` -- [Microsoft Entra ID managed identities and service principals](/docs/security/oidc/#accept-managed-identity-and-service-principal-tokens) - Let services that run as a Microsoft Entra ID managed identity or service principal authenticate to QuestDB Enterprise 4.0.2 with app-only tokens, next to user logins: the Entra ID setup, the claim lists, the app role mapping, how to keep app roles the only way in, and Python examples that ingest and renew the token +- [Tables created by external users](/docs/security/oidc/#tables-created-by-external-users) - Ingestion cannot create a table or add a column for a user who logs in through OIDC, so create the tables in advance, or create them with SQL and `OWNED BY`, naming one of the user's QuestDB groups ### Reference @@ -23,13 +23,17 @@ This page tracks significant updates to the QuestDB documentation. - [`acl.oidc.pkce.required`](/docs/configuration/oidc/#acloidcpkcerequired) - Corrected the property name, previously documented as `acl.oidc.pkce.enabled`, which QuestDB does not recognize - [`acl.oidc.groups.encoded.in.token`](/docs/configuration/oidc/#acloidcgroupsencodedintoken) - Documented that QuestDB Enterprise 4.0.2 rejects expired tokens and tokens without an `exp` claim, which earlier versions accept, and which token QuestDB reads the principal and the groups from - [`acl.oidc.audience`](/docs/configuration/oidc/#acloidcaudience) and [`acl.oidc.cache.ttl`](/docs/configuration/oidc/#acloidccachettl) - Documented that `acl.oidc.audience` takes a single value, which the `aud` claim of a token must be or contain, and what the cache holds when the groups are encoded in the token +- [`acl.oidc.public.keys.expiry`](/docs/configuration/oidc/#acloidcpublickeysexpiry) and [`acl.oidc.state.required`](/docs/configuration/oidc/#acloidcstaterequired) - Documented how long QuestDB uses the signing keys of the OIDC provider before it downloads them again, and the `state` check of the Web Console login ### Updated -- [Mapping user permissions](/docs/security/oidc/#mapping-user-permissions) - Corrected what happens when the groups claim is missing or empty: QuestDB rejects the login. A user is authenticated without permissions only when none of their groups is mapped to a QuestDB group. With the groups encoded in the token, a change to them takes effect when the client presents a new token. The HTTP session of the Web Console is not tied to the token -- [Authentication and authorization flow](/docs/security/oidc/#8-find-user-information) and [non-interactive clients](/docs/security/oidc/#non-interactive-clients) - Corrected which token reaches QuestDB with `acl.oidc.groups.encoded.in.token=true`: the ID token, not the access token +- [OpenID Connect (OIDC)](/docs/security/oidc/) - The identity provider guides moved to their own pages, under OpenID Connect (OIDC) in the sidebar, and the introduction lists what the page covers +- [PingFederate](/docs/security/oidc-pingfederate/) - Moved from the OIDC page to its own page +- [Mapping user permissions](/docs/security/oidc/#mapping-user-permissions) - Corrected what happens when the groups claim is missing or empty: QuestDB rejects the login. A user is authenticated without permissions only when none of their groups is mapped to a QuestDB group. Documented when a change in the identity provider takes effect, that the HTTP session of the Web Console is not tied to the token and does not expire while in use, and how to cut off access immediately +- [Find user information](/docs/security/oidc/#8-find-user-information) and [non-interactive clients](/docs/security/oidc/#non-interactive-clients) - Corrected which token of a user reaches QuestDB with `acl.oidc.groups.encoded.in.token=true`: the ID token rather than the access token. Non-interactive clients also covers the tokens that services obtain for themselves - [Endpoint permissions](/docs/security/rbac/#endpoint-permissions) - The `HTTP` permission also covers QWP over WebSocket. The [authentication methods](/docs/security/rbac/#authentication) and [users and service accounts](/docs/security/rbac/#users-and-service-accounts) describe external users, who log in through OIDC -- [Microsoft Entra ID guide](/docs/security/oidc/#microsoft-entraid) - Uses `preferred_username` as the principal claim instead of the display name, explains when to use `oid` instead, and registers the application as single-tenant +- [Microsoft Entra ID guide](/docs/security/oidc-entra-id/#web-console-sign-in) - Uses `preferred_username` as the principal claim instead of the display name, explains when to use `oid` instead, and registers the application as single-tenant +- [Python client](/docs/connect/clients/python/#authentication-and-tls) - Corrected when bad credentials raise: by default, `connect()` opens its first connections and raises, and `lazy_connect=on` defers the error to the first operation ## September 2026 diff --git a/documentation/configuration/oidc.md b/documentation/configuration/oidc.md index 6f837754d3..458bd756a9 100644 --- a/documentation/configuration/oidc.md +++ b/documentation/configuration/oidc.md @@ -1,6 +1,6 @@ --- title: OpenID Connect (OIDC) -description: Configuration settings for OpenID Connect integration in QuestDB Enterprise. +description: "Configure OIDC in QuestDB Enterprise: provider endpoints, TLS, PKCE, ROPC, token audience, and principal and groups claims with fallback lists." --- :::note @@ -128,6 +128,17 @@ it refuses to start. Use `acl.oidc.pkce.required` instead. Enables or disables the Resource Owner Password Credentials flow. When enabled, this flow must also be configured in the OIDC Provider. +### acl.oidc.state.required + +- **Default**: `false` +- **Reloadable**: no + +Whether the Web Console sends a random `state` parameter in the authorization +request of the Authorization Code Flow, and rejects the redirect back from the +OIDC Provider when it does not carry the same `state`. This ties the redirect +to a login that the same browser started, which protects against cross-site +request forgery on the login redirect. + ## Endpoints ### acl.oidc.authorization.endpoint @@ -144,8 +155,20 @@ Identity Platform. - **Reloadable**: no JSON Web Key Set (JWKS) Endpoint. Provides the list of public keys used to -decode and validate ID tokens issued by the OIDC Provider. The default value -should work for the Ping Identity Platform. +decode and validate the tokens issued by the OIDC Provider, such as ID tokens, +when `acl.oidc.groups.encoded.in.token` is `true`. The default value should +work for the Ping Identity Platform. + +### acl.oidc.public.keys.expiry + +- **Default**: `120000` +- **Reloadable**: no + +How long QuestDB uses the public keys that it downloaded from the JWKS +endpoint, in milliseconds, before it downloads them again. QuestDB also +downloads the keys again when a token names a key ID that it does not know, +such as a key that the OIDC Provider has just started to use. See +[Troubleshooting OIDC logins](/docs/security/oidc/#troubleshooting-oidc-logins). ### acl.oidc.token.endpoint @@ -241,9 +264,10 @@ user, as an array of group names or as a single group name. Required when OIDC is enabled. Since QuestDB Enterprise 4.0.2, accepts a comma-separated list of claims in -priority order, such as `roles,groups`. QuestDB takes the groups from the first -claim on the list that holds at least one group name, and does not combine -groups from several claims. A login is rejected when none of the listed claims +priority order, in the form `claimName[,claimName ...]`, such as +`roles,groups`. Spaces around claim names are trimmed, and empty entries are +skipped. QuestDB takes the groups from the first claim on the list that holds +at least one group name, and does not combine groups from several claims. A login is rejected when none of the listed claims holds a group name. With OIDC enabled, QuestDB refuses to start when the list names a claim twice, or names a claim that `acl.oidc.sub.claim` also lists. @@ -286,8 +310,10 @@ names, such as the `name` claim. See [Choose the principal claim](/docs/security/oidc/#choose-the-principal-claim). Since QuestDB Enterprise 4.0.2, accepts a comma-separated list of claims in -priority order, such as `preferred_username,oid`. QuestDB takes the principal -from the first claim on the list that holds a non-empty value. A login is +priority order, in the form `claimName[,claimName ...]`, such as +`preferred_username,oid`. Spaces around claim names are trimmed, and empty +entries are skipped. QuestDB takes the principal from the first claim on the +list that holds a non-empty value. A login is rejected when none of the listed claims holds one. With OIDC enabled, QuestDB refuses to start when the value is empty, when the list names a claim twice, or when it names a claim that `acl.oidc.groups.claim` also lists. An empty value diff --git a/documentation/connect/clients/c-and-cpp.md b/documentation/connect/clients/c-and-cpp.md index 511bff592a..c453c303a2 100644 --- a/documentation/connect/clients/c-and-cpp.md +++ b/documentation/connect/clients/c-and-cpp.md @@ -332,7 +332,7 @@ following are **not** supported: | Path | Status | Workaround | |---|---|---| -| OIDC token acquisition or in-band refresh | Not supported. The client does not negotiate with an identity provider and has no callback to refresh a token mid-session. | QuestDB itself supports OIDC; see [OpenID Connect](/docs/security/oidc/#non-interactive-clients) for which token to send. Acquire the token out-of-band from your IdP, pass it via `token=...`, and rebuild the pool when the token nears expiry. | +| OIDC token acquisition or in-band refresh | Not supported. The client does not negotiate with an identity provider and has no callback to refresh a token mid-session. | QuestDB itself supports OIDC; see [OpenID Connect](/docs/security/oidc/#non-interactive-clients) for which token to send. Acquire the token out-of-band from your IdP, pass it via `token=...`, and rebuild the pool when the token nears expiry. For Azure services, see [Microsoft Entra ID managed identities and service principals](/docs/security/oidc-entra-id/#managed-identities-and-service-principals). | | Mutual TLS (client certificates) | Not supported. The QuestDB server does not negotiate client certificates regardless of client. | Use bearer-token auth over `wss`. See the connect-string reference's [TLS section](/docs/connect/clients/connect-string/#tls). | | Token rotation mid-session | Not supported. Credentials are presented once during the WebSocket upgrade and are not re-sent. | On token expiry, close the pool and build a fresh one with the new token. | diff --git a/documentation/connect/clients/dotnet.md b/documentation/connect/clients/dotnet.md index d1c9516b5e..f1d7eb2ba1 100644 --- a/documentation/connect/clients/dotnet.md +++ b/documentation/connect/clients/dotnet.md @@ -228,7 +228,7 @@ that purpose; left unset, it inherits `auth_timeout_ms`. | Path | Status | Workaround | |---|---|---| -| OIDC token acquisition or in-band refresh | Not supported by this client. It does not negotiate with an identity provider and has no callback to refresh a token mid-session. | QuestDB itself supports OIDC; see [OpenID Connect](/docs/security/oidc/#non-interactive-clients) for which token to send. Acquire the token out-of-band from your IdP, pass it via `token=...` above, and rebuild the sender / query client when the token nears expiry. | +| OIDC token acquisition or in-band refresh | Not supported by this client. It does not negotiate with an identity provider and has no callback to refresh a token mid-session. | QuestDB itself supports OIDC; see [OpenID Connect](/docs/security/oidc/#non-interactive-clients) for which token to send. Acquire the token out-of-band from your IdP, pass it via `token=...` above, and rebuild the sender / query client when the token nears expiry. For Azure services, see [Microsoft Entra ID managed identities and service principals](/docs/security/oidc-entra-id/#managed-identities-and-service-principals). | | Mutual TLS (client certificates) | Not supported. The QuestDB server does not negotiate client certificates regardless of client. | Use bearer-token auth over `wss://`. See the connect-string reference for the canonical statement. | | Token rotation mid-session | Not supported. Credentials are presented once during the WebSocket upgrade and are not re-sent. | On token expiry, `await sender.DisposeAsync()` and build a fresh sender with the new token. The same applies to `QueryClient`. | diff --git a/documentation/connect/clients/go.md b/documentation/connect/clients/go.md index cd3c375680..94ab8d2483 100644 --- a/documentation/connect/clients/go.md +++ b/documentation/connect/clients/go.md @@ -362,6 +362,8 @@ the ingress and egress WebSocket upgrades. It is a **static credential**: the client sends exactly the string you pass and never refreshes or renews it. Acquire it out of band, for example from the identity provider of the QuestDB Enterprise [OpenID Connect flow](/docs/security/oidc/#non-interactive-clients), and manage its lifetime yourself. +For Azure services, see +[Microsoft Entra ID managed identities and service principals](/docs/security/oidc-entra-id/#managed-identities-and-service-principals). When the token expires or is rotated, construct a new handle with the new token. An expired or rejected token surfaces as an authentication failure (see [Connection-level errors](#connection-level-errors)). It is mutually exclusive diff --git a/documentation/connect/clients/python.md b/documentation/connect/clients/python.md index f16125ae58..16f265c7cb 100644 --- a/documentation/connect/clients/python.md +++ b/documentation/connect/clients/python.md @@ -143,13 +143,17 @@ token = questdb.connect( ``` Authentication happens during the WebSocket upgrade, before any data frames -are exchanged. Bad credentials raise `QuestDBErrorCode.AuthError` from the -first operation that needs the connection, not from `connect()`. Queries and -`dataframe()` calls raise it directly. Row senders connect in the -background, so a fire-and-forget `flush()` can return before the upgrade -fails; the error then surfaces from `flush(wait=True)`, `wait()`, or the -next call on the lease — and immediately through the -[connection listener](#failover-and-errors) as an `AuthFailed` event. +are exchanged. By default, `connect()` opens its first connections before it +returns (see [The pool](#the-pool)), so bad credentials raise +`QuestDBErrorCode.AuthError` from `connect()` itself. With +[`lazy_connect=on`](/docs/connect/clients/connect-string/#pool-keys), +`connect()` opens no connection, and bad credentials raise from the first +operation that needs the connection instead. Queries and `dataframe()` calls +raise it directly. Row senders connect in the background, so a +fire-and-forget `flush()` can return before the upgrade fails. The error then +surfaces from `flush(wait=True)`, `wait()`, or the next call on the lease, and +immediately through the [connection listener](#failover-and-errors) as an +`AuthFailed` event. With `wss`, the default combines the bundled `webpki` root store with the operating-system certificate store. Override it with configuration keys: @@ -171,9 +175,9 @@ following are **not** supported: | Path | Status | Workaround | | --- | --- | --- | -| OIDC token acquisition or in-band refresh | Not supported. The client does not negotiate with an identity provider and cannot refresh a token mid-session. | QuestDB itself supports OIDC; see [OpenID Connect](/docs/security/oidc/#non-interactive-clients) for which token to send. Acquire the token out-of-band from your IdP, pass it via `token=...`, and rebuild the handle when the token nears expiry. For an example with an Entra ID managed identity, see [Microsoft Entra ID managed identities and service principals](/docs/security/oidc/#accept-managed-identity-and-service-principal-tokens). | +| OIDC token acquisition or in-band refresh | Not supported. The client does not negotiate with an identity provider and cannot refresh a token mid-session. | QuestDB itself supports OIDC; see [OpenID Connect](/docs/security/oidc/#non-interactive-clients) for which token to send. Acquire the token out-of-band from your IdP, pass it via `token=...`, and rebuild the handle when the token nears expiry. For an example with an Entra ID managed identity, see [Microsoft Entra ID managed identities and service principals](/docs/security/oidc-entra-id/#managed-identities-and-service-principals). | | Mutual TLS (client certificates) | Not supported. The QuestDB server does not negotiate client certificates regardless of client. | Use bearer-token auth over `wss`. | -| Token rotation mid-session | Not supported. The handle keeps the credentials it was built with and presents them on every connection it opens — including reconnects and failover, so an expired token also breaks mid-session reconnection. | On token expiry, close the handle and build a fresh one with the new token. | +| Token rotation mid-session | Not supported. The handle keeps the credentials it was built with and presents them on every connection it opens — including reconnects and failover, so an expired token also breaks mid-session reconnection. | Before the token expires, build a fresh handle with a new token, then close the old one. See [Request a token in the service](/docs/security/oidc-entra-id/#request-a-token-in-the-service) for an example. | ## The pool @@ -183,10 +187,15 @@ borrows a connection, opening one on demand, and returns it when the lease or result closes; when every slot is busy, a borrow waits for a free one (see [Pool settings](#pool-settings)). -`connect()` performs no blocking network I/O. `dataframe()`, `query()`, -and `server_info()` connect on first use; `sender()` connects in the -background, so call `flush(wait=True)` or `wait()` to surface connection and -delivery errors. +By default, `connect()` opens the warm minimum of connections before it +returns: `sender_pool_min` sender and `query_pool_min` reader connections, one +of each by default (see [Pool settings](#pool-settings)). An unreachable +server or bad credentials therefore raise from `connect()`. With +[`lazy_connect=on`](/docs/connect/clients/connect-string/#pool-keys), +`connect()` performs no network I/O: `dataframe()`, `query()`, and +`server_info()` connect on first use, and `sender()` connects in the +background. In both modes, call `flush(wait=True)` or `wait()` to surface +connection and delivery errors from row senders. ### Choose an API diff --git a/documentation/connect/compatibility/ilp/overview.md b/documentation/connect/compatibility/ilp/overview.md index e2d5887add..9be8189143 100644 --- a/documentation/connect/compatibility/ilp/overview.md +++ b/documentation/connect/compatibility/ilp/overview.md @@ -441,6 +441,11 @@ GRANT add column, insert ON all tables TO ingest_user; -- grants permissions to GRANT add column, insert ON table1, table2 TO ingest_user; -- grants permissions to add columns and insert data to specific tables ``` +External users, who log in through [OpenID Connect](/docs/security/oidc/), +cannot create tables or add columns on ingestion. Create the tables in advance, +with every column that the client sends. See +[Tables created by external users](/docs/security/oidc/#tables-created-by-external-users). + Read more setup details in the [Enterprise quickstart](/docs/getting-started/enterprise-quick-start/#4-ingest-data-influxdb-line-protocol) and the [role-based access control](/docs/security/rbac/) guides. diff --git a/documentation/deployment/azure.md b/documentation/deployment/azure.md index 747e611533..62ec329bbb 100644 --- a/documentation/deployment/azure.md +++ b/documentation/deployment/azure.md @@ -345,11 +345,11 @@ QuestDB Enterprise adds production features for Azure: - **Entra ID SSO** - Single sign-on with Microsoft Entra ID For Entra ID integration, see the -[Microsoft Entra ID OIDC guide](/docs/security/oidc/#microsoft-entraid). +[Microsoft Entra ID guide](/docs/security/oidc-entra-id/). Since QuestDB Enterprise 4.0.2, Azure services that run as a managed identity or a service principal can also authenticate to QuestDB with Entra ID app-only tokens. See -[Microsoft Entra ID managed identities and service principals](/docs/security/oidc/#accept-managed-identity-and-service-principal-tokens). +[Managed identities and service principals](/docs/security/oidc-entra-id/#managed-identities-and-service-principals). See [Enterprise Quick Start](/docs/getting-started/enterprise-quick-start/) for setup. diff --git a/documentation/query/sql/alter-table-add-column.md b/documentation/query/sql/alter-table-add-column.md index 546e0a0e26..a258675875 100644 --- a/documentation/query/sql/alter-table-add-column.md +++ b/documentation/query/sql/alter-table-add-column.md @@ -38,7 +38,10 @@ However, if the `OWNED BY` clause is used, the permissions instead go to the user, group, or service account named in that clause. The `OWNED BY` clause cannot be omitted if the column is added by an external -user, because permissions cannot be granted to them. +user, who logs in through OIDC, because permissions cannot be granted to them. +The owner must then be one of the QuestDB groups that the external user belongs +to. See +[Tables created by external users](/docs/security/oidc/#tables-created-by-external-users). ## Examples diff --git a/documentation/query/sql/create-table.md b/documentation/query/sql/create-table.md index a2a4d39de3..b1cedab389 100644 --- a/documentation/query/sql/create-table.md +++ b/documentation/query/sql/create-table.md @@ -827,7 +827,10 @@ clause is used, the permissions instead go to the user, group, or service account named in that clause. The `OWNED BY` clause cannot be omitted if the table is created by an external -user, because permissions cannot be granted to them. +user, who logs in through OIDC, because permissions cannot be granted to them. +The owner must then be one of the QuestDB groups that the external user belongs +to. See +[Tables created by external users](/docs/security/oidc/#tables-created-by-external-users). ```questdb-sql CREATE GROUP analysts; diff --git a/documentation/security/oidc-entra-id.md b/documentation/security/oidc-entra-id.md new file mode 100644 index 0000000000..0667339720 --- /dev/null +++ b/documentation/security/oidc-entra-id.md @@ -0,0 +1,629 @@ +--- +title: Microsoft Entra ID +sidebar_label: Microsoft Entra ID +description: "Connect Microsoft Entra ID to QuestDB Enterprise: Web Console SSO for users, and app-only tokens for Azure managed identities and service principals." +--- + +import Screenshot from "@theme/Screenshot" +import { EnterpriseNote } from "@site/src/components/EnterpriseNote" + + + OpenID Connect (OIDC) enables SSO and token authentication with external Identity Providers. + + +[Microsoft Entra ID](https://www.microsoft.com/en-gb/security/business/identity-access/microsoft-entra-id), +formerly Azure Active Directory (Azure AD), can be the OpenID Connect (OIDC) +provider of QuestDB Enterprise. This guide sets up one Entra ID application for +two kinds of access: + +- [Web Console sign-in](#web-console-sign-in): users log in to the + [Web Console](/docs/getting-started/web-console/overview/) with single + sign-on (SSO), and get the permissions of the QuestDB groups that their + Entra ID security groups map to. +- [Managed identities and service principals](#managed-identities-and-service-principals): + Azure services authenticate with app-only access tokens, and get the + permissions of the QuestDB groups that their app roles map to. Requires + QuestDB Enterprise 4.0.2 or later. + +For how QuestDB uses OIDC in general, such as the login flow, the claims it +reads, and how it validates tokens, see +[OpenID Connect (OIDC)](/docs/security/oidc/). + +## Web Console sign-in + +This section sets up single sign-on for the Web Console. To also let services +that run as a managed identity or a service principal call QuestDB with +app-only tokens, complete this setup, then follow +[Managed identities and service principals](#managed-identities-and-service-principals). + +:::tip + +To enlarge the images, click or tap them. + +::: + +### Set up the client application in Entra ID + +First thing first, let's pick a name for the client! + +Then head to _Microsoft Entra Admin Center_, and register the application +under _Identity - App registrations - New registration_. + +Under _Supported account types_, select _Accounts in this organizational +directory only_. QuestDB does not check which tenant issued a token, so the +application must not accept accounts from other tenants. See +[Token validation](/docs/security/oidc/#token-validation). + + + +The QuestDB [Web Console](/docs/getting-started/web-console/overview/) is a SPA (Single Page App). + +As a result, it cannot store safely a client secret. + +Instead, it can use PKCE (Proof Key for Code Exchange) to secure the flow. + +When registering the application, select the SPA platform. + +We also have to specify the URL of the [Web Console](/docs/getting-started/web-console/overview/) as Redirect URI. + + + +After clicking _Register_, we have created a client application with the +name _QuestDB_. + +Each application is assigned a unique id (known as Client ID in the +OAuth2 - OIDC standard). The client will identify itself with this id +when sending requests to Entra ID. + + + +We find the platform configurations under _Authentication_. This is the place where +the previously set redirect URI can be viewed and modified. We can also specify +additional redirect URIs, if necessary. + +The redirect URIs of the application are automatically eligible for the +_Authorization Code Flow with PKCE_, which is a special version of the OAuth2 standard's +Authorization Code Flow. It is specifically designed for applications where a client +secret (e.g. a password) could not be kept safely. As single page applications run in +the browser, they fall into this category. + +The redirect URIs are also added to the _CORS_ (Cross-Origin Resource Sharing) policy +of Entra ID. CORS is a mechanism to allow a web page, such as the Web Console, to access +resources from a different domain than the one that served the page. In this context +this means that we let the Web Console to access Entra ID, while its origin is the +HTTP endpoint of QuestDB. + + + +If we scroll down to the bottom of this page, we can also find a section where we +can enable the _Resource Owner Password Credential Flow_. + +This OAuth2 flow is legacy, and should be enabled only if there is a requirement +of connecting to QuestDB using SSO (Single Sign-On) via clients not supporting +redirect based web flows. +This could mean a Postgres client without OAuth2 integration, such as _psql_, or +a standalone in-house client application, or could be just a jupyter notebook. + +The main issue with this flow is that the client application has to be trusted +with the user's login details. The user's credentials are passed to the +application, in this case to QuestDB, and the client application uses these +credentials to authenticate the user by forwarding them to the identity provider, +in this case to Entra ID. + +It is guaranteed that QuestDB does not store the user's credentials in any way. +They are not persisted into the database, not even in encrypted form. +The login details are treated as passthrough information. Only exception is +that server logs can contain the username, logged for audit purposes. + + + +Our next stop is the _Token configuration_, where the OAuth2/OIDC access and ID +tokens can be customized. + +Note that users can be authenticated without customized tokens, but authorization +would prove to be challenging. The user's security groups are not included +in the tokens by default. + +QuestDB can be configured to request the user's groups from the UserInfo +endpoint of the OAuth2 server, but Entra ID cannot be configured to provide +this information via the UserInfo endpoint. +Therefore, we choose to customize the tokens, QuestDB will decode and +validate the ID token, and take the group information from there. + +QuestDB authorization relies on receiving the group memberships of the user. +Entra ID groups should be mapped to QuestDB groups, and permissions can be +granted to the QuestDB groups. Detailed information about group mappings can +be found in the [OIDC integration](/docs/security/oidc/#user-permissions) +documentation. + + + +The customized tokens contain user information which cannot be accessed +without permission. User information is provided by Microsoft Graph, so +the client application needs specific permissions to access +Microsoft Graph APIs. + +These permissions can be configured under _API permissions_. It is important +to note that we will be setting _Delegated_ permissions here, meaning we +are not granting actual permissions to access user data. Instead, each user +logging into QuestDB will have to consent to accessing their user profile. + + + +By default, the _User.Read_ permission is added to the list, but what we +really need is: + - openid: to be able to issue ID tokens + - profile: to access user information + - offline_access: to be able to issue refresh tokens + +By clicking on _Microsoft Graph_ we can select and add these permissions. + + + +The _User.Read_ permission is not needed. It can be removed by clicking +on the `...` at the end of the row, and selecting _Remove permission_ from +the popup menu. + + + +With this we have finished setting up the QuestDB client application +in Entra ID, and now we can wire QuestDB and Entra ID together by +adding OIDC configuration to QuestDB. + +### QuestDB configuration + +The below should be set in QuestDB's `server.conf`: + +```ini title="server.conf" +# enable OIDC +acl.oidc.enabled=true + +# the claim contains the user's sign-in name +acl.oidc.sub.claim=preferred_username + +# the claim contains the user's group memberships +acl.oidc.groups.claim=groups + +# groups are encoded in the token +acl.oidc.groups.encoded.in.token=true + +# OIDC configuration endpoint of Entra ID +acl.oidc.configuration.url=https://login.microsoftonline.com/12345678-1234-1234-1234-123456789abc/v2.0/.well-known/openid-configuration + +# application ID taken from Entra ID +acl.oidc.client.id=8de84b90-1ea5-4e41-9e84-dba860aa01a6 + +# redirect URI, QuestDB's HTTP endpoint +acl.oidc.redirect.uri=http://localhost:9000 + +# OAuth scopes the user has to consent to +acl.oidc.scope=openid profile offline_access + +# enable ROPC flow +# optional, required only if ROPC is enabled in Entra ID +acl.oidc.ropc.flow.enabled=true +``` + +`preferred_username` shows the user's sign-in name in the Web Console and in +the logs. Entra ID can change a sign-in name, and can give it to a new user +later. To identify users by an object ID that never changes, set +`acl.oidc.sub.claim=oid`. See +[Choose the principal claim](/docs/security/oidc/#choose-the-principal-claim). + +The application ID and the OIDC configuration endpoint's URL can be found +in the Overview of the application in Entra ID. + +The application ID is displayed right under the application's name, the +OIDC configuration endpoint is displayed on the panel which opens up when +the _Endpoints_ button is clicked. + + + +### Map groups and grant permissions + +Now we can start QuestDB, and login with the built-in admin to create +group mappings. + +As mentioned earlier, authorization works by mapping Entra ID groups +to QuestDB groups. When the user logs in, QuestDB decodes Entra ID +group memberships from the token, then finds the QuestDB groups +mapped to them, and the user gets the permissions based on the +mapped groups. + +```questdb-sql title="Create a group which is mapped to an Entra ID group" +CREATE GROUP extUsers WITH EXTERNAL ALIAS '87654321-1234-1234-1234-123456789abc'; +``` +The above command maps the Entra ID group identified by object +id `87654321-1234-1234-1234-123456789abc` to a QuestDB group called `extUsers`. + +We should grant the necessary QuestDB endpoint permissions first +to make sure users can access the Web Console, Postgres and ILP +interfaces as required. [Read more about endpoint permissions](/docs/security/rbac/#endpoint-permissions). + +```questdb-sql title="Grant endpoint permissions" +GRANT HTTP, PGWIRE TO extUsers; +``` + +Now we can grant the rest of the permissions as required. We can +grant access to tables, for example. + +```questdb-sql title="Grant database permissions" +GRANT SELECT ON fx_trades, core_price TO extUsers; +``` + +### Confirm group mappings and login + +To test, head to the Web Console and login. + +If all has been wired up well, then login will succeed, and the user +will have the access granted to them. + +## Managed identities and service principals + +Azure services that run as a managed identity or a service principal can +authenticate to QuestDB with app-only access tokens from Entra ID, next to the +users who log in to the Web Console. Each service gets the permissions of the +QuestDB groups that its app roles map to. + +:::warning Requires QuestDB Enterprise 4.0.2 + +Accepting app-only tokens requires QuestDB Enterprise 4.0.2 or later. On +earlier versions, the configuration below rejects every OIDC login, user logins +included. + +::: + +This setup builds on the Web Console setup above: the application registered in +[Set up the client application in Entra ID](#set-up-the-client-application-in-entra-id), +and the [QuestDB configuration](#questdb-configuration). + +App-only tokens identify the caller by the object ID of its service principal, +in the `oid` claim, and list the +[app roles](https://learn.microsoft.com/en-us/entra/identity-platform/howto-add-app-roles-in-apps) +assigned to it in the `roles` claim. They carry no `name` or +`preferred_username` claim. If the service principal is a member of a security +group, its token also carries a `groups` claim, because the QuestDB application +emits group claims, as set up under _Token configuration_ in +[Set up the client application in Entra ID](#set-up-the-client-application-in-entra-id). + +QuestDB treats the service as an external user, not as a QuestDB +[service account](/docs/security/rbac/#users-and-service-accounts), so +permissions cannot be granted to it directly. With the configuration below, +when its token carries no role but carries a `groups` claim, the service gets +the permissions of its Entra ID security groups instead. See +[Restrict access to app roles](#restrict-access-to-app-roles). + +### Configure the application in Entra ID + +1. Make sure that the application is single-tenant, as set up in + [Set up the client application in Entra ID](#set-up-the-client-application-in-entra-id). + QuestDB does not check which tenant issued a token, as described in + [Token validation](/docs/security/oidc/#token-validation). An app role has + the same value in every tenant that uses the application, so with a + multitenant registration, another tenant could assign the role to its own + service principals, and QuestDB would map their tokens to the same QuestDB + groups. +2. Under _Expose an API_, set the Application ID URI, which defaults to + `api://`. Services request their tokens for the scope + `/.default`. A token requested for any other API, such + as Microsoft Graph, carries that API in `aud`, and QuestDB rejects it. +3. Make the application issue v2.0 access tokens: in its _Manifest_, set + `api.requestedAccessTokenVersion` to `2`. The legacy manifest format calls + this property `accessTokenAcceptedVersion`. The `aud` claim of a v2.0 token + always carries the application ID, which + [Token validation](/docs/security/oidc/#token-validation) checks. A v1.0 + token, the default for applications that accept organizational accounts + only, can carry the Application ID URI instead. Web Console logins are not affected, because the + Web Console presents ID tokens. +4. Under _App roles_, define a role for each kind of access, with the + _Applications_ member type. The examples below use the value + `QuestDB.Ingest`. +5. Assign a role to each managed identity or service principal that needs + access. Assign the role to the identity itself: Entra ID does not add a role + assigned to a group to the tokens of the service principals in that group. + + - For the service principal of an app registration, open that registration, + not the QuestDB one. Under _API permissions_, select _Add a permission_, + _APIs my organization uses_, the QuestDB application, and + _Application permissions_. Add the role, and grant admin consent. + - For a managed identity, assign the role through Microsoft Graph, as + described in + [Assign a managed identity to an application role](https://learn.microsoft.com/en-us/entra/identity/managed-identities-azure-resources/how-to-assign-app-role-managed-identity). + +### Configure QuestDB + +Replace the two claim settings of the +[QuestDB configuration](#questdb-configuration) with fallback lists, and +restart QuestDB. Keep `acl.oidc.groups.encoded.in.token=true`, so that QuestDB +validates app-only tokens against the signing keys of Entra ID, the same way as +ID tokens: + +```ini title="server.conf" +# user tokens carry preferred_username, app-only tokens carry oid +acl.oidc.sub.claim=preferred_username,oid + +# app-only tokens carry roles, user tokens carry groups +acl.oidc.groups.claim=roles,groups +``` + +User logins still take the principal from `preferred_username` and the groups +from `groups`. Their tokens carry no `roles` claim, as long as the app roles of +the QuestDB application admit applications only. App-only tokens take the +principal from `oid` and the groups from `roles`, even when they also carry a +`groups` claim. QuestDB rejects a token that carries neither a role nor a +group. See +[How QuestDB picks a claim](/docs/security/oidc/#how-questdb-picks-a-claim). + +If your QuestDB configuration sets `acl.oidc.sub.claim=oid`, as described in +[Choose the principal claim](/docs/security/oidc/#choose-the-principal-claim), +keep it: app-only tokens carry `oid` too, so only `acl.oidc.groups.claim` +needs a list. + +### Map app roles to QuestDB groups + +QuestDB cannot create a table or add a column on ingestion for the service, +because the service is an external user. See +[Tables created by external users](/docs/security/oidc/#tables-created-by-external-users). +As the admin, create the tables that the service writes to, with every column +that it sends: + +```questdb-sql title="Create fx_trades before the service writes to it" +CREATE TABLE fx_trades ( + timestamp TIMESTAMP_NS, + symbol SYMBOL, + side SYMBOL, + price DOUBLE, + quantity DOUBLE +) TIMESTAMP(timestamp) PARTITION BY DAY; +``` + +Map each app role to a QuestDB group by the role's value, the same way as an +Entra ID group, and grant the group what the service needs. A service that +ingests over HTTP or [QWP](/docs/connect/wire-protocols/qwp-ingress-websocket/) +needs the `HTTP` endpoint permission and `INSERT` on its tables: + +```questdb-sql title="Map an app role to a group that ingests into fx_trades" +CREATE GROUP ingestApps WITH EXTERNAL ALIAS 'QuestDB.Ingest'; +GRANT HTTP TO ingestApps; +GRANT INSERT ON fx_trades TO ingestApps; +``` + +A service that queries its tables also needs `SELECT` on them, and a service +that connects to the +[PGWire endpoint](/docs/security/oidc/#oidc-for-the-pgwire-endpoint) also +needs the `PGWIRE` endpoint permission. See +[Endpoint permissions](/docs/security/rbac/#endpoint-permissions) for the other +endpoints. + +### Restrict access to app roles + +Unless _Assignment required_ is enabled for the QuestDB application, any +service principal in the tenant can get a token for QuestDB. A token without a +role falls back to the `groups` claim, so a service principal in a mapped +security group gets the permissions of that group without an app role. To make +an app role the only way for a service to get permissions, do one of the +following: + +- Keep managed identities and service principals out of the security groups + that are mapped to QuestDB groups. +- Enable _Assignment required_ in the _Properties_ of the QuestDB application, + under _Enterprise applications_. Users then need an assignment too, directly + or through a group, to log in to the Web Console. Assigning a group requires + Microsoft Entra ID P1 or P2, and gives access only to the direct members of + the group, not to the members of its nested groups. Entra ID also stops + asking users for consent, so + [grant tenant-wide admin consent](https://learn.microsoft.com/en-us/entra/identity/enterprise-apps/grant-admin-consent) + to the QuestDB application, or users cannot log in. + +### Request a token in the service + +The service requests its token for the QuestDB application. With the +[Azure Identity library](https://learn.microsoft.com/en-us/python/api/overview/azure/identity-readme) +for Python, `DefaultAzureCredential` requests the token as the managed identity +of the Azure resource that the service runs on, or as a service principal set +up in environment variables. For a user-assigned managed identity, set the +`AZURE_CLIENT_ID` environment variable to the client ID of the identity. +Otherwise, the credential uses the system-assigned identity. + +The examples use the QWP API of the `questdb` package, version 5.0 or later. +Install the libraries with +`python3 -m pip install -U questdb azure-identity pandas`. pandas is only +needed for `to_pandas()` in [Verify the setup](#verify-the-setup): + +```python title="Ingest into fx_trades with an app-only token" +import questdb +from azure.identity import DefaultAzureCredential +from questdb import TimestampNanos + +# Application ID URI of the QuestDB application, followed by /.default +scope = "api://8de84b90-1ea5-4e41-9e84-dba860aa01a6/.default" +credential = DefaultAzureCredential() +token = credential.get_token(scope) + +conf = f"wss::addr=questdb.example.com:9000;token={token.token};" +with questdb.connect(conf) as db: + with db.sender() as sender: + sender.row( + "fx_trades", + symbols={"symbol": "EURUSD", "side": "buy"}, + columns={"price": 1.0842, "quantity": 100000.0}, + at=TimestampNanos.now(), + ) + sender.flush(wait=True) +``` + +`flush(wait=True)` returns when the server has accepted the rows. Rows that the +server rejects, for example because the group lacks `INSERT` on the table, go +to the error handler of the connection, which logs them by default. See +[Server rejections](/docs/connect/clients/python/#server-rejections). + +The client presents the same token on every connection that it opens: the +first one, the additional connections of its pool, and the connections that +replace a failed one. Shortly after the token expires, QuestDB rejects these +new connections, as described in +[Token validation](/docs/security/oidc/#token-validation). A long-running +service must therefore switch to a new token before the old one expires: it +connects with the new token, then closes the old handle. +`token.expires_on` is a Unix timestamp. + +The credential caches the token in memory, and requests a new one when the +cached token expires within 5 minutes, or earlier when the token's refresh +time has passed, so calling `get_token()` often is cheap. The following example +looks for a new token in the last 10 minutes of the old one. When it cannot get +a new token or connect with it, it keeps the old token and handle, and tries +again with the next batch: + +```python title="Keep ingesting across token expiry" +import time + +import questdb +from azure.core.exceptions import AzureError +from azure.identity import DefaultAzureCredential +from questdb import QuestDBError, TimestampNanos + +SCOPE = "api://8de84b90-1ea5-4e41-9e84-dba860aa01a6/.default" +ADDR = "questdb.example.com:9000" +# look for a new token when the old one expires within this many seconds +REFRESH_MARGIN_SECONDS = 600 + +credential = DefaultAzureCredential() + + +def on_rejection(error): + # rows that the server rejected, for example for a missing grant + print("rejected:", error.category.tag, error.message) + + +def connect(token): + conf = f"wss::addr={ADDR};token={token.token};" + return questdb.connect(conf, error_handler=on_rejection) + + +def renew(token, db): + # returns the token and the handle to use from now on + try: + new_token = credential.get_token(SCOPE) + # the credential returns the cached token until it renews it + if new_token.token == token.token: + return token, db + new_db = connect(new_token) + except (AzureError, QuestDBError) as e: + # keep the old token and handle, and try again with the next batch + print("token renewal failed:", e) + return token, db + db.close() + return new_token, new_db + + +def next_trades(): + # replace with your source of trades + time.sleep(1) + return [("EURUSD", "buy", 1.0842, 100000.0)] + + +token = credential.get_token(SCOPE) +db = connect(token) +try: + while True: + trades = next_trades() + if time.time() > token.expires_on - REFRESH_MARGIN_SECONDS: + token, db = renew(token, db) + with db.sender() as sender: + for symbol, side, price, quantity in trades: + sender.row( + "fx_trades", + symbols={"symbol": symbol, "side": side}, + columns={"price": price, "quantity": quantity}, + at=TimestampNanos.now(), + ) + sender.flush(wait=True) +finally: + db.close() +``` + +See [Authentication and TLS](/docs/connect/clients/python/#authentication-and-tls) +for the client options. For the +[PGWire endpoint](/docs/security/oidc/#oidc-for-the-pgwire-endpoint), enable +`acl.oidc.pg.token.as.password.enabled`, and send the token as the password of +the `_sso` user. + +### Verify the setup + +`current_user()` returns the principal that QuestDB sees for the service: the +object ID of its service principal, from the `oid` claim. For a managed +identity, that is the _Object (principal) ID_ that the Azure portal shows for +the identity, not its client ID. Run the query from the service while it is +connected: + +```python title="Check the principal of the service" +# db is an open handle, such as the one in the examples above +with db.query("SELECT current_user()") as result: + print(result.to_pandas()) +``` + +If the service cannot connect, the server log gives the reason, as described in +[Troubleshooting OIDC logins](/docs/security/oidc/#troubleshooting-oidc-logins). +For a token that carries neither a role nor a group, the log lists the claims +that the token carries. + +### Change or revoke access + +QuestDB reads the roles from the token, so a change to the roles of a service +takes effect when the service presents a new token. Azure caches managed +identity tokens for around 24 hours, so a role assigned to, or removed from, a +managed identity can take hours to reach QuestDB. See the +[managed identity best practices](https://learn.microsoft.com/en-us/entra/identity/managed-identities-azure-resources/managed-identity-best-practice-recommendations). +To cut a service off without waiting for its token to expire, revoke the +permissions of its QuestDB group. This cuts off every service that holds the +app role, so to cut off one service at a time, give each service an app role, +and a QuestDB group, of its own. diff --git a/documentation/security/oidc-pingfederate.md b/documentation/security/oidc-pingfederate.md new file mode 100644 index 0000000000..6fae93e736 --- /dev/null +++ b/documentation/security/oidc-pingfederate.md @@ -0,0 +1,359 @@ +--- +title: PingFederate +sidebar_label: PingFederate +description: "Set up Web Console single sign-on for QuestDB Enterprise with PingFederate as the OIDC provider, authenticating users over LDAP." +--- + +import Screenshot from "@theme/Screenshot" +import { EnterpriseNote } from "@site/src/components/EnterpriseNote" + + + OpenID Connect (OIDC) enables SSO and token authentication with external Identity Providers. + + +This guide sets up single sign-on (SSO) for the QuestDB Enterprise +[Web Console](/docs/getting-started/web-console/overview/) with +[PingFederate](https://www.pingidentity.com/en/platform/capabilities/authentication-authority/pingfederate.html) +as the OpenID Connect (OIDC) provider. PingFederate authenticates users against +Microsoft Entra ID, formerly Azure Active Directory, over a secure LDAP +connection, and passes their group memberships to QuestDB in the User Info +response. QuestDB maps those groups to its own groups to grant permissions. + +For how QuestDB uses OIDC in general, such as the login flow and how it maps +groups to permissions, see [OpenID Connect (OIDC)](/docs/security/oidc/). + +## Set up PingFederate client + +First thing first, let's pick a name for the client! + + + +The QuestDB [Web Console](/docs/getting-started/web-console/overview/) is a SPA (Single Page App). + +As a result, it cannot store safely a client secret. + +Instead it can use PKCE (Proof Key for Code Exchange) to secure the flow. + +As shown above, leave the client authentication disabled. + +We also have to white list the URL of the [Web Console](/docs/getting-started/web-console/overview/) as a redirection URL: + + + +We can instruct PingFederate to automatically authorize the scopes requested by +the [Web Console](/docs/getting-started/web-console/overview/). + +The user will not be presented the extra window asking for consent after +authentication: + + + +The [Web Console](/docs/getting-started/web-console/overview/) uses the +[Authorization Code Flow](/docs/security/oidc/#authentication-and-authorization-flow), +and refreshes tokens automatically. + +Next, enable the grant types required for this flow: + + + +We've selected: + +- Authorization Code +- Refresh Token +- Access Token Validation (Client is a Resource Server) + +After that, select the token manager for the client. + +The token manager is responsible for issuing access tokens. + +All token related settings should be configured in the token manager. + + + +Finally, enable PKCE - as shown above - and save the settings. + +## Access Token Manager settings + +QuestDB does not require any special setup regarding the access token. + +We recommend that you do not to use shorter tokens than the default 28 +characters. + +As the QuestDB [Web Console](/docs/getting-started/web-console/overview/) refreshes the token automatically, there is no need +for long-lived tokens: + + + +We've selected: + +- Token length: 28 +- Token lifetime: 5 +- Lifetime extension policy: None +- Maximum token lifetime: Null +- Lifetime extension threshold percentage: 30 + +For the next step, we tune the Authorization Server. + +## Authorization Server settings + +These settings relate to the authorization code, refresh token and CORS. + + + +In this section, we've entered: + +- Authorization code timeout: 60 +- Authorization code entropy: 30 +- Client secret retention period: 0 + +Next, ensure the `ROLL REFRESH TOKEN VALUES` option is selected: + + + +It is also important to whitelist the [Web Console](/docs/getting-started/web-console/overview/)'s URL on the CORS list: + + + +## Set up a Microsoft Entra ID Data Source + +PingFederate needs a Data Source setup. + +This is a secure LDAP connection to Microsoft Entra ID, formerly known as Azure +Active Directory. + +The data source needs a: + +- name +- hostname +- port +- username and password for the LDAP connection + + + +We have given it the name EntraDS and it will be applied later. + +## Set up a Password Credential Validator + +Now that PingFederate has an LDAP connection, we can use it for authentication. + +First, create a Password Credential Validator: + + + +We've entered: + +- Instance name: EntraPCV +- Instance ID: EntraPCV +- Selected: LDAP Username Password Credential Validator +- Parent instance: None + +Furthermore, we now declare our previously created data source (`EntraDS`): + + + +This links our data store (`EntraDS`) to our PCV (`EntraPCV`). + +## Set up an Identity Provider + +We can use our PCV once we set up an Identity Provider. + +The IdP will be used to authenticate users against Active Directory using the +LDAP connection. + +We do this in the Type subsection: + + +Next, in the IdP Adapter section... + +Click: Add a new row to Credential Validators. + +Select the PCV (`EntraPCV`) we created. + +Optionally alter number of retries: + + + +## Add groups to OIDC policy management + +QuestDB now needs to know about the user's AD group memberships to find their +permissions. + +Groups are passed to QuestDB inside the User Info object in a custom claim. + +This has to be added in the OpenID Connect Policy Management. + +The field is Multi-Valued, because it is a list of group names. + +Under the Attribute Contract subsection, see: + + + +Next, click to the Attribute Scopes subsection. + +Ensure `groups` is among the `openid` attributes: + + + +Onwards to the Attribute Sources & User Lookup Section. + +From this view, you can add local data stores. + +Note item `test` of type of LDAP: + + + +We created it via the following choices in Add Attribute Source: + + + +Note where we specified the Data Store (`EntraDS`). + +This is also where the directory search parameters are defined. + +Back at the Attribute Sources & User Lookup Section section, note we have set +`email`. + +The source is `LDAP (test)`, while the value is `usePrincipalName`: + + + +And finally! + +In the same Attribute Sources & User Lookup Section... + +Find `groups`. + +Note the definition of Source (`LDAP (test)`) that bridges our various parts. + +The value is `memberOf`. + + + +## Enable Resource Owner Password Credentials (ROPC) flow + +As described in the +[OIDC operations document](/docs/security/oidc/#enable-ropc) +tools - such as `psql` - can be integrated with the OIDC provider using the ROPC flow. + +When setting this flow up, enable the Resource Owner Password Credentials flow in the +client settings. + +Next, create a Resource Owner Credentials Grant Mapping to map values obtained from +the Password Credential Validator (PCV) into the persistent grants. + +When setting this up, select the previously created LDAP Data Source and IdP Adapter, which links +to the existing PCV. + +Then select the `username` attribute of the PCV as `USER_KEY`. + +## Confirm QuestDB mappings and login + +QuestDB requires a mapping, as laid out in the +[OIDC operations document](/docs/security/oidc/#mapping-user-permissions). + +If a given user has the HTTP permission, they will be able to now login via the +[Web Console](/docs/getting-started/web-console/overview/). + +To test, head to `http://localhost:9000` and login. + +If all has been wired up well, then login will succeed. diff --git a/documentation/security/oidc.mdx b/documentation/security/oidc.mdx index 4dbb53ea6f..9a953f50a9 100644 --- a/documentation/security/oidc.mdx +++ b/documentation/security/oidc.mdx @@ -1,6 +1,6 @@ --- title: OpenID Connect (OIDC) Integration -description: "Configure OIDC in QuestDB Enterprise: Web Console SSO, token authentication for clients and services, claim mapping, and Entra ID managed identities." +description: "Configure OIDC in QuestDB Enterprise: Web Console SSO, token authentication for clients and services, claim mapping, token validation, and troubleshooting." --- import Screenshot from "@theme/Screenshot"; @@ -10,20 +10,23 @@ import { EnterpriseNote } from "@site/src/components/EnterpriseNote" OpenID Connect (OIDC) enables SSO and token authentication with external Identity Providers. -OpenID Connect (OIDC) integrates with Identity Providers (IdP) external to -QuestDB. - -It is a convenient way to integrate QuestDB into your enterprise environment. -It provides SSO (Single Sign-On) for the [Web Console](/docs/getting-started/web-console/overview/), -and token authentication for applications and services that connect to -QuestDB. For Azure services that run as a managed identity or a service -principal, see -[Microsoft Entra ID managed identities and service principals](#accept-managed-identity-and-service-principal-tokens). - -Microsoft Active Directory and Azure AD, for example, can be turned into an -Identity Provider. - -Specific installation steps depend on the type of the provider. +OpenID Connect (OIDC) connects QuestDB Enterprise to an external Identity +Provider (IdP), such as Microsoft Entra ID or PingFederate. Users sign in to +the [Web Console](/docs/getting-started/web-console/overview/) with single +sign-on (SSO), and applications and services authenticate with tokens from the +same provider, over HTTP, QWP, or PGWire. External users are not created in +QuestDB: they get the permissions of the QuestDB groups that their groups in +the Identity Provider map to. + +This page covers the login flow, which token a client sends, how QuestDB maps +the claims of a token to a principal and to QuestDB groups, including +[fallback claim lists](#fallback-claim-lists) that accept user and app-only +tokens with one configuration, [token validation](#token-validation), and +[troubleshooting rejected logins](#troubleshooting-oidc-logins). For +step-by-step setup, see the guides for +[Microsoft Entra ID](/docs/security/oidc-entra-id/), which also covers Azure +managed identities and service principals, and +[PingFederate](/docs/security/oidc-pingfederate/). ## Architecture overview @@ -50,7 +53,9 @@ In OAuth2/OIDC terms, the [Web Console](/docs/getting-started/web-console/overvi the _client_, and it is assigned an identifier: the **Client Id**. Each application which integrates via OIDC should be given a different **Client -Id**. +Id**. Clients that send ID tokens to QuestDB are the exception: QuestDB accepts +an ID token only when it was issued for QuestDB's client ID, as described in +[Non-interactive clients](#non-interactive-clients). ### OIDC Provider @@ -78,7 +83,8 @@ endpoints. The database, in OAuth2/OIDC terms the _protected resource_ or _resource server_. -Only processes requests which contain a valid access token. +Only processes requests which contain a valid token: an access token, or, with +`acl.oidc.groups.encoded.in.token=true`, a JWT such as an ID token. ## Authentication and Authorization Flow @@ -351,8 +357,9 @@ special version of the Authorization Code flow designed for mobile apps and single page applications. Regardless of which flow is used by the web or mobile application, the requested -access token can be used for authentication and authorization when communicating -with QuestDB as explained in the [above 7th step](#7-database-access). +token can be used for authentication and authorization when communicating with +QuestDB as explained in the [above 7th step](#7-database-access): the access +token, or the ID token with `acl.oidc.groups.encoded.in.token=true`. ### Jupyter notebook @@ -542,10 +549,40 @@ testldap=> Non-interactive clients are usually jobs or standalone applications, such as a client for ingesting data. It is practical to manage their credentials via an -OAuth2 provider too. +OAuth2 provider too. Such a client requests a token from the provider itself, +and sends it to QuestDB as a bearer token, for example in the +[`token` setting](/docs/connect/clients/connect-string/#auth) of the client's +configuration string, over ILP/HTTP or QWP. -As seen in the Jupyter notebook examples, the clients can request a token -themselves and then use it to authorise data ingestion: +Which token to send depends on where QuestDB reads the user information from, +and on who the client authenticates as: + +- With `acl.oidc.groups.encoded.in.token=false`, QuestDB sends the token to the + provider's User Info endpoint, so send the access token. Some providers + refuse tokens issued to applications at that endpoint, so to accept tokens + that services obtain for themselves, use + `acl.oidc.groups.encoded.in.token=true`. +- With `acl.oidc.groups.encoded.in.token=true`, QuestDB validates the token + itself, so send a JWT that passes [Token validation](#token-validation): + - For a user, send the ID token, because the provider often issues the + access token for another audience. The `aud` claim of an ID token is the + client that requested it, so request the token with QuestDB's client ID, + the value of `acl.oidc.client.id`. + - For a service that authenticates as itself, such as with the client + credentials grant, send the access token that the provider issues to it + for QuestDB. Configure the provider so that the `aud` claim of the token + contains the value of `acl.oidc.audience`, which defaults to QuestDB's + client ID. For Microsoft Entra ID, see + [Managed identities and service principals](/docs/security/oidc-entra-id/#managed-identities-and-service-principals). + +Tokens that the provider issues to a service often carry the principal and the +groups under different claim names than the tokens of users. To accept both +kinds of token with one configuration, list the claims of both. See +[Fallback claim lists](#fallback-claim-lists). + +The following example requests a token for a user with the Resource Owner +Password Credentials grant, then ingests over ILP/HTTP. It assumes that +QuestDB's client ID is `testclient`: ```python import json @@ -579,6 +616,13 @@ with Sender.from_conf(conf) as sender: sender.dataframe(df, table_name="foo", at="ts") ``` +With `acl.oidc.groups.encoded.in.token=true`, send the ID token instead of the +access token: + +```python +conf = f"http::addr=localhost:9000;token={tokens['id_token']};" +``` + Alternatively, a user may rely on QuestDB to authenticate them via the OAuth2 provider when the Resource Owner Password Credentials flow is enabled on the server side: @@ -600,36 +644,11 @@ with Sender.from_conf(conf) as sender: sender.dataframe(df, table_name="foo", at="ts") ``` -Which token to send depends on where QuestDB reads the user information from: - -- With `acl.oidc.groups.encoded.in.token=false`, QuestDB sends the token to the - provider's User Info endpoint, so send the access token. Some providers - refuse tokens issued to applications at that endpoint. -- With `acl.oidc.groups.encoded.in.token=true`, QuestDB validates the token - itself, so send a JWT that passes [Token validation](#token-validation). For - a user, that is the ID token, because the provider often issues the access - token for another audience. The `aud` claim of an ID token is the client - that requested it, so request the token with QuestDB's client ID, the value - of `acl.oidc.client.id`. The examples above assume that it is `testclient`. - -With `acl.oidc.groups.encoded.in.token=true`, the first example above sends the -ID token instead of the access token: - -```python -conf = f"http::addr=localhost:9000;token={tokens['id_token']};" -``` - QuestDB cannot create a table or add a column on ingestion for an external user, so the examples above need the table `foo` to exist, with the columns of `data.csv`. See [Tables created by external users](#tables-created-by-external-users). -Tokens that the provider issues to an application, rather than to a user, often -carry the principal and the groups under different claim names. To accept both -kinds of token with one configuration, list the claims of both. See -[Fallback claim lists](#fallback-claim-lists), and for Microsoft Entra ID, -[Microsoft Entra ID managed identities and service principals](#accept-managed-identity-and-service-principal-tokens). - ## OIDC for the PGWire endpoint If the @@ -742,39 +761,32 @@ login via the [Web Console](/docs/getting-started/web-console/overview/). } ``` -Any change made to the user's group membership in the Identity Provider, QuestDB -will adjust the user's access list. - -:::note - -There may be a slight delay due to the User Info Cache. - -QuestDB will use the cached information until it becomes stale, and gets -updated. - -::: - -The same stands for changes made to the user's status within the Identity -Provider. - -For example, a disabled user will not be kicked out of QuestDB immediately. - -The `acl.oidc.cache.ttl` config option drives how often user information should -be synchronized with the Identity Providers. - -It should be set accordingly to your organization's policies. - -With `acl.oidc.groups.encoded.in.token=true`, QuestDB reads the groups from the -token instead of asking the Identity Provider, so refreshing the cache does not -pick up changes. A change to the user's groups or status takes effect when the -client presents a new token. Until then, QuestDB accepts the old token for up -to 60 seconds plus `acl.oidc.cache.ttl` after it expires, and connections that -are already open stay open. See [Token validation](#token-validation). - -The HTTP session that the Web Console opens at login is not tied to the token. -QuestDB accepts requests in the session, with the groups of the user's latest -login, until the user logs out or the session has been idle for -`http.session.timeout`, 30 minutes by default. +QuestDB picks up a change to the user's groups or status in the Identity +Provider, such as a disabled user, as follows: + +- With the User Info endpoint, the default, when the cached user information + expires. [`acl.oidc.cache.ttl`](/docs/configuration/oidc/#acloidccachettl), + 30 seconds by default, controls how long QuestDB caches it. Set it according + to your organization's policies. +- With `acl.oidc.groups.encoded.in.token=true`, when the client presents a new + token. QuestDB reads the groups from the token instead of asking the Identity + Provider, so refreshing the cache does not pick up changes. Until then, + QuestDB accepts the old token for up to 60 seconds plus `acl.oidc.cache.ttl` + after it expires. See [Token validation](#token-validation). + +In both cases, connections that are already open, such as PGWire connections, +stay open. The HTTP session that the Web Console opens at login is not tied to +the token either. QuestDB accepts requests in the session, with the groups of +the user's latest login, until the user logs out or the session has been idle +for `http.session.timeout`, 30 minutes by default. Every request restarts the +idle timer, so a session in use does not expire. The Web Console logs the user +out when it cannot refresh the token, for example after the user is disabled, +but another client that reuses the session cookie keeps the access for as long +as it stays active. + +To cut off access immediately, revoke the permissions of the QuestDB groups +that the user's groups map to. This takes effect at once for every session and +connection, including those of other users who map to the same groups. ### Tables created by external users @@ -785,9 +797,17 @@ permissions on it. Permissions cannot be granted to external users, so: external user. Create the tables in advance, with every column that the client sends. - An external user that creates a table or adds a column with SQL must name - one of its groups as the owner, with `OWNED BY`. See + the owner with `OWNED BY`: one of the QuestDB groups that its groups map to, + as set up in [Mapping user permissions](#mapping-user-permissions), not the + name of a group in the Identity Provider. See [`CREATE TABLE`](/docs/query/sql/create-table/#owned-by) and [`ALTER TABLE ADD COLUMN`](/docs/query/sql/alter-table-add-column/#owned-by). +- The same applies when an external user creates a view, a materialized view, + a live view, or a service account. See + [`CREATE VIEW`](/docs/query/sql/create-view/#owned-by-enterprise), + [`CREATE MATERIALIZED VIEW`](/docs/query/sql/create-mat-view/#owned-by-clause), + [`CREATE LIVE VIEW`](/docs/query/sql/create-live-view/#owned-by-clause), and + [`CREATE SERVICE ACCOUNT`](/docs/query/sql/acl/create-service-account/). ## User and group claims @@ -848,10 +868,11 @@ Both settings accept a comma-separated list of claim names in priority order, so one configuration can accept tokens that carry the principal and the groups under different claim names. Tokens issued to users and tokens issued to applications often differ this way. For example, Microsoft Entra ID user -tokens, configured as in the [Entra ID guide](#microsoft-entraid), carry +tokens, configured as in the +[Entra ID guide](/docs/security/oidc-entra-id/#web-console-sign-in), carry `preferred_username` and `groups`, while its -[app-only tokens](#accept-managed-identity-and-service-principal-tokens) carry -`oid` and `roles`: +[app-only tokens](/docs/security/oidc-entra-id/#managed-identities-and-service-principals) +carry `oid` and `roles`: ```ini title="server.conf" acl.oidc.sub.claim=preferred_username,oid @@ -888,12 +909,12 @@ token payloads, shortened to the relevant claims, as follows: `roles` comes before `groups` because an app-only token can also carry a `groups` claim, and QuestDB takes the groups from the first claim on the list that holds one. See -[Microsoft Entra ID managed identities and service principals](#accept-managed-identity-and-service-principal-tokens). +[Managed identities and service principals](/docs/security/oidc-entra-id/#managed-identities-and-service-principals). ### How QuestDB picks a claim -The following rules decide which claim QuestDB uses, for a single claim name -and for a list: +In QuestDB Enterprise 4.0.2 and later, the following rules decide which claim +QuestDB uses, for a single claim name and for a list: - The order of the list decides, not the order of the claims in the token. With the configuration in [Fallback claim lists](#fallback-claim-lists), a @@ -907,6 +928,8 @@ and for a list: - Only top-level claims count. A claim nested in another claim is ignored, such as the `roles` that Keycloak puts inside its `realm_access` claim. To use such values, configure the provider to emit them as a top-level claim. + In Keycloak, for example, add a _User Realm Role_ mapper with the + _Token Claim Name_ `roles` to the client that requests the token. - Claim names are case-sensitive. - A listed claim must have the expected shape: a single value, such as a string, for the principal, and a single value or a flat array of values for @@ -915,12 +938,24 @@ and for a list: claim earlier on the list carries a value. - Claims that are not listed are ignored, whatever their shape. +With the configuration in [Fallback claim lists](#fallback-claim-lists), +QuestDB reads these token payloads, shortened to the relevant claims, as +follows: + +| Claims in the token | Result | +| ------------------------------------------------------ | --------------------------------------------------------------------------------------------------------------------------- | +| `"preferred_username": ""`, `"oid": "4c9a4e3b-..."` | Principal `4c9a4e3b-...`, from `oid`: the empty `preferred_username` falls through | +| `"roles": []`, `"groups": ["87654321-..."]` | Groups `87654321-...`, from `groups`: the empty `roles` array falls through | +| `"roles": [null, ""]`, `"groups": ["87654321-..."]` | Groups `87654321-...`, from `groups`: `roles` holds no group name once `null` and `""` are skipped | +| `"roles": ["QuestDB.Ingest"]`, `"groups": {"id": "x"}` | The login fails with `unexpected input format`: `groups` is listed and holds an object, even though `roles` carries a group | + With `acl.oidc.groups.encoded.in.token=true`, QuestDB validates the token before it applies these rules. See [Token validation](#token-validation). ### Startup validation -With OIDC enabled, QuestDB checks both settings at startup. Spaces around claim +In QuestDB Enterprise 4.0.2 and later, with OIDC enabled, QuestDB checks both +settings at startup. Spaces around claim names are trimmed and empty entries are skipped, so `preferred_username, oid,,sub` reads as `preferred_username,oid,sub`. QuestDB refuses to start when: @@ -956,16 +991,16 @@ accepts any token that the provider's keys sign for the expected audience, so make sure that no one outside your organization can obtain tokens for the QuestDB application. With Microsoft Entra ID, register the application as single-tenant, as described in -[Set up the client application in Entra ID](#set-up-the-client-application-in-entra-id). +[Set up the client application in Entra ID](/docs/security/oidc-entra-id/#set-up-the-client-application-in-entra-id). QuestDB checks a token when a request or a connection authenticates with it, and caches the result for up to -[`acl.oidc.cache.ttl`](/docs/configuration/oidc/#acloidccachettl). Because of -the clock skew and the cache, QuestDB can accept a token, on new connections -too, for up to 60 seconds plus `acl.oidc.cache.ttl` after it expires. After -that, it rejects every request and new connection that presents the token. A -PGWire or WebSocket connection that is already open stays open when its token -expires. +[`acl.oidc.cache.ttl`](/docs/configuration/oidc/#acloidccachettl), 30 seconds +by default. Because of the clock skew and the cache, QuestDB can accept a +token, on new connections too, for up to 60 seconds plus `acl.oidc.cache.ttl` +after it expires. After that, it rejects every request and new connection that +presents the token. A PGWire or WebSocket connection that is already open stays +open when its token expires. With `acl.oidc.groups.encoded.in.token=false`, QuestDB sends the token to the provider's User Info endpoint, which validates it. See @@ -996,6 +1031,10 @@ login unless another listed claim carries groups. To avoid this, emit only the groups that are assigned to the application: in the _Token configuration_ of the QuestDB application, select _Groups assigned to the application_, then assign the mapped groups to the application under _Enterprise applications_. +The token then carries only the assigned groups that the user is a direct +member of, so a user who is in a mapped group only through a nested group +loses that group. Assigning groups to an application requires Microsoft Entra +ID P1 or P2. A token that fails [token validation](#token-validation) is rejected the same way, but the server log gives the reason instead of the claim lists, for @@ -1009,15 +1048,17 @@ Request failed [errorMsg=Unable to find public key with key id [...]] `InvalidAudience` usually means that the token was requested for another API, or that Entra ID issued a v1.0 access token. See -[Microsoft Entra ID managed identities and service principals](#accept-managed-identity-and-service-principal-tokens). +[Managed identities and service principals](/docs/security/oidc-entra-id/#managed-identities-and-service-principals). Don't set `acl.oidc.audience` to the Application ID URI to work around it: the setting takes a single value, and the ID tokens of Web Console users carry the application ID, so their logins would fail. `Unable to find public key with key id` means that none of the signing keys that the provider publishes matches the token. QuestDB downloads the keys again -when it meets a key ID that it does not know, so either another provider issued -the token, or QuestDB cannot reach the provider's keys, which it logs as +when they are older than +[`acl.oidc.public.keys.expiry`](/docs/configuration/oidc/#acloidcpublickeysexpiry), +and when it meets a key ID that it does not know, so either another provider +issued the token, or QuestDB cannot reach the provider's keys, which it logs as `Unable to download public keys from OIDC provider`. With `acl.oidc.groups.encoded.in.token=false`, the provider's User Info @@ -1028,6 +1069,11 @@ If the login succeeds but every request is denied, check that at least one of the user's groups is mapped to a QuestDB group that has the endpoint permission. See [Mapping user permissions](#mapping-user-permissions). +If the login succeeds but ingestion fails because a table or a column does not +exist, create it in advance. QuestDB cannot create a table or add a column on +ingestion for an external user. See +[Tables created by external users](#tables-created-by-external-users). + ## Versions before 4.0.2 QuestDB Enterprise versions before 4.0.2 differ as follows: @@ -1071,944 +1117,14 @@ For all OIDC-related configuration options of QuestDB, see ## Identity provider guides {#active-directory} -The following sections are guides for setting up single sign-on (SSO) with various OAuth2 providers, -and for authenticating services with app-only tokens from Microsoft Entra ID. - -### PingFederate - -This document helps set up SSO authentication for the Web Console in -[PingFederate](https://www.pingidentity.com/en/platform/capabilities/authentication-authority/pingfederate.html). - -It is assumed that the Azure Active Directory serves as the Identity Provider -(IdP). - -#### Set up PingFederate client - -First thing first, let's pick a name for the client! - - - -The QuestDB [Web Console](/docs/getting-started/web-console/overview/) is a SPA (Single Page App). - -As a result, it cannot store safely a client secret. - -Instead it can use PKCE (Proof Key for Code Exchange) to secure the flow. - -As shown above, leave the client authentication disabled. - -We also have to white list the URL of the [Web Console](/docs/getting-started/web-console/overview/) as a redirection URL: - - - -We can instruct PingFederate to automatically authorize the scopes requested by -the [Web Console](/docs/getting-started/web-console/overview/). - -The user will not be presented the extra window asking for consent after -authentication: - - - -The [Web Console](/docs/getting-started/web-console/overview/) uses the -[Authorization Code Flow](/docs/security/oidc/#authentication-and-authorization-flow), -and refreshes tokens automatically. - -Next, enable the grant types required for this flow: - - - -We've selected: - -- Authorization Code -- Refresh Token -- Access Token Validation (Client is a Resource Server) - -After that, select the token manager for the client. - -The token manager is responsible for issuing access tokens. - -All token related settings should be configured in the token manager. - - - -Finally, enable PKCE - as shown above - and save the settings. - -#### Access Token Manager settings - -QuestDB does not require any special setup regarding the access token. - -We recommend that you do not to use shorter tokens than the default 28 -characters. - -As the QuestDB [Web Console](/docs/getting-started/web-console/overview/) refreshes the token automatically, there is no need -for long-lived tokens: - - - -We've selected: - -- Token length: 28 -- Token lifetime: 5 -- Lifetime extension policy: None -- Maximum token lifetime: Null -- Lifetime extension threshold percentage: 30 - -For the next step, we tune the Authorization Server. - -#### Authorization Server settings - -These settings relate to the authorization code, refresh token and CORS. - - - -In this section, we've entered: - -- Authorization code timeout: 60 -- Authorization code entropy: 30 -- Client secret retention period: 0 - -Next, ensure the `ROLL REFRESH TOKEN VALUES` option is selected: - - - -It is also important to whitelist the [Web Console](/docs/getting-started/web-console/overview/)'s URL on the CORS list: - - - -#### Set up a Microsoft Entra ID Data Source - -PingFederate needs a Data Source setup. - -This is a secure LDAP connection to Microsoft Entra ID, formerly known as Azure -Active Directory. - -The data source needs a: - -- name -- hostname -- port -- username and password for the LDAP connection - - - -We have given it the name EntraDS and it will be applied later. - -#### Set up a Password Credential Validator - -Now that PingFederate has an LDAP connection, we can use it for authentication. - -First, create a Password Credential Validator: - - - -We've entered: - -- Instance name: EntraPCV -- Instance ID: EntraPCV -- Selected: LDAP Username Password Credential Validator -- Parent instance: None - -Furthermore, we now declare our previously created data source (`EntraDS`): - - - -This links our data store (`EntraDS`) to our PCV (`EntraPCV`). - -#### Set up an Identity Provider - -We can use our PCV once we set up an Identity Provider. - -The IdP will be used to authenticate users against Active Directory using the -LDAP connection. - -We do this in the Type subsection: - - -Next, in the IdP Adapter section... - -Click: Add a new row to Credential Validators. - -Select the PCV (`EntraPCV`) we created. - -Optionally alter number of retries: - - - -#### Add groups to OIDC policy management - -QuestDB now needs to know about the user's AD group memberships to find their -permissions. - -Groups are passed to QuestDB inside the User Info object in a custom claim. - -This has to be added in the OpenID Connect Policy Management. - -The field is Multi-Valued, because it is a list of group names. - -Under the Attribute Contract subsection, see: - - - -Next, click to the Attribute Scopes subsection. - -Ensure `groups` is among the `openid` attributes: - - - -Onwards to the Attribute Sources & User Lookup Section. - -From this view, you can add local data stores. - -Note item `test` of type of LDAP: - - - -We created it via the following choices in Add Attribute Source: - - - -Note where we specified the Data Store (`EntraDS`). - -This is also where the directory search parameters are defined. - -Back at the Attribute Sources & User Lookup Section section, note we have set -`email`. - -The source is `LDAP (test)`, while the value is `usePrincipalName`: - - - -And finally! - -In the same Attribute Sources & User Lookup Section... - -Find `groups`. - -Note the definition of Source (`LDAP (test)`) that bridges our various parts. - -The value is `memberOf`. - - - -#### Enable Resource Owner Password Credentials (ROPC) flow - -As described in the -[OIDC operations document](/docs/security/oidc/#enable-ropc) -tools - such as `psql` - can be integrated with the OIDC provider using the ROPC flow. - -When setting this flow up, enable the Resource Owner Password Credentials flow in the -client settings. - -Next, create a Resource Owner Credentials Grant Mapping to map values obtained from -the Password Credential Validator (PCV) into the persistent grants. - -When setting this up, select the previously created LDAP Data Source and IdP Adapter, which links -to the existing PCV. - -Then select the `username` attribute of the PCV as `USER_KEY`. - -#### Confirm QuestDB mappings and login - -QuestDB requires a mapping, as laid out in the -[OIDC operations document](/docs/security/oidc/#mapping-user-permissions). - -If a given user has the HTTP permission, they will be able to now login via the -[Web Console](/docs/getting-started/web-console/overview/). - -To test, head to `http://localhost:9000` and login. - -If all has been wired up well, then login will succeed. - -
- ### Microsoft Entra ID {#microsoft-entraid} -This document sets up SSO authentication for the [QuestDB Web Console](/docs/getting-started/web-console/overview/) in -[Microsoft Entra ID](https://www.microsoft.com/en-gb/security/business/identity-access/microsoft-entra-id), formerly known as Azure AD. - -To also let services that run as a managed identity or a service principal -call QuestDB with app-only tokens, complete this setup, then follow -[Microsoft Entra ID managed identities and service principals](#accept-managed-identity-and-service-principal-tokens). - - -:::tip - -To enlarge the images, click or tap them. - -::: - -#### Set up the client application in Entra ID - -First thing first, let's pick a name for the client! - -Then head to _Microsoft Entra Admin Center_, and register the application -under _Identity - App registrations - New registration_. - -Under _Supported account types_, select _Accounts in this organizational -directory only_. QuestDB does not check which tenant issued a token, so the -application must not accept accounts from other tenants. See -[Token validation](#token-validation). - - - -The QuestDB [Web Console](/docs/getting-started/web-console/overview/) is a SPA (Single Page App). - -As a result, it cannot store safely a client secret. - -Instead, it can use PKCE (Proof Key for Code Exchange) to secure the flow. - -When registering the application, select the SPA platform. - -We also have to specify the URL of the [Web Console](/docs/getting-started/web-console/overview/) as Redirect URI. - - - -After clicking _Register_, we have created a client application with the -name _QuestDB_. - -Each application is assigned a unique id (known as Client ID in the -OAuth2 - OIDC standard). The client will identify itself with this id -when sending requests to Entra ID. - - - -We find the platform configurations under _Authentication_. This is the place where -the previously set redirect URI can be viewed and modified. We can also specify -additional redirect URIs, if necessary. - -The redirect URIs of the application are automatically eligible for the -_Authorization Code Flow with PKCE_, which is a special version of the OAuth2 standard's -Authorization Code Flow. It is specifically designed for applications where a client -secret (e.g. a password) could not be kept safely. As single page applications run in -the browser, they fall into this category. - -The redirect URIs are also added to the _CORS_ (Cross-Origin Resource Sharing) policy -of Entra ID. CORS is a mechanism to allow a web page, such as the Web Console, to access -resources from a different domain than the one that served the page. In this context -this means that we let the Web Console to access Entra ID, while its origin is the -HTTP endpoint of QuestDB. - - - -If we scroll down to the bottom of this page, we can also find a section where we -can enable the _Resource Owner Password Credential Flow_. - -This OAuth2 flow is legacy, and should be enabled only if there is a requirement -of connecting to QuestDB using SSO (Single Sign-On) via clients not supporting -redirect based web flows. -This could mean a Postgres client without OAuth2 integration, such as _psql_, or -a standalone in-house client application, or could be just a jupyter notebook. - -The main issue with this flow is that the client application has to be trusted -with the user's login details. The user's credentials are passed to the -application, in this case to QuestDB, and the client application uses these -credentials to authenticate the user by forwarding them to the identity provider, -in this case to Entra ID. - -It is guaranteed that QuestDB does not store the user's credentials in any way. -They are not persisted into the database, not even in encrypted form. -The login details are treated as passthrough information. Only exception is -that server logs can contain the username, logged for audit purposes. - - - -Our next stop is the _Token configuration_, where the OAuth2/OIDC access and ID -tokens can be customized. - -Note that users can be authenticated without customized tokens, but authorization -would prove to be challenging. The user's security groups are not included -in the tokens by default. - -QuestDB can be configured to request the user's groups from the UserInfo -endpoint of the OAuth2 server, but Entra ID cannot be configured to provide -this information via the UserInfo endpoint. -Therefore, we choose to customize the tokens, QuestDB will decode and -validate the ID token, and take the group information from there. - -QuestDB authorization relies on receiving the group memberships of the user. -Entra ID groups should be mapped to QuestDB groups, and permissions can be -granted to the QuestDB groups. Detailed information about group mappings can -be found in the [OIDC integration](/docs/security/oidc/#user-permissions) -documentation. - - - -The customized tokens contain user information which cannot be accessed -without permission. User information is provided by Microsoft Graph, so -the client application needs specific permissions to access -Microsoft Graph APIs. +The [Microsoft Entra ID guide](/docs/security/oidc-entra-id/) sets up single +sign-on for the Web Console, and app-only tokens for Azure services that run as +a managed identity or a service principal. -These permissions can be configured under _API permissions_. It is important -to note that we will be setting _Delegated_ permissions here, meaning we -are not granting actual permissions to access user data. Instead, each user -logging into QuestDB will have to consent to accessing their user profile. - - - -By default, the _User.Read_ permission is added to the list, but what we -really need is: - - openid: to be able to issue ID tokens - - profile: to access user information - - offline_access: to be able to issue refresh tokens - -By clicking on _Microsoft Graph_ we can select and add these permissions. - - - -The _User.Read_ permission is not needed. It can be removed by clicking -on the `...` at the end of the row, and selecting _Remove permission_ from -the popup menu. - - - -With this we have finished setting up the QuestDB client application -in Entra ID, and now we can wire QuestDB and Entra ID together by -adding OIDC configuration to QuestDB. - -#### QuestDB configuration - -The below should be set in QuestDB's `server.conf`: - -```shell -# enable OIDC -acl.oidc.enabled=true - -# the claim contains the user's sign-in name -acl.oidc.sub.claim=preferred_username - -# the claim contains the user's group memberships -acl.oidc.groups.claim=groups - -# groups are encoded in the token -acl.oidc.groups.encoded.in.token=true - -# OIDC configuration endpoint of Entra ID -acl.oidc.configuration.url=https://login.microsoftonline.com/12345678-1234-1234-1234-123456789abc/v2.0/.well-known/openid-configuration - -# application ID taken from Entra ID -acl.oidc.client.id=8de84b90-1ea5-4e41-9e84-dba860aa01a6 - -# redirect URI, QuestDB's HTTP endpoint -acl.oidc.redirect.uri=http://localhost:9000 - -# OAuth scopes the user has to consent to -acl.oidc.scope=openid profile offline_access - -# enable ROPC flow -# optional, required only if ROPC is enabled in Entra ID -acl.oidc.ropc.flow.enabled=true -``` - -`preferred_username` shows the user's sign-in name in the Web Console and in -the logs. Entra ID can change a sign-in name, and can give it to a new user -later. To identify users by an object ID that never changes, set -`acl.oidc.sub.claim=oid`. See -[Choose the principal claim](#choose-the-principal-claim). - -The application ID and the OIDC configuration endpoint's URL can be found -in the Overview of the application in Entra ID. - -The application ID is displayed right under the application's name, the -OIDC configuration endpoint is displayed on the panel which opens up when -the _Endpoints_ button is clicked. - - - -#### Map groups and grant permissions - -Now we can start QuestDB, and login with the built-in admin to create -group mappings. - -As mentioned earlier, authorization works by mapping Entra ID groups -to QuestDB groups. When the user logs in, QuestDB decodes Entra ID -group memberships from the token, then finds the QuestDB groups -mapped to them, and the user gets the permissions based on the -mapped groups. - -```questdb-sql title="Create a group which is mapped to an Entra ID group" -CREATE GROUP extUsers WITH EXTERNAL ALIAS '87654321-1234-1234-1234-123456789abc'; -``` -The above command maps the Entra ID group identified by object -id `87654321-1234-1234-1234-123456789abc` to a QuestDB group called `extUsers`. - -We should grant the necessary QuestDB endpoint permissions first -to make sure users can access the Web Console, Postgres and ILP -interfaces as required. [Read more about endpoint permissions](/docs/security/rbac/#endpoint-permissions). - -```questdb-sql title="Grant endpoint permissions" -GRANT HTTP, PGWIRE TO groupName; -``` - -Now we can grant the rest of the permissions as required. We can -grant access to tables, for example. - -```questdb-sql title="Grant database permissions" -GRANT SELECT ON table1, table2 to groupName; -``` - -#### Confirm group mappings and login - -To test, head to the Web Console and login. - -If all has been wired up well, then login will succeed, and the user -will have the access granted to them. - -### Microsoft Entra ID managed identities and service principals {#accept-managed-identity-and-service-principal-tokens} - -Azure services that run as a managed identity or a service principal can -authenticate to QuestDB with app-only access tokens from Entra ID, next to the -users who log in to the Web Console. Each service gets the permissions of the -QuestDB groups that its app roles map to. - -:::warning Requires QuestDB Enterprise 4.0.2 - -Accepting app-only tokens requires QuestDB Enterprise 4.0.2 or later. On -earlier versions, the configuration below rejects every OIDC login, user logins -included. - -::: - -This setup builds on the Web Console setup above: the application registered in -[Set up the client application in Entra ID](#set-up-the-client-application-in-entra-id), -and the [QuestDB configuration](#questdb-configuration). - -App-only tokens identify the caller by the object ID of its service principal, -in the `oid` claim, and list the -[app roles](https://learn.microsoft.com/en-us/entra/identity-platform/howto-add-app-roles-in-apps) -assigned to it in the `roles` claim. They carry no `name` or -`preferred_username` claim. If the service principal is a member of a security -group, its token also carries a `groups` claim, because the QuestDB application -emits group claims, as set up under _Token configuration_ in -[Set up the client application in Entra ID](#set-up-the-client-application-in-entra-id). - -QuestDB treats the service as an external user, not as a QuestDB -[service account](/docs/security/rbac/#users-and-service-accounts), so -permissions cannot be granted to it directly. With the configuration below, -when its token carries no role but carries a `groups` claim, the service gets -the permissions of its Entra ID security groups instead. See -[Restrict access to app roles](#restrict-access-to-app-roles). - -#### Configure the application in Entra ID - -1. Make sure that the application is single-tenant, as described in - [Token validation](#token-validation). An app role has the same value in - every tenant that uses the application, so with a multitenant registration, - another tenant could assign the role to its own service principals, and - QuestDB would map their tokens to the same QuestDB groups. -2. Under _Expose an API_, set the Application ID URI, which defaults to - `api://`. Services request their tokens for the scope - `/.default`. A token requested for any other API, such - as Microsoft Graph, carries that API in `aud`, and QuestDB rejects it. -3. Make the application issue v2.0 access tokens: in its _Manifest_, set - `api.requestedAccessTokenVersion` to `2`. The legacy manifest format calls - this property `accessTokenAcceptedVersion`. The `aud` claim of a v2.0 token - always carries the application ID, which - [Token validation](#token-validation) checks. A v1.0 token, the default for - applications that accept organizational accounts only, can carry the - Application ID URI instead. Web Console logins are not affected, because the - Web Console presents ID tokens. -4. Under _App roles_, define a role for each kind of access, with the - _Applications_ member type. The examples below use the value - `QuestDB.Ingest`. -5. Assign a role to each managed identity or service principal that needs - access. Assign the role to the identity itself: Entra ID does not add a role - assigned to a group to the tokens of the service principals in that group. - - - For the service principal of an app registration, open that registration, - not the QuestDB one. Under _API permissions_, select _Add a permission_, - _APIs my organization uses_, the QuestDB application, and - _Application permissions_. Add the role, and grant admin consent. - - For a managed identity, assign the role through Microsoft Graph, as - described in - [Assign a managed identity to an application role](https://learn.microsoft.com/en-us/entra/identity/managed-identities-azure-resources/how-to-assign-app-role-managed-identity). - -#### Configure QuestDB - -Replace the two claim settings of the -[QuestDB configuration](#questdb-configuration) with fallback lists, and -restart QuestDB. Keep `acl.oidc.groups.encoded.in.token=true`, so that QuestDB -validates app-only tokens against the signing keys of Entra ID, the same way as -ID tokens: - -```ini title="server.conf" -# user tokens carry preferred_username, app-only tokens carry oid -acl.oidc.sub.claim=preferred_username,oid - -# app-only tokens carry roles, user tokens carry groups -acl.oidc.groups.claim=roles,groups -``` - -User logins take the principal from `preferred_username` and the groups from -`groups`, as before. Their tokens carry no `roles` claim, as long as the app -roles of the QuestDB application admit applications only. App-only tokens take -the principal from `oid` and the groups from `roles`, even when they also carry -a `groups` claim. QuestDB rejects a token that carries neither a role nor a -group. See [How QuestDB picks a claim](#how-questdb-picks-a-claim). - -If your QuestDB configuration sets `acl.oidc.sub.claim=oid`, as described in -[Choose the principal claim](#choose-the-principal-claim), keep it: app-only -tokens carry `oid` too, so only `acl.oidc.groups.claim` needs a list. - -#### Map app roles to QuestDB groups - -QuestDB cannot create a table or add a column on ingestion for the service, -because the service is an external user. See -[Tables created by external users](#tables-created-by-external-users). As the -admin, create the tables that the service writes to, with every column that it -sends: - -```questdb-sql title="Create fx_trades before the service writes to it" -CREATE TABLE fx_trades ( - timestamp TIMESTAMP_NS, - symbol SYMBOL, - side SYMBOL, - price DOUBLE, - quantity DOUBLE -) TIMESTAMP(timestamp) PARTITION BY DAY; -``` - -Map each app role to a QuestDB group by the role's value, the same way as an -Entra ID group, and grant the group what the service needs. A service that -ingests over HTTP or [QWP](/docs/connect/wire-protocols/qwp-ingress-websocket/) -needs the `HTTP` endpoint permission and `INSERT` on its tables: - -```questdb-sql title="Map an app role to a group that ingests into fx_trades" -CREATE GROUP ingestApps WITH EXTERNAL ALIAS 'QuestDB.Ingest'; -GRANT HTTP TO ingestApps; -GRANT INSERT ON fx_trades TO ingestApps; -``` - -A service that queries its tables also needs `SELECT` on them, and a service -that connects to the [PGWire endpoint](#oidc-for-the-pgwire-endpoint) also -needs the `PGWIRE` endpoint permission. See -[Endpoint permissions](/docs/security/rbac/#endpoint-permissions) for the other -endpoints. - -#### Restrict access to app roles - -Unless _Assignment required_ is enabled for the QuestDB application, any -service principal in the tenant can get a token for QuestDB. A token without a -role falls back to the `groups` claim, so a service principal in a mapped -security group gets the permissions of that group without an app role. To make -an app role the only way for a service to get permissions, do one of the -following: - -- Keep managed identities and service principals out of the security groups - that are mapped to QuestDB groups. -- Enable _Assignment required_ in the _Properties_ of the QuestDB application, - under _Enterprise applications_. Users then need an assignment too, directly - or through a group, to log in to the Web Console. Entra ID also stops asking - users for consent, so - [grant tenant-wide admin consent](https://learn.microsoft.com/en-us/entra/identity/enterprise-apps/grant-admin-consent) - to the QuestDB application, or users cannot log in. - -#### Request a token in the service - -The service requests its token for the QuestDB application. With the -[Azure Identity library](https://learn.microsoft.com/en-us/python/api/overview/azure/identity-readme) -for Python, `DefaultAzureCredential` requests the token as the managed identity -of the Azure resource that the service runs on, or as a service principal set -up in environment variables. For a user-assigned managed identity, set the -`AZURE_CLIENT_ID` environment variable to the client ID of the identity. -Otherwise, the credential uses the system-assigned identity. - -The examples use the QWP API of the `questdb` package, version 5.0 or later. -Install the libraries with -`python3 -m pip install -U questdb azure-identity pandas`. pandas is only -needed for `to_pandas()` in [Verify the setup](#verify-the-setup): - -```python title="Ingest into fx_trades with an app-only token" -import questdb -from azure.identity import DefaultAzureCredential -from questdb import TimestampNanos - -# Application ID URI of the QuestDB application, followed by /.default -scope = "api://8de84b90-1ea5-4e41-9e84-dba860aa01a6/.default" -credential = DefaultAzureCredential() -token = credential.get_token(scope) - -conf = f"wss::addr=questdb.example.com:9000;token={token.token};" -with questdb.connect(conf) as db: - with db.sender() as sender: - sender.row( - "fx_trades", - symbols={"symbol": "EURUSD", "side": "buy"}, - columns={"price": 1.0842, "quantity": 100000.0}, - at=TimestampNanos.now(), - ) - sender.flush(wait=True) -``` - -`flush(wait=True)` returns when the server has accepted the rows. Rows that the -server rejects, for example because the group lacks `INSERT` on the table, go -to the error handler of the connection, which logs them by default. See -[Server rejections](/docs/connect/clients/python/#server-rejections). - -The client presents the same token on every connection that it opens: the -first one, the additional connections of its pool, and the connections that -replace a failed one. Shortly after the token expires, QuestDB rejects these -new connections, as described in [Token validation](#token-validation). A -long-running service must therefore switch to a new token before the old one -expires: it connects with the new token, then closes the old handle. -`token.expires_on` is a Unix timestamp. - -The credential caches the token in memory, and requests a new one when the -cached token expires within 5 minutes, or earlier when the token's refresh -time has passed, so calling `get_token()` often is cheap. The following example -looks for a new token in the last 10 minutes of the old one. When it cannot get -a new token or connect with it, it keeps the old token and handle, and tries -again with the next batch: - -```python title="Keep ingesting across token expiry" -import time - -import questdb -from azure.core.exceptions import AzureError -from azure.identity import DefaultAzureCredential -from questdb import QuestDBError, TimestampNanos - -SCOPE = "api://8de84b90-1ea5-4e41-9e84-dba860aa01a6/.default" -ADDR = "questdb.example.com:9000" -# look for a new token when the old one expires within this many seconds -REFRESH_MARGIN_SECONDS = 600 - -credential = DefaultAzureCredential() - - -def on_rejection(error): - # rows that the server rejected, for example for a missing grant - print("rejected:", error.category.tag, error.message) - - -def connect(token): - conf = f"wss::addr={ADDR};token={token.token};" - return questdb.connect(conf, error_handler=on_rejection) - - -def renew(token, db): - # returns the token and the handle to use from now on - try: - new_token = credential.get_token(SCOPE) - # the credential returns the cached token until it renews it - if new_token.token == token.token: - return token, db - new_db = connect(new_token) - except (AzureError, QuestDBError) as e: - # keep the old token and handle, and try again with the next batch - print("token renewal failed:", e) - return token, db - db.close() - return new_token, new_db - - -def next_trades(): - # replace with your source of trades - time.sleep(1) - return [("EURUSD", "buy", 1.0842, 100000.0)] - - -token = credential.get_token(SCOPE) -db = connect(token) -try: - while True: - trades = next_trades() - if time.time() > token.expires_on - REFRESH_MARGIN_SECONDS: - token, db = renew(token, db) - with db.sender() as sender: - for symbol, side, price, quantity in trades: - sender.row( - "fx_trades", - symbols={"symbol": symbol, "side": side}, - columns={"price": price, "quantity": quantity}, - at=TimestampNanos.now(), - ) - sender.flush(wait=True) -finally: - db.close() -``` - -See [Authentication and TLS](/docs/connect/clients/python/#authentication-and-tls) -for the client options. For the [PGWire endpoint](#oidc-for-the-pgwire-endpoint), -enable `acl.oidc.pg.token.as.password.enabled`, and send the token as the -password of the `_sso` user. - -#### Verify the setup - -`current_user()` returns the principal that QuestDB sees for the service: the -object ID of its service principal, from the `oid` claim. For a managed -identity, that is the _Object (principal) ID_ that the Azure portal shows for -the identity, not its client ID. Run the query from the service while it is -connected: - -```python title="Check the principal of the service" -with db.query("SELECT current_user()") as result: - print(result.to_pandas()) -``` - -If the service cannot connect, the server log gives the reason, as described in -[Troubleshooting OIDC logins](#troubleshooting-oidc-logins). For a token that -carries neither a role nor a group, the log lists the claims that the token -carries. - -#### Change or revoke access - -QuestDB reads the roles from the token, so a change to the roles of a service -takes effect when the service presents a new token. Azure caches managed -identity tokens for around 24 hours, so a role assigned to, or removed from, a -managed identity can take hours to reach QuestDB. See the -[managed identity best practices](https://learn.microsoft.com/en-us/entra/identity/managed-identities-azure-resources/managed-identity-best-practice-recommendations). -To cut a service off without waiting for its token to expire, revoke the -permissions of its QuestDB group. This cuts off every service that holds the -app role, so to cut off one service at a time, give each service an app role, -and a QuestDB group, of its own. +### PingFederate -
+The [PingFederate guide](/docs/security/oidc-pingfederate/) sets up single +sign-on for the Web Console, with PingFederate authenticating users against +Microsoft Entra ID over LDAP. diff --git a/documentation/security/rbac.md b/documentation/security/rbac.md index 1daee46209..52a5a36820 100644 --- a/documentation/security/rbac.md +++ b/documentation/security/rbac.md @@ -248,7 +248,7 @@ GRANT SWITCH ROLE TO failover_bot; -- SWITCH ROLE, SWITCH STATUS, the endpoint ### Users and service accounts -QuestDB has two types of principals: +QuestDB has two types of local principals: - **Users**: For human individuals. Can belong to multiple groups and inherit permissions from them. Cannot be assumed by others. @@ -266,8 +266,9 @@ Names must be unique across all users, service accounts, and groups. QuestDB Enterprise also accepts external users: people and services that log in through an [OpenID Connect provider](/docs/security/oidc/), such as Azure services that run as a Microsoft Entra ID managed identity. QuestDB does not -store external users, and permissions cannot be granted to them directly. They -get the permissions of the QuestDB groups that their external groups map to. +persist external users, and permissions cannot be granted to them directly. +They get the permissions of the QuestDB groups that their external groups map +to. See [User permissions](/docs/security/oidc/#user-permissions). #### Why service accounts? @@ -331,7 +332,7 @@ dropped, all members lose the permissions they inherited from that group. width={745} /> -QuestDB supports three authentication methods: +QuestDB supports three local authentication methods: | Method | Use case | Endpoints | | ------------------ | ------------------------ | ------------------------- | @@ -344,7 +345,7 @@ tokens from an OpenID Connect provider, such as Microsoft Entra ID. They get their permissions from the QuestDB groups that their external groups map to. See [OpenID Connect (OIDC)](/docs/security/oidc/#user-permissions), and for Azure services, -[Microsoft Entra ID managed identities and service principals](/docs/security/oidc/#accept-managed-identity-and-service-principal-tokens). +[Microsoft Entra ID managed identities and service principals](/docs/security/oidc-entra-id/#managed-identities-and-service-principals). Users can have multiple authentication methods enabled simultaneously: diff --git a/documentation/sidebars.js b/documentation/sidebars.js index e824810980..d4b533d3ed 100644 --- a/documentation/sidebars.js +++ b/documentation/sidebars.js @@ -724,9 +724,30 @@ module.exports = { label: "Role-Based Access Control (RBAC)", }, { - id: "security/oidc", - type: "doc", + type: "category", label: "OpenID Connect (OIDC)", + collapsed: true, + link: { + type: "doc", + id: "security/oidc", + }, + items: [ + { + id: "security/oidc", + type: "doc", + label: "Overview", + }, + { + id: "security/oidc-entra-id", + type: "doc", + label: "Microsoft Entra ID", + }, + { + id: "security/oidc-pingfederate", + type: "doc", + label: "PingFederate", + }, + ], }, { type: "doc", From 88059ea5d828a37f8aaa21d609e3fea24a31d341 Mon Sep 17 00:00:00 2001 From: glasstiger Date: Fri, 9 Oct 2026 13:25:09 +0100 Subject: [PATCH 08/29] docs(oidc): correct rejection, session and ingestion behaviour, gate claim lists on 4.0.2 - Entra app-only guide: a missing INSERT grant or a type mismatch is a terminal rejection, so flush(wait=True) raises QuestDBServerRejectionError as well as calling the error handler. Say that the renewal example stops on it. python.md said that the wait never raises on a server rejection; correct it there too. - Mapping user permissions and Token validation: QuestDB keeps the groups of the last token that it accepted. A rejected token falls back to the Web Console's session cookie, the Web Console's logout ends the session, and open connections outlive their token. Revoking the QuestDB group's permissions is the way to cut off access at once, also for a service whose app role was removed. - External users: ingestion fails when it has to create a table or add a column, QuestDB rolls back the table but keeps the column, and Troubleshooting lists the errors that the client and the log show. - Claim lists: state the 4.0.2 requirement in Fallback claim lists, Non-interactive clients, the Entra Configure QuestDB step and RBAC, and scope the expired-token grace period to 4.0.2. - Non-interactive clients: acl.oidc.groups.encoded.in.token=true also makes the Web Console send the ID token, which must then carry the groups. - Add a minimal server.conf to the OIDC page and the QuestDB configuration to the PingFederate guide. - Logs can hold rejected tokens, token responses and claims: drop the "guaranteed" wording, warn in the token note, and add the Invalid token format error to Troubleshooting. - Entra guide: describe the Token configuration settings in the text, and say that app-only tokens can, not always, carry a groups claim. Co-Authored-By: Claude Opus 5.5 --- documentation/changelog.mdx | 10 +- documentation/connect/clients/python.md | 12 +- .../connect/compatibility/ilp/overview.md | 6 +- documentation/security/oidc-entra-id.md | 78 ++++++--- documentation/security/oidc-pingfederate.md | 30 ++++ documentation/security/oidc.mdx | 155 +++++++++++++----- documentation/security/rbac.md | 15 +- 7 files changed, 221 insertions(+), 85 deletions(-) diff --git a/documentation/changelog.mdx b/documentation/changelog.mdx index 56e2be415e..89649873b1 100644 --- a/documentation/changelog.mdx +++ b/documentation/changelog.mdx @@ -15,7 +15,7 @@ This page tracks significant updates to the QuestDB documentation. - [Microsoft Entra ID](/docs/security/oidc-entra-id/) - New page for Microsoft Entra ID: the Web Console sign-in guide, moved from the OIDC page, and app-only tokens for Azure managed identities and service principals in QuestDB Enterprise 4.0.2, with the app role mapping, how to keep app roles the only way in, and Python examples that ingest and renew the token - [User and group claims](/docs/security/oidc/#user-and-group-claims) - How QuestDB Enterprise 4.0.2 reads the principal and the groups from the user information: fallback claim lists, the rules for missing, `null`, and empty claims, how to choose the principal claim, and the startup checks - [Token validation](/docs/security/oidc/#token-validation), [Troubleshooting OIDC logins](/docs/security/oidc/#troubleshooting-oidc-logins), and [Versions before 4.0.2](/docs/security/oidc/#versions-before-402) - What QuestDB checks in a token and how long it accepts one after it expires, the log messages of rejected logins, how earlier versions differ, and what to check when you upgrade -- [Tables created by external users](/docs/security/oidc/#tables-created-by-external-users) - Ingestion cannot create a table or add a column for a user who logs in through OIDC, so create the tables in advance, or create them with SQL and `OWNED BY`, naming one of the user's QuestDB groups +- [Tables created by external users](/docs/security/oidc/#tables-created-by-external-users) - Ingestion fails when it has to create a table or add a column for a user who logs in through OIDC, so create the tables in advance, or create them with SQL and `OWNED BY`, naming one of the user's QuestDB groups ### Reference @@ -27,13 +27,13 @@ This page tracks significant updates to the QuestDB documentation. ### Updated -- [OpenID Connect (OIDC)](/docs/security/oidc/) - The identity provider guides moved to their own pages, under OpenID Connect (OIDC) in the sidebar, and the introduction lists what the page covers -- [PingFederate](/docs/security/oidc-pingfederate/) - Moved from the OIDC page to its own page -- [Mapping user permissions](/docs/security/oidc/#mapping-user-permissions) - Corrected what happens when the groups claim is missing or empty: QuestDB rejects the login. A user is authenticated without permissions only when none of their groups is mapped to a QuestDB group. Documented when a change in the identity provider takes effect, that the HTTP session of the Web Console is not tied to the token and does not expire while in use, and how to cut off access immediately +- [OpenID Connect (OIDC)](/docs/security/oidc/) - The identity provider guides moved to their own pages, under OpenID Connect (OIDC) in the sidebar, the introduction lists what the page covers, and [Configuration options](/docs/security/oidc/#configuration-options) shows a minimal configuration +- [PingFederate](/docs/security/oidc-pingfederate/) - Moved from the OIDC page to its own page, and added the [QuestDB configuration](/docs/security/oidc-pingfederate/#questdb-configuration) +- [Mapping user permissions](/docs/security/oidc/#mapping-user-permissions) - Corrected what happens when the groups claim is missing or empty: QuestDB rejects the login. A user is authenticated without permissions only when none of their groups is mapped to a QuestDB group. Documented when a change in the identity provider takes effect, that open connections and the HTTP session of the Web Console keep the groups of the last token that QuestDB accepted, and how to cut off access immediately - [Find user information](/docs/security/oidc/#8-find-user-information) and [non-interactive clients](/docs/security/oidc/#non-interactive-clients) - Corrected which token of a user reaches QuestDB with `acl.oidc.groups.encoded.in.token=true`: the ID token rather than the access token. Non-interactive clients also covers the tokens that services obtain for themselves - [Endpoint permissions](/docs/security/rbac/#endpoint-permissions) - The `HTTP` permission also covers QWP over WebSocket. The [authentication methods](/docs/security/rbac/#authentication) and [users and service accounts](/docs/security/rbac/#users-and-service-accounts) describe external users, who log in through OIDC - [Microsoft Entra ID guide](/docs/security/oidc-entra-id/#web-console-sign-in) - Uses `preferred_username` as the principal claim instead of the display name, explains when to use `oid` instead, and registers the application as single-tenant -- [Python client](/docs/connect/clients/python/#authentication-and-tls) - Corrected when bad credentials raise: by default, `connect()` opens its first connections and raises, and `lazy_connect=on` defers the error to the first operation +- [Python client](/docs/connect/clients/python/#authentication-and-tls) - Corrected when bad credentials raise: by default, `connect()` opens its first connections and raises, and `lazy_connect=on` defers the error to the first operation. Corrected that a terminal server rejection, such as a missing grant, raises `QuestDBServerRejectionError` from `flush(wait=True)` ## September 2026 diff --git a/documentation/connect/clients/python.md b/documentation/connect/clients/python.md index 16f265c7cb..ac0d9f0d75 100644 --- a/documentation/connect/clients/python.md +++ b/documentation/connect/clients/python.md @@ -326,11 +326,13 @@ delivery continues in the background. `flush(wait=True)` additionally blocks until the server acknowledges everything published on this lease, and `wait(timeout_millis)` is the standalone barrier with an explicit no-progress timeout (`0` means no deadline; `wait()` returns immediately when the lease -published nothing). The barrier raises only on a terminal connection -failure: server rejections are pushed to the pool's -[rejection handler](#server-rejections) — logged by default — rather than -raised from the wait. Closing the lease (leaving the `with` block) flushes -remaining rows without waiting. +published nothing). Every server rejection goes to the pool's +[rejection handler](#server-rejections), logged by default, and the +store-and-forward queue replays the retriable ones. The barrier raises on a +terminal failure: a terminal connection failure, or a terminal rejection, such +as a schema mismatch or a missing grant, which latches the connection and +raises `QuestDBServerRejectionError`. Closing the lease (leaving the `with` +block) flushes remaining rows without waiting. The pooled lease has no `transaction()`, no `new_buffer()` or `Buffer`, and no manual progress pump; those exist only on the standalone diff --git a/documentation/connect/compatibility/ilp/overview.md b/documentation/connect/compatibility/ilp/overview.md index 9be8189143..7d8ecd3025 100644 --- a/documentation/connect/compatibility/ilp/overview.md +++ b/documentation/connect/compatibility/ilp/overview.md @@ -441,9 +441,9 @@ GRANT add column, insert ON all tables TO ingest_user; -- grants permissions to GRANT add column, insert ON table1, table2 TO ingest_user; -- grants permissions to add columns and insert data to specific tables ``` -External users, who log in through [OpenID Connect](/docs/security/oidc/), -cannot create tables or add columns on ingestion. Create the tables in advance, -with every column that the client sends. See +Ingestion fails when it has to create a table or add a column for an external +user, who logs in through [OpenID Connect](/docs/security/oidc/). Create the +tables in advance, with every column that the client sends. See [Tables created by external users](/docs/security/oidc/#tables-created-by-external-users). Read more setup details in the diff --git a/documentation/security/oidc-entra-id.md b/documentation/security/oidc-entra-id.md index 0667339720..a0a194b6c8 100644 --- a/documentation/security/oidc-entra-id.md +++ b/documentation/security/oidc-entra-id.md @@ -130,10 +130,11 @@ application, in this case to QuestDB, and the client application uses these credentials to authenticate the user by forwarding them to the identity provider, in this case to Entra ID. -It is guaranteed that QuestDB does not store the user's credentials in any way. -They are not persisted into the database, not even in encrypted form. -The login details are treated as passthrough information. Only exception is -that server logs can contain the username, logged for audit purposes. +QuestDB does not persist the user's credentials, not even in encrypted form: +the login details are treated as passthrough information. The server logs can +contain the username, logged for audit purposes, and, when a login fails, the +claims of the user and the response of Entra ID, which can contain tokens. +Restrict access to the server logs. +Select _Add groups claim_, and set it up as follows: + +- Under _Select group types to include in Access, ID, and SAML tokens_, select + _Security groups_. +- Under _Customize token properties by type_, select _Group ID_ for the _ID_ + and the _Access_ token. +- Leave _Emit groups as role claims_ cleared for both. Otherwise, Entra ID + sends the groups in the `roles` claim instead of the `groups` claim that the + [QuestDB configuration](#questdb-configuration) below reads, and mixes the + groups of services with their app roles in + [Managed identities and service principals](#managed-identities-and-service-principals). + The customized tokens contain user information which cannot be accessed without permission. User information is provided by Microsoft Graph, so the client application needs specific permissions to access @@ -332,9 +345,10 @@ in the `oid` claim, and list the [app roles](https://learn.microsoft.com/en-us/entra/identity-platform/howto-add-app-roles-in-apps) assigned to it in the `roles` claim. They carry no `name` or `preferred_username` claim. If the service principal is a member of a security -group, its token also carries a `groups` claim, because the QuestDB application -emits group claims, as set up under _Token configuration_ in +group, its token can also carry a `groups` claim, depending on the group claims +that the QuestDB application emits, as set up under _Token configuration_ in [Set up the client application in Entra ID](#set-up-the-client-application-in-entra-id). +Do not rely on it: grant permissions to services through app roles. QuestDB treats the service as an external user, not as a QuestDB [service account](/docs/security/rbac/#users-and-service-accounts), so @@ -382,11 +396,12 @@ the permissions of its Entra ID security groups instead. See ### Configure QuestDB -Replace the two claim settings of the +On QuestDB Enterprise 4.0.2 or later, replace the two claim settings of the [QuestDB configuration](#questdb-configuration) with fallback lists, and -restart QuestDB. Keep `acl.oidc.groups.encoded.in.token=true`, so that QuestDB -validates app-only tokens against the signing keys of Entra ID, the same way as -ID tokens: +restart QuestDB. Earlier versions read each setting as one claim name, so the +lists below make every OIDC login fail. Keep +`acl.oidc.groups.encoded.in.token=true`, so that QuestDB validates app-only +tokens against the signing keys of Entra ID, the same way as ID tokens: ```ini title="server.conf" # user tokens carry preferred_username, app-only tokens carry oid @@ -411,7 +426,7 @@ needs a list. ### Map app roles to QuestDB groups -QuestDB cannot create a table or add a column on ingestion for the service, +Ingestion fails when it has to create a table or add a column for the service, because the service is an external user. See [Tables created by external users](/docs/security/oidc/#tables-created-by-external-users). As the admin, create the tables that the service writes to, with every column @@ -450,7 +465,8 @@ endpoints. Unless _Assignment required_ is enabled for the QuestDB application, any service principal in the tenant can get a token for QuestDB. A token without a role falls back to the `groups` claim, so a service principal in a mapped -security group gets the permissions of that group without an app role. To make +security group can get the permissions of that group without an app role, if +its token carries the group. To make an app role the only way for a service to get permissions, do one of the following: @@ -502,9 +518,12 @@ with questdb.connect(conf) as db: sender.flush(wait=True) ``` -`flush(wait=True)` returns when the server has accepted the rows. Rows that the -server rejects, for example because the group lacks `INSERT` on the table, go -to the error handler of the connection, which logs them by default. See +`flush(wait=True)` returns when the server has acknowledged the rows. The +client passes every rejection to the error handler of the handle, which logs it +by default. A rejection that resending cannot fix, such as a missing `INSERT` +grant on the table or a value that does not match the column type, is +terminal: `flush(wait=True)` also raises `QuestDBServerRejectionError`, and the +pool retires the connection. See [Server rejections](/docs/connect/clients/python/#server-rejections). The client presents the same token on every connection that it opens: the @@ -540,7 +559,8 @@ credential = DefaultAzureCredential() def on_rejection(error): - # rows that the server rejected, for example for a missing grant + # every rejection; a terminal one, such as a missing grant, also + # raises QuestDBServerRejectionError from flush(wait=True) print("rejected:", error.category.tag, error.message) @@ -591,6 +611,12 @@ finally: db.close() ``` +A terminal rejection raises `QuestDBServerRejectionError` from +`flush(wait=True)` and ends the loop, and the `finally` block closes the +handle. To keep the service running instead, catch the error around the batch, +and decide whether to retry or drop the batch: a batch fails the same way +until you fix the cause of the rejection, such as a missing grant. + See [Authentication and TLS](/docs/connect/clients/python/#authentication-and-tls) for the client options. For the [PGWire endpoint](/docs/security/oidc/#oidc-for-the-pgwire-endpoint), enable @@ -619,11 +645,17 @@ that the token carries. ### Change or revoke access QuestDB reads the roles from the token, so a change to the roles of a service -takes effect when the service presents a new token. Azure caches managed -identity tokens for around 24 hours, so a role assigned to, or removed from, a -managed identity can take hours to reach QuestDB. See the +takes effect when QuestDB accepts a new token of the service. Azure caches +managed identity tokens for around 24 hours, so a role assigned to, or removed +from, a managed identity can take hours to reach QuestDB. See the [managed identity best practices](https://learn.microsoft.com/en-us/entra/identity/managed-identities-azure-resources/managed-identity-best-practice-recommendations). -To cut a service off without waiting for its token to expire, revoke the -permissions of its QuestDB group. This cuts off every service that holds the + +Removing a role does not cut off the connections that the service has already +opened. They stay open when their token expires. If the next token of the +service carries neither a role nor a group, QuestDB rejects it, and the open +connections keep the groups of the last token that QuestDB accepted. To cut a +service off, revoke the permissions of its QuestDB group. This takes effect at +once, on open connections too, but it cuts off every service that holds the app role, so to cut off one service at a time, give each service an app role, -and a QuestDB group, of its own. +and a QuestDB group, of its own. See +[Mapping user permissions](/docs/security/oidc/#mapping-user-permissions). diff --git a/documentation/security/oidc-pingfederate.md b/documentation/security/oidc-pingfederate.md index 6fae93e736..8ab89ef96a 100644 --- a/documentation/security/oidc-pingfederate.md +++ b/documentation/security/oidc-pingfederate.md @@ -346,6 +346,36 @@ to the existing PCV. Then select the `username` attribute of the PCV as `USER_KEY`. +## QuestDB configuration + +Set the following in QuestDB's `server.conf`. QuestDB's default endpoint paths +match PingFederate, so set the host of PingFederate, not the endpoints: + +```ini title="server.conf" +# enable OIDC +acl.oidc.enabled=true + +# the host of PingFederate; also set acl.oidc.port if it is not 443 +acl.oidc.host=pingfederate.example.com + +# the client ID set up in Set up PingFederate client +acl.oidc.client.id=questdb + +# the claim set up in Add groups to OIDC policy management +acl.oidc.groups.claim=groups + +# optional, required only if ROPC is enabled in PingFederate +acl.oidc.ropc.flow.enabled=true +``` + +QuestDB reads the groups from the User Info response, so keep the default, +`acl.oidc.groups.encoded.in.token=false`. It reads the principal from the +`sub` claim by default. To read it from another claim of the User Info +response, set `acl.oidc.sub.claim`. See +[Choose the principal claim](/docs/security/oidc/#choose-the-principal-claim). +For the other settings, such as the TLS settings that a private certificate +authority needs, see [Configuration](/docs/configuration/oidc/). + ## Confirm QuestDB mappings and login QuestDB requires a mapping, as laid out in the diff --git a/documentation/security/oidc.mdx b/documentation/security/oidc.mdx index 9a953f50a9..d98558b613 100644 --- a/documentation/security/oidc.mdx +++ b/documentation/security/oidc.mdx @@ -19,9 +19,10 @@ QuestDB: they get the permissions of the QuestDB groups that their groups in the Identity Provider map to. This page covers the login flow, which token a client sends, how QuestDB maps -the claims of a token to a principal and to QuestDB groups, including -[fallback claim lists](#fallback-claim-lists) that accept user and app-only -tokens with one configuration, [token validation](#token-validation), and +the claims of a token to a principal and to QuestDB groups, including the +[fallback claim lists](#fallback-claim-lists) of QuestDB Enterprise 4.0.2 that +accept user and app-only tokens with one configuration, +[token validation](#token-validation), and [troubleshooting rejected logins](#troubleshooting-oidc-logins). For step-by-step setup, see the guides for [Microsoft Entra ID](/docs/security/oidc-entra-id/), which also covers Azure @@ -244,7 +245,9 @@ instead, as described in step 8. Treat the token as a credential, and serve QuestDB over [TLS](/docs/security/tls/). An access token can be opaque, but an ID token is a signed JWT that anyone who holds it can decode, including the user's name and -groups. +groups. Restrict access to the server logs too: when QuestDB rejects a login, +it can log the token, the response of the provider, or the claims of the user. +See [Troubleshooting OIDC logins](#troubleshooting-oidc-logins). ::: @@ -561,7 +564,12 @@ and on who the client authenticates as: provider's User Info endpoint, so send the access token. Some providers refuse tokens issued to applications at that endpoint, so to accept tokens that services obtain for themselves, use - `acl.oidc.groups.encoded.in.token=true`. + `acl.oidc.groups.encoded.in.token=true`. The setting applies to every + client: the Web Console then sends the ID token, and QuestDB reads the + principal and the groups from it. A provider that returns the groups only + from its User Info endpoint, as set up in the + [PingFederate guide](/docs/security/oidc-pingfederate/), must also put them + into the ID token, or QuestDB rejects every Web Console login. - With `acl.oidc.groups.encoded.in.token=true`, QuestDB validates the token itself, so send a JWT that passes [Token validation](#token-validation): - For a user, send the ID token, because the provider often issues the @@ -577,7 +585,8 @@ and on who the client authenticates as: Tokens that the provider issues to a service often carry the principal and the groups under different claim names than the tokens of users. To accept both -kinds of token with one configuration, list the claims of both. See +kinds of token with one configuration, list the claims of both, which requires +QuestDB Enterprise 4.0.2 or later. See [Fallback claim lists](#fallback-claim-lists). The following example requests a token for a user with the Resource Owner @@ -644,7 +653,7 @@ with Sender.from_conf(conf) as sender: sender.dataframe(df, table_name="foo", at="ts") ``` -QuestDB cannot create a table or add a column on ingestion for an external +Ingestion fails when it has to create a table or add a column for an external user, so the examples above need the table `foo` to exist, with the columns of `data.csv`. See [Tables created by external users](#tables-created-by-external-users). @@ -737,8 +746,9 @@ QuestDB works the list of external groups out from the [user information](#user-and-group-claims). If we take the example used earlier, we will see that the message contains a -claim called `groups`. This name is configurable in QuestDB, and QuestDB can -fall back to other claims when a token does not carry it. See +claim called `groups`. This name is configurable in QuestDB, and since QuestDB +Enterprise 4.0.2, QuestDB can fall back to other claims when the user +information does not carry it, or when it holds no group name. See [User and group claims](#user-and-group-claims). If the groups claim is missing or holds no group name, QuestDB rejects the @@ -761,8 +771,10 @@ login via the [Web Console](/docs/getting-started/web-console/overview/). } ``` -QuestDB picks up a change to the user's groups or status in the Identity -Provider, such as a disabled user, as follows: +QuestDB keeps the groups of the last token that it accepted for each external +user, and every session and connection of the user runs with them. QuestDB +picks up a change to the user's groups or status in the Identity Provider, +such as a disabled user, when it checks a token of the user again: - With the User Info endpoint, the default, when the cached user information expires. [`acl.oidc.cache.ttl`](/docs/configuration/oidc/#acloidccachettl), @@ -770,19 +782,29 @@ Provider, such as a disabled user, as follows: to your organization's policies. - With `acl.oidc.groups.encoded.in.token=true`, when the client presents a new token. QuestDB reads the groups from the token instead of asking the Identity - Provider, so refreshing the cache does not pick up changes. Until then, - QuestDB accepts the old token for up to 60 seconds plus `acl.oidc.cache.ttl` - after it expires. See [Token validation](#token-validation). - -In both cases, connections that are already open, such as PGWire connections, -stay open. The HTTP session that the Web Console opens at login is not tied to -the token either. QuestDB accepts requests in the session, with the groups of -the user's latest login, until the user logs out or the session has been idle -for `http.session.timeout`, 30 minutes by default. Every request restarts the -idle timer, so a session in use does not expire. The Web Console logs the user -out when it cannot refresh the token, for example after the user is disabled, -but another client that reuses the session cookie keeps the access for as long -as it stays active. + Provider, so refreshing the cache does not pick up changes. In QuestDB + Enterprise 4.0.2 and later, QuestDB accepts the old token for up to 60 + seconds plus `acl.oidc.cache.ttl` after it expires. Earlier versions do not + check whether the token has expired. See [Token validation](#token-validation). + +A token that QuestDB rejects does not change the groups, so some access +outlives the token: + +- Connections that are already open, such as PGWire and QWP connections, stay + open when their token expires, and keep running with the groups of the last + token that QuestDB accepted for the user. +- The HTTP session that the Web Console opens at login is not tied to the + token. When QuestDB rejects the token of a request that carries a valid + session cookie, for example because the user is disabled or because the + token no longer carries a group, it accepts the request through the session. + The session ends when the Web Console logs the user out, or after it has + been idle for `http.session.timeout`, 30 minutes by default. Every request + restarts the idle timer, so a session in use does not expire. The Web + Console logs the user out, and ends the session, when it cannot refresh the + token, for example after the user is disabled. A session that the Web + Console does not end, for example because the browser was closed, stays + valid for any client that reuses its cookie, for as long as that client + stays active. To cut off access immediately, revoke the permissions of the QuestDB groups that the user's groups map to. This takes effect at once for every session and @@ -793,9 +815,10 @@ connection, including those of other users who map to the same groups. When a user creates a table or adds a column, QuestDB grants that user all permissions on it. Permissions cannot be granted to external users, so: -- Ingestion, over ILP/HTTP or QWP, cannot create a table or add a column for an - external user. Create the tables in advance, with every column that the - client sends. +- Ingestion, over ILP/HTTP or QWP, fails when it has to create a table or add a + column for an external user. QuestDB rolls back a new table, but a new column + stays in the table even though the request fails. Create the tables in + advance, with every column that the client sends. - An external user that creates a table or adds a column with SQL must name the owner with `OWNED BY`: one of the QuestDB groups that its groups map to, as set up in [Mapping user permissions](#mapping-user-permissions), not the @@ -811,10 +834,11 @@ permissions on it. Permissions cannot be granted to external users, so: ## User and group claims -:::warning Requires QuestDB Enterprise 4.0.2 +:::note -This section describes QuestDB Enterprise 4.0.2 and later. For earlier -versions, see [Versions before 4.0.2](#versions-before-402). +Fallback claim lists, and the rules and startup checks that come with them, +require QuestDB Enterprise 4.0.2 or later. For how earlier versions read the +claims, see [Versions before 4.0.2](#versions-before-402). ::: @@ -864,11 +888,14 @@ new principal. Permissions stay the same, because they come from the groups. ### Fallback claim lists -Both settings accept a comma-separated list of claim names in priority order, -so one configuration can accept tokens that carry the principal and the groups -under different claim names. Tokens issued to users and tokens issued to -applications often differ this way. For example, Microsoft Entra ID user -tokens, configured as in the +In QuestDB Enterprise 4.0.2 and later, `acl.oidc.sub.claim` and +`acl.oidc.groups.claim` accept a comma-separated list of claim names in +priority order, so one configuration can accept tokens that carry the principal +and the groups under different claim names. Earlier versions read the whole +value as one claim name, so a list makes every OIDC login fail. + +Tokens issued to users and tokens issued to applications often differ this +way. For example, Microsoft Entra ID user tokens, configured as in the [Entra ID guide](/docs/security/oidc-entra-id/#web-console-sign-in), carry `preferred_username` and `groups`, while its [app-only tokens](/docs/security/oidc-entra-id/#managed-identities-and-service-principals) @@ -998,9 +1025,11 @@ and caches the result for up to [`acl.oidc.cache.ttl`](/docs/configuration/oidc/#acloidccachettl), 30 seconds by default. Because of the clock skew and the cache, QuestDB can accept a token, on new connections too, for up to 60 seconds plus `acl.oidc.cache.ttl` -after it expires. After that, it rejects every request and new connection that -presents the token. A PGWire or WebSocket connection that is already open stays -open when its token expires. +after it expires. After that, it rejects the token on every request and new +connection. An HTTP request that also carries a valid session cookie, such as +a request of the Web Console, is still accepted through the session, and a +PGWire or WebSocket connection that is already open stays open when its token +expires. See [Mapping user permissions](#mapping-user-permissions). With `acl.oidc.groups.encoded.in.token=false`, QuestDB sends the token to the provider's User Info endpoint, which validates it. See @@ -1061,6 +1090,17 @@ and when it meets a key ID that it does not know, so either another provider issued the token, or QuestDB cannot reach the provider's keys, which it logs as `Unable to download public keys from OIDC provider`. +With `acl.oidc.groups.encoded.in.token=true`, QuestDB accepts only a JWT. It +rejects a token that is not one, such as an opaque access token, and logs the +whole token: + +```text +Request failed [errorMsg=Invalid token format, cannot find header separator, expected format of JWT:
.. [token=...]] +``` + +Send a JWT instead, as described in +[Non-interactive clients](#non-interactive-clients). + With `acl.oidc.groups.encoded.in.token=false`, the provider's User Info endpoint rejects a token that is invalid or expired, and QuestDB rejects the login. @@ -1069,9 +1109,12 @@ If the login succeeds but every request is denied, check that at least one of the user's groups is mapped to a QuestDB group that has the endpoint permission. See [Mapping user permissions](#mapping-user-permissions). -If the login succeeds but ingestion fails because a table or a column does not -exist, create it in advance. QuestDB cannot create a table or add a column on -ingestion for an external user. See +If the login succeeds but ingestion fails when it creates a table or adds a +column, create the table and its columns in advance: ingestion cannot create a +table or add a column for an external user. The error contains +`Cannot grant permissions to an external user`. When ingestion adds a column, +the client can receive `WAL writer is distressed and cannot be used any more` +instead, and the column stays in the table. See [Tables created by external users](#tables-created-by-external-users). ## Versions before 4.0.2 @@ -1110,8 +1153,36 @@ When you upgrade from an earlier version: ## Configuration options +A minimal configuration enables OIDC, points QuestDB at the provider, and +names the client ID and the groups claim: + +```ini title="server.conf" +# enable OIDC +acl.oidc.enabled=true + +# the OIDC discovery document of the provider; alternatively, set +# acl.oidc.host and the endpoint settings +acl.oidc.configuration.url=https://idp.example.com/.well-known/openid-configuration + +# the client ID that the provider assigned to QuestDB +acl.oidc.client.id=questdb + +# required, no default: the claim that carries the groups +acl.oidc.groups.claim=groups + +# optional, sub by default: the claim that carries the principal +# acl.oidc.sub.claim=sub + +# optional, false by default: read the claims from a JWT, such as the ID token, +# instead of calling the User Info endpoint +# acl.oidc.groups.encoded.in.token=true +``` + For all OIDC-related configuration options of QuestDB, see -[Configuration](/docs/configuration/oidc/). +[Configuration](/docs/configuration/oidc/). For complete examples, see the +QuestDB configuration in the +[Microsoft Entra ID](/docs/security/oidc-entra-id/#questdb-configuration) and +[PingFederate](/docs/security/oidc-pingfederate/#questdb-configuration) guides.
diff --git a/documentation/security/rbac.md b/documentation/security/rbac.md index 52a5a36820..f085f6ce73 100644 --- a/documentation/security/rbac.md +++ b/documentation/security/rbac.md @@ -264,12 +264,13 @@ CREATE SERVICE ACCOUNT app_account WITH PASSWORD 'pwd'; Names must be unique across all users, service accounts, and groups. QuestDB Enterprise also accepts external users: people and services that log -in through an [OpenID Connect provider](/docs/security/oidc/), such as Azure -services that run as a Microsoft Entra ID managed identity. QuestDB does not -persist external users, and permissions cannot be granted to them directly. -They get the permissions of the QuestDB groups that their external groups map -to. -See [User permissions](/docs/security/oidc/#user-permissions). +in through an [OpenID Connect provider](/docs/security/oidc/), such as users +who sign in to the Web Console with Microsoft Entra ID, and, since QuestDB +Enterprise 4.0.2, Azure services that run as a Microsoft Entra ID managed +identity. QuestDB does not persist external users, and permissions cannot be +granted to them directly. They get the permissions of the QuestDB groups that +their external groups map to. See +[User permissions](/docs/security/oidc/#user-permissions). #### Why service accounts? @@ -344,7 +345,7 @@ QuestDB Enterprise can also authenticate external users and services with tokens from an OpenID Connect provider, such as Microsoft Entra ID. They get their permissions from the QuestDB groups that their external groups map to. See [OpenID Connect (OIDC)](/docs/security/oidc/#user-permissions), and for -Azure services, +Azure services on QuestDB Enterprise 4.0.2 or later, [Microsoft Entra ID managed identities and service principals](/docs/security/oidc-entra-id/#managed-identities-and-service-principals). Users can have multiple authentication methods enabled simultaneously: From 7f975831c2dd4321649eb27b5ea5bca55b557adc Mon Sep 17 00:00:00 2001 From: glasstiger Date: Fri, 9 Oct 2026 14:25:48 +0100 Subject: [PATCH 09/29] docs(oidc): fix review findings on principals, token audience and token lifetime - ROPC logins: the principal is the username that the client sends, not the claim that acl.oidc.sub.claim names, and group changes take effect after acl.oidc.cache.ttl. Say so in User and group claims, Choose the principal claim, the sub.claim and cache.ttl reference entries, and both provider guides. - Token validation: with acl.oidc.groups.encoded.in.token=false, QuestDB checks neither the audience nor the client of a token, and acl.oidc.audience does not apply. Note it in the audience reference too. - Non-interactive clients: add a client credentials example over QWP, move the password grant examples from ILP/HTTP and questdb.ingress to QWP and fx_trades, use the client ID questdb in every example, and add Token lifetime, the provider-neutral rules for expiring tokens that the client pages link to. - Mapping user permissions: list the endpoint permission of each interface with a GRANT example, and state that external aliases match exactly. Move group changes, sessions and revocation to their own section, and put the minimal configuration right after the intro. - python.md: connect() is eager by default everywhere on the page, and a terminal rejection raises from flush(wait=True). Move the barrier rules out of Value types into Flushing and acknowledgements. - C/C++, .NET and Go clients: build the new client before the token expires, then close the old one. - Entra guide: one app role and one QuestDB group per service, with QuestDB.FxIngest as the example, and REVOKE statements to cut a service off. Co-Authored-By: Claude Opus 5.5 --- documentation/changelog.mdx | 19 +- documentation/configuration/oidc.md | 13 +- documentation/connect/clients/c-and-cpp.md | 2 +- documentation/connect/clients/dotnet.md | 2 +- documentation/connect/clients/go.md | 5 +- documentation/connect/clients/python.md | 122 +++---- documentation/security/oidc-entra-id.md | 35 +- documentation/security/oidc-pingfederate.md | 4 +- documentation/security/oidc.mdx | 347 ++++++++++++++------ 9 files changed, 368 insertions(+), 181 deletions(-) diff --git a/documentation/changelog.mdx b/documentation/changelog.mdx index 89649873b1..dc1a23e21a 100644 --- a/documentation/changelog.mdx +++ b/documentation/changelog.mdx @@ -12,28 +12,31 @@ This page tracks significant updates to the QuestDB documentation. ### New -- [Microsoft Entra ID](/docs/security/oidc-entra-id/) - New page for Microsoft Entra ID: the Web Console sign-in guide, moved from the OIDC page, and app-only tokens for Azure managed identities and service principals in QuestDB Enterprise 4.0.2, with the app role mapping, how to keep app roles the only way in, and Python examples that ingest and renew the token +- [Microsoft Entra ID](/docs/security/oidc-entra-id/) - New page for Microsoft Entra ID: the Web Console sign-in guide, moved from the OIDC page, and app-only tokens for Azure managed identities and service principals in QuestDB Enterprise 4.0.2, with an app role and a QuestDB group for each service, how to keep app roles the only way in, how to cut off one service, and Python examples that ingest and renew the token - [User and group claims](/docs/security/oidc/#user-and-group-claims) - How QuestDB Enterprise 4.0.2 reads the principal and the groups from the user information: fallback claim lists, the rules for missing, `null`, and empty claims, how to choose the principal claim, and the startup checks -- [Token validation](/docs/security/oidc/#token-validation), [Troubleshooting OIDC logins](/docs/security/oidc/#troubleshooting-oidc-logins), and [Versions before 4.0.2](/docs/security/oidc/#versions-before-402) - What QuestDB checks in a token and how long it accepts one after it expires, the log messages of rejected logins, how earlier versions differ, and what to check when you upgrade +- [Token validation](/docs/security/oidc/#token-validation), [Troubleshooting OIDC logins](/docs/security/oidc/#troubleshooting-oidc-logins), and [Versions before 4.0.2](/docs/security/oidc/#versions-before-402) - What QuestDB checks in a token, including that it does not check the audience of a token that it sends to the User Info endpoint, how long it accepts a token after it expires, the log messages of rejected logins, how earlier versions differ, and what to check when you upgrade - [Tables created by external users](/docs/security/oidc/#tables-created-by-external-users) - Ingestion fails when it has to create a table or add a column for a user who logs in through OIDC, so create the tables in advance, or create them with SQL and `OWNED BY`, naming one of the user's QuestDB groups +- [Group changes, sessions, and revoking access](/docs/security/oidc/#group-changes-sessions-and-revoking-access) - When a change in the identity provider takes effect, that open connections and the HTTP session of the Web Console keep the groups of the last token that QuestDB accepted, and how to cut off access immediately +- [Token lifetime](/docs/security/oidc/#token-lifetime) - How QuestDB treats the token of a client as it expires, and how to switch a long-running client to a new token before then ### Reference -- [`acl.oidc.groups.claim`](/docs/configuration/oidc/#acloidcgroupsclaim) and [`acl.oidc.sub.claim`](/docs/configuration/oidc/#acloidcsubclaim) - Documented the comma-separated fallback claim lists of QuestDB Enterprise 4.0.2 and the startup validation. `acl.oidc.groups.claim` is now listed with no default, as it must be set when OIDC is enabled +- [`acl.oidc.groups.claim`](/docs/configuration/oidc/#acloidcgroupsclaim) and [`acl.oidc.sub.claim`](/docs/configuration/oidc/#acloidcsubclaim) - Documented the comma-separated fallback claim lists of QuestDB Enterprise 4.0.2 and the startup validation. `acl.oidc.groups.claim` is now listed with no default, as it must be set when OIDC is enabled. Username and password (ROPC) logins take the principal from the username that the client sends - [`acl.oidc.pkce.required`](/docs/configuration/oidc/#acloidcpkcerequired) - Corrected the property name, previously documented as `acl.oidc.pkce.enabled`, which QuestDB does not recognize - [`acl.oidc.groups.encoded.in.token`](/docs/configuration/oidc/#acloidcgroupsencodedintoken) - Documented that QuestDB Enterprise 4.0.2 rejects expired tokens and tokens without an `exp` claim, which earlier versions accept, and which token QuestDB reads the principal and the groups from -- [`acl.oidc.audience`](/docs/configuration/oidc/#acloidcaudience) and [`acl.oidc.cache.ttl`](/docs/configuration/oidc/#acloidccachettl) - Documented that `acl.oidc.audience` takes a single value, which the `aud` claim of a token must be or contain, and what the cache holds when the groups are encoded in the token +- [`acl.oidc.audience`](/docs/configuration/oidc/#acloidcaudience) and [`acl.oidc.cache.ttl`](/docs/configuration/oidc/#acloidccachettl) - Documented that `acl.oidc.audience` takes a single value, which the `aud` claim of a token must be or contain, and that QuestDB does not use it with the User Info endpoint, and what the cache holds when the groups are encoded in the token or a user logs in with a username and password - [`acl.oidc.public.keys.expiry`](/docs/configuration/oidc/#acloidcpublickeysexpiry) and [`acl.oidc.state.required`](/docs/configuration/oidc/#acloidcstaterequired) - Documented how long QuestDB uses the signing keys of the OIDC provider before it downloads them again, and the `state` check of the Web Console login ### Updated -- [OpenID Connect (OIDC)](/docs/security/oidc/) - The identity provider guides moved to their own pages, under OpenID Connect (OIDC) in the sidebar, the introduction lists what the page covers, and [Configuration options](/docs/security/oidc/#configuration-options) shows a minimal configuration +- [OpenID Connect (OIDC)](/docs/security/oidc/) - The identity provider guides moved to their own pages, under OpenID Connect (OIDC) in the sidebar, the introduction lists what the page covers, and [Configuration options](/docs/security/oidc/#configuration-options), right after the introduction, shows a minimal configuration - [PingFederate](/docs/security/oidc-pingfederate/) - Moved from the OIDC page to its own page, and added the [QuestDB configuration](/docs/security/oidc-pingfederate/#questdb-configuration) -- [Mapping user permissions](/docs/security/oidc/#mapping-user-permissions) - Corrected what happens when the groups claim is missing or empty: QuestDB rejects the login. A user is authenticated without permissions only when none of their groups is mapped to a QuestDB group. Documented when a change in the identity provider takes effect, that open connections and the HTTP session of the Web Console keep the groups of the last token that QuestDB accepted, and how to cut off access immediately -- [Find user information](/docs/security/oidc/#8-find-user-information) and [non-interactive clients](/docs/security/oidc/#non-interactive-clients) - Corrected which token of a user reaches QuestDB with `acl.oidc.groups.encoded.in.token=true`: the ID token rather than the access token. Non-interactive clients also covers the tokens that services obtain for themselves +- [Mapping user permissions](/docs/security/oidc/#mapping-user-permissions) - Corrected what happens when the groups claim is missing or empty: QuestDB rejects the login. A user is authenticated without permissions only when none of their groups is mapped to a QuestDB group. Added the endpoint permission that each interface needs, and how external aliases match +- [Find user information](/docs/security/oidc/#8-find-user-information) and [non-interactive clients](/docs/security/oidc/#non-interactive-clients) - Corrected which token of a user reaches QuestDB with `acl.oidc.groups.encoded.in.token=true`: the ID token rather than the access token. Non-interactive clients also covers the tokens that services obtain for themselves, with client credentials and password grant examples over QWP - [Endpoint permissions](/docs/security/rbac/#endpoint-permissions) - The `HTTP` permission also covers QWP over WebSocket. The [authentication methods](/docs/security/rbac/#authentication) and [users and service accounts](/docs/security/rbac/#users-and-service-accounts) describe external users, who log in through OIDC - [Microsoft Entra ID guide](/docs/security/oidc-entra-id/#web-console-sign-in) - Uses `preferred_username` as the principal claim instead of the display name, explains when to use `oid` instead, and registers the application as single-tenant -- [Python client](/docs/connect/clients/python/#authentication-and-tls) - Corrected when bad credentials raise: by default, `connect()` opens its first connections and raises, and `lazy_connect=on` defers the error to the first operation. Corrected that a terminal server rejection, such as a missing grant, raises `QuestDBServerRejectionError` from `flush(wait=True)` +- [Python client](/docs/connect/clients/python/#authentication-and-tls) - Corrected when bad credentials raise: by default, `connect()` opens its first connections and raises, and `lazy_connect=on` defers the error to the first operation. Corrected that a terminal server rejection, such as a missing grant, raises `QuestDBServerRejectionError` from `flush(wait=True)`, as described in [Flushing and acknowledgements](/docs/connect/clients/python/#flushing-and-acknowledgements) +- [C/C++](/docs/connect/clients/c-and-cpp/), [.NET](/docs/connect/clients/dotnet/), and [Go](/docs/connect/clients/go/) clients - Corrected token rotation: build a new client with a new token before the old token expires, then close the old one ## September 2026 diff --git a/documentation/configuration/oidc.md b/documentation/configuration/oidc.md index 458bd756a9..0f689a158b 100644 --- a/documentation/configuration/oidc.md +++ b/documentation/configuration/oidc.md @@ -32,6 +32,10 @@ single value. Keep the default: the ID tokens that the Web Console sends carry the client ID, so another value makes Web Console logins fail. See [Token validation](/docs/security/oidc/#token-validation). +With `acl.oidc.groups.encoded.in.token=false`, QuestDB does not use this +setting: it accepts any access token that the provider's User Info endpoint +accepts, whatever its audience. + ### acl.oidc.client.id - **Default**: none @@ -252,7 +256,10 @@ milliseconds: the User Info endpoint's response or, when token. This setting controls how often a token is validated again and the user information refreshed. With `acl.oidc.groups.encoded.in.token=true`, the user information comes from the token itself, so a change to it takes effect when -the client presents a new token. +the client presents a new token. For username and password logins through the +ROPC flow, QuestDB caches the result of the login for this long, then requests +a new token from the provider, so with either setting, changes take effect +after this time. ### acl.oidc.groups.claim @@ -309,6 +316,10 @@ every login, so users who share a principal share permissions. Avoid display names, such as the `name` claim. See [Choose the principal claim](/docs/security/oidc/#choose-the-principal-claim). +For username and password logins through the ROPC flow, the principal is the +username that the client sends instead. QuestDB still requires one of the +listed claims to carry a value, but does not use that value. + Since QuestDB Enterprise 4.0.2, accepts a comma-separated list of claims in priority order, in the form `claimName[,claimName ...]`, such as `preferred_username,oid`. Spaces around claim names are trimmed, and empty diff --git a/documentation/connect/clients/c-and-cpp.md b/documentation/connect/clients/c-and-cpp.md index c453c303a2..8eaa21d6f0 100644 --- a/documentation/connect/clients/c-and-cpp.md +++ b/documentation/connect/clients/c-and-cpp.md @@ -334,7 +334,7 @@ following are **not** supported: |---|---|---| | OIDC token acquisition or in-band refresh | Not supported. The client does not negotiate with an identity provider and has no callback to refresh a token mid-session. | QuestDB itself supports OIDC; see [OpenID Connect](/docs/security/oidc/#non-interactive-clients) for which token to send. Acquire the token out-of-band from your IdP, pass it via `token=...`, and rebuild the pool when the token nears expiry. For Azure services, see [Microsoft Entra ID managed identities and service principals](/docs/security/oidc-entra-id/#managed-identities-and-service-principals). | | Mutual TLS (client certificates) | Not supported. The QuestDB server does not negotiate client certificates regardless of client. | Use bearer-token auth over `wss`. See the connect-string reference's [TLS section](/docs/connect/clients/connect-string/#tls). | -| Token rotation mid-session | Not supported. Credentials are presented once during the WebSocket upgrade and are not re-sent. | On token expiry, close the pool and build a fresh one with the new token. | +| Token rotation mid-session | Not supported. Credentials are presented once during the WebSocket upgrade and are not re-sent. | Before the token expires, build a new pool with a new token, then close the old one. QuestDB checks the token again on every new connection, including reconnects, and rejects it once it has expired; see [Token lifetime](/docs/security/oidc/#token-lifetime). | ## Headers diff --git a/documentation/connect/clients/dotnet.md b/documentation/connect/clients/dotnet.md index f1d7eb2ba1..4d5653408b 100644 --- a/documentation/connect/clients/dotnet.md +++ b/documentation/connect/clients/dotnet.md @@ -230,7 +230,7 @@ that purpose; left unset, it inherits `auth_timeout_ms`. |---|---|---| | OIDC token acquisition or in-band refresh | Not supported by this client. It does not negotiate with an identity provider and has no callback to refresh a token mid-session. | QuestDB itself supports OIDC; see [OpenID Connect](/docs/security/oidc/#non-interactive-clients) for which token to send. Acquire the token out-of-band from your IdP, pass it via `token=...` above, and rebuild the sender / query client when the token nears expiry. For Azure services, see [Microsoft Entra ID managed identities and service principals](/docs/security/oidc-entra-id/#managed-identities-and-service-principals). | | Mutual TLS (client certificates) | Not supported. The QuestDB server does not negotiate client certificates regardless of client. | Use bearer-token auth over `wss://`. See the connect-string reference for the canonical statement. | -| Token rotation mid-session | Not supported. Credentials are presented once during the WebSocket upgrade and are not re-sent. | On token expiry, `await sender.DisposeAsync()` and build a fresh sender with the new token. The same applies to `QueryClient`. | +| Token rotation mid-session | Not supported. Credentials are presented once during the WebSocket upgrade and are not re-sent. | Before the token expires, build a new sender with a new token, then `await` the old sender's `DisposeAsync()`. The same applies to `QueryClient`. QuestDB checks the token again on every new connection, including reconnects, and rejects it once it has expired; see [Token lifetime](/docs/security/oidc/#token-lifetime). | ### Production example (TLS + token + multi-host) diff --git a/documentation/connect/clients/go.md b/documentation/connect/clients/go.md index 94ab8d2483..7cf99637c9 100644 --- a/documentation/connect/clients/go.md +++ b/documentation/connect/clients/go.md @@ -364,7 +364,10 @@ Acquire it out of band, for example from the identity provider of the QuestDB En [OpenID Connect flow](/docs/security/oidc/#non-interactive-clients), and manage its lifetime yourself. For Azure services, see [Microsoft Entra ID managed identities and service principals](/docs/security/oidc-entra-id/#managed-identities-and-service-principals). -When the token expires or is rotated, construct a new handle with the new token. +Before the token expires, construct a new handle with a new token, then close +the old one. QuestDB checks the token again on every new connection, including +reconnects, and rejects it once it has expired; see +[Token lifetime](/docs/security/oidc/#token-lifetime). An expired or rejected token surfaces as an authentication failure (see [Connection-level errors](#connection-level-errors)). It is mutually exclusive with `username`/`password`. diff --git a/documentation/connect/clients/python.md b/documentation/connect/clients/python.md index ac0d9f0d75..742d1ed858 100644 --- a/documentation/connect/clients/python.md +++ b/documentation/connect/clients/python.md @@ -91,10 +91,14 @@ db = questdb.connect("ws::addr=localhost:9000;") `connect()` accepts only `ws`/`wss` configuration strings; ILP schemes such as `http::` or `tcp::` belong to the [legacy Sender](#legacy-ilp-clients). -It parses and validates the string without opening a connection: the first -`sender()`, `dataframe()`, or `query()` call opens one, and later calls -reuse pooled connections. Servers without QWP support fail during the -WebSocket upgrade; see +It parses and validates the string, then opens the warm minimum of +connections before it returns (see [The pool](#the-pool)), so an unreachable +server or rejected credentials raise from `connect()`. Later calls reuse +pooled connections. With +[`lazy_connect=on`](/docs/connect/clients/connect-string/#pool-keys), +`connect()` returns without connecting, and the first `sender()`, +`dataframe()`, or `query()` call opens a connection. Servers without QWP +support fail during the WebSocket upgrade; see [protocol versioning](/docs/connect/wire-protocols/overview/#versioning). Instead of a configuration string, `connect()` also takes the equivalent @@ -147,8 +151,8 @@ are exchanged. By default, `connect()` opens its first connections before it returns (see [The pool](#the-pool)), so bad credentials raise `QuestDBErrorCode.AuthError` from `connect()` itself. With [`lazy_connect=on`](/docs/connect/clients/connect-string/#pool-keys), -`connect()` opens no connection, and bad credentials raise from the first -operation that needs the connection instead. Queries and `dataframe()` calls +`connect()` returns without connecting, and bad credentials raise from the +first operation that needs the connection instead. Queries and `dataframe()` calls raise it directly. Row senders connect in the background, so a fire-and-forget `flush()` can return before the upgrade fails. The error then surfaces from `flush(wait=True)`, `wait()`, or the next call on the lease, and @@ -177,7 +181,7 @@ following are **not** supported: | --- | --- | --- | | OIDC token acquisition or in-band refresh | Not supported. The client does not negotiate with an identity provider and cannot refresh a token mid-session. | QuestDB itself supports OIDC; see [OpenID Connect](/docs/security/oidc/#non-interactive-clients) for which token to send. Acquire the token out-of-band from your IdP, pass it via `token=...`, and rebuild the handle when the token nears expiry. For an example with an Entra ID managed identity, see [Microsoft Entra ID managed identities and service principals](/docs/security/oidc-entra-id/#managed-identities-and-service-principals). | | Mutual TLS (client certificates) | Not supported. The QuestDB server does not negotiate client certificates regardless of client. | Use bearer-token auth over `wss`. | -| Token rotation mid-session | Not supported. The handle keeps the credentials it was built with and presents them on every connection it opens — including reconnects and failover, so an expired token also breaks mid-session reconnection. | Before the token expires, build a fresh handle with a new token, then close the old one. See [Request a token in the service](/docs/security/oidc-entra-id/#request-a-token-in-the-service) for an example. | +| Token rotation mid-session | Not supported. The handle keeps the credentials it was built with and presents them on every connection it opens, including reconnects and failover, so an expired token also breaks mid-session reconnection. | Before the token expires, build a fresh handle with a new token, then close the old one. See [Token lifetime](/docs/security/oidc/#token-lifetime) for how QuestDB treats an expiring token, and [Request a token in the service](/docs/security/oidc-entra-id/#request-a-token-in-the-service) for an example. | ## The pool @@ -192,7 +196,7 @@ returns: `sender_pool_min` sender and `query_pool_min` reader connections, one of each by default (see [Pool settings](#pool-settings)). An unreachable server or bad credentials therefore raise from `connect()`. With [`lazy_connect=on`](/docs/connect/clients/connect-string/#pool-keys), -`connect()` performs no network I/O: `dataframe()`, `query()`, and +`connect()` performs no blocking network I/O: `dataframe()`, `query()`, and `server_info()` connect on first use, and `sender()` connects in the background. In both modes, call `flush(wait=True)` or `wait()` to surface connection and delivery errors from row senders. @@ -306,47 +310,6 @@ QWP cannot preserve nulls for `BOOLEAN`, `BYTE`, or `SHORT`. An absent value in one of those columns is received as `false` or `0`; use a wider nullable type when the distinction matters. -:::note Auto-flush - -The pooled sender auto-flushes by default at 1,000 rows, after 100 ms, or when -its estimated encoded size reaches 90% of the current QWP frame limit. Until -the server advertises that limit, the byte threshold is 8 MiB (or the lower -local queue limit). The interval is checked only by `row()`; there is no -background timer that flushes an idle buffer. - -Override the thresholds with `auto_flush_rows`, `auto_flush_interval`, and -`auto_flush_bytes`. Set `auto_flush_bytes=off` to disable only the byte trigger, -or `auto_flush=off` to disable all row-triggered publishing. Auto-flush does not -wait for an acknowledgement; errors propagate from `row()`. - -::: - -`flush()` publishes the buffered rows to the local queue and returns; -delivery continues in the background. `flush(wait=True)` additionally blocks -until the server acknowledges everything published on this lease, and -`wait(timeout_millis)` is the standalone barrier with an explicit no-progress -timeout (`0` means no deadline; `wait()` returns immediately when the lease -published nothing). Every server rejection goes to the pool's -[rejection handler](#server-rejections), logged by default, and the -store-and-forward queue replays the retriable ones. The barrier raises on a -terminal failure: a terminal connection failure, or a terminal rejection, such -as a schema mismatch or a missing grant, which latches the connection and -raises `QuestDBServerRejectionError`. Closing the lease (leaving the `with` -block) flushes remaining rows without waiting. - -The pooled lease has no `transaction()`, no `new_buffer()` or `Buffer`, -and no manual progress pump; those exist only on the standalone -[legacy `Sender`](#legacy-ilp-clients). The rest of the ws delivery -surface is on the lease: `flush_and_get_fsn()` / -`flush_and_keep_and_get_fsn()` return the published frame's sequence -number, `await_acked_fsn(fsn, timeout_millis)` waits for its -acknowledgement, `published_fsn()` / `acked_fsn()` report progress without -blocking, and `poll_error()` / `error_events_dropped()` pull the -rejections recorded since the lease was borrowed. FSNs are watermarks of -the lease's pooled connection — use them while the lease is held; they are -not portable across leases. Use `auto_flush=off` when -`flush_and_get_fsn()` must define exact application batches. - ### Multi-dimensional arrays A numpy array keeps its shape on the wire, so an N-dimensional array lands in an @@ -387,6 +350,50 @@ np.array([[1, 2], [3, 4]], dtype=np.int64) Cast first with `arr.astype(np.float64)` when your source data is `float32` or an integer type. +### Flushing and acknowledgements + +`flush()` publishes the buffered rows to the local queue and returns; +delivery continues in the background. `flush(wait=True)` additionally blocks +until the server acknowledges everything published on this lease, and +`wait(timeout_millis)` is the standalone barrier with an explicit no-progress +timeout (`0` means no deadline; `wait()` returns immediately when the lease +published nothing). Every server rejection goes to the pool's +[rejection handler](#server-rejections), logged by default, and the +store-and-forward queue replays the retriable ones. The barrier raises on a +terminal failure: a terminal connection failure, or a terminal rejection, such +as a schema mismatch or a missing grant, which latches the connection and +raises `QuestDBServerRejectionError`. A terminal rejection that arrives after +the barrier returned raises from the next call on the lease. Closing the lease +(leaving the `with` block) flushes remaining rows without waiting. + +:::note Auto-flush + +The pooled sender auto-flushes by default at 1,000 rows, after 100 ms, or when +its estimated encoded size reaches 90% of the current QWP frame limit. Until +the server advertises that limit, the byte threshold is 8 MiB (or the lower +local queue limit). The interval is checked only by `row()`; there is no +background timer that flushes an idle buffer. + +Override the thresholds with `auto_flush_rows`, `auto_flush_interval`, and +`auto_flush_bytes`. Set `auto_flush_bytes=off` to disable only the byte trigger, +or `auto_flush=off` to disable all row-triggered publishing. Auto-flush does not +wait for an acknowledgement; errors propagate from `row()`. + +::: + +The pooled lease has no `transaction()`, no `new_buffer()` or `Buffer`, +and no manual progress pump; those exist only on the standalone +[legacy `Sender`](#legacy-ilp-clients). The rest of the ws delivery +surface is on the lease: `flush_and_get_fsn()` / +`flush_and_keep_and_get_fsn()` return the published frame's sequence +number, `await_acked_fsn(fsn, timeout_millis)` waits for its +acknowledgement, `published_fsn()` / `acked_fsn()` report progress without +blocking, and `poll_error()` / `error_events_dropped()` pull the +rejections recorded since the lease was borrowed. FSNs are watermarks of +the lease's pooled connection: use them while the lease is held, because they +are not portable across leases. Use `auto_flush=off` when +`flush_and_get_fsn()` must define exact application batches. + ## DataFrame ingestion Use `db.dataframe()` when the data already lives in columns. Each call @@ -654,8 +661,10 @@ background delivery continues, so retry `wait()` rather than flushing the same rows again. `flush(wait=True)` and `wait()` observe the accepted (`Ok`) acknowledgement: -the server took responsibility for the frames. They are pure barriers — -data-fate notification is a separate channel, the +the server took responsibility for the frames. They raise on a terminal +failure, including a terminal rejection, as described in +[Flushing and acknowledgements](#flushing-and-acknowledgements). Every +rejection, terminal or retriable, also goes to the [rejection handler](#server-rejections). A durable-level wait (waiting for object-storage upload on Enterprise deployments) is not exposed on the pooled Python API. The `request_durable_ack=on` connect key is accepted, and @@ -818,10 +827,11 @@ The `db.error_events_delivered` and `db.error_events_dropped` properties report totals. Use the handler for dead-lettering, alerting, and metrics. Terminal -rejections additionally latch the connection: the next call on the affected -sender lease raises `QuestDBServerRejectionError`, and the pool retires the -connection instead of lending it out again. Producer-side abort logic -belongs with that raised error, not in the handler. +rejections additionally latch the connection: `flush(wait=True)` or `wait()` +on the affected sender lease raises `QuestDBServerRejectionError`, or, when +the rejection arrives after that call returned, the next call on the lease +raises it. The pool retires the connection instead of lending it out again. +Producer-side abort logic belongs with that raised error, not in the handler. ### Reader failover @@ -868,7 +878,7 @@ Codes you will most often dispatch on: | `InvalidTimestamp` | Bad `at` value (wrong type, `NaT`). | Pass `TimestampNanos`, a timezone-aware `datetime`, or `ServerTimestamp`. | | `FailoverRetry` | `flush(wait=True)` or `wait()` made no progress within the budget. | Retry the wait; do not re-send the rows. | | `FailoverWouldDuplicate` | Mid-stream failover on an `iter_*` consumer or `polars.DataFrame(result)`. | Discard partial state and rerun the query. | -| `ServerRejection` | Terminal server rejection (schema mismatch, parse error, ...) latched by the connection; raised by the next call on the affected lease. Every rejection, terminal or retriable, is also delivered to the [rejection handler](#server-rejections). | Inspect `sender_error`; fix the data or the schema. | +| `ServerRejection` | Terminal server rejection (schema mismatch, parse error, security error such as a missing grant, ...) latched by the connection; raised by `flush(wait=True)` or `wait()` on the affected lease, or by its next call when the rejection arrives later. Every rejection, terminal or retriable, is also delivered to the [rejection handler](#server-rejections). | Inspect `sender_error`; fix the data or the schema. | | `ProtocolVersionError` | The server lacks a negotiated capability (for example durable ACK). | Drop the option or upgrade the server. | `QuestDBServerRejectionError` marks a terminal server rejection; its diff --git a/documentation/security/oidc-entra-id.md b/documentation/security/oidc-entra-id.md index a0a194b6c8..8009461b24 100644 --- a/documentation/security/oidc-entra-id.md +++ b/documentation/security/oidc-entra-id.md @@ -265,7 +265,9 @@ acl.oidc.ropc.flow.enabled=true `preferred_username` shows the user's sign-in name in the Web Console and in the logs. Entra ID can change a sign-in name, and can give it to a new user later. To identify users by an object ID that never changes, set -`acl.oidc.sub.claim=oid`. See +`acl.oidc.sub.claim=oid`. Users who log in with a username and password +through the ROPC flow, such as with `psql`, get the username that they type as +the principal instead, whatever the claim. See [Choose the principal claim](/docs/security/oidc/#choose-the-principal-claim). The application ID and the OIDC configuration endpoint's URL can be found @@ -379,9 +381,11 @@ the permissions of its Entra ID security groups instead. See token, the default for applications that accept organizational accounts only, can carry the Application ID URI instead. Web Console logins are not affected, because the Web Console presents ID tokens. -4. Under _App roles_, define a role for each kind of access, with the - _Applications_ member type. The examples below use the value - `QuestDB.Ingest`. +4. Under _App roles_, define a role for each service, with the _Applications_ + member type. A role of its own, mapped to a QuestDB group of its own, lets + you cut off one service without affecting the others, as described in + [Change or revoke access](#change-or-revoke-access). The examples below use + the value `QuestDB.FxIngest`, for a service that ingests FX trades. 5. Assign a role to each managed identity or service principal that needs access. Assign the role to the identity itself: Entra ID does not add a role assigned to a group to the tokens of the service principals in that group. @@ -447,10 +451,10 @@ Entra ID group, and grant the group what the service needs. A service that ingests over HTTP or [QWP](/docs/connect/wire-protocols/qwp-ingress-websocket/) needs the `HTTP` endpoint permission and `INSERT` on its tables: -```questdb-sql title="Map an app role to a group that ingests into fx_trades" -CREATE GROUP ingestApps WITH EXTERNAL ALIAS 'QuestDB.Ingest'; -GRANT HTTP TO ingestApps; -GRANT INSERT ON fx_trades TO ingestApps; +```questdb-sql title="Map the app role of the service to a group that ingests into fx_trades" +CREATE GROUP fxIngest WITH EXTERNAL ALIAS 'QuestDB.FxIngest'; +GRANT HTTP TO fxIngest; +GRANT INSERT ON fx_trades TO fxIngest; ``` A service that queries its tables also needs `SELECT` on them, and a service @@ -655,7 +659,14 @@ opened. They stay open when their token expires. If the next token of the service carries neither a role nor a group, QuestDB rejects it, and the open connections keep the groups of the last token that QuestDB accepted. To cut a service off, revoke the permissions of its QuestDB group. This takes effect at -once, on open connections too, but it cuts off every service that holds the -app role, so to cut off one service at a time, give each service an app role, -and a QuestDB group, of its own. See -[Mapping user permissions](/docs/security/oidc/#mapping-user-permissions). +once, on open connections too, for every service that holds the app role, +which is why each service gets an app role and a QuestDB group of its own: + +```questdb-sql title="Cut off the service that ingests into fx_trades" +REVOKE INSERT ON fx_trades FROM fxIngest; +REVOKE HTTP FROM fxIngest; +``` + +Granting the permissions again restores access on the connections that are +still open, and for every token of the service that QuestDB still accepts. See +[Group changes, sessions, and revoking access](/docs/security/oidc/#group-changes-sessions-and-revoking-access). diff --git a/documentation/security/oidc-pingfederate.md b/documentation/security/oidc-pingfederate.md index 8ab89ef96a..e7696c58e8 100644 --- a/documentation/security/oidc-pingfederate.md +++ b/documentation/security/oidc-pingfederate.md @@ -371,7 +371,9 @@ acl.oidc.ropc.flow.enabled=true QuestDB reads the groups from the User Info response, so keep the default, `acl.oidc.groups.encoded.in.token=false`. It reads the principal from the `sub` claim by default. To read it from another claim of the User Info -response, set `acl.oidc.sub.claim`. See +response, set `acl.oidc.sub.claim`. Users who log in with a username and +password through the ROPC flow, such as with `psql`, get the username that +they type as the principal instead. See [Choose the principal claim](/docs/security/oidc/#choose-the-principal-claim). For the other settings, such as the TLS settings that a private certificate authority needs, see [Configuration](/docs/configuration/oidc/). diff --git a/documentation/security/oidc.mdx b/documentation/security/oidc.mdx index d98558b613..1629dbab0e 100644 --- a/documentation/security/oidc.mdx +++ b/documentation/security/oidc.mdx @@ -18,7 +18,8 @@ same provider, over HTTP, QWP, or PGWire. External users are not created in QuestDB: they get the permissions of the QuestDB groups that their groups in the Identity Provider map to. -This page covers the login flow, which token a client sends, how QuestDB maps +Start with the [minimal configuration](#configuration-options). The rest of +this page covers the login flow, which token a client sends, how QuestDB maps the claims of a token to a principal and to QuestDB groups, including the [fallback claim lists](#fallback-claim-lists) of QuestDB Enterprise 4.0.2 that accept user and app-only tokens with one configuration, @@ -29,6 +30,39 @@ step-by-step setup, see the guides for managed identities and service principals, and [PingFederate](/docs/security/oidc-pingfederate/). +## Configuration options + +A minimal configuration enables OIDC, points QuestDB at the provider, and +names the client ID and the groups claim: + +```ini title="server.conf" +# enable OIDC +acl.oidc.enabled=true + +# the OIDC discovery document of the provider; alternatively, set +# acl.oidc.host and the endpoint settings +acl.oidc.configuration.url=https://idp.example.com/.well-known/openid-configuration + +# the client ID that the provider assigned to QuestDB +acl.oidc.client.id=questdb + +# required, no default: the claim that carries the groups +acl.oidc.groups.claim=groups + +# optional, sub by default: the claim that carries the principal +# acl.oidc.sub.claim=sub + +# optional, false by default: read the claims from a JWT, such as the ID token, +# instead of calling the User Info endpoint +# acl.oidc.groups.encoded.in.token=true +``` + +For all OIDC-related configuration options of QuestDB, see +[Configuration](/docs/configuration/oidc/). For complete examples, see the +QuestDB configuration in the +[Microsoft Entra ID](/docs/security/oidc-entra-id/#questdb-configuration) and +[PingFederate](/docs/security/oidc-pingfederate/#questdb-configuration) guides. + ## Architecture overview Altogether, the architecture appears as such: @@ -56,7 +90,10 @@ the _client_, and it is assigned an identifier: the **Client Id**. Each application which integrates via OIDC should be given a different **Client Id**. Clients that send ID tokens to QuestDB are the exception: QuestDB accepts an ID token only when it was issued for QuestDB's client ID, as described in -[Non-interactive clients](#non-interactive-clients). +[Non-interactive clients](#non-interactive-clients). Separate client IDs do not +limit which access tokens QuestDB accepts when it reads the user information +from the User Info endpoint, the default. See +[Token validation](#token-validation). ### OIDC Provider @@ -397,7 +434,7 @@ data = parse.urlencode( { "username": "testuser", "password": "testpwd", "scope": "openid", - "client_id": "testclient" + "client_id": "questdb" } ).encode() req = request.Request(url=url, data=data) req.add_header("Content-Type", "application/x-www-form-urlencoded") @@ -457,7 +494,7 @@ data = parse.urlencode( { "username": user, "password": pwd, "scope": "openid", - "client_id": "testclient" + "client_id": "questdb" } ).encode() req = request.Request(url=url, data=data) req.add_header("Content-Type", "application/x-www-form-urlencoded") @@ -589,74 +626,169 @@ kinds of token with one configuration, list the claims of both, which requires QuestDB Enterprise 4.0.2 or later. See [Fallback claim lists](#fallback-claim-lists). +The examples below ingest into `fx_trades` over QWP with the +[Python client](/docs/connect/clients/python/). Ingestion fails when it has to +create a table or add a column for an external user, so create the table in +advance, with every column that the client sends. See +[Tables created by external users](#tables-created-by-external-users). The +examples connect over TLS with `wss::`; against a server without TLS, use +`ws::`. + +### Client credentials grant + +A service that authenticates as itself requests an access token with the +client credentials grant, using its own client ID and secret, and sends the +token to QuestDB. This example assumes `acl.oidc.groups.encoded.in.token=true`, +and a provider that issues the token as a JWT whose `aud` claim contains the +value of `acl.oidc.audience`: + +```python +import os + +import questdb +import requests +from questdb import TimestampNanos + +token_endpoint = "https://oidc.provider:443/as/token.oauth2" +response = requests.post( + token_endpoint, + data={ + "grant_type": "client_credentials", + "client_id": os.environ["CLIENT_ID"], + "client_secret": os.environ["CLIENT_SECRET"], + # the scope of the QuestDB API, if the provider needs one, such as + # api:///.default for Microsoft Entra ID + "scope": os.environ["QUESTDB_SCOPE"], + }, + timeout=10, +) +response.raise_for_status() +access_token = response.json()["access_token"] + +conf = f"wss::addr=questdb.example.com:9000;token={access_token};" +with questdb.connect(conf) as db: + with db.sender() as sender: + sender.row( + "fx_trades", + symbols={"symbol": "EURUSD", "side": "buy"}, + columns={"price": 1.0842, "quantity": 100000.0}, + at=TimestampNanos.now(), + ) + sender.flush(wait=True) +``` + +The service gets the permissions of the QuestDB groups that the groups or roles +in its token map to, as described in +[Mapping user permissions](#mapping-user-permissions). For Microsoft Entra ID, +see +[Managed identities and service principals](/docs/security/oidc-entra-id/#managed-identities-and-service-principals), +which requests the token without a client secret. + +### Resource Owner Password Credentials grant + The following example requests a token for a user with the Resource Owner -Password Credentials grant, then ingests over ILP/HTTP. It assumes that -QuestDB's client ID is `testclient`: +Password Credentials grant, then ingests over QWP. It requests the token with +QuestDB's client ID, `questdb` in the +[minimal configuration](#configuration-options): ```python -import json import os + +import questdb import requests -import pandas as pd from dotenv import load_dotenv -from questdb.ingress import Sender +from questdb import TimestampNanos load_dotenv() user = os.environ.get("username") pwd = os.environ.get("password") token_endpoint = "https://oidc.provider:443/as/token.oauth2" -response = requests.post(token_endpoint, - data={"grant_type": "password", - "client_id": "testclient", - "username": user, - "password": pwd, - "scope": "openid"}, - headers={"Content-Type": "application/x-www-form-urlencoded"}) - -response_body = response.content.decode("utf-8") -tokens = json.loads(response_body) +response = requests.post( + token_endpoint, + data={ + "grant_type": "password", + "client_id": "questdb", + "username": user, + "password": pwd, + "scope": "openid", + }, + timeout=10, +) +response.raise_for_status() +tokens = response.json() access_token = tokens["access_token"] -conf = f"http::addr=localhost:9000;token={access_token};" -with Sender.from_conf(conf) as sender: - df = pd.read_csv("data.csv") - df["ts"] = pd.to_datetime(df["ts"]) - sender.dataframe(df, table_name="foo", at="ts") +conf = f"wss::addr=questdb.example.com:9000;token={access_token};" +with questdb.connect(conf) as db: + with db.sender() as sender: + sender.row( + "fx_trades", + symbols={"symbol": "GBPUSD", "side": "sell"}, + columns={"price": 1.2634, "quantity": 250000.0}, + at=TimestampNanos.now(), + ) + sender.flush(wait=True) ``` With `acl.oidc.groups.encoded.in.token=true`, send the ID token instead of the access token: ```python -conf = f"http::addr=localhost:9000;token={tokens['id_token']};" +conf = f"wss::addr=questdb.example.com:9000;token={tokens['id_token']};" ``` Alternatively, a user may rely on QuestDB to authenticate them via the OAuth2 provider when the Resource Owner Password Credentials flow is enabled on the -server side: +server side. The client then sends the username and password instead of a +token, and the principal is the username, as described in +[User and group claims](#user-and-group-claims): ```python import os -import pandas as pd + +import questdb from dotenv import load_dotenv -from questdb.ingress import Sender +from questdb import TimestampNanos load_dotenv() user = os.environ.get("username") pwd = os.environ.get("password") -conf = f"http::addr=localhost:9000;username={user};password={pwd};" -with Sender.from_conf(conf) as sender: - df = pd.read_csv("data.csv") - df["ts"] = pd.to_datetime(df["ts"]) - sender.dataframe(df, table_name="foo", at="ts") +conf = f"wss::addr=questdb.example.com:9000;username={user};password={pwd};" +with questdb.connect(conf) as db: + with db.sender() as sender: + sender.row( + "fx_trades", + symbols={"symbol": "GBPUSD", "side": "sell"}, + columns={"price": 1.2634, "quantity": 250000.0}, + at=TimestampNanos.now(), + ) + sender.flush(wait=True) ``` -Ingestion fails when it has to create a table or add a column for an external -user, so the examples above need the table `foo` to exist, with the columns of -`data.csv`. See -[Tables created by external users](#tables-created-by-external-users). +### Token lifetime + +A client configured with a fixed token, such as with the `token` setting of +its configuration string, sends that token on every connection that it opens +and never refreshes it. QuestDB checks a token when a request or a connection +authenticates with it: + +- A connection that is already open, such as a QWP or PGWire connection, + stays open when its token expires. +- Every HTTP request, and every new connection, including a reconnect, a + failover, and an additional connection of a client's pool, authenticates + with the token again. With `acl.oidc.groups.encoded.in.token=true`, QuestDB + Enterprise 4.0.2 and later reject the token from up to 60 seconds plus + `acl.oidc.cache.ttl` after it expires, as described in + [Token validation](#token-validation). Earlier versions do not check whether + the token has expired. With the User Info endpoint, the provider rejects the + expired token once the user information that QuestDB cached for it expires. + +A long-running client must therefore switch to a new token before the old one +expires: request a new token, create a new client with it, then close the old +client. For a Python example, see +[Request a token in the service](/docs/security/oidc-entra-id/#request-a-token-in-the-service). ## OIDC for the PGWire endpoint @@ -743,22 +875,7 @@ mapped QuestDB group instead; see [memory limits](/docs/security/rbac/#memory-limits). QuestDB works the list of external groups out from the -[user information](#user-and-group-claims). - -If we take the example used earlier, we will see that the message contains a -claim called `groups`. This name is configurable in QuestDB, and since QuestDB -Enterprise 4.0.2, QuestDB can fall back to other claims when the user -information does not carry it, or when it holds no group name. See -[User and group claims](#user-and-group-claims). - -If the groups claim is missing or holds no group name, QuestDB rejects the -login. See [Troubleshooting OIDC logins](#troubleshooting-oidc-logins). - -If none of the user's groups is mapped to a QuestDB group, the user is -authenticated, but has no permissions at all. - -The user has to have at least the `HTTP` permission to be able to successfully -login via the [Web Console](/docs/getting-started/web-console/overview/). +[user information](#user-and-group-claims), such as this User Info response: ```json title="User info response example with Active Directory groups" { @@ -771,6 +888,42 @@ login via the [Web Console](/docs/getting-started/web-console/overview/). } ``` +The groups are in a claim called `groups`. This name is configurable in +QuestDB, and since QuestDB Enterprise 4.0.2, QuestDB can fall back to other +claims when the user information does not carry it, or when it holds no group +name. See [User and group claims](#user-and-group-claims). + +If the groups claim is missing or holds no group name, QuestDB rejects the +login. See [Troubleshooting OIDC logins](#troubleshooting-oidc-logins). + +An external alias must equal a group name in the user information exactly, +including its case. The group names depend on the provider: for example, the +distinguished names of Active Directory groups, as above, or, with Microsoft +Entra ID, the object IDs of security groups and the values of app roles, as +described in +[Map groups and grant permissions](/docs/security/oidc-entra-id/#map-groups-and-grant-permissions) +and +[Map app roles to QuestDB groups](/docs/security/oidc-entra-id/#map-app-roles-to-questdb-groups). + +If none of the user's groups is mapped to a QuestDB group, the user is +authenticated, but has no permissions at all. + +Grant each mapped QuestDB group the +[endpoint permission](/docs/security/rbac/#endpoint-permissions) of every +interface that its users connect to, and the permissions on the tables that +they use: + +- `HTTP` for the [Web Console](/docs/getting-started/web-console/overview/), + the REST API, ILP/HTTP, and QWP over WebSocket. +- `PGWIRE` for the [PGWire endpoint](#oidc-for-the-pgwire-endpoint). + +```questdb-sql title="Let the members of a mapped group log in and query fx_trades" +GRANT HTTP, PGWIRE TO groupName; +GRANT SELECT ON fx_trades TO groupName; +``` + +### Group changes, sessions, and revoking access + QuestDB keeps the groups of the last token that it accepted for each external user, and every session and connection of the user runs with them. QuestDB picks up a change to the user's groups or status in the Identity Provider, @@ -787,6 +940,11 @@ such as a disabled user, when it checks a token of the user again: seconds plus `acl.oidc.cache.ttl` after it expires. Earlier versions do not check whether the token has expired. See [Token validation](#token-validation). +For a username and password login through the [ROPC flow](#enable-ropc), +QuestDB requests a new token from the provider itself once the cached result +expires, so with either setting, changes take effect after +`acl.oidc.cache.ttl`. + A token that QuestDB rejects does not change the groups, so some access outlives the token: @@ -864,6 +1022,12 @@ Two settings name the claims that QuestDB reads: QuestDB rejects the login when the user information carries no principal or no group. +A username and password login through the [ROPC flow](#enable-ropc), over HTTP +basic authentication or PGWire, is the exception to the principal claim: its +principal is the username that the client sends. QuestDB still requires one of +the claims that `acl.oidc.sub.claim` lists to carry a value, but does not use +that value. + ### Choose the principal claim QuestDB keeps one in-memory external user for each principal, and every login @@ -886,6 +1050,12 @@ Never use a display name, such as the `name` claim, because different people can share one. Changing the claim on an existing deployment gives every user a new principal. Permissions stay the same, because they come from the groups. +The claim does not apply to username and password logins through the ROPC +flow, which use the username that the client sends as the principal. A person +who logs in to the Web Console, and with a username and password to the PGWire +endpoint, appears as two external users, unless the claim carries the same +username. + ### Fallback claim lists In QuestDB Enterprise 4.0.2 and later, `acl.oidc.sub.claim` and @@ -924,14 +1094,14 @@ token payloads, shortened to the relevant claims, as follows: { "sub": "4c9a4e3b-0d6f-4f43-9a8e-27d2b1f0c5a1", "oid": "4c9a4e3b-0d6f-4f43-9a8e-27d2b1f0c5a1", - "roles": ["QuestDB.Ingest"] + "roles": ["QuestDB.FxIngest"] } ``` -| Token | Principal | Groups | -| -------- | --------------------------------------------------- | ------------------------------ | -| User | `jane.smith@example.com`, from `preferred_username` | `87654321-...`, from `groups` | -| App-only | `4c9a4e3b-...`, from `oid` | `QuestDB.Ingest`, from `roles` | +| Token | Principal | Groups | +| -------- | --------------------------------------------------- | -------------------------------- | +| User | `jane.smith@example.com`, from `preferred_username` | `87654321-...`, from `groups` | +| App-only | `4c9a4e3b-...`, from `oid` | `QuestDB.FxIngest`, from `roles` | `roles` comes before `groups` because an app-only token can also carry a `groups` claim, and QuestDB takes the groups from the first claim on the list @@ -969,12 +1139,12 @@ With the configuration in [Fallback claim lists](#fallback-claim-lists), QuestDB reads these token payloads, shortened to the relevant claims, as follows: -| Claims in the token | Result | -| ------------------------------------------------------ | --------------------------------------------------------------------------------------------------------------------------- | -| `"preferred_username": ""`, `"oid": "4c9a4e3b-..."` | Principal `4c9a4e3b-...`, from `oid`: the empty `preferred_username` falls through | -| `"roles": []`, `"groups": ["87654321-..."]` | Groups `87654321-...`, from `groups`: the empty `roles` array falls through | -| `"roles": [null, ""]`, `"groups": ["87654321-..."]` | Groups `87654321-...`, from `groups`: `roles` holds no group name once `null` and `""` are skipped | -| `"roles": ["QuestDB.Ingest"]`, `"groups": {"id": "x"}` | The login fails with `unexpected input format`: `groups` is listed and holds an object, even though `roles` carries a group | +| Claims in the token | Result | +| -------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------- | +| `"preferred_username": ""`, `"oid": "4c9a4e3b-..."` | Principal `4c9a4e3b-...`, from `oid`: the empty `preferred_username` falls through | +| `"roles": []`, `"groups": ["87654321-..."]` | Groups `87654321-...`, from `groups`: the empty `roles` array falls through | +| `"roles": [null, ""]`, `"groups": ["87654321-..."]` | Groups `87654321-...`, from `groups`: `roles` holds no group name once `null` and `""` are skipped | +| `"roles": ["QuestDB.FxIngest"]`, `"groups": {"id": "x"}` | The login fails with `unexpected input format`: `groups` is listed and holds an object, even though `roles` carries a group | With `acl.oidc.groups.encoded.in.token=true`, QuestDB validates the token before it applies these rules. See [Token validation](#token-validation). @@ -1029,10 +1199,18 @@ after it expires. After that, it rejects the token on every request and new connection. An HTTP request that also carries a valid session cookie, such as a request of the Web Console, is still accepted through the session, and a PGWire or WebSocket connection that is already open stays open when its token -expires. See [Mapping user permissions](#mapping-user-permissions). +expires. See +[Group changes, sessions, and revoking access](#group-changes-sessions-and-revoking-access), +and, for clients, [Token lifetime](#token-lifetime). With `acl.oidc.groups.encoded.in.token=false`, QuestDB sends the token to the -provider's User Info endpoint, which validates it. See +provider's User Info endpoint, which validates it, and checks nothing else. +QuestDB does not check the audience of the token or the client that it was +issued to, and does not use `acl.oidc.audience`: it accepts any access token +that the User Info endpoint accepts. Any application that can obtain such a +token for a user, from the same provider, can therefore call QuestDB as that +user. With `acl.oidc.groups.encoded.in.token=true`, QuestDB checks the `aud` +claim instead, as described above. See [Find user information](#8-find-user-information). ## Troubleshooting OIDC logins @@ -1107,7 +1285,9 @@ login. If the login succeeds but every request is denied, check that at least one of the user's groups is mapped to a QuestDB group that has the endpoint -permission. See [Mapping user permissions](#mapping-user-permissions). +permission of the interface: `HTTP` for the Web Console, the REST API, +ILP/HTTP, and QWP, or `PGWIRE` for the PGWire endpoint. See +[Mapping user permissions](#mapping-user-permissions). If the login succeeds but ingestion fails when it creates a table or adds a column, create the table and its columns in advance: ingestion cannot create a @@ -1151,39 +1331,6 @@ When you upgrade from an earlier version: claim list. Earlier versions reject every OIDC login when a setting lists more than one claim. -## Configuration options - -A minimal configuration enables OIDC, points QuestDB at the provider, and -names the client ID and the groups claim: - -```ini title="server.conf" -# enable OIDC -acl.oidc.enabled=true - -# the OIDC discovery document of the provider; alternatively, set -# acl.oidc.host and the endpoint settings -acl.oidc.configuration.url=https://idp.example.com/.well-known/openid-configuration - -# the client ID that the provider assigned to QuestDB -acl.oidc.client.id=questdb - -# required, no default: the claim that carries the groups -acl.oidc.groups.claim=groups - -# optional, sub by default: the claim that carries the principal -# acl.oidc.sub.claim=sub - -# optional, false by default: read the claims from a JWT, such as the ID token, -# instead of calling the User Info endpoint -# acl.oidc.groups.encoded.in.token=true -``` - -For all OIDC-related configuration options of QuestDB, see -[Configuration](/docs/configuration/oidc/). For complete examples, see the -QuestDB configuration in the -[Microsoft Entra ID](/docs/security/oidc-entra-id/#questdb-configuration) and -[PingFederate](/docs/security/oidc-pingfederate/#questdb-configuration) guides. -
## Identity provider guides {#active-directory} From 9fa710e6867ec824b5e368702ec8760decab5908 Mon Sep 17 00:00:00 2001 From: glasstiger Date: Fri, 9 Oct 2026 16:59:34 +0100 Subject: [PATCH 10/29] docs(oidc): fix token rotation with sf_dir, expiry wording and missing grants - Token rotation: with sf_dir, two clients cannot hold the same store-and-forward slot, so building the new client before closing the old one fails. Token lifetime, the Entra renewal example and the Python, Go, C/C++ and .NET pages now say to flush and close the old client first, then reuse the sender_id so that the new client replays the slot. .NET: call SendAsync() before DisposeAsync(), which does not send. - C/C++, .NET and Go: the client sends the token on every connection that it opens, and QuestDB rejects an expired token on new connections only on 4.0.2 and later or with the User Info endpoint, shortly after it expires. - Non-interactive clients: create fx_trades and grant HTTP and INSERT to the client's group before the ingestion examples. - Entra guide: emitting groups as role claims removes the app roles from the roles claim; it does not mix them. - Changelog: add Python and the sf_dir order to the token rotation entry. Co-Authored-By: Claude Opus 5.5 --- documentation/changelog.mdx | 2 +- documentation/connect/clients/c-and-cpp.md | 2 +- documentation/connect/clients/dotnet.md | 2 +- documentation/connect/clients/go.md | 10 +++++--- documentation/connect/clients/python.md | 2 +- documentation/security/oidc-entra-id.md | 14 ++++++++--- documentation/security/oidc.mdx | 29 ++++++++++++++++++++++ 7 files changed, 51 insertions(+), 10 deletions(-) diff --git a/documentation/changelog.mdx b/documentation/changelog.mdx index dc1a23e21a..a3761d1080 100644 --- a/documentation/changelog.mdx +++ b/documentation/changelog.mdx @@ -36,7 +36,7 @@ This page tracks significant updates to the QuestDB documentation. - [Endpoint permissions](/docs/security/rbac/#endpoint-permissions) - The `HTTP` permission also covers QWP over WebSocket. The [authentication methods](/docs/security/rbac/#authentication) and [users and service accounts](/docs/security/rbac/#users-and-service-accounts) describe external users, who log in through OIDC - [Microsoft Entra ID guide](/docs/security/oidc-entra-id/#web-console-sign-in) - Uses `preferred_username` as the principal claim instead of the display name, explains when to use `oid` instead, and registers the application as single-tenant - [Python client](/docs/connect/clients/python/#authentication-and-tls) - Corrected when bad credentials raise: by default, `connect()` opens its first connections and raises, and `lazy_connect=on` defers the error to the first operation. Corrected that a terminal server rejection, such as a missing grant, raises `QuestDBServerRejectionError` from `flush(wait=True)`, as described in [Flushing and acknowledgements](/docs/connect/clients/python/#flushing-and-acknowledgements) -- [C/C++](/docs/connect/clients/c-and-cpp/), [.NET](/docs/connect/clients/dotnet/), and [Go](/docs/connect/clients/go/) clients - Corrected token rotation: build a new client with a new token before the old token expires, then close the old one +- [Python](/docs/connect/clients/python/), [C/C++](/docs/connect/clients/c-and-cpp/), [.NET](/docs/connect/clients/dotnet/), and [Go](/docs/connect/clients/go/) clients - Corrected token rotation: build a new client with a new token before the old token expires, then close the old one, or close the old one first when it uses `sf_dir` ## September 2026 diff --git a/documentation/connect/clients/c-and-cpp.md b/documentation/connect/clients/c-and-cpp.md index 8eaa21d6f0..61807763e6 100644 --- a/documentation/connect/clients/c-and-cpp.md +++ b/documentation/connect/clients/c-and-cpp.md @@ -334,7 +334,7 @@ following are **not** supported: |---|---|---| | OIDC token acquisition or in-band refresh | Not supported. The client does not negotiate with an identity provider and has no callback to refresh a token mid-session. | QuestDB itself supports OIDC; see [OpenID Connect](/docs/security/oidc/#non-interactive-clients) for which token to send. Acquire the token out-of-band from your IdP, pass it via `token=...`, and rebuild the pool when the token nears expiry. For Azure services, see [Microsoft Entra ID managed identities and service principals](/docs/security/oidc-entra-id/#managed-identities-and-service-principals). | | Mutual TLS (client certificates) | Not supported. The QuestDB server does not negotiate client certificates regardless of client. | Use bearer-token auth over `wss`. See the connect-string reference's [TLS section](/docs/connect/clients/connect-string/#tls). | -| Token rotation mid-session | Not supported. Credentials are presented once during the WebSocket upgrade and are not re-sent. | Before the token expires, build a new pool with a new token, then close the old one. QuestDB checks the token again on every new connection, including reconnects, and rejects it once it has expired; see [Token lifetime](/docs/security/oidc/#token-lifetime). | +| Token rotation mid-session | Not supported. The pool sends the credentials in the WebSocket upgrade of every connection that it opens, including reconnects, and cannot switch to a new token while it runs. | Before the token expires, build a new pool with a new token, then close the old one. With `sf_dir`, flush and close the old pool first, then build the new one with the same `sender_id`: two pools cannot hold the same store-and-forward slot, and the new pool replays what the old one left on disk. On QuestDB Enterprise 4.0.2 and later, or with the User Info endpoint, QuestDB rejects the token on new connections shortly after it expires; see [Token lifetime](/docs/security/oidc/#token-lifetime). | ## Headers diff --git a/documentation/connect/clients/dotnet.md b/documentation/connect/clients/dotnet.md index 4d5653408b..17e3b0cb9b 100644 --- a/documentation/connect/clients/dotnet.md +++ b/documentation/connect/clients/dotnet.md @@ -230,7 +230,7 @@ that purpose; left unset, it inherits `auth_timeout_ms`. |---|---|---| | OIDC token acquisition or in-band refresh | Not supported by this client. It does not negotiate with an identity provider and has no callback to refresh a token mid-session. | QuestDB itself supports OIDC; see [OpenID Connect](/docs/security/oidc/#non-interactive-clients) for which token to send. Acquire the token out-of-band from your IdP, pass it via `token=...` above, and rebuild the sender / query client when the token nears expiry. For Azure services, see [Microsoft Entra ID managed identities and service principals](/docs/security/oidc-entra-id/#managed-identities-and-service-principals). | | Mutual TLS (client certificates) | Not supported. The QuestDB server does not negotiate client certificates regardless of client. | Use bearer-token auth over `wss://`. See the connect-string reference for the canonical statement. | -| Token rotation mid-session | Not supported. Credentials are presented once during the WebSocket upgrade and are not re-sent. | Before the token expires, build a new sender with a new token, then `await` the old sender's `DisposeAsync()`. The same applies to `QueryClient`. QuestDB checks the token again on every new connection, including reconnects, and rejects it once it has expired; see [Token lifetime](/docs/security/oidc/#token-lifetime). | +| Token rotation mid-session | Not supported. The client sends the credentials in the WebSocket upgrade of every connection that it opens, including reconnects, and cannot switch to a new token while it runs. | Before the token expires, build a new sender with a new token, call `SendAsync()` on the old sender, because dispose does not send, then `await` its `DisposeAsync()`. With `sf_dir`, call `SendAsync()` on the old sender and dispose of it first, then build the new one with the same `sender_id`: two senders cannot hold the same store-and-forward slot, and the new sender replays what the old one left on disk. Replace a `QueryClient` the same way: build the new one, then dispose of the old one. On QuestDB Enterprise 4.0.2 and later, or with the User Info endpoint, QuestDB rejects the token on new connections shortly after it expires; see [Token lifetime](/docs/security/oidc/#token-lifetime). | ### Production example (TLS + token + multi-host) diff --git a/documentation/connect/clients/go.md b/documentation/connect/clients/go.md index 7cf99637c9..d8543f0304 100644 --- a/documentation/connect/clients/go.md +++ b/documentation/connect/clients/go.md @@ -365,9 +365,13 @@ Acquire it out of band, for example from the identity provider of the QuestDB En For Azure services, see [Microsoft Entra ID managed identities and service principals](/docs/security/oidc-entra-id/#managed-identities-and-service-principals). Before the token expires, construct a new handle with a new token, then close -the old one. QuestDB checks the token again on every new connection, including -reconnects, and rejects it once it has expired; see -[Token lifetime](/docs/security/oidc/#token-lifetime). +the old one. With `sf_dir`, flush and close the old handle first, then +construct the new one with the same `sender_id`: two handles cannot hold the +same store-and-forward slot, and the new handle replays what the old one left +on disk. The client sends the token on every connection that it opens, +including reconnects. On QuestDB Enterprise 4.0.2 and later, or with the User +Info endpoint, QuestDB rejects it on new connections shortly after it expires; +see [Token lifetime](/docs/security/oidc/#token-lifetime). An expired or rejected token surfaces as an authentication failure (see [Connection-level errors](#connection-level-errors)). It is mutually exclusive with `username`/`password`. diff --git a/documentation/connect/clients/python.md b/documentation/connect/clients/python.md index 742d1ed858..cb1d18a3b3 100644 --- a/documentation/connect/clients/python.md +++ b/documentation/connect/clients/python.md @@ -181,7 +181,7 @@ following are **not** supported: | --- | --- | --- | | OIDC token acquisition or in-band refresh | Not supported. The client does not negotiate with an identity provider and cannot refresh a token mid-session. | QuestDB itself supports OIDC; see [OpenID Connect](/docs/security/oidc/#non-interactive-clients) for which token to send. Acquire the token out-of-band from your IdP, pass it via `token=...`, and rebuild the handle when the token nears expiry. For an example with an Entra ID managed identity, see [Microsoft Entra ID managed identities and service principals](/docs/security/oidc-entra-id/#managed-identities-and-service-principals). | | Mutual TLS (client certificates) | Not supported. The QuestDB server does not negotiate client certificates regardless of client. | Use bearer-token auth over `wss`. | -| Token rotation mid-session | Not supported. The handle keeps the credentials it was built with and presents them on every connection it opens, including reconnects and failover, so an expired token also breaks mid-session reconnection. | Before the token expires, build a fresh handle with a new token, then close the old one. See [Token lifetime](/docs/security/oidc/#token-lifetime) for how QuestDB treats an expiring token, and [Request a token in the service](/docs/security/oidc-entra-id/#request-a-token-in-the-service) for an example. | +| Token rotation mid-session | Not supported. The handle keeps the credentials it was built with and presents them on every connection it opens, including reconnects and failover, so an expired token also breaks mid-session reconnection. | Before the token expires, build a fresh handle with a new token, then close the old one. With `sf_dir`, flush and close the old handle first, then build the new one with the same `sender_id`: two handles cannot hold the same store-and-forward slot, and the new handle replays what the old one left on disk. See [Token lifetime](/docs/security/oidc/#token-lifetime) for how QuestDB treats an expiring token, and [Request a token in the service](/docs/security/oidc-entra-id/#request-a-token-in-the-service) for an example. | ## The pool diff --git a/documentation/security/oidc-entra-id.md b/documentation/security/oidc-entra-id.md index 8009461b24..ed7bc3f9b0 100644 --- a/documentation/security/oidc-entra-id.md +++ b/documentation/security/oidc-entra-id.md @@ -177,9 +177,10 @@ Select _Add groups claim_, and set it up as follows: and the _Access_ token. - Leave _Emit groups as role claims_ cleared for both. Otherwise, Entra ID sends the groups in the `roles` claim instead of the `groups` claim that the - [QuestDB configuration](#questdb-configuration) below reads, and mixes the - groups of services with their app roles in - [Managed identities and service principals](#managed-identities-and-service-principals). + [QuestDB configuration](#questdb-configuration) below reads, and the `roles` + claim no longer carries the app roles that + [Managed identities and service principals](#managed-identities-and-service-principals) + relies on. The customized tokens contain user information which cannot be accessed without permission. User information is provided by Microsoft Graph, so @@ -615,6 +616,13 @@ finally: db.close() ``` +The example keeps its store-and-forward data in memory, the default. With +`sf_dir`, `connect()` fails while the old handle holds the slot of the same +`sender_id`, so `renew()` keeps the old token until it expires. Close the old +handle first instead, then connect with the new token and the same +`sender_id`. The new handle replays what the old one left on disk, as +described in [Token lifetime](/docs/security/oidc/#token-lifetime). + A terminal rejection raises `QuestDBServerRejectionError` from `flush(wait=True)` and ends the loop, and the `finally` block closes the handle. To keep the service running instead, catch the error around the batch, diff --git a/documentation/security/oidc.mdx b/documentation/security/oidc.mdx index 1629dbab0e..c5644ffe26 100644 --- a/documentation/security/oidc.mdx +++ b/documentation/security/oidc.mdx @@ -634,6 +634,26 @@ advance, with every column that the client sends. See examples connect over TLS with `wss::`; against a server without TLS, use `ws::`. +As the admin, create the table, and grant the QuestDB group that the client's +groups map to the `HTTP` endpoint permission and `INSERT` on the table: + +```questdb-sql title="Create fx_trades and let the client's group ingest into it" +CREATE TABLE fx_trades ( + timestamp TIMESTAMP_NS, + symbol SYMBOL, + side SYMBOL, + price DOUBLE, + quantity DOUBLE +) TIMESTAMP(timestamp) PARTITION BY DAY; + +CREATE GROUP fxIngest WITH EXTERNAL ALIAS 'fx-ingest'; +GRANT HTTP TO fxIngest; +GRANT INSERT ON fx_trades TO fxIngest; +``` + +Replace `fx-ingest` with a group or role name that the client's token carries, +as described in [Mapping user permissions](#mapping-user-permissions). + ### Client credentials grant A service that authenticates as itself requests an access token with the @@ -790,6 +810,15 @@ expires: request a new token, create a new client with it, then close the old client. For a Python example, see [Request a token in the service](/docs/security/oidc-entra-id/#request-a-token-in-the-service). +A client that keeps its +[store-and-forward](/docs/connect/clients/connect-string/#sf-keys) data on +disk, with `sf_dir`, must switch in the other order. Two clients cannot hold +the same slot, so a new client with the `sender_id` of the old client fails to +start while the old client is open. Flush and close the old client first, then +create the new client with the new token and the same `sender_id`. The new +client replays what the old client left on disk, as described in +[Sender restart and replay](/docs/connect/clients/connect-string/#sender-restart-and-replay). + ## OIDC for the PGWire endpoint If the From 3b4389a2d90187b2f6c36e0da015848fab634afa Mon Sep 17 00:00:00 2001 From: glasstiger Date: Fri, 9 Oct 2026 18:19:06 +0100 Subject: [PATCH 11/29] docs(oidc): fix sf_dir token rotation, mode choice, startup and title findings - Entra ID guide: with sf_dir, renew() never switches tokens, because the old handle holds the store-and-forward slot. Say so, and add a variant that closes the old handle first, then retries the connection with a current token instead of reusing the closed handle. Token lifetime states the retry rule for every client. - Entra ID guide: QuestDB rejects a login without a groups claim. Drop the statement that users authenticate without customized tokens. - Title the provider guides "Microsoft Entra ID (Azure AD) OIDC setup" and "PingFederate OIDC setup", and add OIDC and Azure AD to the Entra ID description. - OIDC page: add "Where QuestDB reads the user information", comparing the User Info endpoint with reading the claims from the token, and add acl.oidc.groups.encoded.in.token=true to the fallback claim list snippets. The minimal configuration says Entra ID needs it. - Document that QuestDB downloads the discovery document at startup and does not start without it, that with acl.oidc.host it starts while the provider is unreachable, and that the two settings cannot be combined. - C/C++ client: the pool connects eagerly by default, as the 7.0.0 headers say; lazy_connect=on defers the errors to the first borrow. - Changelog entries for the above. Co-Authored-By: Claude Opus 5.5 --- documentation/changelog.mdx | 5 +- documentation/configuration/oidc.md | 16 +++- documentation/connect/clients/c-and-cpp.md | 42 +++++++---- documentation/security/oidc-entra-id.md | 84 ++++++++++++++++----- documentation/security/oidc-pingfederate.md | 2 +- documentation/security/oidc.mdx | 69 +++++++++++++++-- 6 files changed, 173 insertions(+), 45 deletions(-) diff --git a/documentation/changelog.mdx b/documentation/changelog.mdx index a3761d1080..9db2172c4c 100644 --- a/documentation/changelog.mdx +++ b/documentation/changelog.mdx @@ -12,7 +12,8 @@ This page tracks significant updates to the QuestDB documentation. ### New -- [Microsoft Entra ID](/docs/security/oidc-entra-id/) - New page for Microsoft Entra ID: the Web Console sign-in guide, moved from the OIDC page, and app-only tokens for Azure managed identities and service principals in QuestDB Enterprise 4.0.2, with an app role and a QuestDB group for each service, how to keep app roles the only way in, how to cut off one service, and Python examples that ingest and renew the token +- [Microsoft Entra ID](/docs/security/oidc-entra-id/) - New page for Microsoft Entra ID: the Web Console sign-in guide, moved from the OIDC page, and app-only tokens for Azure managed identities and service principals in QuestDB Enterprise 4.0.2, with an app role and a QuestDB group for each service, how to keep app roles the only way in, how to cut off one service, and Python examples that ingest and renew the token, also with store-and-forward on disk +- [Where QuestDB reads the user information](/docs/security/oidc/#where-questdb-reads-the-user-information) - Compares the User Info endpoint with reading the claims from the token: which token each client sends, what QuestDB checks, and when a change to the groups takes effect - [User and group claims](/docs/security/oidc/#user-and-group-claims) - How QuestDB Enterprise 4.0.2 reads the principal and the groups from the user information: fallback claim lists, the rules for missing, `null`, and empty claims, how to choose the principal claim, and the startup checks - [Token validation](/docs/security/oidc/#token-validation), [Troubleshooting OIDC logins](/docs/security/oidc/#troubleshooting-oidc-logins), and [Versions before 4.0.2](/docs/security/oidc/#versions-before-402) - What QuestDB checks in a token, including that it does not check the audience of a token that it sends to the User Info endpoint, how long it accepts a token after it expires, the log messages of rejected logins, how earlier versions differ, and what to check when you upgrade - [Tables created by external users](/docs/security/oidc/#tables-created-by-external-users) - Ingestion fails when it has to create a table or add a column for a user who logs in through OIDC, so create the tables in advance, or create them with SQL and `OWNED BY`, naming one of the user's QuestDB groups @@ -26,6 +27,7 @@ This page tracks significant updates to the QuestDB documentation. - [`acl.oidc.groups.encoded.in.token`](/docs/configuration/oidc/#acloidcgroupsencodedintoken) - Documented that QuestDB Enterprise 4.0.2 rejects expired tokens and tokens without an `exp` claim, which earlier versions accept, and which token QuestDB reads the principal and the groups from - [`acl.oidc.audience`](/docs/configuration/oidc/#acloidcaudience) and [`acl.oidc.cache.ttl`](/docs/configuration/oidc/#acloidccachettl) - Documented that `acl.oidc.audience` takes a single value, which the `aud` claim of a token must be or contain, and that QuestDB does not use it with the User Info endpoint, and what the cache holds when the groups are encoded in the token or a user logs in with a username and password - [`acl.oidc.public.keys.expiry`](/docs/configuration/oidc/#acloidcpublickeysexpiry) and [`acl.oidc.state.required`](/docs/configuration/oidc/#acloidcstaterequired) - Documented how long QuestDB uses the signing keys of the OIDC provider before it downloads them again, and the `state` check of the Web Console login +- [`acl.oidc.configuration.url`](/docs/configuration/oidc/#acloidcconfigurationurl) and [`acl.oidc.host`](/docs/configuration/oidc/#acloidchost) - Documented that QuestDB downloads the discovery document at startup and does not start without it, that with `acl.oidc.host` QuestDB starts while the provider is unreachable, and that the two settings cannot be combined ### Updated @@ -36,6 +38,7 @@ This page tracks significant updates to the QuestDB documentation. - [Endpoint permissions](/docs/security/rbac/#endpoint-permissions) - The `HTTP` permission also covers QWP over WebSocket. The [authentication methods](/docs/security/rbac/#authentication) and [users and service accounts](/docs/security/rbac/#users-and-service-accounts) describe external users, who log in through OIDC - [Microsoft Entra ID guide](/docs/security/oidc-entra-id/#web-console-sign-in) - Uses `preferred_username` as the principal claim instead of the display name, explains when to use `oid` instead, and registers the application as single-tenant - [Python client](/docs/connect/clients/python/#authentication-and-tls) - Corrected when bad credentials raise: by default, `connect()` opens its first connections and raises, and `lazy_connect=on` defers the error to the first operation. Corrected that a terminal server rejection, such as a missing grant, raises `QuestDBServerRejectionError` from `flush(wait=True)`, as described in [Flushing and acknowledgements](/docs/connect/clients/python/#flushing-and-acknowledgements) +- [C/C++ client](/docs/connect/clients/c-and-cpp/#connecting) - Corrected when connection and authentication errors surface: by default, the pool connects when it is created and the errors surface there, and `lazy_connect=on` defers them - [Python](/docs/connect/clients/python/), [C/C++](/docs/connect/clients/c-and-cpp/), [.NET](/docs/connect/clients/dotnet/), and [Go](/docs/connect/clients/go/) clients - Corrected token rotation: build a new client with a new token before the old token expires, then close the old one, or close the old one first when it uses `sf_dir` ## September 2026 diff --git a/documentation/configuration/oidc.md b/documentation/configuration/oidc.md index 0f689a158b..1a38dde7fc 100644 --- a/documentation/configuration/oidc.md +++ b/documentation/configuration/oidc.md @@ -52,6 +52,15 @@ enabled. URL where the OpenID Provider's configuration information can be loaded in JSON format. Should always end with `/.well-known/openid-configuration`. +QuestDB downloads the document when it starts, and takes the provider's +endpoints from it. QuestDB does not start when it cannot download or parse the +document. The error starts with +`Unable to download OIDC provider configuration from` or +`Unable to parse OIDC provider configuration`. To let QuestDB start while the +provider is unreachable, set [`acl.oidc.host`](#acloidchost) and the +[endpoint settings](#endpoints) instead. QuestDB refuses to start when both +this setting and `acl.oidc.host` are set. + ### acl.oidc.enabled - **Default**: `false` @@ -65,8 +74,11 @@ configuration options must also be set. - **Default**: none - **Reloadable**: no -OIDC provider hostname. Required when OIDC is enabled, unless the OIDC -configuration URL is set. +OIDC provider hostname. Required when OIDC is enabled, unless +[`acl.oidc.configuration.url`](#acloidcconfigurationurl) is set. The two +settings cannot be combined: QuestDB refuses to start when both are set. With +`acl.oidc.host`, QuestDB starts even when the provider is unreachable, and only +OIDC logins fail until the provider is reachable again. ### acl.oidc.http.timeout diff --git a/documentation/connect/clients/c-and-cpp.md b/documentation/connect/clients/c-and-cpp.md index 61807763e6..9313c2a4ca 100644 --- a/documentation/connect/clients/c-and-cpp.md +++ b/documentation/connect/clients/c-and-cpp.md @@ -251,8 +251,8 @@ threads, close it at shutdown. You don't open connections yourself: **borrow** a lease (a sender or reader), use it on one thread, then **return** it (recycles the connection) or **drop** it (retires it). In C++ the borrowed sender wrapper and pooled reader return on -destruction; `drop_on_return()` forces a drop. The pool connects lazily, and -reconnects and fails over on its own. +destruction; `drop_on_return()` forces a drop. The pool opens its first +connections when you create it, and reconnects and fails over on its own. ### Which borrow? @@ -271,14 +271,19 @@ the schema server-side. The connect string uses a QWP/WebSocket scheme: `ws` / `wss`. For auth and TLS keys, see the -[connect string reference](/docs/connect/clients/connect-string/). The pool is -**lazy**: `questdb_db_connect` (C) and the `questdb::pool` constructor (C++) -parse and validate the string but perform no blocking network I/O, so auth / -TLS / connect errors surface from the first borrow, not at construction. The -one exception is the on-disk send queue (`sf_dir`, see -[Durability and backpressure](#durability-and-backpressure)): at connect the -pool reopens any queue directories a previous run left holding unacknowledged -rows, then reconnects and resends them in the background. +[connect string reference](/docs/connect/clients/connect-string/). By default, +the pool connects **eagerly**: `questdb_db_connect` (C) and the +`questdb::pool` constructor (C++) parse and validate the string, then open the +warm minimum of connections (`sender_pool_min` senders and `query_pool_min` +readers, one of each by default) before they return, so auth / TLS / connect +errors surface from `questdb_db_connect` or the constructor. With +[`lazy_connect=on`](/docs/connect/clients/connect-string/#pool-keys), they +perform no blocking network I/O: senders connect in the background, readers +connect on the first borrow, and the errors surface from those operations +instead. With the on-disk send queue (`sf_dir`, see +[Durability and backpressure](#durability-and-backpressure)), the pool also +reopens, at connect, any queue directories a previous run left holding +unacknowledged rows, then reconnects and resends them in the background. @@ -320,10 +325,13 @@ wss::addr=db.example.com:9000;token=your_bearer_token; # bearer token (E rejects it. - **`auth_timeout_ms`** (default 15000) bounds the WebSocket upgrade. -Because the pool connects lazily, a bad credential surfaces as -`line_sender_error_auth_error` from the **first borrow** on each connection. -Handle it there, not at `connect` (see -[Which errors mean what](#which-errors-mean-what)). +Because the pool connects eagerly by default, a bad credential surfaces as +`line_sender_error_auth_error` from `questdb_db_connect` or the `questdb::pool` +constructor. With `lazy_connect=on`, it surfaces later: from a reader's first +borrow, or, because senders connect in the background, from a sender's wait or +a later call on it. A connection that the pool opens later, such as a +reconnect, can also fail authentication, for example after the token expires +(see [Which errors mean what](#which-errors-mean-what)). ### Unsupported auth paths @@ -1591,7 +1599,8 @@ int main() // Enterprise; the connect fails against a server without replication. "request_durable_ack=on;"}; - // First borrow opens the connection: give it the failover budget. + // If the pool has to open a connection, retry it within the + // failover budget. auto sender = pool.borrow_sender_with_retry(pool.reconnect_max_duration_ms()); @@ -1640,7 +1649,8 @@ int main(void) db = questdb_db_connect(conf, strlen(conf), &err); if (!db) goto on_error; - /* First borrow opens the connection: give it the failover budget. */ + /* If the pool has to open a connection, retry it within the failover + budget. */ sender = questdb_db_borrow_sender_with_retry( db, questdb_db_reconnect_max_duration_ms(db), &err); if (!sender) goto on_error; diff --git a/documentation/security/oidc-entra-id.md b/documentation/security/oidc-entra-id.md index ed7bc3f9b0..0078755782 100644 --- a/documentation/security/oidc-entra-id.md +++ b/documentation/security/oidc-entra-id.md @@ -1,7 +1,7 @@ --- -title: Microsoft Entra ID +title: Microsoft Entra ID (Azure AD) OIDC setup sidebar_label: Microsoft Entra ID -description: "Connect Microsoft Entra ID to QuestDB Enterprise: Web Console SSO for users, and app-only tokens for Azure managed identities and service principals." +description: "Set up Microsoft Entra ID (Azure AD) OIDC for QuestDB Enterprise: Web Console SSO, and app-only tokens for managed identities and service principals." --- import Screenshot from "@theme/Screenshot" @@ -146,15 +146,15 @@ width={600} Our next stop is the _Token configuration_, where the OAuth2/OIDC access and ID tokens can be customized. -Note that users can be authenticated without customized tokens, but authorization -would prove to be challenging. The user's security groups are not included -in the tokens by default. +Entra ID does not include the user's security groups in the tokens by default, +and QuestDB rejects a login whose user information carries no group, so the +tokens must be customized. -QuestDB can be configured to request the user's groups from the UserInfo -endpoint of the OAuth2 server, but Entra ID cannot be configured to provide -this information via the UserInfo endpoint. -Therefore, we choose to customize the tokens, QuestDB will decode and -validate the ID token, and take the group information from there. +QuestDB can also request the user's groups from the User Info endpoint of the +OAuth2 server, but Entra ID cannot be configured to return them from that +endpoint. Therefore, QuestDB decodes and validates the ID token, and takes the +groups from it, with `acl.oidc.groups.encoded.in.token=true` in the +[QuestDB configuration](#questdb-configuration) below. QuestDB authorization relies on receiving the group memberships of the user. Entra ID groups should be mapped to QuestDB groups, and permissions can be @@ -409,6 +409,9 @@ lists below make every OIDC login fail. Keep tokens against the signing keys of Entra ID, the same way as ID tokens: ```ini title="server.conf" +# unchanged: QuestDB validates the tokens itself and reads the claims from them +acl.oidc.groups.encoded.in.token=true + # user tokens carry preferred_username, app-only tokens carry oid acl.oidc.sub.claim=preferred_username,oid @@ -616,19 +619,66 @@ finally: db.close() ``` -The example keeps its store-and-forward data in memory, the default. With -`sf_dir`, `connect()` fails while the old handle holds the slot of the same -`sender_id`, so `renew()` keeps the old token until it expires. Close the old -handle first instead, then connect with the new token and the same -`sender_id`. The new handle replays what the old one left on disk, as -described in [Token lifetime](/docs/security/oidc/#token-lifetime). - A terminal rejection raises `QuestDBServerRejectionError` from `flush(wait=True)` and ends the loop, and the `finally` block closes the handle. To keep the service running instead, catch the error around the batch, and decide whether to retry or drop the batch: a batch fails the same way until you fix the cause of the rejection, such as a missing grant. +The example keeps its store-and-forward data in memory, the default. With +[`sf_dir`](/docs/connect/clients/connect-string/#sf-keys), the client keeps the +rows that QuestDB has not acknowledged on disk, in a slot named after its +`sender_id`, and two handles cannot hold the same slot. `connect()` with the +new token then raises `QuestDBError`, because the old handle still holds the +slot, so `renew()` above never switches to the new token, and the service fails +once it has to open a new connection after QuestDB stops accepting the old +token. Close the old handle first instead, then connect with the new token and +the same `sender_id`: the new handle takes over the slot, and replays what the +old one left on disk. The old handle is closed by then, so when the new +connection fails, retry it instead of falling back to the old handle. With +`sf_dir`, replace `connect()` and `renew()` in the example with: + +```python title="Switch to a new token with store-and-forward on disk" +# the store-and-forward slot of the service +SF_CONF = "sf_dir=/var/lib/fx-ingest/sf;sender_id=fx-ingest;" + + +def connect(token): + conf = f"wss::addr={ADDR};token={token.token};{SF_CONF}" + return questdb.connect(conf, error_handler=on_rejection) + + +def renew(token, db): + # returns the token and the handle to use from now on + try: + new_token = credential.get_token(SCOPE) + except AzureError as e: + # keep the old token and handle, and try again with the next batch + print("token renewal failed:", e) + return token, db + # the credential returns the cached token until it renews it + if new_token.token == token.token: + return token, db + # release the slot; the rows that QuestDB has not acknowledged stay on disk + db.close() + while True: + try: + # the new handle takes over the slot and replays its rows + return new_token, connect(new_token) + except QuestDBError as e: + # the old handle is closed, so retry instead of falling back to it + print("connect failed, retrying in 5 seconds:", e) + time.sleep(5) + try: + # an outage can outlast the token, so get a current one + new_token = credential.get_token(SCOPE) + except AzureError as e: + print("token renewal failed:", e) +``` + +For the rules that apply to every client, see +[Token lifetime](/docs/security/oidc/#token-lifetime). + See [Authentication and TLS](/docs/connect/clients/python/#authentication-and-tls) for the client options. For the [PGWire endpoint](/docs/security/oidc/#oidc-for-the-pgwire-endpoint), enable diff --git a/documentation/security/oidc-pingfederate.md b/documentation/security/oidc-pingfederate.md index e7696c58e8..21cab0547e 100644 --- a/documentation/security/oidc-pingfederate.md +++ b/documentation/security/oidc-pingfederate.md @@ -1,5 +1,5 @@ --- -title: PingFederate +title: PingFederate OIDC setup sidebar_label: PingFederate description: "Set up Web Console single sign-on for QuestDB Enterprise with PingFederate as the OIDC provider, authenticating users over LDAP." --- diff --git a/documentation/security/oidc.mdx b/documentation/security/oidc.mdx index c5644ffe26..8992ba8888 100644 --- a/documentation/security/oidc.mdx +++ b/documentation/security/oidc.mdx @@ -18,9 +18,12 @@ same provider, over HTTP, QWP, or PGWire. External users are not created in QuestDB: they get the permissions of the QuestDB groups that their groups in the Identity Provider map to. -Start with the [minimal configuration](#configuration-options). The rest of -this page covers the login flow, which token a client sends, how QuestDB maps -the claims of a token to a principal and to QuestDB groups, including the +Start with the [minimal configuration](#configuration-options), and decide +[where QuestDB reads the user information](#where-questdb-reads-the-user-information): +from the provider's User Info endpoint, or from the token that the client +sends. The rest of this page covers the login flow, which token a client +sends, how QuestDB maps the claims of a token to a principal and to QuestDB +groups, including the [fallback claim lists](#fallback-claim-lists) of QuestDB Enterprise 4.0.2 that accept user and app-only tokens with one configuration, [token validation](#token-validation), and @@ -39,8 +42,8 @@ names the client ID and the groups claim: # enable OIDC acl.oidc.enabled=true -# the OIDC discovery document of the provider; alternatively, set -# acl.oidc.host and the endpoint settings +# the OIDC discovery document of the provider, which QuestDB downloads at +# startup; alternatively, set acl.oidc.host and the endpoint settings acl.oidc.configuration.url=https://idp.example.com/.well-known/openid-configuration # the client ID that the provider assigned to QuestDB @@ -53,16 +56,55 @@ acl.oidc.groups.claim=groups # acl.oidc.sub.claim=sub # optional, false by default: read the claims from a JWT, such as the ID token, -# instead of calling the User Info endpoint +# instead of calling the User Info endpoint; required for Microsoft Entra ID # acl.oidc.groups.encoded.in.token=true ``` +QuestDB downloads the discovery document when it starts, and does not start +when it cannot download or parse it. The error starts with +`Unable to download OIDC provider configuration from` or +`Unable to parse OIDC provider configuration`. To let QuestDB start while the +provider is unreachable, set `acl.oidc.host` and the +[endpoint settings](/docs/configuration/oidc/#endpoints) instead: QuestDB then +starts without the provider, and only OIDC logins fail until the provider is +reachable again. QuestDB refuses to start when both +`acl.oidc.configuration.url` and `acl.oidc.host` are set. + For all OIDC-related configuration options of QuestDB, see [Configuration](/docs/configuration/oidc/). For complete examples, see the QuestDB configuration in the [Microsoft Entra ID](/docs/security/oidc-entra-id/#questdb-configuration) and [PingFederate](/docs/security/oidc-pingfederate/#questdb-configuration) guides. +## Where QuestDB reads the user information + +`acl.oidc.groups.encoded.in.token` decides where QuestDB reads the principal +and the groups of an external user from, and with it, which token a client +sends and what QuestDB checks: + +| | User Info endpoint (default) | Token (`acl.oidc.groups.encoded.in.token=true`) | +| --- | --- | --- | +| Claims come from | the provider's User Info response for the token | the payload of a JWT: the ID token for Web Console and ROPC logins, or the token that a client sends | +| The Web Console sends | the access token | the ID token | +| Other clients send | an access token | for a user, an ID token requested with QuestDB's client ID; for a service, an access token whose `aud` contains `acl.oidc.audience` | +| The token is validated by | the provider's User Info endpoint | QuestDB: signature, `aud`, `exp` (QuestDB Enterprise 4.0.2 and later), and `sub` | +| `acl.oidc.audience` | not used | must be in the `aud` claim | +| A change to the groups applies | after `acl.oidc.cache.ttl` | when the client presents a new token | +| Suits providers that | return the groups from the User Info endpoint, such as PingFederate | put the groups into the tokens, such as Microsoft Entra ID, and issue tokens that services request for themselves | + +The setting applies to every client, the Web Console included. Microsoft Entra +ID does not return the groups from its User Info endpoint, so with Entra ID, +set `acl.oidc.groups.encoded.in.token=true`: with the default, QuestDB rejects +every login. A provider that returns the groups only from its User Info +endpoint needs the default, unless you also configure it to put the groups into +the ID token. + +For which token each client sends, see +[Non-interactive clients](#non-interactive-clients). For what QuestDB checks, +see [Token validation](#token-validation), and for when a change in the +Identity Provider takes effect, see +[Group changes, sessions, and revoking access](#group-changes-sessions-and-revoking-access). + ## Architecture overview Altogether, the architecture appears as such: @@ -807,8 +849,7 @@ authenticates with it: A long-running client must therefore switch to a new token before the old one expires: request a new token, create a new client with it, then close the old -client. For a Python example, see -[Request a token in the service](/docs/security/oidc-entra-id/#request-a-token-in-the-service). +client. A client that keeps its [store-and-forward](/docs/connect/clients/connect-string/#sf-keys) data on @@ -818,6 +859,12 @@ start while the old client is open. Flush and close the old client first, then create the new client with the new token and the same `sender_id`. The new client replays what the old client left on disk, as described in [Sender restart and replay](/docs/connect/clients/connect-string/#sender-restart-and-replay). +The old client is closed by then, so when the new client fails to start, for +example while QuestDB is unreachable, retry it instead of falling back to the +old client, and request a current token for each attempt, because an outage +can outlast the token. +[Request a token in the service](/docs/security/oidc-entra-id/#request-a-token-in-the-service) +shows both orders in Python. ## OIDC for the PGWire endpoint @@ -1101,7 +1148,13 @@ way. For example, Microsoft Entra ID user tokens, configured as in the carry `oid` and `roles`: ```ini title="server.conf" +# Entra ID puts the groups and the app roles into the tokens +acl.oidc.groups.encoded.in.token=true + +# user tokens carry preferred_username, app-only tokens carry oid acl.oidc.sub.claim=preferred_username,oid + +# app-only tokens carry roles, user tokens carry groups acl.oidc.groups.claim=roles,groups ``` From 62a29bf74c606b64024297579d3ac62e16fca75f Mon Sep 17 00:00:00 2001 From: glasstiger Date: Fri, 9 Oct 2026 19:47:41 +0100 Subject: [PATCH 12/29] docs(oidc): move the OIDC pages under security/oidc/ and document expired Web Console ID tokens - Move the OIDC overview and the Entra ID and PingFederate guides to security/oidc/{index,entra-id,pingfederate}.mdx, the paths that #522 uses, so neither PR moves a published URL again. The overview stays at /docs/security/oidc/. - Add "When QuestDB stops accepting a token" under Token validation as the one place for the expiry window, and link to it from Token lifetime, Group changes and the Entra guide instead of restating it. - Document that the Web Console refreshes on the access token's expiry but sends the ID token when the groups are in the token, so with Entra ID's default lifetimes 4.0.2 logs ExpiredSignature and accepts the requests through the session. Covered in the new section, step 6, Troubleshooting, When you upgrade, and the Entra guide. - Give the Python client page the same 4.0.2 / User Info endpoint qualifier as the C/C++, .NET and Go pages. - Fix the Enable ROPC example, which used parse and json without importing them. Co-Authored-By: Claude Opus 5.5 --- documentation/changelog.mdx | 6 +- documentation/connect/clients/c-and-cpp.md | 2 +- documentation/connect/clients/dotnet.md | 2 +- documentation/connect/clients/go.md | 2 +- documentation/connect/clients/python.md | 4 +- documentation/deployment/azure.md | 4 +- .../{oidc-entra-id.md => oidc/entra-id.mdx} | 16 +- .../security/{oidc.mdx => oidc/index.mdx} | 146 +++++++++++------- .../pingfederate.mdx} | 0 documentation/security/rbac.md | 2 +- documentation/sidebars.js | 8 +- 11 files changed, 119 insertions(+), 73 deletions(-) rename documentation/security/{oidc-entra-id.md => oidc/entra-id.mdx} (97%) rename documentation/security/{oidc.mdx => oidc/index.mdx} (91%) rename documentation/security/{oidc-pingfederate.md => oidc/pingfederate.mdx} (100%) diff --git a/documentation/changelog.mdx b/documentation/changelog.mdx index 9db2172c4c..f66ec68b01 100644 --- a/documentation/changelog.mdx +++ b/documentation/changelog.mdx @@ -12,7 +12,7 @@ This page tracks significant updates to the QuestDB documentation. ### New -- [Microsoft Entra ID](/docs/security/oidc-entra-id/) - New page for Microsoft Entra ID: the Web Console sign-in guide, moved from the OIDC page, and app-only tokens for Azure managed identities and service principals in QuestDB Enterprise 4.0.2, with an app role and a QuestDB group for each service, how to keep app roles the only way in, how to cut off one service, and Python examples that ingest and renew the token, also with store-and-forward on disk +- [Microsoft Entra ID](/docs/security/oidc/entra-id/) - New page for Microsoft Entra ID: the Web Console sign-in guide, moved from the OIDC page, and app-only tokens for Azure managed identities and service principals in QuestDB Enterprise 4.0.2, with an app role and a QuestDB group for each service, how to keep app roles the only way in, how to cut off one service, and Python examples that ingest and renew the token, also with store-and-forward on disk - [Where QuestDB reads the user information](/docs/security/oidc/#where-questdb-reads-the-user-information) - Compares the User Info endpoint with reading the claims from the token: which token each client sends, what QuestDB checks, and when a change to the groups takes effect - [User and group claims](/docs/security/oidc/#user-and-group-claims) - How QuestDB Enterprise 4.0.2 reads the principal and the groups from the user information: fallback claim lists, the rules for missing, `null`, and empty claims, how to choose the principal claim, and the startup checks - [Token validation](/docs/security/oidc/#token-validation), [Troubleshooting OIDC logins](/docs/security/oidc/#troubleshooting-oidc-logins), and [Versions before 4.0.2](/docs/security/oidc/#versions-before-402) - What QuestDB checks in a token, including that it does not check the audience of a token that it sends to the User Info endpoint, how long it accepts a token after it expires, the log messages of rejected logins, how earlier versions differ, and what to check when you upgrade @@ -32,11 +32,11 @@ This page tracks significant updates to the QuestDB documentation. ### Updated - [OpenID Connect (OIDC)](/docs/security/oidc/) - The identity provider guides moved to their own pages, under OpenID Connect (OIDC) in the sidebar, the introduction lists what the page covers, and [Configuration options](/docs/security/oidc/#configuration-options), right after the introduction, shows a minimal configuration -- [PingFederate](/docs/security/oidc-pingfederate/) - Moved from the OIDC page to its own page, and added the [QuestDB configuration](/docs/security/oidc-pingfederate/#questdb-configuration) +- [PingFederate](/docs/security/oidc/pingfederate/) - Moved from the OIDC page to its own page, and added the [QuestDB configuration](/docs/security/oidc/pingfederate/#questdb-configuration) - [Mapping user permissions](/docs/security/oidc/#mapping-user-permissions) - Corrected what happens when the groups claim is missing or empty: QuestDB rejects the login. A user is authenticated without permissions only when none of their groups is mapped to a QuestDB group. Added the endpoint permission that each interface needs, and how external aliases match - [Find user information](/docs/security/oidc/#8-find-user-information) and [non-interactive clients](/docs/security/oidc/#non-interactive-clients) - Corrected which token of a user reaches QuestDB with `acl.oidc.groups.encoded.in.token=true`: the ID token rather than the access token. Non-interactive clients also covers the tokens that services obtain for themselves, with client credentials and password grant examples over QWP - [Endpoint permissions](/docs/security/rbac/#endpoint-permissions) - The `HTTP` permission also covers QWP over WebSocket. The [authentication methods](/docs/security/rbac/#authentication) and [users and service accounts](/docs/security/rbac/#users-and-service-accounts) describe external users, who log in through OIDC -- [Microsoft Entra ID guide](/docs/security/oidc-entra-id/#web-console-sign-in) - Uses `preferred_username` as the principal claim instead of the display name, explains when to use `oid` instead, and registers the application as single-tenant +- [Microsoft Entra ID guide](/docs/security/oidc/entra-id/#web-console-sign-in) - Uses `preferred_username` as the principal claim instead of the display name, explains when to use `oid` instead, and registers the application as single-tenant - [Python client](/docs/connect/clients/python/#authentication-and-tls) - Corrected when bad credentials raise: by default, `connect()` opens its first connections and raises, and `lazy_connect=on` defers the error to the first operation. Corrected that a terminal server rejection, such as a missing grant, raises `QuestDBServerRejectionError` from `flush(wait=True)`, as described in [Flushing and acknowledgements](/docs/connect/clients/python/#flushing-and-acknowledgements) - [C/C++ client](/docs/connect/clients/c-and-cpp/#connecting) - Corrected when connection and authentication errors surface: by default, the pool connects when it is created and the errors surface there, and `lazy_connect=on` defers them - [Python](/docs/connect/clients/python/), [C/C++](/docs/connect/clients/c-and-cpp/), [.NET](/docs/connect/clients/dotnet/), and [Go](/docs/connect/clients/go/) clients - Corrected token rotation: build a new client with a new token before the old token expires, then close the old one, or close the old one first when it uses `sf_dir` diff --git a/documentation/connect/clients/c-and-cpp.md b/documentation/connect/clients/c-and-cpp.md index 9313c2a4ca..215a1b23b3 100644 --- a/documentation/connect/clients/c-and-cpp.md +++ b/documentation/connect/clients/c-and-cpp.md @@ -340,7 +340,7 @@ following are **not** supported: | Path | Status | Workaround | |---|---|---| -| OIDC token acquisition or in-band refresh | Not supported. The client does not negotiate with an identity provider and has no callback to refresh a token mid-session. | QuestDB itself supports OIDC; see [OpenID Connect](/docs/security/oidc/#non-interactive-clients) for which token to send. Acquire the token out-of-band from your IdP, pass it via `token=...`, and rebuild the pool when the token nears expiry. For Azure services, see [Microsoft Entra ID managed identities and service principals](/docs/security/oidc-entra-id/#managed-identities-and-service-principals). | +| OIDC token acquisition or in-band refresh | Not supported. The client does not negotiate with an identity provider and has no callback to refresh a token mid-session. | QuestDB itself supports OIDC; see [OpenID Connect](/docs/security/oidc/#non-interactive-clients) for which token to send. Acquire the token out-of-band from your IdP, pass it via `token=...`, and rebuild the pool when the token nears expiry. For Azure services, see [Microsoft Entra ID managed identities and service principals](/docs/security/oidc/entra-id/#managed-identities-and-service-principals). | | Mutual TLS (client certificates) | Not supported. The QuestDB server does not negotiate client certificates regardless of client. | Use bearer-token auth over `wss`. See the connect-string reference's [TLS section](/docs/connect/clients/connect-string/#tls). | | Token rotation mid-session | Not supported. The pool sends the credentials in the WebSocket upgrade of every connection that it opens, including reconnects, and cannot switch to a new token while it runs. | Before the token expires, build a new pool with a new token, then close the old one. With `sf_dir`, flush and close the old pool first, then build the new one with the same `sender_id`: two pools cannot hold the same store-and-forward slot, and the new pool replays what the old one left on disk. On QuestDB Enterprise 4.0.2 and later, or with the User Info endpoint, QuestDB rejects the token on new connections shortly after it expires; see [Token lifetime](/docs/security/oidc/#token-lifetime). | diff --git a/documentation/connect/clients/dotnet.md b/documentation/connect/clients/dotnet.md index 17e3b0cb9b..805502f12c 100644 --- a/documentation/connect/clients/dotnet.md +++ b/documentation/connect/clients/dotnet.md @@ -228,7 +228,7 @@ that purpose; left unset, it inherits `auth_timeout_ms`. | Path | Status | Workaround | |---|---|---| -| OIDC token acquisition or in-band refresh | Not supported by this client. It does not negotiate with an identity provider and has no callback to refresh a token mid-session. | QuestDB itself supports OIDC; see [OpenID Connect](/docs/security/oidc/#non-interactive-clients) for which token to send. Acquire the token out-of-band from your IdP, pass it via `token=...` above, and rebuild the sender / query client when the token nears expiry. For Azure services, see [Microsoft Entra ID managed identities and service principals](/docs/security/oidc-entra-id/#managed-identities-and-service-principals). | +| OIDC token acquisition or in-band refresh | Not supported by this client. It does not negotiate with an identity provider and has no callback to refresh a token mid-session. | QuestDB itself supports OIDC; see [OpenID Connect](/docs/security/oidc/#non-interactive-clients) for which token to send. Acquire the token out-of-band from your IdP, pass it via `token=...` above, and rebuild the sender / query client when the token nears expiry. For Azure services, see [Microsoft Entra ID managed identities and service principals](/docs/security/oidc/entra-id/#managed-identities-and-service-principals). | | Mutual TLS (client certificates) | Not supported. The QuestDB server does not negotiate client certificates regardless of client. | Use bearer-token auth over `wss://`. See the connect-string reference for the canonical statement. | | Token rotation mid-session | Not supported. The client sends the credentials in the WebSocket upgrade of every connection that it opens, including reconnects, and cannot switch to a new token while it runs. | Before the token expires, build a new sender with a new token, call `SendAsync()` on the old sender, because dispose does not send, then `await` its `DisposeAsync()`. With `sf_dir`, call `SendAsync()` on the old sender and dispose of it first, then build the new one with the same `sender_id`: two senders cannot hold the same store-and-forward slot, and the new sender replays what the old one left on disk. Replace a `QueryClient` the same way: build the new one, then dispose of the old one. On QuestDB Enterprise 4.0.2 and later, or with the User Info endpoint, QuestDB rejects the token on new connections shortly after it expires; see [Token lifetime](/docs/security/oidc/#token-lifetime). | diff --git a/documentation/connect/clients/go.md b/documentation/connect/clients/go.md index d8543f0304..7b5fda6d34 100644 --- a/documentation/connect/clients/go.md +++ b/documentation/connect/clients/go.md @@ -363,7 +363,7 @@ client sends exactly the string you pass and never refreshes or renews it. Acquire it out of band, for example from the identity provider of the QuestDB Enterprise [OpenID Connect flow](/docs/security/oidc/#non-interactive-clients), and manage its lifetime yourself. For Azure services, see -[Microsoft Entra ID managed identities and service principals](/docs/security/oidc-entra-id/#managed-identities-and-service-principals). +[Microsoft Entra ID managed identities and service principals](/docs/security/oidc/entra-id/#managed-identities-and-service-principals). Before the token expires, construct a new handle with a new token, then close the old one. With `sf_dir`, flush and close the old handle first, then construct the new one with the same `sender_id`: two handles cannot hold the diff --git a/documentation/connect/clients/python.md b/documentation/connect/clients/python.md index cb1d18a3b3..f5656424be 100644 --- a/documentation/connect/clients/python.md +++ b/documentation/connect/clients/python.md @@ -179,9 +179,9 @@ following are **not** supported: | Path | Status | Workaround | | --- | --- | --- | -| OIDC token acquisition or in-band refresh | Not supported. The client does not negotiate with an identity provider and cannot refresh a token mid-session. | QuestDB itself supports OIDC; see [OpenID Connect](/docs/security/oidc/#non-interactive-clients) for which token to send. Acquire the token out-of-band from your IdP, pass it via `token=...`, and rebuild the handle when the token nears expiry. For an example with an Entra ID managed identity, see [Microsoft Entra ID managed identities and service principals](/docs/security/oidc-entra-id/#managed-identities-and-service-principals). | +| OIDC token acquisition or in-band refresh | Not supported. The client does not negotiate with an identity provider and cannot refresh a token mid-session. | QuestDB itself supports OIDC; see [OpenID Connect](/docs/security/oidc/#non-interactive-clients) for which token to send. Acquire the token out-of-band from your IdP, pass it via `token=...`, and rebuild the handle when the token nears expiry. For an example with an Entra ID managed identity, see [Microsoft Entra ID managed identities and service principals](/docs/security/oidc/entra-id/#managed-identities-and-service-principals). | | Mutual TLS (client certificates) | Not supported. The QuestDB server does not negotiate client certificates regardless of client. | Use bearer-token auth over `wss`. | -| Token rotation mid-session | Not supported. The handle keeps the credentials it was built with and presents them on every connection it opens, including reconnects and failover, so an expired token also breaks mid-session reconnection. | Before the token expires, build a fresh handle with a new token, then close the old one. With `sf_dir`, flush and close the old handle first, then build the new one with the same `sender_id`: two handles cannot hold the same store-and-forward slot, and the new handle replays what the old one left on disk. See [Token lifetime](/docs/security/oidc/#token-lifetime) for how QuestDB treats an expiring token, and [Request a token in the service](/docs/security/oidc-entra-id/#request-a-token-in-the-service) for an example. | +| Token rotation mid-session | Not supported. The handle keeps the credentials it was built with and presents them on every connection it opens, including reconnects and failover, and cannot switch to a new token while it runs. | Before the token expires, build a fresh handle with a new token, then close the old one. With `sf_dir`, flush and close the old handle first, then build the new one with the same `sender_id`: two handles cannot hold the same store-and-forward slot, and the new handle replays what the old one left on disk. On QuestDB Enterprise 4.0.2 and later, or with the User Info endpoint, QuestDB rejects the token on new connections shortly after it expires, so reconnects and failover then fail too; see [Token lifetime](/docs/security/oidc/#token-lifetime). For an example, see [Request a token in the service](/docs/security/oidc/entra-id/#request-a-token-in-the-service). | ## The pool diff --git a/documentation/deployment/azure.md b/documentation/deployment/azure.md index 62ec329bbb..0a74607205 100644 --- a/documentation/deployment/azure.md +++ b/documentation/deployment/azure.md @@ -345,11 +345,11 @@ QuestDB Enterprise adds production features for Azure: - **Entra ID SSO** - Single sign-on with Microsoft Entra ID For Entra ID integration, see the -[Microsoft Entra ID guide](/docs/security/oidc-entra-id/). +[Microsoft Entra ID guide](/docs/security/oidc/entra-id/). Since QuestDB Enterprise 4.0.2, Azure services that run as a managed identity or a service principal can also authenticate to QuestDB with Entra ID app-only tokens. See -[Managed identities and service principals](/docs/security/oidc-entra-id/#managed-identities-and-service-principals). +[Managed identities and service principals](/docs/security/oidc/entra-id/#managed-identities-and-service-principals). See [Enterprise Quick Start](/docs/getting-started/enterprise-quick-start/) for setup. diff --git a/documentation/security/oidc-entra-id.md b/documentation/security/oidc/entra-id.mdx similarity index 97% rename from documentation/security/oidc-entra-id.md rename to documentation/security/oidc/entra-id.mdx index 0078755782..55c8171273 100644 --- a/documentation/security/oidc-entra-id.md +++ b/documentation/security/oidc/entra-id.mdx @@ -324,6 +324,19 @@ To test, head to the Web Console and login. If all has been wired up well, then login will succeed, and the user will have the access granted to them. +:::note + +In QuestDB Enterprise 4.0.2 and later, the server log can show +`Failed to decode JWT token Error(ExpiredSignature)` for Web Console users. +By default, Entra ID issues ID tokens that are valid for one hour, and access +tokens that are valid for 60 to 90 minutes. The Web Console requests new +tokens shortly before the access token expires, so it can keep sending an +expired ID token for up to half an hour. QuestDB accepts these requests +through the session of the Web Console. See +[When QuestDB stops accepting a token](/docs/security/oidc/#when-questdb-stops-accepting-a-token). + +::: + ## Managed identities and service principals Azure services that run as a managed identity or a service principal can @@ -538,7 +551,8 @@ The client presents the same token on every connection that it opens: the first one, the additional connections of its pool, and the connections that replace a failed one. Shortly after the token expires, QuestDB rejects these new connections, as described in -[Token validation](/docs/security/oidc/#token-validation). A long-running +[When QuestDB stops accepting a token](/docs/security/oidc/#when-questdb-stops-accepting-a-token). +A long-running service must therefore switch to a new token before the old one expires: it connects with the new token, then closes the old handle. `token.expires_on` is a Unix timestamp. diff --git a/documentation/security/oidc.mdx b/documentation/security/oidc/index.mdx similarity index 91% rename from documentation/security/oidc.mdx rename to documentation/security/oidc/index.mdx index 8992ba8888..d09e03c483 100644 --- a/documentation/security/oidc.mdx +++ b/documentation/security/oidc/index.mdx @@ -1,5 +1,6 @@ --- title: OpenID Connect (OIDC) Integration +slug: /security/oidc description: "Configure OIDC in QuestDB Enterprise: Web Console SSO, token authentication for clients and services, claim mapping, token validation, and troubleshooting." --- @@ -29,9 +30,9 @@ accept user and app-only tokens with one configuration, [token validation](#token-validation), and [troubleshooting rejected logins](#troubleshooting-oidc-logins). For step-by-step setup, see the guides for -[Microsoft Entra ID](/docs/security/oidc-entra-id/), which also covers Azure +[Microsoft Entra ID](/docs/security/oidc/entra-id/), which also covers Azure managed identities and service principals, and -[PingFederate](/docs/security/oidc-pingfederate/). +[PingFederate](/docs/security/oidc/pingfederate/). ## Configuration options @@ -73,8 +74,8 @@ reachable again. QuestDB refuses to start when both For all OIDC-related configuration options of QuestDB, see [Configuration](/docs/configuration/oidc/). For complete examples, see the QuestDB configuration in the -[Microsoft Entra ID](/docs/security/oidc-entra-id/#questdb-configuration) and -[PingFederate](/docs/security/oidc-pingfederate/#questdb-configuration) guides. +[Microsoft Entra ID](/docs/security/oidc/entra-id/#questdb-configuration) and +[PingFederate](/docs/security/oidc/pingfederate/#questdb-configuration) guides. ## Where QuestDB reads the user information @@ -130,12 +131,11 @@ In OAuth2/OIDC terms, the [Web Console](/docs/getting-started/web-console/overvi the _client_, and it is assigned an identifier: the **Client Id**. Each application which integrates via OIDC should be given a different **Client -Id**. Clients that send ID tokens to QuestDB are the exception: QuestDB accepts -an ID token only when it was issued for QuestDB's client ID, as described in -[Non-interactive clients](#non-interactive-clients). Separate client IDs do not -limit which access tokens QuestDB accepts when it reads the user information -from the User Info endpoint, the default. See -[Token validation](#token-validation). +Id**, except clients that send ID tokens to QuestDB: they request them with +QuestDB's client ID, as described in +[Non-interactive clients](#non-interactive-clients). With the User Info +endpoint, the default, separate client IDs do not restrict which access tokens +QuestDB accepts; see [Token validation](#token-validation). ### OIDC Provider @@ -293,8 +293,11 @@ If the PKCE check is passed, the Web Console receives the ID and access tokens. There is a third token in the response too, the refresh token. -The refresh token is used by the Web Console to refresh the access token before -it expires. +The Web Console uses the refresh token to request new tokens shortly before the +access token expires. With `acl.oidc.groups.encoded.in.token=true`, it sends +QuestDB the ID token rather than the access token, and the ID token can expire +first, as described in +[When QuestDB stops accepting a token](#when-questdb-stops-accepting-a-token). Without the refresh token mechanism, the user would be forced to re-authenticate when the access token expires. @@ -564,8 +567,9 @@ the OAuth2 provider. ```python from dotenv import load_dotenv import os -from urllib import request +from urllib import request, parse import base64 +import json load_dotenv() user = os.environ.get("username") @@ -647,7 +651,7 @@ and on who the client authenticates as: client: the Web Console then sends the ID token, and QuestDB reads the principal and the groups from it. A provider that returns the groups only from its User Info endpoint, as set up in the - [PingFederate guide](/docs/security/oidc-pingfederate/), must also put them + [PingFederate guide](/docs/security/oidc/pingfederate/), must also put them into the ID token, or QuestDB rejects every Web Console login. - With `acl.oidc.groups.encoded.in.token=true`, QuestDB validates the token itself, so send a JWT that passes [Token validation](#token-validation): @@ -660,7 +664,7 @@ and on who the client authenticates as: for QuestDB. Configure the provider so that the `aud` claim of the token contains the value of `acl.oidc.audience`, which defaults to QuestDB's client ID. For Microsoft Entra ID, see - [Managed identities and service principals](/docs/security/oidc-entra-id/#managed-identities-and-service-principals). + [Managed identities and service principals](/docs/security/oidc/entra-id/#managed-identities-and-service-principals). Tokens that the provider issues to a service often carry the principal and the groups under different claim names than the tokens of users. To accept both @@ -743,7 +747,7 @@ The service gets the permissions of the QuestDB groups that the groups or roles in its token map to, as described in [Mapping user permissions](#mapping-user-permissions). For Microsoft Entra ID, see -[Managed identities and service principals](/docs/security/oidc-entra-id/#managed-identities-and-service-principals), +[Managed identities and service principals](/docs/security/oidc/entra-id/#managed-identities-and-service-principals), which requests the token without a client secret. ### Resource Owner Password Credentials grant @@ -833,19 +837,12 @@ with questdb.connect(conf) as db: A client configured with a fixed token, such as with the `token` setting of its configuration string, sends that token on every connection that it opens -and never refreshes it. QuestDB checks a token when a request or a connection -authenticates with it: - -- A connection that is already open, such as a QWP or PGWire connection, - stays open when its token expires. -- Every HTTP request, and every new connection, including a reconnect, a - failover, and an additional connection of a client's pool, authenticates - with the token again. With `acl.oidc.groups.encoded.in.token=true`, QuestDB - Enterprise 4.0.2 and later reject the token from up to 60 seconds plus - `acl.oidc.cache.ttl` after it expires, as described in - [Token validation](#token-validation). Earlier versions do not check whether - the token has expired. With the User Info endpoint, the provider rejects the - expired token once the user information that QuestDB cached for it expires. +and never refreshes it. A connection that is already open, such as a QWP or +PGWire connection, stays open when its token expires. Every HTTP request and +every new connection, including a reconnect, a failover, and an additional +connection of a client's pool, authenticates with the token again, and fails +once QuestDB stops accepting the token, as described in +[When QuestDB stops accepting a token](#when-questdb-stops-accepting-a-token). A long-running client must therefore switch to a new token before the old one expires: request a new token, create a new client with it, then close the old @@ -863,7 +860,7 @@ The old client is closed by then, so when the new client fails to start, for example while QuestDB is unreachable, retry it instead of falling back to the old client, and request a current token for each attempt, because an outage can outlast the token. -[Request a token in the service](/docs/security/oidc-entra-id/#request-a-token-in-the-service) +[Request a token in the service](/docs/security/oidc/entra-id/#request-a-token-in-the-service) shows both orders in Python. ## OIDC for the PGWire endpoint @@ -977,9 +974,9 @@ including its case. The group names depend on the provider: for example, the distinguished names of Active Directory groups, as above, or, with Microsoft Entra ID, the object IDs of security groups and the values of app roles, as described in -[Map groups and grant permissions](/docs/security/oidc-entra-id/#map-groups-and-grant-permissions) +[Map groups and grant permissions](/docs/security/oidc/entra-id/#map-groups-and-grant-permissions) and -[Map app roles to QuestDB groups](/docs/security/oidc-entra-id/#map-app-roles-to-questdb-groups). +[Map app roles to QuestDB groups](/docs/security/oidc/entra-id/#map-app-roles-to-questdb-groups). If none of the user's groups is mapped to a QuestDB group, the user is authenticated, but has no permissions at all. @@ -1011,10 +1008,9 @@ such as a disabled user, when it checks a token of the user again: to your organization's policies. - With `acl.oidc.groups.encoded.in.token=true`, when the client presents a new token. QuestDB reads the groups from the token instead of asking the Identity - Provider, so refreshing the cache does not pick up changes. In QuestDB - Enterprise 4.0.2 and later, QuestDB accepts the old token for up to 60 - seconds plus `acl.oidc.cache.ttl` after it expires. Earlier versions do not - check whether the token has expired. See [Token validation](#token-validation). + Provider, so refreshing the cache does not pick up changes, and the old token + keeps its groups until QuestDB stops accepting it, as described in + [When QuestDB stops accepting a token](#when-questdb-stops-accepting-a-token). For a username and password login through the [ROPC flow](#enable-ropc), QuestDB requests a new token from the provider itself once the cached result @@ -1142,9 +1138,9 @@ value as one claim name, so a list makes every OIDC login fail. Tokens issued to users and tokens issued to applications often differ this way. For example, Microsoft Entra ID user tokens, configured as in the -[Entra ID guide](/docs/security/oidc-entra-id/#web-console-sign-in), carry +[Entra ID guide](/docs/security/oidc/entra-id/#web-console-sign-in), carry `preferred_username` and `groups`, while its -[app-only tokens](/docs/security/oidc-entra-id/#managed-identities-and-service-principals) +[app-only tokens](/docs/security/oidc/entra-id/#managed-identities-and-service-principals) carry `oid` and `roles`: ```ini title="server.conf" @@ -1188,7 +1184,7 @@ token payloads, shortened to the relevant claims, as follows: `roles` comes before `groups` because an app-only token can also carry a `groups` claim, and QuestDB takes the groups from the first claim on the list that holds one. See -[Managed identities and service principals](/docs/security/oidc-entra-id/#managed-identities-and-service-principals). +[Managed identities and service principals](/docs/security/oidc/entra-id/#managed-identities-and-service-principals). ### How QuestDB picks a claim @@ -1270,20 +1266,7 @@ accepts any token that the provider's keys sign for the expected audience, so make sure that no one outside your organization can obtain tokens for the QuestDB application. With Microsoft Entra ID, register the application as single-tenant, as described in -[Set up the client application in Entra ID](/docs/security/oidc-entra-id/#set-up-the-client-application-in-entra-id). - -QuestDB checks a token when a request or a connection authenticates with it, -and caches the result for up to -[`acl.oidc.cache.ttl`](/docs/configuration/oidc/#acloidccachettl), 30 seconds -by default. Because of the clock skew and the cache, QuestDB can accept a -token, on new connections too, for up to 60 seconds plus `acl.oidc.cache.ttl` -after it expires. After that, it rejects the token on every request and new -connection. An HTTP request that also carries a valid session cookie, such as -a request of the Web Console, is still accepted through the session, and a -PGWire or WebSocket connection that is already open stays open when its token -expires. See -[Group changes, sessions, and revoking access](#group-changes-sessions-and-revoking-access), -and, for clients, [Token lifetime](#token-lifetime). +[Set up the client application in Entra ID](/docs/security/oidc/entra-id/#set-up-the-client-application-in-entra-id). With `acl.oidc.groups.encoded.in.token=false`, QuestDB sends the token to the provider's User Info endpoint, which validates it, and checks nothing else. @@ -1295,6 +1278,44 @@ user. With `acl.oidc.groups.encoded.in.token=true`, QuestDB checks the `aud` claim instead, as described above. See [Find user information](#8-find-user-information). +### When QuestDB stops accepting a token + +QuestDB checks a token when a request or a new connection authenticates with +it, and caches the result for up to +[`acl.oidc.cache.ttl`](/docs/configuration/oidc/#acloidccachettl), 30 seconds +by default: + +- With `acl.oidc.groups.encoded.in.token=true`, QuestDB Enterprise 4.0.2 and + later accept a token for up to 60 seconds, the allowed clock skew, plus + `acl.oidc.cache.ttl` after it expires. After that, they reject it on every + request and new connection, including reconnects. Earlier versions do not + check whether the token has expired. +- With the User Info endpoint, the provider rejects an expired token. QuestDB + asks the provider again once the user information that it cached for the + token expires. + +Some access outlives the token. A PGWire or WebSocket connection, such as a QWP +connection, that is already open stays open when its token expires, and an HTTP +request that also carries a valid session cookie, such as a request of the Web +Console, is accepted through the session. See +[Group changes, sessions, and revoking access](#group-changes-sessions-and-revoking-access). +For what a client must do before its token expires, see +[Token lifetime](#token-lifetime). + +The Web Console requests new tokens shortly before the access token expires, +but with `acl.oidc.groups.encoded.in.token=true`, it sends the ID token. When +the provider issues ID tokens that expire before its access tokens, the Web +Console keeps sending the ID token after it has expired, until it requests new +tokens. Microsoft Entra ID issues such tokens by default: ID tokens are valid +for one hour, and access tokens for a random 60 to 90 minutes. QuestDB +Enterprise 4.0.2 and later then reject the ID token, log +`Failed to decode JWT token Error(ExpiredSignature)`, and accept the request +through the session of the Web Console, which keeps the groups of the last +token that QuestDB accepted. When the session has expired too, for example +because the Web Console sent no request for longer than +`http.session.timeout`, QuestDB rejects the request, and the Web Console logs +the user out. + ## Troubleshooting OIDC logins When the user information carries none of the listed principal claims, or none @@ -1337,11 +1358,19 @@ Request failed [errorMsg=Unable to find public key with key id [...]] `InvalidAudience` usually means that the token was requested for another API, or that Entra ID issued a v1.0 access token. See -[Managed identities and service principals](/docs/security/oidc-entra-id/#managed-identities-and-service-principals). +[Managed identities and service principals](/docs/security/oidc/entra-id/#managed-identities-and-service-principals). Don't set `acl.oidc.audience` to the Application ID URI to work around it: the setting takes a single value, and the ID tokens of Web Console users carry the application ID, so their logins would fail. +`ExpiredSignature` means that the token has expired. A client configured with a +fixed token must switch to a new token before then, as described in +[Token lifetime](#token-lifetime). For Web Console users, the error can also +mean that the Web Console still sends an ID token that has expired, which it +does with Microsoft Entra ID. QuestDB then accepts the requests through the +session of the Web Console. See +[When QuestDB stops accepting a token](#when-questdb-stops-accepting-a-token). + `Unable to find public key with key id` means that none of the signing keys that the provider publishes matches the token. QuestDB downloads the keys again when they are older than @@ -1402,7 +1431,10 @@ When you upgrade from an earlier version: - With `acl.oidc.groups.encoded.in.token=true`, QuestDB rejects expired tokens and tokens without an `exp` claim. A client that keeps using a token after it expires, such as a client configured with a fixed token, must request a new - token before the old one expires. + token before the old one expires. With Microsoft Entra ID, the server log + also shows `ExpiredSignature` errors for Web Console users, whose ID tokens + expire before the Web Console requests new tokens. See + [When QuestDB stops accepting a token](#when-questdb-stops-accepting-a-token). - QuestDB refuses to start with an empty `acl.oidc.sub.claim`, or with a claim that is listed twice or in both settings. See [Startup validation](#startup-validation). @@ -1419,12 +1451,12 @@ When you upgrade from an earlier version: ### Microsoft Entra ID {#microsoft-entraid} -The [Microsoft Entra ID guide](/docs/security/oidc-entra-id/) sets up single +The [Microsoft Entra ID guide](/docs/security/oidc/entra-id/) sets up single sign-on for the Web Console, and app-only tokens for Azure services that run as a managed identity or a service principal. ### PingFederate -The [PingFederate guide](/docs/security/oidc-pingfederate/) sets up single +The [PingFederate guide](/docs/security/oidc/pingfederate/) sets up single sign-on for the Web Console, with PingFederate authenticating users against Microsoft Entra ID over LDAP. diff --git a/documentation/security/oidc-pingfederate.md b/documentation/security/oidc/pingfederate.mdx similarity index 100% rename from documentation/security/oidc-pingfederate.md rename to documentation/security/oidc/pingfederate.mdx diff --git a/documentation/security/rbac.md b/documentation/security/rbac.md index f085f6ce73..14e596686c 100644 --- a/documentation/security/rbac.md +++ b/documentation/security/rbac.md @@ -346,7 +346,7 @@ tokens from an OpenID Connect provider, such as Microsoft Entra ID. They get their permissions from the QuestDB groups that their external groups map to. See [OpenID Connect (OIDC)](/docs/security/oidc/#user-permissions), and for Azure services on QuestDB Enterprise 4.0.2 or later, -[Microsoft Entra ID managed identities and service principals](/docs/security/oidc-entra-id/#managed-identities-and-service-principals). +[Microsoft Entra ID managed identities and service principals](/docs/security/oidc/entra-id/#managed-identities-and-service-principals). Users can have multiple authentication methods enabled simultaneously: diff --git a/documentation/sidebars.js b/documentation/sidebars.js index d4b533d3ed..87e088dc30 100644 --- a/documentation/sidebars.js +++ b/documentation/sidebars.js @@ -729,21 +729,21 @@ module.exports = { collapsed: true, link: { type: "doc", - id: "security/oidc", + id: "security/oidc/index", }, items: [ { - id: "security/oidc", + id: "security/oidc/index", type: "doc", label: "Overview", }, { - id: "security/oidc-entra-id", + id: "security/oidc/entra-id", type: "doc", label: "Microsoft Entra ID", }, { - id: "security/oidc-pingfederate", + id: "security/oidc/pingfederate", type: "doc", label: "PingFederate", }, From d2bc437183d6f9e66cd2c176bd73ccf24e0da3da Mon Sep 17 00:00:00 2001 From: glasstiger Date: Fri, 9 Oct 2026 20:56:43 +0100 Subject: [PATCH 13/29] docs(oidc): fix review findings on token rotation, refresh tokens, PGWire and claim order - Token lifetime and the Python, C/C++ and Go pages: wait until QuestDB has acknowledged what the old client sent before closing it, because a closed client discards the rows that are not acknowledged within close_flush_timeout_millis. - Web Console: it refreshes its tokens before a query only when it holds a refresh token, and logs the user out when the provider refuses. Without a refresh token, a disabled user keeps working through the session. Document offline_access in the minimal configuration and in acl.oidc.scope. - PGWire: which token to send in each mode, that the principal and the groups come from the token, the setting as a server.conf block, and TLS. - Fallback claim lists: a user token that carries roles takes its groups from roles alone, so put the Keycloak realm-role mapper on the client of the service only. Entra guide: user-assignable app roles do the same. - sf_dir: a terminally rejected batch stays in the slot and is sent again before later batches, so don't send it again (Entra guide, Python page). Describe what an ingesting service sees during a revoke. - Entra guide: finish the setup before a managed identity requests its first token, because Azure caches its tokens for about 24 hours. - PingFederate: the LDAP source is Microsoft Entra Domain Services or Active Directory, not Entra ID. Keep the old data source anchor, and fix userPrincipalName. - Rust client: connect is eager by default, as in the 7.x core, and lazy_connect=on defers the errors. - Changelog entries for the above. Co-Authored-By: Claude Opus 5.5 --- documentation/changelog.mdx | 15 +-- documentation/configuration/oidc.md | 8 ++ documentation/connect/clients/c-and-cpp.md | 2 +- documentation/connect/clients/go.md | 7 +- documentation/connect/clients/python.md | 10 +- documentation/connect/clients/rust.md | 28 +++-- documentation/security/oidc/entra-id.mdx | 34 +++++- documentation/security/oidc/index.mdx | 114 ++++++++++++++----- documentation/security/oidc/pingfederate.mdx | 17 +-- 9 files changed, 173 insertions(+), 62 deletions(-) diff --git a/documentation/changelog.mdx b/documentation/changelog.mdx index f66ec68b01..51efafa4a0 100644 --- a/documentation/changelog.mdx +++ b/documentation/changelog.mdx @@ -14,11 +14,11 @@ This page tracks significant updates to the QuestDB documentation. - [Microsoft Entra ID](/docs/security/oidc/entra-id/) - New page for Microsoft Entra ID: the Web Console sign-in guide, moved from the OIDC page, and app-only tokens for Azure managed identities and service principals in QuestDB Enterprise 4.0.2, with an app role and a QuestDB group for each service, how to keep app roles the only way in, how to cut off one service, and Python examples that ingest and renew the token, also with store-and-forward on disk - [Where QuestDB reads the user information](/docs/security/oidc/#where-questdb-reads-the-user-information) - Compares the User Info endpoint with reading the claims from the token: which token each client sends, what QuestDB checks, and when a change to the groups takes effect -- [User and group claims](/docs/security/oidc/#user-and-group-claims) - How QuestDB Enterprise 4.0.2 reads the principal and the groups from the user information: fallback claim lists, the rules for missing, `null`, and empty claims, how to choose the principal claim, and the startup checks +- [User and group claims](/docs/security/oidc/#user-and-group-claims) - How QuestDB Enterprise 4.0.2 reads the principal and the groups from the user information: fallback claim lists, how to order them so that user tokens keep their groups, the rules for missing, `null`, and empty claims, how to choose the principal claim, and the startup checks - [Token validation](/docs/security/oidc/#token-validation), [Troubleshooting OIDC logins](/docs/security/oidc/#troubleshooting-oidc-logins), and [Versions before 4.0.2](/docs/security/oidc/#versions-before-402) - What QuestDB checks in a token, including that it does not check the audience of a token that it sends to the User Info endpoint, how long it accepts a token after it expires, the log messages of rejected logins, how earlier versions differ, and what to check when you upgrade - [Tables created by external users](/docs/security/oidc/#tables-created-by-external-users) - Ingestion fails when it has to create a table or add a column for a user who logs in through OIDC, so create the tables in advance, or create them with SQL and `OWNED BY`, naming one of the user's QuestDB groups -- [Group changes, sessions, and revoking access](/docs/security/oidc/#group-changes-sessions-and-revoking-access) - When a change in the identity provider takes effect, that open connections and the HTTP session of the Web Console keep the groups of the last token that QuestDB accepted, and how to cut off access immediately -- [Token lifetime](/docs/security/oidc/#token-lifetime) - How QuestDB treats the token of a client as it expires, and how to switch a long-running client to a new token before then +- [Group changes, sessions, and revoking access](/docs/security/oidc/#group-changes-sessions-and-revoking-access) - When a change in the identity provider takes effect, that open connections and the HTTP session of the Web Console keep the groups of the last token that QuestDB accepted, that the Web Console logs out a disabled user only when the provider issued it a refresh token, and how to cut off access immediately +- [Token lifetime](/docs/security/oidc/#token-lifetime) - How QuestDB treats the token of a client as it expires, and how to switch a long-running client to a new token before then without losing the rows that QuestDB has not acknowledged ### Reference @@ -32,14 +32,15 @@ This page tracks significant updates to the QuestDB documentation. ### Updated - [OpenID Connect (OIDC)](/docs/security/oidc/) - The identity provider guides moved to their own pages, under OpenID Connect (OIDC) in the sidebar, the introduction lists what the page covers, and [Configuration options](/docs/security/oidc/#configuration-options), right after the introduction, shows a minimal configuration -- [PingFederate](/docs/security/oidc/pingfederate/) - Moved from the OIDC page to its own page, and added the [QuestDB configuration](/docs/security/oidc/pingfederate/#questdb-configuration) +- [PingFederate](/docs/security/oidc/pingfederate/) - Moved from the OIDC page to its own page, and added the [QuestDB configuration](/docs/security/oidc/pingfederate/#questdb-configuration). Corrected the directory that PingFederate reads the users from over LDAP: Microsoft Entra Domain Services or an Active Directory domain, as Microsoft Entra ID has no LDAP endpoint +- [OIDC for the PGWire endpoint](/docs/security/oidc/#oidc-for-the-pgwire-endpoint) - Documented which token to send in the password field, that QuestDB takes the principal and the groups from the token, and that the connection should use TLS - [Mapping user permissions](/docs/security/oidc/#mapping-user-permissions) - Corrected what happens when the groups claim is missing or empty: QuestDB rejects the login. A user is authenticated without permissions only when none of their groups is mapped to a QuestDB group. Added the endpoint permission that each interface needs, and how external aliases match - [Find user information](/docs/security/oidc/#8-find-user-information) and [non-interactive clients](/docs/security/oidc/#non-interactive-clients) - Corrected which token of a user reaches QuestDB with `acl.oidc.groups.encoded.in.token=true`: the ID token rather than the access token. Non-interactive clients also covers the tokens that services obtain for themselves, with client credentials and password grant examples over QWP - [Endpoint permissions](/docs/security/rbac/#endpoint-permissions) - The `HTTP` permission also covers QWP over WebSocket. The [authentication methods](/docs/security/rbac/#authentication) and [users and service accounts](/docs/security/rbac/#users-and-service-accounts) describe external users, who log in through OIDC - [Microsoft Entra ID guide](/docs/security/oidc/entra-id/#web-console-sign-in) - Uses `preferred_username` as the principal claim instead of the display name, explains when to use `oid` instead, and registers the application as single-tenant -- [Python client](/docs/connect/clients/python/#authentication-and-tls) - Corrected when bad credentials raise: by default, `connect()` opens its first connections and raises, and `lazy_connect=on` defers the error to the first operation. Corrected that a terminal server rejection, such as a missing grant, raises `QuestDBServerRejectionError` from `flush(wait=True)`, as described in [Flushing and acknowledgements](/docs/connect/clients/python/#flushing-and-acknowledgements) -- [C/C++ client](/docs/connect/clients/c-and-cpp/#connecting) - Corrected when connection and authentication errors surface: by default, the pool connects when it is created and the errors surface there, and `lazy_connect=on` defers them -- [Python](/docs/connect/clients/python/), [C/C++](/docs/connect/clients/c-and-cpp/), [.NET](/docs/connect/clients/dotnet/), and [Go](/docs/connect/clients/go/) clients - Corrected token rotation: build a new client with a new token before the old token expires, then close the old one, or close the old one first when it uses `sf_dir` +- [Python client](/docs/connect/clients/python/#authentication-and-tls) - Corrected when bad credentials raise: by default, `connect()` opens its first connections and raises, and `lazy_connect=on` defers the error to the first operation. Corrected that a terminal server rejection, such as a missing grant, raises `QuestDBServerRejectionError` from `flush(wait=True)`, as described in [Flushing and acknowledgements](/docs/connect/clients/python/#flushing-and-acknowledgements), and that with `sf_dir` the rejected rows stay on disk and are sent again before later batches, as described in [Server rejections](/docs/connect/clients/python/#server-rejections) +- [C/C++](/docs/connect/clients/c-and-cpp/#connecting) and [Rust](/docs/connect/clients/rust/#connecting) clients - Corrected when connection and authentication errors surface: by default, the pool connects when it is created and the errors surface there, and `lazy_connect=on` defers them +- [Python](/docs/connect/clients/python/), [C/C++](/docs/connect/clients/c-and-cpp/), [.NET](/docs/connect/clients/dotnet/), and [Go](/docs/connect/clients/go/) clients - Corrected token rotation: build a new client with a new token before the old token expires, then close the old one once QuestDB has acknowledged what it sent, or close the old one first when it uses `sf_dir` ## September 2026 diff --git a/documentation/configuration/oidc.md b/documentation/configuration/oidc.md index 1a38dde7fc..876b6ebc80 100644 --- a/documentation/configuration/oidc.md +++ b/documentation/configuration/oidc.md @@ -111,6 +111,14 @@ location where it was loaded from (`window.location.href`). The OIDC server asks consent for the scopes listed in this property. The scope `openid` is mandatory and must always be included. +The Web Console requests these scopes when the user logs in. Add +`offline_access` if the provider issues a refresh token only for that scope, +such as Microsoft Entra ID. Without a refresh token, the Web Console cannot +renew its tokens, and does not log the user out when the user is disabled in +the provider. See +[Credentials received](/docs/security/oidc/#6-credentials-received) in the +OIDC guide. + ## Authentication flows ### acl.oidc.pg.token.as.password.enabled diff --git a/documentation/connect/clients/c-and-cpp.md b/documentation/connect/clients/c-and-cpp.md index 215a1b23b3..fb49ed1853 100644 --- a/documentation/connect/clients/c-and-cpp.md +++ b/documentation/connect/clients/c-and-cpp.md @@ -342,7 +342,7 @@ following are **not** supported: |---|---|---| | OIDC token acquisition or in-band refresh | Not supported. The client does not negotiate with an identity provider and has no callback to refresh a token mid-session. | QuestDB itself supports OIDC; see [OpenID Connect](/docs/security/oidc/#non-interactive-clients) for which token to send. Acquire the token out-of-band from your IdP, pass it via `token=...`, and rebuild the pool when the token nears expiry. For Azure services, see [Microsoft Entra ID managed identities and service principals](/docs/security/oidc/entra-id/#managed-identities-and-service-principals). | | Mutual TLS (client certificates) | Not supported. The QuestDB server does not negotiate client certificates regardless of client. | Use bearer-token auth over `wss`. See the connect-string reference's [TLS section](/docs/connect/clients/connect-string/#tls). | -| Token rotation mid-session | Not supported. The pool sends the credentials in the WebSocket upgrade of every connection that it opens, including reconnects, and cannot switch to a new token while it runs. | Before the token expires, build a new pool with a new token, then close the old one. With `sf_dir`, flush and close the old pool first, then build the new one with the same `sender_id`: two pools cannot hold the same store-and-forward slot, and the new pool replays what the old one left on disk. On QuestDB Enterprise 4.0.2 and later, or with the User Info endpoint, QuestDB rejects the token on new connections shortly after it expires; see [Token lifetime](/docs/security/oidc/#token-lifetime). | +| Token rotation mid-session | Not supported. The pool sends the credentials in the WebSocket upgrade of every connection that it opens, including reconnects, and cannot switch to a new token while it runs. | Before the token expires, build a new pool with a new token, then close the old one once its senders have waited for their acknowledgements (`qwp_sender_wait`, C++ `wait()`): closing a pool discards the rows that QuestDB has not acknowledged within `close_flush_timeout_millis`. With `sf_dir`, flush and close the old pool first, then build the new one with the same `sender_id`: two pools cannot hold the same store-and-forward slot, and the new pool replays what the old one left on disk. On QuestDB Enterprise 4.0.2 and later, or with the User Info endpoint, QuestDB rejects the token on new connections shortly after it expires; see [Token lifetime](/docs/security/oidc/#token-lifetime). | ## Headers diff --git a/documentation/connect/clients/go.md b/documentation/connect/clients/go.md index 7b5fda6d34..d79b84a0dc 100644 --- a/documentation/connect/clients/go.md +++ b/documentation/connect/clients/go.md @@ -364,8 +364,11 @@ Acquire it out of band, for example from the identity provider of the QuestDB En [OpenID Connect flow](/docs/security/oidc/#non-interactive-clients), and manage its lifetime yourself. For Azure services, see [Microsoft Entra ID managed identities and service principals](/docs/security/oidc/entra-id/#managed-identities-and-service-principals). -Before the token expires, construct a new handle with a new token, then close -the old one. With `sf_dir`, flush and close the old handle first, then +Before the token expires, construct a new handle with a new token, wait until +QuestDB has acknowledged what the old handle sent (`FlushAndGetSequence` and +`AwaitAckedFsn`), then close the old one: its final flush is best-effort and +bounded by `close_flush_timeout_millis`. With `sf_dir`, flush and close the old +handle first, then construct the new one with the same `sender_id`: two handles cannot hold the same store-and-forward slot, and the new handle replays what the old one left on disk. The client sends the token on every connection that it opens, diff --git a/documentation/connect/clients/python.md b/documentation/connect/clients/python.md index f5656424be..13bfcbff53 100644 --- a/documentation/connect/clients/python.md +++ b/documentation/connect/clients/python.md @@ -181,7 +181,7 @@ following are **not** supported: | --- | --- | --- | | OIDC token acquisition or in-band refresh | Not supported. The client does not negotiate with an identity provider and cannot refresh a token mid-session. | QuestDB itself supports OIDC; see [OpenID Connect](/docs/security/oidc/#non-interactive-clients) for which token to send. Acquire the token out-of-band from your IdP, pass it via `token=...`, and rebuild the handle when the token nears expiry. For an example with an Entra ID managed identity, see [Microsoft Entra ID managed identities and service principals](/docs/security/oidc/entra-id/#managed-identities-and-service-principals). | | Mutual TLS (client certificates) | Not supported. The QuestDB server does not negotiate client certificates regardless of client. | Use bearer-token auth over `wss`. | -| Token rotation mid-session | Not supported. The handle keeps the credentials it was built with and presents them on every connection it opens, including reconnects and failover, and cannot switch to a new token while it runs. | Before the token expires, build a fresh handle with a new token, then close the old one. With `sf_dir`, flush and close the old handle first, then build the new one with the same `sender_id`: two handles cannot hold the same store-and-forward slot, and the new handle replays what the old one left on disk. On QuestDB Enterprise 4.0.2 and later, or with the User Info endpoint, QuestDB rejects the token on new connections shortly after it expires, so reconnects and failover then fail too; see [Token lifetime](/docs/security/oidc/#token-lifetime). For an example, see [Request a token in the service](/docs/security/oidc/entra-id/#request-a-token-in-the-service). | +| Token rotation mid-session | Not supported. The handle keeps the credentials it was built with and presents them on every connection it opens, including reconnects and failover, and cannot switch to a new token while it runs. | Before the token expires, build a fresh handle with a new token, then close the old one once its senders have flushed with `flush(wait=True)`: closing a handle discards the rows that QuestDB has not acknowledged within `close_flush_timeout_millis`. With `sf_dir`, flush and close the old handle first, then build the new one with the same `sender_id`: two handles cannot hold the same store-and-forward slot, and the new handle replays what the old one left on disk. On QuestDB Enterprise 4.0.2 and later, or with the User Info endpoint, QuestDB rejects the token on new connections shortly after it expires, so reconnects and failover then fail too; see [Token lifetime](/docs/security/oidc/#token-lifetime). For an example, see [Request a token in the service](/docs/security/oidc/entra-id/#request-a-token-in-the-service). | ## The pool @@ -833,6 +833,12 @@ the rejection arrives after that call returned, the next call on the lease raises it. The pool retires the connection instead of lending it out again. Producer-side abort logic belongs with that raised error, not in the handler. +Without `sf_dir`, the rejected rows are dropped. With `sf_dir`, they stay in +the store-and-forward slot, and the next connection that takes over the slot +sends them again before any later batch. Later batches then fail the same way +until QuestDB accepts the rejected rows, for example once you grant a missing +permission, so don't send the rows again. + ### Reader failover Reader failover has a separate policy: @@ -878,7 +884,7 @@ Codes you will most often dispatch on: | `InvalidTimestamp` | Bad `at` value (wrong type, `NaT`). | Pass `TimestampNanos`, a timezone-aware `datetime`, or `ServerTimestamp`. | | `FailoverRetry` | `flush(wait=True)` or `wait()` made no progress within the budget. | Retry the wait; do not re-send the rows. | | `FailoverWouldDuplicate` | Mid-stream failover on an `iter_*` consumer or `polars.DataFrame(result)`. | Discard partial state and rerun the query. | -| `ServerRejection` | Terminal server rejection (schema mismatch, parse error, security error such as a missing grant, ...) latched by the connection; raised by `flush(wait=True)` or `wait()` on the affected lease, or by its next call when the rejection arrives later. Every rejection, terminal or retriable, is also delivered to the [rejection handler](#server-rejections). | Inspect `sender_error`; fix the data or the schema. | +| `ServerRejection` | Terminal server rejection (schema mismatch, parse error, security error such as a missing grant, ...) latched by the connection; raised by `flush(wait=True)` or `wait()` on the affected lease, or by its next call when the rejection arrives later. Every rejection, terminal or retriable, is also delivered to the [rejection handler](#server-rejections). | Inspect `sender_error`; fix the data, the schema, or the grant. With `sf_dir`, don't send the rows again: they stay on disk and are sent first (see [Server rejections](#server-rejections)). | | `ProtocolVersionError` | The server lacks a negotiated capability (for example durable ACK). | Drop the option or upgrade the server. | `QuestDBServerRejectionError` marks a terminal server rejection; its diff --git a/documentation/connect/clients/rust.md b/documentation/connect/clients/rust.md index 20fb11dbb2..fd937d4788 100644 --- a/documentation/connect/clients/rust.md +++ b/documentation/connect/clients/rust.md @@ -125,16 +125,21 @@ use questdb::QuestDb; let db = QuestDb::connect("ws::addr=localhost:9000;")?; ``` -`QuestDb::connect` parses the connect string and doesn't normally perform any -blocking network I/O. The exception to that happens when store-and-forward -durability is in use, and the client detects unsent data left over from a -previously crashed client. The pool may reopen the dirty slots at construction -and start replaying them in the background. - -The ingestion sender creates its local producer on first borrow and connects its +By default, `QuestDb::connect` parses the connect string, then opens the warm +minimum of connections before it returns: `sender_pool_min` senders and +`query_pool_min` readers, one of each by default. An unreachable server, +rejected credentials, or a TLS error therefore fails `connect()` itself. +Senders honour `initial_connect_retry` while they connect, and readers fail +fast. With store-and-forward on disk, the pool also reopens the slots that a +previous client left holding unsent data, and replays them in the background. + +With [`lazy_connect=on`](/docs/connect/clients/connect-string/#pool-keys), +`connect()` performs no blocking network I/O, and `query_pool_min` defaults to +`0`. A borrowed sender then creates its local producer at once and connects its delivery runner in the background, so it can queue data while the server is -temporarily unavailable. Reader errors and direct Arrow/Polars transport errors -surface from the borrow or during operation rather than from `connect()`. +unavailable, and reader errors surface from the first borrow. In both modes, +direct Arrow/Polars transport errors surface from the borrow or during +operation rather than from `connect()`. ## Authentication and TLS @@ -770,14 +775,15 @@ and is not part of that thread-bound borrow list. | Key | Default | Guidance | | --- | --- | --- | -| `sender_pool_min` | `1` | Warm minimum of ingestion connections retained once they have been opened. Set it near steady concurrent use. | +| `sender_pool_min` | `1` | Warm minimum of ingestion connections, opened by `connect()` unless `lazy_connect=on`. Set it near steady concurrent use. | | `sender_pool_max` | `4` | Ingestion pool growth cap. Set it at or above peak concurrent sender borrows. | -| `query_pool_min` | `1` | Warm minimum of reader connections retained once they have been opened. | +| `query_pool_min` | `1`, or `0` with `lazy_connect=on` | Warm minimum of reader connections, opened by `connect()` unless `lazy_connect=on`. | | `query_pool_max` | `4` | Reader pool growth cap. Set it at or above peak concurrent reader borrows. | | `acquire_timeout_ms` | `5000` | How long a borrow waits for a returned connection once its pool is at its cap. `0` fails fast. | | `idle_timeout_ms` | `60000` | Idle lifetime for connections above the warm minimum. | | `pool_reap` | `auto` | Use `manual` only when your application will call `reap_idle()`. | | `initial_connect_retry` | `off` | `on`/`sync` retries initial connection synchronously; `async` starts background retry. | +| `lazy_connect` | `off` | `on` makes `connect()` open no connections, so that it tolerates a server that is down at startup. See [Connecting](#connecting). | Setting an ingress `reconnect_*` key without explicitly setting `initial_connect_retry` promotes the initial mode to synchronous retry. See the diff --git a/documentation/security/oidc/entra-id.mdx b/documentation/security/oidc/entra-id.mdx index 55c8171273..63ad6885f7 100644 --- a/documentation/security/oidc/entra-id.mdx +++ b/documentation/security/oidc/entra-id.mdx @@ -330,8 +330,9 @@ In QuestDB Enterprise 4.0.2 and later, the server log can show `Failed to decode JWT token Error(ExpiredSignature)` for Web Console users. By default, Entra ID issues ID tokens that are valid for one hour, and access tokens that are valid for 60 to 90 minutes. The Web Console requests new -tokens shortly before the access token expires, so it can keep sending an -expired ID token for up to half an hour. QuestDB accepts these requests +tokens, with the refresh token that Entra ID issues for the `offline_access` +scope, when the access token expires within 30 seconds, so it can keep sending +an expired ID token for up to half an hour. QuestDB accepts these requests through the session of the Web Console. See [When QuestDB stops accepting a token](/docs/security/oidc/#when-questdb-stops-accepting-a-token). @@ -412,6 +413,18 @@ the permissions of its Entra ID security groups instead. See described in [Assign a managed identity to an application role](https://learn.microsoft.com/en-us/entra/identity/managed-identities-azure-resources/how-to-assign-app-role-managed-identity). +:::warning + +Complete these steps before a managed identity requests its first token for +QuestDB. Azure caches managed identity tokens for +[around 24 hours](https://learn.microsoft.com/en-us/entra/identity/managed-identities-azure-resources/managed-identity-best-practice-recommendations), +and you cannot make it issue a new token before the cached one expires. A +token that the service requested before the role assignment or the manifest +change carries no role, or the audience of a v1.0 token, and QuestDB rejects +it until Azure issues a new one. + +::: + ### Configure QuestDB On QuestDB Enterprise 4.0.2 or later, replace the two claim settings of the @@ -434,7 +447,10 @@ acl.oidc.groups.claim=roles,groups User logins still take the principal from `preferred_username` and the groups from `groups`. Their tokens carry no `roles` claim, as long as the app roles of -the QuestDB application admit applications only. App-only tokens take the +the QuestDB application admit applications only. If an app role also admits +users and groups, the tokens of the users who hold it carry a `roles` claim, +and QuestDB takes their groups from `roles` alone, without their security +groups. App-only tokens take the principal from `oid` and the groups from `roles`, even when they also carry a `groups` claim. QuestDB rejects a token that carries neither a role nor a group. See @@ -639,6 +655,12 @@ handle. To keep the service running instead, catch the error around the batch, and decide whether to retry or drop the batch: a batch fails the same way until you fix the cause of the rejection, such as a missing grant. +With store-and-forward on disk, described next, the client keeps the rejected +batch instead: it stays in the slot, and the next connection that takes over +the slot, in this handle or a new one, sends it again before any later batch. +Later batches then fail the same way until QuestDB accepts the rejected batch, +for a missing grant once you grant it, so don't send the batch again. + The example keeps its store-and-forward data in memory, the default. With [`sf_dir`](/docs/connect/clients/connect-string/#sf-keys), the client keeps the rows that QuestDB has not acknowledged on disk, in a slot named after its @@ -739,6 +761,12 @@ REVOKE INSERT ON fx_trades FROM fxIngest; REVOKE HTTP FROM fxIngest; ``` +After the revoke, QuestDB rejects the next batch of the service: +`flush(wait=True)` raises `QuestDBServerRejectionError`, and the examples above +stop. Without `sf_dir`, the rejected rows are lost. With `sf_dir`, they stay on +disk, and land once you grant the permissions again and the service connects +with the same `sender_id`. + Granting the permissions again restores access on the connections that are still open, and for every token of the service that QuestDB still accepts. See [Group changes, sessions, and revoking access](/docs/security/oidc/#group-changes-sessions-and-revoking-access). diff --git a/documentation/security/oidc/index.mdx b/documentation/security/oidc/index.mdx index d09e03c483..982cd20aff 100644 --- a/documentation/security/oidc/index.mdx +++ b/documentation/security/oidc/index.mdx @@ -59,6 +59,11 @@ acl.oidc.groups.claim=groups # optional, false by default: read the claims from a JWT, such as the ID token, # instead of calling the User Info endpoint; required for Microsoft Entra ID # acl.oidc.groups.encoded.in.token=true + +# optional, openid by default: the scopes that the Web Console requests; add +# offline_access if the provider issues a refresh token only for that scope, +# such as Microsoft Entra ID, so that the Web Console can renew its tokens +# acl.oidc.scope=openid offline_access ``` QuestDB downloads the discovery document when it starts, and does not start @@ -291,16 +296,24 @@ grant_type=authorization_code&code=1L344XEY5XRka1j4ySNa8bVQSLf71as9uGLEuv_A&clie If the PKCE check is passed, the Web Console receives the ID and access tokens. -There is a third token in the response too, the refresh token. +The response can carry a third token too, the refresh token. Some providers, +such as Microsoft Entra ID, issue it only when the Web Console requests the +`offline_access` scope, which +[`acl.oidc.scope`](/docs/configuration/oidc/#acloidcscope) sets. -The Web Console uses the refresh token to request new tokens shortly before the -access token expires. With `acl.oidc.groups.encoded.in.token=true`, it sends -QuestDB the ID token rather than the access token, and the ID token can expire -first, as described in +Before it sends a query, the Web Console uses the refresh token to request new +tokens when the access token expires within 30 seconds. With +`acl.oidc.groups.encoded.in.token=true`, it sends QuestDB the ID token rather +than the access token, and the ID token can expire first, as described in [When QuestDB stops accepting a token](#when-questdb-stops-accepting-a-token). +When the provider refuses to issue new tokens, for example because the user is +disabled, the Web Console logs the user out. -Without the refresh token mechanism, the user would be forced to re-authenticate -when the access token expires. +Without a refresh token, the Web Console never requests new tokens. Once QuestDB +stops accepting the tokens of the login, it accepts the requests of the Web +Console through its HTTP session, until the session expires and the Web Console +logs the user out, as described in +[Group changes, sessions, and revoking access](#group-changes-sessions-and-revoking-access). The validity of the tokens are configurable inside the OIDC Provider. @@ -845,8 +858,12 @@ once QuestDB stops accepting the token, as described in [When QuestDB stops accepting a token](#when-questdb-stops-accepting-a-token). A long-running client must therefore switch to a new token before the old one -expires: request a new token, create a new client with it, then close the old -client. +expires: request a new token, create a new client with it, wait until the old +client has delivered what it sent, then close the old client. A QWP client that +is closed discards the rows that QuestDB has not acknowledged within +`close_flush_timeout_millis`, unless it keeps them on disk with `sf_dir`, so +wait for the acknowledgement first, for example with `flush(wait=True)` in the +[Python client](/docs/connect/clients/python/#flushing-and-acknowledgements). A client that keeps its [store-and-forward](/docs/connect/clients/connect-string/#sf-keys) data on @@ -866,20 +883,45 @@ shows both orders in Python. ## OIDC for the PGWire endpoint If the -Resource Owner Password Credentials (ROPC) flow is not an option, we can still authenticate via OIDC on the PGWire endpoint. -However, in this case the client's responsibility to source the token required for authentication. -This method works wherever a Postgres client library is available, including jupyter notebooks. +Resource Owner Password Credentials (ROPC) flow is not an option, clients +can still authenticate on the PGWire endpoint with OIDC, using a token that they +request from the provider themselves. This works with any PostgreSQL client +library, including in Jupyter notebooks. + +Enable token authentication for the PGWire endpoint in the server +configuration: + +```ini title="server.conf" +acl.oidc.pg.token.as.password.enabled=true +``` -Token authentication for the PGWire endpoint should be enabled by adding the `acl.oidc.pg.token.as.password.enabled=true` setting to the server configuration. +The client sends the token in the password field, and the string `_sso` in the +username field, or leaves the username empty if it can. QuestDB validates the +token the same way as over HTTP, and takes the principal and the groups from +it, not from the username field. Send the token that an HTTP client would send: -The token should be sent in the password field, while the username field should contain the string `_sso`, or left empty if that is an option: +- With the User Info endpoint, the default, an access token. +- With `acl.oidc.groups.encoded.in.token=true`, for a user, an ID token + requested with QuestDB's client ID, and for a service, an access token whose + `aud` claim contains `acl.oidc.audience`. QuestDB rejects a token for another + audience, such as a user's access token for another API, and logs + `InvalidAudience`. + +See [Non-interactive clients](#non-interactive-clients). Like a password, the +token crosses the network in the clear unless the connection uses +[TLS](/docs/security/tls/#enabling-tls-for-pgwire), so require TLS on the +client: ```python import psycopg as pg +# a token that the client requested from the provider, as described above token = "token_requested_from_the_oauth2_provider" -conn_str = f"user=_sso password={token} host=localhost port=8812 dbname=qdb" +conn_str = ( + f"user=_sso password={token} host=questdb.example.com port=8812 " + "dbname=qdb sslmode=require" +) with pg.connect(conn_str, autocommit=True) as connection: with connection.cursor() as cur: cur.execute('select current_user()') @@ -1030,11 +1072,14 @@ outlives the token: The session ends when the Web Console logs the user out, or after it has been idle for `http.session.timeout`, 30 minutes by default. Every request restarts the idle timer, so a session in use does not expire. The Web - Console logs the user out, and ends the session, when it cannot refresh the - token, for example after the user is disabled. A session that the Web - Console does not end, for example because the browser was closed, stays - valid for any client that reuses its cookie, for as long as that client - stays active. + Console logs the user out, and ends the session, when the provider refuses + to refresh its tokens, for example after the user is disabled. It refreshes + them only with a refresh token, as described in + [step 6](#6-credentials-received): without one, a disabled user keeps + working through the session for as long as they stay active. A session that + the Web Console does not end, for example because the browser was closed, + stays valid for any client that reuses its cookie, for as long as that + client stays active. To cut off access immediately, revoke the permissions of the QuestDB groups that the user's groups map to. This takes effect at once for every session and @@ -1183,7 +1228,12 @@ token payloads, shortened to the relevant claims, as follows: `roles` comes before `groups` because an app-only token can also carry a `groups` claim, and QuestDB takes the groups from the first claim on the list -that holds one. See +that holds one. For the same reason, a user token that carries a `roles` claim +takes its groups from `roles` alone, and the groups in its `groups` claim no +longer count. Entra ID puts no `roles` claim into user tokens as long as the +app roles of the QuestDB application admit applications only. With any +provider, make sure that the first claim on the list appears only in the +tokens that should take their groups from it. See [Managed identities and service principals](/docs/security/oidc/entra-id/#managed-identities-and-service-principals). ### How QuestDB picks a claim @@ -1204,7 +1254,11 @@ QuestDB uses, for a single claim name and for a list: such as the `roles` that Keycloak puts inside its `realm_access` claim. To use such values, configure the provider to emit them as a top-level claim. In Keycloak, for example, add a _User Realm Role_ mapper with the - _Token Claim Name_ `roles` to the client that requests the token. + _Token Claim Name_ `roles` to the client of the service that requests the + token. Don't add it to the client that the Web Console uses, the one that + `acl.oidc.client.id` names: every Keycloak user has the default realm roles, + so with `roles,groups`, the tokens of Web Console users would take their + groups from `roles`. - Claim names are case-sensitive. - A listed claim must have the expected shape: a single value, such as a string, for the principal, and a single value or a flat array of values for @@ -1302,13 +1356,15 @@ Console, is accepted through the session. See For what a client must do before its token expires, see [Token lifetime](#token-lifetime). -The Web Console requests new tokens shortly before the access token expires, -but with `acl.oidc.groups.encoded.in.token=true`, it sends the ID token. When -the provider issues ID tokens that expire before its access tokens, the Web -Console keeps sending the ID token after it has expired, until it requests new -tokens. Microsoft Entra ID issues such tokens by default: ID tokens are valid -for one hour, and access tokens for a random 60 to 90 minutes. QuestDB -Enterprise 4.0.2 and later then reject the ID token, log +The Web Console requests new tokens when the access token expires within 30 +seconds, but with `acl.oidc.groups.encoded.in.token=true`, it sends the ID +token. When the provider issues ID tokens that expire before its access tokens, +the Web Console keeps sending the ID token after it has expired, until it +requests new tokens. Microsoft Entra ID issues such tokens by default: ID +tokens are valid for one hour, and access tokens for a random 60 to 90 minutes. +Without a refresh token, the Web Console never requests new tokens, and keeps +sending the expired token. QuestDB Enterprise 4.0.2 and later then reject the +ID token, log `Failed to decode JWT token Error(ExpiredSignature)`, and accept the request through the session of the Web Console, which keeps the groups of the last token that QuestDB accepted. When the session has expired too, for example diff --git a/documentation/security/oidc/pingfederate.mdx b/documentation/security/oidc/pingfederate.mdx index 21cab0547e..935eed7dbf 100644 --- a/documentation/security/oidc/pingfederate.mdx +++ b/documentation/security/oidc/pingfederate.mdx @@ -15,9 +15,12 @@ This guide sets up single sign-on (SSO) for the QuestDB Enterprise [Web Console](/docs/getting-started/web-console/overview/) with [PingFederate](https://www.pingidentity.com/en/platform/capabilities/authentication-authority/pingfederate.html) as the OpenID Connect (OIDC) provider. PingFederate authenticates users against -Microsoft Entra ID, formerly Azure Active Directory, over a secure LDAP -connection, and passes their group memberships to QuestDB in the User Info -response. QuestDB maps those groups to its own groups to grant permissions. +an Active Directory domain over secure LDAP (LDAPS), and passes their group +memberships to QuestDB in the User Info response. QuestDB maps those groups to +its own groups to grant permissions. The domain in this guide is a +[Microsoft Entra Domain Services](https://learn.microsoft.com/en-us/entra/identity/domain-services/overview) +managed domain with secure LDAP enabled, because Microsoft Entra ID itself has +no LDAP endpoint. An on-premises Active Directory works the same way. For how QuestDB uses OIDC in general, such as the login flow and how it maps groups to permissions, see [OpenID Connect (OIDC)](/docs/security/oidc/). @@ -159,12 +162,12 @@ It is also important to whitelist the [Web Console](/docs/getting-started/web-co width={500} /> -## Set up a Microsoft Entra ID Data Source +## Set up an Active Directory data source {#set-up-a-microsoft-entra-id-data-source} PingFederate needs a Data Source setup. -This is a secure LDAP connection to Microsoft Entra ID, formerly known as Azure -Active Directory. +This is a secure LDAP connection to the Active Directory domain, here a +Microsoft Entra Domain Services managed domain. The data source needs a: @@ -303,7 +306,7 @@ This is also where the directory search parameters are defined. Back at the Attribute Sources & User Lookup Section section, note we have set `email`. -The source is `LDAP (test)`, while the value is `usePrincipalName`: +The source is `LDAP (test)`, while the value is `userPrincipalName`: Date: Wed, 14 Oct 2026 23:03:09 +0100 Subject: [PATCH 14/29] docs(oidc): document issuer validation, session credentials, ID token renewal and Java token providers Align the OIDC, RBAC and client pages with questdb/questdb-enterprise#1267, questdb/questdb#7799, questdb/ui#603 and questdb/java-questdb-client#102. - Issuer validation (QuestDB Enterprise 4.0.2): acl.oidc.issuer and acl.oidc.issuer.validation.enabled, a new "The expected issuer" section with where the issuer comes from, the issuers of Entra ID v1.0 and v2.0 tokens, the common/organizations templates, issuers that depend on the host name of the request, the opt-out and its advisory, the InvalidIssuer and MissingRequiredClaim("iss") log lines, and the upgrade notes. acl.oidc.host needs acl.oidc.issuer unless validation is disabled. PGWire tokens need the expected issuer too. - HTTP sessions: a request with a credential that QuestDB rejects, or cannot check because the provider is unreachable, gets 401 and ends its session. Sessions end with the password or REST token that opened them, or when the user is disabled or dropped, and on a replica when they are next used. acl.oidc.cache.ttl reuses only the check of a password that QuestDB verifies itself, even with OIDC disabled. - Provider outages: with the User Info endpoint, Web Console users are logged out once acl.oidc.cache.ttl has passed; with tokens validated locally, once the ID token expires; ROPC logins need the token endpoint in both modes. - Web Console: it renews the ID token by its exp claim, keeps its token while it cannot reach the provider, and without a refresh token logs the user out when the token of the login expires. Remove the stale "logged out about an hour after the login" text from the OIDC page and the Entra ID guide; Entra ID limits the refresh tokens of single-page applications to 24 hours. Its password login sends a REST token that survives a password change, so drop the user's REST tokens to log the user out. - Java client: "Rotating tokens" with token_provider=azure, the questdb-client-azure artifact, RefreshingTokenProvider and how token failures are handled, "Connection health", the token provider keys in the connect string reference, and a Java example in the Entra ID guide. - With the User Info endpoint, QuestDB treats the access token as opaque. - Changelog entries for the above. Co-Authored-By: Claude Opus 5.5 --- documentation/changelog.mdx | 23 +- documentation/configuration/oidc.md | 100 +++++- .../connect/clients/connect-string.md | 45 +++ documentation/connect/clients/java.md | 200 ++++++++++- documentation/security/oidc/entra-id.mdx | 187 ++++++++-- documentation/security/oidc/index.mdx | 325 +++++++++++++----- documentation/security/rbac.md | 21 ++ 7 files changed, 772 insertions(+), 129 deletions(-) diff --git a/documentation/changelog.mdx b/documentation/changelog.mdx index 51efafa4a0..399c9edcd5 100644 --- a/documentation/changelog.mdx +++ b/documentation/changelog.mdx @@ -12,22 +12,27 @@ This page tracks significant updates to the QuestDB documentation. ### New -- [Microsoft Entra ID](/docs/security/oidc/entra-id/) - New page for Microsoft Entra ID: the Web Console sign-in guide, moved from the OIDC page, and app-only tokens for Azure managed identities and service principals in QuestDB Enterprise 4.0.2, with an app role and a QuestDB group for each service, how to keep app roles the only way in, how to cut off one service, and Python examples that ingest and renew the token, also with store-and-forward on disk +- [Microsoft Entra ID](/docs/security/oidc/entra-id/) - New page for Microsoft Entra ID: the Web Console sign-in guide, moved from the OIDC page, and app-only tokens for Azure managed identities and service principals in QuestDB Enterprise 4.0.2, with an app role and a QuestDB group for each service, how to keep app roles the only way in, how to cut off one service, a Java example in which the client requests and renews the token itself with `token_provider=azure`, and Python examples that ingest and renew the token, also with store-and-forward on disk - [Where QuestDB reads the user information](/docs/security/oidc/#where-questdb-reads-the-user-information) - Compares the User Info endpoint with reading the claims from the token: which token each client sends, what QuestDB checks, and when a change to the groups takes effect - [User and group claims](/docs/security/oidc/#user-and-group-claims) - How QuestDB Enterprise 4.0.2 reads the principal and the groups from the user information: fallback claim lists, how to order them so that user tokens keep their groups, the rules for missing, `null`, and empty claims, how to choose the principal claim, and the startup checks -- [Token validation](/docs/security/oidc/#token-validation), [Troubleshooting OIDC logins](/docs/security/oidc/#troubleshooting-oidc-logins), and [Versions before 4.0.2](/docs/security/oidc/#versions-before-402) - What QuestDB checks in a token, including that it does not check the audience of a token that it sends to the User Info endpoint, how long it accepts a token after it expires, the log messages of rejected logins, how earlier versions differ, and what to check when you upgrade +- [Token validation](/docs/security/oidc/#token-validation), [Troubleshooting OIDC logins](/docs/security/oidc/#troubleshooting-oidc-logins), and [Versions before 4.0.2](/docs/security/oidc/#versions-before-402) - What QuestDB checks in a token, including that it treats the access token as opaque and leaves its validation to the User Info endpoint, the issuer that QuestDB Enterprise 4.0.2 requires in a JWT and where it comes from, how long it accepts a token after it expires, the log messages of rejected logins, how earlier versions differ, and what to check when you upgrade - [Tables created by external users](/docs/security/oidc/#tables-created-by-external-users) - Ingestion fails when it has to create a table or add a column for a user who logs in through OIDC, so create the tables in advance, or create them with SQL and `OWNED BY`, naming one of the user's QuestDB groups -- [Group changes, sessions, and revoking access](/docs/security/oidc/#group-changes-sessions-and-revoking-access) - When a change in the identity provider takes effect, that open connections and the HTTP session of the Web Console keep the groups of the last token that QuestDB accepted, that the Web Console logs out a disabled user only when the provider issued it a refresh token, and how to cut off access immediately -- [Token lifetime](/docs/security/oidc/#token-lifetime) - How QuestDB treats the token of a client as it expires, and how to switch a long-running client to a new token before then without losing the rows that QuestDB has not acknowledged +- [Group changes, sessions, and revoking access](/docs/security/oidc/#group-changes-sessions-and-revoking-access) - When a change in the identity provider takes effect, that open connections keep the groups of the last token that QuestDB accepted, that in QuestDB Enterprise 4.0.2 an HTTP session ends when a request carries a token that QuestDB rejects or cannot check, so the Web Console logs the user out once QuestDB stops accepting its token, also during an outage of the provider, and how to cut off access immediately +- [The expected issuer](/docs/security/oidc/#the-expected-issuer) - QuestDB Enterprise 4.0.2 accepts a JWT only from the issuer of the provider's discovery document, or from `acl.oidc.issuer`: when to set it, including for providers that put the host name of a request into the issuer, the issuers of Microsoft Entra ID v1.0 and v2.0 tokens, and the opt-out for providers that issue tokens without an `iss` claim +- [Token lifetime](/docs/security/oidc/#token-lifetime) - How QuestDB treats the token of a client as it expires, and how to switch a long-running client to a new token before then without losing the rows that QuestDB has not acknowledged, unless the Java client renews the token itself +- [Rotating tokens](/docs/connect/clients/java/#rotating-tokens) - The Java client requests bearer tokens and renews them before they expire with a token provider: `token_provider=azure` and the `questdb-client-azure` artifact for Microsoft Entra ID, `RefreshingTokenProvider` for other identity providers, and how the client handles a token that it cannot get or that QuestDB rejects +- [Connection health](/docs/connect/clients/java/#connection-health) - `QuestDB.health()`, `Sender.health()`, and `QwpQueryClient.health()` of the Java client report whether a client reaches the server, since when it has not, and the last failure ### Reference - [`acl.oidc.groups.claim`](/docs/configuration/oidc/#acloidcgroupsclaim) and [`acl.oidc.sub.claim`](/docs/configuration/oidc/#acloidcsubclaim) - Documented the comma-separated fallback claim lists of QuestDB Enterprise 4.0.2 and the startup validation. `acl.oidc.groups.claim` is now listed with no default, as it must be set when OIDC is enabled. Username and password (ROPC) logins take the principal from the username that the client sends - [`acl.oidc.pkce.required`](/docs/configuration/oidc/#acloidcpkcerequired) - Corrected the property name, previously documented as `acl.oidc.pkce.enabled`, which QuestDB does not recognize -- [`acl.oidc.groups.encoded.in.token`](/docs/configuration/oidc/#acloidcgroupsencodedintoken) - Documented that QuestDB Enterprise 4.0.2 rejects expired tokens and tokens without an `exp` claim, which earlier versions accept, and which token QuestDB reads the principal and the groups from -- [`acl.oidc.audience`](/docs/configuration/oidc/#acloidcaudience) and [`acl.oidc.cache.ttl`](/docs/configuration/oidc/#acloidccachettl) - Documented that `acl.oidc.audience` takes a single value, which the `aud` claim of a token must be or contain, and that QuestDB does not use it with the User Info endpoint, and what the cache holds when the groups are encoded in the token or a user logs in with a username and password +- [`acl.oidc.issuer`](/docs/configuration/oidc/#acloidcissuer) and [`acl.oidc.issuer.validation.enabled`](/docs/configuration/oidc/#acloidcissuervalidationenabled) - New settings of QuestDB Enterprise 4.0.2: the issuer that the `iss` claim of a token must equal, which defaults to the issuer of the discovery document, and the opt-out of the issuer check +- [`acl.oidc.groups.encoded.in.token`](/docs/configuration/oidc/#acloidcgroupsencodedintoken) - Documented that QuestDB Enterprise 4.0.2 rejects expired tokens, tokens without an `exp` claim, and tokens from another issuer, which earlier versions accept, and which token QuestDB reads the principal and the groups from +- [`acl.oidc.audience`](/docs/configuration/oidc/#acloidcaudience) and [`acl.oidc.cache.ttl`](/docs/configuration/oidc/#acloidccachettl) - Documented that `acl.oidc.audience` takes a single value, which the `aud` claim of a token must be or contain, and that QuestDB does not use it with the User Info endpoint, what the cache holds when the groups are encoded in the token or a user logs in with a username and password, and that in QuestDB Enterprise 4.0.2 it also sets how long an HTTP session reuses the check of the password of a database user or the built-in admin, even when OIDC is disabled - [`acl.oidc.public.keys.expiry`](/docs/configuration/oidc/#acloidcpublickeysexpiry) and [`acl.oidc.state.required`](/docs/configuration/oidc/#acloidcstaterequired) - Documented how long QuestDB uses the signing keys of the OIDC provider before it downloads them again, and the `state` check of the Web Console login -- [`acl.oidc.configuration.url`](/docs/configuration/oidc/#acloidcconfigurationurl) and [`acl.oidc.host`](/docs/configuration/oidc/#acloidchost) - Documented that QuestDB downloads the discovery document at startup and does not start without it, that with `acl.oidc.host` QuestDB starts while the provider is unreachable, and that the two settings cannot be combined +- [`acl.oidc.configuration.url`](/docs/configuration/oidc/#acloidcconfigurationurl) and [`acl.oidc.host`](/docs/configuration/oidc/#acloidchost) - Documented that QuestDB downloads the discovery document at startup and does not start without it, that with `acl.oidc.host` QuestDB starts while the provider is unreachable, that QuestDB Enterprise 4.0.2 reads the expected issuer from the discovery document and needs `acl.oidc.issuer` with `acl.oidc.host`, unless issuer validation is disabled, and that the two settings cannot be combined +- [Token providers](/docs/connect/clients/connect-string/#token-providers) - The `token_provider`, `azure_resource`, `azure_credential`, and `azure_client_id` keys of the Java client, which make it request and renew Microsoft Entra ID tokens ### Updated @@ -37,7 +42,9 @@ This page tracks significant updates to the QuestDB documentation. - [Mapping user permissions](/docs/security/oidc/#mapping-user-permissions) - Corrected what happens when the groups claim is missing or empty: QuestDB rejects the login. A user is authenticated without permissions only when none of their groups is mapped to a QuestDB group. Added the endpoint permission that each interface needs, and how external aliases match - [Find user information](/docs/security/oidc/#8-find-user-information) and [non-interactive clients](/docs/security/oidc/#non-interactive-clients) - Corrected which token of a user reaches QuestDB with `acl.oidc.groups.encoded.in.token=true`: the ID token rather than the access token. Non-interactive clients also covers the tokens that services obtain for themselves, with client credentials and password grant examples over QWP - [Endpoint permissions](/docs/security/rbac/#endpoint-permissions) - The `HTTP` permission also covers QWP over WebSocket. The [authentication methods](/docs/security/rbac/#authentication) and [users and service accounts](/docs/security/rbac/#users-and-service-accounts) describe external users, who log in through OIDC -- [Microsoft Entra ID guide](/docs/security/oidc/entra-id/#web-console-sign-in) - Uses `preferred_username` as the principal claim instead of the display name, explains when to use `oid` instead, and registers the application as single-tenant +- [Microsoft Entra ID guide](/docs/security/oidc/entra-id/#web-console-sign-in) - Uses `preferred_username` as the principal claim instead of the display name, explains when to use `oid` instead, registers the application as single-tenant, explains the issuer that QuestDB Enterprise 4.0.2 reads from the tenant's discovery URL, and that the Web Console renews its tokens with the refresh token of the `offline_access` scope, which Entra ID limits to 24 hours +- [Credentials received](/docs/security/oidc/#6-credentials-received) and [When QuestDB stops accepting a token](/docs/security/oidc/#when-questdb-stops-accepting-a-token) - The Web Console of QuestDB Enterprise 4.0.2 renews the ID token before its `exp` claim passes, keeps its token while it cannot reach the provider, and without a refresh token, logs the user out once the token of the login expires +- [Managing authentication](/docs/security/rbac/#managing-authentication) - In QuestDB Enterprise 4.0.2, an HTTP session ends with the credential that opened it: when the password is changed or removed, the REST token is dropped or expires, or the user is disabled or dropped. Also how to log a user out of the Web Console, which authenticates with a REST token that it creates at login - [Python client](/docs/connect/clients/python/#authentication-and-tls) - Corrected when bad credentials raise: by default, `connect()` opens its first connections and raises, and `lazy_connect=on` defers the error to the first operation. Corrected that a terminal server rejection, such as a missing grant, raises `QuestDBServerRejectionError` from `flush(wait=True)`, as described in [Flushing and acknowledgements](/docs/connect/clients/python/#flushing-and-acknowledgements), and that with `sf_dir` the rejected rows stay on disk and are sent again before later batches, as described in [Server rejections](/docs/connect/clients/python/#server-rejections) - [C/C++](/docs/connect/clients/c-and-cpp/#connecting) and [Rust](/docs/connect/clients/rust/#connecting) clients - Corrected when connection and authentication errors surface: by default, the pool connects when it is created and the errors surface there, and `lazy_connect=on` defers them - [Python](/docs/connect/clients/python/), [C/C++](/docs/connect/clients/c-and-cpp/), [.NET](/docs/connect/clients/dotnet/), and [Go](/docs/connect/clients/go/) clients - Corrected token rotation: build a new client with a new token before the old token expires, then close the old one once QuestDB has acknowledged what it sent, or close the old one first when it uses `sf_dir` diff --git a/documentation/configuration/oidc.md b/documentation/configuration/oidc.md index 876b6ebc80..7b9aba37b0 100644 --- a/documentation/configuration/oidc.md +++ b/documentation/configuration/oidc.md @@ -1,6 +1,6 @@ --- title: OpenID Connect (OIDC) -description: "Configure OIDC in QuestDB Enterprise: provider endpoints, TLS, PKCE, ROPC, token audience, and principal and groups claims with fallback lists." +description: "Configure OIDC in QuestDB Enterprise: provider endpoints, TLS, PKCE, ROPC, token audience and issuer, and principal and groups claims with fallback lists." --- :::note @@ -33,8 +33,8 @@ the client ID, so another value makes Web Console logins fail. See [Token validation](/docs/security/oidc/#token-validation). With `acl.oidc.groups.encoded.in.token=false`, QuestDB does not use this -setting: it accepts any access token that the provider's User Info endpoint -accepts, whatever its audience. +setting. It treats the access token as opaque, even when it is a JWT, and +leaves its validation to the provider's User Info endpoint. ### acl.oidc.client.id @@ -53,8 +53,10 @@ URL where the OpenID Provider's configuration information can be loaded in JSON format. Should always end with `/.well-known/openid-configuration`. QuestDB downloads the document when it starts, and takes the provider's -endpoints from it. QuestDB does not start when it cannot download or parse the -document. The error starts with +endpoints from it. Since QuestDB Enterprise 4.0.2, it also takes the expected +issuer of the tokens from the `issuer` field of the document, unless +[`acl.oidc.issuer`](#acloidcissuer) is set. QuestDB does not start when it +cannot download or parse the document. The error starts with `Unable to download OIDC provider configuration from` or `Unable to parse OIDC provider configuration`. To let QuestDB start while the provider is unreachable, set [`acl.oidc.host`](#acloidchost) and the @@ -80,6 +82,12 @@ settings cannot be combined: QuestDB refuses to start when both are set. With `acl.oidc.host`, QuestDB starts even when the provider is unreachable, and only OIDC logins fail until the provider is reachable again. +Without a discovery document, QuestDB has no issuer of the tokens to read. With +`acl.oidc.groups.encoded.in.token=true`, QuestDB Enterprise 4.0.2 and later +therefore refuse to start unless [`acl.oidc.issuer`](#acloidcissuer) is set, or +[`acl.oidc.issuer.validation.enabled`](#acloidcissuervalidationenabled) is +`false`. + ### acl.oidc.http.timeout - **Default**: `30000` @@ -87,6 +95,64 @@ OIDC logins fail until the provider is reachable again. OIDC provider HTTP request timeout in milliseconds. +### acl.oidc.issuer + +- **Default**: none (the `issuer` of the discovery document) +- **Reloadable**: no + +The issuer that the `iss` claim of a token must equal when +`acl.oidc.groups.encoded.in.token` is `true`, in QuestDB Enterprise 4.0.2 and +later. The comparison is exact and case-sensitive, and includes the path and +any trailing slash. When not set, QuestDB uses the `issuer` field of the +discovery document that it downloads from +[`acl.oidc.configuration.url`](#acloidcconfigurationurl). When set, it takes +precedence over the discovery document. + +Set it when: + +- QuestDB uses [`acl.oidc.host`](#acloidchost) and the endpoint settings, + which provide no issuer. +- The discovery document names a template instead of an issuer, such as + `https://login.microsoftonline.com/{tenantid}/v2.0`. Set the issuer of your + tenant, not a template or a wildcard. +- The discovery document names another issuer than the tokens, because the + provider puts the host name of each request into the issuer, such as + Keycloak without a fixed hostname, and QuestDB reaches the provider through + another URL than the clients. Set the issuer that the tokens carry. + +With issuer validation enabled, QuestDB refuses to start when it has no +expected issuer: + +```text +required to set acl.oidc.issuer or provide an issuer in the OIDC discovery document when acl.oidc.groups.encoded.in.token and acl.oidc.issuer.validation.enabled are true +``` + +Not used with `acl.oidc.groups.encoded.in.token=false`, or with +[`acl.oidc.issuer.validation.enabled=false`](#acloidcissuervalidationenabled). +See [The expected issuer](/docs/security/oidc/#the-expected-issuer). + +### acl.oidc.issuer.validation.enabled + +- **Default**: `true` +- **Reloadable**: no + +Whether QuestDB checks the `iss` claim of a token when +`acl.oidc.groups.encoded.in.token` is `true`, in QuestDB Enterprise 4.0.2 and +later. When `false`, QuestDB neither requires the `iss` claim nor compares it, +even when [`acl.oidc.issuer`](#acloidcissuer) is set or the discovery document +provides an issuer, and starts without an expected issuer. It still checks the +signature, the audience, the expiry, and the subject of the token, and logs an +advisory at startup. + +Without the check, QuestDB accepts the tokens of any issuer or tenant that the +provider signs with the same keys for the expected audience. Keep the default, +unless the provider issues tokens without an `iss` claim. + +Accepts `true` or `false`, in any letter case. QuestDB refuses to start with +any other value, an empty value included, even when OIDC is disabled: +`invalid configuration value [key=acl.oidc.issuer.validation.enabled, value=...]`. +See [The expected issuer](/docs/security/oidc/#the-expected-issuer). + ### acl.oidc.port - **Default**: `443` @@ -114,7 +180,9 @@ scope `openid` is mandatory and must always be included. The Web Console requests these scopes when the user logs in. Add `offline_access` if the provider issues a refresh token only for that scope, such as Microsoft Entra ID. Without a refresh token, the Web Console cannot -renew its tokens, and does not log the user out when the user is disabled in +renew its tokens: it logs the user out once QuestDB stops accepting the token +of the login, for example when the token expires, about an hour after the login +with Microsoft Entra ID. This can be later than when the user is disabled in the provider. See [Credentials received](/docs/security/oidc/#6-credentials-received) in the OIDC guide. @@ -281,6 +349,18 @@ ROPC flow, QuestDB caches the result of the login for this long, then requests a new token from the provider, so with either setting, changes take effect after this time. +Since QuestDB Enterprise 4.0.2, this also sets how long an HTTP session reuses +the check of a password that QuestDB verifies itself, the password of a +database user or of the built-in admin, even when OIDC is disabled. A request +that carries the session cookie and the same password is checked again once +this time has passed since the session last checked it. Changing or removing +the password ends the session at once, as described in +[Managing authentication](/docs/security/rbac/#managing-authentication). +QuestDB checks OIDC tokens, username and password logins through the ROPC flow, +and REST tokens on every request, the OIDC ones through the cache described +above. See +[Group changes, sessions, and revoking access](/docs/security/oidc/#group-changes-sessions-and-revoking-access). + ### acl.oidc.groups.claim - **Default**: none @@ -318,9 +398,11 @@ token itself, as described in [Token validation](/docs/security/oidc/#token-validation). Set to `true` if the OIDC Provider encodes group memberships directly into the token. -Since QuestDB Enterprise 4.0.2, QuestDB also rejects expired tokens and tokens -without an `exp` claim. Earlier versions do not check the `exp` claim of the -token. +Since QuestDB Enterprise 4.0.2, QuestDB also rejects expired tokens, tokens +without an `exp` claim, and, unless +[`acl.oidc.issuer.validation.enabled`](#acloidcissuervalidationenabled) is +`false`, tokens whose `iss` claim is not the expected issuer. Earlier versions +check neither the `exp` nor the `iss` claim of the token. ### acl.oidc.sub.claim diff --git a/documentation/connect/clients/connect-string.md b/documentation/connect/clients/connect-string.md index c027700dd5..f4cf221cd2 100644 --- a/documentation/connect/clients/connect-string.md +++ b/documentation/connect/clients/connect-string.md @@ -167,6 +167,7 @@ caveats), follow the section links from the [Key index](#key-index). | Enable TLS | both | `addr` with `wss` schema | `tls_verify`, `tls_roots`, `tls_roots_password` | | Basic-auth credentials | both | `username`, `password` | `auth_timeout_ms` | | Bearer-token credentials | both | `token` | `auth_timeout_ms` | +| Entra ID tokens that the client renews (Java) | both | `token_provider=azure`, `azure_resource` | `azure_credential`, `azure_client_id` | | Multi-host failover | both | `addr=h1,h2,…` | `target`, `zone`, `reconnect_*` (ingress), `failover_*` (egress) | | Query only the primary (freshest data) | egress | `target=primary` | — | | Query only replicas (offload primary) | egress | `target=replica` | — | @@ -235,6 +236,46 @@ the TLS handshake is server-authenticated only. `tls_roots` / `tls_roots_password` configure server-cert trust, not client identity. Use `token=` or `username=` / `password=` for client authentication. +### Token providers {#token-providers} + +*Applies to: ingress and egress. Java client only.* + +A token provider requests bearer tokens from an identity provider, caches +them, and renews them in the background before they expire, so every +connection, reconnect, and failover presents a valid token. Use it instead of +a fixed `token` for tokens that expire, such as the tokens of a Microsoft Entra +ID managed identity or service principal: + +``` +wss::addr=node-a:9000,node-b:9000;token_provider=azure;azure_resource=api://;azure_credential=managed_identity; +``` + +- `token_provider`: the provider. `azure` requests tokens from Microsoft Entra + ID through the Azure Identity library, and needs the `questdb-client-azure` + artifact. Requires the `wss` schema. Cannot be combined with `token`, + `username` / `password`, or a token provider that the application passes to + the client. +- `azure_resource`: required with `token_provider=azure`. The Application ID + URI, such as `api://`, or the application ID of the QuestDB + application in Entra ID. The client requests the scope + `/.default`. +- `azure_credential`: the Azure credential that requests the token. `default`, + the default, is `DefaultAzureCredential`, a discovery chain meant for + development. `managed_identity`, `workload_identity`, and `environment`, a + service principal configured with the `AZURE_TENANT_ID`, `AZURE_CLIENT_ID`, + and `AZURE_CLIENT_SECRET` environment variables or a certificate, use that + credential alone, which is recommended in production. +- `azure_client_id`: the client ID of a user-assigned managed identity or of a + workload identity. Cannot be combined with `azure_credential=environment`. + +The keys hold no secret, so a connect string that uses them is safe to log, and +to keep in `QDB_CLIENT_CONF`. Clients built from equivalent connect strings +share one provider in a process. See +[Rotating tokens](/docs/connect/clients/java/#rotating-tokens) in the Java +client reference, and +[Managed identities and service principals](/docs/security/oidc/entra-id/#managed-identities-and-service-principals) +for the Entra ID setup. + ## TLS {#tls} *Applies to: ingress and egress.* @@ -857,6 +898,9 @@ description and behaviour notes. | `auto_flush_bytes` | size | Java `0` (off) / .NET `8m` (Rust: rejected) | [Auto-flushing](#auto-flush) | | `auto_flush_interval` | int (ms) / `off` | `100` (Rust: rejected) | [Auto-flushing](#auto-flush) | | `auto_flush_rows` | int / `off` | `1000` (Rust: rejected) | [Auto-flushing](#auto-flush) | +| `azure_client_id` | string | unset | [Token providers](#token-providers) (Java only) | +| `azure_credential` | enum (`default` / `managed_identity` / `workload_identity` / `environment`) | `default` | [Token providers](#token-providers) (Java only) | +| `azure_resource` | string | unset (required with `token_provider=azure`) | [Token providers](#token-providers) (Java only) | | `buffer_pool_size` | int (≥ 1) | `4` | [Query client keys](#egress-keys) | | `catch_up_cap_gap_min_escalation_window_millis` | int (ms) | `300000` (5 min) | [Store-and-forward](#sf-keys) | | `client_id` | string | client-specific | [Query client keys](#egress-keys) | @@ -916,6 +960,7 @@ description and behaviour notes. | `tls_roots_password` | string | unset (JKS / PKCS#12 only) | [TLS](#tls) | | `tls_verify` | enum (`on` / `unsafe_off`) | `on` | [TLS](#tls) | | `token` | string | unset | [Authentication](#auth) | +| `token_provider` | enum (`azure`) | unset | [Token providers](#token-providers) (Java only) | | `transaction` | enum (`on` / `off`) | `off` | [Store-and-forward](#sf-keys) | | `user` | string | unset | [Authentication](#auth) (alias of `username`) | | `username` | string | unset | [Authentication](#auth) | diff --git a/documentation/connect/clients/java.md b/documentation/connect/clients/java.md index 1df7d8239f..845f5d5bee 100644 --- a/documentation/connect/clients/java.md +++ b/documentation/connect/clients/java.md @@ -234,14 +234,16 @@ a pool of each client type — [`Sender`](#data-ingestion)s for ingestion and query clients for SQL — plus a background housekeeper thread that closes idle and over-age connections. -The interface has exactly five members: +The interface has these members: | Method | Returns | Purpose | |--------|---------|---------| | `connect(String)` | `QuestDB` | Static factory. One connect string drives both ingress and egress (schema must be `ws`/`wss`). | +| `connect(String, HttpTokenProvider)` | `QuestDB` | Static factory with a provider that supplies the bearer token, such as a `RefreshingTokenProvider`. See [Rotating tokens](#rotating-tokens). | | `builder()` | `QuestDBBuilder` | Pool sizes and timeouts. | | `borrowSender()` | `Sender` | Lease a sender from the pool; `close()` flushes and returns it. | | `borrowQuery()` | `Query` | Lease a query handle from the pool; `close()` returns it. | +| `health()` | `ConnectionHealth.Aggregate` | Connection health of every pooled sender and query client. See [Connection health](#connection-health). | | `close()` | `void` | Shut down pools and disconnect every underlying client. Idempotent. | A `QuestDB` instance is safe to share across threads. Borrows and query @@ -335,6 +337,7 @@ Builder methods: | `errorHandler(SenderErrorHandler)` | — | Async ingest rejections from across the whole sender pool, delivered on the senders' I/O threads. | | `connectionListener(SenderConnectionListener)` | — | Connect and disconnect events across the sender pool. | | `drainerListener(BackgroundDrainerListener)` | — | Store-and-forward background drainer events. | +| `httpTokenProvider(HttpTokenProvider)` | — | Supplies the bearer token of every connection, such as from a `RefreshingTokenProvider`. Cannot be combined with `token`, `username`/`password`, or `token_provider` in the configuration. See [Rotating tokens](#rotating-tokens). | `fromConfig(...)` is required; `build()` throws `IllegalStateException("configuration is required; call fromConfig()")` without @@ -395,7 +398,149 @@ try (QuestDB db = QuestDB.connect( The token is sent as an `Authorization: Bearer YOUR_BEARER_TOKEN` header on both the ingress and egress WebSocket upgrades. It is mutually exclusive with -`username`/`password`. +`username`/`password`. The client sends the same token on every connection that +it opens, so once the token expires, new connections, reconnects, and +failovers fail. For tokens that expire, use a +[token provider](#rotating-tokens). + +### Rotating tokens + +The tokens of an identity provider expire, for example after one to 24 hours +with Microsoft Entra ID. A **token provider** requests tokens from the identity +provider instead of a fixed `token`, caches them, and renews them in the +background at about half of their lifetime, so every connection, reconnect, +and failover presents a valid token. A connection that is already open is not +authenticated again. + +#### Microsoft Entra ID + +Add the `questdb-client-azure` artifact, at the same version as +`questdb-client`. It brings the Azure Identity library, and registers +`token_provider=azure`: + + + + ( + + {` + org.questdb + questdb-client-azure + ${release.name} +`} + + )} /> + + + ( + + {`implementation 'org.questdb:questdb-client-azure:${release.name}'`} + + )} /> + + + +```java +try (QuestDB db = QuestDB.connect( + "wss::addr=db.example.com:9000;" + + "token_provider=azure;" + + "azure_resource=api://;" + + "azure_credential=managed_identity;")) { + // ... +} +``` + +| Key | Description | +|-----|-------------| +| `token_provider` | `azure`. Requires the `wss` schema. Cannot be combined with `token`, `username`/`password`, or a provider passed to `httpTokenProvider(...)`. | +| `azure_resource` | Required. The Application ID URI, such as `api://`, or the application ID of the QuestDB application in Entra ID. The client requests the scope `/.default`. | +| `azure_credential` | The Azure credential that requests the token: `default` (the default), `managed_identity`, `workload_identity`, or `environment`. | +| `azure_client_id` | The client ID of a user-assigned managed identity or of a workload identity. Cannot be combined with `azure_credential=environment`. | + +`azure_credential=default` uses `DefaultAzureCredential`, a discovery chain +meant for development: a service principal configured in environment +variables, a workload identity, a managed identity, then developer tools, such +as the Azure CLI. The chain checks the managed identity endpoint once, without +retrying, so at startup, an outage of that endpoint looks like a host without +a credential. The client logs a warning about this when it starts. In +production, name the credential: `managed_identity`, `workload_identity`, or `environment`, a +service principal configured with `AZURE_TENANT_ID`, `AZURE_CLIENT_ID`, and +`AZURE_CLIENT_SECRET` or a certificate. A managed identity named this way +retries an unreachable endpoint, and fails fast when the identity is not +assigned to the host. + +The connect string holds no secret, so it is safe to log, and to keep in +`QDB_CLIENT_CONF`. Every sender, query client, and pooled connection built from +equivalent connect strings shares one provider in a process: one thread that +renews the token, and one request to Entra ID at a time. For the Entra ID +setup and the QuestDB configuration, see +[Managed identities and service principals](/docs/security/oidc/entra-id/#managed-identities-and-service-principals). + +#### Other identity providers + +Wrap a `TokenSource`, the function that requests a token from your identity +provider, in a `RefreshingTokenProvider`, and pass the provider to +`QuestDB.connect(String, HttpTokenProvider)`, +`QuestDB.builder().httpTokenProvider(...)`, +`Sender.builder(...).httpTokenProvider(...)`, or +`QwpQueryClient.withBearerTokenProvider(...)`. The application owns the +provider: close it after the clients that use it. + +```java +import io.questdb.client.QuestDB; +import io.questdb.client.cutlass.auth.ExpiringToken; +import io.questdb.client.cutlass.auth.RefreshingTokenProvider; + +RefreshingTokenProvider tokens = RefreshingTokenProvider.builder(() -> { + // runs on the provider's own thread: request a token from your + // identity provider, such as with the client credentials grant + MyToken token = requestToken(); + return new ExpiringToken(token.value(), token.expiresAtEpochMillis()); +}).build(); +try (QuestDB db = QuestDB.connect("wss::addr=db.example.com:9000;", tokens)) { + // ... +} finally { + tokens.close(); +} +``` + +`requestToken()` and `MyToken` stand for your own code. `ExpiringToken` takes +the token, without the `Bearer ` prefix, and the time when it expires, in +milliseconds since the epoch. The provider requests its first token as soon as +it is built, and `awaitReady(timeoutMillis)` returns `true` once it holds a +valid one, so an application can wait for it at startup. + +A source reports a failure by throwing `TokenUnavailableException.retryable(...)` +for network trouble, throttling, or a `5xx` response, or +`TokenUnavailableException.permanent(...)` for a missing or wrong +configuration. Any other exception counts as retryable. Don't put a token or a +raw response body into the message. The provider keeps retrying with backoff, +and keeps handing out the current token while it is still valid. + +The provider hands out a token only while more than 60 seconds of its lifetime +are left. A token that arrives with less counts as a failed request, whose +error names `handout_floor`. If your identity provider issues tokens that +short, lower the floor with `handoutFloorMillis(...)` on the builder. + +#### How the client handles token failures + +- When QuestDB rejects a token with `401`, the client asks the provider for a + new token, and when it gets a different one, retries the same server once, + immediately. A `403` does not trigger a retry. +- At startup, with `initial_connect_retry=off`, the default, the client fails + when it cannot get a token. With `on`, it retries a retryable failure of the + provider within `reconnect_max_duration_millis`, and fails at once on a + permanent one. With `async`, it keeps retrying in the background. +- Once a sender has connected, it keeps retrying when it cannot get a token or + QuestDB rejects one, buffers rows in [store-and-forward](#store-and-forward) + meanwhile, and reports each failure to the error handler and in its + [connection health](#connection-health). It never fails on its own: an + application that prefers a hard failure watches the error handler or the + connection health, and closes the sender itself. +- A query whose client has to connect, and cannot get a token, fails. The next + query tries again. ### HTTP basic auth @@ -1320,7 +1465,13 @@ What is and isn't carried on `onError`: - **Authentication failure**: `401`/`403` HTTP response before the WebSocket upgrade completes. Terminal across all endpoints. The borrow that - triggered the connect rethrows `LineSenderException`. + triggered the connect rethrows `LineSenderException`. With a + [token provider](#rotating-tokens), a `401` first makes the client request + a new token and retry the same endpoint once. +- **Credential unavailable**: the [token provider](#rotating-tokens) cannot + supply a token, for example because the identity provider is unreachable. + The client contacts no endpoint, and handles the failure as described in + [How the client handles token failures](#how-the-client-handles-token-failures). - **Malformed frames**: `QwpDecodeException` or WebSocket close with a terminal code. - **Role mismatch**: `QwpRoleMismatchException` when all endpoints report @@ -1422,7 +1573,11 @@ client and gives you a fresh one on your next query. ### Which failures are retried - **Authentication failures** (a bad token or wrong credentials) stop - immediately on every host — retrying cannot help. + immediately on every host, because retrying cannot help. With a + [token provider](#rotating-tokens), a `401` first makes the client request a + new token and retry the same host once. A sender that has already connected + keeps retrying instead, as described in + [How the client handles token failures](#how-the-client-handles-token-failures). - **Network and availability failures** (connection refused, TLS errors, a `5xx` from the server, a mid-query drop) are treated as temporary and fed into the reconnect and failover loops. @@ -1468,6 +1623,42 @@ For the full list of connect-string keys, see the [multi-host failover](/docs/connect/clients/connect-string#failover-keys) sections of the connect string reference. +### Connection health + +A sender rides out an outage, such as an unreachable cluster or a token that +QuestDB rejects, and keeps accepting rows while it retries. To find out that it +has not reached the server, check its connection health. Reading it never +blocks, and never contains a credential, so it can back a health endpoint: + +```java +import io.questdb.client.ConnectionHealth; + +// every pooled sender and query client of the handle +ConnectionHealth.Aggregate health = db.health(); +if (health.count(ConnectionHealth.State.RECONNECTING) > 0) { + long since = health.getOldestOutageSinceEpochMillis(); + ConnectionHealth.Failure failure = health.getLastFailure(); + System.err.printf("QuestDB unreachable since %d: %s%n", since, failure); +} +``` + +`Sender.health()` and `QwpQueryClient.health()` return the `ConnectionHealth` +of one client. Only the WebSocket (QWP) sender tracks it; the other transports +throw `UnsupportedOperationException`. + +| Accessor | Description | +|----------|-------------| +| `getState()` | `CONNECTING` (no successful connect yet), `CONNECTED`, `RECONNECTING` (the connection was lost, and the client is connecting again; for a query client, the last connect failed, and the next query tries again), `FAILED` (terminal), or `CLOSED`. | +| `getLastConnectedAtEpochMillis()` | Time of the last successful connect, or `ConnectionHealth.NONE` (`-1`). | +| `getOutageSinceEpochMillis()` | Start of the current outage, or `NONE` while connected. | +| `getFailedRounds()` | Failed connect rounds, walks over the endpoints in `addr` that ended without a connection, since the outage started. | +| `getLastFailure()` | The last failed connect round, kept after the connection recovers, or `null`. `getFailureClass()` returns `CREDENTIAL_UNAVAILABLE`, `AUTH_REJECTED` (with the status in `getStatusCode()`), `ROLE_REJECTED`, `TRANSPORT`, or `OTHER`, and `getMessage()` and `getEpochMillis()` return the message and the time. | + +`ConnectionHealth.Aggregate` counts the connections in each state with +`count(State)`, and returns the oldest outage with +`getOldestOutageSinceEpochMillis()` and the most recent failure with +`getLastFailure()`. + ## Concurrency `QuestDB` is the only thread-safe handle in the library. `Sender` and @@ -1592,6 +1783,7 @@ Common WebSocket-specific options: | `compression` | `raw` | Egress batch compression (`raw`, `zstd`, `auto`). | | `compression_level` | `1` | Egress zstd level (`1`–`22`, clamped server-side to `1`–`9`). | | `client_id` | unset | Egress client label, sent as the `X-QWP-Client-Id` header; set in production so support can correlate sessions. | +| `token_provider` | unset | Bearer-token provider that requests and renews the token, `wss` only: `azure`, with `azure_resource`, `azure_credential`, and `azure_client_id`. See [Rotating tokens](#rotating-tokens). | Pool keys (read by the `QuestDB` facade; the two clients accept and ignore them): diff --git a/documentation/security/oidc/entra-id.mdx b/documentation/security/oidc/entra-id.mdx index 63ad6885f7..b94f963c11 100644 --- a/documentation/security/oidc/entra-id.mdx +++ b/documentation/security/oidc/entra-id.mdx @@ -5,6 +5,9 @@ description: "Set up Microsoft Entra ID (Azure AD) OIDC for QuestDB Enterprise: --- import Screenshot from "@theme/Screenshot" +import Tabs from "@theme/Tabs" +import TabItem from "@theme/TabItem" +import CodeBlock from "@theme/CodeBlock" import { EnterpriseNote } from "@site/src/components/EnterpriseNote" @@ -50,9 +53,14 @@ Then head to _Microsoft Entra Admin Center_, and register the application under _Identity - App registrations - New registration_. Under _Supported account types_, select _Accounts in this organizational -directory only_. QuestDB does not check which tenant issued a token, so the -application must not accept accounts from other tenants. See -[Token validation](/docs/security/oidc/#token-validation). +directory only_. QuestDB Enterprise 4.0.2 and later accept only the tokens of +the tenant in the discovery URL of the +[QuestDB configuration](#questdb-configuration), because they check the issuer +of every token, as described in +[The expected issuer](/docs/security/oidc/#the-expected-issuer). Earlier +versions, and later ones with issuer validation disabled, do not check which +tenant issued a token, so with them, the application must not accept accounts +from other tenants. /v2.0`, from the OIDC +configuration endpoint, and accept only the tokens that carry it. Use the +endpoint of your tenant, with the tenant ID in the URL, as above. The +endpoints of `common` and `organizations` name the template +`https://login.microsoftonline.com/{tenantid}/v2.0` instead, which no token +carries, so QuestDB would reject every login with `InvalidIssuer`. See +[The expected issuer](/docs/security/oidc/#the-expected-issuer). + The application ID and the OIDC configuration endpoint's URL can be found in the Overview of the application in Entra ID. @@ -326,14 +343,16 @@ will have the access granted to them. :::note -In QuestDB Enterprise 4.0.2 and later, the server log can show -`Failed to decode JWT token Error(ExpiredSignature)` for Web Console users. -By default, Entra ID issues ID tokens that are valid for one hour, and access -tokens that are valid for 60 to 90 minutes. The Web Console requests new -tokens, with the refresh token that Entra ID issues for the `offline_access` -scope, when the access token expires within 30 seconds, so it can keep sending -an expired ID token for up to half an hour. QuestDB accepts these requests -through the session of the Web Console. See +The Web Console renews its tokens before the ID token expires, with the refresh +token that Entra ID issues for the `offline_access` scope in `acl.oidc.scope`. +Without `offline_access`, the Web Console cannot renew them: QuestDB +Enterprise 4.0.2 and later reject the ID token once it expires, one hour after +the login by default, log `Failed to decode JWT token Error(ExpiredSignature)`, +and end the session of the Web Console, which logs the user out. Entra ID also +limits the refresh tokens of single-page applications to +[24 hours](https://learn.microsoft.com/en-us/entra/identity-platform/refresh-tokens), +so the Web Console logs users out once a day. While their Entra ID session is +valid, logging in again usually does not ask for their credentials. See [When QuestDB stops accepting a token](/docs/security/oidc/#when-questdb-stops-accepting-a-token). ::: @@ -378,12 +397,13 @@ the permissions of its Entra ID security groups instead. See 1. Make sure that the application is single-tenant, as set up in [Set up the client application in Entra ID](#set-up-the-client-application-in-entra-id). - QuestDB does not check which tenant issued a token, as described in - [Token validation](/docs/security/oidc/#token-validation). An app role has - the same value in every tenant that uses the application, so with a - multitenant registration, another tenant could assign the role to its own - service principals, and QuestDB would map their tokens to the same QuestDB - groups. + An app role has the same value in every tenant that uses the application, + so with a multitenant registration, another tenant could assign the role to + its own service principals. QuestDB rejects their tokens, because their + issuer names the other tenant, as described in + [The expected issuer](/docs/security/oidc/#the-expected-issuer). With + `acl.oidc.issuer.validation.enabled=false`, it would map them to the same + QuestDB groups. 2. Under _Expose an API_, set the Application ID URI, which defaults to `api://`. Services request their tokens for the scope `/.default`. A token requested for any other API, such @@ -391,11 +411,14 @@ the permissions of its Entra ID security groups instead. See 3. Make the application issue v2.0 access tokens: in its _Manifest_, set `api.requestedAccessTokenVersion` to `2`. The legacy manifest format calls this property `accessTokenAcceptedVersion`. The `aud` claim of a v2.0 token - always carries the application ID, which - [Token validation](/docs/security/oidc/#token-validation) checks. A v1.0 - token, the default for applications that accept organizational accounts - only, can carry the Application ID URI instead. Web Console logins are not affected, because the - Web Console presents ID tokens. + always carries the application ID, and its `iss` claim carries + `https://login.microsoftonline.com//v2.0`, the issuer of the ID + tokens. [Token validation](/docs/security/oidc/#token-validation) checks + both. A v1.0 token, the default for applications that accept + organizational accounts only, carries the issuer + `https://sts.windows.net//`, which QuestDB rejects with + `InvalidIssuer`, and can carry the Application ID URI in `aud`. Web Console + logins are not affected, because the Web Console presents ID tokens. 4. Under _App roles_, define a role for each service, with the _Applications_ member type. A role of its own, mapped to a QuestDB group of its own, lets you cut off one service without affecting the others, as described in @@ -420,8 +443,8 @@ QuestDB. Azure caches managed identity tokens for [around 24 hours](https://learn.microsoft.com/en-us/entra/identity/managed-identities-azure-resources/managed-identity-best-practice-recommendations), and you cannot make it issue a new token before the cached one expires. A token that the service requested before the role assignment or the manifest -change carries no role, or the audience of a v1.0 token, and QuestDB rejects -it until Azure issues a new one. +change carries no role, or the issuer and the audience of a v1.0 token, and +QuestDB rejects it until Azure issues a new one. ::: @@ -520,7 +543,107 @@ following: ### Request a token in the service -The service requests its token for the QuestDB application. With the +The service requests its token for the QuestDB application, for the scope +`/.default`, as the managed identity of the Azure resource +that it runs on, or as a service principal. The +[Java client](#with-the-java-client) requests the token and renews it by +itself. With the [Python client](#with-the-python-client), the service requests +the token, and switches to a new token before the old one expires. + +#### With the Java client + +With `token_provider=azure`, the +[Java client](/docs/connect/clients/java/#rotating-tokens) requests the token +from Entra ID by itself, and renews it in the background at about half of its +lifetime, so every connection that it opens presents a valid token. Add the +`questdb-client-azure` artifact, which brings the Azure Identity library, at +the same version as `questdb-client`: + + + + ( + + {` + org.questdb + questdb-client-azure + ${release.name} +`} + + )} /> + + + ( + + {`implementation 'org.questdb:questdb-client-azure:${release.name}'`} + + )} /> + + + +```java title="Ingest into fx_trades with an app-only token" +import io.questdb.client.QuestDB; +import io.questdb.client.Sender; + +public class FxIngest { + public static void main(String[] args) { + String conf = "wss::addr=questdb.example.com:9000;" + + "token_provider=azure;" + // Application ID URI of the QuestDB application + + "azure_resource=api://8de84b90-1ea5-4e41-9e84-dba860aa01a6;" + // the managed identity of the Azure resource + + "azure_credential=managed_identity;"; + try (QuestDB db = QuestDB.builder() + .fromConfig(conf) + // batches that QuestDB rejects, such as for a missing grant + .errorHandler(error -> System.err.printf("rejected: %s %s%n", + error.getCategory(), error.getServerMessage())) + .build()) { + try (Sender sender = db.borrowSender()) { + sender.table("fx_trades") + .symbol("symbol", "EURUSD") + .symbol("side", "buy") + .doubleColumn("price", 1.0842) + .doubleColumn("quantity", 100000.0) + .atNow(); + } + } + } +} +``` + +- `azure_resource` is the Application ID URI that you set under + _Expose an API_, as described in + [Configure the application in Entra ID](#configure-the-application-in-entra-id), + or the application ID. The client requests the scope + `/.default`. +- `azure_credential=managed_identity` requests the token as the managed + identity of the Azure resource. For a user-assigned managed identity, also + set `azure_client_id` to the client ID of the identity. Otherwise, the client + uses the system-assigned identity. For a workload identity in Azure + Kubernetes Service, use `workload_identity`. For a service principal, use + `environment`, which reads `AZURE_TENANT_ID`, `AZURE_CLIENT_ID`, and + `AZURE_CLIENT_SECRET` or a certificate from the environment. Without + `azure_credential`, the client uses `DefaultAzureCredential`, a discovery + chain meant for development, which does not ride out an outage of the + managed identity endpoint at startup. +- The connect string holds no secret, so it is safe to log, and to keep in the + `QDB_CLIENT_CONF` environment variable. +- When QuestDB rejects a token with `401`, the client asks Entra ID for a new + token, and when it gets a different one, retries the same server once. A + sender that has connected keeps buffering rows while it cannot get a token or + QuestDB rejects it, and reports each failure to the error handler and in its + [connection health](/docs/connect/clients/java/#connection-health). + +The client keeps using the same `QuestDB` handle across token renewals, so a +service can keep its store-and-forward data on disk with `sf_dir` without the +switch between handles that the Python examples below need. + +#### With the Python client + +With the [Azure Identity library](https://learn.microsoft.com/en-us/python/api/overview/azure/identity-readme) for Python, `DefaultAzureCredential` requests the token as the managed identity of the Azure resource that the service runs on, or as a service principal set @@ -735,6 +858,9 @@ with db.query("SELECT current_user()") as result: print(result.to_pandas()) ``` +With the Java client, run the query with `db.borrowQuery()`, as described in +[Querying with Query and Completion](/docs/connect/clients/java/#querying-with-query-and-completion). + If the service cannot connect, the server log gives the reason, as described in [Troubleshooting OIDC logins](/docs/security/oidc/#troubleshooting-oidc-logins). For a token that carries neither a role nor a group, the log lists the claims @@ -761,11 +887,12 @@ REVOKE INSERT ON fx_trades FROM fxIngest; REVOKE HTTP FROM fxIngest; ``` -After the revoke, QuestDB rejects the next batch of the service: -`flush(wait=True)` raises `QuestDBServerRejectionError`, and the examples above -stop. Without `sf_dir`, the rejected rows are lost. With `sf_dir`, they stay on -disk, and land once you grant the permissions again and the service connects -with the same `sender_id`. +After the revoke, QuestDB rejects the next batch of the service. In the Python +examples above, `flush(wait=True)` raises `QuestDBServerRejectionError`, and +the examples stop. Without `sf_dir`, the rejected rows are lost. With `sf_dir`, +they stay on disk, and land once you grant the permissions again and the +service connects with the same `sender_id`. The Java client reports the +rejection to the error handler of its `QuestDB` handle. Granting the permissions again restores access on the connections that are still open, and for every token of the service that QuestDB still accepts. See diff --git a/documentation/security/oidc/index.mdx b/documentation/security/oidc/index.mdx index 982cd20aff..342091414e 100644 --- a/documentation/security/oidc/index.mdx +++ b/documentation/security/oidc/index.mdx @@ -73,7 +73,12 @@ when it cannot download or parse it. The error starts with provider is unreachable, set `acl.oidc.host` and the [endpoint settings](/docs/configuration/oidc/#endpoints) instead: QuestDB then starts without the provider, and only OIDC logins fail until the provider is -reachable again. QuestDB refuses to start when both +reachable again. Without a discovery document, QuestDB has no issuer of the +tokens to read, so with `acl.oidc.groups.encoded.in.token=true`, QuestDB +Enterprise 4.0.2 and later refuse to start unless +[`acl.oidc.issuer`](/docs/configuration/oidc/#acloidcissuer) is set or issuer +validation is disabled, as described in +[The expected issuer](#the-expected-issuer). QuestDB refuses to start when both `acl.oidc.configuration.url` and `acl.oidc.host` are set. For all OIDC-related configuration options of QuestDB, see @@ -93,8 +98,9 @@ sends and what QuestDB checks: | Claims come from | the provider's User Info response for the token | the payload of a JWT: the ID token for Web Console and ROPC logins, or the token that a client sends | | The Web Console sends | the access token | the ID token | | Other clients send | an access token | for a user, an ID token requested with QuestDB's client ID; for a service, an access token whose `aud` contains `acl.oidc.audience` | -| The token is validated by | the provider's User Info endpoint | QuestDB: signature, `aud`, `exp` (QuestDB Enterprise 4.0.2 and later), and `sub` | +| The token is validated by | the provider's User Info endpoint | QuestDB: signature, `aud`, `sub`, and in QuestDB Enterprise 4.0.2 and later, `exp` and `iss` | | `acl.oidc.audience` | not used | must be in the `aud` claim | +| `acl.oidc.issuer` | not used | the `iss` claim must equal it; defaults to the issuer of the discovery document (4.0.2 and later) | | A change to the groups applies | after `acl.oidc.cache.ttl` | when the client presents a new token | | Suits providers that | return the groups from the User Info endpoint, such as PingFederate | put the groups into the tokens, such as Microsoft Entra ID, and issue tokens that services request for themselves | @@ -138,9 +144,7 @@ the _client_, and it is assigned an identifier: the **Client Id**. Each application which integrates via OIDC should be given a different **Client Id**, except clients that send ID tokens to QuestDB: they request them with QuestDB's client ID, as described in -[Non-interactive clients](#non-interactive-clients). With the User Info -endpoint, the default, separate client IDs do not restrict which access tokens -QuestDB accepts; see [Token validation](#token-validation). +[Non-interactive clients](#non-interactive-clients). ### OIDC Provider @@ -302,18 +306,23 @@ such as Microsoft Entra ID, issue it only when the Web Console requests the [`acl.oidc.scope`](/docs/configuration/oidc/#acloidcscope) sets. Before it sends a query, the Web Console uses the refresh token to request new -tokens when the access token expires within 30 seconds. With -`acl.oidc.groups.encoded.in.token=true`, it sends QuestDB the ID token rather -than the access token, and the ID token can expire first, as described in -[When QuestDB stops accepting a token](#when-questdb-stops-accepting-a-token). +tokens when the token that it sends QuestDB expires within 30 seconds: the +access token or, with `acl.oidc.groups.encoded.in.token=true`, the ID token. It +reads the expiry of the ID token from the token's `exp` claim, and the expiry +of the access token from the `expires_in` field of the token response. + When the provider refuses to issue new tokens, for example because the user is -disabled, the Web Console logs the user out. +disabled, the Web Console logs the user out. When it cannot reach the provider, +or the provider answers with `429` or a `5xx` status, the Web Console keeps +sending the current token, warns the user, and tries again with a growing +delay of up to 30 seconds. If the token expires and the Web Console still +cannot get new tokens, it logs the user out. Without a refresh token, the Web Console never requests new tokens. Once QuestDB -stops accepting the tokens of the login, it accepts the requests of the Web -Console through its HTTP session, until the session expires and the Web Console -logs the user out, as described in -[Group changes, sessions, and revoking access](#group-changes-sessions-and-revoking-access). +stops accepting the token of the login, for example when the token expires, it +rejects the requests of the Web Console with `401 Unauthorized` and ends its +HTTP session, and the Web Console logs the user out, as described in +[When QuestDB stops accepting a token](#when-questdb-stops-accepting-a-token). The validity of the tokens are configurable inside the OIDC Provider. @@ -651,7 +660,9 @@ client for ingesting data. It is practical to manage their credentials via an OAuth2 provider too. Such a client requests a token from the provider itself, and sends it to QuestDB as a bearer token, for example in the [`token` setting](/docs/connect/clients/connect-string/#auth) of the client's -configuration string, over ILP/HTTP or QWP. +configuration string, over ILP/HTTP or QWP. The Java client can also request +the token and renew it by itself, with a +[token provider](/docs/connect/clients/java/#rotating-tokens). Which token to send depends on where QuestDB reads the user information from, and on who the client authenticates as: @@ -676,7 +687,9 @@ and on who the client authenticates as: credentials grant, send the access token that the provider issues to it for QuestDB. Configure the provider so that the `aud` claim of the token contains the value of `acl.oidc.audience`, which defaults to QuestDB's - client ID. For Microsoft Entra ID, see + client ID, and so that its `iss` claim is the expected issuer, as + described in [The expected issuer](#the-expected-issuer). For Microsoft + Entra ID, see [Managed identities and service principals](/docs/security/oidc/entra-id/#managed-identities-and-service-principals). Tokens that the provider issues to a service often carry the principal and the @@ -719,7 +732,7 @@ A service that authenticates as itself requests an access token with the client credentials grant, using its own client ID and secret, and sends the token to QuestDB. This example assumes `acl.oidc.groups.encoded.in.token=true`, and a provider that issues the token as a JWT whose `aud` claim contains the -value of `acl.oidc.audience`: +value of `acl.oidc.audience`, and whose `iss` claim is the expected issuer: ```python import os @@ -857,7 +870,13 @@ connection of a client's pool, authenticates with the token again, and fails once QuestDB stops accepting the token, as described in [When QuestDB stops accepting a token](#when-questdb-stops-accepting-a-token). -A long-running client must therefore switch to a new token before the old one +The [Java client](/docs/connect/clients/java/#rotating-tokens) can renew the +token by itself instead: with a token provider, such as `token_provider=azure` +for Microsoft Entra ID, it requests a new token in the background before the +current one expires, and presents it on every connection that it opens after +that. + +Any other long-running client must switch to a new token before the old one expires: request a new token, create a new client with it, wait until the old client has delivered what it sent, then close the old client. A QWP client that is closed discards the rows that QuestDB has not acknowledged within @@ -903,9 +922,11 @@ it, not from the username field. Send the token that an HTTP client would send: - With the User Info endpoint, the default, an access token. - With `acl.oidc.groups.encoded.in.token=true`, for a user, an ID token requested with QuestDB's client ID, and for a service, an access token whose - `aud` claim contains `acl.oidc.audience`. QuestDB rejects a token for another + `aud` claim contains `acl.oidc.audience`, and whose `iss` claim is the + [expected issuer](#the-expected-issuer). QuestDB rejects a token for another audience, such as a user's access token for another API, and logs - `InvalidAudience`. + `InvalidAudience`, or `InvalidIssuer` when the token also comes from another + issuer. See [Non-interactive clients](#non-interactive-clients). Like a password, the token crosses the network in the clear unless the connection uses @@ -1064,23 +1085,58 @@ outlives the token: - Connections that are already open, such as PGWire and QWP connections, stay open when their token expires, and keep running with the groups of the last - token that QuestDB accepted for the user. -- The HTTP session that the Web Console opens at login is not tied to the - token. When QuestDB rejects the token of a request that carries a valid - session cookie, for example because the user is disabled or because the - token no longer carries a group, it accepts the request through the session. - The session ends when the Web Console logs the user out, or after it has - been idle for `http.session.timeout`, 30 minutes by default. Every request - restarts the idle timer, so a session in use does not expire. The Web - Console logs the user out, and ends the session, when the provider refuses - to refresh its tokens, for example after the user is disabled. It refreshes - them only with a refresh token, as described in - [step 6](#6-credentials-received): without one, a disabled user keeps - working through the session for as long as they stay active. A session that - the Web Console does not end, for example because the browser was closed, - stays valid for any client that reuses its cookie, for as long as that + token that QuestDB accepted for the user. A WebSocket connection that a + browser opened with the cookie of an HTTP session also stays open when the + session ends. +- An HTTP session, such as the one that the Web Console opens at login, + accepts the requests that carry its cookie without a token. QuestDB checks + an OIDC token only when a request carries it, so it does not end such a + session when the token expires. The session ends when the Web Console logs + the user out, or after it has been idle for `http.session.timeout`, 30 + minutes by default. Every request that QuestDB accepts restarts the idle + timer, so a session in use does not expire. A session that the Web Console + does not end, for example because the browser was closed, stays valid for + any client that reuses its cookie without a token, for as long as that client stays active. +The Web Console itself sends its token with every query, so its session lasts +only as long as QuestDB accepts the token. QuestDB checks the token of every +request that carries one, and reuses the result of a check for up to +`acl.oidc.cache.ttl`, as described in +[When QuestDB stops accepting a token](#when-questdb-stops-accepting-a-token). +When it rejects the token, for example because the token has expired, because +the provider rejects it, or because the user information no longer carries a +group, QuestDB rejects the request with `401 Unauthorized` and ends the +session, and the Web Console logs the user out. The Web Console also logs the +user out when the provider refuses to refresh its tokens, for example after +the user is disabled. It refreshes them only with a refresh token, as described +in [step 6](#6-credentials-received): without one, a disabled user keeps +working until QuestDB stops accepting the token of the login, when the token +expires or, with the User Info endpoint, when the provider rejects it. +Versions before 4.0.2 accept a request with a rejected token through its +session instead, as described in [Versions before 4.0.2](#versions-before-402). + +QuestDB also rejects a token that it cannot check because the provider cannot +be reached or answers with an error, so an outage of the provider can log users +out of the Web Console: + +- With the User Info endpoint, QuestDB asks the provider about a token again + once `acl.oidc.cache.ttl` has passed since it last did. During an outage, it + then rejects the token, and the Web Console logs the user out. +- With `acl.oidc.groups.encoded.in.token=true`, QuestDB validates tokens with + the signing keys that it has already downloaded, so during an outage, it + rejects only the tokens signed with a key that it has not downloaded yet. + The Web Console cannot renew its tokens during the outage, and logs the user + out once the ID token has expired, as described in + [step 6](#6-credentials-received). +- With either setting, username and password logins through the + [ROPC flow](#enable-ropc) fail during an outage once `acl.oidc.cache.ttl` + has passed since QuestDB last checked them, because QuestDB checks them with + the provider's token endpoint. + +Versions before 4.0.2 accept a request that carries such a token through its +HTTP session instead. + To cut off access immediately, revoke the permissions of the QuestDB groups that the user's groups map to. This takes effect at once for every session and connection, including those of other users who map to the same groups. @@ -1309,29 +1365,96 @@ itself before it reads any claim: [`acl.oidc.audience`](/docs/configuration/oidc/#acloidcaudience), which defaults to the client ID. The setting takes a single value. A token whose `aud` claim is a list is accepted when the list contains that value. +- Its `iss` claim must be the expected issuer, by default the issuer of the + provider's discovery document. Versions before 4.0.2 do not check `iss`. See + [The expected issuer](#the-expected-issuer). - It must carry an `exp` claim that has not passed. QuestDB allows 60 seconds of clock skew. Versions before 4.0.2 do not check `exp`. See [Versions before 4.0.2](#versions-before-402). - It must carry a non-empty `sub` claim, whatever claims `acl.oidc.sub.claim` lists. -QuestDB does not check the issuer (`iss`) or the tenant of the token. It -accepts any token that the provider's keys sign for the expected audience, so -make sure that no one outside your organization can obtain tokens for the -QuestDB application. With Microsoft Entra ID, register the application as -single-tenant, as described in -[Set up the client application in Entra ID](/docs/security/oidc/entra-id/#set-up-the-client-application-in-entra-id). - -With `acl.oidc.groups.encoded.in.token=false`, QuestDB sends the token to the -provider's User Info endpoint, which validates it, and checks nothing else. -QuestDB does not check the audience of the token or the client that it was -issued to, and does not use `acl.oidc.audience`: it accepts any access token -that the User Info endpoint accepts. Any application that can obtain such a -token for a user, from the same provider, can therefore call QuestDB as that -user. With `acl.oidc.groups.encoded.in.token=true`, QuestDB checks the `aud` -claim instead, as described above. See +With `acl.oidc.groups.encoded.in.token=false`, the default, QuestDB expects an +access token, and treats it as opaque, even when it is a JWT: QuestDB does not +decode the token, and leaves its validation to the provider. It sends the token +to the provider's User Info endpoint, which validates it and returns the user +information. `acl.oidc.audience` and `acl.oidc.issuer` do not apply. See [Find user information](#8-find-user-information). +### The expected issuer + +In QuestDB Enterprise 4.0.2 and later, with +`acl.oidc.groups.encoded.in.token=true`, QuestDB accepts tokens from a single +issuer, the expected issuer: + +- [`acl.oidc.issuer`](/docs/configuration/oidc/#acloidcissuer), when it is + set. +- Otherwise, the `issuer` field of the discovery document that QuestDB + downloads from `acl.oidc.configuration.url` at startup. QuestDB does not + derive the issuer from the URL. + +The `iss` claim of a token must be a single string that equals the expected +issuer exactly. The comparison is case-sensitive, and includes the path and any +trailing slash. QuestDB rejects a token without an `iss` claim, or with another +issuer, like a token that fails any other check, and logs the reason, as +described in [Troubleshooting OIDC logins](#troubleshooting-oidc-logins). A +provider that serves several tenants, such as Microsoft Entra ID, puts the +tenant into the issuer, so QuestDB accepts only the tokens of the tenant that +the expected issuer names. + +Set `acl.oidc.issuer` when: + +- QuestDB uses `acl.oidc.host` and the endpoint settings instead of a + discovery document. QuestDB then has no issuer to read, and refuses to start + with this error, which also stops QuestDB when the discovery document has no + issuer: + + ```text + required to set acl.oidc.issuer or provide an issuer in the OIDC discovery document when acl.oidc.groups.encoded.in.token and acl.oidc.issuer.validation.enabled are true + ``` + +- The discovery document names a template instead of an issuer, such as + `https://login.microsoftonline.com/{tenantid}/v2.0` in the discovery + documents of the `common` and `organizations` endpoints of Microsoft Entra + ID. No token carries a template, so QuestDB rejects every token. Set the + issuer of your tenant instead. +- The provider puts the host name of each request into the issuer, such as + Keycloak without a fixed hostname, and QuestDB reaches the provider through + another URL than the clients, such as an internal one. The discovery document + that QuestDB downloads then names another issuer than the tokens. Set the + issuer that the tokens carry. + +The setting takes a single value, so every token that QuestDB accepts must +carry the same issuer. The issuer can depend on the kind of token, not just on +the provider. Microsoft Entra ID puts +`https://login.microsoftonline.com//v2.0` into v2.0 tokens, and +`https://sts.windows.net//` into v1.0 tokens. With the v2.0 +discovery URL of the +[Entra ID guide](/docs/security/oidc/entra-id/#questdb-configuration), the ID +tokens of Web Console users are v2.0 tokens, but the access tokens for the +QuestDB application are v1.0 tokens, unless its manifest requests v2.0 access +tokens. To accept both the ID tokens of +Web Console users and the access tokens of services, make the provider issue +access tokens with the issuer of its ID tokens. For Entra ID, see +[Managed identities and service principals](/docs/security/oidc/entra-id/#managed-identities-and-service-principals). + +For a provider that issues tokens without an `iss` claim, turn the check off +with +[`acl.oidc.issuer.validation.enabled=false`](/docs/configuration/oidc/#acloidcissuervalidationenabled). +QuestDB then ignores the `iss` claim, needs no expected issuer to start, and +still checks the signature, the audience, the expiry, and the subject. It logs +this advisory at startup: + +```text +OIDC JWT issuer validation is disabled; tokens from other issuers or tenants may authenticate when signing keys and audience are shared +``` + +Without the check, QuestDB accepts any token that the provider's keys sign for +the expected audience, so make sure that no one outside your organization can +obtain tokens for the QuestDB application. With Microsoft Entra ID, register +the application as single-tenant, as described in +[Set up the client application in Entra ID](/docs/security/oidc/entra-id/#set-up-the-client-application-in-entra-id). + ### When QuestDB stops accepting a token QuestDB checks a token when a request or a new connection authenticates with @@ -1350,27 +1473,30 @@ by default: Some access outlives the token. A PGWire or WebSocket connection, such as a QWP connection, that is already open stays open when its token expires, and an HTTP -request that also carries a valid session cookie, such as a request of the Web -Console, is accepted through the session. See +session accepts the requests that carry its cookie without a token. A request +that carries a token that QuestDB no longer accepts fails, even with a valid +session cookie, and ends the session. See [Group changes, sessions, and revoking access](#group-changes-sessions-and-revoking-access). For what a client must do before its token expires, see [Token lifetime](#token-lifetime). -The Web Console requests new tokens when the access token expires within 30 -seconds, but with `acl.oidc.groups.encoded.in.token=true`, it sends the ID -token. When the provider issues ID tokens that expire before its access tokens, -the Web Console keeps sending the ID token after it has expired, until it -requests new tokens. Microsoft Entra ID issues such tokens by default: ID -tokens are valid for one hour, and access tokens for a random 60 to 90 minutes. +With a refresh token, the Web Console requests new tokens before the token that +it sends QuestDB expires, the ID token with +`acl.oidc.groups.encoded.in.token=true`, as described in +[step 6](#6-credentials-received). When the clock of the browser is behind, +QuestDB can still reject the token as expired first: the Web Console then +renews its tokens and sends the request again. + Without a refresh token, the Web Console never requests new tokens, and keeps -sending the expired token. QuestDB Enterprise 4.0.2 and later then reject the -ID token, log -`Failed to decode JWT token Error(ExpiredSignature)`, and accept the request -through the session of the Web Console, which keeps the groups of the last -token that QuestDB accepted. When the session has expired too, for example -because the Web Console sent no request for longer than -`http.session.timeout`, QuestDB rejects the request, and the Web Console logs -the user out. +sending the token of the login after it has expired. QuestDB Enterprise 4.0.2 +and later then reject the expired ID token, log +`Failed to decode JWT token Error(ExpiredSignature)`, reject the request with +`401 Unauthorized`, and end the session of the Web Console. The Web Console +logs the user out, and the user has to log in again. With Microsoft Entra ID, +whose ID tokens are valid for one hour by default, this happens about an hour +after the login, unless +[`acl.oidc.scope`](/docs/configuration/oidc/#acloidcscope) includes +`offline_access`, the scope that Entra ID issues refresh tokens for. ## Troubleshooting OIDC logins @@ -1408,23 +1534,43 @@ example: ```text Request failed [errorMsg=Failed to decode JWT token Error(InvalidAudience)] +Request failed [errorMsg=Failed to decode JWT token Error(InvalidIssuer)] +Request failed [errorMsg=Failed to decode JWT token Error(MissingRequiredClaim("iss"))] Request failed [errorMsg=Failed to decode JWT token Error(ExpiredSignature)] Request failed [errorMsg=Unable to find public key with key id [...]] ``` `InvalidAudience` usually means that the token was requested for another API, -or that Entra ID issued a v1.0 access token. See +or, with issuer validation disabled, that Entra ID issued a v1.0 access token. + +`InvalidIssuer` means that the `iss` claim of the token is not the expected +issuer, and `MissingRequiredClaim("iss")` that the token carries no `iss` +claim, as described in [The expected issuer](#the-expected-issuer). QuestDB +checks the expiry before the issuer, and the issuer before the audience, and +logs the first check that fails. With Microsoft Entra ID, `InvalidIssuer` +usually means a v1.0 access token, whose issuer is +`https://sts.windows.net//`, or a token of another tenant. When +every login fails with it, compare the `iss` claim of a token with the +expected issuer: `acl.oidc.issuer`, or the `issuer` field of the discovery +document. If the discovery document names a template rather than your tenant, +or another host than the `iss` claim of the tokens, set `acl.oidc.issuer`. + +For a v1.0 access token of Entra ID, make the application issue v2.0 tokens, as +described in [Managed identities and service principals](/docs/security/oidc/entra-id/#managed-identities-and-service-principals). -Don't set `acl.oidc.audience` to the Application ID URI to work around it: the -setting takes a single value, and the ID tokens of Web Console users carry the -application ID, so their logins would fail. +Don't set `acl.oidc.audience` to the Application ID URI, or `acl.oidc.issuer` +to the issuer of v1.0 tokens, to work around it: each setting takes a single +value, and the ID tokens of Web Console users carry the application ID and the +v2.0 issuer, so their logins would fail. `ExpiredSignature` means that the token has expired. A client configured with a fixed token must switch to a new token before then, as described in -[Token lifetime](#token-lifetime). For Web Console users, the error can also -mean that the Web Console still sends an ID token that has expired, which it -does with Microsoft Entra ID. QuestDB then accepts the requests through the -session of the Web Console. See +[Token lifetime](#token-lifetime). For Web Console users, it usually means that +the Web Console has no refresh token, because +[`acl.oidc.scope`](/docs/configuration/oidc/#acloidcscope) does not include the +scope that the provider issues refresh tokens for, such as `offline_access` +with Microsoft Entra ID. QuestDB then rejects the requests, and the Web Console +logs the user out once the ID token of the login has expired. See [When QuestDB stops accepting a token](#when-questdb-stops-accepting-a-token). `Unable to find public key with key id` means that none of the signing keys @@ -1473,7 +1619,13 @@ QuestDB Enterprise versions before 4.0.2 differ as follows: - With `acl.oidc.groups.encoded.in.token=true`, they reject any token that does not carry a `groups` array, whatever `acl.oidc.groups.claim` names. They do not check the `exp` claim, so they accept expired tokens and tokens without - an `exp` claim. + an `exp` claim. They do not check the `iss` claim either, so they accept the + tokens of any issuer that the provider's keys sign for the expected + audience, such as the tokens of another Microsoft Entra ID tenant. +- When an HTTP request carries a session cookie and a token that they reject, + or cannot check because the provider is unreachable, they accept the request + through the session. The Web Console keeps working with a token that they no + longer accept, until its session expires. - They read a `null` claim as the value `null`. Every account whose principal claim is `null` logs in as the same principal, `null`, and `null` and empty group names count as groups. @@ -1487,10 +1639,27 @@ When you upgrade from an earlier version: - With `acl.oidc.groups.encoded.in.token=true`, QuestDB rejects expired tokens and tokens without an `exp` claim. A client that keeps using a token after it expires, such as a client configured with a fixed token, must request a new - token before the old one expires. With Microsoft Entra ID, the server log - also shows `ExpiredSignature` errors for Web Console users, whose ID tokens - expire before the Web Console requests new tokens. See + token before the old one expires. The Web Console renews the ID token before + it expires, but only with a refresh token: without one, it logs the user out + when the ID token of the login expires, about an hour after the login with + Microsoft Entra ID, where earlier versions accept the expired ID token. To + keep users logged in, add the scope that the provider issues refresh tokens + for, such as `offline_access` with Entra ID, to + [`acl.oidc.scope`](/docs/configuration/oidc/#acloidcscope). See [When QuestDB stops accepting a token](#when-questdb-stops-accepting-a-token). +- With `acl.oidc.groups.encoded.in.token=true`, QuestDB rejects tokens whose + `iss` claim is missing or is not the expected issuer, and refuses to start + when it has no expected issuer, such as with `acl.oidc.host`: set + `acl.oidc.issuer`. With Microsoft Entra ID, it rejects v1.0 access tokens. + For a provider that issues tokens without an `iss` claim, set + `acl.oidc.issuer.validation.enabled=false`. See + [The expected issuer](#the-expected-issuer). +- A request that carries a session cookie and a token that QuestDB rejects, or + cannot check because the provider is unreachable, fails with + `401 Unauthorized`, and ends the session, instead of running through the + session. An outage of the provider can therefore log users out of the Web + Console, as described in + [Group changes, sessions, and revoking access](#group-changes-sessions-and-revoking-access). - QuestDB refuses to start with an empty `acl.oidc.sub.claim`, or with a claim that is listed twice or in both settings. See [Startup validation](#startup-validation). diff --git a/documentation/security/rbac.md b/documentation/security/rbac.md index 14e596686c..633946811c 100644 --- a/documentation/security/rbac.md +++ b/documentation/security/rbac.md @@ -575,6 +575,27 @@ ALTER USER username DROP TOKEN TYPE REST 'token_value_here'; -- Drop specific t Removing all authentication methods (password and tokens) effectively disables the user - they can no longer connect to the database. +In QuestDB Enterprise 4.0.2 and later, an HTTP session ends with the +credential that opened it, or the one that it checked last: when the password +is changed or removed, when the REST token is dropped or expires, or when the +user or service account is disabled or dropped, even if the client sends only +the session cookie. A session of the built-in admin ends when a configuration +reload changes `acl.admin.password`. A request that carries a credential that +QuestDB rejects fails with `401 Unauthorized`, and ends the session whose +cookie it carries. A replica ends the sessions of a disabled or dropped user or +service account only when they are next used, so a session that is not used +while the user is disabled survives if the user is enabled again. + +The Web Console logs in with the password, then sends a REST token that it +creates at login with every later request. The token is valid for one day, and +is extended while in use. A password change ends the session of the Web +Console, but not the token, which opens a new session. To log a user out of +the Web Console, drop the user's REST tokens with +`ALTER USER username DROP TOKEN TYPE REST`, which also drops the REST tokens +that the user's other clients use. To cut the user off from every client, +disable the user. For external users, see +[Group changes, sessions, and revoking access](/docs/security/oidc/#group-changes-sessions-and-revoking-access). + ### Viewing information ```questdb-sql From 73b8f5a8e34666e9612499f77550ca4cbbb26306 Mon Sep 17 00:00:00 2001 From: glasstiger Date: Thu, 15 Oct 2026 11:17:32 +0100 Subject: [PATCH 15/29] docs(oidc): fix review findings on log lines, Web Console renewal, upgrade notes and Java startup - Troubleshooting: QuestDB does not log the token of a request without a JWT, so correct that log line. Add MissingRequiredClaim("exp"), InvalidSignature, Sub is not set in token and a missing key ID, state the full check order, and split the section into subsections. - The expected issuer: acl.oidc.issuer for a provider reached through an internal URL fixes token validation only, because the Web Console uses the endpoints of the same discovery document, so recommend a fixed public hostname. Add acl.oidc.host and opt-out examples. - Web Console: one Web Console token renewal section for the console side and When QuestDB stops accepting a token for the server side, replacing the copies in step 6, Group changes, troubleshooting, the upgrade notes, the Entra ID guide and acl.oidc.scope. Add the missing 4.0.2 qualifiers, the renewal and resend after a 401, and the logout after the token has expired. - Upgrade: a warning after the introduction, and one Upgrade to 4.0.2 list that gives, for each change, the earlier behaviour and what to do. - Define token mode and User Info mode, shorten the introduction, correct the PingFederate stub, the client credentials scope example and the principal claim guidance, and add dedup notes where replay is recommended. - Java client: token providers and connection health need Java client 1.3.10 or later; lazy_connect starts a QuestDB handle while the identity provider is down; a connected sender retries authentication failures with any credential. Correct the error policies and the error handler of the QuestDB handle, and the connection health example. Document the checked exceptions of a TokenSource, getLastFailure(), the wait limits, AzureTokenSource and orphan drainers. - connect-string: the Java exception to terminal authentication failures on reconnect, the lazy_connect conflicts, and reconnect_* promotion for standalone senders only in the C, C++, Python and Rust clients. - rust.md: token rotation, lazy_connect conflicts and reconnect_* promotion. - Document http.session.timeout. State the owner that an external user names in CREATE VIEW, CREATE LIVE VIEW, CREATE MATERIALIZED VIEW and CREATE SERVICE ACCOUNT. Shorten the changelog to one line per change. Co-Authored-By: Claude Opus 5.5 --- documentation/changelog.mdx | 62 +- documentation/configuration/http-server.md | 13 + documentation/configuration/oidc.md | 37 +- .../connect/clients/connect-string.md | 21 +- documentation/connect/clients/java.md | 190 ++++-- documentation/connect/clients/rust.md | 25 +- .../query/sql/acl/create-service-account.md | 4 +- documentation/query/sql/create-live-view.md | 6 +- documentation/query/sql/create-mat-view.md | 4 +- documentation/query/sql/create-view.md | 6 + documentation/security/oidc/entra-id.mdx | 73 ++- documentation/security/oidc/index.mdx | 597 ++++++++++-------- documentation/security/oidc/pingfederate.mdx | 6 + 13 files changed, 648 insertions(+), 396 deletions(-) diff --git a/documentation/changelog.mdx b/documentation/changelog.mdx index 399c9edcd5..7a8fa8cb6f 100644 --- a/documentation/changelog.mdx +++ b/documentation/changelog.mdx @@ -12,42 +12,44 @@ This page tracks significant updates to the QuestDB documentation. ### New -- [Microsoft Entra ID](/docs/security/oidc/entra-id/) - New page for Microsoft Entra ID: the Web Console sign-in guide, moved from the OIDC page, and app-only tokens for Azure managed identities and service principals in QuestDB Enterprise 4.0.2, with an app role and a QuestDB group for each service, how to keep app roles the only way in, how to cut off one service, a Java example in which the client requests and renews the token itself with `token_provider=azure`, and Python examples that ingest and renew the token, also with store-and-forward on disk -- [Where QuestDB reads the user information](/docs/security/oidc/#where-questdb-reads-the-user-information) - Compares the User Info endpoint with reading the claims from the token: which token each client sends, what QuestDB checks, and when a change to the groups takes effect -- [User and group claims](/docs/security/oidc/#user-and-group-claims) - How QuestDB Enterprise 4.0.2 reads the principal and the groups from the user information: fallback claim lists, how to order them so that user tokens keep their groups, the rules for missing, `null`, and empty claims, how to choose the principal claim, and the startup checks -- [Token validation](/docs/security/oidc/#token-validation), [Troubleshooting OIDC logins](/docs/security/oidc/#troubleshooting-oidc-logins), and [Versions before 4.0.2](/docs/security/oidc/#versions-before-402) - What QuestDB checks in a token, including that it treats the access token as opaque and leaves its validation to the User Info endpoint, the issuer that QuestDB Enterprise 4.0.2 requires in a JWT and where it comes from, how long it accepts a token after it expires, the log messages of rejected logins, how earlier versions differ, and what to check when you upgrade -- [Tables created by external users](/docs/security/oidc/#tables-created-by-external-users) - Ingestion fails when it has to create a table or add a column for a user who logs in through OIDC, so create the tables in advance, or create them with SQL and `OWNED BY`, naming one of the user's QuestDB groups -- [Group changes, sessions, and revoking access](/docs/security/oidc/#group-changes-sessions-and-revoking-access) - When a change in the identity provider takes effect, that open connections keep the groups of the last token that QuestDB accepted, that in QuestDB Enterprise 4.0.2 an HTTP session ends when a request carries a token that QuestDB rejects or cannot check, so the Web Console logs the user out once QuestDB stops accepting its token, also during an outage of the provider, and how to cut off access immediately -- [The expected issuer](/docs/security/oidc/#the-expected-issuer) - QuestDB Enterprise 4.0.2 accepts a JWT only from the issuer of the provider's discovery document, or from `acl.oidc.issuer`: when to set it, including for providers that put the host name of a request into the issuer, the issuers of Microsoft Entra ID v1.0 and v2.0 tokens, and the opt-out for providers that issue tokens without an `iss` claim -- [Token lifetime](/docs/security/oidc/#token-lifetime) - How QuestDB treats the token of a client as it expires, and how to switch a long-running client to a new token before then without losing the rows that QuestDB has not acknowledged, unless the Java client renews the token itself -- [Rotating tokens](/docs/connect/clients/java/#rotating-tokens) - The Java client requests bearer tokens and renews them before they expire with a token provider: `token_provider=azure` and the `questdb-client-azure` artifact for Microsoft Entra ID, `RefreshingTokenProvider` for other identity providers, and how the client handles a token that it cannot get or that QuestDB rejects -- [Connection health](/docs/connect/clients/java/#connection-health) - `QuestDB.health()`, `Sender.health()`, and `QwpQueryClient.health()` of the Java client report whether a client reaches the server, since when it has not, and the last failure +- [User and group claims](/docs/security/oidc/#user-and-group-claims) - Fallback claim lists of QuestDB Enterprise 4.0.2, how QuestDB picks a claim, and the startup checks +- [Token validation](/docs/security/oidc/#token-validation) - What QuestDB checks in a token, the expected issuer, and when QuestDB stops accepting an expired token +- [Web Console token renewal](/docs/security/oidc/#web-console-token-renewal) - When the Web Console renews its tokens, and when it logs the user out +- [Troubleshooting OIDC logins](/docs/security/oidc/#troubleshooting-oidc-logins) - The log lines of rejected logins and what they mean +- [Upgrade to 4.0.2](/docs/security/oidc/#upgrade-to-402) - How earlier versions differ, and what to check when you upgrade +- [Where QuestDB reads the user information](/docs/security/oidc/#where-questdb-reads-the-user-information) - The User Info endpoint compared with reading the claims from the token +- [Group changes, sessions, and revoking access](/docs/security/oidc/#group-changes-sessions-and-revoking-access) - When a change in the identity provider takes effect, and how to cut off access +- [Tables created by external users](/docs/security/oidc/#tables-created-by-external-users) - Create tables in advance, or with `OWNED BY` and a QuestDB group of the user +- [Token lifetime](/docs/security/oidc/#token-lifetime) - Switch a long-running client to a new token without losing rows +- [Rotating tokens](/docs/connect/clients/java/#rotating-tokens) - The Java client requests and renews bearer tokens with `token_provider=azure` or a `RefreshingTokenProvider` +- [Connection health](/docs/connect/clients/java/#connection-health) - The Java client reports whether it reaches the server, and its last failure ### Reference -- [`acl.oidc.groups.claim`](/docs/configuration/oidc/#acloidcgroupsclaim) and [`acl.oidc.sub.claim`](/docs/configuration/oidc/#acloidcsubclaim) - Documented the comma-separated fallback claim lists of QuestDB Enterprise 4.0.2 and the startup validation. `acl.oidc.groups.claim` is now listed with no default, as it must be set when OIDC is enabled. Username and password (ROPC) logins take the principal from the username that the client sends -- [`acl.oidc.pkce.required`](/docs/configuration/oidc/#acloidcpkcerequired) - Corrected the property name, previously documented as `acl.oidc.pkce.enabled`, which QuestDB does not recognize -- [`acl.oidc.issuer`](/docs/configuration/oidc/#acloidcissuer) and [`acl.oidc.issuer.validation.enabled`](/docs/configuration/oidc/#acloidcissuervalidationenabled) - New settings of QuestDB Enterprise 4.0.2: the issuer that the `iss` claim of a token must equal, which defaults to the issuer of the discovery document, and the opt-out of the issuer check -- [`acl.oidc.groups.encoded.in.token`](/docs/configuration/oidc/#acloidcgroupsencodedintoken) - Documented that QuestDB Enterprise 4.0.2 rejects expired tokens, tokens without an `exp` claim, and tokens from another issuer, which earlier versions accept, and which token QuestDB reads the principal and the groups from -- [`acl.oidc.audience`](/docs/configuration/oidc/#acloidcaudience) and [`acl.oidc.cache.ttl`](/docs/configuration/oidc/#acloidccachettl) - Documented that `acl.oidc.audience` takes a single value, which the `aud` claim of a token must be or contain, and that QuestDB does not use it with the User Info endpoint, what the cache holds when the groups are encoded in the token or a user logs in with a username and password, and that in QuestDB Enterprise 4.0.2 it also sets how long an HTTP session reuses the check of the password of a database user or the built-in admin, even when OIDC is disabled -- [`acl.oidc.public.keys.expiry`](/docs/configuration/oidc/#acloidcpublickeysexpiry) and [`acl.oidc.state.required`](/docs/configuration/oidc/#acloidcstaterequired) - Documented how long QuestDB uses the signing keys of the OIDC provider before it downloads them again, and the `state` check of the Web Console login -- [`acl.oidc.configuration.url`](/docs/configuration/oidc/#acloidcconfigurationurl) and [`acl.oidc.host`](/docs/configuration/oidc/#acloidchost) - Documented that QuestDB downloads the discovery document at startup and does not start without it, that with `acl.oidc.host` QuestDB starts while the provider is unreachable, that QuestDB Enterprise 4.0.2 reads the expected issuer from the discovery document and needs `acl.oidc.issuer` with `acl.oidc.host`, unless issuer validation is disabled, and that the two settings cannot be combined -- [Token providers](/docs/connect/clients/connect-string/#token-providers) - The `token_provider`, `azure_resource`, `azure_credential`, and `azure_client_id` keys of the Java client, which make it request and renew Microsoft Entra ID tokens +- [`acl.oidc.sub.claim`](/docs/configuration/oidc/#acloidcsubclaim) and [`acl.oidc.groups.claim`](/docs/configuration/oidc/#acloidcgroupsclaim) - Documented fallback claim lists and their startup validation +- [`acl.oidc.issuer`](/docs/configuration/oidc/#acloidcissuer) and [`acl.oidc.issuer.validation.enabled`](/docs/configuration/oidc/#acloidcissuervalidationenabled) - New settings for the expected issuer of a token +- [`acl.oidc.pkce.required`](/docs/configuration/oidc/#acloidcpkcerequired) - Corrected the name of the property, previously documented as `acl.oidc.pkce.enabled` +- [`acl.oidc.state.required`](/docs/configuration/oidc/#acloidcstaterequired) and [`acl.oidc.public.keys.expiry`](/docs/configuration/oidc/#acloidcpublickeysexpiry) - Documented both properties +- [`acl.oidc.cache.ttl`](/docs/configuration/oidc/#acloidccachettl) - Documented what QuestDB caches, and that it also bounds how long an HTTP session reuses a password check +- [`http.session.timeout`](/docs/configuration/http-server/#httpsessiontimeout) - Documented how long an idle HTTP session stays valid +- [Token providers](/docs/connect/clients/connect-string/#token-providers) - The `token_provider` and `azure_*` keys of the Java client + +### Guides + +- [Microsoft Entra ID](/docs/security/oidc/entra-id/) - Moved to its own page, and added app-only tokens for Azure managed identities and service principals +- [PingFederate](/docs/security/oidc/pingfederate/) - Moved to its own page, added the QuestDB configuration, and corrected the directory that PingFederate reads users from ### Updated -- [OpenID Connect (OIDC)](/docs/security/oidc/) - The identity provider guides moved to their own pages, under OpenID Connect (OIDC) in the sidebar, the introduction lists what the page covers, and [Configuration options](/docs/security/oidc/#configuration-options), right after the introduction, shows a minimal configuration -- [PingFederate](/docs/security/oidc/pingfederate/) - Moved from the OIDC page to its own page, and added the [QuestDB configuration](/docs/security/oidc/pingfederate/#questdb-configuration). Corrected the directory that PingFederate reads the users from over LDAP: Microsoft Entra Domain Services or an Active Directory domain, as Microsoft Entra ID has no LDAP endpoint -- [OIDC for the PGWire endpoint](/docs/security/oidc/#oidc-for-the-pgwire-endpoint) - Documented which token to send in the password field, that QuestDB takes the principal and the groups from the token, and that the connection should use TLS -- [Mapping user permissions](/docs/security/oidc/#mapping-user-permissions) - Corrected what happens when the groups claim is missing or empty: QuestDB rejects the login. A user is authenticated without permissions only when none of their groups is mapped to a QuestDB group. Added the endpoint permission that each interface needs, and how external aliases match -- [Find user information](/docs/security/oidc/#8-find-user-information) and [non-interactive clients](/docs/security/oidc/#non-interactive-clients) - Corrected which token of a user reaches QuestDB with `acl.oidc.groups.encoded.in.token=true`: the ID token rather than the access token. Non-interactive clients also covers the tokens that services obtain for themselves, with client credentials and password grant examples over QWP -- [Endpoint permissions](/docs/security/rbac/#endpoint-permissions) - The `HTTP` permission also covers QWP over WebSocket. The [authentication methods](/docs/security/rbac/#authentication) and [users and service accounts](/docs/security/rbac/#users-and-service-accounts) describe external users, who log in through OIDC -- [Microsoft Entra ID guide](/docs/security/oidc/entra-id/#web-console-sign-in) - Uses `preferred_username` as the principal claim instead of the display name, explains when to use `oid` instead, registers the application as single-tenant, explains the issuer that QuestDB Enterprise 4.0.2 reads from the tenant's discovery URL, and that the Web Console renews its tokens with the refresh token of the `offline_access` scope, which Entra ID limits to 24 hours -- [Credentials received](/docs/security/oidc/#6-credentials-received) and [When QuestDB stops accepting a token](/docs/security/oidc/#when-questdb-stops-accepting-a-token) - The Web Console of QuestDB Enterprise 4.0.2 renews the ID token before its `exp` claim passes, keeps its token while it cannot reach the provider, and without a refresh token, logs the user out once the token of the login expires -- [Managing authentication](/docs/security/rbac/#managing-authentication) - In QuestDB Enterprise 4.0.2, an HTTP session ends with the credential that opened it: when the password is changed or removed, the REST token is dropped or expires, or the user is disabled or dropped. Also how to log a user out of the Web Console, which authenticates with a REST token that it creates at login -- [Python client](/docs/connect/clients/python/#authentication-and-tls) - Corrected when bad credentials raise: by default, `connect()` opens its first connections and raises, and `lazy_connect=on` defers the error to the first operation. Corrected that a terminal server rejection, such as a missing grant, raises `QuestDBServerRejectionError` from `flush(wait=True)`, as described in [Flushing and acknowledgements](/docs/connect/clients/python/#flushing-and-acknowledgements), and that with `sf_dir` the rejected rows stay on disk and are sent again before later batches, as described in [Server rejections](/docs/connect/clients/python/#server-rejections) -- [C/C++](/docs/connect/clients/c-and-cpp/#connecting) and [Rust](/docs/connect/clients/rust/#connecting) clients - Corrected when connection and authentication errors surface: by default, the pool connects when it is created and the errors surface there, and `lazy_connect=on` defers them -- [Python](/docs/connect/clients/python/), [C/C++](/docs/connect/clients/c-and-cpp/), [.NET](/docs/connect/clients/dotnet/), and [Go](/docs/connect/clients/go/) clients - Corrected token rotation: build a new client with a new token before the old token expires, then close the old one once QuestDB has acknowledged what it sent, or close the old one first when it uses `sf_dir` +- [OpenID Connect (OIDC)](/docs/security/oidc/) - Split into an overview and provider guides, with a minimal configuration and an upgrade warning for QuestDB Enterprise 4.0.2 +- [Mapping user permissions](/docs/security/oidc/#mapping-user-permissions) - Corrected that a login without groups is rejected, and added the endpoint permission of each interface +- [Non-interactive clients](/docs/security/oidc/#non-interactive-clients) - Which token each client sends, with client credentials and password grant examples over QWP +- [Managing authentication](/docs/security/rbac/#managing-authentication) - HTTP sessions of QuestDB Enterprise 4.0.2 end with the credential that opened them +- [Java client](/docs/connect/clients/java/#ingestion-errors) - Corrected the error policies, and that the `QuestDB` handle accepts an error handler +- [Python](/docs/connect/clients/python/#the-pool), [C/C++](/docs/connect/clients/c-and-cpp/#connecting), and [Rust](/docs/connect/clients/rust/#connecting) clients - Corrected that the pool connects when it is created, unless `lazy_connect=on` +- [Python client](/docs/connect/clients/python/#flushing-and-acknowledgements) - Corrected that a terminal server rejection raises from `flush(wait=True)` +- [Client token rotation](/docs/security/oidc/#token-lifetime) - Corrected how the Python, C/C++, .NET, Go, and Rust clients switch to a new token +- [`CREATE VIEW`](/docs/query/sql/create-view/#owned-by-enterprise), [`CREATE LIVE VIEW`](/docs/query/sql/create-live-view/#owned-by-clause), and [`CREATE MATERIALIZED VIEW`](/docs/query/sql/create-mat-view/#owned-by-clause) - Documented the owner that an external user must name ## September 2026 diff --git a/documentation/configuration/http-server.md b/documentation/configuration/http-server.md index cb8971ee49..c736ba003b 100644 --- a/documentation/configuration/http-server.md +++ b/documentation/configuration/http-server.md @@ -81,6 +81,19 @@ occurs. Changing the default value is strongly discouraged. Password for HTTP Basic Authentication in QuestDB Open Source. QuestDB Enterprise supports more advanced authentication via RBAC. +### http.session.timeout + +- **Default**: `30m` +- **Reloadable**: no + +How long an HTTP session stays valid without a request. A client opens a +session, the `qdb_session` cookie, by sending `session=true`, as the Web +Console does at login. Every request that QuestDB authorizes with the session +restarts the timer, so a session in use does not expire. In QuestDB +Enterprise, a session also ends when the credential that opened it is +revoked, as described in +[Managing authentication](/docs/security/rbac/#managing-authentication). + ### http.user - **Default**: N/A diff --git a/documentation/configuration/oidc.md b/documentation/configuration/oidc.md index 7b9aba37b0..b84004e99d 100644 --- a/documentation/configuration/oidc.md +++ b/documentation/configuration/oidc.md @@ -1,6 +1,6 @@ --- title: OpenID Connect (OIDC) -description: "Configure OIDC in QuestDB Enterprise: provider endpoints, TLS, PKCE, ROPC, token audience and issuer, and principal and groups claims with fallback lists." +description: "Reference for the acl.oidc.* settings of QuestDB Enterprise: provider endpoints, TLS, PKCE, ROPC, token audience and issuer, and claim lists." --- :::note @@ -14,7 +14,9 @@ Management. The database can be integrated with any OAuth2/OIDC Identity Provider (IdP). For detailed information about OIDC, see the -[OpenID Connect (OIDC) integration guide](/docs/security/oidc). +[OpenID Connect (OIDC) integration guide](/docs/security/oidc). For complete +provider setups, see the [Microsoft Entra ID](/docs/security/oidc/entra-id/) +and [PingFederate](/docs/security/oidc/pingfederate/) guides. ## General @@ -118,7 +120,10 @@ Set it when: - The discovery document names another issuer than the tokens, because the provider puts the host name of each request into the issuer, such as Keycloak without a fixed hostname, and QuestDB reaches the provider through - another URL than the clients. Set the issuer that the tokens carry. + another URL than the clients. Set the issuer that the tokens carry. The Web + Console still sends browsers to the endpoints of that discovery document, so + for Web Console logins, give the provider a fixed public hostname instead, + as described in [The expected issuer](/docs/security/oidc/#the-expected-issuer). With issuer validation enabled, QuestDB refuses to start when it has no expected issuer: @@ -180,12 +185,12 @@ scope `openid` is mandatory and must always be included. The Web Console requests these scopes when the user logs in. Add `offline_access` if the provider issues a refresh token only for that scope, such as Microsoft Entra ID. Without a refresh token, the Web Console cannot -renew its tokens: it logs the user out once QuestDB stops accepting the token -of the login, for example when the token expires, about an hour after the login -with Microsoft Entra ID. This can be later than when the user is disabled in -the provider. See -[Credentials received](/docs/security/oidc/#6-credentials-received) in the -OIDC guide. +renew its tokens, and on QuestDB Enterprise 4.0.2 and later, it logs the user +out once QuestDB stops accepting the token of the login, about an hour after +the login with Microsoft Entra ID. Until then, a user who is disabled in the +provider can keep access. See +[Web Console token renewal](/docs/security/oidc/#web-console-token-renewal) in +the OIDC guide. ## Authentication flows @@ -231,6 +236,11 @@ OIDC Provider when it does not carry the same `state`. This ties the redirect to a login that the same browser started, which protects against cross-site request forgery on the login redirect. +Enable it when your security policy or your provider requires the `state` +parameter. The PKCE check, which +[`acl.oidc.pkce.required`](#acloidcpkcerequired) enables by default, already +ties the authorization code to the browser that started the login. + ## Endpoints ### acl.oidc.authorization.endpoint @@ -356,10 +366,11 @@ that carries the session cookie and the same password is checked again once this time has passed since the session last checked it. Changing or removing the password ends the session at once, as described in [Managing authentication](/docs/security/rbac/#managing-authentication). -QuestDB checks OIDC tokens, username and password logins through the ROPC flow, -and REST tokens on every request, the OIDC ones through the cache described -above. See -[Group changes, sessions, and revoking access](/docs/security/oidc/#group-changes-sessions-and-revoking-access). +The session does not reuse the check of any other credential. QuestDB checks a +REST token on every request that carries it, and an OIDC token or an ROPC login +through the cache described above, which asks the provider again once this +time has passed since the provider last checked the credential. See +[When QuestDB stops accepting a token](/docs/security/oidc/#when-questdb-stops-accepting-a-token). ### acl.oidc.groups.claim diff --git a/documentation/connect/clients/connect-string.md b/documentation/connect/clients/connect-string.md index f4cf221cd2..e6371a5088 100644 --- a/documentation/connect/clients/connect-string.md +++ b/documentation/connect/clients/connect-string.md @@ -152,7 +152,10 @@ ws::addr=node-a:9000;reconnect_max_duration_millis=120000; The 2-minute reconnect budget covers both the *first* connect and any subsequent reconnect: setting any explicit `reconnect_*` key implicitly -turns on `initial_connect_retry`. See +turns on `initial_connect_retry`. In the C, C++, Python, and Rust clients, +this applies to a standalone sender only: their pooled facade honours only an +explicit `initial_connect_retry`, and tolerates a server that is down at +startup with [`lazy_connect=on`](#pool-keys). See [Ingress reconnect](#reconnect-keys). ## Recipes {#recipes} @@ -238,7 +241,7 @@ the TLS handshake is server-authenticated only. `tls_roots` / ### Token providers {#token-providers} -*Applies to: ingress and egress. Java client only.* +*Applies to: ingress and egress. Java client only, 1.3.10 or later.* A token provider requests bearer tokens from an identity provider, caches them, and renews them in the background before they expire, so every @@ -684,7 +687,11 @@ architecture is that a producer survives an arbitrarily long outage. Auth failures during reconnect (authentication rejected, version mismatch, durable-ack mismatch, non-101 upgrade without a role hint) are immediately -terminal — the loop does not retry them. +terminal: the loop does not retry them. The Java client is the exception for +authentication, upgrade, and durable-ack rejections: it treats them as terminal +only for the first connect of a sender and for orphan drainers, and a sender +that has connected keeps retrying them, as described in +[How the client handles token failures](/docs/connect/clients/java/#how-the-client-handles-token-failures). ### Egress failover {#egress-failover} @@ -815,7 +822,11 @@ explicit setter always wins over the string. - `lazy_connect` — when `on`, the pool defers opening its first connection until the first borrow, so construction succeeds against a server that is down. This is the supported way to tolerate a server that starts after your - application. Default: `off`. + application, and in the Java client, an identity provider that is + unreachable when a token provider requests its first token. In the Java, C, + C++, Python, and Rust clients, it cannot be combined with `query_pool_min` + above `0`, or with an `initial_connect_retry` other than `async`: the client + rejects the configuration. Default: `off`. ## Error handling {#error-handling} @@ -918,7 +929,7 @@ description and behaviour notes. | `failover_max_attempts` | int | `8` | [Egress failover](#reconnect-keys) | | `failover_max_duration_ms` | int (ms) | `30000` | [Egress failover](#reconnect-keys) | | `init_buf_size` | size | `65536` (64 KiB) | [Buffer sizing](#buffer) | -| `initial_connect_retry` | enum (`off` / `on` / `async`) | `off` (auto-promoted to `on` when any explicit `reconnect_*` key is set) | [Ingress reconnect](#reconnect-keys) | +| `initial_connect_retry` | enum (`off` / `on` / `async`) | `off` (auto-promoted to `on` when any explicit `reconnect_*` key is set, except in the pooled facade of the C, C++, Python, and Rust clients) | [Ingress reconnect](#reconnect-keys) | | `initial_credit` | int (bytes) | `0` (unbounded) | [Query client keys](#egress-keys) | | `housekeeper_interval_ms` | int (ms) | `5000` | [Connection pool](#pool-keys) | | `idle_timeout_ms` | int (ms) | `60000` (`0` ⇒ infinite) | [Connection pool](#pool-keys) | diff --git a/documentation/connect/clients/java.md b/documentation/connect/clients/java.md index 845f5d5bee..8a78db95ea 100644 --- a/documentation/connect/clients/java.md +++ b/documentation/connect/clients/java.md @@ -44,6 +44,10 @@ Key capabilities: async [`Completion`](#completion) handles, and byte-credit flow control. - **Failover**: multi-endpoint connections with automatic reconnect across rolling upgrades and primary migrations. +- **Token renewal**: with a token provider, such as `token_provider=azure` for + Microsoft Entra ID managed identities and service principals, the client + requests bearer tokens and renews them before they expire. See + [Rotating tokens](#rotating-tokens). :::tip Legacy transports @@ -399,9 +403,11 @@ try (QuestDB db = QuestDB.connect( The token is sent as an `Authorization: Bearer YOUR_BEARER_TOKEN` header on both the ingress and egress WebSocket upgrades. It is mutually exclusive with `username`/`password`. The client sends the same token on every connection that -it opens, so once the token expires, new connections, reconnects, and -failovers fail. For tokens that expire, use a -[token provider](#rotating-tokens). +it opens, so once QuestDB stops accepting the token, new connections, +reconnects, and failovers fail. QuestDB Enterprise 4.0.2 and later, and the +User Info endpoint, reject a token shortly after it expires; see +[Token lifetime](/docs/security/oidc/#token-lifetime). For tokens that expire, +use a [token provider](#rotating-tokens). ### Rotating tokens @@ -410,7 +416,7 @@ with Microsoft Entra ID. A **token provider** requests tokens from the identity provider instead of a fixed `token`, caches them, and renews them in the background at about half of their lifetime, so every connection, reconnect, and failover presents a valid token. A connection that is already open is not -authenticated again. +authenticated again. Token providers require Java client 1.3.10 or later. #### Microsoft Entra ID @@ -474,10 +480,34 @@ assigned to the host. The connect string holds no secret, so it is safe to log, and to keep in `QDB_CLIENT_CONF`. Every sender, query client, and pooled connection built from equivalent connect strings shares one provider in a process: one thread that -renews the token, and one request to Entra ID at a time. For the Entra ID -setup and the QuestDB configuration, see +renews the token, and one request to Entra ID at a time. The provider stays +open for 60 seconds after the last client that uses it closes, so that a client +built soon after reuses its token. For the keys, see +[Token providers](/docs/connect/clients/connect-string/#token-providers) in the +connect string reference. For the Entra ID setup and the QuestDB configuration, +see [Managed identities and service principals](/docs/security/oidc/entra-id/#managed-identities-and-service-principals). +To use an Azure credential that `azure_credential` cannot select, such as a +certificate credential, wrap it in an `AzureTokenSource`, and pass it to a +`RefreshingTokenProvider`, as described in +[Other identity providers](#other-identity-providers): + +```java +import com.azure.core.credential.TokenCredential; +import com.azure.identity.ClientCertificateCredentialBuilder; +import io.questdb.client.azure.AzureTokenSource; +import io.questdb.client.cutlass.auth.RefreshingTokenProvider; + +TokenCredential credential = new ClientCertificateCredentialBuilder() + .tenantId("") + .clientId("") + .pemCertificate("/etc/fx-ingest/client.pem") + .build(); +RefreshingTokenProvider tokens = RefreshingTokenProvider.builder( + new AzureTokenSource(credential, "api://")).build(); +``` + #### Other identity providers Wrap a `TokenSource`, the function that requests a token from your identity @@ -489,15 +519,22 @@ provider, in a `RefreshingTokenProvider`, and pass the provider to provider: close it after the clients that use it. ```java +import java.io.IOException; + import io.questdb.client.QuestDB; import io.questdb.client.cutlass.auth.ExpiringToken; import io.questdb.client.cutlass.auth.RefreshingTokenProvider; +import io.questdb.client.cutlass.auth.TokenUnavailableException; RefreshingTokenProvider tokens = RefreshingTokenProvider.builder(() -> { // runs on the provider's own thread: request a token from your // identity provider, such as with the client credentials grant - MyToken token = requestToken(); - return new ExpiringToken(token.value(), token.expiresAtEpochMillis()); + try { + MyToken token = requestToken(); + return new ExpiringToken(token.value(), token.expiresAtEpochMillis()); + } catch (IOException e) { + throw TokenUnavailableException.retryable("token endpoint unreachable"); + } }).build(); try (QuestDB db = QuestDB.connect("wss::addr=db.example.com:9000;", tokens)) { // ... @@ -508,37 +545,84 @@ try (QuestDB db = QuestDB.connect("wss::addr=db.example.com:9000;", tokens)) { `requestToken()` and `MyToken` stand for your own code. `ExpiringToken` takes the token, without the `Bearer ` prefix, and the time when it expires, in -milliseconds since the epoch. The provider requests its first token as soon as -it is built, and `awaitReady(timeoutMillis)` returns `true` once it holds a -valid one, so an application can wait for it at startup. +milliseconds since the epoch. The source cannot throw checked exceptions: +catch them, as the example does with the `IOException` of an HTTP client, and +throw a `TokenUnavailableException` instead, which is unchecked. A source reports a failure by throwing `TokenUnavailableException.retryable(...)` for network trouble, throttling, or a `5xx` response, or `TokenUnavailableException.permanent(...)` for a missing or wrong -configuration. Any other exception counts as retryable. Don't put a token or a -raw response body into the message. The provider keeps retrying with backoff, -and keeps handing out the current token while it is still valid. +configuration, such as a wrong client secret. Pass the `Retry-After` of a +throttled response with `retryable(message, retryAfterMillis)`. Any other +exception counts as retryable. Don't put a token or a raw response body into +the message. The provider keeps retrying with backoff, whatever the +classification, and keeps handing out the current token while it is still +valid. + +The provider requests its first token as soon as it is built. +`awaitReady(timeoutMillis)` returns `true` once it holds a valid one, and +`false` when the timeout passes, also after a permanent failure. To fail fast +at startup, for example on a wrong client secret, check `getLastFailure()`: + +```java +if (!tokens.awaitReady(30_000)) { + RefreshingTokenProvider.FetchFailure failure = tokens.getLastFailure(); + if (failure != null && !failure.isRetryable()) { + tokens.close(); + throw new IllegalStateException("cannot get a token: " + failure.getMessage()); + } + // a retryable failure: start anyway, and let the provider keep trying +} +``` + +`getConsecutiveFailures()` and `getTokenExpiresAtEpochMillis()` report the +state of the provider too, such as for a health check. The provider hands out a token only while more than 60 seconds of its lifetime are left. A token that arrives with less counts as a failed request, whose error names `handout_floor`. If your identity provider issues tokens that -short, lower the floor with `handoutFloorMillis(...)` on the builder. +short, lower the floor with `handoutFloorMillis(...)` on the builder. A +connection that needs a token waits up to 30 seconds for one when the provider +holds none, set with `coldWaitMillis(...)`, and up to 5 seconds for a new +token after a `401`, set with `forcedWaitMillis(...)`. #### How the client handles token failures - When QuestDB rejects a token with `401`, the client asks the provider for a new token, and when it gets a different one, retries the same server once, - immediately. A `403` does not trigger a retry. -- At startup, with `initial_connect_retry=off`, the default, the client fails - when it cannot get a token. With `on`, it retries a retryable failure of the - provider within `reconnect_max_duration_millis`, and fails at once on a - permanent one. With `async`, it keeps retrying in the background. + immediately. The provider requests a new token for a rejected one at most + once every 30 seconds. With `token_provider=azure`, Azure Identity can + return the token that it has cached, and then the client does not retry. A + `403` does not trigger a retry. +- At startup, a sender with `initial_connect_retry=off`, the default, fails + when it cannot get a token within 30 seconds. With `on`, it retries a + retryable failure of the provider within `reconnect_max_duration_millis`, + and fails at once on a permanent one. With `async`, it keeps retrying in the + background. +- A `QuestDB` handle also connects `query_pool_min` query clients, one by + default, when it starts, and a query client that cannot get a token fails + without retrying. `QuestDB.connect()` and `build()` therefore fail while the + identity provider is unreachable, whatever `initial_connect_retry` says. To + start anyway, set `lazy_connect=true`: senders then connect in the + background, as with `initial_connect_retry=async`, and query clients on the + first query. `lazy_connect=true` rejects an explicit `query_pool_min` above + `0`, and an `initial_connect_retry` other than `async`. - Once a sender has connected, it keeps retrying when it cannot get a token or - QuestDB rejects one, buffers rows in [store-and-forward](#store-and-forward) - meanwhile, and reports each failure to the error handler and in its - [connection health](#connection-health). It never fails on its own: an - application that prefers a hard failure watches the error handler or the - connection health, and closes the sender itself. + QuestDB rejects one, with any credential, not only with a token provider. It + buffers rows in [store-and-forward](#store-and-forward) meanwhile, and + reports each failure in its [connection health](#connection-health) and to + the error handler, as a `SECURITY_ERROR` with the `RETRIABLE` policy, whose + message starts with `ws-upgrade-failed:` or `credential-unavailable:`. It + never fails on its own: an application that prefers a hard failure watches + the error handler or the connection health, and closes the sender itself. + Without `sf_dir`, closing it discards the rows that QuestDB has not + acknowledged. +- An orphan drainer, enabled with `drain_orphans=on`, gives up instead. When + QuestDB keeps rejecting its credential, it sets the slot aside behind a + `.failed` file and reports `DATA_LOSS`: at once with a fixed credential, and + with a token provider, once the rejections have lasted for + `reconnect_max_duration_millis`, at most 5 minutes, and at least six + attempts. The rows stay on disk. - A query whose client has to connect, and cannot get a token, fails. The next query tries again. @@ -1379,16 +1463,15 @@ apply to the query side only and are ignored by the ingest side. WebSocket ingestion uses an asynchronous error model. Batch rejections are delivered via the `SenderErrorHandler` callback, not thrown from `flush()`. -The `QuestDB` connect-string path does not yet expose this callback — if -your application needs structured `SenderError` reports, drop down to the -[low-level `Sender.builder`](#sender-low-level-primitive) directly and -skip the pool. The structured fields on `SenderError` are the same -regardless of how the sender was created: +Register the handler with `QuestDB.builder().errorHandler(...)` for every +sender of the pool, or with `errorHandler(...)` on the builder of a +[low-level `Sender`](#sender-low-level-primitive). The structured fields on +`SenderError` are the same regardless of how the sender was created: | Field | Accessor | Description | |-------|----------|-------------| -| Category | `getCategory()` | `SCHEMA_MISMATCH`, `PARSE_ERROR`, `INTERNAL_ERROR`, `SECURITY_ERROR`, `WRITE_ERROR`, `PROTOCOL_VIOLATION`, or `UNKNOWN` | -| Policy | `getAppliedPolicy()` | `DROP_AND_CONTINUE` (batch dropped, sender continues) or `HALT` (next API call throws `LineSenderServerException`) | +| Category | `getCategory()` | `SCHEMA_MISMATCH`, `PARSE_ERROR`, `INTERNAL_ERROR`, `SECURITY_ERROR`, `WRITE_ERROR`, `NOT_WRITABLE`, `DICTIONARY_GAP`, `PROTOCOL_VIOLATION`, `DATA_LOSS`, or `UNKNOWN` | +| Policy | `getAppliedPolicy()` | `RETRIABLE` (the client reconnects and replays the batch from store-and-forward), `RETRIABLE_OTHER` (the same, on the next endpoint), `TERMINAL` (the next API call throws `LineSenderServerException`, and the rejected rows stay in store-and-forward), or `ABANDONED` (with `DATA_LOSS` only: the rows can never be sent, and stay on disk under `getQuarantinedPath()`) | | Server message | `getServerMessage()` | Human-readable error text from the server (may be null) | | Table name | `getTableName()` | The rejected table (null for multi-table batches) | | FSN range | `getFromFsn()` / `getToFsn()` | Frame sequence number span identifying the rejected batch | @@ -1398,7 +1481,15 @@ regardless of how the sender was created: The error handler runs on a dedicated dispatcher thread, never on the I/O or producer thread. -When a sender owned by `QuestDB` enters a terminal `HALT` state, the next +By default, schema mismatches, parse errors, security errors such as a missing +grant, and protocol violations are `TERMINAL`. Write errors, internal errors, +dictionary gaps, and unknown categories are `RETRIABLE`, and a node that cannot +serve writes is `RETRIABLE_OTHER`. A connected sender also reports each +reconnect whose credential QuestDB rejects, or for which it cannot get a token, +as a `RETRIABLE` `SECURITY_ERROR`, as described in +[How the client handles token failures](#how-the-client-handles-token-failures). + +When a sender owned by `QuestDB` enters the `TERMINAL` state, the next producer-thread call throws `LineSenderServerException`. The pool detects the failure on close/return and replaces the failed sender with a fresh one on the next borrow. @@ -1464,10 +1555,13 @@ What is and isn't carried on `onError`: ### Connection-level errors - **Authentication failure**: `401`/`403` HTTP response before the WebSocket - upgrade completes. Terminal across all endpoints. The borrow that - triggered the connect rethrows `LineSenderException`. With a - [token provider](#rotating-tokens), a `401` first makes the client request - a new token and retry the same endpoint once. + upgrade completes. Terminal across all endpoints for the first connect of a + sender and for a query client: the borrow that triggered the connect + rethrows `LineSenderException`. A sender that has connected before keeps + retrying instead, as described in + [How the client handles token failures](#how-the-client-handles-token-failures). + With a [token provider](#rotating-tokens), a `401` first makes the client + request a new token and retry the same endpoint once. - **Credential unavailable**: the [token provider](#rotating-tokens) cannot supply a token, for example because the identity provider is unreachable. The client contacts no endpoint, and handles the failure as described in @@ -1573,10 +1667,11 @@ client and gives you a fresh one on your next query. ### Which failures are retried - **Authentication failures** (a bad token or wrong credentials) stop - immediately on every host, because retrying cannot help. With a - [token provider](#rotating-tokens), a `401` first makes the client request a - new token and retry the same host once. A sender that has already connected - keeps retrying instead, as described in + immediately on every host for a query and for the first connect of a sender, + because retrying cannot help. With a [token provider](#rotating-tokens), a + `401` first makes the client request a new token and retry the same host + once. A sender that has already connected keeps retrying with any + credential, as described in [How the client handles token failures](#how-the-client-handles-token-failures). - **Network and availability failures** (connection refused, TLS errors, a `5xx` from the server, a mid-query drop) are treated as temporary and fed into @@ -1627,21 +1722,26 @@ sections of the connect string reference. A sender rides out an outage, such as an unreachable cluster or a token that QuestDB rejects, and keeps accepting rows while it retries. To find out that it -has not reached the server, check its connection health. Reading it never -blocks, and never contains a credential, so it can back a health endpoint: +has not reached the server, check its connection health, which requires Java +client 1.3.10 or later. Reading it never blocks, and never contains a +credential, so it can back a health endpoint: ```java import io.questdb.client.ConnectionHealth; // every pooled sender and query client of the handle ConnectionHealth.Aggregate health = db.health(); -if (health.count(ConnectionHealth.State.RECONNECTING) > 0) { - long since = health.getOldestOutageSinceEpochMillis(); +long since = health.getOldestOutageSinceEpochMillis(); +if (since != ConnectionHealth.NONE) { + // a client has not connected yet, is reconnecting, or has failed ConnectionHealth.Failure failure = health.getLastFailure(); System.err.printf("QuestDB unreachable since %d: %s%n", since, failure); } ``` +The aggregate counts query clients too: a query client whose last connect +failed is in an outage until a later query connects it. + `Sender.health()` and `QwpQueryClient.health()` return the `ConnectionHealth` of one client. Only the WebSocket (QWP) sender tracks it; the other transports throw `UnsupportedOperationException`. diff --git a/documentation/connect/clients/rust.md b/documentation/connect/clients/rust.md index fd937d4788..2b6a0078be 100644 --- a/documentation/connect/clients/rust.md +++ b/documentation/connect/clients/rust.md @@ -135,7 +135,9 @@ previous client left holding unsent data, and replays them in the background. With [`lazy_connect=on`](/docs/connect/clients/connect-string/#pool-keys), `connect()` performs no blocking network I/O, and `query_pool_min` defaults to -`0`. A borrowed sender then creates its local producer at once and connects its +`0`. It rejects an explicit `query_pool_min` above `0`, and an +`initial_connect_retry` other than `async`. A borrowed sender then creates its +local producer at once and connects its delivery runner in the background, so it can queue data while the server is unavailable, and reader errors surface from the first borrow. In both modes, direct Arrow/Polars transport errors surface from the borrow or during @@ -157,6 +159,21 @@ let token = QuestDb::connect( )?; ``` +A `token` is a static credential: the client sends it on every connection that +it opens, including reconnects, and never renews it. Acquire it from the +identity provider of the QuestDB Enterprise +[OpenID Connect flow](/docs/security/oidc/#non-interactive-clients), and for +Azure services, see +[Microsoft Entra ID managed identities and service principals](/docs/security/oidc/entra-id/#managed-identities-and-service-principals). +Before the token expires, connect a new `QuestDb` with a new token, wait until +QuestDB has acknowledged what the old senders sent, with +`wait(AckLevel::Ok, timeout)`, then close the old handle. With `sf_dir`, close +the old handle first, then connect the new one with the same `sender_id`: two +handles cannot hold the same store-and-forward slot, and the new handle replays +what the old one left on disk. On QuestDB Enterprise 4.0.2 and later, or with +the User Info endpoint, QuestDB rejects the token on new connections shortly +after it expires; see [Token lifetime](/docs/security/oidc/#token-lifetime). + With the default crate features, the TLS root set is `webpki_roots`. Other choices have feature requirements: @@ -785,8 +802,10 @@ and is not part of that thread-bound borrow list. | `initial_connect_retry` | `off` | `on`/`sync` retries initial connection synchronously; `async` starts background retry. | | `lazy_connect` | `off` | `on` makes `connect()` open no connections, so that it tolerates a server that is down at startup. See [Connecting](#connecting). | -Setting an ingress `reconnect_*` key without explicitly setting -`initial_connect_retry` promotes the initial mode to synchronous retry. See the +For a standalone `Sender`, setting an ingress `reconnect_*` key without +explicitly setting `initial_connect_retry` promotes the initial mode to +synchronous retry. A pool, and so `QuestDb::connect`, honours only an +explicitly set `initial_connect_retry`. See the [connect string reference](/docs/connect/clients/connect-string/) for the full grammar and all limits. diff --git a/documentation/query/sql/acl/create-service-account.md b/documentation/query/sql/acl/create-service-account.md index ce9230ec9b..b61b21b443 100644 --- a/documentation/query/sql/acl/create-service-account.md +++ b/documentation/query/sql/acl/create-service-account.md @@ -50,7 +50,9 @@ clause is present, in which case the permission is granted to the user or group specified in the clause. The `OWNED BY` clause cannot be omitted if the service account is created by -an external user, because permissions cannot be granted to them. +an external user, because permissions cannot be granted to them. The owner must +then be one of the QuestDB groups that the external user belongs to. See +[Tables created by external users](/docs/security/oidc/#tables-created-by-external-users). ## Examples diff --git a/documentation/query/sql/create-live-view.md b/documentation/query/sql/create-live-view.md index 6c6fd579dd..26b4ba9c43 100644 --- a/documentation/query/sql/create-live-view.md +++ b/documentation/query/sql/create-live-view.md @@ -306,7 +306,11 @@ including `DROP LIVE VIEW`, with the `GRANT` option. ### OWNED BY clause -Assign ownership to a user, group, or service account: +Assign ownership to a user, group, or service account. The `OWNED BY` clause +cannot be omitted if the live view is created by an external user, who logs in +through OIDC, because permissions cannot be granted to them. The owner must +then be one of the QuestDB groups that the external user belongs to. See +[Tables created by external users](/docs/security/oidc/#tables-created-by-external-users). ```questdb-sql CREATE GROUP analysts; diff --git a/documentation/query/sql/create-mat-view.md b/documentation/query/sql/create-mat-view.md index 0dd9739535..6bf7d2cf16 100644 --- a/documentation/query/sql/create-mat-view.md +++ b/documentation/query/sql/create-mat-view.md @@ -362,7 +362,9 @@ CREATE MATERIALIZED VIEW trades_hourly AS ( :::note External users (authenticated via external identity providers) must specify the -`OWNED BY` clause when creating materialized views. +`OWNED BY` clause when creating materialized views. The owner must then be one +of the QuestDB groups that the external user belongs to. See +[Tables created by external users](/docs/security/oidc/#tables-created-by-external-users). ::: ### Permission examples diff --git a/documentation/query/sql/create-view.md b/documentation/query/sql/create-view.md index 33eacf2c68..8efde0c864 100644 --- a/documentation/query/sql/create-view.md +++ b/documentation/query/sql/create-view.md @@ -217,6 +217,12 @@ When a user creates a new view, they are automatically assigned all view level permissions with the `GRANT` option for that view. This behavior can be overridden using `OWNED BY`. +The `OWNED BY` clause cannot be omitted if the view is created by an external +user, who logs in through OIDC, because permissions cannot be granted to them. +The owner must then be one of the QuestDB groups that the external user belongs +to. See +[Tables created by external users](/docs/security/oidc/#tables-created-by-external-users). + ```questdb-sql CREATE GROUP analysts; CREATE VIEW trades_summary AS ( diff --git a/documentation/security/oidc/entra-id.mdx b/documentation/security/oidc/entra-id.mdx index b94f963c11..cda86cd81c 100644 --- a/documentation/security/oidc/entra-id.mdx +++ b/documentation/security/oidc/entra-id.mdx @@ -190,6 +190,11 @@ Select _Add groups claim_, and set it up as follows: [Managed identities and service principals](#managed-identities-and-service-principals) relies on. +Entra ID leaves the `groups` claim out of the tokens of a user who belongs to +more than 200 groups, and QuestDB then rejects the login. To emit only the +groups that are assigned to the application instead, see +[Missing principal or groups claims](/docs/security/oidc/#missing-principal-or-groups-claims). + The customized tokens contain user information which cannot be accessed without permission. User information is provided by Microsoft Graph, so the client application needs specific permissions to access @@ -339,21 +344,22 @@ GRANT SELECT ON fx_trades, core_price TO extUsers; To test, head to the Web Console and login. If all has been wired up well, then login will succeed, and the user -will have the access granted to them. +will have the access granted to them. If the login fails, the server log gives +the reason, as described in +[Troubleshooting OIDC logins](/docs/security/oidc/#troubleshooting-oidc-logins). :::note -The Web Console renews its tokens before the ID token expires, with the refresh -token that Entra ID issues for the `offline_access` scope in `acl.oidc.scope`. -Without `offline_access`, the Web Console cannot renew them: QuestDB -Enterprise 4.0.2 and later reject the ID token once it expires, one hour after -the login by default, log `Failed to decode JWT token Error(ExpiredSignature)`, -and end the session of the Web Console, which logs the user out. Entra ID also -limits the refresh tokens of single-page applications to +Entra ID issues the refresh token, with which the Web Console renews its +tokens, only for the `offline_access` scope in `acl.oidc.scope`. Without it, +the Web Console cannot renew its tokens, and on QuestDB Enterprise 4.0.2 and +later, it logs users out when the ID token of their login expires, one hour +after the login by default. Entra ID also limits the refresh tokens of +single-page applications to [24 hours](https://learn.microsoft.com/en-us/entra/identity-platform/refresh-tokens), so the Web Console logs users out once a day. While their Entra ID session is valid, logging in again usually does not ask for their credentials. See -[When QuestDB stops accepting a token](/docs/security/oidc/#when-questdb-stops-accepting-a-token). +[Web Console token renewal](/docs/security/oidc/#web-console-token-renewal). ::: @@ -393,7 +399,7 @@ when its token carries no role but carries a `groups` claim, the service gets the permissions of its Entra ID security groups instead. See [Restrict access to app roles](#restrict-access-to-app-roles). -### Configure the application in Entra ID +### Add app roles and v2.0 tokens in Entra ID {#configure-the-application-in-entra-id} 1. Make sure that the application is single-tenant, as set up in [Set up the client application in Entra ID](#set-up-the-client-application-in-entra-id). @@ -448,7 +454,7 @@ QuestDB rejects it until Azure issues a new one. ::: -### Configure QuestDB +### Accept app-only tokens in QuestDB {#configure-questdb} On QuestDB Enterprise 4.0.2 or later, replace the two claim settings of the [QuestDB configuration](#questdb-configuration) with fallback lists, and @@ -557,7 +563,7 @@ With `token_provider=azure`, the from Entra ID by itself, and renews it in the background at about half of its lifetime, so every connection that it opens presents a valid token. Add the `questdb-client-azure` artifact, which brings the Azure Identity library, at -the same version as `questdb-client`: +the same version as `questdb-client`, 1.3.10 or later: /.default for Microsoft Entra ID - "scope": os.environ["QUESTDB_SCOPE"], + # api:///.default for Microsoft Entra ID; requests + # leaves the field out when QUESTDB_SCOPE is not set + "scope": os.environ.get("QUESTDB_SCOPE"), }, timeout=10, ) @@ -823,8 +813,7 @@ with questdb.connect(conf) as db: sender.flush(wait=True) ``` -With `acl.oidc.groups.encoded.in.token=true`, send the ID token instead of the -access token: +In token mode, send the ID token instead of the access token: ```python conf = f"wss::addr=questdb.example.com:9000;token={tokens['id_token']};" @@ -899,6 +888,12 @@ can outlast the token. [Request a token in the service](/docs/security/oidc/entra-id/#request-a-token-in-the-service) shows both orders in Python. +Replay is at-least-once: after a reconnect or a switch to a new client, +QuestDB can receive rows that it has already stored. To avoid duplicates, +declare [`DEDUP UPSERT KEYS`](/docs/concepts/deduplication/) on the table, +covering the identity of a row, such as a trade ID. See +[Delivery semantics](/docs/concepts/delivery-semantics/). + ## OIDC for the PGWire endpoint If the @@ -919,10 +914,10 @@ username field, or leaves the username empty if it can. QuestDB validates the token the same way as over HTTP, and takes the principal and the groups from it, not from the username field. Send the token that an HTTP client would send: -- With the User Info endpoint, the default, an access token. -- With `acl.oidc.groups.encoded.in.token=true`, for a user, an ID token - requested with QuestDB's client ID, and for a service, an access token whose - `aud` claim contains `acl.oidc.audience`, and whose `iss` claim is the +- In User Info mode, the default, an access token. +- In token mode, for a user, an ID token requested with QuestDB's client ID, + and for a service, an access token whose `aud` claim contains + `acl.oidc.audience`, and whose `iss` claim is the [expected issuer](#the-expected-issuer). QuestDB rejects a token for another audience, such as a user's access token for another API, and logs `InvalidAudience`, or `InvalidIssuer` when the token also comes from another @@ -1065,77 +1060,27 @@ user, and every session and connection of the user runs with them. QuestDB picks up a change to the user's groups or status in the Identity Provider, such as a disabled user, when it checks a token of the user again: -- With the User Info endpoint, the default, when the cached user information - expires. [`acl.oidc.cache.ttl`](/docs/configuration/oidc/#acloidccachettl), - 30 seconds by default, controls how long QuestDB caches it. Set it according - to your organization's policies. -- With `acl.oidc.groups.encoded.in.token=true`, when the client presents a new - token. QuestDB reads the groups from the token instead of asking the Identity - Provider, so refreshing the cache does not pick up changes, and the old token - keeps its groups until QuestDB stops accepting it, as described in - [When QuestDB stops accepting a token](#when-questdb-stops-accepting-a-token). +- In User Info mode, when the cached user information expires. + [`acl.oidc.cache.ttl`](/docs/configuration/oidc/#acloidccachettl), 30 + seconds by default, controls how long QuestDB caches it. Set it according to + your organization's policies. +- In token mode, when the client presents a new token. QuestDB reads the groups + from the token instead of asking the Identity Provider, so refreshing the + cache does not pick up changes, and the old token keeps its groups until + QuestDB stops accepting it. For a username and password login through the [ROPC flow](#enable-ropc), QuestDB requests a new token from the provider itself once the cached result -expires, so with either setting, changes take effect after -`acl.oidc.cache.ttl`. - -A token that QuestDB rejects does not change the groups, so some access -outlives the token: +expires, so in both modes, changes take effect after `acl.oidc.cache.ttl`. -- Connections that are already open, such as PGWire and QWP connections, stay - open when their token expires, and keep running with the groups of the last - token that QuestDB accepted for the user. A WebSocket connection that a - browser opened with the cookie of an HTTP session also stays open when the - session ends. -- An HTTP session, such as the one that the Web Console opens at login, - accepts the requests that carry its cookie without a token. QuestDB checks - an OIDC token only when a request carries it, so it does not end such a - session when the token expires. The session ends when the Web Console logs - the user out, or after it has been idle for `http.session.timeout`, 30 - minutes by default. Every request that QuestDB accepts restarts the idle - timer, so a session in use does not expire. A session that the Web Console - does not end, for example because the browser was closed, stays valid for - any client that reuses its cookie without a token, for as long as that - client stays active. - -The Web Console itself sends its token with every query, so its session lasts -only as long as QuestDB accepts the token. QuestDB checks the token of every -request that carries one, and reuses the result of a check for up to -`acl.oidc.cache.ttl`, as described in +A token that QuestDB rejects does not change the groups. Connections that are +already open, such as PGWire and QWP connections, and HTTP sessions that +receive requests without a token, keep running with the groups of the last +token that QuestDB accepted for the user, as described in [When QuestDB stops accepting a token](#when-questdb-stops-accepting-a-token). -When it rejects the token, for example because the token has expired, because -the provider rejects it, or because the user information no longer carries a -group, QuestDB rejects the request with `401 Unauthorized` and ends the -session, and the Web Console logs the user out. The Web Console also logs the -user out when the provider refuses to refresh its tokens, for example after -the user is disabled. It refreshes them only with a refresh token, as described -in [step 6](#6-credentials-received): without one, a disabled user keeps -working until QuestDB stops accepting the token of the login, when the token -expires or, with the User Info endpoint, when the provider rejects it. -Versions before 4.0.2 accept a request with a rejected token through its -session instead, as described in [Versions before 4.0.2](#versions-before-402). - -QuestDB also rejects a token that it cannot check because the provider cannot -be reached or answers with an error, so an outage of the provider can log users -out of the Web Console: - -- With the User Info endpoint, QuestDB asks the provider about a token again - once `acl.oidc.cache.ttl` has passed since it last did. During an outage, it - then rejects the token, and the Web Console logs the user out. -- With `acl.oidc.groups.encoded.in.token=true`, QuestDB validates tokens with - the signing keys that it has already downloaded, so during an outage, it - rejects only the tokens signed with a key that it has not downloaded yet. - The Web Console cannot renew its tokens during the outage, and logs the user - out once the ID token has expired, as described in - [step 6](#6-credentials-received). -- With either setting, username and password logins through the - [ROPC flow](#enable-ropc) fail during an outage once `acl.oidc.cache.ttl` - has passed since QuestDB last checked them, because QuestDB checks them with - the provider's token endpoint. - -Versions before 4.0.2 accept a request that carries such a token through its -HTTP session instead. +In the Web Console, a disabled user is logged out when the provider refuses to +renew the tokens, or when QuestDB stops accepting the token of the login, as +described in [Web Console token renewal](#web-console-token-renewal). To cut off access immediately, revoke the permissions of the QuestDB groups that the user's groups map to. This takes effect at once for every session and @@ -1178,9 +1123,9 @@ principal, which identifies the user, and the groups the user belongs to. The user information is: - By default, the response of the provider's User Info endpoint. -- With `acl.oidc.groups.encoded.in.token=true`, the payload of a JWT: the ID - token, for Web Console logins and in the ROPC flow, or the token that a - client presents, such as an Entra ID app-only access token. +- In token mode, the payload of a JWT: the ID token, for Web Console logins and + in the ROPC flow, or the token that a client presents, such as an Entra ID + app-only access token. Two settings name the claims that QuestDB reads: @@ -1223,6 +1168,10 @@ Never use a display name, such as the `name` claim, because different people can share one. Changing the claim on an existing deployment gives every user a new principal. Permissions stay the same, because they come from the groups. +Avoid principals that can equal the name of a QuestDB user, service account, +or group. Once such an external user has logged in, statements that look the +name up, such as `SHOW PERMISSIONS`, fail with `Ambiguous user name`. + The claim does not apply to username and password logins through the ROPC flow, which use the username that the client sends as the principal. A person who logs in to the Web Console, and with a username and password to the PGWire @@ -1334,8 +1283,8 @@ follows: | `"roles": [null, ""]`, `"groups": ["87654321-..."]` | Groups `87654321-...`, from `groups`: `roles` holds no group name once `null` and `""` are skipped | | `"roles": ["QuestDB.FxIngest"]`, `"groups": {"id": "x"}` | The login fails with `unexpected input format`: `groups` is listed and holds an object, even though `roles` carries a group | -With `acl.oidc.groups.encoded.in.token=true`, QuestDB validates the token before -it applies these rules. See [Token validation](#token-validation). +In token mode, QuestDB validates the token before it applies these rules. See +[Token validation](#token-validation). ### Startup validation @@ -1356,8 +1305,7 @@ refuses to start when: ## Token validation -With `acl.oidc.groups.encoded.in.token=true`, QuestDB validates the token -itself before it reads any claim: +In token mode, QuestDB validates the token itself before it reads any claim: - The token must be a JWT whose `kid` header names one of the signing keys that the provider publishes. @@ -1374,8 +1322,8 @@ itself before it reads any claim: - It must carry a non-empty `sub` claim, whatever claims `acl.oidc.sub.claim` lists. -With `acl.oidc.groups.encoded.in.token=false`, the default, QuestDB expects an -access token, and treats it as opaque, even when it is a JWT: QuestDB does not +In User Info mode, the default, QuestDB expects an access token, and treats it +as opaque, even when it is a JWT: QuestDB does not decode the token, and leaves its validation to the provider. It sends the token to the provider's User Info endpoint, which validates it and returns the user information. `acl.oidc.audience` and `acl.oidc.issuer` do not apply. See @@ -1383,8 +1331,7 @@ information. `acl.oidc.audience` and `acl.oidc.issuer` do not apply. See ### The expected issuer -In QuestDB Enterprise 4.0.2 and later, with -`acl.oidc.groups.encoded.in.token=true`, QuestDB accepts tokens from a single +In token mode, QuestDB Enterprise 4.0.2 and later accept tokens from a single issuer, the expected issuer: - [`acl.oidc.issuer`](/docs/configuration/oidc/#acloidcissuer), when it is @@ -1422,10 +1369,37 @@ Set `acl.oidc.issuer` when: Keycloak without a fixed hostname, and QuestDB reaches the provider through another URL than the clients, such as an internal one. The discovery document that QuestDB downloads then names another issuer than the tokens. Set the - issuer that the tokens carry. + issuer that the tokens carry. This fixes token validation only: the Web + Console sends browsers to the authorization and token endpoints of the same + discovery document, so Web Console logins fail unless browsers can reach the + internal URL too. For the Web Console, give the provider a fixed public + hostname, such as with the `hostname` option of Keycloak, and let QuestDB + reach the provider at that hostname, for example through DNS that resolves it + to an internal address. The discovery document then names the issuer of the + tokens, and `acl.oidc.issuer` is not needed. + +For example, to validate the tokens of a Keycloak realm with `acl.oidc.host` +instead of a discovery document: -The setting takes a single value, so every token that QuestDB accepts must -carry the same issuer. The issuer can depend on the kind of token, not just on +```ini title="server.conf" +acl.oidc.enabled=true +acl.oidc.client.id=questdb +acl.oidc.groups.claim=groups +acl.oidc.groups.encoded.in.token=true + +# the provider and its endpoints, instead of acl.oidc.configuration.url +acl.oidc.host=keycloak.example.com +acl.oidc.authorization.endpoint=/realms/markets/protocol/openid-connect/auth +acl.oidc.token.endpoint=/realms/markets/protocol/openid-connect/token +acl.oidc.userinfo.endpoint=/realms/markets/protocol/openid-connect/userinfo +acl.oidc.public.keys.endpoint=/realms/markets/protocol/openid-connect/certs + +# required without a discovery document: the iss claim of the tokens +acl.oidc.issuer=https://keycloak.example.com/realms/markets +``` + +`acl.oidc.issuer` takes a single value, so every token that QuestDB accepts +must carry the same issuer. The issuer can depend on the kind of token, not just on the provider. Microsoft Entra ID puts `https://login.microsoftonline.com//v2.0` into v2.0 tokens, and `https://sts.windows.net//` into v1.0 tokens. With the v2.0 @@ -1438,12 +1412,17 @@ Web Console users and the access tokens of services, make the provider issue access tokens with the issuer of its ID tokens. For Entra ID, see [Managed identities and service principals](/docs/security/oidc/entra-id/#managed-identities-and-service-principals). -For a provider that issues tokens without an `iss` claim, turn the check off -with -[`acl.oidc.issuer.validation.enabled=false`](/docs/configuration/oidc/#acloidcissuervalidationenabled). -QuestDB then ignores the `iss` claim, needs no expected issuer to start, and -still checks the signature, the audience, the expiry, and the subject. It logs -this advisory at startup: +For a provider that issues tokens without an `iss` claim, turn the check off: + +```ini title="server.conf" +acl.oidc.issuer.validation.enabled=false +``` + +With +[`acl.oidc.issuer.validation.enabled=false`](/docs/configuration/oidc/#acloidcissuervalidationenabled), +QuestDB ignores the `iss` claim, needs no expected issuer to start, and still +checks the signature, the audience, the expiry, and the subject. It logs this +advisory at startup: ```text OIDC JWT issuer validation is disabled; tokens from other issuers or tenants may authenticate when signing keys and audience are shared @@ -1462,44 +1441,94 @@ it, and caches the result for up to [`acl.oidc.cache.ttl`](/docs/configuration/oidc/#acloidccachettl), 30 seconds by default: -- With `acl.oidc.groups.encoded.in.token=true`, QuestDB Enterprise 4.0.2 and - later accept a token for up to 60 seconds, the allowed clock skew, plus - `acl.oidc.cache.ttl` after it expires. After that, they reject it on every - request and new connection, including reconnects. Earlier versions do not - check whether the token has expired. -- With the User Info endpoint, the provider rejects an expired token. QuestDB - asks the provider again once the user information that it cached for the - token expires. - -Some access outlives the token. A PGWire or WebSocket connection, such as a QWP -connection, that is already open stays open when its token expires, and an HTTP -session accepts the requests that carry its cookie without a token. A request -that carries a token that QuestDB no longer accepts fails, even with a valid -session cookie, and ends the session. See -[Group changes, sessions, and revoking access](#group-changes-sessions-and-revoking-access). -For what a client must do before its token expires, see +- In token mode, QuestDB Enterprise 4.0.2 and later accept a token for up to + 60 seconds, the allowed clock skew, plus `acl.oidc.cache.ttl` after it + expires. After that, they reject it on every request and new connection, + including reconnects. Earlier versions do not check whether the token has + expired. +- In User Info mode, the provider rejects an expired token. QuestDB asks the + provider again once the user information that it cached for the token + expires. + +QuestDB also rejects a token that it cannot check, because the provider cannot +be reached or answers with an error: + +- In User Info mode, once `acl.oidc.cache.ttl` has passed since QuestDB last + asked the provider about the token. +- In token mode, only a token signed with a key that QuestDB has not + downloaded yet: QuestDB validates tokens with the signing keys that it has + already downloaded. +- Username and password logins through the [ROPC flow](#enable-ropc) fail in + both modes once `acl.oidc.cache.ttl` has passed since QuestDB last checked + them, because QuestDB checks them with the provider's token endpoint. + +In QuestDB Enterprise 4.0.2 and later, a request that carries a token that +QuestDB rejects, or cannot check, fails with `401 Unauthorized`, even with a +valid session cookie, and ends the HTTP session. An outage of the provider can +therefore log users out of the Web Console. Versions before 4.0.2 accept such +a request through its session instead. + +Some access outlives the token: + +- Connections that are already open, such as PGWire and QWP connections, stay + open when their token expires, and keep running with the groups of the last + token that QuestDB accepted for the user. A WebSocket connection that a + browser opened with the cookie of an HTTP session also stays open when the + session ends. +- An HTTP session, such as the one that the Web Console opens at login, + accepts the requests that carry its cookie without a token. QuestDB checks + an OIDC token only when a request carries it, so it does not end such a + session when the token expires. The session ends when the client logs out, + or after it has been idle for + [`http.session.timeout`](/docs/configuration/http-server/#httpsessiontimeout), + 30 minutes by default. Every request that QuestDB accepts restarts the idle + timer. A session that the Web Console does not end, for example because the + browser was closed, stays valid for any client that reuses its cookie + without a token, for as long as that client stays active. + +For how the Web Console renews its token, see +[Web Console token renewal](#web-console-token-renewal). For what other +clients must do before their token expires, see [Token lifetime](#token-lifetime). -With a refresh token, the Web Console requests new tokens before the token that -it sends QuestDB expires, the ID token with -`acl.oidc.groups.encoded.in.token=true`, as described in -[step 6](#6-credentials-received). When the clock of the browser is behind, -QuestDB can still reject the token as expired first: the Web Console then +### Web Console token renewal + +The Web Console sends its token with every query: the access token in User +Info mode, or the ID token in token mode. It renews its tokens only with a +refresh token. Some providers, such as Microsoft Entra ID, issue one only for +the `offline_access` scope, which +[`acl.oidc.scope`](/docs/configuration/oidc/#acloidcscope) sets. + +With a refresh token, the Web Console requests new tokens before it sends a +query, when the token that it sends expires within 30 seconds. It reads the +expiry of the ID token from the token's `exp` claim, and the expiry of the +access token from the `expires_in` field of the token response. It does not +renew a token whose expiry it does not know. When the clock of the browser is +behind, QuestDB can reject the token as expired first: the Web Console then renews its tokens and sends the request again. -Without a refresh token, the Web Console never requests new tokens, and keeps -sending the token of the login after it has expired. QuestDB Enterprise 4.0.2 -and later then reject the expired ID token, log -`Failed to decode JWT token Error(ExpiredSignature)`, reject the request with -`401 Unauthorized`, and end the session of the Web Console. The Web Console -logs the user out, and the user has to log in again. With Microsoft Entra ID, -whose ID tokens are valid for one hour by default, this happens about an hour -after the login, unless -[`acl.oidc.scope`](/docs/configuration/oidc/#acloidcscope) includes -`offline_access`, the scope that Entra ID issues refresh tokens for. +- When the provider refuses to issue new tokens, for example because the user + is disabled or the refresh token has expired, the Web Console logs the user + out, and the login page shows the reason that the provider gives. +- When the Web Console cannot reach the provider, or the provider answers with + `429` or a `5xx` status, it keeps sending the current token while the token + is valid, warns the user, and tries again with a growing delay of up to 30 + seconds. Once the token has expired, the Web Console stops sending queries + with it, and after three more failed attempts, it logs the user out. + +Without a refresh token, the Web Console keeps sending the token of the login +after it has expired. QuestDB Enterprise 4.0.2 and later then reject the token, +in token mode with `Failed to decode JWT token Error(ExpiredSignature)` in the +log, answer with `401 Unauthorized`, and end the session, and the Web Console +logs the user out. With Microsoft Entra ID, whose ID tokens are valid for one +hour by default, this happens about an hour after the login. Earlier versions +keep accepting the requests of the Web Console, as described in +[Versions before 4.0.2](#versions-before-402). ## Troubleshooting OIDC logins +### Missing principal or groups claims + When the user information carries none of the listed principal claims, or none of the listed groups claims, QuestDB rejects the login. HTTP clients receive `401 Unauthorized`, and PGWire clients receive `invalid username/password`. The @@ -1528,6 +1557,8 @@ member of, so a user who is in a mapped group only through a nested group loses that group. Assigning groups to an application requires Microsoft Entra ID P1 or P2. +### Token validation errors + A token that fails [token validation](#token-validation) is rejected the same way, but the server log gives the reason instead of the claim lists, for example: @@ -1536,24 +1567,31 @@ example: Request failed [errorMsg=Failed to decode JWT token Error(InvalidAudience)] Request failed [errorMsg=Failed to decode JWT token Error(InvalidIssuer)] Request failed [errorMsg=Failed to decode JWT token Error(MissingRequiredClaim("iss"))] +Request failed [errorMsg=Failed to decode JWT token Error(MissingRequiredClaim("exp"))] Request failed [errorMsg=Failed to decode JWT token Error(ExpiredSignature)] +Request failed [errorMsg=Failed to decode JWT token Error(InvalidSignature)] Request failed [errorMsg=Unable to find public key with key id [...]] ``` +QuestDB checks the signature of the token first, then that the token carries +the `sub`, `aud`, and `exp` claims, and with issuer validation, the `iss` +claim, then the expiry, the issuer, and the audience, in this order, and logs +the first check that fails. For example, an expired token without an `iss` +claim logs `MissingRequiredClaim("iss")`. + `InvalidAudience` usually means that the token was requested for another API, or, with issuer validation disabled, that Entra ID issued a v1.0 access token. `InvalidIssuer` means that the `iss` claim of the token is not the expected issuer, and `MissingRequiredClaim("iss")` that the token carries no `iss` -claim, as described in [The expected issuer](#the-expected-issuer). QuestDB -checks the expiry before the issuer, and the issuer before the audience, and -logs the first check that fails. With Microsoft Entra ID, `InvalidIssuer` -usually means a v1.0 access token, whose issuer is -`https://sts.windows.net//`, or a token of another tenant. When -every login fails with it, compare the `iss` claim of a token with the -expected issuer: `acl.oidc.issuer`, or the `issuer` field of the discovery -document. If the discovery document names a template rather than your tenant, -or another host than the `iss` claim of the tokens, set `acl.oidc.issuer`. +claim, as described in [The expected issuer](#the-expected-issuer). With +Microsoft Entra ID, `InvalidIssuer` usually means a v1.0 access token, whose +issuer is `https://sts.windows.net//`, or a token of another +tenant. When every login fails with it, compare the `iss` claim of a token +with the expected issuer: `acl.oidc.issuer`, or the `issuer` field of the +discovery document. If the discovery document names a template rather than +your tenant, or another host than the `iss` claim of the tokens, set +`acl.oidc.issuer`. For a v1.0 access token of Entra ID, make the application issue v2.0 tokens, as described in @@ -1563,15 +1601,16 @@ to the issuer of v1.0 tokens, to work around it: each setting takes a single value, and the ID tokens of Web Console users carry the application ID and the v2.0 issuer, so their logins would fail. -`ExpiredSignature` means that the token has expired. A client configured with a -fixed token must switch to a new token before then, as described in -[Token lifetime](#token-lifetime). For Web Console users, it usually means that -the Web Console has no refresh token, because -[`acl.oidc.scope`](/docs/configuration/oidc/#acloidcscope) does not include the -scope that the provider issues refresh tokens for, such as `offline_access` -with Microsoft Entra ID. QuestDB then rejects the requests, and the Web Console -logs the user out once the ID token of the login has expired. See -[When QuestDB stops accepting a token](#when-questdb-stops-accepting-a-token). +`ExpiredSignature` means that the token has expired, and +`MissingRequiredClaim("exp")` that it carries no `exp` claim, which QuestDB +Enterprise 4.0.2 and later require. A client configured with a fixed token must +switch to a new token before it expires, as described in +[Token lifetime](#token-lifetime). For Web Console users, `ExpiredSignature` +usually means that the Web Console has no refresh token. See +[Web Console token renewal](#web-console-token-renewal). + +`InvalidSignature` means that the signature of the token does not match the +signing key that its `kid` header names. `Unable to find public key with key id` means that none of the signing keys that the provider publishes matches the token. QuestDB downloads the keys again @@ -1579,22 +1618,27 @@ when they are older than [`acl.oidc.public.keys.expiry`](/docs/configuration/oidc/#acloidcpublickeysexpiry), and when it meets a key ID that it does not know, so either another provider issued the token, or QuestDB cannot reach the provider's keys, which it logs as -`Unable to download public keys from OIDC provider`. +`Unable to download public keys from OIDC provider`. A token whose header names +no key ID logs `Key id is missing from JWT token header`. + +QuestDB also rejects a token whose `sub` claim is missing or empty, whatever +claims `acl.oidc.sub.claim` lists. It then logs a JSON error that names `sub`, +or `Sub is not set in token`. -With `acl.oidc.groups.encoded.in.token=true`, QuestDB accepts only a JWT. It -rejects a token that is not one, such as an opaque access token, and logs the -whole token: +In token mode, QuestDB accepts only a JWT. It rejects a token that is not one, +such as an opaque access token, and logs: ```text -Request failed [errorMsg=Invalid token format, cannot find header separator, expected format of JWT:
.. [token=...]] +Request failed [errorMsg=Invalid token format, cannot find header separator, expected format of JWT:
..] ``` -Send a JWT instead, as described in -[Non-interactive clients](#non-interactive-clients). +A token with a single `.` logs `cannot find signature separator` instead. Send +a JWT, as described in [Non-interactive clients](#non-interactive-clients). -With `acl.oidc.groups.encoded.in.token=false`, the provider's User Info -endpoint rejects a token that is invalid or expired, and QuestDB rejects the -login. +In User Info mode, the provider's User Info endpoint rejects a token that is +invalid or expired, and QuestDB rejects the login. + +### Requests denied after a successful login If the login succeeds but every request is denied, check that at least one of the user's groups is mapped to a QuestDB group that has the endpoint @@ -1602,6 +1646,8 @@ permission of the interface: `HTTP` for the Web Console, the REST API, ILP/HTTP, and QWP, or `PGWIRE` for the PGWire endpoint. See [Mapping user permissions](#mapping-user-permissions). +### Ingestion cannot create a table or add a column + If the login succeeds but ingestion fails when it creates a table or adds a column, create the table and its columns in advance: ingestion cannot create a table or add a column for an external user. The error contains @@ -1612,63 +1658,61 @@ instead, and the column stays in the table. See ## Versions before 4.0.2 -QuestDB Enterprise versions before 4.0.2 differ as follows: - -- They read the whole value of each setting as a single claim name, so a claim - list makes every OIDC login fail. -- With `acl.oidc.groups.encoded.in.token=true`, they reject any token that does - not carry a `groups` array, whatever `acl.oidc.groups.claim` names. They do - not check the `exp` claim, so they accept expired tokens and tokens without - an `exp` claim. They do not check the `iss` claim either, so they accept the - tokens of any issuer that the provider's keys sign for the expected - audience, such as the tokens of another Microsoft Entra ID tenant. -- When an HTTP request carries a session cookie and a token that they reject, - or cannot check because the provider is unreachable, they accept the request - through the session. The Web Console keeps working with a token that they no - longer accept, until its session expires. -- They read a `null` claim as the value `null`. Every account whose principal - claim is `null` logs in as the same principal, `null`, and `null` and empty - group names count as groups. -- They reject a login when a claim that is not listed holds an array of - objects or nested arrays. -- They start with an empty `acl.oidc.sub.claim`, or with the same claim in both - settings, and then reject every OIDC login. - -When you upgrade from an earlier version: - -- With `acl.oidc.groups.encoded.in.token=true`, QuestDB rejects expired tokens - and tokens without an `exp` claim. A client that keeps using a token after it - expires, such as a client configured with a fixed token, must request a new - token before the old one expires. The Web Console renews the ID token before - it expires, but only with a refresh token: without one, it logs the user out - when the ID token of the login expires, about an hour after the login with - Microsoft Entra ID, where earlier versions accept the expired ID token. To - keep users logged in, add the scope that the provider issues refresh tokens - for, such as `offline_access` with Entra ID, to - [`acl.oidc.scope`](/docs/configuration/oidc/#acloidcscope). See +QuestDB Enterprise 4.0.2 changes how QuestDB validates tokens, reads claims, +and handles HTTP sessions. For each change, the list below gives how earlier +versions behave, and what to check when you upgrade. + +### Upgrade to 4.0.2 {#upgrade-to-402} + +- **Token expiry.** In token mode, earlier versions do not check the `exp` + claim, so they accept expired tokens and tokens without an `exp` claim. + QuestDB Enterprise 4.0.2 rejects them, as described in [When QuestDB stops accepting a token](#when-questdb-stops-accepting-a-token). -- With `acl.oidc.groups.encoded.in.token=true`, QuestDB rejects tokens whose - `iss` claim is missing or is not the expected issuer, and refuses to start - when it has no expected issuer, such as with `acl.oidc.host`: set - `acl.oidc.issuer`. With Microsoft Entra ID, it rejects v1.0 access tokens. - For a provider that issues tokens without an `iss` claim, set + A client configured with a fixed token must switch to a new token before the + old one expires, as described in [Token lifetime](#token-lifetime). The Web + Console renews its tokens only with a refresh token: to keep users logged + in, add the scope that the provider issues refresh tokens for, such as + `offline_access` with Microsoft Entra ID, to + [`acl.oidc.scope`](/docs/configuration/oidc/#acloidcscope). See + [Web Console token renewal](#web-console-token-renewal). +- **Issuer.** In token mode, earlier versions do not check the `iss` claim, so + they accept the tokens of any issuer that the provider's keys sign for the + expected audience, such as the tokens of another Microsoft Entra ID tenant. + QuestDB Enterprise 4.0.2 rejects tokens whose `iss` claim is missing or is + not the expected issuer, including Microsoft Entra ID v1.0 access tokens + with the v2.0 discovery URL, and refuses to start when it has no expected + issuer, such as with `acl.oidc.host`. Set `acl.oidc.issuer`, or, for a + provider whose tokens carry no `iss` claim, `acl.oidc.issuer.validation.enabled=false`. See [The expected issuer](#the-expected-issuer). -- A request that carries a session cookie and a token that QuestDB rejects, or - cannot check because the provider is unreachable, fails with - `401 Unauthorized`, and ends the session, instead of running through the - session. An outage of the provider can therefore log users out of the Web - Console, as described in - [Group changes, sessions, and revoking access](#group-changes-sessions-and-revoking-access). -- QuestDB refuses to start with an empty `acl.oidc.sub.claim`, or with a claim - that is listed twice or in both settings. See +- **HTTP sessions.** When a request carries a session cookie and a token that + they reject, or cannot check because the provider is unreachable, earlier + versions accept the request through the session, so the Web Console keeps + working with a token that they no longer accept, until its session expires. + QuestDB Enterprise 4.0.2 rejects the request with `401 Unauthorized` and ends + the session, so an outage of the provider can log users out of the Web + Console. See + [When QuestDB stops accepting a token](#when-questdb-stops-accepting-a-token). +- **Claim lists.** Earlier versions read the whole value of each setting as a + single claim name, so a claim list makes every OIDC login fail. In a + replicated cluster, upgrade every node before you change a setting to a + claim list. +- **Groups in tokens.** In token mode, earlier versions reject any token that + does not carry a `groups` array, whatever `acl.oidc.groups.claim` names. + QuestDB Enterprise 4.0.2 reads the claims that the setting lists. +- **`null` and empty claims.** Earlier versions read a `null` claim as the + value `null`: every account whose principal claim is `null` logs in as the + same principal, `null`, and `null` and empty group names count as groups. + QuestDB Enterprise 4.0.2 rejects such logins, unless another listed claim + carries a value. See [How QuestDB picks a claim](#how-questdb-picks-a-claim). +- **Unlisted claims.** Earlier versions reject a login when a claim that is + not listed holds an array of objects or nested arrays. QuestDB Enterprise + 4.0.2 ignores the claims that are not listed, whatever their shape. +- **Startup checks.** Earlier versions start with an empty + `acl.oidc.sub.claim`, or with the same claim in both settings, and then + reject every OIDC login. QuestDB Enterprise 4.0.2 refuses to start instead, + and also when a setting lists the same claim twice. See [Startup validation](#startup-validation). -- QuestDB rejects logins whose principal claim is `null`, or whose groups claim - holds only `null` or empty group names, unless another listed claim carries a - value. Earlier versions let them log in. -- In a replicated cluster, upgrade every node before you change a setting to a - claim list. Earlier versions reject every OIDC login when a setting lists - more than one claim.
@@ -1683,5 +1727,6 @@ a managed identity or a service principal. ### PingFederate The [PingFederate guide](/docs/security/oidc/pingfederate/) sets up single -sign-on for the Web Console, with PingFederate authenticating users against -Microsoft Entra ID over LDAP. +sign-on for the Web Console, with PingFederate authenticating users against an +Active Directory domain, such as Microsoft Entra Domain Services, over secure +LDAP. diff --git a/documentation/security/oidc/pingfederate.mdx b/documentation/security/oidc/pingfederate.mdx index 935eed7dbf..27dc77c4db 100644 --- a/documentation/security/oidc/pingfederate.mdx +++ b/documentation/security/oidc/pingfederate.mdx @@ -25,6 +25,12 @@ no LDAP endpoint. An on-premises Active Directory works the same way. For how QuestDB uses OIDC in general, such as the login flow and how it maps groups to permissions, see [OpenID Connect (OIDC)](/docs/security/oidc/). +This guide covers Web Console logins, and username and password logins through +the ROPC flow, with QuestDB reading the groups from the User Info endpoint of +PingFederate. It does not set up tokens that services request for themselves: +those usually need QuestDB to read the groups from a JWT instead, as described +in [Non-interactive clients](/docs/security/oidc/#non-interactive-clients). + ## Set up PingFederate client First thing first, let's pick a name for the client! From ad98bb9373070fb668c0c4e4364c3f3a398c3a81 Mon Sep 17 00:00:00 2001 From: glasstiger Date: Thu, 15 Oct 2026 11:23:35 +0100 Subject: [PATCH 16/29] dummy commit to trigger PR sync --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index b3870a783a..e85b133ecb 100644 --- a/README.md +++ b/README.md @@ -33,7 +33,7 @@ docker-compose up ## Contributing -You'll need to sign a quick CLA. +You will need to sign a quick CLA. Pretty standard, [check it out](https://cla-assistant.io/questdb/documentation). From 72c623c16b6fa8db2437bec4e12a456a434b0e0a Mon Sep 17 00:00:00 2001 From: glasstiger Date: Thu, 15 Oct 2026 12:52:12 +0100 Subject: [PATCH 17/29] docs(oidc): fix token recovery and client reference inconsistencies --- .../connect/clients/connect-string.md | 38 ++- documentation/security/oidc/entra-id.mdx | 221 +++++++++--------- documentation/security/oidc/index.mdx | 118 ++++++++-- 3 files changed, 247 insertions(+), 130 deletions(-) diff --git a/documentation/connect/clients/connect-string.md b/documentation/connect/clients/connect-string.md index e6371a5088..5caa89a390 100644 --- a/documentation/connect/clients/connect-string.md +++ b/documentation/connect/clients/connect-string.md @@ -687,10 +687,22 @@ architecture is that a producer survives an arbitrarily long outage. Auth failures during reconnect (authentication rejected, version mismatch, durable-ack mismatch, non-101 upgrade without a role hint) are immediately -terminal: the loop does not retry them. The Java client is the exception for -authentication, upgrade, and durable-ack rejections: it treats them as terminal -only for the first connect of a sender and for orphan drainers, and a sender -that has connected keeps retrying them, as described in +terminal in clients other than Java: the loop does not retry them. In Java, +a sender that has connected keeps retrying authentication, upgrade, and +durable-ACK rejections. On the first connect, those rejections are terminal. + +Java orphan drainers have a separate policy: + +- Authentication rejection with a fixed credential, and non-retriable upgrade + rejection, fail immediately. +- With a token provider, `401`/`403` rejections receive bounded retries with + fresh credentials before the drainer quarantines the slot. +- A durable-ACK capability mismatch receives bounded retries to allow a + rolling upgrade to settle before the drainer quarantines the slot. +- Role rejections while all reachable servers are replicas are transient: + the drainer keeps retrying rather than quarantining the slot. + +For the token-rejection thresholds and reporting, see [How the client handles token failures](/docs/connect/clients/java/#how-the-client-handles-token-failures). ### Egress failover {#egress-failover} @@ -819,14 +831,16 @@ explicit setter always wins over the string. (30 min). - `housekeeper_interval_ms` — how often the housekeeper checks for idle and over-age connections. Default: `5000`. -- `lazy_connect` — when `on`, the pool defers opening its first connection - until the first borrow, so construction succeeds against a server that is - down. This is the supported way to tolerate a server that starts after your - application, and in the Java client, an identity provider that is - unreachable when a token provider requests its first token. In the Java, C, - C++, Python, and Rust clients, it cannot be combined with `query_pool_min` - above `0`, or with an `initial_connect_retry` other than `async`: the client - rejects the configuration. Default: `off`. +- `lazy_connect` — when `on`, construction does not wait for a connection, + so the application can start while the server is down. In the Java, C, + C++, Python, and Rust clients, senders connect asynchronously in the + background, while query connections open on first use. Networking can + therefore begin before the first borrow. In Java, this also lets the + application start while the identity provider is unreachable when a token + provider requests its first token. In these clients, `lazy_connect` cannot + be combined with `query_pool_min` above `0`, or with an + `initial_connect_retry` other than `async`: the client rejects the + configuration. Default: `off`. ## Error handling {#error-handling} diff --git a/documentation/security/oidc/entra-id.mdx b/documentation/security/oidc/entra-id.mdx index cda86cd81c..e953bd99e2 100644 --- a/documentation/security/oidc/entra-id.mdx +++ b/documentation/security/oidc/entra-id.mdx @@ -717,153 +717,163 @@ first one, the additional connections of its pool, and the connections that replace a failed one. Shortly after the token expires, QuestDB rejects these new connections, as described in [When QuestDB stops accepting a token](/docs/security/oidc/#when-questdb-stops-accepting-a-token). -A long-running -service must therefore switch to a new token before the old one expires: it -connects with the new token, then closes the old handle. -`token.expires_on` is a Unix timestamp. +A long-running service must therefore switch to a new token before the old +one expires. `token.expires_on` is a Unix timestamp. + +The short example above keeps its store-and-forward queue in memory. Closing +that handle can discard unacknowledged rows, so replacing an in-memory handle +does not guarantee lossless recovery across an outage and token expiry. For +pending rows to survive handle replacement, use +[`sf_dir`](/docs/connect/clients/connect-string/#sf-keys) and retain the same +`sender_id`. Two handles cannot hold the same disk slot: release every sender +lease and close the old handle before opening its replacement. The new handle +replays the rows left on disk. The credential caches the token in memory, and requests a new one when the cached token expires within 5 minutes, or earlier when the token's refresh -time has passed, so calling `get_token()` often is cheap. The following example -looks for a new token in the last 10 minutes of the old one. When it cannot get -a new token or connect with it, it keeps the old token and handle, and tries -again with the next batch: - -```python title="Keep ingesting across token expiry" +time has passed, so calling `get_token()` often is cheap. The following +single-producer example looks for a new token in the last 10 minutes of the +old one. It retries token acquisition and transient connection failures, +including at startup. While token acquisition or reconnection is retrying, +it pauses ingestion. + +After publishing a batch once with `flush()`, the example calls `wait(5000)`. +This is a five-second no-progress timeout, not a wall-clock deadline. On +`FailoverRetry`, the rows stay queued; the example returns to the outer loop +so that renewal remains reachable. It does not submit the timed-out batch +again. If the old token expires during the wait and the connection fails with +`AuthError`, it releases the lease and replaces the handle with a current +token. Other authentication errors and terminal server rejections stop the +example instead of being retried indefinitely. + +```python title="Rotate tokens with disk-backed store-and-forward" import time import questdb from azure.core.exceptions import AzureError from azure.identity import DefaultAzureCredential -from questdb import QuestDBError, TimestampNanos +from questdb import QuestDBError, QuestDBErrorCode, TimestampNanos SCOPE = "api://8de84b90-1ea5-4e41-9e84-dba860aa01a6/.default" ADDR = "questdb.example.com:9000" -# look for a new token when the old one expires within this many seconds +SF_CONF = "sf_dir=/var/lib/fx-ingest/sf;sender_id=fx-ingest;" REFRESH_MARGIN_SECONDS = 600 +RETRY_SECONDS = 5 +WAIT_TIMEOUT_MILLIS = 5000 +TRANSIENT_CONNECT_ERRORS = { + QuestDBErrorCode.CouldNotResolveAddr, + QuestDBErrorCode.SocketError, + QuestDBErrorCode.ConnectTimeout, + QuestDBErrorCode.FailoverRetry, +} credential = DefaultAzureCredential() def on_rejection(error): - # every rejection; a terminal one, such as a missing grant, also - # raises QuestDBServerRejectionError from flush(wait=True) print("rejected:", error.category.tag, error.message) -def connect(token): - conf = f"wss::addr={ADDR};token={token.token};" - return questdb.connect(conf, error_handler=on_rejection) +def get_token_with_retry(): + while True: + try: + return credential.get_token(SCOPE) + except AzureError as e: + print("token request failed, retrying:", e) + time.sleep(RETRY_SECONDS) + + +def connect_with_retry(): + while True: + # Obtain a current token on every attempt: the outage may outlast it. + token = get_token_with_retry() + conf = ( + f"wss::addr={ADDR};token={token.token};{SF_CONF}" + "sender_pool_min=1;sender_pool_max=1;query_pool_min=0;" + "auto_flush=off;close_flush_timeout_millis=1000;" + ) + try: + return token, questdb.connect(conf, error_handler=on_rejection) + except QuestDBError as e: + expired = time.time() >= token.expires_on + if e.code not in TRANSIENT_CONNECT_ERRORS and not ( + e.code is QuestDBErrorCode.AuthError and expired + ): + raise + print("connect failed, retrying:", e) + time.sleep(RETRY_SECONDS) def renew(token, db): - # returns the token and the handle to use from now on - try: - new_token = credential.get_token(SCOPE) - # the credential returns the cached token until it renews it - if new_token.token == token.token: - return token, db - new_db = connect(new_token) - except (AzureError, QuestDBError) as e: - # keep the old token and handle, and try again with the next batch - print("token renewal failed:", e) + new_token = get_token_with_retry() + if new_token.token == token.token: return token, db + # No sender lease is held here. Release the slot before reopening it. db.close() - return new_token, new_db + return connect_with_retry() def next_trades(): - # replace with your source of trades + # Replace with your source of trades and its acknowledgement policy. time.sleep(1) return [("EURUSD", "buy", 1.0842, 100000.0)] -token = credential.get_token(SCOPE) -db = connect(token) +token, db = connect_with_retry() try: while True: - trades = next_trades() if time.time() > token.expires_on - REFRESH_MARGIN_SECONDS: token, db = renew(token, db) + replace_handle = False with db.sender() as sender: - for symbol, side, price, quantity in trades: + for symbol, side, price, quantity in next_trades(): sender.row( "fx_trades", symbols={"symbol": symbol, "side": side}, columns={"price": price, "quantity": quantity}, at=TimestampNanos.now(), ) - sender.flush(wait=True) + # Publish once. Do not rebuild this batch after a wait timeout. + sender.flush() + try: + sender.wait(WAIT_TIMEOUT_MILLIS) + except QuestDBError as e: + if e.code is QuestDBErrorCode.FailoverRetry: + print("no acknowledgement progress; rows remain on disk") + elif ( + e.code is QuestDBErrorCode.AuthError + and time.time() >= token.expires_on + ): + replace_handle = True + else: + raise + if replace_handle: + # The lease has been returned; the replacement replays its rows. + db.close() + token, db = connect_with_retry() finally: db.close() ``` -A terminal rejection raises `QuestDBServerRejectionError` from -`flush(wait=True)` and ends the loop, and the `finally` block closes the -handle. To keep the service running instead, catch the error around the batch, -and decide whether to retry or drop the batch: a batch fails the same way -until you fix the cause of the rejection, such as a missing grant. - -With store-and-forward on disk, described next, the client keeps the rejected -batch instead: it stays in the slot, and the next connection that takes over -the slot, in this handle or a new one, sends it again before any later batch. -Later batches then fail the same way until QuestDB accepts the rejected batch, -for a missing grant once you grant it, so don't send the batch again. - -The example keeps its store-and-forward data in memory, the default. With -[`sf_dir`](/docs/connect/clients/connect-string/#sf-keys), the client keeps the -rows that QuestDB has not acknowledged on disk, in a slot named after its -`sender_id`, and two handles cannot hold the same slot. `connect()` with the -new token then raises `QuestDBError`, because the old handle still holds the -slot, so `renew()` above never switches to the new token, and the service fails -once it has to open a new connection after QuestDB stops accepting the old -token. Close the old handle first instead, then connect with the new token and -the same `sender_id`: the new handle takes over the slot, and replays what the -old one left on disk. The old handle is closed by then, so when the new -connection fails, retry it instead of falling back to the old handle. With -`sf_dir`, replace `connect()` and `renew()` in the example with: - -```python title="Switch to a new token with store-and-forward on disk" -# the store-and-forward slot of the service -SF_CONF = "sf_dir=/var/lib/fx-ingest/sf;sender_id=fx-ingest;" - - -def connect(token): - conf = f"wss::addr={ADDR};token={token.token};{SF_CONF}" - return questdb.connect(conf, error_handler=on_rejection) - - -def renew(token, db): - # returns the token and the handle to use from now on - try: - new_token = credential.get_token(SCOPE) - except AzureError as e: - # keep the old token and handle, and try again with the next batch - print("token renewal failed:", e) - return token, db - # the credential returns the cached token until it renews it - if new_token.token == token.token: - return token, db - # release the slot; the rows that QuestDB has not acknowledged stay on disk - db.close() - while True: - try: - # the new handle takes over the slot and replays its rows - return new_token, connect(new_token) - except QuestDBError as e: - # the old handle is closed, so retry instead of falling back to it - print("connect failed, retrying in 5 seconds:", e) - time.sleep(5) - try: - # an outage can outlast the token, so get a current one - new_token = credential.get_token(SCOPE) - except AzureError as e: - print("token renewal failed:", e) -``` - -Replay is at-least-once, so the table can receive rows that it has already -stored. For the rules that apply to every client, including how to avoid -duplicates with `DEDUP UPSERT KEYS`, see +After a wait timeout, the example can queue later batches while the server is +unreachable. Store-and-forward is bounded: if the disk queue fills, +publication can fail, and this example stops. Size and monitor the queue for +your outage budget. Store-and-forward protects only rows already accepted by +the local queue; a real trade source also needs a policy for retaining and +acknowledging records not yet published to it. See +[Delivery and durability](/docs/connect/clients/python/#delivery-and-durability). + +A terminal rejection raises `QuestDBServerRejectionError` and ends the loop; +the `finally` block closes the handle. The rejected batch stays on disk and +is replayed before later batches when the service restarts with the same +`sender_id`. Fix the cause, such as a missing grant, before restarting. Do not +submit the rejected batch again. + +Keep `sf_dir` on storage that survives service or container replacement. The +Python client's disk queue protects against a client-process crash, not a +host crash or power loss. Replay is at-least-once, so the table can receive +rows that it has already stored. For the rules that apply to every client, +including how to avoid duplicates with `DEDUP UPSERT KEYS`, see [Token lifetime](/docs/security/oidc/#token-lifetime). See [Authentication and TLS](/docs/connect/clients/python/#authentication-and-tls) @@ -916,11 +926,12 @@ REVOKE HTTP FROM fxIngest; ``` After the revoke, QuestDB rejects the next batch of the service. In the Python -examples above, `flush(wait=True)` raises `QuestDBServerRejectionError`, and -the examples stop. Without `sf_dir`, the rejected rows are lost. With `sf_dir`, -they stay on disk, and land once you grant the permissions again and the -service connects with the same `sender_id`. In the Java client, the rejection -reaches the error handler of the `QuestDB` handle as a terminal +examples above, `flush(wait=True)` or `wait()` raises +`QuestDBServerRejectionError`, and the examples stop. Without `sf_dir`, the +rejected rows are lost. With `sf_dir`, they stay on disk, and land once you +grant the permissions again and the service connects with the same +`sender_id`. In the Java client, the rejection reaches the error handler of +the `QuestDB` handle as a terminal `SECURITY_ERROR`, the next call on the borrowed sender throws `LineSenderServerException`, and the pool replaces the sender on its next borrow. diff --git a/documentation/security/oidc/index.mdx b/documentation/security/oidc/index.mdx index 515954745c..5e9732605b 100644 --- a/documentation/security/oidc/index.mdx +++ b/documentation/security/oidc/index.mdx @@ -5,6 +5,7 @@ description: "Configure OIDC in QuestDB Enterprise: Web Console SSO, token authe --- import Screenshot from "@theme/Screenshot"; +import Link from "@docusaurus/Link"; import { EnterpriseNote } from "@site/src/components/EnterpriseNote" @@ -1462,11 +1463,17 @@ be reached or answers with an error: both modes once `acl.oidc.cache.ttl` has passed since QuestDB last checked them, because QuestDB checks them with the provider's token endpoint. -In QuestDB Enterprise 4.0.2 and later, a request that carries a token that -QuestDB rejects, or cannot check, fails with `401 Unauthorized`, even with a -valid session cookie, and ends the HTTP session. An outage of the provider can -therefore log users out of the Web Console. Versions before 4.0.2 accept such -a request through its session instead. +For ordinary HTTP requests in QuestDB Enterprise 4.0.2 and later, a token +that QuestDB rejects, or cannot check, causes `401 Unauthorized`, even with a +valid session cookie, and ends that HTTP session. An outage of the provider +can therefore log users out of the Web Console. Versions before 4.0.2 accept +such a request through its session instead. + +A browser QWP WebSocket upgrade that supplies a credential through the +WebSocket credential subprotocol is handled separately: a rejected credential +fails the upgrade without falling back to the session cookie. This path does +not create, rotate, or destroy an HTTP session, so the failed upgrade does not +end an existing cookie session. Some access outlives the token: @@ -1685,14 +1692,15 @@ versions behave, and what to check when you upgrade. provider whose tokens carry no `iss` claim, `acl.oidc.issuer.validation.enabled=false`. See [The expected issuer](#the-expected-issuer). -- **HTTP sessions.** When a request carries a session cookie and a token that - they reject, or cannot check because the provider is unreachable, earlier - versions accept the request through the session, so the Web Console keeps - working with a token that they no longer accept, until its session expires. - QuestDB Enterprise 4.0.2 rejects the request with `401 Unauthorized` and ends - the session, so an outage of the provider can log users out of the Web - Console. See - [When QuestDB stops accepting a token](#when-questdb-stops-accepting-a-token). +- **HTTP sessions.** When an ordinary HTTP request carries a session cookie + and a token that they reject, or cannot check because the provider is + unreachable, earlier versions accept the request through the session, so + the Web Console keeps working with a token that they no longer accept, + until its session expires. QuestDB Enterprise 4.0.2 rejects the request with + `401 Unauthorized` and ends the session, so an outage of the provider can log + users out of the Web Console. A browser QWP upgrade using the credential + subprotocol instead fails without using or destroying the cookie session. + See [When QuestDB stops accepting a token](#when-questdb-stops-accepting-a-token). - **Claim lists.** Earlier versions read the whole value of each setting as a single claim name, so a claim list makes every OIDC login fail. In a replicated cluster, upgrade every node before you change a setting to a @@ -1724,9 +1732,93 @@ The [Microsoft Entra ID guide](/docs/security/oidc/entra-id/) sets up single sign-on for the Web Console, and app-only tokens for Azure services that run as a managed identity or a service principal. +Continue with a setup step: + +- + Set up the client application in Entra ID + +- + QuestDB configuration + +- + Map groups and grant permissions + +- + Confirm group mappings and login + + ### PingFederate The [PingFederate guide](/docs/security/oidc/pingfederate/) sets up single sign-on for the Web Console, with PingFederate authenticating users against an Active Directory domain, such as Microsoft Entra Domain Services, over secure LDAP. + +Continue with a setup step: + +- + Set up PingFederate client + +- + Access Token Manager settings + +- + Authorization Server settings + +- + Set up an Active Directory data source + +- + Set up a Password Credential Validator + +- + Set up an Identity Provider + +- + Add groups to OIDC policy management + +- + Enable Resource Owner Password Credentials (ROPC) flow + +- + Confirm QuestDB mappings and login + From 872f7606ed706691a7707e7f2fc29de50163b8de Mon Sep 17 00:00:00 2001 From: glasstiger Date: Thu, 15 Oct 2026 13:28:21 +0100 Subject: [PATCH 18/29] docs(oidc): fix review findings and Java client version selection --- documentation/changelog.mdx | 1 + documentation/configuration/oidc.md | 2 + .../connect/clients/connect-string.md | 7 +- documentation/connect/clients/java.md | 25 ++- documentation/security/oidc/entra-id.mdx | 5 +- .../fetch-java-client-release/index.test.cjs | 170 ++++++++++++++++++ plugins/fetch-java-client-release/index.ts | 16 +- plugins/raw-markdown/convert-components.js | 4 +- shared/java-client-version.js | 31 ++++ 9 files changed, 241 insertions(+), 20 deletions(-) create mode 100644 plugins/fetch-java-client-release/index.test.cjs create mode 100644 shared/java-client-version.js diff --git a/documentation/changelog.mdx b/documentation/changelog.mdx index 7a8fa8cb6f..c88b2fcddb 100644 --- a/documentation/changelog.mdx +++ b/documentation/changelog.mdx @@ -46,6 +46,7 @@ This page tracks significant updates to the QuestDB documentation. - [Non-interactive clients](/docs/security/oidc/#non-interactive-clients) - Which token each client sends, with client credentials and password grant examples over QWP - [Managing authentication](/docs/security/rbac/#managing-authentication) - HTTP sessions of QuestDB Enterprise 4.0.2 end with the credential that opened them - [Java client](/docs/connect/clients/java/#ingestion-errors) - Corrected the error policies, and that the `QuestDB` handle accepts an error handler +- [Java token providers](/docs/connect/clients/java/#rotating-tokens) - Clarified how token-wait budgets combine and which connection-string settings must be redacted before logging - [Python](/docs/connect/clients/python/#the-pool), [C/C++](/docs/connect/clients/c-and-cpp/#connecting), and [Rust](/docs/connect/clients/rust/#connecting) clients - Corrected that the pool connects when it is created, unless `lazy_connect=on` - [Python client](/docs/connect/clients/python/#flushing-and-acknowledgements) - Corrected that a terminal server rejection raises from `flush(wait=True)` - [Client token rotation](/docs/security/oidc/#token-lifetime) - Corrected how the Python, C/C++, .NET, Go, and Rust clients switch to a new token diff --git a/documentation/configuration/oidc.md b/documentation/configuration/oidc.md index b84004e99d..fb9ea2626a 100644 --- a/documentation/configuration/oidc.md +++ b/documentation/configuration/oidc.md @@ -203,6 +203,8 @@ When enabled, the PGWire endpoint supports OIDC authentication. The OAuth2 token should be sent in the password field, while the username field should contain the string `_sso`, or left empty if that is an option. + + ### acl.oidc.pkce.required - **Default**: `true` diff --git a/documentation/connect/clients/connect-string.md b/documentation/connect/clients/connect-string.md index 5caa89a390..b0f6720c1a 100644 --- a/documentation/connect/clients/connect-string.md +++ b/documentation/connect/clients/connect-string.md @@ -271,9 +271,10 @@ wss::addr=node-a:9000,node-b:9000;token_provider=azure;azure_resource=api:// + Object.hasOwn(mocks, id) ? mocks[id] : localRequire(id), + console: { error() {}, warn() {} }, + ...globals, + }, + { filename }, + ) + return module.exports +} + +test("release tags resolve to stable versions at or above the minimum", () => { + const cases = [ + [undefined, minimum], + [null, minimum], + [42, minimum], + ["", minimum], + ["latest", minimum], + ["invalid", minimum], + ["v1.0.1", minimum], + ["1.3.8", minimum], + ["1.3.9", minimum], + ["1.3.10", minimum], + ["v1.3.10", minimum], + ["v1.3.11", "1.3.11"], + ["1.4.0", "1.4.0"], + ["v1.10.0", "1.10.0"], + ["2.0.0", "2.0.0"], + ["v1.3.10-rc.1", minimum], + ["v2.0.0-SNAPSHOT", minimum], + ] + for (const [tag, expected] of cases) { + assert.equal(resolveJavaClientVersion(tag), expected, `tag: ${tag}`) + } +}) + +const response = (body) => ({ ok: true, json: async () => body }) +const scenarios = [ + ["older release", async () => response({ tag_name: "v1.3.8" }), minimum], + ["minimum release", async () => response({ tag_name: "v1.3.10" }), minimum], + ["newer release", async () => response({ tag_name: "v1.3.11" }), "1.3.11"], + ["missing tag", async () => response({}), minimum], + ["invalid tag", async () => response({ tag_name: 42 }), minimum], + [ + "prerelease tag", + async () => response({ tag_name: "v2.0.0-rc.1" }), + minimum, + ], + ["HTTP failure", async () => ({ ok: false, status: 503 }), minimum], + [ + "network failure", + async () => { + throw new Error("offline") + }, + minimum, + ], + [ + "malformed JSON", + async () => ({ + ok: true, + json: async () => { + throw new SyntaxError("invalid JSON") + }, + }), + minimum, + ], +] + +for (const nativeFetch of [true, false]) { + for (const [label, fetchStub, expected] of scenarios) { + test(`HTML plugin (${nativeFetch ? "native fetch" : "node-fetch"}): ${label}`, async () => { + const { default: plugin } = loadModule( + "plugins/fetch-java-client-release/index.ts", + { "node-fetch": fetchStub }, + { fetch: nativeFetch ? fetchStub : undefined }, + ) + const instance = plugin() + const content = await instance.loadContent() + assert.equal(content.name, expected) + let published + await instance.contentLoaded({ + content, + actions: { + setGlobalData: (data) => { + published = data + }, + }, + }) + assert.equal(published.release.name, expected) + }) + } +} + +for (const [label, tag, expected] of [ + ["older release", "v1.3.8", minimum], + ["minimum release", "v1.3.10", minimum], + ["newer release", "v1.3.11", "1.3.11"], + ["missing tag", undefined, minimum], + ["network failure", null, minimum], +]) { + test(`raw Markdown keeps core and Azure dependencies aligned: ${label}`, async () => { + const https = { + get(_url, _options, callback) { + const request = new EventEmitter() + queueMicrotask(() => { + if (tag === null) { + request.emit("error", new Error("offline")) + } else { + const res = new EventEmitter() + callback(res) + res.emit("data", JSON.stringify({ tag_name: tag })) + res.emit("end") + } + }) + return request + }, + } + const { convertAllComponents } = loadModule( + "plugins/raw-markdown/convert-components.js", + { https }, + ) + const filename = path.join(root, "documentation/connect/clients/java.md") + const converted = await convertAllComponents( + fs.readFileSync(filename, "utf8"), + path.dirname(filename), + path.join(root, "documentation"), + ) + const versions = [ + ...converted.matchAll(/([^<]+)<\/version>/g), + ].map((match) => match[1]) + assert.deepEqual(versions, [expected, expected]) + for (const artifact of ["questdb-client", "questdb-client-azure"]) { + assert.ok(converted.includes(`org.questdb:${artifact}:${expected}`)) + } + assert.ok(!converted.includes("${release.name}")) + }) +} diff --git a/plugins/fetch-java-client-release/index.ts b/plugins/fetch-java-client-release/index.ts index b440c350e6..fdd1cbd40c 100644 --- a/plugins/fetch-java-client-release/index.ts +++ b/plugins/fetch-java-client-release/index.ts @@ -1,12 +1,16 @@ import type { Plugin } from '@docusaurus/types' import nodeFetch from 'node-fetch' +import { + MINIMUM_JAVA_CLIENT_VERSION, + resolveJavaClientVersion, +} from '../../shared/java-client-version' type Release = { name: string } const DEFAULT_RELEASE: Release = { - name: '1.0.1', + name: MINIMUM_JAVA_CLIENT_VERSION, } async function fetchLatestJavaClientRelease(): Promise { @@ -27,8 +31,9 @@ async function fetchLatestJavaClientRelease(): Promise { } const data = await response.json() - const tagName = (data as { tag_name?: string }).tag_name ?? '' - return { name: tagName.replace(/^v/, '') || DEFAULT_RELEASE.name } + return { + name: resolveJavaClientVersion((data as { tag_name?: string }).tag_name), + } } catch (error) { console.error('Failed to fetch latest Java client release:', error) return DEFAULT_RELEASE @@ -47,8 +52,9 @@ async function fetchLatestJavaClientRelease(): Promise { } const data = await response.json() - const tagName = (data as { tag_name?: string }).tag_name ?? '' - return { name: tagName.replace(/^v/, '') || DEFAULT_RELEASE.name } + return { + name: resolveJavaClientVersion((data as { tag_name?: string }).tag_name), + } } catch (error) { console.error('Failed to fetch latest Java client release:', error) return DEFAULT_RELEASE diff --git a/plugins/raw-markdown/convert-components.js b/plugins/raw-markdown/convert-components.js index fc576b441f..95adee0bcc 100644 --- a/plugins/raw-markdown/convert-components.js +++ b/plugins/raw-markdown/convert-components.js @@ -2,6 +2,7 @@ const fs = require("fs") const path = require("path") const clients = require("../../shared/clients.json") const ilpClients = require("../../shared/ilp_clients.json") +const { resolveJavaClientVersion } = require("../../shared/java-client-version") /** * Parses JSX props from a string like: alt="test" height={300} src="..." @@ -321,7 +322,8 @@ async function fetchReleaseVersion() { } async function fetchJavaClientVersion() { - return fetchGitHubRelease("questdb/java-questdb-client", { stripV: true }) + const tag = await fetchGitHubRelease("questdb/java-questdb-client", { stripV: true }) + return resolveJavaClientVersion(tag) } function convertInterpolateReleaseData(content, releaseVersion) { diff --git a/shared/java-client-version.js b/shared/java-client-version.js new file mode 100644 index 0000000000..ca9814af98 --- /dev/null +++ b/shared/java-client-version.js @@ -0,0 +1,31 @@ +// Token-provider examples require this version of both Java client artifacts. +const MINIMUM_JAVA_CLIENT_VERSION = "1.3.10" + +/** + * Resolve a stable release tag without falling below the documented API version. + * @param {unknown} tag + * @returns {string} + */ +function resolveJavaClientVersion(tag) { + const version = typeof tag === "string" ? tag.replace(/^v/, "") : "" + if (!/^\d+\.\d+\.\d+$/.test(version)) { + return MINIMUM_JAVA_CLIENT_VERSION + } + + const parts = version.split(".").map(Number) + const minimum = MINIMUM_JAVA_CLIENT_VERSION.split(".").map(Number) + for (let i = 0; i < minimum.length; i++) { + if (parts[i] > minimum[i]) { + return version + } + if (parts[i] < minimum[i]) { + return MINIMUM_JAVA_CLIENT_VERSION + } + } + return version +} + +module.exports = { + MINIMUM_JAVA_CLIENT_VERSION, + resolveJavaClientVersion, +} From a80642312f3cfb534ef9ca9a65aeb1e708b57fe8 Mon Sep 17 00:00:00 2001 From: glasstiger Date: Thu, 15 Oct 2026 14:05:25 +0100 Subject: [PATCH 19/29] docs(clients): align authentication recovery and startup retries --- documentation/changelog.mdx | 4 +- .../connect/clients/connect-string.md | 72 ++++++++------ documentation/connect/clients/java.md | 13 ++- .../wire-protocols/qwp-client-behavior.md | 95 ++++++++++++++----- .../wire-protocols/qwp-egress-websocket.md | 9 +- .../wire-protocols/qwp-ingress-websocket.md | 22 +++-- .../client-failover/concepts.md | 35 +++++-- .../store-and-forward/operating-and-tuning.md | 8 +- documentation/security/oidc/entra-id.mdx | 15 +-- 9 files changed, 185 insertions(+), 88 deletions(-) diff --git a/documentation/changelog.mdx b/documentation/changelog.mdx index c88b2fcddb..cc0c50a217 100644 --- a/documentation/changelog.mdx +++ b/documentation/changelog.mdx @@ -46,7 +46,9 @@ This page tracks significant updates to the QuestDB documentation. - [Non-interactive clients](/docs/security/oidc/#non-interactive-clients) - Which token each client sends, with client credentials and password grant examples over QWP - [Managing authentication](/docs/security/rbac/#managing-authentication) - HTTP sessions of QuestDB Enterprise 4.0.2 end with the credential that opened them - [Java client](/docs/connect/clients/java/#ingestion-errors) - Corrected the error policies, and that the `QuestDB` handle accepts an error handler -- [Java token providers](/docs/connect/clients/java/#rotating-tokens) - Clarified how token-wait budgets combine and which connection-string settings must be redacted before logging +- [Java token providers](/docs/connect/clients/java/#rotating-tokens) - Clarified token-wait budgets, which `401` responses trigger refresh, and which connection-string settings must be redacted before logging +- [Client failover](/docs/high-availability/client-failover/concepts/#authentication-is-cluster-wide) - Distinguished authentication rejection during startup, established Java ingestion, query reconnects, and orphan recovery +- [Connection strings](/docs/connect/clients/connect-string/#reconnect-keys) - Corrected pooled initial-retry settings and clarified when lazy pools begin connecting in Java, C/C++, Python, and Rust - [Python](/docs/connect/clients/python/#the-pool), [C/C++](/docs/connect/clients/c-and-cpp/#connecting), and [Rust](/docs/connect/clients/rust/#connecting) clients - Corrected that the pool connects when it is created, unless `lazy_connect=on` - [Python client](/docs/connect/clients/python/#flushing-and-acknowledgements) - Corrected that a terminal server rejection raises from `flush(wait=True)` - [Client token rotation](/docs/security/oidc/#token-lifetime) - Corrected how the Python, C/C++, .NET, Go, and Rust clients switch to a new token diff --git a/documentation/connect/clients/connect-string.md b/documentation/connect/clients/connect-string.md index 39fbc06f7b..eb50d39f0d 100644 --- a/documentation/connect/clients/connect-string.md +++ b/documentation/connect/clients/connect-string.md @@ -147,16 +147,18 @@ wss::addr=node-a:443,node-b:443;target=replica;zone=eu-west-1a; ### Tolerate a slow or restarting server at startup ``` -ws::addr=node-a:9000;reconnect_max_duration_millis=120000; +ws::addr=node-a:9000;query_pool_min=0;initial_connect_retry=on;reconnect_max_duration_millis=120000; ``` -The 2-minute reconnect budget covers both the *first* connect and any -subsequent reconnect: setting any explicit `reconnect_*` key implicitly -turns on `initial_connect_retry`. In the C, C++, Python, and Rust clients, -this applies to a standalone sender only: their pooled facade honours only an -explicit `initial_connect_retry`, and tolerates a server that is down at -startup with [`lazy_connect=on`](#pool-keys). See -[Ingress reconnect](#reconnect-keys). +The sender retries its first connection synchronously for up to 2 minutes. +The explicit `initial_connect_retry=on` also enables this in the pooled +facade of the C, C++, Python, and Rust clients, where `reconnect_*` keys +alone do not enable initial retries. `query_pool_min=0` prevents the query +pool's separate, fail-fast startup connection, so this example configures +ingest startup only. Once connected, the sender retries transport outages +indefinitely; the 2-minute budget does not limit subsequent reconnects. +For non-blocking pool startup, use [`lazy_connect=on`](#pool-keys) instead. +See [Ingress reconnect](#reconnect-keys). ## Recipes {#recipes} @@ -182,8 +184,9 @@ caveats), follow the section links from the [Key index](#key-index). | Run multiple senders sharing one `sf_dir` | ingress | `sf_dir`, `sender_id` | unique `sender_id` per sender | | Orphan recovery for crashed senders | ingress | `drain_orphans=on` | `max_background_drainers` | | End-to-end durable acknowledgement | ingress | `request_durable_ack=on` | `durable_ack_keepalive_interval_millis` | -| Tune ingress reconnect backoff | ingress | — | `reconnect_initial_backoff_millis`, `reconnect_max_backoff_millis`, `reconnect_max_duration_millis` (any of these also implies `initial_connect_retry=on`) | -| Force fail-fast on initial connect | ingress | `initial_connect_retry=off` | overrides the implicit promotion from any explicit `reconnect_*` key | +| Tune ingress reconnect backoff | ingress | — | `reconnect_initial_backoff_millis`, `reconnect_max_backoff_millis`, `reconnect_max_duration_millis`; pooled C/C++/Python/Rust clients need an explicit initial retry mode | +| Force fail-fast on initial connect | ingress | `initial_connect_retry=off` | overrides implicit promotion where supported; see [Ingress reconnect](#reconnect-keys) | +| Retry initial ingest connection synchronously | ingress | `initial_connect_retry=on` | `reconnect_max_duration_millis`; use `query_pool_min=0` to skip separate query startup | | Retry initial connect in background | ingress | `initial_connect_retry=async` | `reconnect_*` | | Fast `close()` without drain | ingress | `close_flush_timeout_millis=0` | — | | Disable per-query egress failover | egress | `failover=off` | — | @@ -652,16 +655,15 @@ architecture is that a producer survives an arbitrarily long outage. - `reconnect_initial_backoff_millis` — initial wait between reconnect attempts. Backoff grows exponentially up to `reconnect_max_backoff_millis`. - Default: `100`. Setting this enables `initial_connect_retry=on` implicitly; - see below. + Default: `100`. See the implicit-promotion rules below for its effect on + initial connect. - `reconnect_max_backoff_millis` — cap on per-attempt backoff. - Default: `5000` (5 s). Setting this enables - `initial_connect_retry=on` implicitly; see below. + Default: `5000` (5 s). See the implicit-promotion rules below. - `reconnect_max_duration_millis` — time budget for the **blocking sync initial connect** (`initial_connect_retry=on`): once exceeded, the constructor gives up and returns the error. The running loop and the `async` initial connect never consult it. Default: `300000` (5 min). - Setting this enables `initial_connect_retry=on` implicitly; see below. + See the implicit-promotion rules below. - `initial_connect_retry` — whether the client retries the initial connect attempt on failure. - `off` (default, alias `false`) — fail fast on initial connect failure. @@ -671,13 +673,16 @@ architecture is that a producer survives an arbitrarily long outage. the background indefinitely, surfacing only genuine terminal failures (auth reject, durable-ack mismatch) via the error inbox. - **Implicit promotion.** Setting any explicit `reconnect_*` key without - also choosing an `initial_connect_retry` mode promotes - `initial_connect_retry` to `on` automatically, so the budget also covers - the *first* connect attempt — without the promotion, - `reconnect_max_duration_millis` would be inert. To keep fail-fast - behaviour on the first connect while still tuning the backoff, set - `initial_connect_retry=off` explicitly; the explicit setting wins. + **Implicit promotion.** In Java, .NET, and Go, setting any explicit + `reconnect_*` key without also choosing an `initial_connect_retry` mode + promotes the initial mode to `on`. In C, C++, Python, and Rust, this + promotion applies only to a standalone sender, not to the pooled facade. + Set `initial_connect_retry=on` explicitly for blocking initial retries + in those pools, or use `lazy_connect=on` for non-blocking startup. Query + connections have separate fail-fast startup behavior; set + `query_pool_min=0` to skip their prewarm. An explicit + `initial_connect_retry=off` always wins over implicit promotion, letting + you tune reconnect backoff while retaining fail-fast sender startup. - `close_flush_timeout_millis` — `close()` blocks up to this many milliseconds waiting for buffered frames to drain. Set to `0` or `-1` for fast close (skip the drain). **The default differs by client**: `60000` @@ -835,15 +840,20 @@ explicit setter always wins over the string. - `housekeeper_interval_ms` — how often the housekeeper checks for idle and over-age connections. Default: `5000`. - `lazy_connect` — when `on`, construction does not wait for a connection, - so the application can start while the server is down. In the Java, C, - C++, Python, and Rust clients, senders connect asynchronously in the - background, while query connections open on first use. Networking can - therefore begin before the first borrow. In Java, this also lets the - application start while the identity provider is unreachable when a token - provider requests its first token. In these clients, `lazy_connect` cannot - be combined with `query_pool_min` above `0`, or with an - `initial_connect_retry` other than `async`: the client rejects the - configuration. Default: `off`. + so the application can start while the server is down. + - Java can prewarm its configured minimum senders asynchronously before + the first borrow. This also lets the application start while the + identity provider is unreachable when a token provider requests its + first token. + - C, C++, Python, and Rust skip sender prewarming: a clean pool creates + senders on first borrow, then connects them asynchronously. Recovery + of existing disk-backed slots can start before the first borrow. + - In these clients, query connections open on first use, and + `lazy_connect` cannot be combined with `query_pool_min` above `0`, or + with an `initial_connect_retry` other than `async`: the client rejects + the configuration. + + Default: `off`. ## Error handling {#error-handling} diff --git a/documentation/connect/clients/java.md b/documentation/connect/clients/java.md index 1b9f4c1817..b792f30647 100644 --- a/documentation/connect/clients/java.md +++ b/documentation/connect/clients/java.md @@ -586,7 +586,7 @@ are left. A token that arrives with less counts as a failed request, whose error names `handout_floor`. If your identity provider issues tokens that short, lower the floor with `handoutFloorMillis(...)` on the builder. A connection that needs a token waits up to 30 seconds when the provider holds no -usable token, set with `coldWaitMillis(...)`. After a `401`, +usable token, set with `coldWaitMillis(...)`. After a refreshable `401`, `onTokenRejected()` waits up to 5 seconds for a refresh, set with `forcedWaitMillis(...)`. The client then calls `getToken()` again. If the cached token became unusable during the forced-refresh wait, this call can add the @@ -595,12 +595,15 @@ excluding network time. #### How the client handles token failures -- When QuestDB rejects a token with `401`, the client asks the provider for a - new token, and when it gets a different one, retries the same server once, +- When QuestDB rejects a token with `401` and the response has no Bearer + `error` or has `error="invalid_token"`, the client asks the provider for a + new token. If it gets a different one, it retries the same server once, immediately. The provider requests a new token for a rejected one at most once every 30 seconds. With `token_provider=azure`, Azure Identity can - return the token that it has cached, and then the client does not retry. A - `403` does not trigger a retry. + return its cached token; an unchanged token does not earn this immediate + retry. A `403` or any other Bearer error, such as `insufficient_scope`, + does not trigger this refresh/retry. These rules govern the one-time + token refresh, not the established sender's reconnect loop described below. - At startup, a sender with `initial_connect_retry=off`, the default, fails when it cannot get a token within 30 seconds. With `on`, it retries a retryable failure of the provider within `reconnect_max_duration_millis`, diff --git a/documentation/connect/wire-protocols/qwp-client-behavior.md b/documentation/connect/wire-protocols/qwp-client-behavior.md index c4e73a3d5e..9b52d94ccb 100644 --- a/documentation/connect/wire-protocols/qwp-client-behavior.md +++ b/documentation/connect/wire-protocols/qwp-client-behavior.md @@ -137,11 +137,13 @@ query execution after a connection exists. This naming trips people up | `on` / `true` / `sync` | `SYNC` | retry loop on caller thread, bounded by `reconnect_max_duration_millis` (blocks) | | `async` | `ASYNC` | returns immediately; I/O thread retries in background | -**Default resolution ⚠:** if you don't set `initial_connect_retry` explicitly but -you *do* set any `reconnect_*` knob, the mode becomes `SYNC` — so a "resilience" -knob silently turns startup into a multi-minute **blocking** retry. If no -`reconnect_*` knob is set either, the mode is `OFF`. Always set -`initial_connect_retry` explicitly to avoid this ([sharp edge #1](#known-sharp-edges)). +**Default resolution ⚠:** in Java, setting any `reconnect_*` knob without an +explicit `initial_connect_retry` promotes the mode to `SYNC`, so tuning +reconnects can turn startup into a multi-minute **blocking** retry. In C, C++, +Python, and Rust, this promotion applies to standalone senders only; their +pooled facades require an explicit initial mode. With no promotion or explicit +mode, the default is `OFF`. Set `initial_connect_retry` explicitly to avoid +ambiguity ([sharp edge #1](#known-sharp-edges)). ### Facade prewarm @@ -279,11 +281,11 @@ here. | # | Sharp edge | Status | | --- | --- | --- | -| 1 | `initial_connect_retry` is implicitly promoted to `SYNC` when any `reconnect_*` knob is set — a resilience knob silently makes startup block. | Candidate | +| 1 | Java and standalone C/C++/Python/Rust senders promote `initial_connect_retry` to `SYNC` when any `reconnect_*` knob is set; pooled C/C++/Python/Rust clients do not. Set the initial mode explicitly. | Candidate | | 2 | `reconnect_max_duration_millis` is named as if it governs reconnection, but a running sender never consults it — it bounds only the blocking initial connect. | Candidate (naming) | | 3 | `failover` sounds like it covers startup but only affects post-connect query `execute()`. Queries have no async/lazy initial connect at all. | Candidate | | 4 | No first-class write-only facade: a write-only user must still supply a query config and remember `query_pool_min=0`, or use `lazy_connect=true`. | Candidate | -| 5 | A single endpoint returning `401`/`403` is treated as cluster-wide terminal and aborts the whole endpoint walk, even at startup, even if other endpoints would accept the credentials. | Intended (documented), revisit | +| 5 | Authentication rejection ends the current endpoint walk. Java startup, established senders, queries, and orphan drainers have different [recovery policies](#authentication-rejection-and-credential-recovery). | Intended; client- and lifecycle-specific | | 6 | Query `serverInfoTimeoutMs` has no config key, so a facade query client cannot tune it. | Candidate | | 7 | The simplest API (`fromConfig` + async) has the worst error visibility — terminal async failures surface only on later producer calls or at `close()`. | Candidate | | 8 | `SenderProgressHandler` has no builder setter on either surface; it must be installed post-construction via `QwpWebSocketSender.setProgressHandler`. | Candidate | @@ -344,10 +346,14 @@ With `initial_connect_retry=async`: - Terminal errors go to a configured `SenderErrorHandler`; without one they surface on later producer calls or at close-time. -A sender in async mode does not give up because time passed. What ends it is a -**terminal** condition — authentication rejection, a durable-ack capability -mismatch, or the poison-frame detector — or the producer hitting -`sf_max_total_bytes` and exhausting `sf_append_deadline_millis` on `append()`. +A sender in async mode does not give up because time passed. Before its first +successful connection, authentication rejection (after any token-provider retry) +or a durable-ack capability mismatch is terminal. Established Java senders +instead retry these rejections; see +[Authentication rejection and credential recovery](#authentication-rejection-and-credential-recovery). +The poison-frame detector can still terminate a sender. If the buffer reaches +`sf_max_total_bytes`, `append()` blocks and then throws after +`sf_append_deadline_millis`; the sender itself keeps retrying. ### Reconnect and outage handling @@ -356,13 +362,17 @@ wall-clock give-up and no budget-exhaustion event. Backoff grows from `reconnect_initial_backoff_millis` to `reconnect_max_backoff_millis` and stays there; the loop rotates through the endpoints in `addr` as it goes. -`reconnect_max_duration_millis` has exactly two consumers, neither of which is -the steady-state loop: +`reconnect_max_duration_millis` bounds startup and specific Java orphan-recovery +budgets, not the steady-state foreground loop: -1. The **blocking sync initial connect** (`initial_connect_retry=on` / `sync`), - which gives up and throws when the budget expires. -2. The background drainer's **durable-ack capability-gap budget**, used when an - orphan slot repeatedly lands on a node that cannot serve durable acks. +1. The **blocking sync initial connect** (`initial_connect_retry=on` / `sync`) + gives up and throws when the budget expires. +2. The Java background drainer uses it as a **durable-ack capability-gap budget** + when an orphan repeatedly lands on a node that cannot serve durable acks. +3. With a token provider, the Java drainer also derives a bounded authentication + rejection window from it. See + [How the client handles token failures](/docs/connect/clients/java/#how-the-client-handles-token-failures) + for the time and attempt thresholds. The practical consequence: what bounds your tolerance of a long outage is **buffer capacity, not time**. In SF mode that is `sf_max_total_bytes` against @@ -380,6 +390,33 @@ and apply back-pressure to the producer rather than dropping data. ::: +### Authentication rejection and credential recovery + +The Java reference client separates a failed endpoint walk from the lifetime of +the sender or query. A `401`/`403` stops the current walk rather than continuing +to other hosts with the rejected credential. + +With a token provider, a `401` with no Bearer `error` or with +`error="invalid_token"` can first receive one same-endpoint retry, but only if +the provider supplies a different token. A `403` or another Bearer error, such +as `insufficient_scope`, does not trigger this refresh/retry. Any remaining +rejection follows these Java lifecycle rules: + +| Lifecycle | Authentication rejection behavior | +| --- | --- | +| Sender before its first successful connection | Terminal, including with `initial_connect_retry=async`. | +| Established foreground sender | Retry with backoff indefinitely while retaining buffered rows, even with fixed credentials. Report the outage through connection health and the error handler. | +| Query initial connect or execution-time reconnect | Terminal for that connection attempt or query; no continuation to later endpoints. | +| Orphan drainer with fixed credentials | Quarantine the slot immediately behind a `.failed` file. | +| Orphan drainer with a token provider | Retry under a bounded time/attempt policy, then quarantine if rejections persist. | + +Other language clients still treat authentication rejection as terminal even for +established senders. See [Ingress reconnect](/docs/connect/clients/connect-string/#reconnect-keys) +for client differences, and +[How the client handles token failures](/docs/connect/clients/java/#how-the-client-handles-token-failures) +for Java refresh limits, orphan thresholds, and reporting. The established +sender's reconnect loop is independent of the one-time token refresh. + ### Ingest endpoint walk (`addr=a:9000,b:9000,...`) | Per-endpoint result | Sender behavior | @@ -389,11 +426,11 @@ and apply back-pressure to the producer rather than dropping data. | TLS session/certificate failure | transport error; try next endpoint | | HTTP upgrade timeout / non-auth transport error | try next endpoint | | `421` with `X-QuestDB-Role: REPLICA` | role reject; try next endpoint | -| `401` / `403` auth failure | **terminal**; do not try later endpoints ⚠ | -| durable-ack requested but unsupported | terminal mismatch | +| `401` / `403` auth failure | End the current walk after any token-provider retry; Java recovery depends on [lifecycle](#authentication-rejection-and-credential-recovery). | +| durable-ack requested but unsupported | Terminal at startup; established Java foreground senders retry, while orphan drainers use a bounded capability-gap budget. | | successful write upgrade | bind this endpoint | | all endpoints fail transport | throw / retry per initial/reconnect mode | -| all endpoints role-reject as replicas | `QwpRoleMismatchException` | +| all endpoints role-reject as replicas | Initial connect throws `QwpRoleMismatchException`; the running sender retries. | ### Query client initial connect @@ -417,7 +454,7 @@ Query initial-connect endpoint matrix: | --- | --- | | DNS / TCP / TLS failure | record transport error; try next endpoint | | HTTP upgrade timeout | transport error; try next endpoint | -| HTTP `401` / `403` | **terminal** `QwpAuthFailedException`; do not try later ⚠ | +| HTTP `401` / `403` | Java may first retry the same endpoint with a refreshed token; otherwise terminal `QwpAuthFailedException`, with no later endpoints tried. | | HTTP `421` + role header | role reject; try next endpoint | | upgrade ok but no `SERVER_INFO` before timeout | transport error; try next | | `SERVER_INFO` role ≠ `target` | role reject; try next endpoint | @@ -439,7 +476,9 @@ With `failover=on`: - Bounded by `failover_max_attempts` (default `8`, incl. original) **and** `failover_max_duration_ms` (default `30000`; `0` disables the duration cap). - Backoff: `failover_backoff_initial_ms=50`, `failover_backoff_max_ms=1000`. -- Auth failure during failover reconnect is terminal and reported to the handler. +- Auth rejection during failover reconnect is terminal and reported to the + handler after any eligible same-endpoint token-provider retry. See + [Authentication rejection and credential recovery](#authentication-rejection-and-credential-recovery). With `failover=off`, a transport failure is reported to the handler with no reconnect/replay. @@ -464,7 +503,7 @@ reconnect/replay. | server down, default mode | no `reconnect_*`, no async | one attempt; build throws | | server down, reconnect duration set, no mode | `reconnect_max_duration_millis=...` | **synchronous** retry; build blocks ⚠ | | server down, async | `initial_connect_retry=async` | build returns; I/O thread retries | -| server returns `401`/`403` | any mode | terminal auth failure; no endpoint continuation | +| server returns `401`/`403` | any mode | terminal at startup after any eligible same-endpoint token-provider retry; no continuation to later endpoints | | server appears later, any delay | `initial_connect_retry=async` | buffered frames sent and ACKed — the retry loop has no deadline | | server never appears, buffer fills | async + `sf_max_total_bytes` reached | `append()` blocks, then throws after `sf_append_deadline_millis`; the sender itself stays alive and keeps retrying | @@ -479,7 +518,7 @@ reconnect/replay. | upgrades but no `SERVER_INFO` | Yes | after `serverInfoTimeoutMs` (builder only) | | primary/standalone while `target=replica` | Yes | role mismatch | | `421` role reject | Yes | try next | -| `401`/`403` | **No** | auth treated as cluster-wide terminal ⚠ | +| `401`/`403` | **No** | Java may retry the same endpoint after eligible token refresh; a remaining rejection is terminal. | | broken shared TLS/trust store | No | every endpoint fails | | all endpoints down | No | `all QWP endpoints unreachable` | | reachable but none match `target` | No | `QwpRoleMismatchException` | @@ -555,5 +594,9 @@ source states the contract directly: > converts it into the durable-ack capability-gap budget. Neither bounds this > loop's steady-state reconnect. -`QwpAuthFailedException` and `WebSocketUpgradeException` raised inside the loop -are terminal across all endpoints. Everything else is retried. +Before the first successful connection, `QwpAuthFailedException` (after any +eligible token-provider retry) and non-role `WebSocketUpgradeException` failures +are terminal. An established foreground sender instead retries these failures +with backoff while retaining buffered data. Orphan drainers return control to +their owner, which applies its separate quarantine policy. See +[Authentication rejection and credential recovery](#authentication-rejection-and-credential-recovery). diff --git a/documentation/connect/wire-protocols/qwp-egress-websocket.md b/documentation/connect/wire-protocols/qwp-egress-websocket.md index 53d66107b9..7f3d078a52 100644 --- a/documentation/connect/wire-protocols/qwp-egress-websocket.md +++ b/documentation/connect/wire-protocols/qwp-egress-websocket.md @@ -796,8 +796,13 @@ section of the connect string reference. The shared failover primitives Key behaviors: -- Authentication errors are terminal at any host; the reconnect loop does not - continue past them. +- Authentication rejections stop the endpoint walk rather than trying other + hosts. With a token provider, Java can first retry the same host once if a + refreshable `401` produces a different token. A `403` or a Bearer error + other than `invalid_token` does not trigger that refresh/retry. If the + retry is unavailable or fails, the authentication rejection is terminal + for the query. See + [How the client handles token failures](/docs/connect/clients/java/#how-the-client-handles-token-failures). - A `CANCEL` acknowledged with `QUERY_ERROR(CANCELLED)` routes through the normal error path, not the transport-error path, so it never triggers failover. diff --git a/documentation/connect/wire-protocols/qwp-ingress-websocket.md b/documentation/connect/wire-protocols/qwp-ingress-websocket.md index 0e4d9417fd..57db1ec84d 100644 --- a/documentation/connect/wire-protocols/qwp-ingress-websocket.md +++ b/documentation/connect/wire-protocols/qwp-ingress-websocket.md @@ -1159,15 +1159,23 @@ Key behaviors: every host's zone tier is equivalent and selection is based on health state only. The `zone=` connect-string key is accepted but silently ignored, so a connect string shared with egress clients works unchanged on ingress. -- **Authentication errors are terminal** at any host (`401`/`403`). The - reconnect loop does not continue past them. +- **Authentication rejections depend on the client and lifecycle.** In Java, + a token provider can retry a refreshable `401` once on the same host with a + different token. If that retry is unavailable or fails, authentication + rejection is terminal before the sender's first successful connection. + Once a foreground Java sender has connected, it instead keeps buffering + and retrying rejected credentials with backoff, even with a fixed + credential. Orphan drainers have a separate bounded policy. See + [How the client handles token failures](/docs/connect/clients/java/#how-the-client-handles-token-failures). + Other clients treat `401`/`403` rejections as terminal, including during + reconnect. - **`421 + X-QuestDB-Role`** is a role reject: transient if the role is `PRIMARY_CATCHUP`, topology-level otherwise. -- **All other upgrade errors are transient** and feed into the reconnect loop, - including `404`, `426`, `503`, generic 4xx/5xx, TCP/TLS failures, - mid-stream send/recv errors, and an upgrade response that advertises a QWP - version outside the client's supported range (per-endpoint, so a host on a - rolling upgrade does not lock the client out of compatible peers). +- **Transport errors are transient**, including TCP/TLS failures and + mid-stream send/recv errors. For other HTTP upgrade rejections, version + mismatches, and durable-ACK capability rejections, the client and lifecycle + determine whether a failed endpoint round is terminal or retried. See + [Ingress reconnect](/docs/connect/clients/connect-string/#reconnect-keys). :::note Enterprise diff --git a/documentation/high-availability/client-failover/concepts.md b/documentation/high-availability/client-failover/concepts.md index be95b64a9b..666e709825 100644 --- a/documentation/high-availability/client-failover/concepts.md +++ b/documentation/high-availability/client-failover/concepts.md @@ -187,7 +187,7 @@ will not help. | Condition | Why terminal | |---|---| -| HTTP `401` / `403` on upgrade | Credentials are cluster-wide; retrying floods server logs without recovery. | +| HTTP `401` / `403` during initial connect or query reconnect, after any token-provider retry | The rejected credential stops the endpoint walk. Established Java senders and orphan drainers have [different recovery policies](#authentication-is-cluster-wide). | | Server-status reject (SF) | Application-layer reject; replay reproduces the same response. | ### Topology — handled inside the round @@ -232,14 +232,31 @@ when at least one peer is healthy." ## Authentication is cluster-wide -A `401` or `403` on the HTTP upgrade is terminal — the client does not retry -other hosts. The assumption is that auth credentials are configured -identically across the cluster, so a credential failure against one node is -a credential failure against all of them. Retrying would spam every peer's -audit log without recovering. - -If your deployment has per-host credentials, that is unsupported and outside -the failover model — split the workload into one connect string per credential. +Credentials are assumed to be configured identically across the cluster. +A `401` or `403` therefore ends the current endpoint walk rather than +continuing to other hosts with the rejected credential. Whether the client +retries later depends on the client and lifecycle: + +- With a token provider, Java can first retry the same host once when a + refreshable `401` produces a different token. A `403` or a Bearer error + other than `invalid_token` does not trigger that refresh/retry. +- Before a Java sender's first successful connection, or when a query client + connects or reconnects, any remaining authentication rejection is terminal. +- An established foreground Java sender instead keeps buffering rows and + retrying with backoff, even with a fixed credential. It reports the outage + through connection health and the error handler; applications that need a + hard failure must monitor these and close the sender themselves. +- A Java orphan drainer fails immediately on a rejected fixed credential. + With a token provider, it retries under a bounded policy before + quarantining the slot behind a `.failed` file; the rows stay on disk. +- Other clients treat authentication rejection as terminal even after the + first connection. + +See [How the client handles token failures](/docs/connect/clients/java/#how-the-client-handles-token-failures) +for the refresh rules, drainer thresholds, and reporting. + +Per-host credentials are outside the failover model. Split such workloads +into one connect string per credential. ## Next steps diff --git a/documentation/high-availability/store-and-forward/operating-and-tuning.md b/documentation/high-availability/store-and-forward/operating-and-tuning.md index 9b69e434ca..fcc8bce59c 100644 --- a/documentation/high-availability/store-and-forward/operating-and-tuning.md +++ b/documentation/high-availability/store-and-forward/operating-and-tuning.md @@ -225,7 +225,13 @@ and exits when the orphan's `ackedFsn ≥ publishedFsn`. - **Reconnect budget exhausted.** Drainer writes `.failed` with reason, releases the lock, exits. -- **Auth-terminal upgrade error.** Same. +- **Authentication rejection.** In Java, a fixed credential that is rejected + fails the drainer immediately. With a token provider, `401`/`403` + rejections receive bounded retries before the drainer writes `.failed` + and exits. This differs from an established foreground Java sender, + which keeps retrying. See + [How the client handles token failures](/docs/connect/clients/java/#how-the-client-handles-token-failures) + for the retry thresholds and reporting. - **Irrecoverable corruption.** Same. `.failed` slots are excluded from auto-drain on subsequent scans — diff --git a/documentation/security/oidc/entra-id.mdx b/documentation/security/oidc/entra-id.mdx index f773ae8303..70dcc33283 100644 --- a/documentation/security/oidc/entra-id.mdx +++ b/documentation/security/oidc/entra-id.mdx @@ -638,12 +638,15 @@ public class FxIngest { - The Azure provider keys contain no secret. You can keep the connect string in the `QDB_CLIENT_CONF` environment variable. Redact any other sensitive settings, such as `tls_roots_password`, before logging it. -- When QuestDB rejects a token with `401`, the client asks Azure Identity for a - new token, at most once every 30 seconds, and when it gets a different one, - retries the same server once. Azure Identity can return the token that it - has cached, and then the client does not retry. A sender that has connected - keeps buffering rows while it cannot get a token or QuestDB rejects it, and - reports each failure to the error handler and in its +- When QuestDB rejects a token with `401` and the response has no Bearer + `error` or has `error="invalid_token"`, the client asks Azure Identity for a + new token, at most once every 30 seconds. If it gets a different one, it + retries the same server once. Azure Identity can return its cached token; + an unchanged token does not earn this immediate retry. A `403` or any + other Bearer error, such as `insufficient_scope`, does not trigger this + refresh/retry. Independently of this one-time retry, a sender that has + connected keeps buffering rows while it cannot get a token or QuestDB + rejects it, and reports each failure to the error handler and in its [connection health](/docs/connect/clients/java/#connection-health). - When the handle starts, it waits up to 30 seconds for the first token, and fails when Entra ID cannot issue one. To start while Entra ID or QuestDB is From bc6737e920c635102f88cdd9faffba945bc0777f Mon Sep 17 00:00:00 2001 From: glasstiger Date: Thu, 15 Oct 2026 14:54:57 +0100 Subject: [PATCH 20/29] docs: clarify QWP acknowledgements and authentication retries --- .../connect/clients/connect-string.md | 17 +++-- documentation/connect/clients/java.md | 76 ++++++++++++++----- documentation/security/oidc/entra-id.mdx | 8 +- 3 files changed, 73 insertions(+), 28 deletions(-) diff --git a/documentation/connect/clients/connect-string.md b/documentation/connect/clients/connect-string.md index eb50d39f0d..5e2d93172b 100644 --- a/documentation/connect/clients/connect-string.md +++ b/documentation/connect/clients/connect-string.md @@ -692,11 +692,18 @@ architecture is that a producer survives an arbitrarily long outage. This is the shutdown data-loss window. Setting it to `0` skips the drain entirely and drops un-ACKed batches on every clean shutdown. -Auth failures during reconnect (authentication rejected, version mismatch, -durable-ack mismatch, non-101 upgrade without a role hint) are immediately -terminal in clients other than Java: the loop does not retry them. In Java, -a sender that has connected keeps retrying authentication, upgrade, and -durable-ACK rejections. On the first connect, those rejections are terminal. +Authentication rejection, durable-ACK mismatch, and a non-101 upgrade +response without a role hint are immediately terminal during reconnect in +clients other than Java: the loop does not retry them. In Java, a sender +that has connected keeps retrying these rejections. On the first connect, +these rejections are terminal, after any eligible +[token-provider refresh and retry](/docs/connect/clients/java/#how-the-client-handles-token-failures). + +In Java, a protocol-version mismatch is transient, including at startup. +With `initial_connect_retry=on`, the client retries it within +`reconnect_max_duration_millis`; with `async`, it retries in the background +indefinitely. A sender that has already connected also retries version +mismatches indefinitely, allowing a rolling upgrade to settle. Java orphan drainers have a separate policy: diff --git a/documentation/connect/clients/java.md b/documentation/connect/clients/java.md index b792f30647..5031973c43 100644 --- a/documentation/connect/clients/java.md +++ b/documentation/connect/clients/java.md @@ -629,10 +629,17 @@ excluding network time. acknowledged. - An orphan drainer, enabled with `drain_orphans=on`, gives up instead. When QuestDB keeps rejecting its credential, it sets the slot aside behind a - `.failed` file and reports `DATA_LOSS`: at once with a fixed credential, and - with a token provider, once the rejections have lasted for - `reconnect_max_duration_millis`, at most 5 minutes, and at least six - attempts. The rows stay on disk. + `.failed` file and reports `DATA_LOSS`. With a fixed credential, this happens + at once. With a token provider, it happens when either condition is met: + - At least six authentication rejections have occurred, and the rejections + have persisted for at least the smaller of + `reconnect_max_duration_millis` and 5 minutes. + - There have been 256 authentication rejections in one episode without ACK + progress, regardless of the persistence interval. + + ACK progress resets the episode, including the rejection count and + persistence clock. Intervening failures of another kind reset only the + persistence clock, not the count. The rows stay on disk. - A query whose client has to connect, and cannot get a token, fails. The next query tries again. @@ -1095,27 +1102,51 @@ ws::addr=localhost:9000;sf_dir=/var/lib/questdb/sf;request_durable_ack=on; `flush()` returns once the batch is handed to the send engine, not once the server has acknowledged it. ACKs arrive asynchronously on the I/O loop. To -bridge between publish and acknowledgement, every published frame is assigned a -frame sequence number (FSN). `flushAndGetSequence()` returns -the highest FSN the call published, and `awaitAckedFsn(targetFsn, -timeoutMillis)` blocks until the server has acknowledged up to that FSN: +wait for all rows published by a QWP sender, call `drain(timeoutMillis)` while +that sender is borrowed. It flushes any buffered rows, then waits for the +server to acknowledge all published frames, including frames published by an +earlier automatic flush: + +```java +try (Sender sender = db.borrowSender()) { + sender.table("trades").doubleColumn("price", 42.0).atNow(); + if (!sender.drain(10_000)) { + throw new IllegalStateException("Timed out waiting for QuestDB acknowledgements"); + } +} +``` + +A `false` result means the acknowledgement wait timed out, not that delivery +succeeded. It does not cancel the published frames: the sender may still +deliver them later. + +For per-call correlation, every published frame has a frame sequence number +(FSN). `flushAndGetSequence()` returns the highest FSN published by that call, +and `awaitAckedFsn(targetFsn, timeoutMillis)` waits for acknowledgement up to +that FSN: ```java try (Sender sender = db.borrowSender()) { sender.table("trades").doubleColumn("price", 42.0).atNow(); long fsn = sender.flushAndGetSequence(); if (fsn >= 0 && !sender.awaitAckedFsn(fsn, 10_000)) { - // timed out waiting for the server ACK + throw new IllegalStateException("Timed out waiting for frame " + fsn); } } ``` +This FSN example waits only for frames published by its explicit flush. A +return value of `-1` means that call published nothing; it does **not** mean +previously auto-flushed rows were acknowledged. Use `drain()` for the +all-published-rows barrier. + Related accessors: | Method | Returns | |--------|---------| -| `flushAndGetSequence()` | Highest FSN published by this call. `-1` if nothing was published. | -| `getAckedFsn()` | Highest FSN the server has acknowledged. `-1` if no batch has been published yet. | +| `drain(timeoutMillis)` | Flushes buffered rows and waits for all published frames. `true` if acknowledged, `false` on timeout. | +| `flushAndGetSequence()` | Highest FSN published by this call. `-1` if nothing was published by this call. | +| `getAckedFsn()` | Highest FSN the server has acknowledged. `-1` if no frame has been acknowledged yet. | | `awaitAckedFsn(fsn, timeoutMillis)` | Block until `getAckedFsn()` reaches `fsn`, or the timeout elapses. | When `request_durable_ack=on` is set, `getAckedFsn()` advances after the @@ -1124,9 +1155,10 @@ FSN span is reported on `SenderError.getFromFsn()` / `getToFsn()` for rejected batches, so the value returned by `flushAndGetSequence()` is also the correlation key for async error reports. -These methods are no-ops on transports that do not track frame sequence -numbers (HTTP, TCP, UDP): `flushAndGetSequence()` and `getAckedFsn()` -return `-1`, and `awaitAckedFsn()` returns immediately. +On transports that do not track frame sequence numbers (HTTP, TCP, UDP), +`flushAndGetSequence()` flushes and returns `-1`, `getAckedFsn()` returns `-1`, +and `awaitAckedFsn()` returns immediately. On those transports, `drain()` +flushes and returns `true` without a frame-acknowledgement wait. ## Querying with `Query` and `Completion` @@ -1571,8 +1603,10 @@ What is and isn't carried on `onError`: rethrows `LineSenderException`. A sender that has connected before keeps retrying instead, as described in [How the client handles token failures](#how-the-client-handles-token-failures). - With a [token provider](#rotating-tokens), a `401` first makes the client - request a new token and retry the same endpoint once. + With a [token provider](#how-the-client-handles-token-failures), a `401` can + trigger a refresh only when the response has no Bearer `error` or has + `error="invalid_token"`. The client retries the same endpoint once only if + the provider returns a different token. - **Credential unavailable**: the [token provider](#rotating-tokens) cannot supply a token, for example because the identity provider is unreachable. The client contacts no endpoint, and handles the failure as described in @@ -1680,10 +1714,12 @@ client and gives you a fresh one on your next query. - **Authentication failures** (a bad token or wrong credentials) stop immediately on every host for a query and for the first connect of a sender, - because retrying cannot help. With a [token provider](#rotating-tokens), a - `401` first makes the client request a new token and retry the same host - once. A sender that has already connected keeps retrying with any - credential, as described in + because retrying cannot help. With a + [token provider](#how-the-client-handles-token-failures), a `401` can trigger + a refresh only when the response has no Bearer `error` or has + `error="invalid_token"`. The client retries the same host once only if the + provider returns a different token. A sender that has already connected + keeps retrying with any credential, as described in [How the client handles token failures](#how-the-client-handles-token-failures). - **Network and availability failures** (connection refused, TLS errors, a `5xx` from the server, a mid-query drop) are treated as temporary and fed into diff --git a/documentation/security/oidc/entra-id.mdx b/documentation/security/oidc/entra-id.mdx index 70dcc33283..1c09ccc189 100644 --- a/documentation/security/oidc/entra-id.mdx +++ b/documentation/security/oidc/entra-id.mdx @@ -660,9 +660,11 @@ connect string, such as `sf_dir=/var/lib/fx-ingest/sf;`, as described in [Store-and-forward](/docs/connect/clients/java/#store-and-forward). Closing a borrowed sender flushes its rows and returns it to the pool, and closing the handle waits up to `close_flush_timeout_millis`, 60 seconds by default, for -QuestDB to acknowledge them. To confirm that a batch has reached QuestDB, -wait for its acknowledgement, as described in -[Awaiting acknowledgements](/docs/connect/clients/java/#awaiting-acknowledgements). +QuestDB to acknowledge them. To confirm that all rows published by a sender +have reached QuestDB, call `sender.drain(timeoutMillis)` while the sender is +borrowed, and treat a `false` result as a timeout, not a delivery confirmation. +This also waits for rows published by an earlier automatic flush, as described +in [Awaiting acknowledgements](/docs/connect/clients/java/#awaiting-acknowledgements). #### With the Python client From c60a6e62f5f7d0b7d966256a3fa207ccb3faa59f Mon Sep 17 00:00:00 2001 From: glasstiger Date: Thu, 15 Oct 2026 15:18:26 +0100 Subject: [PATCH 21/29] docs: address OIDC and Java client review findings --- documentation/changelog.mdx | 2 +- documentation/connect/clients/java.md | 22 +++-- documentation/security/oidc/entra-id.mdx | 10 ++ documentation/security/oidc/index.mdx | 81 ++++++++-------- .../raw-markdown/convert-components.test.cjs | 95 +++++++++++++++++++ 5 files changed, 160 insertions(+), 50 deletions(-) create mode 100644 plugins/raw-markdown/convert-components.test.cjs diff --git a/documentation/changelog.mdx b/documentation/changelog.mdx index cc0c50a217..6d912ea58a 100644 --- a/documentation/changelog.mdx +++ b/documentation/changelog.mdx @@ -46,7 +46,7 @@ This page tracks significant updates to the QuestDB documentation. - [Non-interactive clients](/docs/security/oidc/#non-interactive-clients) - Which token each client sends, with client credentials and password grant examples over QWP - [Managing authentication](/docs/security/rbac/#managing-authentication) - HTTP sessions of QuestDB Enterprise 4.0.2 end with the credential that opened them - [Java client](/docs/connect/clients/java/#ingestion-errors) - Corrected the error policies, and that the `QuestDB` handle accepts an error handler -- [Java token providers](/docs/connect/clients/java/#rotating-tokens) - Clarified token-wait budgets, which `401` responses trigger refresh, and which connection-string settings must be redacted before logging +- [Java token providers](/docs/connect/clients/java/#rotating-tokens) - Clarified certificate credential configuration, query authentication exceptions, token-wait budgets, which `401` responses trigger refresh, and which connection-string settings must be redacted before logging - [Client failover](/docs/high-availability/client-failover/concepts/#authentication-is-cluster-wide) - Distinguished authentication rejection during startup, established Java ingestion, query reconnects, and orphan recovery - [Connection strings](/docs/connect/clients/connect-string/#reconnect-keys) - Corrected pooled initial-retry settings and clarified when lazy pools begin connecting in Java, C/C++, Python, and Rust - [Python](/docs/connect/clients/python/#the-pool), [C/C++](/docs/connect/clients/c-and-cpp/#connecting), and [Rust](/docs/connect/clients/rust/#connecting) clients - Corrected that the pool connects when it is created, unless `lazy_connect=on` diff --git a/documentation/connect/clients/java.md b/documentation/connect/clients/java.md index 5031973c43..aa3ac9a23f 100644 --- a/documentation/connect/clients/java.md +++ b/documentation/connect/clients/java.md @@ -491,9 +491,10 @@ connect string reference. For the Entra ID setup and the QuestDB configuration, see [Managed identities and service principals](/docs/security/oidc/entra-id/#managed-identities-and-service-principals). -To use an Azure credential that `azure_credential` cannot select, such as a -certificate credential, wrap it in an `AzureTokenSource`, and pass it to a -`RefreshingTokenProvider`, as described in +Certificates configured through Azure's environment variables can use +`azure_credential=environment`. To construct or customize an Azure credential +in code instead, such as the certificate credential below, wrap it in an +`AzureTokenSource`, and pass it to a `RefreshingTokenProvider`, as described in [Other identity providers](#other-identity-providers): ```java @@ -1599,9 +1600,12 @@ What is and isn't carried on `onError`: - **Authentication failure**: `401`/`403` HTTP response before the WebSocket upgrade completes. Terminal across all endpoints for the first connect of a - sender and for a query client: the borrow that triggered the connect - rethrows `LineSenderException`. A sender that has connected before keeps - retrying instead, as described in + sender and for a query client. A synchronous sender connect reports + `LineSenderException`. If `db.borrowQuery()` creates a query client and its + connection fails, it throws `QueryException` with the connection failure as + its cause. A failed reconnect during query execution is reported to the + handler's `onError(...)`, and `Completion.await()` throws `QueryException`. + A sender that has connected before keeps retrying instead, as described in [How the client handles token failures](#how-the-client-handles-token-failures). With a [token provider](#how-the-client-handles-token-failures), a `401` can trigger a refresh only when the response has no Bearer `error` or has @@ -1771,8 +1775,10 @@ sections of the connect string reference. A sender rides out an outage, such as an unreachable cluster or a token that QuestDB rejects, and keeps accepting rows while it retries. To find out that it has not reached the server, check its connection health, which requires Java -client 1.3.10 or later. Reading it never blocks, and never contains a -credential, so it can back a health endpoint: +client 1.3.10 or later. Health reads do not perform network I/O or wait for a +connection attempt. The `db.health()` aggregate acquires pool locks to collect +each client's snapshot; it is not lock-free. The snapshots contain no +credential, so they can back a health endpoint: ```java import io.questdb.client.ConnectionHealth; diff --git a/documentation/security/oidc/entra-id.mdx b/documentation/security/oidc/entra-id.mdx index 1c09ccc189..c2f07acbb5 100644 --- a/documentation/security/oidc/entra-id.mdx +++ b/documentation/security/oidc/entra-id.mdx @@ -589,6 +589,16 @@ the same version as `questdb-client`, 1.3.10 or later: +`token_provider=azure` requires a TLS-secured `wss::` endpoint. Before running +this example, configure +[HTTP server TLS](/docs/security/tls/#enabling-tls-for-http-server-rest-api) +or a TLS-terminating proxy that supports WebSocket upgrades. Replace +`questdb.example.com:9000` with the secure host and port reachable from your +service. For a private CA, configure the +[Java trust store](/docs/connect/clients/java/#tls-with-custom-trust-store). +If enabling TLS changes the Web Console URL, update its registered redirect +URI in Entra ID and `acl.oidc.redirect.uri` to match. + ```java title="Ingest into fx_trades with an app-only token" import io.questdb.client.QuestDB; import io.questdb.client.Sender; diff --git a/documentation/security/oidc/index.mdx b/documentation/security/oidc/index.mdx index 5e9732605b..c5616ae1c8 100644 --- a/documentation/security/oidc/index.mdx +++ b/documentation/security/oidc/index.mdx @@ -5,7 +5,6 @@ description: "Configure OIDC in QuestDB Enterprise: Web Console SSO, token authe --- import Screenshot from "@theme/Screenshot"; -import Link from "@docusaurus/Link"; import { EnterpriseNote } from "@site/src/components/EnterpriseNote" @@ -887,7 +886,7 @@ example while QuestDB is unreachable, retry it instead of falling back to the old client, and request a current token for each attempt, because an outage can outlast the token. [Request a token in the service](/docs/security/oidc/entra-id/#request-a-token-in-the-service) -shows both orders in Python. +shows disk-backed token rotation in Python. Replay is at-least-once: after a reconnect or a switch to a new client, QuestDB can receive rows that it has already stored. To avoid duplicates, @@ -1734,30 +1733,30 @@ a managed identity or a service principal. Continue with a setup step: -- Set up the client application in Entra ID - -- +- QuestDB configuration - -- +- Map groups and grant permissions - -- +- Confirm group mappings and login - + ### PingFederate @@ -1768,57 +1767,57 @@ LDAP. Continue with a setup step: -- Set up PingFederate client - -- +- Access Token Manager settings - -- +- Authorization Server settings - -- +- Set up an Active Directory data source - -- +- Set up a Password Credential Validator - -- +- Set up an Identity Provider - -- +- Add groups to OIDC policy management - -- +- Enable Resource Owner Password Credentials (ROPC) flow - -- +- Confirm QuestDB mappings and login - + diff --git a/plugins/raw-markdown/convert-components.test.cjs b/plugins/raw-markdown/convert-components.test.cjs new file mode 100644 index 0000000000..9ed9491b27 --- /dev/null +++ b/plugins/raw-markdown/convert-components.test.cjs @@ -0,0 +1,95 @@ +const assert = require("node:assert/strict") +const { EventEmitter } = require("node:events") +const fs = require("node:fs") +const { createRequire } = require("node:module") +const path = require("node:path") +const test = require("node:test") +const vm = require("node:vm") + +// Exercise the real converter without fetching release data over the network. +function loadConverter() { + const filename = path.join(__dirname, "convert-components.js") + const localRequire = createRequire(filename) + const https = { + get(_url, _options, callback) { + const request = new EventEmitter() + queueMicrotask(() => { + const response = new EventEmitter() + callback(response) + response.emit("data", JSON.stringify({ tag_name: "v1.3.10" })) + response.emit("end") + }) + return request + }, + } + const module = { exports: {} } + vm.runInNewContext( + fs.readFileSync(filename, "utf8"), + { + module, + exports: module.exports, + require: (id) => (id === "https" ? https : localRequire(id)), + console, + }, + { filename }, + ) + return module.exports +} + +test("OIDC raw Markdown preserves usable legacy provider-step links", async () => { + const { convertAllComponents, removeImports, normalizeNewLines } = + loadConverter() + const docsRoot = path.resolve(__dirname, "../../documentation") + const filename = path.join(docsRoot, "security/oidc/index.mdx") + const converted = normalizeNewLines( + removeImports( + await convertAllComponents( + fs.readFileSync(filename, "utf8"), + path.dirname(filename), + docsRoot, + ), + ), + ) + const anchors = new Map() + for (const [, attributes, label] of converted.matchAll( + /]*)>([\s\S]*?)<\/a>/g, + )) { + const id = attributes.match(/\bid="([^"]+)"/)?.[1] + const href = attributes.match(/\bhref="([^"]+)"/)?.[1] + if (id) { + assert.ok(label.trim(), `link label for ${id}`) + assert.ok(!anchors.has(id), `unique legacy anchor ${id}`) + anchors.set(id, href) + } + } + + const providerSteps = { + "entra-id": [ + "set-up-the-client-application-in-entra-id", + "questdb-configuration", + "map-groups-and-grant-permissions", + "confirm-group-mappings-and-login", + ], + pingfederate: [ + "set-up-pingfederate-client", + "access-token-manager-settings", + "authorization-server-settings", + "set-up-a-microsoft-entra-id-data-source", + "set-up-a-password-credential-validator", + "set-up-an-identity-provider", + "add-groups-to-oidc-policy-management", + "enable-resource-owner-password-credentials-ropc-flow", + "confirm-questdb-mappings-and-login", + ], + } + for (const [provider, steps] of Object.entries(providerSteps)) { + for (const id of steps) { + assert.equal( + anchors.get(id), + `/docs/security/oidc/${provider}/#${id}`, + `legacy anchor ${id} links to the provider setup step`, + ) + } + } + assert.doesNotMatch(converted, /<\/?Link\b/) +}) From 76062c0d56b6fdf587e425bd4985820a8d6e7350 Mon Sep 17 00:00:00 2001 From: glasstiger Date: Thu, 15 Oct 2026 15:47:45 +0100 Subject: [PATCH 22/29] docs: clarify OIDC cache checks and make PingFederate ROPC opt-in --- documentation/changelog.mdx | 4 ++-- documentation/configuration/oidc.md | 10 +++++++--- documentation/security/oidc/pingfederate.mdx | 4 ++-- 3 files changed, 11 insertions(+), 7 deletions(-) diff --git a/documentation/changelog.mdx b/documentation/changelog.mdx index 6d912ea58a..259a8bd02d 100644 --- a/documentation/changelog.mdx +++ b/documentation/changelog.mdx @@ -30,14 +30,14 @@ This page tracks significant updates to the QuestDB documentation. - [`acl.oidc.issuer`](/docs/configuration/oidc/#acloidcissuer) and [`acl.oidc.issuer.validation.enabled`](/docs/configuration/oidc/#acloidcissuervalidationenabled) - New settings for the expected issuer of a token - [`acl.oidc.pkce.required`](/docs/configuration/oidc/#acloidcpkcerequired) - Corrected the name of the property, previously documented as `acl.oidc.pkce.enabled` - [`acl.oidc.state.required`](/docs/configuration/oidc/#acloidcstaterequired) and [`acl.oidc.public.keys.expiry`](/docs/configuration/oidc/#acloidcpublickeysexpiry) - Documented both properties -- [`acl.oidc.cache.ttl`](/docs/configuration/oidc/#acloidccachettl) - Documented what QuestDB caches, and that it also bounds how long an HTTP session reuses a password check +- [`acl.oidc.cache.ttl`](/docs/configuration/oidc/#acloidccachettl) - Documented what QuestDB caches, distinguished provider checks from JWT revalidation in token mode, and explained how long an HTTP session reuses a password check - [`http.session.timeout`](/docs/configuration/http-server/#httpsessiontimeout) - Documented how long an idle HTTP session stays valid - [Token providers](/docs/connect/clients/connect-string/#token-providers) - The `token_provider` and `azure_*` keys of the Java client ### Guides - [Microsoft Entra ID](/docs/security/oidc/entra-id/) - Moved to its own page, and added app-only tokens for Azure managed identities and service principals -- [PingFederate](/docs/security/oidc/pingfederate/) - Moved to its own page, added the QuestDB configuration, and corrected the directory that PingFederate reads users from +- [PingFederate](/docs/security/oidc/pingfederate/) - Moved to its own page, added the QuestDB configuration with ROPC disabled unless explicitly enabled, and corrected the directory that PingFederate reads users from ### Updated diff --git a/documentation/configuration/oidc.md b/documentation/configuration/oidc.md index fb9ea2626a..fe6c79c522 100644 --- a/documentation/configuration/oidc.md +++ b/documentation/configuration/oidc.md @@ -370,9 +370,13 @@ the password ends the session at once, as described in [Managing authentication](/docs/security/rbac/#managing-authentication). The session does not reuse the check of any other credential. QuestDB checks a REST token on every request that carries it, and an OIDC token or an ROPC login -through the cache described above, which asks the provider again once this -time has passed since the provider last checked the credential. See -[When QuestDB stops accepting a token](/docs/security/oidc/#when-questdb-stops-accepting-a-token). +through the cache described above. Once a cache entry expires, User Info mode +calls the provider's User Info endpoint again, and ROPC requests a new token +from the provider in either mode. For bearer tokens in token mode, QuestDB +instead revalidates the JWT against signing keys and rereads its existing +claims. It does not ask the provider about the user's current status or group +memberships; updated group claims require the client to present a new token. +See [When QuestDB stops accepting a token](/docs/security/oidc/#when-questdb-stops-accepting-a-token). ### acl.oidc.groups.claim diff --git a/documentation/security/oidc/pingfederate.mdx b/documentation/security/oidc/pingfederate.mdx index 27dc77c4db..0c65812e2b 100644 --- a/documentation/security/oidc/pingfederate.mdx +++ b/documentation/security/oidc/pingfederate.mdx @@ -373,8 +373,8 @@ acl.oidc.client.id=questdb # the claim set up in Add groups to OIDC policy management acl.oidc.groups.claim=groups -# optional, required only if ROPC is enabled in PingFederate -acl.oidc.ropc.flow.enabled=true +# optional: uncomment only if you enabled ROPC in PingFederate +# acl.oidc.ropc.flow.enabled=true ``` QuestDB reads the groups from the User Info response, so keep the default, From 9a71cf2cc0af111d0f49e1109ae09465fdb1b91a Mon Sep 17 00:00:00 2001 From: glasstiger Date: Thu, 15 Oct 2026 16:14:44 +0100 Subject: [PATCH 23/29] docs: clarify OIDC refresh requirements and verified PGWire TLS --- documentation/changelog.mdx | 3 ++- documentation/security/oidc/index.mdx | 17 ++++++++++++++--- 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/documentation/changelog.mdx b/documentation/changelog.mdx index 259a8bd02d..e7478c812c 100644 --- a/documentation/changelog.mdx +++ b/documentation/changelog.mdx @@ -14,7 +14,7 @@ This page tracks significant updates to the QuestDB documentation. - [User and group claims](/docs/security/oidc/#user-and-group-claims) - Fallback claim lists of QuestDB Enterprise 4.0.2, how QuestDB picks a claim, and the startup checks - [Token validation](/docs/security/oidc/#token-validation) - What QuestDB checks in a token, the expected issuer, and when QuestDB stops accepting an expired token -- [Web Console token renewal](/docs/security/oidc/#web-console-token-renewal) - When the Web Console renews its tokens, and when it logs the user out +- [Web Console token renewal](/docs/security/oidc/#web-console-token-renewal) - When the Web Console renews its tokens, the ID-token requirement in token mode, and when it logs the user out - [Troubleshooting OIDC logins](/docs/security/oidc/#troubleshooting-oidc-logins) - The log lines of rejected logins and what they mean - [Upgrade to 4.0.2](/docs/security/oidc/#upgrade-to-402) - How earlier versions differ, and what to check when you upgrade - [Where QuestDB reads the user information](/docs/security/oidc/#where-questdb-reads-the-user-information) - The User Info endpoint compared with reading the claims from the token @@ -44,6 +44,7 @@ This page tracks significant updates to the QuestDB documentation. - [OpenID Connect (OIDC)](/docs/security/oidc/) - Split into an overview and provider guides, with a minimal configuration and an upgrade warning for QuestDB Enterprise 4.0.2 - [Mapping user permissions](/docs/security/oidc/#mapping-user-permissions) - Corrected that a login without groups is rejected, and added the endpoint permission of each interface - [Non-interactive clients](/docs/security/oidc/#non-interactive-clients) - Which token each client sends, with client credentials and password grant examples over QWP +- [OIDC for PGWire](/docs/security/oidc/#oidc-for-the-pgwire-endpoint) - Require certificate and hostname verification in the bearer-token example, with a trusted CA certificate bundle - [Managing authentication](/docs/security/rbac/#managing-authentication) - HTTP sessions of QuestDB Enterprise 4.0.2 end with the credential that opened them - [Java client](/docs/connect/clients/java/#ingestion-errors) - Corrected the error policies, and that the `QuestDB` handle accepts an error handler - [Java token providers](/docs/connect/clients/java/#rotating-tokens) - Clarified certificate credential configuration, query authentication exceptions, token-wait budgets, which `401` responses trigger refresh, and which connection-string settings must be redacted before logging diff --git a/documentation/security/oidc/index.mdx b/documentation/security/oidc/index.mdx index c5616ae1c8..5d15e2eb0e 100644 --- a/documentation/security/oidc/index.mdx +++ b/documentation/security/oidc/index.mdx @@ -925,8 +925,12 @@ it, not from the username field. Send the token that an HTTP client would send: See [Non-interactive clients](#non-interactive-clients). Like a password, the token crosses the network in the clear unless the connection uses -[TLS](/docs/security/tls/#enabling-tls-for-pgwire), so require TLS on the -client: +[TLS](/docs/security/tls/#enabling-tls-for-pgwire), so require TLS and verify +the server's identity on the client. Use `sslmode=verify-full` to check both +the certificate chain and that the certificate matches the `host` name. +`sslmode=require` encrypts the connection but does not require hostname +verification. Set `sslrootcert` to a trusted CA certificate bundle on the +client, including your private CA if the server uses one: ```python import psycopg as pg @@ -936,7 +940,8 @@ token = "token_requested_from_the_oauth2_provider" conn_str = ( f"user=_sso password={token} host=questdb.example.com port=8812 " - "dbname=qdb sslmode=require" + "dbname=qdb sslmode=verify-full " + "sslrootcert=/path/to/trusted-ca.pem" ) with pg.connect(conn_str, autocommit=True) as connection: with connection.cursor() as cur: @@ -1513,6 +1518,12 @@ renew a token whose expiry it does not know. When the clock of the browser is behind, QuestDB can reject the token as expired first: the Web Console then renews its tokens and sends the request again. +In token mode, every refresh response must include an `id_token`, even if it +contains a new access token. If the ID token is missing, the Web Console logs +the user out with **Missing ID token** instead of keeping the previous ID +token. Configure the provider to issue ID tokens for the refresh-token grant, +and ensure `acl.oidc.scope` includes `openid`. + - When the provider refuses to issue new tokens, for example because the user is disabled or the refresh token has expired, the Web Console logs the user out, and the login page shows the reason that the provider gives. From 77c98d6b185005c5085a817abe0198310ecd707c Mon Sep 17 00:00:00 2001 From: glasstiger Date: Thu, 15 Oct 2026 16:46:52 +0100 Subject: [PATCH 24/29] docs: clarify OIDC setup and JWKS refresh behavior --- documentation/changelog.mdx | 2 +- documentation/configuration/oidc.md | 16 ++++++++++++---- documentation/security/oidc/index.mdx | 10 +++++++--- 3 files changed, 20 insertions(+), 8 deletions(-) diff --git a/documentation/changelog.mdx b/documentation/changelog.mdx index e7478c812c..3379baa09f 100644 --- a/documentation/changelog.mdx +++ b/documentation/changelog.mdx @@ -29,7 +29,7 @@ This page tracks significant updates to the QuestDB documentation. - [`acl.oidc.sub.claim`](/docs/configuration/oidc/#acloidcsubclaim) and [`acl.oidc.groups.claim`](/docs/configuration/oidc/#acloidcgroupsclaim) - Documented fallback claim lists and their startup validation - [`acl.oidc.issuer`](/docs/configuration/oidc/#acloidcissuer) and [`acl.oidc.issuer.validation.enabled`](/docs/configuration/oidc/#acloidcissuervalidationenabled) - New settings for the expected issuer of a token - [`acl.oidc.pkce.required`](/docs/configuration/oidc/#acloidcpkcerequired) - Corrected the name of the property, previously documented as `acl.oidc.pkce.enabled` -- [`acl.oidc.state.required`](/docs/configuration/oidc/#acloidcstaterequired) and [`acl.oidc.public.keys.expiry`](/docs/configuration/oidc/#acloidcpublickeysexpiry) - Documented both properties +- [`acl.oidc.state.required`](/docs/configuration/oidc/#acloidcstaterequired) and [`acl.oidc.public.keys.expiry`](/docs/configuration/oidc/#acloidcpublickeysexpiry) - Documented both properties, including JWKS refresh timing, outage delays, and retained signing keys - [`acl.oidc.cache.ttl`](/docs/configuration/oidc/#acloidccachettl) - Documented what QuestDB caches, distinguished provider checks from JWT revalidation in token mode, and explained how long an HTTP session reuses a password check - [`http.session.timeout`](/docs/configuration/http-server/#httpsessiontimeout) - Documented how long an idle HTTP session stays valid - [Token providers](/docs/connect/clients/connect-string/#token-providers) - The `token_provider` and `azure_*` keys of the Java client diff --git a/documentation/configuration/oidc.md b/documentation/configuration/oidc.md index fe6c79c522..8911287b69 100644 --- a/documentation/configuration/oidc.md +++ b/documentation/configuration/oidc.md @@ -268,10 +268,18 @@ work for the Ping Identity Platform. - **Default**: `120000` - **Reloadable**: no -How long QuestDB uses the public keys that it downloaded from the JWKS -endpoint, in milliseconds, before it downloads them again. QuestDB also -downloads the keys again when a token names a key ID that it does not know, -such as a key that the OIDC Provider has just started to use. See +The interval, in milliseconds, before QuestDB refreshes the signing keys +that it downloaded from the JWKS endpoint. After startup, QuestDB refreshes +the keys during authentication, not on a background timer. It also refreshes +them when a token names a key ID that it does not know, such as a key that +the OIDC Provider has just started to use. + +This is a refresh interval, not a hard lifetime for cached keys. If a refresh +fails, QuestDB retains its previously downloaded keys and can still validate +tokens signed with them. The authentication request that performs the +refresh may wait for a slow or unreachable provider until the HTTP request +timeout, [`acl.oidc.http.timeout`](#acloidchttptimeout), expires, even when +its signing key is already cached. The timeout is 30 seconds by default. See [Troubleshooting OIDC logins](/docs/security/oidc/#troubleshooting-oidc-logins). ### acl.oidc.token.endpoint diff --git a/documentation/security/oidc/index.mdx b/documentation/security/oidc/index.mdx index 5d15e2eb0e..8de6375cd1 100644 --- a/documentation/security/oidc/index.mdx +++ b/documentation/security/oidc/index.mdx @@ -21,8 +21,10 @@ the Identity Provider map to. Start with the [minimal configuration](#configuration-options), then choose [where QuestDB reads the user information](#where-questdb-reads-the-user-information): -the provider's User Info endpoint, or the token that the client sends. To -accept user and service tokens with one configuration, see +the provider's User Info endpoint, or the token that the client sends. Then +[map the provider's groups to QuestDB groups](#mapping-user-permissions) and +grant endpoint and table permissions before testing access. To accept user +and service tokens with one configuration, see [Fallback claim lists](#fallback-claim-lists). For rejected logins, see [Troubleshooting OIDC logins](#troubleshooting-oidc-logins). For step-by-step setup, see the guides for [Microsoft Entra ID](/docs/security/oidc/entra-id/), @@ -1462,7 +1464,9 @@ be reached or answers with an error: asked the provider about the token. - In token mode, only a token signed with a key that QuestDB has not downloaded yet: QuestDB validates tokens with the signing keys that it has - already downloaded. + already downloaded. These keys are retained if a refresh fails, but + refreshing can delay authentication; see + [JWKS refresh behavior](/docs/configuration/oidc/#acloidcpublickeysexpiry). - Username and password logins through the [ROPC flow](#enable-ropc) fail in both modes once `acl.oidc.cache.ttl` has passed since QuestDB last checked them, because QuestDB checks them with the provider's token endpoint. From 853b8c97c893ac14debba7b074eea484accb5c36 Mon Sep 17 00:00:00 2001 From: glasstiger Date: Thu, 15 Oct 2026 18:11:39 +0100 Subject: [PATCH 25/29] docs(qwp): clarify all-replica startup retry behavior --- documentation/connect/wire-protocols/qwp-client-behavior.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/documentation/connect/wire-protocols/qwp-client-behavior.md b/documentation/connect/wire-protocols/qwp-client-behavior.md index 9b52d94ccb..f4d27e3bba 100644 --- a/documentation/connect/wire-protocols/qwp-client-behavior.md +++ b/documentation/connect/wire-protocols/qwp-client-behavior.md @@ -430,7 +430,7 @@ sender's reconnect loop is independent of the one-time token refresh. | durable-ack requested but unsupported | Terminal at startup; established Java foreground senders retry, while orphan drainers use a bounded capability-gap budget. | | successful write upgrade | bind this endpoint | | all endpoints fail transport | throw / retry per initial/reconnect mode | -| all endpoints role-reject as replicas | Initial connect throws `QwpRoleMismatchException`; the running sender retries. | +| all endpoints role-reject as replicas | OFF startup throws `QwpRoleMismatchException`; SYNC startup retries within its budget; ASYNC startup and running senders retry in the background. | ### Query client initial connect From d960544e4359b8ed3ce20c86d448150706836b5c Mon Sep 17 00:00:00 2001 From: glasstiger Date: Thu, 15 Oct 2026 19:57:53 +0100 Subject: [PATCH 26/29] fix: correct OIDC renewal guidance and raw Markdown tab hierarchy --- documentation/changelog.mdx | 2 +- documentation/security/oidc/index.mdx | 14 ++-- plugins/raw-markdown/convert-components.js | 8 +- .../raw-markdown/convert-components.test.cjs | 74 +++++++++++++++++++ 4 files changed, 88 insertions(+), 10 deletions(-) diff --git a/documentation/changelog.mdx b/documentation/changelog.mdx index 8ab97a0001..4dfeaf0377 100644 --- a/documentation/changelog.mdx +++ b/documentation/changelog.mdx @@ -14,7 +14,7 @@ This page tracks significant updates to the QuestDB documentation. - [User and group claims](/docs/security/oidc/#user-and-group-claims) - Fallback claim lists of QuestDB Enterprise 4.0.2, how QuestDB picks a claim, and the startup checks - [Token validation](/docs/security/oidc/#token-validation) - What QuestDB checks in a token, the expected issuer, and when QuestDB stops accepting an expired token -- [Web Console token renewal](/docs/security/oidc/#web-console-token-renewal) - When the Web Console renews its tokens, the ID-token requirement in token mode, and when it logs the user out +- [Web Console token renewal](/docs/security/oidc/#web-console-token-renewal) - When the Web Console renews its tokens, the ID-token requirement in token mode, how transient renewal failures pause queries without logging users out, and which errors end the session - [Troubleshooting OIDC logins](/docs/security/oidc/#troubleshooting-oidc-logins) - The log lines of rejected logins and what they mean - [Upgrade to 4.0.2](/docs/security/oidc/#upgrade-to-402) - How earlier versions differ, and what to check when you upgrade - [Where QuestDB reads the user information](/docs/security/oidc/#where-questdb-reads-the-user-information) - The User Info endpoint compared with reading the claims from the token diff --git a/documentation/security/oidc/index.mdx b/documentation/security/oidc/index.mdx index 8de6375cd1..dab7ce52c4 100644 --- a/documentation/security/oidc/index.mdx +++ b/documentation/security/oidc/index.mdx @@ -1531,11 +1531,15 @@ and ensure `acl.oidc.scope` includes `openid`. - When the provider refuses to issue new tokens, for example because the user is disabled or the refresh token has expired, the Web Console logs the user out, and the login page shows the reason that the provider gives. -- When the Web Console cannot reach the provider, or the provider answers with - `429` or a `5xx` status, it keeps sending the current token while the token - is valid, warns the user, and tries again with a growing delay of up to 30 - seconds. Once the token has expired, the Web Console stops sending queries - with it, and after three more failed attempts, it logs the user out. +- When token renewal times out, the Web Console cannot reach the provider, + or the provider answers with `429` or a `5xx` status, the Web Console keeps + the login and warns the user. While the current token is valid, queries + continue to use it. Subsequent query attempts retry renewal with a growing + delay of up to 30 seconds between attempts. If the token expires before + renewal succeeds, the Web Console pauses queries instead of sending the + expired token. A query blocked by renewal is reported as not sent and has + not executed on the database. Transient failures do not log the user out, + however many attempts fail. Once renewal succeeds, queries can run again. Without a refresh token, the Web Console keeps sending the token of the login after it has expired. QuestDB Enterprise 4.0.2 and later then reject the token, diff --git a/plugins/raw-markdown/convert-components.js b/plugins/raw-markdown/convert-components.js index 95adee0bcc..87279f6493 100644 --- a/plugins/raw-markdown/convert-components.js +++ b/plugins/raw-markdown/convert-components.js @@ -161,7 +161,7 @@ function convertDocButton(content) { } /** - * Converts and to markdown table + * Converts and to labeled Markdown blocks */ function convertTabs(content) { return replaceComponent(content, 'Tabs', (match) => { @@ -195,8 +195,8 @@ function convertTabs(content) { tabItems.push({ value, content }) } - // Convert to sections with headings instead of table - // This handles complex content better (nested tables, code blocks, etc.) + // Labels must not change the surrounding heading hierarchy: tabs can + // appear at any depth, with prose and sibling sections following them. let sections = "\n\n" for (const valueObj of valuesArray) { @@ -204,7 +204,7 @@ function convertTabs(content) { const tabItem = tabItems.find(item => item.value === valueObj.value) const content = tabItem ? tabItem.content.trim() : "" - sections += `### ${label}\n\n${content}\n\n` + sections += `**${label}**\n\n${content}\n\n` } return sections diff --git a/plugins/raw-markdown/convert-components.test.cjs b/plugins/raw-markdown/convert-components.test.cjs index 9ed9491b27..f687f69edc 100644 --- a/plugins/raw-markdown/convert-components.test.cjs +++ b/plugins/raw-markdown/convert-components.test.cjs @@ -93,3 +93,77 @@ test("OIDC raw Markdown preserves usable legacy provider-step links", async () = } assert.doesNotMatch(converted, /<\/?Link\b/) }) + +test("nested dependency tabs preserve the surrounding heading hierarchy", async () => { + const { convertAllComponents, normalizeNewLines } = loadConverter() + const source = `### Request a token + +#### With the Java client + + + + +\`\`\`xml +questdb-client-azure +\`\`\` + + + + +\`\`\`text +implementation 'org.questdb:questdb-client-azure:1.3.10' +\`\`\` + + + + +Java ingestion instructions. + +#### With the Python client + +Python ingestion instructions. +` + const docsRoot = path.resolve(__dirname, "../../documentation") + const converted = normalizeNewLines( + await convertAllComponents(source, docsRoot, docsRoot), + ) + assert.deepEqual(converted.match(/^#{1,6} .+$/gm), [ + "### Request a token", + "#### With the Java client", + "#### With the Python client", + ]) + assert.match( + converted, + /\*\*Maven\*\*\s+```xml\nquestdb-client-azure<\/dependency>\n```/, + ) + assert.match( + converted, + /\*\*Gradle\*\*\s+```text\nimplementation 'org.questdb:questdb-client-azure:1\.3\.10'\n```/, + ) + assert.match(converted, /Java ingestion instructions\./) + assert.match(converted, /Python ingestion instructions\./) + assert.doesNotMatch(converted, /<\/?(?:Tabs|TabItem)\b/) +}) + +for (const page of ["security/oidc/entra-id.mdx", "connect/clients/java.md"]) { + test(`${page} keeps dependency labels out of its Markdown outline`, async () => { + const { convertAllComponents } = loadConverter() + const docsRoot = path.resolve(__dirname, "../../documentation") + const filename = path.join(docsRoot, page) + const source = fs.readFileSync(filename, "utf8") + const converted = await convertAllComponents( + source, + path.dirname(filename), + docsRoot, + ) + assert.deepEqual( + converted.match(/^#{1,6} .+$/gm), + source.match(/^#{1,6} .+$/gm), + ) + assert.match(converted, /\*\*Maven\*\*/) + assert.match(converted, /\*\*Gradle\*\*/) + assert.match(converted, /questdb-client-azure<\/artifactId>/) + assert.match(converted, /1\.3\.10<\/version>/) + assert.doesNotMatch(converted, /\$\{release\.name\}|<\/?(?:Tabs|TabItem)\b/) + }) +} From 615a84f2554d6e71f1eeb866a85bdedfa35109f7 Mon Sep 17 00:00:00 2001 From: glasstiger Date: Fri, 16 Oct 2026 12:12:08 +0100 Subject: [PATCH 27/29] docs(oidc): document device flow for client applications --- documentation/changelog.mdx | 3 + documentation/configuration/oidc.md | 22 + documentation/connect/clients/c-and-cpp.md | 45 +- .../connect/clients/connect-string.md | 10 +- documentation/connect/clients/java.md | 35 ++ documentation/connect/clients/python.md | 44 +- documentation/connect/clients/rust.md | 48 ++ documentation/security/oidc/device-flow.mdx | 522 ++++++++++++++++++ documentation/security/oidc/entra-id.mdx | 27 +- documentation/security/oidc/index.mdx | 62 ++- documentation/sidebars.js | 5 + 11 files changed, 794 insertions(+), 29 deletions(-) create mode 100644 documentation/security/oidc/device-flow.mdx diff --git a/documentation/changelog.mdx b/documentation/changelog.mdx index 4dfeaf0377..3facae9c5c 100644 --- a/documentation/changelog.mdx +++ b/documentation/changelog.mdx @@ -12,6 +12,7 @@ This page tracks significant updates to the QuestDB documentation. ### New +- [OIDC device flow](/docs/security/oidc/device-flow/) - Human sign-in from Java, Python, Rust, C, and C++, with provider discovery, checked ingestion examples, silent refresh, reauthentication, and optional token persistence - [User and group claims](/docs/security/oidc/#user-and-group-claims) - Fallback claim lists of QuestDB Enterprise 4.0.2, how QuestDB picks a claim, and the startup checks - [Token validation](/docs/security/oidc/#token-validation) - What QuestDB checks in a token, the expected issuer, and when QuestDB stops accepting an expired token - [Web Console token renewal](/docs/security/oidc/#web-console-token-renewal) - When the Web Console renews its tokens, the ID-token requirement in token mode, how transient renewal failures pause queries without logging users out, and which errors end the session @@ -28,6 +29,7 @@ This page tracks significant updates to the QuestDB documentation. ### Reference +- [`acl.oidc.device.authorization.endpoint`](/docs/configuration/oidc/#acloidcdeviceauthorizationendpoint) - Documented the optional device endpoint, discovery-document behavior, and client discovery through `/settings` - [`acl.oidc.sub.claim`](/docs/configuration/oidc/#acloidcsubclaim) and [`acl.oidc.groups.claim`](/docs/configuration/oidc/#acloidcgroupsclaim) - Documented fallback claim lists and their startup validation - [`acl.oidc.issuer`](/docs/configuration/oidc/#acloidcissuer) and [`acl.oidc.issuer.validation.enabled`](/docs/configuration/oidc/#acloidcissuervalidationenabled) - New settings for the expected issuer of a token - [`acl.oidc.pkce.required`](/docs/configuration/oidc/#acloidcpkcerequired) - Corrected the name of the property, previously documented as `acl.oidc.pkce.enabled` @@ -46,6 +48,7 @@ This page tracks significant updates to the QuestDB documentation. ### Updated +- [Java](/docs/connect/clients/java/#oidc-device-flow), [Python](/docs/connect/clients/python/#oidc-device-flow), [Rust](/docs/connect/clients/rust/#oidc-device-flow), and [C/C++](/docs/connect/clients/c-and-cpp/#oidc-device-flow) clients - Added device-flow provider integration and distinguished silent renewal from fixed-token replacement - [OpenID Connect (OIDC)](/docs/security/oidc/) - Split into an overview and provider guides, with a minimal configuration and an upgrade warning for QuestDB Enterprise 4.0.2 - [Mapping user permissions](/docs/security/oidc/#mapping-user-permissions) - Corrected that a login without groups is rejected, and added the endpoint permission of each interface - [Non-interactive clients](/docs/security/oidc/#non-interactive-clients) - Which token each client sends, with client credentials and password grant examples over QWP diff --git a/documentation/configuration/oidc.md b/documentation/configuration/oidc.md index 8911287b69..ca7b544762 100644 --- a/documentation/configuration/oidc.md +++ b/documentation/configuration/oidc.md @@ -253,6 +253,28 @@ ties the authorization code to the browser that started the login. OIDC Authorization Endpoint. The default value should work for the Ping Identity Platform. +### acl.oidc.device.authorization.endpoint + +- **Default**: none +- **Reloadable**: no + +The provider's device authorization endpoint, used by the +[Java, Python, Rust, C, and C++ device-flow helpers](/docs/security/oidc/device-flow/). +With [`acl.oidc.host`](#acloidchost) and manual endpoint configuration, set +this to the provider's endpoint path. QuestDB combines it with the configured +host, port, and TLS mode. There is no provider-independent default. + +With [`acl.oidc.configuration.url`](#acloidcconfigurationurl), QuestDB reads +`device_authorization_endpoint` from the provider's discovery document +instead. A missing, empty, or blank value leaves this endpoint unset and does +not prevent startup. An invalid nonblank endpoint is rejected like the other +discovered endpoint URLs. + +When present, QuestDB includes the resolved URL in `/settings` so clients can +discover it. The client, not QuestDB, sends device authorization and token +requests to the provider. Setting this property does not enable the grant at +the provider or enable QuestDB's ROPC flow. + ### acl.oidc.public.keys.endpoint - **Default**: `/pf/JWKS` diff --git a/documentation/connect/clients/c-and-cpp.md b/documentation/connect/clients/c-and-cpp.md index fb49ed1853..2dae962e87 100644 --- a/documentation/connect/clients/c-and-cpp.md +++ b/documentation/connect/clients/c-and-cpp.md @@ -333,16 +333,45 @@ a later call on it. A connection that the pool opens later, such as a reconnect, can also fail authentication, for example after the token expires (see [Which errors mean what](#which-errors-mean-what)). -### Unsupported auth paths - -The client supports only HTTP basic auth and static bearer-token auth. The -following are **not** supported: - -| Path | Status | Workaround | +### OIDC device flow + +With matching OIDC-enabled headers and library, the client can acquire and +refresh tokens for a human user through the OAuth 2.0 Device Authorization +Grant. The user authorizes a displayed code in a browser on any device; the +application does not collect their password. + +- **C++:** include ``, build an auth object with + `questdb::oidc::builder::from_questdb(url).build()`, call `auth.sign_in()`, + then construct `questdb::pool{conf, auth}`. +- **C:** include ``, build an auth with + `questdb_oidc_builder_from_questdb` and `questdb_oidc_builder_build`, then + call `questdb_oidc_auth_sign_in`. Initialize `questdb_db_connect_options`, + set its `oidc_auth` field, and pass it to `questdb_db_connect_ex`. + +The pool retains shared auth state and resolves a current token on each +sender/reader connection or reconnect. Provider calls can refresh silently but +never display a prompt. Sign in before starting the pool, and do not combine +the provider with `token=`, `username`, or `password`. + +For complete checked-error examples, provider setup, client-build availability, +persistence, and reauthentication, see the +[C++](/docs/security/oidc/device-flow/#cpp) and +[C](/docs/security/oidc/device-flow/#c) device-flow examples. For standalone +senders, use C++ `opts::oidc_auth(auth)` or C +`line_sender_opts_oidc_auth(options, auth, &error)`. + +### Authentication support {#unsupported-auth-paths} + +The client supports HTTP basic auth, static bearer tokens, and OIDC device flow: + +| Path | Status | Usage | |---|---|---| -| OIDC token acquisition or in-band refresh | Not supported. The client does not negotiate with an identity provider and has no callback to refresh a token mid-session. | QuestDB itself supports OIDC; see [OpenID Connect](/docs/security/oidc/#non-interactive-clients) for which token to send. Acquire the token out-of-band from your IdP, pass it via `token=...`, and rebuild the pool when the token nears expiry. For Azure services, see [Microsoft Entra ID managed identities and service principals](/docs/security/oidc/entra-id/#managed-identities-and-service-principals). | +| OIDC device acquisition and refresh | Supported with a shared device auth object. | Sign in explicitly, then attach the provider to the pool or standalone sender. See [Device flow](#oidc-device-flow). | | Mutual TLS (client certificates) | Not supported. The QuestDB server does not negotiate client certificates regardless of client. | Use bearer-token auth over `wss`. See the connect-string reference's [TLS section](/docs/connect/clients/connect-string/#tls). | -| Token rotation mid-session | Not supported. The pool sends the credentials in the WebSocket upgrade of every connection that it opens, including reconnects, and cannot switch to a new token while it runs. | Before the token expires, build a new pool with a new token, then close the old one once its senders have waited for their acknowledgements (`qwp_sender_wait`, C++ `wait()`): closing a pool discards the rows that QuestDB has not acknowledged within `close_flush_timeout_millis`. With `sf_dir`, flush and close the old pool first, then build the new one with the same `sender_id`: two pools cannot hold the same store-and-forward slot, and the new pool replays what the old one left on disk. On QuestDB Enterprise 4.0.2 and later, or with the User Info endpoint, QuestDB rejects the token on new connections shortly after it expires; see [Token lifetime](/docs/security/oidc/#token-lifetime). | +| Changing a fixed `token=` value in place | Not supported; a device-flow provider can renew tokens without replacing the pool. | With a fixed token, build a new pool before expiry and wait for the old senders' acknowledgements before closing it. With `sf_dir`, flush and close the old pool first, then build the new one with the same `sender_id` so it can replay retained rows. See [Token lifetime](/docs/security/oidc/#token-lifetime). | + +Device flow is a human login. For unattended Azure services, see +[Managed identities and service principals](/docs/security/oidc/entra-id/#managed-identities-and-service-principals). ## Headers diff --git a/documentation/connect/clients/connect-string.md b/documentation/connect/clients/connect-string.md index 5e2d93172b..a7403ff107 100644 --- a/documentation/connect/clients/connect-string.md +++ b/documentation/connect/clients/connect-string.md @@ -242,9 +242,17 @@ the TLS handshake is server-authenticated only. `tls_roots` / `tls_roots_password` configure server-cert trust, not client identity. Use `token=` or `username=` / `password=` for client authentication. +For interactive human sign-in, the Java, Python, Rust, C, and C++ clients also +accept a programmatically supplied +[OIDC device-flow provider](/docs/security/oidc/device-flow/). Sign in explicitly +and attach the auth object or callback to the client; do not copy its token +into a fixed `token=` string. Device flow is not a `token_provider=oidc` +connect-string option, and must not be combined with static credentials. + ### Token providers {#token-providers} -*Applies to: ingress and egress. Java client only, 1.3.10 or later.* +*Applies to: ingress and egress. The connect-string provider keys below are +Java-only, 1.3.10 or later.* A token provider requests bearer tokens from an identity provider, caches them, and renews them in the background before they expire, so every diff --git a/documentation/connect/clients/java.md b/documentation/connect/clients/java.md index aa3ac9a23f..83a87c0905 100644 --- a/documentation/connect/clients/java.md +++ b/documentation/connect/clients/java.md @@ -44,6 +44,9 @@ Key capabilities: async [`Completion`](#completion) handles, and byte-credit flow control. - **Failover**: multi-endpoint connections with automatic reconnect across rolling upgrades and primary migrations. +- **OIDC device sign-in**: authenticate a human user from a CLI, desktop + application, or remote process without collecting a password. See + [OIDC device flow](#oidc-device-flow). - **Token renewal**: with a token provider, such as `token_provider=azure` for Microsoft Entra ID managed identities and service principals, the client requests bearer tokens and renews them before they expire. See @@ -409,6 +412,38 @@ User Info endpoint, reject a token shortly after it expires; see [Token lifetime](/docs/security/oidc/#token-lifetime). For tokens that expire, use a [token provider](#rotating-tokens). +### OIDC device flow + +`OidcDeviceAuth` in the core `questdb-client` artifact signs a user in with the +OAuth 2.0 Device Authorization Grant. It displays a verification URL and code; +the user can authorize in a browser on another device. No Azure adapter is +required. Sign in explicitly, then pass the provider to the handle: + +```java +import io.questdb.client.QuestDB; +import io.questdb.client.cutlass.auth.OidcDeviceAuth; + +try (OidcDeviceAuth auth = OidcDeviceAuth.fromQuestDB( + "https://questdb.example.com:9000")) { + auth.signIn(); + try (QuestDB db = QuestDB.connect( + "wss::addr=questdb.example.com:9000;", auth::getToken)) { + // Both sender and query connections use the shared provider. + } +} +``` + +`getToken()` can refresh silently but never prompts. Request `offline_access` +if the provider needs that scope to issue a refresh token. Without a usable +cached or refreshable credential, call `signIn()` again outside transport +callbacks. Do not copy its result into a fixed `token=` string. Close the +handle before closing the auth object. + +For provider setup, a complete ingestion example, token persistence, and +reauthentication, see [OIDC device flow](/docs/security/oidc/device-flow/#java). +For unattended service identities rather than a human login, use the rotating +token providers below. + ### Rotating tokens The tokens of an identity provider expire, for example after one to 24 hours diff --git a/documentation/connect/clients/python.md b/documentation/connect/clients/python.md index 13bfcbff53..05a860a113 100644 --- a/documentation/connect/clients/python.md +++ b/documentation/connect/clients/python.md @@ -172,16 +172,48 @@ operating-system certificate store. Override it with configuration keys: See the [connect string reference](/docs/connect/clients/connect-string/) for the full grammar. -### Unsupported auth paths +### OIDC device flow -The client supports only HTTP basic auth and static bearer-token auth. The -following are **not** supported: +With an OIDC-enabled client build, `questdb.auth.OidcDeviceAuth` signs a human +user in from a terminal or notebook, including a remote kernel. The user opens +a verification URL and enters a code; the application does not collect their +password. Sign in explicitly, then share the provider with the pool: -| Path | Status | Workaround | +```python +import questdb +from questdb.auth import OidcDeviceAuth + +with OidcDeviceAuth.from_questdb("https://questdb.example.com:9000") as auth: + auth.sign_in() + with questdb.connect( + "wss::addr=questdb.example.com:9000;", oidc_auth=auth + ) as db: + # Senders and queries obtain current tokens from the same auth state. + pass +``` + +`auth.token()` and transport callbacks never prompt. They return a cached or +silently refreshed token, or raise `OidcInteractionRequired` when an explicit +`sign_in()` is needed. For a standalone ILP/HTTP sender, pass `oidc_auth=auth` +to `Sender.from_conf(...)`. Do not combine it with static credentials or close +the auth provider while attached transports are still in use. + +See [OIDC device flow](/docs/security/oidc/device-flow/#python) for client-build +availability, provider prerequisites, a complete ingestion example, persistence, +and reauthentication. Device flow requires a person; for unattended Azure +services, see +[Managed identities and service principals](/docs/security/oidc/entra-id/#managed-identities-and-service-principals). + +### Authentication support {#unsupported-auth-paths} + +The client supports HTTP basic auth, static bearer tokens, and device-flow +authentication through a shared provider: + +| Path | Status | Usage | | --- | --- | --- | -| OIDC token acquisition or in-band refresh | Not supported. The client does not negotiate with an identity provider and cannot refresh a token mid-session. | QuestDB itself supports OIDC; see [OpenID Connect](/docs/security/oidc/#non-interactive-clients) for which token to send. Acquire the token out-of-band from your IdP, pass it via `token=...`, and rebuild the handle when the token nears expiry. For an example with an Entra ID managed identity, see [Microsoft Entra ID managed identities and service principals](/docs/security/oidc/entra-id/#managed-identities-and-service-principals). | +| OIDC device acquisition and refresh | Supported with `OidcDeviceAuth`. | Sign in explicitly and pass `oidc_auth=auth`. Refresh requires a refresh token from the provider. See [Device flow](#oidc-device-flow). | | Mutual TLS (client certificates) | Not supported. The QuestDB server does not negotiate client certificates regardless of client. | Use bearer-token auth over `wss`. | -| Token rotation mid-session | Not supported. The handle keeps the credentials it was built with and presents them on every connection it opens, including reconnects and failover, and cannot switch to a new token while it runs. | Before the token expires, build a fresh handle with a new token, then close the old one once its senders have flushed with `flush(wait=True)`: closing a handle discards the rows that QuestDB has not acknowledged within `close_flush_timeout_millis`. With `sf_dir`, flush and close the old handle first, then build the new one with the same `sender_id`: two handles cannot hold the same store-and-forward slot, and the new handle replays what the old one left on disk. On QuestDB Enterprise 4.0.2 and later, or with the User Info endpoint, QuestDB rejects the token on new connections shortly after it expires, so reconnects and failover then fail too; see [Token lifetime](/docs/security/oidc/#token-lifetime). For an example, see [Request a token in the service](/docs/security/oidc/entra-id/#request-a-token-in-the-service). | +| Changing a fixed `token=` value in place | Not supported; this restriction does not apply to `oidc_auth`. | Before a fixed token expires, build a fresh handle with a new token, then close the old one once its senders have flushed with `flush(wait=True)`. Closing a handle discards rows not acknowledged within `close_flush_timeout_millis`. With `sf_dir`, flush and close the old handle first, then build the new one with the same `sender_id` so it can replay the retained rows. See [Token lifetime](/docs/security/oidc/#token-lifetime). | ## The pool diff --git a/documentation/connect/clients/rust.md b/documentation/connect/clients/rust.md index 2b6a0078be..914d703fca 100644 --- a/documentation/connect/clients/rust.md +++ b/documentation/connect/clients/rust.md @@ -174,6 +174,54 @@ what the old one left on disk. On QuestDB Enterprise 4.0.2 and later, or with the User Info endpoint, QuestDB rejects the token on new connections shortly after it expires; see [Token lifetime](/docs/security/oidc/#token-lifetime). +### OIDC device flow + +With an OIDC-enabled client build and the `oidc` feature, use +`questdb::oidc::OidcDeviceAuth` for an interactive human login. It displays a +verification URL and code that the user authorizes on any device. Sign in +explicitly, then pass a provider closure to +`QuestDb::connect_with_token_provider` so both ingestion and query connections +can obtain a current token: + +```text title="Cargo.toml" +[dependencies] +questdb-rs = { version = "7", features = ["oidc"] } +``` + +```rust +use std::sync::Arc; +use questdb::{oidc::OidcDeviceAuth, QuestDb}; + +fn main() -> questdb::Result<()> { + let auth = Arc::new( + OidcDeviceAuth::from_questdb("https://questdb.example.com:9000").build()?, + ); + auth.sign_in()?; + let db = QuestDb::connect_with_token_provider( + "wss::addr=questdb.example.com:9000;", + { + let auth = Arc::clone(&auth); + move || auth.token() + }, + )?; + // Borrow senders and readers from db; they share the auth state. + drop(db); + Ok(()) +} +``` + +`token()` can refresh silently but never starts an interactive sign-in. +Without a usable cached or refreshable credential it returns an OIDC +`InteractionRequired` error; call `sign_in()` outside transport callbacks. +The closure's `Arc` keeps the auth object alive for the pool. Do not also set +static credentials in the configuration string. + +See [OIDC device flow](/docs/security/oidc/device-flow/#rust) for client-build +availability, a complete ingestion example, discovery, persistence, and +reauthentication. The `oidc` feature is not enabled by default. + +### TLS + With the default crate features, the TLS root set is `webpki_roots`. Other choices have feature requirements: diff --git a/documentation/security/oidc/device-flow.mdx b/documentation/security/oidc/device-flow.mdx new file mode 100644 index 0000000000..432a792a57 --- /dev/null +++ b/documentation/security/oidc/device-flow.mdx @@ -0,0 +1,522 @@ +--- +title: OIDC device flow for client applications +sidebar_label: Device flow +description: "Sign in to QuestDB Enterprise from Java, Python, Rust, C, and C++ with OIDC device flow, then reuse and refresh tokens without storing a password." +--- + +The OAuth 2.0 Device Authorization Grant +([RFC 8628](https://www.rfc-editor.org/rfc/rfc8628)), also called *device flow*, +lets a user sign in to a QuestDB client without giving the application their +password. The client shows a verification URL and a short code. The user +opens the URL in a browser, on the same machine or another device, and completes +the identity provider's sign-in, including MFA when required. The client polls +the provider for tokens and uses them to authenticate to QuestDB Enterprise. + +Use device flow for a notebook, CLI, or desktop application acting as a human +user, including a remote kernel or container without a local browser. The +client only needs outbound connections; it does not run a local callback server +or receive a browser redirect. Device flow still requires a person to authorize +access. For unattended services, use +[managed identities or service principals](/docs/security/oidc/entra-id/#managed-identities-and-service-principals) +or the [client credentials grant](/docs/security/oidc/#client-credentials-grant) +instead. + +## Prerequisites + +- QuestDB Enterprise with [OIDC configured](/docs/security/oidc/#configuration-options), + reachable over HTTPS. If necessary, configure + [HTTP server TLS](/docs/security/tls/#enabling-tls-for-http-server-rest-api) + or a TLS-terminating proxy that supports WebSocket upgrades. +- An identity provider that supports device authorization. Enable the device + grant for the client ID used by QuestDB, and allow public-client + authentication without an application secret. Device flow does not use the + Web Console's redirect URI and does not require + [`acl.oidc.ropc.flow.enabled`](/docs/configuration/oidc/#acloidcropcflowenabled). +- The `openid` scope so the provider returns an ID token. Request + `offline_access` if the provider requires it to issue a refresh token. Set + [`acl.oidc.scope`](/docs/configuration/oidc/#acloidcscope), for example to + `openid offline_access`, and keep any additional scopes needed for group + claims. +- A user whose provider groups map to QuestDB groups with the required endpoint + and table permissions. A successful device sign-in alone does not grant + database access. See [Prepare the database](#prepare-the-database). + +:::note Client availability + +Use an OIDC-enabled client build. The Java examples use `questdb-client` +1.3.10 or later; no Azure adapter is required. The Python APIs described here +are part of [py-questdb-client#133](https://github.com/questdb/py-questdb-client/pull/133). +The Rust, C, and C++ APIs are part of +[c-questdb-client#183](https://github.com/questdb/c-questdb-client/pull/183). +Until those changes are released, use a build containing them rather than an +older package that supports only static tokens. In Rust, also enable the +`oidc` crate feature. + +::: + +### Discover the identity provider + +The helpers discover the client ID, scope, token endpoint, audience, and +`groups.encoded.in.token` setting from the QuestDB server's public `/settings` +endpoint. QuestDB advertises the device authorization endpoint when the +provider's discovery document contains `device_authorization_endpoint`. With +manual endpoint configuration, set +[`acl.oidc.device.authorization.endpoint`](/docs/configuration/oidc/#acloidcdeviceauthorizationendpoint). +A missing device endpoint does not stop QuestDB from serving other OIDC flows, +but the client cannot start device flow without one. + +If the server does not advertise that endpoint, supply the provider's issuer +explicitly. The helper can then discover it from the issuer's +`/.well-known/openid-configuration` document: + +| Client | Issuer override | +| --- | --- | +| Java | `new OidcDeviceAuth.DiscoveryOptions().issuer("https://idp.example.com")`, passed as the second argument to `fromQuestDB(...)` | +| Python | `OidcDeviceAuth.from_questdb(url, issuer="https://idp.example.com")` | +| Rust | `OidcDeviceAuth::from_questdb(url).issuer("https://idp.example.com").build()?` | +| C++ | `questdb::oidc::builder::from_questdb(url).issuer("https://idp.example.com").build()` | +| C | `questdb_oidc_builder_issuer(builder, issuer, issuer_len, &error)` before building the auth object | + +Use an issuer obtained from your administrator, not one inferred from an +untrusted server. An issuer override pins the endpoints to the issuer's origin; +settings-sourced endpoints must also satisfy the client's issuer-path checks. +If the provider intentionally hosts endpoints elsewhere, configure its trusted +endpoints explicitly instead of weakening TLS verification. + +For Microsoft Entra ID, complete the +[Web Console and group setup](/docs/security/oidc/entra-id/#web-console-sign-in), +then enable **Allow public client flows** under the application's +**Authentication** settings. Tenant policy must also permit device code flow. +Use `openid profile offline_access` and token mode so the helpers send the ID +token, not a Microsoft Graph access token. This is a user login, not the +app-only role setup for managed identities. See +[Microsoft's device authorization documentation](https://learn.microsoft.com/en-us/entra/identity-platform/v2-oauth2-device-code). + +### Private CA trust + +If QuestDB or the provider uses a private certificate authority, configure +trust for the auth helper as well as for the database transport. The +connection string's `tls_roots` setting does not configure the helper that +fetches `/settings` and calls the provider: + +| Client | Auth helper configuration | +| --- | --- | +| Java | Pass `DiscoveryOptions.tlsConfig(tls)` to `fromQuestDB(...)`, where `tls` is an `io.questdb.client.ClientTlsConfiguration` with your trust store and `TLS_VALIDATION_MODE_FULL` | +| Python | Pass `ca_bundle="/path/to/ca.pem"` to `from_questdb(...)` | +| Rust | Add `.ca_bundle("/path/to/ca.pem")` to the builder | +| C++ | Add `.ca_bundle("/path/to/ca.pem")` to the builder | +| C | Call `questdb_oidc_builder_ca_bundle(builder, path, path_len, &error)` before building | + +Keep certificate and hostname verification enabled. A helper's trust settings +apply to its QuestDB discovery request and its provider requests, so the trust +store must cover both. See the client pages for the +[database transport's TLS settings](/docs/connect/clients/connect-string/#tls). + +### Prepare the database + +The examples below write the same `fx_trades` schema. As the admin, create it +before running a client: ingestion cannot create tables or columns for an +[external user](/docs/security/oidc/#tables-created-by-external-users). + +```questdb-sql title="Create the table and grant the user's mapped group access" +CREATE TABLE fx_trades ( + timestamp TIMESTAMP_NS, + symbol SYMBOL, + side SYMBOL, + price DOUBLE, + quantity DOUBLE +) TIMESTAMP(timestamp) PARTITION BY DAY; + +CREATE GROUP fxUsers WITH EXTERNAL ALIAS 'fx-users'; +GRANT HTTP TO fxUsers; +GRANT INSERT, SELECT ON fx_trades TO fxUsers; +``` + +Replace `fx-users` with a group value in the user's token or User Info response. +For Entra ID security groups, use the group's object ID. A user without a +populated groups claim is rejected, even if the device authorization succeeded. +See [Mapping user permissions](/docs/security/oidc/#mapping-user-permissions). + +## Sign in and connect + +Replace `questdb.example.com:9000` in both the discovery URL and the connection +string with your secure QuestDB endpoint. Sign in explicitly before opening +connections. Keep the auth object alive until its connections have closed. +Do not combine a provider with `token=`, `username`, or `password` in the +connection string. + +The examples use QWP for ingestion; the same shared provider supplies the query +connections in each pool. The provider returns the access token in User Info +mode and the ID token in token mode, matching the server's configuration. + +### Java + +Add the [Java client dependency](/docs/connect/clients/java/#quick-start). +`OidcDeviceAuth` is in the core `questdb-client` artifact, not +`questdb-client-azure`: + +```java +import io.questdb.client.QuestDB; +import io.questdb.client.Sender; +import io.questdb.client.cutlass.auth.OidcDeviceAuth; + +public class DeviceFlowIngest { + public static void main(String[] args) { + try (OidcDeviceAuth auth = OidcDeviceAuth.fromQuestDB( + "https://questdb.example.com:9000")) { + auth.signIn(); + try (QuestDB db = QuestDB.connect( + "wss::addr=questdb.example.com:9000;", auth::getToken)) { + try (Sender sender = db.borrowSender()) { + sender.table("fx_trades") + .symbol("symbol", "EURUSD") + .symbol("side", "buy") + .doubleColumn("price", 1.0842) + .doubleColumn("quantity", 100000.0) + .atNow(); + if (!sender.drain(30_000)) { + throw new IllegalStateException("Acknowledgement timed out"); + } + } + } + } + } +} +``` + +`signIn()` may prompt; `getToken()` only returns a cached or silently refreshed +token. Passing `auth::getToken` lets new connections and reconnects obtain a +current token without rebuilding the handle. For a standalone ILP/HTTP sender, +use `httpTokenProvider(auth::getToken)` instead of a fixed `httpToken(...)`. + +By default, Java prints the URL and code and attempts to open a browser. On a +remote or shared host, disable browser launch with +`-Dquestdb.client.oidc.open.browser=false`, or pass +`new OidcDeviceAuth.DiscoveryOptions().prompt(DeviceCodePrompt.SYSTEM_OUT)` +with `io.questdb.client.cutlass.auth.DeviceCodePrompt` imported. + +### Python + +Install an [OIDC-enabled Python client](#prerequisites). The device-flow helper +requires no additional OAuth Python dependency: + +```python +import questdb +from questdb import TimestampNanos +from questdb.auth import OidcDeviceAuth + +with OidcDeviceAuth.from_questdb("https://questdb.example.com:9000") as auth: + auth.sign_in() + with questdb.connect( + "wss::addr=questdb.example.com:9000;", oidc_auth=auth + ) as db: + with db.sender() as sender: + sender.row( + "fx_trades", + symbols={"symbol": "EURUSD", "side": "buy"}, + columns={"price": 1.0842, "quantity": 100000.0}, + at=TimestampNanos.now(), + ) + sender.flush(wait=True) +``` + +`sign_in()` presents the prompt in the terminal or notebook. +`auth.token()` and transport operations never prompt. Use `open_browser=False` +in `from_questdb(...)` to suppress browser launch; a notebook normally shows +the challenge in the kernel's output rather than opening a browser on the +kernel's host. For a standalone ILP/HTTP sender, pass `oidc_auth=auth` to +`questdb.Sender.from_conf("https::addr=questdb.example.com:9000;", ...)`. + +### Rust + +Enable OIDC in addition to the default transport and TLS features: + +```text title="Cargo.toml" +[dependencies] +questdb-rs = { version = "7", features = ["oidc"] } +``` + +```rust +use std::sync::Arc; + +use questdb::{ + ingress::{AckLevel, TimestampNanos}, + oidc::OidcDeviceAuth, + QuestDb, +}; + +fn main() -> questdb::Result<()> { + let auth = Arc::new( + OidcDeviceAuth::from_questdb("https://questdb.example.com:9000").build()?, + ); + auth.sign_in()?; + + let db = QuestDb::connect_with_token_provider( + "wss::addr=questdb.example.com:9000;", + { + let auth = Arc::clone(&auth); + move || auth.token() + }, + )?; + { + let mut sender = db.borrow_sender()?; + let mut buffer = sender.new_buffer(); + buffer + .table("fx_trades")? + .symbol("symbol", "EURUSD")? + .symbol("side", "buy")? + .column_f64("price", 1.0842)? + .column_f64("quantity", 100000.0)? + .at(TimestampNanos::now())?; + sender.flush_buffer_and_wait(&mut buffer, AckLevel::Ok)?; + } + Ok(()) +} +``` + +The `Arc` keeps one auth state shared by the pool's token-provider closure. +`sign_in()` is interactive; `token()` is not. For a standalone sender, use +`SenderBuilder::http_token_provider` for ILP/HTTP or +`SenderBuilder::qwp_ws_token_provider` for QWP. Set `.open_browser(false)` on +the OIDC builder when the process should only display the URL and code. + +### C++ {#cpp} + +Use matching OIDC-enabled headers and the native library from the +[C/C++ client](/docs/connect/clients/c-and-cpp/#cmake-setup): + +```cpp +#include +#include +#include + +#include +#include + +using namespace questdb::ingress::literals; + +int main() +{ + try + { + auto auth = questdb::oidc::builder::from_questdb( + "https://questdb.example.com:9000").build(); + auth.sign_in(); + questdb::pool pool{"wss::addr=questdb.example.com:9000;", auth}; + { + auto sender = pool.borrow_sender(); + auto buffer = sender.new_buffer(); + buffer.table("fx_trades"_tn) + .symbol("symbol"_cn, "EURUSD"_utf8) + .symbol("side"_cn, "buy"_utf8) + .column("price"_cn, 1.0842) + .column("quantity"_cn, 100000.0) + .at(questdb::ingress::timestamp_nanos::now()); + sender.flush_and_wait(buffer); + } + } + catch (const std::exception& error) + { + std::cerr << error.what() << '\n'; + return 1; + } +} +``` + +The pool retains shared ownership of the auth state. Its connection callbacks +refresh silently and never run device sign-in. For a standalone sender, +configure `questdb::ingress::opts::oidc_auth(auth)`. To suppress browser launch, +set `.open_browser(false)` on the OIDC builder. + +### C + +Initialize `questdb_db_connect_options` with its initializer before setting +`oidc_auth`. Check each fallible operation and release the pool and auth handles +on both success and failure: + +```c +#include +#include +#include + +#include +#include + +int main(void) +{ + const char* url = "https://questdb.example.com:9000"; + const char* conf = "wss::addr=questdb.example.com:9000;"; + questdb_error* error = NULL; + questdb_oidc_builder* builder = NULL; + questdb_oidc_auth* auth = NULL; + questdb_db* db = NULL; + qwp_sender* sender = NULL; + line_sender_buffer* buffer = NULL; + int result = 1; + + builder = questdb_oidc_builder_from_questdb(url, strlen(url), &error); + if (!builder) goto cleanup; + auth = questdb_oidc_builder_build(builder, &error); + if (!auth) goto cleanup; + if (!questdb_oidc_auth_sign_in(auth, &error)) goto cleanup; + + questdb_db_connect_options options; + questdb_db_connect_options_init(&options, sizeof options); + options.oidc_auth = auth; + db = questdb_db_connect_ex(conf, strlen(conf), &options, &error); + if (!db) goto cleanup; + sender = questdb_db_borrow_sender(db, &error); + if (!sender) goto cleanup; + buffer = questdb_db_new_buffer(db, &error); + if (!buffer) goto cleanup; + if (!line_sender_buffer_table( + buffer, QDB_TABLE_NAME_LITERAL("fx_trades"), &error)) goto cleanup; + if (!line_sender_buffer_symbol(buffer, QDB_COLUMN_NAME_LITERAL("symbol"), + QDB_UTF8_LITERAL("EURUSD"), &error)) goto cleanup; + if (!line_sender_buffer_symbol(buffer, QDB_COLUMN_NAME_LITERAL("side"), + QDB_UTF8_LITERAL("buy"), &error)) goto cleanup; + if (!line_sender_buffer_column_f64(buffer, QDB_COLUMN_NAME_LITERAL("price"), + 1.0842, &error)) goto cleanup; + if (!line_sender_buffer_column_f64(buffer, QDB_COLUMN_NAME_LITERAL("quantity"), + 100000.0, &error)) goto cleanup; + if (!line_sender_buffer_at_nanos( + buffer, line_sender_now_nanos(), &error)) goto cleanup; + if (!qwp_sender_flush_buffer_and_wait( + sender, buffer, qwpws_ack_level_ok, &error)) goto cleanup; + result = 0; + +cleanup: + if (error) + { + size_t length = 0; + const char* message = questdb_error_msg(error, &length); + fprintf(stderr, "%.*s\n", (int)length, message); + } + line_sender_buffer_free(buffer); + if (sender) questdb_db_return_sender(db, sender); + questdb_db_close(db); + questdb_oidc_auth_free(auth); + questdb_oidc_builder_free(builder); + questdb_error_free(error); + return result; +} +``` + +For a standalone sender, attach the auth with +`line_sender_opts_oidc_auth(options, auth, &error)`. Disable browser launch with +`questdb_oidc_builder_open_browser(builder, false, &error)` before building the +auth object. Without a custom event handler, the native client prints the +verification URL and code to standard error. + +## Token renewal and reauthentication + +Keep **interactive sign-in** separate from **token retrieval**: + +1. Run `signIn()` / `sign_in()` explicitly while a user can respond. It can + reuse a cached token or refresh silently; otherwise it starts device flow. +2. Attach the auth object or a token-provider callback to the transport. + Connection and reconnect callbacks can return cached tokens or refresh them, + but never start an interactive sign-in. +3. A refresh requires a refresh token from the provider. Request + `offline_access` where required; it does not guarantee one will be issued. +4. If the token expires without a usable refresh token, or the provider requires + user interaction again, token retrieval fails. Python raises + `OidcInteractionRequired`; Rust and C/C++ report an OIDC + `InteractionRequired` error kind; Java throws `OidcAuthException` with a + sign-in instruction. Run explicit sign-in again outside transport callbacks. + Handle any separate transport-level failure before resuming ingestion; an + authentication helper does not undo a sender's terminal state. + +Silent refresh performs network I/O and can block token retrieval, a flush, +or connection setup. The auth helper's HTTP timeout and waits for another +thread's refresh are separate from the transport's request and reconnect +budgets. Configure those provider waits for the amount of backpressure the +application can tolerate; non-interactive does not mean non-blocking. + +Do not put `auth.token()` or `auth.signIn()` into a `token=` string and expect +renewal. That copies one token: future connections keep using it after expiry. +Use the provider integrations above. Device-flow refresh is demand-driven; it +is not the background half-life renewal of Java's +[`RefreshingTokenProvider`](/docs/connect/clients/java/#rotating-tokens). + +For SQL over PGWire, enable +[`acl.oidc.pg.token.as.password.enabled`](/docs/configuration/oidc/#acloidcpgtokenaspasswordenabled), +connect as `_sso`, and supply a current token as the password. Pull a fresh token +when opening **each new connection**. An already-open connection is not +reauthenticated on token renewal. The Python helper also provides +`questdb.auth.sqlalchemy_engine` and `questdb.auth.psycopg_connect`; the former +resolves the provider for each new pooled connection, while the latter captures +the token when connecting. Use certificate and hostname verification as in +[OIDC for PGWire](/docs/security/oidc/#oidc-for-the-pgwire-endpoint). + +## Persisting tokens and signing out + +Tokens stay in memory by default. To resume a login after a process restart, +opt into a token store: + +| Client | Enable the bundled file store | +| --- | --- | +| Java | Pass `new OidcDeviceAuth.DiscoveryOptions().tokenStore(FileTokenStore.atDefaultLocation())` to `fromQuestDB(...)`; import `io.questdb.client.cutlass.auth.FileTokenStore` | +| Python | Pass `token_store=FileTokenStore.at_default_location()` to `from_questdb(...)`; import it from `questdb.auth` | +| Rust | Add `.token_store(FileTokenStore::at_default_location()?)` to the builder; import it from `questdb::oidc` | +| C++ | Add `.default_file_token_store()` to the builder | +| C | Call `questdb_oidc_builder_default_file_token_store(builder, &error)` before building | + +The default directory is `~/.questdb/oidc-tokens/`. Python and native clients +accept an absolute directory override in `QUESTDB_CLIENT_OIDC_TOKEN_STORE_DIR`; +Java uses the JVM system property `questdb.client.oidc.token.store.dir`. +A stored refresh token lets a restarted process refresh silently rather than +prompt again, only while that credential remains valid. + +:::warning File persistence stores credentials + +The bundled file stores write **unencrypted JSON**, including long-lived refresh +tokens. Protect the directory and never commit, log, or share its contents. +On POSIX, directories and files use owner-only permissions. Python and native +file-store mutations currently require Unix; use in-memory authentication on +other platforms. Java's file store on Windows relies on the profile directory's +ACLs rather than enforcing POSIX modes. + +A store entry identifies an OIDC configuration, not a separate application +user. Use separate store directories for different people sharing one OS +account. File permissions protect against credential theft and substitution; +the file format is not an encrypted or authenticated vault. + +::: + +Clear the local cached login with Java `clearCache()`, Python `clear()`, Rust +`clear()`, C++ `clear()`, or C `questdb_oidc_auth_clear(...)`. These operations +also remove the configured persisted entry, but **do not revoke tokens at the +identity provider or terminate existing database connections**. For immediate +access removal, see +[Group changes, sessions, and revoking access](/docs/security/oidc/#group-changes-sessions-and-revoking-access). +Close transports before closing the auth provider. Explicitly closing a Python +or native auth provider also prevents its attached transports from acquiring +more tokens; recreate both provider and transports to recover. Closing releases +resources; it is not an identity-provider logout. + +## Security and troubleshooting + +- Only discover settings from a QuestDB server you trust, over HTTPS. Those + settings designate where the client sends the device code and refresh token. + The provider's credential endpoints also require HTTPS outside loopback + development environments. Configure private CA trust rather than disabling + certificate or hostname verification. The connection string's TLS settings + do not automatically configure the auth helper's HTTP client. +- Treat device codes and verification links as login challenges. Only approve a + flow you initiated, verify the provider and account before accepting it, and + do not publish the challenge in shared logs. A headless client can display + the code without launching a browser. +- If discovery cannot find the device endpoint, check the provider's discovery + document, the server's device-endpoint setting, or configure a trusted issuer + explicitly. Enabling QuestDB's ROPC setting does not enable device flow. +- If token mode reports a missing ID token, include `openid` and ensure the + provider issues ID tokens on both device authorization and refresh. Do not + substitute an access token issued for a different API. +- If device authorization succeeds but QuestDB rejects the connection, check + audience, issuer, group claims, and mapped endpoint/table grants. See + [Troubleshooting OIDC logins](/docs/security/oidc/#troubleshooting-oidc-logins). +- If the code expires or the user declines authorization, start a new explicit + sign-in only when the user is ready. A refresh failure must not cause an + automatic loop of interactive prompts. Retain unacknowledged source data; + obtaining a token does not confirm ingestion delivery. diff --git a/documentation/security/oidc/entra-id.mdx b/documentation/security/oidc/entra-id.mdx index c2f07acbb5..d49e490549 100644 --- a/documentation/security/oidc/entra-id.mdx +++ b/documentation/security/oidc/entra-id.mdx @@ -271,9 +271,9 @@ acl.oidc.redirect.uri=http://localhost:9000 # OAuth scopes the user has to consent to acl.oidc.scope=openid profile offline_access -# enable ROPC flow -# optional, required only if ROPC is enabled in Entra ID -acl.oidc.ropc.flow.enabled=true +# optional: only for clients sending username/password credentials to QuestDB +# not required for Web Console sign-in or device flow +# acl.oidc.ropc.flow.enabled=true ``` `preferred_username` shows the user's sign-in name in the Web Console and in @@ -363,6 +363,27 @@ valid, logging in again usually does not ask for their credentials. See ::: +## Device flow for users + +For Java, Python, Rust, C, or C++ applications acting as a human user, use +[OIDC device flow](/docs/security/oidc/device-flow/) rather than collecting +the user's password. The client displays a URL and code, and the user signs +in through Entra ID in a browser on any device. It works from remote notebook +kernels and applications without a local browser callback. + +After the Web Console and group setup above, enable **Allow public client +flows** in the application's **Authentication** settings, and ensure tenant +policy permits device code flow. Keep `openid profile offline_access` in +`acl.oidc.scope` and token mode enabled. The client uses QuestDB's client ID +and presents the ID token, whose group claims map to the same QuestDB groups +as the Web Console user's claims. Requesting `offline_access` lets Entra ID +issue a refresh token, subject to its policies. + +This flow needs no client secret, Web Console redirect, or QuestDB ROPC +setting. A person must authorize the initial login, and sign in again when +silent refresh is no longer possible. For services that must run without a +person, use the managed-identity or service-principal flow below instead. + ## Managed identities and service principals Azure services that run as a managed identity or a service principal can diff --git a/documentation/security/oidc/index.mdx b/documentation/security/oidc/index.mdx index dab7ce52c4..3ac7d7bbcd 100644 --- a/documentation/security/oidc/index.mdx +++ b/documentation/security/oidc/index.mdx @@ -29,7 +29,10 @@ and service tokens with one configuration, see [Troubleshooting OIDC logins](#troubleshooting-oidc-logins). For step-by-step setup, see the guides for [Microsoft Entra ID](/docs/security/oidc/entra-id/), which also covers Azure managed identities and service principals, and -[PingFederate](/docs/security/oidc/pingfederate/). +[PingFederate](/docs/security/oidc/pingfederate/). To sign a human in from a +notebook, CLI, or application without a browser callback, see +[OIDC device flow for clients](/docs/security/oidc/device-flow/), with Java, +Python, Rust, C, and C++ examples. :::warning Upgrading to QuestDB Enterprise 4.0.2 @@ -69,9 +72,9 @@ acl.oidc.groups.claim=groups # instead of calling the User Info endpoint; required for Microsoft Entra ID # acl.oidc.groups.encoded.in.token=true -# optional, openid by default: the scopes that the Web Console requests; add -# offline_access if the provider issues a refresh token only for that scope, -# such as Microsoft Entra ID, so that the Web Console can renew its tokens +# optional, openid by default: scopes for the Web Console and device-flow +# clients that discover their settings from QuestDB; add offline_access if +# the provider requires it to issue refresh tokens, such as Microsoft Entra ID # acl.oidc.scope=openid offline_access ``` @@ -461,8 +464,29 @@ token can be used for authentication and authorization when communicating with QuestDB as explained in the [above 7th step](#7-database-access): the access token, or in token mode, the ID token. +### Device flow for client applications + +For a notebook, CLI, or application acting as a human user, the +[Java, Python, Rust, C, and C++ device-flow helpers](/docs/security/oidc/device-flow/) +run the OAuth 2.0 Device Authorization Grant. The client displays a URL and a +code, and the user signs in with the identity provider on any device. No +password or browser redirect passes through the application. Sign in +explicitly, then attach the shared auth provider so connections can obtain +cached or silently refreshed tokens without prompting from background threads. + +This is not an unattended service flow. Use the +[client credentials grant](#client-credentials-grant) or an Azure +[managed identity](/docs/security/oidc/entra-id/#managed-identities-and-service-principals) +for jobs that must start without a person present. + ### Jupyter notebook +The Python client's [device-flow helper](/docs/security/oidc/device-flow/#python) +works in local and remote kernels, including notebooks without JupyterHub. +It supports explicit sign-in and silent refresh without collecting the user's +SSO password. Prefer it to the legacy ROPC example below when the provider +supports device flow. + JupyterHub can integrate with OAuth2 providers using OAuthenticator, as described in its [documentation](https://jupyterhub.readthedocs.io/en/stable/explanation/oauth.html). @@ -629,11 +653,17 @@ with pg.connect(conn_str, autocommit=True) as connection: ### CLI, standalone applications -When using CLI tools, such as `psql`, or standalone applications like Microsoft -Access, the best option may be the Resource Owner Password Credentials flow. +For applications that can integrate a client auth helper, use +[device flow](/docs/security/oidc/device-flow/) to sign a user in without +collecting their password. A tool that accepts only PGWire credentials can +connect as `_sso` with an externally obtained device-flow token as the +password; see [OIDC for PGWire](#oidc-for-the-pgwire-endpoint). -The user logs in with their SSO credentials, and the server validates the -details with the OAuth2 provider: +For tools that cannot integrate token acquisition, such as an unmodified +`psql` login, the legacy Resource Owner Password Credentials flow is another +option when explicitly enabled by the administrator and provider. The user +logs in with their SSO credentials, and the server validates them with the +OAuth2 provider: ```shell % psql -h localhost -p 8812 -U testuser @@ -647,6 +677,10 @@ testldap=> ## Non-interactive clients +This section covers jobs and services acting as themselves, or integrations +using the legacy password grant. For a person signing in from a client +application, see [Device flow](/docs/security/oidc/device-flow/) instead. + Non-interactive clients are usually jobs or standalone applications, such as a client for ingesting data. It is practical to manage their credentials via an OAuth2 provider too. Such a client requests a token from the provider itself, @@ -867,9 +901,15 @@ for Microsoft Entra ID, it requests a new token in the background before the current one expires, and presents it on every connection that it opens after that. -Any other long-running client must switch to a new token before the old one -expires: request a new token, create a new client with it, wait until the old -client has delivered what it sent, then close the old client. A QWP client that +The [device-flow helpers](/docs/security/oidc/device-flow/#token-renewal-and-reauthentication) +for Java, Python, Rust, C, and C++ can also supply rotating tokens without +rebuilding the client. They refresh on demand when the provider has issued a +usable refresh token, and require explicit user sign-in again when it has not. + +Without a provider, a long-running client configured with a fixed token must +switch before that token expires: request a new token, create a new client +with it, wait until the old client has delivered what it sent, then close the +old client. A QWP client that is closed discards the rows that QuestDB has not acknowledged within `close_flush_timeout_millis`, unless it keeps them on disk with `sf_dir`, so wait for the acknowledgement first, for example with `flush(wait=True)` in the diff --git a/documentation/sidebars.js b/documentation/sidebars.js index 6999c9e322..b172a89ae8 100644 --- a/documentation/sidebars.js +++ b/documentation/sidebars.js @@ -743,6 +743,11 @@ module.exports = { type: "doc", label: "Overview", }, + { + id: "security/oidc/device-flow", + type: "doc", + label: "Device flow", + }, { id: "security/oidc/entra-id", type: "doc", From c88aba16d6f7a66101f0c4c2daf3f0e8fa12874c Mon Sep 17 00:00:00 2001 From: glasstiger Date: Fri, 16 Oct 2026 13:10:37 +0100 Subject: [PATCH 28/29] docs(oidc): fix recovery guidance and device-flow configuration --- .../connect/clients/connect-string.md | 47 +++++++++-- .../wire-protocols/qwp-client-behavior.md | 55 +++++++------ .../wire-protocols/qwp-ingress-websocket.md | 15 +++- .../client-failover/concepts.md | 33 +++++--- documentation/security/oidc/device-flow.mdx | 81 ++++++++++++++++--- documentation/sidebars.js | 2 +- 6 files changed, 178 insertions(+), 55 deletions(-) diff --git a/documentation/connect/clients/connect-string.md b/documentation/connect/clients/connect-string.md index a7403ff107..1949812d26 100644 --- a/documentation/connect/clients/connect-string.md +++ b/documentation/connect/clients/connect-string.md @@ -700,12 +700,30 @@ architecture is that a producer survives an arbitrarily long outage. This is the shutdown data-loss window. Setting it to `0` skips the drain entirely and drops un-ACKed batches on every clean shutdown. -Authentication rejection, durable-ACK mismatch, and a non-101 upgrade -response without a role hint are immediately terminal during reconnect in -clients other than Java: the loop does not retry them. In Java, a sender -that has connected keeps retrying these rejections. On the first connect, -these rejections are terminal, after any eligible -[token-provider refresh and retry](/docs/connect/clients/java/#how-the-client-handles-token-failures). +Authentication rejection is terminal before a sender's first successful +connection, after any eligible token-provider refresh and retry. Established +Java senders instead keep retrying server upgrade `401`/`403` rejections with +capped backoff while retaining buffered rows, even with fixed credentials. +Rust, C, and C++ builds containing +[c-questdb-client#183](https://github.com/questdb/c-questdb-client/pull/183) +and Python builds containing +[py-questdb-client#133](https://github.com/questdb/py-questdb-client/pull/133) +share this established-sender policy. Older native releases may still treat +authentication rejection as terminal after the first connection. + +Native clients emit `auth_failed` connection events, and an acknowledgement +wait that times out identifies the rejection. Applications requiring a hard +failure must monitor the outage and close the sender themselves. This retry +policy covers server upgrade rejections, not a closed or misconfigured token +provider; those native provider failures remain terminal. Query authentication +rejections remain terminal for the connection attempt or query after any +eligible token retry, rather than starting an indefinite recovery loop. + +A durable-ACK mismatch or non-101 upgrade response without a role hint remains +terminal during native reconnect. Java senders that have already connected +retry these rejections; at initial connect they are terminal. See +[How the client handles token failures](/docs/connect/clients/java/#how-the-client-handles-token-failures) +for Java's token-provider refresh and retry rules. In Java, a protocol-version mismatch is transient, including at startup. With `initial_connect_retry=on`, the client retries it within @@ -724,9 +742,24 @@ Java orphan drainers have a separate policy: - Role rejections while all reachable servers are replicas are transient: the drainer keeps retrying rather than quarantining the slot. -For the token-rejection thresholds and reporting, see +For the Java token-rejection thresholds and reporting, see [How the client handles token failures](/docs/connect/clients/java/#how-the-client-handles-token-failures). +Native orphan drainers in the builds listed above also distinguish fixed and +rotating credentials: + +- A rejected fixed credential retires the slot immediately for the current + session. +- With a token provider, the drainer retries until at least six rejections have + persisted for `min(reconnect_max_duration_millis, 300000)` milliseconds, or + until 256 rejections occur without acknowledgement progress. Other failure + types reset the persistence timer but not the rejection count; new + acknowledgement progress resets both. + +Retiring a native slot records `.last_error` and stops draining it for the +rest of the session, without writing Java's `.failed` sentinel. Unacknowledged +rows remain on disk, and the next session tries the slot again. + ### Egress failover {#egress-failover} These keys control the per-`Execute()` reconnect loop on the QWP query diff --git a/documentation/connect/wire-protocols/qwp-client-behavior.md b/documentation/connect/wire-protocols/qwp-client-behavior.md index f4d27e3bba..9c7f5e24f3 100644 --- a/documentation/connect/wire-protocols/qwp-client-behavior.md +++ b/documentation/connect/wire-protocols/qwp-client-behavior.md @@ -348,8 +348,8 @@ With `initial_connect_retry=async`: A sender in async mode does not give up because time passed. Before its first successful connection, authentication rejection (after any token-provider retry) -or a durable-ack capability mismatch is terminal. Established Java senders -instead retry these rejections; see +or a durable-ack capability mismatch is terminal. After a sender has connected, +recovery depends on the rejection and client build; see [Authentication rejection and credential recovery](#authentication-rejection-and-credential-recovery). The poison-frame detector can still terminate a sender. If the buffer reaches `sf_max_total_bytes`, `append()` blocks and then throws after @@ -392,30 +392,39 @@ and apply back-pressure to the producer rather than dropping data. ### Authentication rejection and credential recovery -The Java reference client separates a failed endpoint walk from the lifetime of -the sender or query. A `401`/`403` stops the current walk rather than continuing -to other hosts with the rejected credential. +A `401`/`403` ends the current endpoint walk rather than continuing to other +hosts with the rejected credential. This does not necessarily end the sender's +lifetime. -With a token provider, a `401` with no Bearer `error` or with -`error="invalid_token"` can first receive one same-endpoint retry, but only if -the provider supplies a different token. A `403` or another Bearer error, such -as `insufficient_scope`, does not trigger this refresh/retry. Any remaining -rejection follows these Java lifecycle rules: +Java and native clients with the token-provider changes below can first retry a +`401` once on the same endpoint when there is no Bearer `error` or when +`error="invalid_token"`, but only if the provider supplies a different token. +A `403` or another Bearer error, such as `insufficient_scope`, does not trigger +this refresh/retry. -| Lifecycle | Authentication rejection behavior | -| --- | --- | -| Sender before its first successful connection | Terminal, including with `initial_connect_retry=async`. | -| Established foreground sender | Retry with backoff indefinitely while retaining buffered rows, even with fixed credentials. Report the outage through connection health and the error handler. | -| Query initial connect or execution-time reconnect | Terminal for that connection attempt or query; no continuation to later endpoints. | -| Orphan drainer with fixed credentials | Quarantine the slot immediately behind a `.failed` file. | -| Orphan drainer with a token provider | Retry under a bounded time/attempt policy, then quarantine if rejections persist. | - -Other language clients still treat authentication rejection as terminal even for -established senders. See [Ingress reconnect](/docs/connect/clients/connect-string/#reconnect-keys) -for client differences, and +Here, **native clients** means Rust, C, and C++ builds containing +[c-questdb-client#183](https://github.com/questdb/c-questdb-client/pull/183), and +Python builds containing +[py-questdb-client#133](https://github.com/questdb/py-questdb-client/pull/133), +which uses that native core. After any eligible token retry: + +| Lifecycle | Java | Native clients | +| --- | --- | --- | +| Sender before its first successful connection | Terminal, including with `initial_connect_retry=async`. | Terminal, including with `initial_connect_retry=async`. | +| Established foreground sender | Retry with backoff indefinitely while retaining buffered rows, even with fixed credentials. Report through connection health and the error handler. | Retry with capped backoff indefinitely while retaining buffered rows, even with fixed credentials. Emit `auth_failed` connection events; an acknowledgement timeout names the rejection. | +| Query initial connect or execution-time reconnect | Terminal for that connection attempt or query; no continuation to later endpoints. | Terminal for that connection attempt or query; no continuation to later endpoints. | +| Orphan drainer with fixed credentials | Quarantine the slot immediately behind a `.failed` file. | Retire the slot immediately for the current session and record `.last_error`. | +| Orphan drainer with a token provider | Retry under a bounded time/attempt policy, then quarantine if rejections persist. | Retry under a bounded time/attempt policy, then retire the slot for the current session and record `.last_error`. No `.failed` sentinel is written; the next session tries the slot again. | + +Older native releases may still treat authentication rejection as terminal for +established senders. See +[Ingress reconnect](/docs/connect/clients/connect-string/#reconnect-keys) +for the native drainer policy, and [How the client handles token failures](/docs/connect/clients/java/#how-the-client-handles-token-failures) for Java refresh limits, orphan thresholds, and reporting. The established -sender's reconnect loop is independent of the one-time token refresh. +sender's reconnect loop is independent of the one-time token refresh. Retrying +a server's `401`/`403` does not make a closed or misconfigured native token +provider recoverable; those provider failures remain terminal. ### Ingest endpoint walk (`addr=a:9000,b:9000,...`) @@ -426,7 +435,7 @@ sender's reconnect loop is independent of the one-time token refresh. | TLS session/certificate failure | transport error; try next endpoint | | HTTP upgrade timeout / non-auth transport error | try next endpoint | | `421` with `X-QuestDB-Role: REPLICA` | role reject; try next endpoint | -| `401` / `403` auth failure | End the current walk after any token-provider retry; Java recovery depends on [lifecycle](#authentication-rejection-and-credential-recovery). | +| `401` / `403` auth failure | End the current walk after any token-provider retry; subsequent recovery depends on [client build and lifecycle](#authentication-rejection-and-credential-recovery). | | durable-ack requested but unsupported | Terminal at startup; established Java foreground senders retry, while orphan drainers use a bounded capability-gap budget. | | successful write upgrade | bind this endpoint | | all endpoints fail transport | throw / retry per initial/reconnect mode | diff --git a/documentation/connect/wire-protocols/qwp-ingress-websocket.md b/documentation/connect/wire-protocols/qwp-ingress-websocket.md index 57db1ec84d..af8e5e9956 100644 --- a/documentation/connect/wire-protocols/qwp-ingress-websocket.md +++ b/documentation/connect/wire-protocols/qwp-ingress-websocket.md @@ -1165,10 +1165,17 @@ Key behaviors: rejection is terminal before the sender's first successful connection. Once a foreground Java sender has connected, it instead keeps buffering and retrying rejected credentials with backoff, even with a fixed - credential. Orphan drainers have a separate bounded policy. See - [How the client handles token failures](/docs/connect/clients/java/#how-the-client-handles-token-failures). - Other clients treat `401`/`403` rejections as terminal, including during - reconnect. + credential. Rust, C, and C++ builds containing + [c-questdb-client#183](https://github.com/questdb/c-questdb-client/pull/183), + and Python builds containing + [py-questdb-client#133](https://github.com/questdb/py-questdb-client/pull/133), + also retry server upgrade `401`/`403` indefinitely after the first successful + connection, retaining buffered rows. Initial authentication rejection remains + terminal after any eligible token retry. Older native releases may still + treat reconnect authentication rejection as terminal. Orphan drainers have + separate bounded policies, including different Java and native slot-retirement + behavior. See + [Authentication rejection and credential recovery](/docs/connect/wire-protocols/qwp-client-behavior/#authentication-rejection-and-credential-recovery). - **`421 + X-QuestDB-Role`** is a role reject: transient if the role is `PRIMARY_CATCHUP`, topology-level otherwise. - **Transport errors are transient**, including TCP/TLS failures and diff --git a/documentation/high-availability/client-failover/concepts.md b/documentation/high-availability/client-failover/concepts.md index 666e709825..c8ad927b88 100644 --- a/documentation/high-availability/client-failover/concepts.md +++ b/documentation/high-availability/client-failover/concepts.md @@ -187,7 +187,7 @@ will not help. | Condition | Why terminal | |---|---| -| HTTP `401` / `403` during initial connect or query reconnect, after any token-provider retry | The rejected credential stops the endpoint walk. Established Java senders and orphan drainers have [different recovery policies](#authentication-is-cluster-wide). | +| HTTP `401` / `403` during initial connect or query reconnect, after any token-provider retry | The rejected credential stops the endpoint walk. Established senders and orphan drainers have [client-specific recovery policies](#authentication-is-cluster-wide). | | Server-status reject (SF) | Application-layer reject; replay reproduces the same response. | ### Topology — handled inside the round @@ -240,20 +240,35 @@ retries later depends on the client and lifecycle: - With a token provider, Java can first retry the same host once when a refreshable `401` produces a different token. A `403` or a Bearer error other than `invalid_token` does not trigger that refresh/retry. -- Before a Java sender's first successful connection, or when a query client +- Before a sender's first successful connection, or when a query client connects or reconnects, any remaining authentication rejection is terminal. - An established foreground Java sender instead keeps buffering rows and retrying with backoff, even with a fixed credential. It reports the outage - through connection health and the error handler; applications that need a - hard failure must monitor these and close the sender themselves. + through connection health and the error handler. +- Rust, C, and C++ builds containing + [c-questdb-client#183](https://github.com/questdb/c-questdb-client/pull/183), + and Python builds containing + [py-questdb-client#133](https://github.com/questdb/py-questdb-client/pull/133), + also retry server upgrade `401`/`403` indefinitely on established senders, + retaining buffered rows. They emit `auth_failed` connection events, and + acknowledgement timeouts identify the rejection. A closed or misconfigured + native token provider remains terminal. Older native releases may still + treat server authentication rejection as terminal after the first connection. +- Applications that need a hard failure must monitor the established sender's + outage and close it themselves; an acknowledgement timeout does not stop its + reconnect loop. - A Java orphan drainer fails immediately on a rejected fixed credential. With a token provider, it retries under a bounded policy before quarantining the slot behind a `.failed` file; the rows stay on disk. -- Other clients treat authentication rejection as terminal even after the - first connection. - -See [How the client handles token failures](/docs/connect/clients/java/#how-the-client-handles-token-failures) -for the refresh rules, drainer thresholds, and reporting. +- Native orphan drainers in the builds above also stop immediately on a + rejected fixed credential and retry rotating credentials under a bounded + policy. They record `.last_error` and retire the slot for the current session + without writing a `.failed` sentinel; the next session tries it again. + +See [Ingress reconnect](/docs/connect/clients/connect-string/#reconnect-keys) +for native drainer thresholds, and +[How the client handles token failures](/docs/connect/clients/java/#how-the-client-handles-token-failures) +for Java refresh rules, drainer thresholds, and reporting. Per-host credentials are outside the failover model. Split such workloads into one connect string per credential. diff --git a/documentation/security/oidc/device-flow.mdx b/documentation/security/oidc/device-flow.mdx index 432a792a57..8d37de44c6 100644 --- a/documentation/security/oidc/device-flow.mdx +++ b/documentation/security/oidc/device-flow.mdx @@ -56,7 +56,7 @@ older package that supports only static tokens. In Rust, also enable the ### Discover the identity provider -The helpers discover the client ID, scope, token endpoint, audience, and +The helpers discover the client ID, scope, token endpoint, and `groups.encoded.in.token` setting from the QuestDB server's public `/settings` endpoint. QuestDB advertises the device authorization endpoint when the provider's discovery document contains `device_authorization_endpoint`. With @@ -78,10 +78,23 @@ explicitly. The helper can then discover it from the issuer's | C | `questdb_oidc_builder_issuer(builder, issuer, issuer_len, &error)` before building the auth object | Use an issuer obtained from your administrator, not one inferred from an -untrusted server. An issuer override pins the endpoints to the issuer's origin; -settings-sourced endpoints must also satisfy the client's issuer-path checks. -If the provider intentionally hosts endpoints elsewhere, configure its trusted -endpoints explicitly instead of weakening TLS verification. +untrusted server. The issuer override is a trust pin, not a guarantee that all +credential requests stay on the issuer's origin: + +- Endpoints read from QuestDB's `/settings` must be on the issuer's origin and + under its path. This prevents a different tenant on the same provider host + from receiving credentials. +- Endpoints obtained from the pinned issuer's own HTTPS discovery document can + be outside that origin or path, provided the document's `issuer` matches the + requested issuer. The provider's document is authoritative for those endpoints. +- Python, Rust, C, and C++ also accept a settings-sourced endpoint outside the + issuer's origin or path when the issuer's discovery document confirms that + endpoint. Java still applies the origin/path checks to settings-sourced + endpoints, even if the provider's document would confirm them. + +The device authorization and token endpoints must still share an origin. If +Java rejects a valid settings-sourced endpoint, configure trusted endpoints +explicitly with `OidcDeviceAuth.builder()` instead of weakening TLS verification. For Microsoft Entra ID, complete the [Web Console and group setup](/docs/security/oidc/entra-id/#web-console-sign-in), @@ -92,6 +105,27 @@ token, not a Microsoft Graph access token. This is a user login, not the app-only role setup for managed identities. See [Microsoft's device authorization documentation](https://learn.microsoft.com/en-us/entra/identity-platform/v2-oauth2-device-code). +### Configure an audience + +QuestDB's `/settings` does not advertise +[`acl.oidc.audience`](/docs/configuration/oidc/#acloidcaudience). If the provider +requires an audience request parameter to issue the token QuestDB expects, +configure it explicitly in the auth helper: + +| Client | Audience configuration | +| --- | --- | +| Java | `.audience("questdb")` on `OidcDeviceAuth.builder()` | +| Python | `audience="questdb"` in `OidcDeviceAuth.from_questdb(...)` | +| Rust | `.audience("questdb")` on the OIDC builder | +| C++ | `.audience("questdb")` on the OIDC builder | +| C | `questdb_oidc_builder_audience(builder, audience, audience_len, &error)` before building | + +Replace `questdb` with the provider's required audience. Java's discovery factory +has no audience override. When using its explicit builder, also supply trusted +`.clientId(...)`, `.deviceAuthorizationEndpoint(...)`, and `.tokenEndpoint(...)` +values, and set `.scope(...)` and `.groupsInToken(...)` to match the server. +Configure `.tlsConfig(...)` as needed for private CA trust. + ### Private CA trust If QuestDB or the provider uses a private certificate authority, configure @@ -427,12 +461,6 @@ Keep **interactive sign-in** separate from **token retrieval**: Handle any separate transport-level failure before resuming ingestion; an authentication helper does not undo a sender's terminal state. -Silent refresh performs network I/O and can block token retrieval, a flush, -or connection setup. The auth helper's HTTP timeout and waits for another -thread's refresh are separate from the transport's request and reconnect -budgets. Configure those provider waits for the amount of backpressure the -application can tolerate; non-interactive does not mean non-blocking. - Do not put `auth.token()` or `auth.signIn()` into a `token=` string and expect renewal. That copies one token: future connections keep using it after expiry. Use the provider integrations above. Device-flow refresh is demand-driven; it @@ -449,6 +477,37 @@ resolves the provider for each new pooled connection, while the latter captures the token when connecting. Use certificate and hostname verification as in [OIDC for PGWire](/docs/security/oidc/#oidc-for-the-pgwire-endpoint). +### Provider timeouts + +Silent refresh performs network I/O and can block token retrieval, a flush, +or connection setup. The auth helper's HTTP timeout and waits for another +thread's refresh are separate from the transport's request and reconnect +budgets. Configure the helper's timeout for the amount of backpressure the +application can tolerate; non-interactive does not mean non-blocking. + +| Client | HTTP timeout configuration | Units | Default | +| --- | --- | --- | --- | +| Java | `.httpTimeoutMillis(30000)` on `OidcDeviceAuth.builder()` | Milliseconds | `30000` | +| Python | `timeout=30` in `OidcDeviceAuth.from_questdb(...)` | Seconds | `30` | +| Rust | `.timeout(std::time::Duration::from_secs(30))` on the OIDC builder | `Duration` | 30 seconds | +| C++ | `.timeout_ms(30000)` on the OIDC builder | Milliseconds | `30000` | +| C | `questdb_oidc_builder_timeout_ms(builder, 30000, &error)` before building | Milliseconds | `30000` | + +Timeouts must be positive and no greater than 120 seconds. They bound individual +HTTP requests or phases, not the entire interactive sign-in, which can last +until the device code expires. Java's `fromQuestDB(...)` factory uses the +30-second default and `DiscoveryOptions` has no timeout setter; use an explicit +builder with the trusted configuration described under +[Configure an audience](#configure-an-audience) to change it. + +Waiting behind another thread's silent refresh on the same auth object can take +up to **six times the HTTP timeout**, or **180 seconds by default**. This wait +budget is derived from the timeout; it is not an independent setting. Token-store +coordination and the caller's own refresh can add further waiting, so this is +not a total token-acquisition or flush deadline. Transport request and reconnect +budgets do not cap these provider waits. Size backpressure against the longer +wait, not just the per-request timeout. + ## Persisting tokens and signing out Tokens stay in memory by default. To resume a login after a process restart, diff --git a/documentation/sidebars.js b/documentation/sidebars.js index b172a89ae8..9782e82a45 100644 --- a/documentation/sidebars.js +++ b/documentation/sidebars.js @@ -740,7 +740,7 @@ module.exports = { items: [ { id: "security/oidc/index", - type: "doc", + type: "ref", label: "Overview", }, { From 0208d130c79cecef86e9f38a5a52bee5b0921568 Mon Sep 17 00:00:00 2001 From: glasstiger Date: Fri, 16 Oct 2026 13:45:24 +0100 Subject: [PATCH 29/29] docs(oidc): restore LLM indexing and align native recovery guidance --- documentation/connect/clients/python.md | 26 +++++++++++++---- documentation/connect/clients/rust.md | 18 ++++++++++-- scripts/generate-llms-files.js | 16 +++++++---- scripts/generate-llms-files.test.cjs | 38 +++++++++++++++++++++++++ 4 files changed, 83 insertions(+), 15 deletions(-) create mode 100644 scripts/generate-llms-files.test.cjs diff --git a/documentation/connect/clients/python.md b/documentation/connect/clients/python.md index 05a860a113..b0e40bf8f5 100644 --- a/documentation/connect/clients/python.md +++ b/documentation/connect/clients/python.md @@ -791,12 +791,26 @@ uses these keys: | `reconnect_initial_backoff_millis` | `100` | | `reconnect_max_backoff_millis` | `5000` | -Authentication failures, protocol-version failures, and terminal data -rejections are not retried. When a `flush(wait=True)` or `wait()` makes no -progress within the budget it raises `QuestDBErrorCode.FailoverRetry`: the -rows remain queued and background delivery continues, so retry the wait or -extend the budget — do not rebuild and flush the same rows, which would -duplicate them once the original frames land. +Protocol-version failures and terminal data rejections are not retried. +Authentication rejection is terminal before a sender's first successful +connection and for query initial connects and reconnects, after any eligible +token-provider refresh and retry. + +In builds containing +[py-questdb-client#133](https://github.com/questdb/py-questdb-client/pull/133), +established senders instead retry server upgrade `401`/`403` rejections +indefinitely with capped backoff, retaining buffered rows even with fixed +credentials. They emit `auth_failed` connection events. Applications requiring +a hard stop must monitor the outage and [close the pool](#closing). Older +builds may still treat these rejections as terminal. A closed or misconfigured +token provider remains terminal. See +[Authentication rejection and credential recovery](/docs/connect/wire-protocols/qwp-client-behavior/#authentication-rejection-and-credential-recovery). + +When a `flush(wait=True)` or `wait()` makes no progress within the budget it +raises `QuestDBErrorCode.FailoverRetry`: the rows remain queued and background +delivery continues, so retry the wait or extend the budget. Do not rebuild +and flush the same rows, which would duplicate them once the original frames +land. To observe connection state, pass a listener when connecting. It receives one `ConnectionEvent` per state transition — `kind` diff --git a/documentation/connect/clients/rust.md b/documentation/connect/clients/rust.md index 914d703fca..95a8e23c19 100644 --- a/documentation/connect/clients/rust.md +++ b/documentation/connect/clients/rust.md @@ -924,9 +924,21 @@ ingestion retry budget uses these keys: | `reconnect_initial_backoff_millis` | `100` | | `reconnect_max_backoff_millis` | `5000` | -Authentication failures, protocol-version failures, and terminal data -rejections are not retried. A terminal writer is retired when its borrow is -dropped; there is no pooled `must_close()` or async-error polling API. +Protocol-version failures and terminal data rejections are not retried. A +terminal writer is retired when its borrow is dropped; there is no pooled +`must_close()` or async-error polling API. Authentication rejection is terminal +before a sender's first successful connection and for query initial connects +and reconnects, after any eligible token-provider refresh and retry. + +In builds containing +[c-questdb-client#183](https://github.com/questdb/c-questdb-client/pull/183), +established senders instead retry server upgrade `401`/`403` rejections +indefinitely with capped backoff, retaining buffered rows even with fixed +credentials. They emit `auth_failed` connection events. Applications requiring +a hard stop must monitor the outage and [close the pool](#closing). Older +builds may still treat these rejections as terminal. A closed or misconfigured +token provider remains terminal. See +[Authentication rejection and credential recovery](/docs/connect/wire-protocols/qwp-client-behavior/#authentication-rejection-and-credential-recovery). Reader failover has a separate policy: diff --git a/scripts/generate-llms-files.js b/scripts/generate-llms-files.js index a1968ba778..17f531129c 100644 --- a/scripts/generate-llms-files.js +++ b/scripts/generate-llms-files.js @@ -109,7 +109,7 @@ function processForLlmsTxt(items, indent = 0, isTopLevel = false) { } result += '\n' - } else if (item.type === 'doc') { + } else if (item.type === 'doc' || item.type === 'ref') { const docId = item.id const docPath = path.join('./documentation', docId + '.md') const { title, description } = extractFrontmatter(docPath) @@ -171,8 +171,12 @@ function generateLlmsFiles() { console.log(` - Size: ${(llmsOutput.length / 1024).toFixed(2)} KB`) } -try { - generateLlmsFiles() -} catch (error) { - console.error('Error generating llms files:', error) -} \ No newline at end of file +if (require.main === module) { + try { + generateLlmsFiles() + } catch (error) { + console.error('Error generating llms files:', error) + } +} + +module.exports = { processForLlmsTxt } diff --git a/scripts/generate-llms-files.test.cjs b/scripts/generate-llms-files.test.cjs new file mode 100644 index 0000000000..8b21da40ea --- /dev/null +++ b/scripts/generate-llms-files.test.cjs @@ -0,0 +1,38 @@ +const assert = require("node:assert/strict") +const test = require("node:test") +const { processForLlmsTxt } = require("./generate-llms-files") +const { docs } = require("../documentation/sidebars") + +const overviewId = "security/oidc/index" +const overviewUrl = "https://questdb.com/docs/security/oidc.md" + +// A reference has the same metadata and canonical URL as a normal doc entry. +test("doc references preserve labels, descriptions, and custom slugs", () => { + const entry = { id: overviewId, label: "Overview" } + const output = processForLlmsTxt([{ ...entry, type: "ref" }]) + + assert.equal(output, processForLlmsTxt([{ ...entry, type: "doc" }])) + assert.ok(output.startsWith(`- [Overview](${overviewUrl}): `)) + assert.ok(output.includes("Configure OIDC in QuestDB Enterprise:")) +}) + +test("unlabelled references use the document title", () => { + const output = processForLlmsTxt([{ type: "ref", id: overviewId }]) + + assert.ok( + output.startsWith( + `- [OpenID Connect (OIDC) Integration](${overviewUrl}): `, + ), + ) +}) + +test("the real sidebar includes the canonical OIDC overview exactly once", () => { + const output = processForLlmsTxt(docs, 0, true) + + assert.equal(output.split(overviewUrl).length - 1, 1) + assert.ok(output.includes(` - [Overview](${overviewUrl}): Configure OIDC`)) + for (const guide of ["device-flow", "entra-id", "pingfederate"]) { + const url = `https://questdb.com/docs/security/oidc/${guide}.md` + assert.equal(output.split(url).length - 1, 1, guide) + } +})