From 2e6e8d3e0bb74a425bbc4695c2bc92f56c2aec43 Mon Sep 17 00:00:00 2001 From: xmagda03 Date: Tue, 28 Jul 2026 16:05:55 +0200 Subject: [PATCH 1/5] API: Add v2 endpoints with query-parameter EIDs - Add v2_router with EID passed as query parameter instead of path - Fixes 404 errors for EIDs containing '/' (e.g., IPv6 CIDR notation) - New endpoints under /entity/v2/ prefix - Add tests in tests/test_api/test_v2_api.py - Update docs/api.md with v2 API documentation - v1 API remains unchanged and functional --- docs/api.md | 263 +++++++++++++++++++++++++++++ dp3/api/main.py | 1 + dp3/api/routers/entity.py | 309 ++++++++++++++++++++++++++++++++++ tests/test_api/test_v2_api.py | 125 ++++++++++++++ 4 files changed, 698 insertions(+) create mode 100644 tests/test_api/test_v2_api.py diff --git a/docs/api.md b/docs/api.md index e09ef1f6..72ff2a46 100644 --- a/docs/api.md +++ b/docs/api.md @@ -9,6 +9,8 @@ For routine same-host reads and writes, prefer `dp3 sh` or the generated `/get`](#get-entities): get current snapshots of entities of entity type @@ -22,6 +24,27 @@ There are several API endpoints: - [`GET /entity//_/distinct/`](#get-distinct-values): get distinct attribute values and their counts based on latest snapshots - [`DELETE /entity//`](#delete-eid-data): delete entity data for given id - [`POST /entity///ttl`](#extend-ttls): extend TTLs of the specified entity + +**Note:** The v1 API has a limitation where Entity IDs (EIDs) containing `/` characters (e.g., IPv6 CIDR notation like `2001:db8:f00::/64`) cannot be accessed via the REST API because `/` is interpreted as a path separator. For such EIDs, use the v2 API below. + +### v2 API (Query-parameter EIDs) + +The v2 API addresses the limitation of EIDs containing special characters by passing the EID as a query parameter instead of in the URL path. + +- [`GET /entity/v2//get`](#v2-get-entities): list entities (same as v1) +- [`GET /entity/v2//count`](#v2-count-entities): count entities (same as v1) +- [`GET /entity/v2//raw/get`](#v2-get-raw-datapoints): get raw datapoints (same as v1) +- [`GET /entity/v2//_/distinct/`](#v2-get-distinct-values): get distinct values (same as v1) +- [`GET /entity/v2//`](#v2-get-eid-data): get entity data with `?eid=X` +- [`GET /entity/v2//master`](#v2-get-master-record): get master record with `?eid=X` +- [`GET /entity/v2//snapshots`](#v2-get-snapshots): get snapshots with `?eid=X` +- [`GET /entity/v2//attr`](#v2-get-attr-value): get attribute with `?eid=X&attr=name` +- [`POST /entity/v2//attr`](#v2-set-attr-value): set attribute with `?eid=X&attr=name` +- [`POST /entity/v2//ttl`](#v2-extend-ttls): extend TTL with `?eid=X` +- [`DELETE /entity/v2//`](#v2-delete-eid-data): delete entity with `?eid=X` + +### Other Endpoints + - [`GET /entities`](#entities): list entity configuration - [`GET /control/`](#control): send a pre-defined action into execution queue. - [`GET /telemetry/sources_validity`](#source-validity): get timestamps of latest data from each source @@ -760,3 +783,243 @@ Only queues belonging to the current DP³ application are returned. ] } ``` + +--- + +## v2 API: Query-Parameter EIDs + +The v2 API provides an alternative way to access entities where the Entity ID (EID) is passed as a query parameter instead of in the URL path. This allows EIDs containing special characters like `/` (e.g., IPv6 CIDR notation `2001:db8:f00::/64`). + +### Why v2 API? + +In the v1 API, EIDs are part of the URL path: +``` +GET /entity/ipv6_64prefix/2001:db8:f00::/64/snapshots +``` + +This fails because `/64` is interpreted as a new path segment, causing a 404 error. + +In the v2 API, the EID is a query parameter: +``` +GET /entity/v2/ipv6_64prefix/snapshots?eid=2001:db8:f00::/64 +``` + +This works correctly because the entire EID value is passed as a query string. + +--- + +### v2 Get Entities + +Same as v1 `/entity/{etype}/get`. See v1 documentation above. + +### v2 Count Entities + +Same as v1 `/entity/{etype}/count`. See v1 documentation above. + +### v2 Get Raw Datapoints + +Same as v1 `/entity/{etype}/raw/get`. See v1 documentation above. + +### v2 Get Distinct Values + +Same as v1 `/entity/{etype}/_/distinct/{attr}`. See v1 documentation above. + +--- + +### v2 Get EID Data + +Get data of an entity identified by `etype` and `eid`. + +#### Request + +`GET /entity/v2/{etype}/?eid=` + +#### Query Parameters + +| Parameter | Description | Required | +|-----------|-------------|----------| +| `eid` | Entity ID (can contain special characters like `/`) | Yes | +| `date_from` | Start date for filtering (ISO 8601) | No | +| `date_to` | End date for filtering (ISO 8601) | No | + +#### Example + +```bash +# IPv6 CIDR notation EID +curl "http://localhost:8000/entity/v2/ipv6_64prefix/?eid=2001:db8:f00::/64" + +# Path-like EID +curl "http://localhost:8000/entity/v2/user/?eid=admin/root" +``` + +#### Response + +**`200 OK`**: Same structure as v1 `/entity/{etype}/{eid}` response. + +--- + +### v2 Get Master Record + +Get the master record of an entity. + +#### Request + +`GET /entity/v2/{etype}/master?eid=` + +#### Query Parameters + +| Parameter | Description | Required | +|-----------|-------------|----------| +| `eid` | Entity ID | Yes | +| `date_from` | Start date for filtering | No | +| `date_to` | End date for filtering | No | + +#### Example + +```bash +curl "http://localhost:8000/entity/v2/ipv6_64prefix/master?eid=2001:db8:f00::/64" +``` + +--- + +### v2 Get Snapshots + +Get snapshot history of an entity. + +#### Request + +`GET /entity/v2/{etype}/snapshots?eid=` + +#### Query Parameters + +| Parameter | Description | Required | +|-----------|-------------|----------| +| `eid` | Entity ID | Yes | +| `date_from` | Start date for filtering | No | +| `date_to` | End date for filtering | No | +| `skip` | Number of snapshots to skip (pagination) | No, default=0 | +| `limit` | Maximum snapshots to return (0=no limit) | No, default=0 | + +#### Example + +```bash +curl "http://localhost:8000/entity/v2/ipv6_64prefix/snapshots?eid=2001:db8:f00::/64&limit=10" +``` + +--- + +### v2 Get Attribute Value + +Get attribute value for an entity. + +#### Request + +`GET /entity/v2/{etype}/attr?eid=&attr=` + +#### Query Parameters + +| Parameter | Description | Required | +|-----------|-------------|----------| +| `eid` | Entity ID | Yes | +| `attr` | Attribute name | Yes | +| `date_from` | Start date for history | No | +| `date_to` | End date for history | No | + +#### Example + +```bash +curl "http://localhost:8000/entity/v2/ipv6_64prefix/attr?eid=2001:db8:f00::/64&attr=network" +``` + +--- + +### v2 Set Attribute Value + +Set attribute value for an entity. + +#### Request + +`POST /entity/v2/{etype}/attr?eid=&attr=` + +#### Query Parameters + +| Parameter | Description | Required | +|-----------|-------------|----------| +| `eid` | Entity ID | Yes | +| `attr` | Attribute name | Yes | + +#### Request Body + +```json +{ + "value": "new_value" +} +``` + +#### Example + +```bash +curl -X POST "http://localhost:8000/entity/v2/example/attr?eid=test_id&attr=hostname" \ + -H "Content-Type: application/json" \ + -d '{"value": "new_hostname"}' +``` + +--- + +### v2 Extend TTLs + +Extend TTLs of an entity. + +#### Request + +`POST /entity/v2/{etype}/ttl?eid=` + +#### Query Parameters + +| Parameter | Description | Required | +|-----------|-------------|----------| +| `eid` | Entity ID | Yes | + +#### Request Body + +```json +{ + "default": "2025-12-31T23:59:59Z" +} +``` + +#### Example + +```bash +curl -X POST "http://localhost:8000/entity/v2/example/ttl?eid=test_id" \ + -H "Content-Type: application/json" \ + -d '{"default": "2025-12-31T23:59:59Z"}' +``` + +--- + +### v2 Delete Entity Data + +Delete the master record and snapshots of an entity. + +#### Request + +`DELETE /entity/v2/{etype}/?eid=` + +#### Query Parameters + +| Parameter | Description | Required | +|-----------|-------------|----------| +| `eid` | Entity ID | Yes | + +#### Example + +```bash +curl -X DELETE "http://localhost:8000/entity/v2/example/?eid=test_id" +``` + +--- + +### v2 API Testing + +See [`tests/test_api/test_v2_api.py`](../tests/test_api/test_v2_api.py) for comprehensive test examples. diff --git a/dp3/api/main.py b/dp3/api/main.py index 3eb1fda7..441c41a8 100644 --- a/dp3/api/main.py +++ b/dp3/api/main.py @@ -49,6 +49,7 @@ async def validation_exception_handler(request, exc): # Register routers app.include_router(entity.router, prefix="/entity", tags=["Entity"]) +app.include_router(entity.v2_router, prefix="/entity/v2", tags=["Entity v2"]) app.include_router(control.router, prefix="/control", tags=["Control"]) app.include_router(telemetry.router, prefix="/telemetry", tags=["Telemetry"]) app.include_router(root.router) diff --git a/dp3/api/routers/entity.py b/dp3/api/routers/entity.py index 3b2ed28e..c1e0b6bb 100644 --- a/dp3/api/routers/entity.py +++ b/dp3/api/routers/entity.py @@ -46,6 +46,14 @@ async def parse_eid(etype: str, eid: str) -> EntityId: ParsedEid = Annotated[EntityId, Depends(parse_eid)] +async def parse_eid_from_query(etype: str, eid: str) -> EntityId: + """Parse EID from query parameter (for v2 API).""" + try: + return cast(EntityId, EntityIdAdapter.validate_python({"etype": etype, "eid": eid})) + except ValidationError as e: + raise RequestValidationError(["query", "eid"], e.errors()[0]["msg"]) from e + + def _parse_optional_eid(etype: str, eid: str | None) -> Any: """Parse optional entity id query parameter for entity-scoped endpoints.""" if eid is None: @@ -570,3 +578,304 @@ async def delete_eid_record(e: ParsedEid) -> SuccessResponse: TASK_WRITER.put_task(task, False) return SuccessResponse() + + +# ============================================================================= +# v2 API Router - EID passed as query parameter +# ============================================================================= + +v2_router = APIRouter(dependencies=[Depends(check_etype)]) + + +@v2_router.get( + "/{etype}/get", + responses={400: {"description": "Query can't be processed", "model": ErrorResponse}}, +) +async def v2_get_entity_type_eids( + etype: str, + fulltext_filters: Json = None, + generic_filter: Json = None, + skip: NonNegativeInt = 0, + limit: NonNegativeInt = 20, + sort: Annotated[ + list[str] | None, + Query(description="example: hostname:-1 for descending sort by hostname"), + ] = None, +) -> EntityEidList: + """List latest snapshots of all `id`s present in database under `etype`. + + Same as v1 `/entity/{etype}/get` endpoint. See v1 documentation for details. + """ + fulltext_filters, generic_filter = _validate_snapshot_filters(fulltext_filters, generic_filter) + sort_criteria = _validate_sort_params(etype, sort) + + try: + cursor = DB.snapshots.find_latest(etype, fulltext_filters, generic_filter) + + if sort_criteria: + sort_spec = [("last." + attr, direction) for attr, direction in sort_criteria] + cursor = cursor.sort(sort_spec) + + cursor_page = cursor.skip(skip).limit(limit) + except DatabaseError as e: + raise HTTPException(status_code=400, detail=str(e)) from e + + time_created = None + result = [r["last"] for r in cursor_page] + for r in result: + time_created = r["_time_created"] + del r["_time_created"] + + return EntityEidList(time_created=time_created, count=len(result), data=result) + + +@v2_router.get( + "/{etype}/count", + responses={400: {"description": "Query can't be processed", "model": ErrorResponse}}, +) +async def v2_count_entity_type_eids( + etype: str, + fulltext_filters: Json = None, + generic_filter: Json = None, +) -> EntityEidCount: + """Count latest snapshots of all `id`s present in database under `etype`. + + Same as v1 `/entity/{etype}/count` endpoint. See v1 documentation for details. + """ + fulltext_filters, generic_filter = _validate_snapshot_filters(fulltext_filters, generic_filter) + + try: + count = DB.snapshots.count_latest(etype, fulltext_filters, generic_filter) + except DatabaseError as e: + raise HTTPException(status_code=400, detail=str(e)) from e + + return EntityEidCount(total_count=count) + + +@v2_router.get( + "/{etype}/raw/get", + responses={400: {"description": "Query can't be processed", "model": ErrorResponse}}, +) +async def v2_get_entity_type_raw_datapoints( + etype: str, + eid: str | None = None, + attr: str | None = None, + src: str | None = None, + skip: NonNegativeInt = 0, + limit: NonNegativeInt = 20, +) -> EntityRawDataPage: + """List raw datapoints from the current raw collection for troubleshooting ingestion. + + Same as v1 `/entity/{etype}/raw/get` endpoint. See v1 documentation for details. + """ + if attr is not None and attr not in MODEL_SPEC.attribs(etype): + raise RequestValidationError(["query", "attr"], f"Attribute '{attr}' doesn't exist") + + parsed_eid = _parse_optional_eid(etype, eid) + + try: + datapoints = list( + DB.find_raw_datapoints( + etype, + eid=parsed_eid, + attr=attr, + src=src, + skip=skip, + limit=limit, + ) + ) + except DatabaseError as e: + raise HTTPException(status_code=400, detail=str(e)) from e + + return EntityRawDataPage( + count=len(datapoints), + data=[_raw_datapoint_to_response(datapoint) for datapoint in datapoints], + ) + + +@v2_router.get( + "/{etype}/_/distinct/{attr}", + responses={400: {"description": "Query can't be processed", "model": ErrorResponse}}, +) +async def v2_get_distinct_attribute_values(etype: str, attr: str) -> dict[JsonVal, int]: + """Gets distinct attribute values and their counts based on latest snapshots. + + Same as v1 `/entity/{etype}/_/distinct/{attr}` endpoint. See v1 documentation for details. + """ + try: + return DB.snapshots.get_distinct_val_count(etype, attr) + except DatabaseError as e: + raise HTTPException(status_code=400, detail=str(e)) from e + + +@v2_router.get("/{etype}/") +async def v2_get_eid_data( + etype: str, + eid: Annotated[str, Query(description="Entity ID")], + date_from: AwareDatetime | None = None, + date_to: AwareDatetime | None = None, +) -> EntityEidData: + """Get data of the entity identified by `etype` and `eid`. + + Contains all snapshots and master record. Snapshots are ordered by ascending creation time. + + **v2 API**: EID is passed as query parameter (`?eid=X`) instead of path. + This allows EIDs containing special characters like `/` (e.g., IPv6 CIDR notation). + """ + e = await parse_eid_from_query(etype, eid) + master_record = get_eid_master_record_handler(e, date_from, date_to) + snapshots = get_eid_snapshots_handler(e, date_from, date_to) + empty = not master_record and len(snapshots) == 0 + return EntityEidData(empty=empty, master_record=master_record, snapshots=snapshots) + + +@v2_router.get("/{etype}/master") +async def v2_get_eid_master_record( + etype: str, + eid: Annotated[str, Query(description="Entity ID")], + date_from: AwareDatetime | None = None, + date_to: AwareDatetime | None = None, +) -> EntityEidMasterRecord: + """Get the master record of the entity identified by `etype` and `eid`. + + **v2 API**: EID is passed as query parameter (`?eid=X`) instead of path. + This allows EIDs containing special characters like `/` (e.g., IPv6 CIDR notation). + """ + e = await parse_eid_from_query(etype, eid) + return get_eid_master_record_handler(e, date_from, date_to) + + +@v2_router.get("/{etype}/snapshots") +async def v2_get_eid_snapshots( + etype: str, + eid: Annotated[str, Query(description="Entity ID")], + date_from: AwareDatetime | None = None, + date_to: AwareDatetime | None = None, + skip: NonNegativeInt = 0, + limit: NonNegativeInt = 0, +) -> EntityEidSnapshots: + """Get snapshots of the entity identified by `etype` and `eid`. + + Supports optional pagination via `skip` and `limit`. + Setting `limit` to `0` returns all matching snapshots. + + **v2 API**: EID is passed as query parameter (`?eid=X`) instead of path. + This allows EIDs containing special characters like `/` (e.g., IPv6 CIDR notation). + """ + e = await parse_eid_from_query(etype, eid) + return get_eid_snapshots_handler(e, date_from, date_to, skip, limit) + + +@v2_router.get("/{etype}/attr") +async def v2_get_eid_attr_value( + etype: str, + eid: Annotated[str, Query(description="Entity ID")], + attr: Annotated[str, Query(description="Attribute name")], + date_from: AwareDatetime | None = None, + date_to: AwareDatetime | None = None, +) -> EntityEidAttrValueOrHistory: + """Get attribute value. + + Value is either of: + - current value: in case of plain attribute + - current value and history: in case of observation attribute + - history: in case of timeseries attribute + + **v2 API**: EID is passed as query parameter (`?eid=X&attr=name`) instead of path. + This allows EIDs containing special characters like `/` (e.g., IPv6 CIDR notation). + """ + # Check if attribute exists + if attr not in MODEL_SPEC.attribs(etype): + raise RequestValidationError(["query", "attr"], f"Attribute '{attr}' doesn't exist") + + e = await parse_eid_from_query(etype, eid) + value_or_history = DB.get_value_or_history(e.type, attr, e.id, t1=date_from, t2=date_to) + + return EntityEidAttrValueOrHistory( + attr_type=MODEL_SPEC.attr(e.type, attr).t, **value_or_history + ) + + +@v2_router.post("/{etype}/attr") +async def v2_set_eid_attr_value( + etype: str, + eid: Annotated[str, Query(description="Entity ID")], + attr: Annotated[str, Query(description="Attribute name")], + request: Request, +) -> SuccessResponse: + """Set current value of attribute. + + Internally just creates datapoint for specified attribute and value. + + This endpoint is meant for `editable` plain attributes -- for direct user edit on DP3 web UI. + + **v2 API**: EID is passed as query parameter (`?eid=X&attr=name`) instead of path. + This allows EIDs containing special characters like `/` (e.g., IPv6 CIDR notation). + """ + # Check if attribute exists + if attr not in MODEL_SPEC.attribs(etype): + raise RequestValidationError(["query", "attr"], f"Attribute '{attr}' doesn't exist") + + try: + body = EntityEidAttrValue.model_validate(await request.json()) + except ValueError as e: + raise RequestValidationError(["body"], str(e)) from e + + # Construct datapoint + try: + dp = DataPoint( + type=etype, + id=eid, + attr=attr, + v=body.value, + t1=datetime.now(UTC), + src=f"{request.client.host} via API", + ) + dp3_dp = api_to_dp3_datapoint(dp.model_dump()) + except ValidationError as e: + raise RequestValidationError(["body", "value"], e.errors()[0]["msg"]) from e + + # This shouldn't fail + with task_context(MODEL_SPEC): + task = DataPointTask(etype=etype, eid=eid, data_points=[dp3_dp]) + + TASK_WRITER.put_task(task, False) + return SuccessResponse() + + +@v2_router.post("/{etype}/ttl") +async def v2_extend_eid_ttls( + etype: str, + eid: Annotated[str, Query(description="Entity ID")], + body: dict[str, AwareDatetime], +) -> SuccessResponse: + """Extend TTLs of the specified entity. + + **v2 API**: EID is passed as query parameter (`?eid=X`) instead of path. + This allows EIDs containing special characters like `/` (e.g., IPv6 CIDR notation). + """ + e = await parse_eid_from_query(etype, eid) + with task_context(MODEL_SPEC): + task = DataPointTask(etype=e.type, eid=e.id, ttl_tokens=body) + TASK_WRITER.put_task(task, False) + return SuccessResponse() + + +@v2_router.delete("/{etype}/") +async def v2_delete_eid_record( + etype: str, + eid: Annotated[str, Query(description="Entity ID")], +) -> SuccessResponse: + """Delete the master record and snapshots of the specified entity. + + Notice that this does not delete any raw datapoints, + or block the re-creation of the entity if new datapoints are received. + + **v2 API**: EID is passed as query parameter (`?eid=X`) instead of path. + This allows EIDs containing special characters like `/` (e.g., IPv6 CIDR notation). + """ + e = await parse_eid_from_query(etype, eid) + with task_context(MODEL_SPEC): + task = DataPointTask(etype=e.type, eid=e.id, delete=True) + TASK_WRITER.put_task(task, False) + return SuccessResponse() diff --git a/tests/test_api/test_v2_api.py b/tests/test_api/test_v2_api.py new file mode 100644 index 00000000..379d3498 --- /dev/null +++ b/tests/test_api/test_v2_api.py @@ -0,0 +1,125 @@ +"""Tests for v2 API endpoints with query-parameter EIDs. + +These tests verify that the v2 API correctly handles EIDs containing special +characters like '/' (e.g., IPv6 CIDR notation: 2001:db8:f00::/64). +""" + +from datetime import UTC + +import pytest +from fastapi.testclient import TestClient + +from dp3.api.main import app + + +@pytest.fixture +def client(): + """Create a test client for the DP3 API.""" + return TestClient(app, raise_server_exceptions=False) + + +class TestV2APIBasicEndpoints: + """Test basic v2 API endpoints that don't require EID in path.""" + + def test_v2_get_entity_type_eids(self, client): + """Test v2 GET /entity/v2/{etype}/get endpoint.""" + # This will return empty data or error if etype doesn't exist, + # but verifies the route is registered and accessible + response = client.get("/entity/v2/example/get") + # Should not be 404 (route not found) + assert response.status_code != 404 + + def test_v2_count_entity_type_eids(self, client): + """Test v2 GET /entity/v2/{etype}/count endpoint.""" + response = client.get("/entity/v2/example/count") + assert response.status_code != 404 + + def test_v2_get_distinct_attribute_values(self, client): + """Test v2 GET /entity/v2/{etype}/_/distinct/{attr} endpoint.""" + response = client.get("/entity/v2/example/_/distinct/some_attr") + assert response.status_code != 404 + + +class TestV2APIWithSpecialCharacterEIDs: + """Test v2 API endpoints with EIDs containing special characters.""" + + def test_v2_snapshots_with_ipv6_cidr(self, client): + """Test v2 snapshots endpoint with IPv6 CIDR notation EID. + + This is the primary use case that motivated the v2 API - EIDs + containing '/' should work when passed as query parameter. + """ + ipv6_cidr = "2001:db8:f00::/64" + response = client.get(f"/entity/v2/ipv6_64prefix/snapshots?eid={ipv6_cidr}") + # Should not be 404 - the route should match + # Response may be 422 (validation) or 400 (no data) or 200 (success) + assert response.status_code != 404 + + def test_v2_master_with_ipv6_cidr(self, client): + """Test v2 master endpoint with IPv6 CIDR notation EID.""" + ipv6_cidr = "2001:db8:f00::/64" + response = client.get(f"/entity/v2/ipv6_64prefix/master?eid={ipv6_cidr}") + assert response.status_code != 404 + + def test_v2_get_with_path_like_eid(self, client): + """Test v2 get endpoint with path-like EID (e.g., 'foo/bar/baz').""" + path_eid = "foo/bar/baz" + response = client.get(f"/entity/v2/some_type/?eid={path_eid}") + assert response.status_code != 404 + + def test_v2_attr_get_with_special_eid(self, client): + """Test v2 attribute get endpoint with special character EID.""" + special_eid = "user/name" + response = client.get(f"/entity/v2/user/attr?eid={special_eid}&attr=hostname") + assert response.status_code != 404 + + +class TestV2APIValidation: + """Test v2 API validation behavior.""" + + def test_v2_missing_eid_param_returns_422(self, client): + """Test that missing EID query parameter returns 422 Unprocessable Entity.""" + response = client.get("/entity/v2/example/master") + assert response.status_code == 422 + + def test_v2_invalid_eid_format_returns_422(self, client): + """Test that invalid EID format returns appropriate error.""" + # Invalid EID depending on entity type's expected format + response = client.get("/entity/v2/example/master?eid=") + # Empty string may be rejected by validation + assert response.status_code in (422, 400) + + def test_v2_nonexistent_etype_returns_422(self, client): + """Test that nonexistent entity type returns 422.""" + response = client.get("/entity/v2/nonexistent_type/master?eid=some_id") + assert response.status_code == 422 + + +class TestV2APIPostEndpoints: + """Test v2 POST endpoints.""" + + def test_v2_set_attr_value(self, client): + """Test v2 set attribute value endpoint.""" + response = client.post( + "/entity/v2/example/attr?eid=test_id&attr=hostname", json={"value": "new_hostname"} + ) + # Should not be 404 - route should match + assert response.status_code != 404 + + def test_v2_extend_ttl(self, client): + """Test v2 extend TTL endpoint.""" + from datetime import datetime, timedelta + + future_time = (datetime.now(UTC) + timedelta(hours=1)).isoformat() + response = client.post("/entity/v2/example/ttl?eid=test_id", json={"default": future_time}) + assert response.status_code != 404 + + +class TestV2APIDeleteEndpoint: + """Test v2 DELETE endpoint.""" + + def test_v2_delete_entity(self, client): + """Test v2 delete entity endpoint.""" + response = client.delete("/entity/v2/example/?eid=test_id") + # Should not be 404 - route should match + assert response.status_code != 404 From 89cdbcd2d5f8074ceeaeb9db4977086e17c3f648 Mon Sep 17 00:00:00 2001 From: Ondrej Sedlacek Date: Tue, 28 Jul 2026 21:29:44 +0100 Subject: [PATCH 2/5] API: register v2 routes before catch-all routes The v1 entity router contains dynamic /{etype}/{eid} routes. While v2 is mounted below /entity/v2, registering v1 first lets those catch-all routes consume requests intended for v2 and interpret "v2" as an entity type. Mount v2 first so its routes take precedence. Also accept entity GET and DELETE requests without a trailing slash, avoiding redirects and supporting both URL forms consistently. --- dp3/api/main.py | 2 +- dp3/api/routers/entity.py | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/dp3/api/main.py b/dp3/api/main.py index 441c41a8..3bc8e672 100644 --- a/dp3/api/main.py +++ b/dp3/api/main.py @@ -48,8 +48,8 @@ async def validation_exception_handler(request, exc): ) # Register routers -app.include_router(entity.router, prefix="/entity", tags=["Entity"]) app.include_router(entity.v2_router, prefix="/entity/v2", tags=["Entity v2"]) +app.include_router(entity.router, prefix="/entity", tags=["Entity"]) app.include_router(control.router, prefix="/control", tags=["Control"]) app.include_router(telemetry.router, prefix="/telemetry", tags=["Telemetry"]) app.include_router(root.router) diff --git a/dp3/api/routers/entity.py b/dp3/api/routers/entity.py index c1e0b6bb..1705a56b 100644 --- a/dp3/api/routers/entity.py +++ b/dp3/api/routers/entity.py @@ -708,6 +708,7 @@ async def v2_get_distinct_attribute_values(etype: str, attr: str) -> dict[JsonVa raise HTTPException(status_code=400, detail=str(e)) from e +@v2_router.get("/{etype}", include_in_schema=False) @v2_router.get("/{etype}/") async def v2_get_eid_data( etype: str, @@ -861,6 +862,7 @@ async def v2_extend_eid_ttls( return SuccessResponse() +@v2_router.delete("/{etype}", include_in_schema=False) @v2_router.delete("/{etype}/") async def v2_delete_eid_record( etype: str, From abe8d90a2730954ec6a950d03dbc5c8949b60a5d Mon Sep 17 00:00:00 2001 From: Ondrej Sedlacek Date: Tue, 28 Jul 2026 21:31:20 +0100 Subject: [PATCH 3/5] Tests: use unittest for v2 API coverage Replace isolated TestClient smoke checks with the repository's unittest API harness so the tests are discovered by the standard API test runner and exercise the running API, worker, queue, and database together. Create an entity whose string EID contains path separators, then verify its read, update, TTL, and delete flows through v2. This proves query parameters preserve the EID end to end instead of merely checking that routes do not return 404. Keep datetime imports at module scope and cover validation plus both trailing-slash forms. --- tests/test_api/test_v2_api.py | 214 +++++++++++++++++----------------- 1 file changed, 106 insertions(+), 108 deletions(-) diff --git a/tests/test_api/test_v2_api.py b/tests/test_api/test_v2_api.py index 379d3498..790bf962 100644 --- a/tests/test_api/test_v2_api.py +++ b/tests/test_api/test_v2_api.py @@ -1,125 +1,123 @@ -"""Tests for v2 API endpoints with query-parameter EIDs. +"""Integration tests for v2 API endpoints with query-parameter EIDs.""" -These tests verify that the v2 API correctly handles EIDs containing special -characters like '/' (e.g., IPv6 CIDR notation: 2001:db8:f00::/64). -""" +from datetime import UTC, datetime, timedelta +from uuid import uuid4 -from datetime import UTC +import common +import requests -import pytest -from fastapi.testclient import TestClient -from dp3.api.main import app +class V2APIIntegration(common.APITest): + """Verify that v2 routes preserve EIDs containing path separators.""" + etype = "test_entity_type" + attr = "test_attr_string" -@pytest.fixture -def client(): - """Create a test client for the DP3 API.""" - return TestClient(app, raise_server_exceptions=False) + @classmethod + def setUpClass(cls) -> None: + super().setUpClass() + cls.eid = f"v2/path/{uuid4()}" + cls.initial_value = "v2 initial value" + response = cls.push_datapoints( + [ + { + "type": cls.etype, + "id": cls.eid, + "attr": cls.attr, + "v": cls.initial_value, + "src": "v2-api-test", + } + ] + ) + if response.status_code != 200: + raise RuntimeError(f"Failed to push v2 API test datapoint: {response.text}") -class TestV2APIBasicEndpoints: - """Test basic v2 API endpoints that don't require EID in path.""" - - def test_v2_get_entity_type_eids(self, client): - """Test v2 GET /entity/v2/{etype}/get endpoint.""" - # This will return empty data or error if etype doesn't exist, - # but verifies the route is registered and accessible - response = client.get("/entity/v2/example/get") - # Should not be 404 (route not found) - assert response.status_code != 404 - - def test_v2_count_entity_type_eids(self, client): - """Test v2 GET /entity/v2/{etype}/count endpoint.""" - response = client.get("/entity/v2/example/count") - assert response.status_code != 404 - - def test_v2_get_distinct_attribute_values(self, client): - """Test v2 GET /entity/v2/{etype}/_/distinct/{attr} endpoint.""" - response = client.get("/entity/v2/example/_/distinct/some_attr") - assert response.status_code != 404 - - -class TestV2APIWithSpecialCharacterEIDs: - """Test v2 API endpoints with EIDs containing special characters.""" - - def test_v2_snapshots_with_ipv6_cidr(self, client): - """Test v2 snapshots endpoint with IPv6 CIDR notation EID. - - This is the primary use case that motivated the v2 API - EIDs - containing '/' should work when passed as query parameter. - """ - ipv6_cidr = "2001:db8:f00::/64" - response = client.get(f"/entity/v2/ipv6_64prefix/snapshots?eid={ipv6_cidr}") - # Should not be 404 - the route should match - # Response may be 422 (validation) or 400 (no data) or 200 (success) - assert response.status_code != 404 - - def test_v2_master_with_ipv6_cidr(self, client): - """Test v2 master endpoint with IPv6 CIDR notation EID.""" - ipv6_cidr = "2001:db8:f00::/64" - response = client.get(f"/entity/v2/ipv6_64prefix/master?eid={ipv6_cidr}") - assert response.status_code != 404 - - def test_v2_get_with_path_like_eid(self, client): - """Test v2 get endpoint with path-like EID (e.g., 'foo/bar/baz').""" - path_eid = "foo/bar/baz" - response = client.get(f"/entity/v2/some_type/?eid={path_eid}") - assert response.status_code != 404 - - def test_v2_attr_get_with_special_eid(self, client): - """Test v2 attribute get endpoint with special character EID.""" - special_eid = "user/name" - response = client.get(f"/entity/v2/user/attr?eid={special_eid}&attr=hostname") - assert response.status_code != 404 - - -class TestV2APIValidation: - """Test v2 API validation behavior.""" - - def test_v2_missing_eid_param_returns_422(self, client): - """Test that missing EID query parameter returns 422 Unprocessable Entity.""" - response = client.get("/entity/v2/example/master") - assert response.status_code == 422 - - def test_v2_invalid_eid_format_returns_422(self, client): - """Test that invalid EID format returns appropriate error.""" - # Invalid EID depending on entity type's expected format - response = client.get("/entity/v2/example/master?eid=") - # Empty string may be rejected by validation - assert response.status_code in (422, 400) - - def test_v2_nonexistent_etype_returns_422(self, client): - """Test that nonexistent entity type returns 422.""" - response = client.get("/entity/v2/nonexistent_type/master?eid=some_id") - assert response.status_code == 422 + @classmethod + def v2_url(cls, suffix: str = "") -> str: + return f"{common.base_url}/entity/v2/{cls.etype}{suffix}" + @classmethod + def get_v2(cls, suffix: str = "", **params) -> requests.Response: + return common.retry_request_on_error( + lambda: requests.get(cls.v2_url(suffix), params=params, timeout=5) + ) -class TestV2APIPostEndpoints: - """Test v2 POST endpoints.""" + @classmethod + def post_v2(cls, suffix: str, json, **params) -> requests.Response: + return common.retry_request_on_error( + lambda: requests.post(cls.v2_url(suffix), params=params, json=json, timeout=5) + ) - def test_v2_set_attr_value(self, client): - """Test v2 set attribute value endpoint.""" - response = client.post( - "/entity/v2/example/attr?eid=test_id&attr=hostname", json={"value": "new_hostname"} + @classmethod + def delete_v2(cls, suffix: str = "", **params) -> requests.Response: + return common.retry_request_on_error( + lambda: requests.delete(cls.v2_url(suffix), params=params, timeout=5) ) - # Should not be 404 - route should match - assert response.status_code != 404 - def test_v2_extend_ttl(self, client): - """Test v2 extend TTL endpoint.""" - from datetime import datetime, timedelta + def test_01_get_entity_data_with_trailing_slash(self): + response = self.query_expected_value( + lambda: self.get_v2("/", eid=self.eid), + lambda result: result.status_code == 200 + and result.json()["master_record"].get(self.attr, {}).get("v") == self.initial_value, + msg="Timed out waiting for the v2 entity data endpoint.", + ) + self.assertEqual(response.json()["master_record"][self.attr]["v"], self.initial_value) + + def test_02_get_entity_data_without_trailing_slash(self): + response = self.get_v2(eid=self.eid) + self.assertEqual(response.status_code, 200, msg=response.text) + self.assertEqual(response.json()["master_record"][self.attr]["v"], self.initial_value) + + def test_03_get_master_record(self): + response = self.get_v2("/master", eid=self.eid) + self.assertEqual(response.status_code, 200, msg=response.text) + self.assertEqual(response.json()[self.attr]["v"], self.initial_value) + + def test_04_get_snapshots(self): + response = self.get_v2("/snapshots", eid=self.eid) + self.assertEqual(response.status_code, 200, msg=response.text) + self.assertIsInstance(response.json(), list) + + def test_05_get_attribute(self): + response = self.get_v2("/attr", eid=self.eid, attr=self.attr) + self.assertEqual(response.status_code, 200, msg=response.text) + self.assertEqual(response.json()["current_value"], self.initial_value) + + def test_06_set_attribute(self): + updated_value = "v2 updated value" + response = self.post_v2("/attr", {"value": updated_value}, eid=self.eid, attr=self.attr) + self.assertEqual(response.status_code, 200, msg=response.text) + + self.query_expected_value( + lambda: self.get_v2("/attr", eid=self.eid, attr=self.attr), + lambda result: result.status_code == 200 + and result.json().get("current_value") == updated_value, + msg="Timed out waiting for the v2 attribute update.", + ) + def test_07_extend_ttl(self): future_time = (datetime.now(UTC) + timedelta(hours=1)).isoformat() - response = client.post("/entity/v2/example/ttl?eid=test_id", json={"default": future_time}) - assert response.status_code != 404 - + response = self.post_v2("/ttl", {"manual": future_time}, eid=self.eid) + self.assertEqual(response.status_code, 200, msg=response.text) + + def test_08_missing_eid_returns_422(self): + response = self.get_v2("/master") + self.assertEqual(response.status_code, 422, msg=response.text) + + def test_09_nonexistent_etype_returns_422(self): + response = common.retry_request_on_error( + lambda: requests.get( + f"{common.base_url}/entity/v2/nonexistent_type/master", + params={"eid": self.eid}, + timeout=5, + ) + ) + self.assertEqual(response.status_code, 422, msg=response.text) -class TestV2APIDeleteEndpoint: - """Test v2 DELETE endpoint.""" + def test_99_delete_entity_with_and_without_trailing_slash(self): + response = self.delete_v2(eid=self.eid) + self.assertEqual(response.status_code, 200, msg=response.text) - def test_v2_delete_entity(self, client): - """Test v2 delete entity endpoint.""" - response = client.delete("/entity/v2/example/?eid=test_id") - # Should not be 404 - route should match - assert response.status_code != 404 + response = self.delete_v2("/", eid=self.eid) + self.assertEqual(response.status_code, 200, msg=response.text) From c56f1a9a3abfdd7e0126ef7e5ec1aa74b90f1d9f Mon Sep 17 00:00:00 2001 From: Ondrej Sedlacek Date: Tue, 28 Jul 2026 21:47:56 +0100 Subject: [PATCH 4/5] API: move v2 entity routes outside v1 namespace Using /entity/v2 leaves the version marker inside the v1 namespace, where it can conflict with dynamic /entity/{etype}/{eid} routes and makes correctness depend on router registration order. Move the endpoints to /v2/entity so the API versions have disjoint prefixes. Update integration tests and documentation to use the new paths. Let curl encode query parameters in the examples to demonstrate safe handling of EIDs that contain URL-significant characters. --- docs/api.md | 77 ++++++++++++++++++++--------------- dp3/api/main.py | 2 +- tests/test_api/test_v2_api.py | 4 +- 3 files changed, 48 insertions(+), 35 deletions(-) diff --git a/docs/api.md b/docs/api.md index 72ff2a46..683abc9e 100644 --- a/docs/api.md +++ b/docs/api.md @@ -31,17 +31,17 @@ There are several API endpoints: The v2 API addresses the limitation of EIDs containing special characters by passing the EID as a query parameter instead of in the URL path. -- [`GET /entity/v2//get`](#v2-get-entities): list entities (same as v1) -- [`GET /entity/v2//count`](#v2-count-entities): count entities (same as v1) -- [`GET /entity/v2//raw/get`](#v2-get-raw-datapoints): get raw datapoints (same as v1) -- [`GET /entity/v2//_/distinct/`](#v2-get-distinct-values): get distinct values (same as v1) -- [`GET /entity/v2//`](#v2-get-eid-data): get entity data with `?eid=X` -- [`GET /entity/v2//master`](#v2-get-master-record): get master record with `?eid=X` -- [`GET /entity/v2//snapshots`](#v2-get-snapshots): get snapshots with `?eid=X` -- [`GET /entity/v2//attr`](#v2-get-attr-value): get attribute with `?eid=X&attr=name` -- [`POST /entity/v2//attr`](#v2-set-attr-value): set attribute with `?eid=X&attr=name` -- [`POST /entity/v2//ttl`](#v2-extend-ttls): extend TTL with `?eid=X` -- [`DELETE /entity/v2//`](#v2-delete-eid-data): delete entity with `?eid=X` +- [`GET /v2/entity//get`](#v2-get-entities): list entities (same as v1) +- [`GET /v2/entity//count`](#v2-count-entities): count entities (same as v1) +- [`GET /v2/entity//raw/get`](#v2-get-raw-datapoints): get raw datapoints (same as v1) +- [`GET /v2/entity//_/distinct/`](#v2-get-distinct-values): get distinct values (same as v1) +- [`GET /v2/entity//`](#v2-get-eid-data): get entity data with `?eid=X` +- [`GET /v2/entity//master`](#v2-get-master-record): get master record with `?eid=X` +- [`GET /v2/entity//snapshots`](#v2-get-snapshots): get snapshots with `?eid=X` +- [`GET /v2/entity//attr`](#v2-get-attr-value): get attribute with `?eid=X&attr=name` +- [`POST /v2/entity//attr`](#v2-set-attr-value): set attribute with `?eid=X&attr=name` +- [`POST /v2/entity//ttl`](#v2-extend-ttls): extend TTL with `?eid=X` +- [`DELETE /v2/entity//`](#v2-delete-eid-data): delete entity with `?eid=X` ### Other Endpoints @@ -801,10 +801,12 @@ This fails because `/64` is interpreted as a new path segment, causing a 404 err In the v2 API, the EID is a query parameter: ``` -GET /entity/v2/ipv6_64prefix/snapshots?eid=2001:db8:f00::/64 +GET /v2/entity/ipv6_64prefix/snapshots?eid=2001:db8:f00::/64 ``` -This works correctly because the entire EID value is passed as a query string. +This works correctly because the entire EID value is passed as a query string. Query +parameter values still need to be URL-encoded as usual; the examples below let `curl` handle +encoding rather than constructing query strings manually. --- @@ -832,7 +834,7 @@ Get data of an entity identified by `etype` and `eid`. #### Request -`GET /entity/v2/{etype}/?eid=` +`GET /v2/entity/{etype}/?eid=` #### Query Parameters @@ -846,10 +848,12 @@ Get data of an entity identified by `etype` and `eid`. ```bash # IPv6 CIDR notation EID -curl "http://localhost:8000/entity/v2/ipv6_64prefix/?eid=2001:db8:f00::/64" +curl --get "http://localhost:8000/v2/entity/ipv6_64prefix/" \ + --data-urlencode "eid=2001:db8:f00::/64" # Path-like EID -curl "http://localhost:8000/entity/v2/user/?eid=admin/root" +curl --get "http://localhost:8000/v2/entity/user/" \ + --data-urlencode "eid=admin/root" ``` #### Response @@ -864,7 +868,7 @@ Get the master record of an entity. #### Request -`GET /entity/v2/{etype}/master?eid=` +`GET /v2/entity/{etype}/master?eid=` #### Query Parameters @@ -877,7 +881,8 @@ Get the master record of an entity. #### Example ```bash -curl "http://localhost:8000/entity/v2/ipv6_64prefix/master?eid=2001:db8:f00::/64" +curl --get "http://localhost:8000/v2/entity/ipv6_64prefix/master" \ + --data-urlencode "eid=2001:db8:f00::/64" ``` --- @@ -888,7 +893,7 @@ Get snapshot history of an entity. #### Request -`GET /entity/v2/{etype}/snapshots?eid=` +`GET /v2/entity/{etype}/snapshots?eid=` #### Query Parameters @@ -903,7 +908,9 @@ Get snapshot history of an entity. #### Example ```bash -curl "http://localhost:8000/entity/v2/ipv6_64prefix/snapshots?eid=2001:db8:f00::/64&limit=10" +curl --get "http://localhost:8000/v2/entity/ipv6_64prefix/snapshots" \ + --data-urlencode "eid=2001:db8:f00::/64" \ + --data-urlencode "limit=10" ``` --- @@ -914,7 +921,7 @@ Get attribute value for an entity. #### Request -`GET /entity/v2/{etype}/attr?eid=&attr=` +`GET /v2/entity/{etype}/attr?eid=&attr=` #### Query Parameters @@ -928,7 +935,9 @@ Get attribute value for an entity. #### Example ```bash -curl "http://localhost:8000/entity/v2/ipv6_64prefix/attr?eid=2001:db8:f00::/64&attr=network" +curl --get "http://localhost:8000/v2/entity/ipv6_64prefix/attr" \ + --data-urlencode "eid=2001:db8:f00::/64" \ + --data-urlencode "attr=network" ``` --- @@ -939,7 +948,7 @@ Set attribute value for an entity. #### Request -`POST /entity/v2/{etype}/attr?eid=&attr=` +`POST /v2/entity/{etype}/attr?eid=&attr=` #### Query Parameters @@ -959,9 +968,11 @@ Set attribute value for an entity. #### Example ```bash -curl -X POST "http://localhost:8000/entity/v2/example/attr?eid=test_id&attr=hostname" \ - -H "Content-Type: application/json" \ - -d '{"value": "new_hostname"}' +curl --request POST "http://localhost:8000/v2/entity/example/attr" \ + --url-query "eid=test_id" \ + --url-query "attr=hostname" \ + --header "Content-Type: application/json" \ + --data '{"value": "new_hostname"}' ``` --- @@ -972,7 +983,7 @@ Extend TTLs of an entity. #### Request -`POST /entity/v2/{etype}/ttl?eid=` +`POST /v2/entity/{etype}/ttl?eid=` #### Query Parameters @@ -991,9 +1002,10 @@ Extend TTLs of an entity. #### Example ```bash -curl -X POST "http://localhost:8000/entity/v2/example/ttl?eid=test_id" \ - -H "Content-Type: application/json" \ - -d '{"default": "2025-12-31T23:59:59Z"}' +curl --request POST "http://localhost:8000/v2/entity/example/ttl" \ + --url-query "eid=test_id" \ + --header "Content-Type: application/json" \ + --data '{"default": "2025-12-31T23:59:59Z"}' ``` --- @@ -1004,7 +1016,7 @@ Delete the master record and snapshots of an entity. #### Request -`DELETE /entity/v2/{etype}/?eid=` +`DELETE /v2/entity/{etype}/?eid=` #### Query Parameters @@ -1015,7 +1027,8 @@ Delete the master record and snapshots of an entity. #### Example ```bash -curl -X DELETE "http://localhost:8000/entity/v2/example/?eid=test_id" +curl --request DELETE "http://localhost:8000/v2/entity/example/" \ + --url-query "eid=test_id" ``` --- diff --git a/dp3/api/main.py b/dp3/api/main.py index 3bc8e672..e8c13228 100644 --- a/dp3/api/main.py +++ b/dp3/api/main.py @@ -48,7 +48,7 @@ async def validation_exception_handler(request, exc): ) # Register routers -app.include_router(entity.v2_router, prefix="/entity/v2", tags=["Entity v2"]) +app.include_router(entity.v2_router, prefix="/v2/entity", tags=["Entity v2"]) app.include_router(entity.router, prefix="/entity", tags=["Entity"]) app.include_router(control.router, prefix="/control", tags=["Control"]) app.include_router(telemetry.router, prefix="/telemetry", tags=["Telemetry"]) diff --git a/tests/test_api/test_v2_api.py b/tests/test_api/test_v2_api.py index 790bf962..22c318e7 100644 --- a/tests/test_api/test_v2_api.py +++ b/tests/test_api/test_v2_api.py @@ -35,7 +35,7 @@ def setUpClass(cls) -> None: @classmethod def v2_url(cls, suffix: str = "") -> str: - return f"{common.base_url}/entity/v2/{cls.etype}{suffix}" + return f"{common.base_url}/v2/entity/{cls.etype}{suffix}" @classmethod def get_v2(cls, suffix: str = "", **params) -> requests.Response: @@ -108,7 +108,7 @@ def test_08_missing_eid_returns_422(self): def test_09_nonexistent_etype_returns_422(self): response = common.retry_request_on_error( lambda: requests.get( - f"{common.base_url}/entity/v2/nonexistent_type/master", + f"{common.base_url}/v2/entity/nonexistent_type/master", params={"eid": self.eid}, timeout=5, ) From bb71a8bca63c1993698bf7ac875bcaef28f4ce89 Mon Sep 17 00:00:00 2001 From: Ondrej Sedlacek Date: Tue, 28 Jul 2026 22:44:34 +0100 Subject: [PATCH 5/5] API: share v1 and v2 entity endpoint behavior V1 and v2 differ only in whether EIDs arrive in the path or query, so delegate both route sets to shared handlers while preserving transport-specific validation. --- dp3/api/routers/entity.py | 472 ++++++++++++-------------------------- 1 file changed, 148 insertions(+), 324 deletions(-) diff --git a/dp3/api/routers/entity.py b/dp3/api/routers/entity.py index 1705a56b..ccddcc27 100644 --- a/dp3/api/routers/entity.py +++ b/dp3/api/routers/entity.py @@ -1,5 +1,6 @@ import re from datetime import UTC, datetime +from inspect import cleandoc from typing import Annotated, Any, cast from fastapi import APIRouter, Depends, HTTPException, Query, Request @@ -29,14 +30,14 @@ async def check_etype(etype: str): - """Middleware to check entity type existence""" + """Validate the entity type as a FastAPI dependency.""" if etype not in MODEL_SPEC.entities: raise RequestValidationError(["path", "etype"], f"Entity type '{etype}' doesn't exist") return etype async def parse_eid(etype: str, eid: str) -> EntityId: - """Middleware to parse EID""" + """Parse a path EID as a FastAPI dependency.""" try: return cast(EntityId, EntityIdAdapter.validate_python({"etype": etype, "eid": eid})) except ValidationError as e: @@ -46,7 +47,9 @@ async def parse_eid(etype: str, eid: str) -> EntityId: ParsedEid = Annotated[EntityId, Depends(parse_eid)] -async def parse_eid_from_query(etype: str, eid: str) -> EntityId: +async def parse_eid_from_query( + etype: str, eid: Annotated[str, Query(description="Entity ID")] +) -> EntityId: """Parse EID from query parameter (for v2 API).""" try: return cast(EntityId, EntityIdAdapter.validate_python({"etype": etype, "eid": eid})) @@ -54,6 +57,9 @@ async def parse_eid_from_query(etype: str, eid: str) -> EntityId: raise RequestValidationError(["query", "eid"], e.errors()[0]["msg"]) from e +ParsedQueryEid = Annotated[EntityId, Depends(parse_eid_from_query)] + + def _parse_optional_eid(etype: str, eid: str | None) -> Any: """Parse optional entity id query parameter for entity-scoped endpoints.""" if eid is None: @@ -120,7 +126,90 @@ def get_eid_snapshots_handler( ) +def get_eid_data_handler( + e: EntityId, + date_from: AwareDatetime | None = None, + date_to: AwareDatetime | None = None, +) -> EntityEidData: + """Handler for getting all data of EID.""" + master_record = get_eid_master_record_handler(e, date_from, date_to) + snapshots = get_eid_snapshots_handler(e, date_from, date_to) + return EntityEidData( + empty=not master_record and len(snapshots) == 0, + master_record=master_record, + snapshots=snapshots, + ) + + +def get_eid_attr_value_handler( + e: EntityId, + attr: str, + attr_location: str, + date_from: AwareDatetime | None = None, + date_to: AwareDatetime | None = None, +) -> EntityEidAttrValueOrHistory: + """Handler for getting an attribute value or history of EID.""" + if attr not in MODEL_SPEC.attribs(e.type): + raise RequestValidationError([attr_location, "attr"], f"Attribute '{attr}' doesn't exist") + + value_or_history = DB.get_value_or_history(e.type, attr, e.id, t1=date_from, t2=date_to) + return EntityEidAttrValueOrHistory( + attr_type=MODEL_SPEC.attr(e.type, attr).t, **value_or_history + ) + + +async def set_eid_attr_value_handler( + e: EntityId, attr: str, attr_location: str, request: Request +) -> SuccessResponse: + """Handler for setting the current value of an EID attribute.""" + if attr not in MODEL_SPEC.attribs(e.type): + raise RequestValidationError([attr_location, "attr"], f"Attribute '{attr}' doesn't exist") + + try: + body = EntityEidAttrValue.model_validate(await request.json()) + except ValueError as exc: + raise RequestValidationError(["body"], str(exc)) from exc + + try: + datapoint = DataPoint( + type=e.type, + id=e.id, + attr=attr, + v=body.value, + t1=datetime.now(UTC), + src=f"{request.client.host} via API", + ) + dp3_datapoint = api_to_dp3_datapoint(datapoint.model_dump()) + except ValidationError as exc: + raise RequestValidationError(["body", "value"], exc.errors()[0]["msg"]) from exc + + with task_context(MODEL_SPEC): + task = DataPointTask(etype=e.type, eid=e.id, data_points=[dp3_datapoint]) + + TASK_WRITER.put_task(task, False) + return SuccessResponse() + + +def extend_eid_ttls_handler(e: EntityId, body: dict[str, AwareDatetime]) -> SuccessResponse: + """Handler for extending TTLs of EID.""" + with task_context(MODEL_SPEC): + task = DataPointTask(etype=e.type, eid=e.id, ttl_tokens=body) + + TASK_WRITER.put_task(task, False) + return SuccessResponse() + + +def delete_eid_record_handler(e: EntityId) -> SuccessResponse: + """Handler for deleting the master record and snapshots of EID.""" + with task_context(MODEL_SPEC): + task = DataPointTask(etype=e.type, eid=e.id, delete=True) + + TASK_WRITER.put_task(task, False) + return SuccessResponse() + + router = APIRouter(dependencies=[Depends(check_etype)]) +v2_router = APIRouter(dependencies=[Depends(check_etype)]) # Compile regex patterns once at module level for reuse @@ -233,6 +322,11 @@ def _validate_sort_params(etype: str, sort: list[str] | None) -> list[tuple[str, "/{etype}/get", responses={400: {"description": "Query can't be processed", "model": ErrorResponse}}, ) +@v2_router.get( + "/{etype}/get", + name="v2_get_entity_type_eids", + responses={400: {"description": "Query can't be processed", "model": ErrorResponse}}, +) async def get_entity_type_eids( etype: str, fulltext_filters: Json = None, @@ -360,6 +454,11 @@ async def get_entity_type_eids( "/{etype}/count", responses={400: {"description": "Query can't be processed", "model": ErrorResponse}}, ) +@v2_router.get( + "/{etype}/count", + name="v2_count_entity_type_eids", + responses={400: {"description": "Query can't be processed", "model": ErrorResponse}}, +) async def count_entity_type_eids( etype: str, fulltext_filters: Json = None, @@ -387,6 +486,11 @@ async def count_entity_type_eids( "/{etype}/raw/get", responses={400: {"description": "Query can't be processed", "model": ErrorResponse}}, ) +@v2_router.get( + "/{etype}/raw/get", + name="v2_get_entity_type_raw_datapoints", + responses={400: {"description": "Query can't be processed", "model": ErrorResponse}}, +) async def get_entity_type_raw_datapoints( etype: str, eid: str | None = None, @@ -430,15 +534,9 @@ async def get_eid_data( Contains all snapshots and master record. Snapshots are ordered by ascending creation time. - Combines function of `/{etype}/{eid}/master` and `/{etype}/{eid}/snapshots`. + Combines the master-record and snapshots endpoints. """ - master_record = get_eid_master_record_handler(e, date_from, date_to) - snapshots = get_eid_snapshots_handler(e, date_from, date_to) - - # Whether this eid contains any data - empty = not master_record and len(snapshots) == 0 - - return EntityEidData(empty=empty, master_record=master_record, snapshots=snapshots) + return get_eid_data_handler(e, date_from, date_to) @router.get("/{etype}/{eid}/master") @@ -479,60 +577,18 @@ async def get_eid_attr_value( - current value and history: in case of observation attribute - history: in case of timeseries attribute """ - # Check if attribute exists - if attr not in MODEL_SPEC.attribs(e.type): - raise RequestValidationError(["path", "attr"], f"Attribute '{attr}' doesn't exist") - - value_or_history = DB.get_value_or_history(e.type, attr, e.id, t1=date_from, t2=date_to) - - return EntityEidAttrValueOrHistory( - attr_type=MODEL_SPEC.attr(e.type, attr).t, **value_or_history - ) + return get_eid_attr_value_handler(e, attr, "path", date_from, date_to) @router.post("/{etype}/{eid}/set/{attr}") -async def set_eid_attr_value(etype: str, eid: str, attr: str, request: Request) -> SuccessResponse: +async def set_eid_attr_value(e: ParsedEid, attr: str, request: Request) -> SuccessResponse: """Set current value of attribute Internally just creates datapoint for specified attribute and value. This endpoint is meant for `editable` plain attributes -- for direct user edit on DP3 web UI. """ - # Check if attribute exists - if attr not in MODEL_SPEC.attribs(etype): - raise RequestValidationError(["path", "attr"], f"Attribute '{attr}' doesn't exist") - - try: - body = EntityEidAttrValue.model_validate(await request.json()) - except ValueError as e: - raise RequestValidationError(["body"], str(e)) from e - - # Construct datapoint - try: - dp = DataPoint( - type=etype, - id=eid, - attr=attr, - v=body.value, - t1=datetime.now(UTC), - src=f"{request.client.host} via API", - ) - dp3_dp = api_to_dp3_datapoint(dp.model_dump()) - except ValidationError as e: - raise RequestValidationError(["body", "value"], e.errors()[0]["msg"]) from e - - # This shouldn't fail - with task_context(MODEL_SPEC): - task = DataPointTask(etype=etype, eid=eid, data_points=[dp3_dp]) - - # Push tasks to task queue - TASK_WRITER.put_task(task, False) - - # Datapoints from this endpoint are intentionally not logged using `DPLogger`. - # If for some reason, in the future, they need to be, just copy code from data ingestion - # endpoint. - - return SuccessResponse() + return await set_eid_attr_value_handler(e, attr, "path", request) @router.get( @@ -555,14 +611,7 @@ async def get_distinct_attribute_values(etype: str, attr: str) -> dict[JsonVal, @router.post("/{etype}/{eid}/ttl") async def extend_eid_ttls(e: ParsedEid, body: dict[str, AwareDatetime]) -> SuccessResponse: """Extend TTLs of the specified entity""" - # Construct task - with task_context(MODEL_SPEC): - task = DataPointTask(etype=e.type, eid=e.id, ttl_tokens=body) - - # Push tasks to task queue - TASK_WRITER.put_task(task, False) - - return SuccessResponse() + return extend_eid_ttls_handler(e, body) @router.delete("/{etype}/{eid}") @@ -572,312 +621,87 @@ async def delete_eid_record(e: ParsedEid) -> SuccessResponse: Notice that this does not delete any raw datapoints, or block the re-creation of the entity if new datapoints are received. """ - # Create a "delete" task and push it to task queue - with task_context(MODEL_SPEC): - task = DataPointTask(etype=e.type, eid=e.id, delete=True) - TASK_WRITER.put_task(task, False) - - return SuccessResponse() - - -# ============================================================================= -# v2 API Router - EID passed as query parameter -# ============================================================================= - -v2_router = APIRouter(dependencies=[Depends(check_etype)]) - - -@v2_router.get( - "/{etype}/get", - responses={400: {"description": "Query can't be processed", "model": ErrorResponse}}, -) -async def v2_get_entity_type_eids( - etype: str, - fulltext_filters: Json = None, - generic_filter: Json = None, - skip: NonNegativeInt = 0, - limit: NonNegativeInt = 20, - sort: Annotated[ - list[str] | None, - Query(description="example: hostname:-1 for descending sort by hostname"), - ] = None, -) -> EntityEidList: - """List latest snapshots of all `id`s present in database under `etype`. - - Same as v1 `/entity/{etype}/get` endpoint. See v1 documentation for details. - """ - fulltext_filters, generic_filter = _validate_snapshot_filters(fulltext_filters, generic_filter) - sort_criteria = _validate_sort_params(etype, sort) - - try: - cursor = DB.snapshots.find_latest(etype, fulltext_filters, generic_filter) - - if sort_criteria: - sort_spec = [("last." + attr, direction) for attr, direction in sort_criteria] - cursor = cursor.sort(sort_spec) - - cursor_page = cursor.skip(skip).limit(limit) - except DatabaseError as e: - raise HTTPException(status_code=400, detail=str(e)) from e - - time_created = None - result = [r["last"] for r in cursor_page] - for r in result: - time_created = r["_time_created"] - del r["_time_created"] - - return EntityEidList(time_created=time_created, count=len(result), data=result) - - -@v2_router.get( - "/{etype}/count", - responses={400: {"description": "Query can't be processed", "model": ErrorResponse}}, -) -async def v2_count_entity_type_eids( - etype: str, - fulltext_filters: Json = None, - generic_filter: Json = None, -) -> EntityEidCount: - """Count latest snapshots of all `id`s present in database under `etype`. - - Same as v1 `/entity/{etype}/count` endpoint. See v1 documentation for details. - """ - fulltext_filters, generic_filter = _validate_snapshot_filters(fulltext_filters, generic_filter) + return delete_eid_record_handler(e) - try: - count = DB.snapshots.count_latest(etype, fulltext_filters, generic_filter) - except DatabaseError as e: - raise HTTPException(status_code=400, detail=str(e)) from e - return EntityEidCount(total_count=count) - - -@v2_router.get( - "/{etype}/raw/get", - responses={400: {"description": "Query can't be processed", "model": ErrorResponse}}, +_V2_EID_QUERY_DESCRIPTION = ( + "**v2 API**: EID is passed as query parameter (`?eid=X`) instead of path.\n" + "This allows EIDs containing special characters like `/` (e.g., IPv6 CIDR notation)." ) -async def v2_get_entity_type_raw_datapoints( - etype: str, - eid: str | None = None, - attr: str | None = None, - src: str | None = None, - skip: NonNegativeInt = 0, - limit: NonNegativeInt = 20, -) -> EntityRawDataPage: - """List raw datapoints from the current raw collection for troubleshooting ingestion. - Same as v1 `/entity/{etype}/raw/get` endpoint. See v1 documentation for details. - """ - if attr is not None and attr not in MODEL_SPEC.attribs(etype): - raise RequestValidationError(["query", "attr"], f"Attribute '{attr}' doesn't exist") - parsed_eid = _parse_optional_eid(etype, eid) +def _v2desc(v1_endpoint: Any) -> str: + """Combine v1 endpoint documentation with the v2 EID transport note.""" + return f"{cleandoc(v1_endpoint.__doc__ or '')}\n\n{_V2_EID_QUERY_DESCRIPTION}" - try: - datapoints = list( - DB.find_raw_datapoints( - etype, - eid=parsed_eid, - attr=attr, - src=src, - skip=skip, - limit=limit, - ) - ) - except DatabaseError as e: - raise HTTPException(status_code=400, detail=str(e)) from e - return EntityRawDataPage( - count=len(datapoints), - data=[_raw_datapoint_to_response(datapoint) for datapoint in datapoints], - ) - - -@v2_router.get( - "/{etype}/_/distinct/{attr}", - responses={400: {"description": "Query can't be processed", "model": ErrorResponse}}, -) -async def v2_get_distinct_attribute_values(etype: str, attr: str) -> dict[JsonVal, int]: - """Gets distinct attribute values and their counts based on latest snapshots. - - Same as v1 `/entity/{etype}/_/distinct/{attr}` endpoint. See v1 documentation for details. - """ - try: - return DB.snapshots.get_distinct_val_count(etype, attr) - except DatabaseError as e: - raise HTTPException(status_code=400, detail=str(e)) from e - - -@v2_router.get("/{etype}", include_in_schema=False) -@v2_router.get("/{etype}/") +@v2_router.get("/{etype}", include_in_schema=False, description=_v2desc(get_eid_data)) +@v2_router.get("/{etype}/", description=_v2desc(get_eid_data)) async def v2_get_eid_data( - etype: str, - eid: Annotated[str, Query(description="Entity ID")], + e: ParsedQueryEid, date_from: AwareDatetime | None = None, date_to: AwareDatetime | None = None, ) -> EntityEidData: - """Get data of the entity identified by `etype` and `eid`. + return get_eid_data_handler(e, date_from, date_to) - Contains all snapshots and master record. Snapshots are ordered by ascending creation time. - **v2 API**: EID is passed as query parameter (`?eid=X`) instead of path. - This allows EIDs containing special characters like `/` (e.g., IPv6 CIDR notation). - """ - e = await parse_eid_from_query(etype, eid) - master_record = get_eid_master_record_handler(e, date_from, date_to) - snapshots = get_eid_snapshots_handler(e, date_from, date_to) - empty = not master_record and len(snapshots) == 0 - return EntityEidData(empty=empty, master_record=master_record, snapshots=snapshots) - - -@v2_router.get("/{etype}/master") +@v2_router.get("/{etype}/master", description=_v2desc(get_eid_master_record)) async def v2_get_eid_master_record( - etype: str, - eid: Annotated[str, Query(description="Entity ID")], + e: ParsedQueryEid, date_from: AwareDatetime | None = None, date_to: AwareDatetime | None = None, ) -> EntityEidMasterRecord: - """Get the master record of the entity identified by `etype` and `eid`. - - **v2 API**: EID is passed as query parameter (`?eid=X`) instead of path. - This allows EIDs containing special characters like `/` (e.g., IPv6 CIDR notation). - """ - e = await parse_eid_from_query(etype, eid) return get_eid_master_record_handler(e, date_from, date_to) -@v2_router.get("/{etype}/snapshots") +@v2_router.get("/{etype}/snapshots", description=_v2desc(get_eid_snapshots)) async def v2_get_eid_snapshots( - etype: str, - eid: Annotated[str, Query(description="Entity ID")], + e: ParsedQueryEid, date_from: AwareDatetime | None = None, date_to: AwareDatetime | None = None, skip: NonNegativeInt = 0, limit: NonNegativeInt = 0, ) -> EntityEidSnapshots: - """Get snapshots of the entity identified by `etype` and `eid`. - - Supports optional pagination via `skip` and `limit`. - Setting `limit` to `0` returns all matching snapshots. - - **v2 API**: EID is passed as query parameter (`?eid=X`) instead of path. - This allows EIDs containing special characters like `/` (e.g., IPv6 CIDR notation). - """ - e = await parse_eid_from_query(etype, eid) return get_eid_snapshots_handler(e, date_from, date_to, skip, limit) -@v2_router.get("/{etype}/attr") +@v2_router.get("/{etype}/attr", description=_v2desc(get_eid_attr_value)) async def v2_get_eid_attr_value( - etype: str, - eid: Annotated[str, Query(description="Entity ID")], + e: ParsedQueryEid, attr: Annotated[str, Query(description="Attribute name")], date_from: AwareDatetime | None = None, date_to: AwareDatetime | None = None, ) -> EntityEidAttrValueOrHistory: - """Get attribute value. - - Value is either of: - - current value: in case of plain attribute - - current value and history: in case of observation attribute - - history: in case of timeseries attribute - - **v2 API**: EID is passed as query parameter (`?eid=X&attr=name`) instead of path. - This allows EIDs containing special characters like `/` (e.g., IPv6 CIDR notation). - """ - # Check if attribute exists - if attr not in MODEL_SPEC.attribs(etype): - raise RequestValidationError(["query", "attr"], f"Attribute '{attr}' doesn't exist") - - e = await parse_eid_from_query(etype, eid) - value_or_history = DB.get_value_or_history(e.type, attr, e.id, t1=date_from, t2=date_to) - - return EntityEidAttrValueOrHistory( - attr_type=MODEL_SPEC.attr(e.type, attr).t, **value_or_history - ) + return get_eid_attr_value_handler(e, attr, "query", date_from, date_to) -@v2_router.post("/{etype}/attr") +@v2_router.post("/{etype}/attr", description=_v2desc(set_eid_attr_value)) async def v2_set_eid_attr_value( - etype: str, - eid: Annotated[str, Query(description="Entity ID")], + e: ParsedQueryEid, attr: Annotated[str, Query(description="Attribute name")], request: Request, ) -> SuccessResponse: - """Set current value of attribute. - - Internally just creates datapoint for specified attribute and value. - - This endpoint is meant for `editable` plain attributes -- for direct user edit on DP3 web UI. - - **v2 API**: EID is passed as query parameter (`?eid=X&attr=name`) instead of path. - This allows EIDs containing special characters like `/` (e.g., IPv6 CIDR notation). - """ - # Check if attribute exists - if attr not in MODEL_SPEC.attribs(etype): - raise RequestValidationError(["query", "attr"], f"Attribute '{attr}' doesn't exist") + return await set_eid_attr_value_handler(e, attr, "query", request) - try: - body = EntityEidAttrValue.model_validate(await request.json()) - except ValueError as e: - raise RequestValidationError(["body"], str(e)) from e - # Construct datapoint - try: - dp = DataPoint( - type=etype, - id=eid, - attr=attr, - v=body.value, - t1=datetime.now(UTC), - src=f"{request.client.host} via API", - ) - dp3_dp = api_to_dp3_datapoint(dp.model_dump()) - except ValidationError as e: - raise RequestValidationError(["body", "value"], e.errors()[0]["msg"]) from e - - # This shouldn't fail - with task_context(MODEL_SPEC): - task = DataPointTask(etype=etype, eid=eid, data_points=[dp3_dp]) - - TASK_WRITER.put_task(task, False) - return SuccessResponse() +v2_router.add_api_route( + "/{etype}/_/distinct/{attr}", + get_distinct_attribute_values, + methods=["GET"], + name="v2_get_distinct_attribute_values", + responses={400: {"description": "Query can't be processed", "model": ErrorResponse}}, +) -@v2_router.post("/{etype}/ttl") +@v2_router.post("/{etype}/ttl", description=_v2desc(extend_eid_ttls)) async def v2_extend_eid_ttls( - etype: str, - eid: Annotated[str, Query(description="Entity ID")], + e: ParsedQueryEid, body: dict[str, AwareDatetime], ) -> SuccessResponse: - """Extend TTLs of the specified entity. + return extend_eid_ttls_handler(e, body) - **v2 API**: EID is passed as query parameter (`?eid=X`) instead of path. - This allows EIDs containing special characters like `/` (e.g., IPv6 CIDR notation). - """ - e = await parse_eid_from_query(etype, eid) - with task_context(MODEL_SPEC): - task = DataPointTask(etype=e.type, eid=e.id, ttl_tokens=body) - TASK_WRITER.put_task(task, False) - return SuccessResponse() - - -@v2_router.delete("/{etype}", include_in_schema=False) -@v2_router.delete("/{etype}/") -async def v2_delete_eid_record( - etype: str, - eid: Annotated[str, Query(description="Entity ID")], -) -> SuccessResponse: - """Delete the master record and snapshots of the specified entity. - Notice that this does not delete any raw datapoints, - or block the re-creation of the entity if new datapoints are received. - - **v2 API**: EID is passed as query parameter (`?eid=X`) instead of path. - This allows EIDs containing special characters like `/` (e.g., IPv6 CIDR notation). - """ - e = await parse_eid_from_query(etype, eid) - with task_context(MODEL_SPEC): - task = DataPointTask(etype=e.type, eid=e.id, delete=True) - TASK_WRITER.put_task(task, False) - return SuccessResponse() +@v2_router.delete("/{etype}", include_in_schema=False, description=_v2desc(delete_eid_record)) +@v2_router.delete("/{etype}/", description=_v2desc(delete_eid_record)) +async def v2_delete_eid_record(e: ParsedQueryEid) -> SuccessResponse: + return delete_eid_record_handler(e)