diff --git a/.badges/operations.svg b/.badges/operations.svg index 09d958417f..1b59e78b44 100644 --- a/.badges/operations.svg +++ b/.badges/operations.svg @@ -1,4 +1,4 @@ - + @@ -12,7 +12,7 @@ PARITY entries PARITY entries - 6388 - 6388 + 6398 + 6398 diff --git a/.badges/parity.svg b/.badges/parity.svg index cf9055da3d..fd1c261004 100644 --- a/.badges/parity.svg +++ b/.badges/parity.svg @@ -1,4 +1,4 @@ - + @@ -12,7 +12,7 @@ parity parity - 158 A · 2 B · 2 C - 158 A · 2 B · 2 C + 158 A · 2 B · 3 C + 158 A · 2 B · 3 C diff --git a/.badges/services.svg b/.badges/services.svg index f255ce9805..fa0eb1f907 100644 --- a/.badges/services.svg +++ b/.badges/services.svg @@ -1,4 +1,4 @@ - + @@ -12,7 +12,7 @@ AWS services AWS services - 164 - 164 + 165 + 165 diff --git a/AZURE.md b/AZURE.md index 3944e668f0..943f51ffc5 100644 --- a/AZURE.md +++ b/AZURE.md @@ -82,7 +82,7 @@ That said, MQTT's fixed port (`1883`) happens to fall *outside* `--port-range-st - **M0 (done)** — `pkgs/azureauth` (SharedKey canonicalization + fixed devstoreaccount1 constants); `services/azureblob` wired into `cli.go` and bound to its own fixed port (synchronous bind, fail-fast, no `PortAlloc` fallback — see section 4's port-selection note); Create/Delete/List Container, Put/Get/Delete Blob, List Blobs; `PARITY.md`; unit tests + Go integration tests using `azure-sdk-for-go`. See `services/azureblob/README.md`/`PARITY.md` for current status and known gaps. - **M1** — Blob completeness: properties/metadata, block-blob multipart (Put Block/Put Block List), conditional headers (`If-Match`/`If-None-Match`), error-mapping table (mirrors `services/sqs`'s `errorDetails` pattern). - **M2 (done)** — `services/azurequeue`: List/Create/Delete Queue, full message lifecycle (Put/Get/Peek/Delete/Update/Clear Messages), visibility timeout with pop-receipt rotation, message TTL swept by a background `Janitor`, wired into `cli.go` and bound to its own fixed port (10001, Azurite's own Queue port; synchronous bind, fail-fast, no `PortAlloc` fallback -- mirrors Azure Blob's M0 port strategy, see section 4). See `services/azurequeue/README.md`/`PARITY.md` for current status and known gaps. -- **M3** — `services/azuretable`: table CRUD, entity insert/get/query/update/merge/delete, `$filter` subset (eq/ne/lt/gt/and/or on partition/row key plus scalar properties), ETag-based optimistic concurrency. +- **M3 (done)** — `services/azuretable`: table CRUD (Create/Delete/List Table), full entity lifecycle (Insert/Get/Query/Replace/Merge/Delete), a hand-written `$filter` lexer/parser/evaluator (eq/ne/lt/le/gt/ge, and/or/not, parentheses, every OData literal form -- string/int/Int64/float/bool/datetime/guid/binary -- against `PartitionKey`/`RowKey`/`Timestamp` and scalar properties), all eight EDM property types (String/Int32/Int64/Double/Boolean/DateTime/Guid/Binary) with `@odata.type` annotation round-tripping matching `azure-sdk-for-go/sdk/data/aztables`'s own client-side inference, and ETag-based optimistic concurrency (If-Match `*`/specific/absent -> upsert), wired into `cli.go` and bound to its own fixed port (10002, Azurite's own Table port; synchronous bind, fail-fast, no `PortAlloc` fallback -- mirrors Blob's M0 and Queue's M2 port strategy, see section 4). Batch (`$batch` multipart/mixed changesets) and continuation-token pagination are deferred -- see `services/azuretable/PARITY.md`. (Note the milestone-numbering skew already flagged in `services/azurequeue/PARITY.md`'s `deferred:` entry: this repo's section-8 list calls Table "M3", while the implementation task that built it used the internal name "M2".) See `services/azuretable/README.md`/`PARITY.md` for current status and known gaps. - **M4** — `services/cosmosdb`: database/container CRUD (with partition-key-path declaration), document CRUD, SQL-subset query engine, fake RU/session-token/etag headers; scope the op list against cosmium (github.com/pikami/cosmium) as reference prior art; integration tests against `azure-sdk-for-go`, `azure-sdk-for-js`, and `azure-cosmos` (Python). - **M5** — Docs/polish: root README services table + badges/icons, `docs/services/*.md` guides, a docker-compose example under `examples/`, and the `test/e2e` cross-SDK smoke suite covering all four services. diff --git a/README.md b/README.md index 67c49e26d0..a84163d6c9 100644 --- a/README.md +++ b/README.md @@ -692,6 +692,7 @@ Every service links to its own page with a coverage breakdown — audited operat | [AppStream 2.0](services/appstream/README.md) | A | 44 | clean | | [Azureblob](services/azureblob/README.md) | C | 8 | 8 gaps; 2 deferred | | [Azurequeue](services/azurequeue/README.md) | C | 9 | 7 gaps; 1 deferred | +| [Azuretable](services/azuretable/README.md) | C | 10 | 6 gaps; 3 deferred | | [Cloudfrontkeyvaluestore](services/cloudfrontkeyvaluestore/README.md) | B | 6 | 3 gaps; 1 structural gap | | [Directconnect](services/directconnect/README.md) | A | 64 | 3 gaps; 8 structural gaps; 1 deferred | | [Grafana](services/grafana/README.md) | A | 25 | 2 gaps; 1 structural gap | diff --git a/cli.go b/cli.go index 11b08aa868..a220e51334 100644 --- a/cli.go +++ b/cli.go @@ -91,6 +91,7 @@ import ( awsconfigbackend "github.com/blackbirdworks/gopherstack/services/awsconfig" azureblobbackend "github.com/blackbirdworks/gopherstack/services/azureblob" azurequeuebackend "github.com/blackbirdworks/gopherstack/services/azurequeue" + azuretablebackend "github.com/blackbirdworks/gopherstack/services/azuretable" backupbackend "github.com/blackbirdworks/gopherstack/services/backup" batchbackend "github.com/blackbirdworks/gopherstack/services/batch" bedrockbackend "github.com/blackbirdworks/gopherstack/services/bedrock" @@ -462,6 +463,7 @@ type CLI struct { StepFunctions sfnbackend.Settings `embed:"" prefix:"stepfunctions-"` AzureBlob azureblobbackend.Settings `embed:"" prefix:"azure-blob-"` AzureQueue azurequeuebackend.Settings `embed:"" prefix:"azure-queue-"` + AzureTable azuretablebackend.Settings `embed:"" prefix:"azure-table-"` PortRangeStart int ` name:"port-range-start" env:"PORT_RANGE_START" default:"10000" help:"Start of the port range for resource endpoints."` //nolint:lll // config struct tags are intentionally verbose PortRangeEnd int ` name:"port-range-end" env:"PORT_RANGE_END" default:"10100" help:"End (exclusive) of the port range for resource endpoints."` //nolint:lll // config struct tags are intentionally verbose EC2DockerSSHPortMin int ` name:"ec2-docker-ssh-port-min" env:"EC2_DOCKER_SSH_PORT_MIN" default:"0" help:"Lower bound of the host TCP port range used to map EC2-docker SSH (0 = let Docker pick)."` //nolint:lll // config struct tags are intentionally verbose @@ -535,6 +537,11 @@ func (c *CLI) GetAzureQueueSettings() azurequeuebackend.Settings { return c.AzureQueue } +// GetAzureTableSettings returns Azure Table settings (azuretable.ConfigProvider). +func (c *CLI) GetAzureTableSettings() azuretablebackend.Settings { + return c.AzureTable +} + // GetS3Endpoint returns the configured S3 endpoint (s3.ConfigProvider). func (c *CLI) GetS3Endpoint() string { s3Port := strings.TrimPrefix(c.Port, ":") @@ -1897,6 +1904,17 @@ func reserveFixedServicePorts(ctx context.Context, log *slog.Logger, alloc *port log.WarnContext(ctx, "failed to reserve AzureQueue's fixed port in the shared pool", "port", cli.AzureQueue.Port, "error", err) } + + // AzureTable's dedicated listener (services/azuretable) binds its own + // fixed, protocol-conventional default port (10002, matching Azurite's + // own Table service port) the same way AzureBlob/AzureQueue do above -- + // see those calls' comments and AZURE.md section 4 for the full + // rationale. It sits in the same PortRangeStart/PortRangeEnd default + // range, so it needs the same reservation. + if err := alloc.Reserve(cli.AzureTable.Port, "azuretable"); err != nil { + log.WarnContext(ctx, "failed to reserve AzureTable's fixed port in the shared pool", + "port", cli.AzureTable.Port, "error", err) + } } // setupPortAllocatorWithReservations builds the shared port allocator and @@ -3632,6 +3650,7 @@ func getMostRecentServiceProviders() []service.Provider { return []service.Provider{ &azureblobbackend.Provider{}, &azurequeuebackend.Provider{}, + &azuretablebackend.Provider{}, &pinpointbackend.Provider{}, &pipesbackend.Provider{}, &accessanalyzerbackend.Provider{}, diff --git a/cli_azuretable_port_reservation_test.go b/cli_azuretable_port_reservation_test.go new file mode 100644 index 0000000000..6787291db8 --- /dev/null +++ b/cli_azuretable_port_reservation_test.go @@ -0,0 +1,59 @@ +package main + +import ( + "log/slog" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/blackbirdworks/gopherstack/pkgs/portalloc" + azuretablebackend "github.com/blackbirdworks/gopherstack/services/azuretable" +) + +// TestReserveFixedServicePorts_AzureTable is a sibling to +// cli_azureblob_port_reservation_test.go's TestReserveFixedServicePorts, +// covering AzureTable's own fixed-port reservation instead of restructuring +// that file's AzureBlob-only test table: services/azuretable binds its +// dedicated listener directly via net.Listen, not through PortAlloc, but its +// default port (10002) sits inside PortRangeStart/PortRangeEnd's own default +// range (10000-10100). Without reserving it, PortAlloc could still hand that +// same port number to an unrelated caller (e.g. ElastiCache), which would +// only surface later as a confusing address-in-use failure. See AZURE.md +// section 4 and pkgs/portalloc.Allocator.Reserve's doc comment. +func TestReserveFixedServicePorts_AzureTable(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + azurePort int + rangeStart int + rangeEnd int + wantBlockedFromPool bool + }{ + { + name: "default azure table port collides with default pool range", + azurePort: azuretablebackend.DefaultPort, rangeStart: 10000, rangeEnd: 10100, + wantBlockedFromPool: true, + }, + { + name: "custom azure table port outside a custom pool range", + azurePort: 9998, rangeStart: 10000, rangeEnd: 10100, + wantBlockedFromPool: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + alloc, err := portalloc.New(tt.rangeStart, tt.rangeEnd) + require.NoError(t, err) + + cli := CLI{AzureTable: azuretablebackend.Settings{Port: tt.azurePort}} + reserveFixedServicePorts(t.Context(), slog.Default(), alloc, cli) + + assert.Equal(t, tt.wantBlockedFromPool, alloc.IsAllocated(tt.azurePort), tt.name) + }) + } +} diff --git a/go.mod b/go.mod index 001f9239ce..195ab829c8 100644 --- a/go.mod +++ b/go.mod @@ -208,6 +208,8 @@ require github.com/aws/aws-sdk-go-v2/service/omics v1.49.5 require github.com/aws/aws-sdk-go-v2/service/cleanrooms v1.49.4 require ( + github.com/Azure/azure-sdk-for-go/sdk/azcore v1.22.0 + github.com/Azure/azure-sdk-for-go/sdk/data/aztables v1.4.1 github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.8.0 github.com/Azure/azure-sdk-for-go/sdk/storage/azqueue v1.0.1 github.com/aws/aws-sdk-go-v2/service/account v1.35.4 @@ -229,7 +231,6 @@ require ( ) require ( - github.com/Azure/azure-sdk-for-go/sdk/azcore v1.22.0 // indirect github.com/Azure/azure-sdk-for-go/sdk/internal v1.12.0 // indirect github.com/dustin/go-humanize v1.0.1 // indirect github.com/ncruces/go-strftime v1.0.0 // indirect diff --git a/go.sum b/go.sum index c2bec44393..64a0ace3b7 100644 --- a/go.sum +++ b/go.sum @@ -8,6 +8,8 @@ github.com/Azure/azure-sdk-for-go/sdk/azcore v1.22.0 h1:aokoqcHvaGjiM3VpjKDfMMnF github.com/Azure/azure-sdk-for-go/sdk/azcore v1.22.0/go.mod h1:/WYEx9pcM9Y+Dd/APJaNlSvVSvzl54rrMdZT5+Oi2LM= github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.13.1 h1:Hk5QBxZQC1jb2Fwj6mpzme37xbCDdNTxU7O9eb5+LB4= github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.13.1/go.mod h1:IYus9qsFobWIc2YVwe/WPjcnyCkPKtnHAqUYeebc8z0= +github.com/Azure/azure-sdk-for-go/sdk/data/aztables v1.4.1 h1:j0hhYS006eJ54vusoap0f2NVZ1YY3QnaAEnLM68f0SQ= +github.com/Azure/azure-sdk-for-go/sdk/data/aztables v1.4.1/go.mod h1:AdtInaXmK8eYmbjezRWgLz+Qs46nc9Up9GWGwteWNfw= github.com/Azure/azure-sdk-for-go/sdk/internal v1.12.0 h1:fhqpLE3UEXi9lPaBRpQ6XuRW0nU7hgg4zlmZZa+a9q4= github.com/Azure/azure-sdk-for-go/sdk/internal v1.12.0/go.mod h1:7dCRMLwisfRH3dBupKeNCioWYUZ4SS09Z14H+7i8ZoY= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.8.1 h1:/Zt+cDPnpC3OVDm/JKLOs7M2DKmLRIIp3XIx9pHHiig= diff --git a/pkgs/persistence/testdata/snapshot_inventory.json b/pkgs/persistence/testdata/snapshot_inventory.json index 29e104aa22..0c30505cef 100644 --- a/pkgs/persistence/testdata/snapshot_inventory.json +++ b/pkgs/persistence/testdata/snapshot_inventory.json @@ -2531,6 +2531,22 @@ ], "version": 1 }, + "azuretable": { + "fields": [ + "EntityProperty.Type EdmType", + "EntityProperty.Value any", + "backendSnapshot.Tables map[string]*storedTable `json:\"tables\"`", + "entityCompositeKey.PartitionKey string", + "entityCompositeKey.RowKey string", + "storedEntity.PartitionKey string", + "storedEntity.Properties map[string]EntityProperty", + "storedEntity.RowKey string", + "storedEntity.Timestamp time.Time", + "storedTable.Entities map[entityCompositeKey]*storedEntity", + "storedTable.Name string" + ], + "version": 2 + }, "backup": { "fields": [ "AdvancedBackupSetting.BackupOptions map[string]string `json:\"backupOptions,omitempty\"`", diff --git a/services/azuretable/PARITY.md b/services/azuretable/PARITY.md new file mode 100644 index 0000000000..720e04e0ae --- /dev/null +++ b/services/azuretable/PARITY.md @@ -0,0 +1,139 @@ +--- +service: azuretable +sdk_module: azure-sdk-for-go/sdk/data/aztables@v1.4.1 +last_audit_commit: 3219e576 +last_audit_date: 2026-09-04 +overall: C +# Per-op or per-op-family status. Values: ok | partial | gap | deferred. +# wire=response/request shape vs SDK; errors=code+HTTP status; state=real mutate/read; persist=in backendSnapshot. +ops: + CreateTable: {wire: ok, errors: ok, state: ok, persist: ok, note: "POST //Tables, body {\"TableName\":\"..\"}. 201 + entity body, or 204 + Preference-Applied when Prefer: return-no-content is sent (aztables' default). 409 TableAlreadyExists on duplicate -- unlike services/azurequeue's CreateQueue, there is no metadata-identical-retry idempotency exception."} + DeleteTable: {wire: ok, errors: ok, state: ok, persist: ok, note: "DELETE //Tables('name'), including the '' escaped-quote form. 404 TableNotFound if absent."} + ListTables: {wire: ok, errors: ok, state: ok, persist: n/a, note: "GET //Tables, sorted by name. No prefix/continuation-token pagination -- returns all tables in one page."} + InsertEntity: {wire: ok, errors: ok, state: ok, persist: ok, note: "POST //. 201 + entity body + ETag, or 204 + Preference-Applied under Prefer: return-no-content. 409 EntityAlreadyExists; 404 TableNotFound. Missing PartitionKey/RowKey -> 400 InvalidInput; empty-string keys are accepted (matches real Azure)."} + GetEntity: {wire: ok, errors: ok, state: ok, persist: n/a, note: "GET //
(PartitionKey='..',RowKey='..'). 200 + entity + ETag; 404 ResourceNotFound. $select is honored for custom properties; PartitionKey/RowKey/Timestamp are always returned regardless of $select (see Notes)."} + QueryEntities: {wire: ok, errors: ok, state: ok, persist: n/a, note: "GET //
() or //
, both with optional $filter/$top/$select. Results ordered by (PartitionKey, RowKey). No continuation-token pagination -- returns everything matching in one page, capped by $top if given."} + ReplaceEntity: {wire: ok, errors: ok, state: ok, persist: ok, note: "PUT //
(..). No If-Match -> Insert-Or-Replace (upsert). If-Match: * -> unconditional replace, 404 ResourceNotFound if absent. If-Match: -> 412 UpdateConditionNotSatisfied on mismatch, 404 if absent. Always full-body replace (drops properties not in the new body)."} + MergeEntity: {wire: ok, errors: ok, state: ok, persist: ok, note: "PATCH (aztables' real wire method), literal MERGE, or POST/PUT/PATCH carrying an X-Http-Method: MERGE tunneling header (honored only on those three methods, never GET/DELETE) //
(..), same If-Match semantics as ReplaceEntity but merges properties (unlisted properties survive) instead of replacing wholesale."} + DeleteEntity: {wire: ok, errors: ok, state: ok, persist: ok, note: "DELETE //
(..). If-Match is mandatory (400 InvalidInput if absent) -- * or a specific ETag; 412 on mismatch, 404 ResourceNotFound if absent."} + Batch: {wire: deferred, errors: ok, state: deferred, persist: n/a, note: "POST //$batch (multipart/mixed changesets) returns a clean 501 NotImplemented pointing at this file, rather than a confusing 404/400. See deferred section."} +families: + auth: {status: partial, note: "Identical stance to services/azurequeue: checkAuth parses a present Authorization header via pkgs/azureauth.ParseAuthorizationHeader (structural only, SharedKey and SharedKeyLite both accepted). Verification is not enforced; an absent or malformed header is still accepted."} + wire_protocol: {status: ok, note: "REST+JSON/OData. Honors the Accept header's odata= level (nometadata/minimalmetadata/fullmetadata), responding with a matching Content-Type. x-ms-version (pinned to 2019-02-02, the literal value azure-sdk-for-go/sdk/data/aztables' generated client sends and expects), x-ms-request-id, Date, and DataServiceVersion: 3.0; are set on every response; x-ms-error-code is set on every error."} + odata_filter: {status: ok, note: "Hand-written lexer -> recursive-descent parser -> AST -> evaluator (odata_filter.go/odata_filter_eval.go), modeled on services/dynamodb/expr. Supports eq/ne/lt/le/gt/ge, and/or/not, parentheses, and every literal form (quoted string with '' escape, integer, Int64 'L' suffix, float, true/false, datetime'..', guid'..', X'..'/binary'..'). Recursion is depth-bounded (maxFilterDepth=100), counted per genuine nesting level ('not' and '(...)' only, not per precedence-climbing layer -- see Notes) against stack-overflow DoS. A comparison against a missing property evaluates false, never an error; a parse error always surfaces as 400 InvalidInput, never a panic. Two Int64 operands are compared as int64 directly, not via a blanket float64 conversion, so magnitudes beyond 2^53 compare correctly (see Notes)."} + edm_types: {status: ok, note: "Edm.String/Int32/Int64/Double/Boolean/DateTime/Guid/Binary all round-trip. Int64/DateTime/Guid/Binary emit a Prop@odata.type annotation on read (matching real Table Storage and aztables' own EDMEntity.MarshalJSON); Int32/Double/Boolean/String do not. Unannotated bare-number decode tries Int32 first, then falls back to Double, exactly mirroring aztables' own client-side inference -- a whole-number Double (e.g. 4.0) is therefore indistinguishable from an Int32 without an explicit annotation, an inherent wire-protocol ambiguity aztables itself has (its own MarshalJSON never annotates plain float64 either). EdmBinary values are deep-copied on every store/return path (insert, merge, get, query) so a caller mutating a []byte it passed in or received back can never alias/corrupt stored state."} + etag_and_timestamp: {status: ok, note: "ETag format W/\"datetime''\" derived from Timestamp. Every mutation advances Timestamp by at least 100ns past its previous value even if the injected clock returns the same instant twice in a row, guaranteeing a distinct ETag on every write (see store.go's bumpTimestamp) -- covered by TestInMemoryBackend_ETagChangesOnEveryWrite."} + persistence: {status: ok, note: "Snapshot version 2 (bumped from 1 -- see Notes). Edm.Int64 is snapshotted as a decimal string, not a bare JSON number: a float64 number loses precision above 2^53, corrupting large Int64 values across a save/restore cycle. Entity identity within a table is keyed by entityCompositeKey (a {PartitionKey, RowKey} struct, persisted via MarshalText as a JSON string array), not a NUL-delimited string, closing a key-collision class where two different (PartitionKey, RowKey) pairs could hash to the same delimited string. Restore initializes a table's Entities map when the snapshot carries \"Entities\": null (legal JSON, decodes to a nil map) rather than leaving it nil, which would panic the first time anything inserts into it."} + routing_isolation: {status: ok, note: "Runs on its own dedicated *http.Server, bound synchronously in StartWorker to a fixed port (default 10002 via --azure-table-port/AZURE_TABLE_PORT, matching Azurite's own Table service port; no fallback pool -- fails fast if unavailable, mirroring services/azureblob and services/azurequeue). cli.go's reserveFixedServicePorts additionally reserves this port in the shared PortAlloc pool since 10002 sits inside --port-range-start/--port-range-end's own default range."} + observability: {status: ok, note: "StartWorker wraps its Echo handler with telemetry.WrapEchoHandler so ExtractOperation/ExtractResource feed Prometheus metrics. InMemoryBackend uses *lockmetrics.RWMutex instead of raw sync.RWMutex, matching repo convention."} +gaps: + - "No continuation-token pagination on List Tables or Query Entities -- both return every matching result in one page. x-ms-continuation-NextPartitionKey/NextRowKey response headers are not set." + - "$select is honored for custom properties, but PartitionKey/RowKey/Timestamp are always returned regardless of the $select list (real Table Storage honors $select literally for these too); documented deviation chosen for simplicity and because every SDK round-trip needs the key properties anyway." + - "A whole-number Edm.Double value (e.g. 4.0) round-trips as Edm.Int32 when written without an explicit @odata.type annotation -- an inherent ambiguity in the unannotated-number wire format that aztables' own client has too (see families.edm_types)." + - "No SAS / Set-Get Table ACL support." + - "No queue-style janitor: Table Storage entities have no TTL/expiry concept, so there is nothing to sweep (this is a deliberate scope decision, not an oversight -- see provider.go's Provider doc comment)." + - "Auth verification is not enforced -- see families.auth." + All gaps above are intentional MVP scope per AZURE.md's M3 entry (see AZURE.md section 8; note the milestone numbering there differs from this task's internal M2 naming), not oversights. +deferred: + - "Batch (POST //$batch, multipart/mixed changesets) is explicitly out of scope for this milestone -- returns a clean 501 NotImplemented rather than attempting a partial implementation. Matches services/azureblob's M0 multipart-upload deferral pattern." + - "Continuation-token pagination for List Tables/Query Entities (see gaps)." + - "Initial implementation pass (2026-09-04): seeded this service from scratch per AZURE.md M3 (see AZURE.md section 8; note the milestone numbering there differs from this task's internal M2 naming). Structurally mirrors services/azurequeue's M1/M2 implementation and PARITY.md format; no prior audit history to reconcile." +leaks: {status: clean, note: "The dedicated *http.Server started by StartWorker is stopped by Shutdown via srv.Shutdown(ctx) (falling back to srv.Close() on a graceful-shutdown error, both logged), mirroring services/azurequeue and cli.go's own top-level server lifecycle. No background goroutines beyond the listener itself -- there is no janitor (see gaps)."} +--- + +## Notes + +### Why Azure Table gets its own port, separate from Blob and Queue +Azure Table's REST path shape (`//`) shares the same +ambiguity with Azure Blob and Queue that motivated each of their own +dedicated ports (see `services/azureblob/PARITY.md` and +`services/azurequeue/PARITY.md`'s identical notes, and AZURE.md section 4) +-- multiplexing Table onto either of their ports would reintroduce exactly +that collision. `StartWorker` synchronously binds a fixed port (default +`10002`, Azurite's own Table-service default, overridable via +`--azure-table-port`/`AZURE_TABLE_PORT`) before standing up its own +`*echo.Echo` + `*http.Server`, with no fallback into the shared +`--port-range-start`/`--port-range-end` `PortAlloc` pool if the bind fails. +`cli.go`'s `reserveFixedServicePorts` additionally reserves `10002` in that +shared pool at startup. + +### No janitor +Table Storage entities carry no TTL, visibility timeout, or lease concept -- +nothing analogous to Queue's message expiry or Blob's lease expiry. There is +therefore no background sweep to run, and (deliberately, unlike +`services/azurequeue`/`services/azureblob`) no `janitor.go` in this package. + +### `$filter` grammar +``` +expr := orExpr +orExpr := andExpr ('or' andExpr)* +andExpr := unary ('and' unary)* +unary := 'not' unary | primary +primary := '(' expr ')' | comparison +comparison := operand ('eq'|'ne'|'lt'|'le'|'gt'|'ge') operand +operand := identifier | literal +literal := 'quoted string' (with '' escape) | integer | integer'L' | float | true | false + | datetime'' | guid'' | X'' | binary'' +``` +Implemented as a real lexer -> recursive-descent parser -> AST -> evaluator +(`odata_filter.go`/`odata_filter_eval.go`), modeled on +`services/dynamodb/expr`'s identical shape -- not string matching. Recursion +depth is bounded (`maxFilterDepth = 100`) so a maliciously (or accidentally) +deeply-nested filter fails with a parse error instead of overflowing the +stack. Identifiers resolve against `PartitionKey`/`RowKey`/`Timestamp` first, +then custom properties; a comparison against a property the entity doesn't +have evaluates to `false`, matching real Table Storage semantics, never an +error. Comparisons are type-aware: numeric operands (Int32/Int64/Double) are +compared numerically regardless of which numeric type each side is, strings +lexicographically, datetimes chronologically, and booleans only support +`eq`/`ne`. A type mismatch between the two operands (e.g. a string compared +against a number) evaluates to `false` rather than erroring. A parse error of +any kind -- unbalanced parens, a trailing operator, an empty filter string, an +invalid literal -- surfaces as `400 InvalidInput`, never a panic and never a +500. + +**Depth counting (fixed in review).** `parseOr` -> `parseAnd` -> `parseUnary` +-> `parsePrimary` is one precedence-climbing layer per grammar rule, not one +nesting level -- a bare `Age eq 1` with no parens or `not` anywhere still +passes through all four. An earlier version incremented the depth counter on +every one of those layers, so `maxFilterDepth = 100` was actually exhausted +by roughly 25 nested parentheses, not 100 -- a bound that didn't mean what +its name said. `depth` is now threaded through unchanged across the routine +same-level calls and incremented only at the two places genuine nesting +happens: `parseUnary`'s `not` branch and `parsePrimary`'s `(...)` branch. +`TestParseFilter_ModeratelyNestedParensAccepted` (50 levels, must parse) and +`TestParseFilter_DeepNestingBounded` (500 levels, must reject) cover both +sides of the corrected bound. + +**Int64 comparison precision (fixed in review).** Comparing two integer +operands (Int32 and/or Int64) converts neither through `float64` when both +are integer-typed: it compares their `int64` values directly. A blanket +`float64(intVal)` conversion -- this package's original approach -- silently +rounds any Int64 magnitude beyond `2^53` (float64's mantissa width), so e.g. +`Big eq 9007199254740992L` would incorrectly match a stored +`9007199254740993`. A comparison involving an actual `Edm.Double` operand +still goes through `float64`, since Double itself is already an inexact +64-bit float with no wider exact common type to compare against. See +`TestEvaluateFilter_Int64PrecisionNotLostInComparison`. + +### EDM property typing +See `families.edm_types` above. `models.go`'s `EntityProperty` carries an +explicit `EdmType` alongside its Go value so read/write and `$filter` +evaluation never have to guess; `entity_ops.go`'s `decodeProperty` mirrors +`azure-sdk-for-go/sdk/data/aztables`'s own `EDMEntity.UnmarshalJSON` +inference logic exactly (try `Edm.Int32` first for an unannotated bare +number, then fall through to the JSON value's natural type) so unmodified +SDK round trips match byte-for-byte in the cases that matter. + +### ETag / Timestamp monotonicity +See `families.etag_and_timestamp` above -- this is exactly the bug class +M1's (`services/azurequeue`) review bots caught elsewhere in this project: +two mutations to the same entity within the same clock tick must not produce +identical ETags, so `store.go`'s `bumpTimestamp` forces at least a 100ns +forward step past the entity's previous `Timestamp` even when the injected +clock hasn't advanced. + +## More + +- [Full parity audit](PARITY.md) +- [All services](../../README.md#services) diff --git a/services/azuretable/README.md b/services/azuretable/README.md new file mode 100644 index 0000000000..2f30aa2785 --- /dev/null +++ b/services/azuretable/README.md @@ -0,0 +1,34 @@ + +# Azuretable + +**Parity grade: C** · SDK `azure-sdk-for-go/sdk/data/aztables@v1.4.1` · last audited 2026-09-04 (`3219e576`) + +## Coverage + +| Metric | Value | +| --- | --- | +| PARITY entries audited | 10 (9 ok, 1 deferred) | +| Feature families | 8 (7 ok, 1 partial) | +| Known gaps | 6 | +| Deferred items | 3 | +| Resource leaks | clean | + +### Known gaps + +- No continuation-token pagination on List Tables or Query Entities -- both return every matching result in one page. x-ms-continuation-NextPartitionKey/NextRowKey response headers are not set. +- $select is honored for custom properties, but PartitionKey/RowKey/Timestamp are always returned regardless of the $select list (real Table Storage honors $select literally for these too); documented deviation chosen for simplicity and because every SDK round-trip needs the key properties anyway. +- A whole-number Edm.Double value (e.g. 4.0) round-trips as Edm.Int32 when written without an explicit @odata.type annotation -- an inherent ambiguity in the unannotated-number wire format that aztables' own client has too (see families.edm_types). +- No SAS / Set-Get Table ACL support. +- No queue-style janitor: Table Storage entities have no TTL/expiry concept, so there is nothing to sweep (this is a deliberate scope decision, not an oversight -- see provider.go's Provider doc comment). +- Auth verification is not enforced -- see families.auth. All gaps above are intentional MVP scope per AZURE.md's M3 entry (see AZURE.md section 8; note the milestone numbering there differs from this task's internal M2 naming), not oversights. + +### Deferred + +- Batch (POST //$batch, multipart/mixed changesets) is explicitly out of scope for this milestone -- returns a clean 501 NotImplemented rather than attempting a partial implementation. Matches services/azureblob's M0 multipart-upload deferral pattern. +- Continuation-token pagination for List Tables/Query Entities (see gaps). +- Initial implementation pass (2026-09-04): seeded this service from scratch per AZURE.md M3 (see AZURE.md section 8; note the milestone numbering there differs from this task's internal M2 naming). Structurally mirrors services/azurequeue's M1/M2 implementation and PARITY.md format; no prior audit history to reconcile. + +## More + +- [Full parity audit](PARITY.md) +- [All services](../../README.md#services) diff --git a/services/azuretable/coverage_test.go b/services/azuretable/coverage_test.go new file mode 100644 index 0000000000..015e3a4fb7 --- /dev/null +++ b/services/azuretable/coverage_test.go @@ -0,0 +1,145 @@ +package azuretable_test + +import ( + "context" + "fmt" + "net" + "net/http" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/blackbirdworks/gopherstack/services/azuretable" +) + +// freeEphemeralPort returns a port that was free at the moment of the call +// (bound briefly via net.Listen("tcp", ":0") then released immediately). +// Mirrors services/azurequeue's identical helper. +func freeEphemeralPort(t *testing.T) int { + t.Helper() + + l, err := net.Listen("tcp", ":0") + require.NoError(t, err) + + addr, ok := l.Addr().(*net.TCPAddr) + require.True(t, ok) + require.NoError(t, l.Close()) + + return addr.Port +} + +// reserveEphemeralPort binds and holds a real TCP port until the test ends, +// for tests that need a guaranteed-busy port to exercise a bind-failure path. +func reserveEphemeralPort(t *testing.T) int { + t.Helper() + + l, err := net.Listen("tcp", ":0") + require.NoError(t, err) + t.Cleanup(func() { _ = l.Close() }) + + addr, ok := l.Addr().(*net.TCPAddr) + require.True(t, ok) + + return addr.Port +} + +// TestStartWorker_BindsAndServes exercises the real synchronous bind: it +// starts the dedicated listener on a concrete port, makes a real HTTP +// request against it to prove the telemetry-wrapped handler is actually +// reachable, then shuts it down. +func TestStartWorker_BindsAndServes(t *testing.T) { + t.Parallel() + + port := freeEphemeralPort(t) + + backend := azuretable.NewInMemoryBackend() + h := azuretable.NewHandler(backend) + h.Port = port + + ctx := t.Context() + require.NoError(t, h.StartWorker(ctx)) + + t.Cleanup(func() { + // t.Context() is already canceled by the time a Cleanup-registered + // function runs (see testing.T.Context's doc comment), so it cannot + // serve as this timeout's parent -- an already-done parent would + // make shutdownCtx expire immediately, always forcing Shutdown's + // srv.Close() fallback instead of exercising its graceful path. + // context.Background() bounded by an explicit short timeout is the + // correct (not merely tolerated) choice here. + shutdownCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + h.Shutdown(shutdownCtx) + }) + + url := fmt.Sprintf("http://127.0.0.1:%d/%s/Tables", port, testAccount) + + require.Eventually(t, func() bool { + req, reqErr := http.NewRequestWithContext(ctx, http.MethodGet, url, http.NoBody) + if reqErr != nil { + return false + } + + resp, err := http.DefaultClient.Do(req) + if err != nil { + return false + } + defer resp.Body.Close() + + return resp.StatusCode == http.StatusOK + }, 2*time.Second, 10*time.Millisecond, "dedicated listener should become reachable") +} + +// TestStartWorker_BindFailureIsSynchronous is a regression test for the +// port-check race: binding an already-listening port must fail +// synchronously from StartWorker itself. +func TestStartWorker_BindFailureIsSynchronous(t *testing.T) { + t.Parallel() + + port := reserveEphemeralPort(t) + + h := azuretable.NewHandler(azuretable.NewInMemoryBackend()) + h.Port = port + + err := h.StartWorker(t.Context()) + require.Error(t, err) + assert.Contains(t, err.Error(), "bind port") +} + +// TestShutdown_NilServerIsNoop covers Shutdown's early return when +// StartWorker was never called (h.srv is nil). +func TestShutdown_NilServerIsNoop(t *testing.T) { + t.Parallel() + + h := newTestHandler(t) + + assert.NotPanics(t, func() { + h.Shutdown(t.Context()) + }) +} + +// TestShutdown_ForcesCloseOnGracefulTimeout covers Shutdown's fallback path: +// an already-expired context makes srv.Shutdown return immediately with an +// error, forcing the srv.Close() fallback. +func TestShutdown_ForcesCloseOnGracefulTimeout(t *testing.T) { + t.Parallel() + + h := azuretable.NewHandler(azuretable.NewInMemoryBackend()) + h.Port = 0 + + require.NoError(t, h.StartWorker(t.Context())) + + expiredCtx, cancel := context.WithDeadline(t.Context(), time.Now().Add(-time.Second)) + defer cancel() + + assert.NotPanics(t, func() { + h.Shutdown(expiredCtx) + }) + + // A second Shutdown call must also be a safe no-op (h.srv was cleared). + assert.NotPanics(t, func() { + h.Shutdown(t.Context()) + }) +} diff --git a/services/azuretable/entity_ops.go b/services/azuretable/entity_ops.go new file mode 100644 index 0000000000..d522957d36 --- /dev/null +++ b/services/azuretable/entity_ops.go @@ -0,0 +1,666 @@ +package azuretable + +import ( + "encoding/base64" + "encoding/json" + "errors" + "fmt" + "net/http" + "strconv" + "strings" + "time" + + "github.com/labstack/echo/v5" + + "github.com/blackbirdworks/gopherstack/pkgs/httputils" +) + +// devstoreAccountName is Azurite's well-known development storage account +// name, used to build odata.type values in fullmetadata responses (e.g. +// "devstoreaccount1.Tables"). See AZURE.md section 5 and pkgs/azureauth. +const devstoreAccountName = "devstoreaccount1" + +// entityTimeLayout formats an Edm.DateTime property/Timestamp value on the +// wire: a variable-precision (trailing zeros trimmed) RFC3339 string, +// matching aztables' own EDMDateTime.MarshalText layout exactly so its +// time.Parse round-trips cleanly. +const entityTimeLayout = "2006-01-02T15:04:05.9999999Z" + +// maxQueryTop is the largest $top value queryEntities honors; a caller- +// supplied value beyond this is clamped, not rejected, since larger values +// are harmless (this backend already returns everything in one page -- see +// PARITY.md's pagination gap) but an absurd value should never be used to +// pre-size an allocation. +const maxQueryTop = 100000 + +// --- Path/key-predicate parsing --- + +// unquoteODataString unquotes a single '...'-delimited OData string literal +// (escaped by doubling: a single quote written twice in a row means one +// literal quote), such as the table-name literal in DELETE +// //Tables('foo'). Returns ("", false) for anything else +// (missing/mismatched quotes, an unescaped quote inside). +func unquoteODataString(s string) (string, bool) { + if len(s) < 2 || s[0] != '\'' || s[len(s)-1] != '\'' { + return "", false + } + + inner := s[1 : len(s)-1] + + var b strings.Builder + + i := 0 + for i < len(inner) { + if inner[i] == '\'' { + if i+1 < len(inner) && inner[i+1] == '\'' { + b.WriteByte('\'') + i += 2 + + continue + } + + return "", false + } + + b.WriteByte(inner[i]) + i++ + } + + return b.String(), true +} + +// escapeODataKey escapes a key value for embedding back into a +// single-quoted OData literal (the inverse of unquoteODataString). +func escapeODataKey(s string) string { + return strings.ReplaceAll(s, "'", "''") +} + +// splitTopLevelCommas splits s on commas that are not inside a +// single-quoted literal. Quote-parity tracking (not full escape-aware +// parsing) is sufficient here: an escaped (doubled) quote toggles parity +// twice, correctly leaving the parser "still inside" the literal it started +// in. +func splitTopLevelCommas(s string) []string { + var parts []string + + var b strings.Builder + + inQuote := false + + for i := range len(s) { + c := s[i] + + switch { + case c == '\'': + inQuote = !inQuote + + b.WriteByte(c) + case c == ',' && !inQuote: + parts = append(parts, b.String()) + b.Reset() + default: + b.WriteByte(c) + } + } + + parts = append(parts, b.String()) + + return parts +} + +// parseEntityKeyPredicate parses an entity key predicate +// ("PartitionKey='p',RowKey='r'", in either key order) into its two values. +// Returns ok=false for anything malformed. +func parseEntityKeyPredicate(predicate string) (string, string, bool) { + parts := splitTopLevelCommas(predicate) + if len(parts) != 2 { //nolint:mnd // exactly PartitionKey and RowKey + return "", "", false + } + + var partitionKey, rowKey string + + var havePK, haveRK bool + + for _, part := range parts { + key, value, splitOK := strings.Cut(part, "=") + if !splitOK { + return "", "", false + } + + key = strings.TrimSpace(key) + + unquoted, unquoteOK := unquoteODataString(strings.TrimSpace(value)) + if !unquoteOK { + return "", "", false + } + + switch key { + case partitionKeyProperty: + partitionKey, havePK = unquoted, true + case rowKeyProperty: + rowKey, haveRK = unquoted, true + default: + return "", "", false + } + } + + if !havePK || !haveRK { + return "", "", false + } + + return partitionKey, rowKey, true +} + +// --- Entity body decode (request -> EntityProperty map) --- + +// decodeEntityBody parses an entity JSON request body into its +// PartitionKey/RowKey (if present) and typed custom properties. +// "@odata.type"-annotated properties are decoded per their declared EDM +// type; unannotated ones are inferred the same way +// azure-sdk-for-go/sdk/data/aztables's own EDMEntity.UnmarshalJSON infers +// them client-side: try Edm.Int32 first, then fall back to the JSON value's +// natural type (float64 -> Edm.Double, bool -> Edm.Boolean, string -> +// Edm.String). "Timestamp" is silently ignored (server-managed; a client- +// supplied value never overwrites it -- the server always wins). Any +// "odata.*"/"@odata.type" metadata key is skipped. +func decodeEntityBody(body []byte) (string, string, bool, bool, map[string]EntityProperty, error) { + var raw map[string]json.RawMessage + + if err := json.Unmarshal(body, &raw); err != nil { + return "", "", false, false, nil, fmt.Errorf("%w: %w", ErrInvalidEntityProperty, err) + } + + partitionKey, hasPK, err := decodeSystemKeyProperty(raw, partitionKeyProperty) + if err != nil { + return "", "", false, false, nil, err + } + + rowKey, hasRK, err := decodeSystemKeyProperty(raw, rowKeyProperty) + if err != nil { + return "", "", false, false, nil, err + } + + props, err := decodeCustomProperties(raw) + if err != nil { + return "", "", false, false, nil, err + } + + return partitionKey, rowKey, hasPK, hasRK, props, nil +} + +// decodeSystemKeyProperty extracts the string-valued system property name +// (PartitionKey or RowKey) from raw, if present. +func decodeSystemKeyProperty(raw map[string]json.RawMessage, name string) (string, bool, error) { + rawVal, ok := raw[name] + if !ok { + return "", false, nil + } + + var s string + if err := json.Unmarshal(rawVal, &s); err != nil { + return "", false, fmt.Errorf("%w: %s must be a string", ErrInvalidEntityProperty, name) + } + + return s, true, nil +} + +// isSystemOrMetadataKey reports whether key is a system property +// (PartitionKey/RowKey/Timestamp), an "@odata.type" annotation, or other +// "odata."-prefixed metadata -- none of which decodeCustomProperties treats +// as a user-defined entity property. +func isSystemOrMetadataKey(key string) bool { + if strings.HasSuffix(key, "@odata.type") || strings.HasPrefix(key, "odata.") { + return true + } + + switch key { + case partitionKeyProperty, rowKeyProperty, timestampProperty: + return true + default: + return false + } +} + +// decodeCustomProperties decodes every user-defined (non-system, +// non-metadata) property in raw into its typed EntityProperty. +func decodeCustomProperties(raw map[string]json.RawMessage) (map[string]EntityProperty, error) { + props := make(map[string]EntityProperty, len(raw)) + + for key, rawVal := range raw { + if isSystemOrMetadataKey(key) || string(rawVal) == "null" { + continue + } + + edmType := "" + + if annRaw, ok := raw[key+"@odata.type"]; ok { + if err := json.Unmarshal(annRaw, &edmType); err != nil { + return nil, fmt.Errorf("%w: malformed %s@odata.type", ErrInvalidEntityProperty, key) + } + } + + prop, err := decodeProperty(edmType, rawVal) + if err != nil { + return nil, err + } + + props[key] = prop + } + + return props, nil +} + +// decodeProperty decodes raw into a typed EntityProperty per its (possibly +// empty) "@odata.type" annotation value edmType. +func decodeProperty(edmType string, raw json.RawMessage) (EntityProperty, error) { + switch EdmType(edmType) { + case EdmString, EdmInt32, EdmDouble, EdmBoolean: + return decodeScalarProperty(EdmType(edmType), raw) + case EdmInt64, EdmDateTime, EdmGUID, EdmBinary: + return decodeStringEncodedProperty(EdmType(edmType), raw) + case "": + return decodeUnannotatedProperty(raw) + default: + return EntityProperty{}, fmt.Errorf("%w: unknown @odata.type %q", ErrInvalidEntityProperty, edmType) + } +} + +// decodeScalarProperty decodes the four EDM types whose wire representation +// is a bare, natively-typed JSON value (string/number/number/bool). +func decodeScalarProperty(edmType EdmType, raw json.RawMessage) (EntityProperty, error) { + switch edmType { + case EdmString: + var s string + if err := json.Unmarshal(raw, &s); err != nil { + return EntityProperty{}, fmt.Errorf("%w: not a string", ErrInvalidEntityProperty) + } + + return EntityProperty{Type: EdmString, Value: s}, nil + case EdmInt32: + var n int32 + if err := json.Unmarshal(raw, &n); err != nil { + return EntityProperty{}, fmt.Errorf("%w: not an Int32", ErrInvalidEntityProperty) + } + + return EntityProperty{Type: EdmInt32, Value: n}, nil + case EdmDouble: + var f float64 + if err := json.Unmarshal(raw, &f); err != nil { + return EntityProperty{}, fmt.Errorf("%w: not a Double", ErrInvalidEntityProperty) + } + + return EntityProperty{Type: EdmDouble, Value: f}, nil + case EdmBoolean: + var b bool + if err := json.Unmarshal(raw, &b); err != nil { + return EntityProperty{}, fmt.Errorf("%w: not a Boolean", ErrInvalidEntityProperty) + } + + return EntityProperty{Type: EdmBoolean, Value: b}, nil + default: + return EntityProperty{}, fmt.Errorf("%w: unsupported scalar type %q", ErrInvalidEntityProperty, edmType) + } +} + +// decodeStringEncodedProperty decodes the four EDM types whose wire +// representation is always a JSON string carrying an encoded value (decimal +// digits, an RFC3339 timestamp, a UUID, or base64), requiring the +// "@odata.type" annotation to disambiguate from Edm.String. +func decodeStringEncodedProperty(edmType EdmType, raw json.RawMessage) (EntityProperty, error) { + var s string + if err := json.Unmarshal(raw, &s); err != nil { + return EntityProperty{}, fmt.Errorf("%w: %s must be a string", ErrInvalidEntityProperty, edmType) + } + + switch edmType { + case EdmInt64: + n, err := strconv.ParseInt(s, 10, 64) + if err != nil { + return EntityProperty{}, fmt.Errorf("%w: invalid Int64 %q", ErrInvalidEntityProperty, s) + } + + return EntityProperty{Type: EdmInt64, Value: n}, nil + case EdmDateTime: + t, err := time.Parse(entityTimeLayout, s) + if err != nil { + return EntityProperty{}, fmt.Errorf("%w: invalid DateTime %q", ErrInvalidEntityProperty, s) + } + + return EntityProperty{Type: EdmDateTime, Value: t}, nil + case EdmGUID: + return EntityProperty{Type: EdmGUID, Value: s}, nil + case EdmBinary: + b, err := base64.StdEncoding.DecodeString(s) + if err != nil { + return EntityProperty{}, fmt.Errorf("%w: invalid base64 %q", ErrInvalidEntityProperty, s) + } + + return EntityProperty{Type: EdmBinary, Value: b}, nil + default: + return EntityProperty{}, fmt.Errorf("%w: unsupported string-encoded type %q", ErrInvalidEntityProperty, edmType) + } +} + +// decodeUnannotatedProperty infers a bare (unannotated) property's EDM type, +// mirroring aztables' own client-side inference exactly: try Int32 first +// (so a decimal-point-free number becomes Edm.Int32), then fall back to the +// JSON value's natural Go type. +func decodeUnannotatedProperty(raw json.RawMessage) (EntityProperty, error) { + var i32 int32 + if err := json.Unmarshal(raw, &i32); err == nil { + return EntityProperty{Type: EdmInt32, Value: i32}, nil + } + + var f float64 + if err := json.Unmarshal(raw, &f); err == nil { + return EntityProperty{Type: EdmDouble, Value: f}, nil + } + + var b bool + if err := json.Unmarshal(raw, &b); err == nil { + return EntityProperty{Type: EdmBoolean, Value: b}, nil + } + + var s string + if err := json.Unmarshal(raw, &s); err == nil { + return EntityProperty{Type: EdmString, Value: s}, nil + } + + return EntityProperty{}, fmt.Errorf("%w: unsupported JSON value %s", ErrInvalidEntityProperty, string(raw)) +} + +// --- Entity body encode (EntityProperty map -> response) --- + +// encodeEntity builds an entity's OData JSON response body at the given +// metadata level. select, if non-empty, is a comma-separated $select +// projection list; PartitionKey/RowKey/Timestamp are always included +// regardless of select (they're the entity's identity and are cheap/always +// safe to return -- see PARITY.md's $select note), while custom properties +// are filtered to the requested names. +func (h *Handler) encodeEntity(info EntityInfo, table, level, selectParam string) map[string]any { + m := map[string]any{} + + if level != odataLevelNoMetadata { + endpoint := h.serviceEndpoint() + m["odata.metadata"] = endpoint + "/$metadata#" + table + "/@Element" + m["odata.etag"] = info.ETag + + if level == odataLevelFullMetadata { + keyPredicate := partitionKeyProperty + "='" + escapeODataKey(info.PartitionKey) + + "'," + rowKeyProperty + "='" + escapeODataKey(info.RowKey) + "'" + m["odata.type"] = devstoreAccountName + "." + table + m["odata.id"] = endpoint + "/" + table + "(" + keyPredicate + ")" + m["odata.editLink"] = table + "(" + keyPredicate + ")" + } + } + + m[partitionKeyProperty] = info.PartitionKey + m[rowKeyProperty] = info.RowKey + m[timestampProperty] = info.Timestamp.UTC().Format(entityTimeLayout) + + selected := selectSet(selectParam) + for name, prop := range info.Properties { + if selected != nil && !selected[name] { + continue + } + + encodePropertyInto(m, name, prop) + } + + return m +} + +func selectSet(selectParam string) map[string]bool { + if selectParam == "" { + return nil + } + + names := strings.Split(selectParam, ",") + set := make(map[string]bool, len(names)) + + for _, n := range names { + set[strings.TrimSpace(n)] = true + } + + return set +} + +//nolint:cyclop // per-EDM-type dispatch; splitting would obscure it +func encodePropertyInto(m map[string]any, name string, prop EntityProperty) { + switch prop.Type { + case EdmString: + if s, ok := prop.Value.(string); ok { + m[name] = s + } + case EdmInt32: + if n, ok := prop.Value.(int32); ok { + m[name] = n + } + case EdmDouble: + if f, ok := prop.Value.(float64); ok { + m[name] = f + } + case EdmBoolean: + if b, ok := prop.Value.(bool); ok { + m[name] = b + } + case EdmInt64: + if n, ok := prop.Value.(int64); ok { + m[name] = strconv.FormatInt(n, 10) + m[name+"@odata.type"] = string(EdmInt64) + } + case EdmDateTime: + if t, ok := prop.Value.(time.Time); ok { + m[name] = t.UTC().Format(entityTimeLayout) + m[name+"@odata.type"] = string(EdmDateTime) + } + case EdmGUID: + if s, ok := prop.Value.(string); ok { + m[name] = s + m[name+"@odata.type"] = string(EdmGUID) + } + case EdmBinary: + if b, ok := prop.Value.([]byte); ok { + m[name] = base64.StdEncoding.EncodeToString(b) + m[name+"@odata.type"] = string(EdmBinary) + } + } +} + +// --- HTTP handlers --- + +func (h *Handler) insertEntity(c *echo.Context, table string) error { + r := c.Request() + + body, err := httputils.ReadBody(r) + if err != nil { + return h.writeError(c, http.StatusInternalServerError, "InternalError", "Failed to read request body.") + } + + partitionKey, rowKey, hasPK, hasRK, props, decErr := decodeEntityBody(body) + if decErr != nil { + return h.writeError(c, http.StatusBadRequest, "InvalidInput", "One of the request inputs is not valid.") + } + + if !hasPK || !hasRK { + return h.writeError(c, http.StatusBadRequest, "InvalidInput", + "The values are not specified for all the key properties of the entity.") + } + + info, err := h.Backend.InsertEntity(table, partitionKey, rowKey, props) + + switch { + case err == nil: + case errors.Is(err, ErrTableNotFound): + return h.writeTableNotFoundError(c) + case errors.Is(err, ErrEntityAlreadyExists): + return h.writeError(c, http.StatusConflict, "EntityAlreadyExists", "The specified entity already exists.") + default: + return h.writeError(c, http.StatusInternalServerError, "InternalError", err.Error()) + } + + c.Response().Header().Set("ETag", info.ETag) + + if r.Header.Get("Prefer") == preferReturnNoContent { + c.Response().Header().Set("Preference-Applied", preferReturnNoContent) + + return c.NoContent(http.StatusNoContent) + } + + level := odataLevelFromAccept(r.Header.Get("Accept")) + + return h.writeJSON(c, http.StatusCreated, h.encodeEntity(info, table, level, "")) +} + +func (h *Handler) getEntity(c *echo.Context, table, partitionKey, rowKey string) error { + info, err := h.Backend.GetEntity(table, partitionKey, rowKey) + + switch { + case err == nil: + case errors.Is(err, ErrTableNotFound): + return h.writeTableNotFoundError(c) + case errors.Is(err, ErrEntityNotFound): + return h.writeResourceNotFoundError(c) + default: + return h.writeError(c, http.StatusInternalServerError, "InternalError", err.Error()) + } + + c.Response().Header().Set("ETag", info.ETag) + + level := odataLevelFromAccept(c.Request().Header.Get("Accept")) + + return h.writeJSON(c, http.StatusOK, h.encodeEntity(info, table, level, c.QueryParam("$select"))) +} + +func (h *Handler) queryEntities(c *echo.Context, table string) error { + top, topErr := parseTop(c.QueryParam("$top")) + if topErr != nil { + return h.writeError(c, http.StatusBadRequest, "InvalidInput", "The value for $top is invalid.") + } + + var filter Node + + if filterParam := c.QueryParam("$filter"); filterParam != "" { + node, parseErr := ParseFilter(filterParam) + if parseErr != nil { + return h.writeError(c, http.StatusBadRequest, "InvalidInput", "The specified $filter is invalid.") + } + + filter = node + } + + infos, err := h.Backend.QueryEntities(table, filter, top) + if err != nil { + return h.writeTableNotFoundError(c) + } + + level := odataLevelFromAccept(c.Request().Header.Get("Accept")) + selectParam := c.QueryParam("$select") + + values := make([]map[string]any, 0, len(infos)) + for _, info := range infos { + values = append(values, h.encodeEntity(info, table, level, selectParam)) + } + + return h.writeJSON(c, http.StatusOK, map[string]any{"value": values}) +} + +// parseTop parses the $top query parameter, defaulting to 0 (unlimited) +// when absent and clamping (never erroring on) an oversized value, per +// maxQueryTop's doc comment. A negative value is rejected. +func parseTop(raw string) (int, error) { + if raw == "" { + return 0, nil + } + + n, err := strconv.Atoi(raw) + if err != nil || n < 0 { + return 0, ErrInvalidEntityProperty + } + + if n > maxQueryTop { + n = maxQueryTop + } + + return n, nil +} + +func (h *Handler) replaceEntity(c *echo.Context, table, partitionKey, rowKey string) error { + return h.putOrMergeEntity(c, table, partitionKey, rowKey, h.Backend.ReplaceEntity) +} + +func (h *Handler) mergeEntity(c *echo.Context, table, partitionKey, rowKey string) error { + return h.putOrMergeEntity(c, table, partitionKey, rowKey, h.Backend.MergeEntity) +} + +// mutateEntityFunc is the shape ReplaceEntity/MergeEntity share, so +// putOrMergeEntity can dispatch to either one generically. +type mutateEntityFunc func( + table, partitionKey, rowKey string, props map[string]EntityProperty, ifMatch string, +) (EntityInfo, error) + +func (h *Handler) putOrMergeEntity( + c *echo.Context, table, partitionKey, rowKey string, mutate mutateEntityFunc, +) error { + r := c.Request() + + body, err := httputils.ReadBody(r) + if err != nil { + return h.writeError(c, http.StatusInternalServerError, "InternalError", "Failed to read request body.") + } + + _, _, _, _, props, decErr := decodeEntityBody(body) + if decErr != nil { + return h.writeError(c, http.StatusBadRequest, "InvalidInput", "One of the request inputs is not valid.") + } + + ifMatch := r.Header.Get("If-Match") + + info, mutateErr := mutate(table, partitionKey, rowKey, props, ifMatch) + + switch { + case mutateErr == nil: + case errors.Is(mutateErr, ErrTableNotFound): + return h.writeTableNotFoundError(c) + case errors.Is(mutateErr, ErrEntityNotFound): + return h.writeResourceNotFoundError(c) + case errors.Is(mutateErr, ErrETagMismatch): + return h.writeError(c, http.StatusPreconditionFailed, "UpdateConditionNotSatisfied", + "The update condition specified in the request was not satisfied.") + default: + return h.writeError(c, http.StatusInternalServerError, "InternalError", mutateErr.Error()) + } + + c.Response().Header().Set("ETag", info.ETag) + + return c.NoContent(http.StatusNoContent) +} + +func (h *Handler) deleteEntity(c *echo.Context, table, partitionKey, rowKey string) error { + ifMatch := c.Request().Header.Get("If-Match") + if ifMatch == "" { + return h.writeError(c, http.StatusBadRequest, "InvalidInput", "An If-Match header is required for delete.") + } + + err := h.Backend.DeleteEntity(table, partitionKey, rowKey, ifMatch) + + switch { + case err == nil: + return c.NoContent(http.StatusNoContent) + case errors.Is(err, ErrTableNotFound): + return h.writeTableNotFoundError(c) + case errors.Is(err, ErrEntityNotFound): + return h.writeResourceNotFoundError(c) + case errors.Is(err, ErrETagMismatch): + return h.writeError(c, http.StatusPreconditionFailed, "UpdateConditionNotSatisfied", + "The update condition specified in the request was not satisfied.") + default: + return h.writeError(c, http.StatusInternalServerError, "InternalError", err.Error()) + } +} + +// writeResourceNotFoundError maps a missing-entity StorageBackend error to +// the corresponding Azure error code/status. +func (h *Handler) writeResourceNotFoundError(c *echo.Context) error { + return h.writeError(c, http.StatusNotFound, "ResourceNotFound", "The specified resource does not exist.") +} diff --git a/services/azuretable/entity_ops_test.go b/services/azuretable/entity_ops_test.go new file mode 100644 index 0000000000..1d8fcc5005 --- /dev/null +++ b/services/azuretable/entity_ops_test.go @@ -0,0 +1,601 @@ +package azuretable_test + +import ( + "encoding/base64" + "encoding/json" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/labstack/echo/v5" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/blackbirdworks/gopherstack/services/azuretable" +) + +func TestInsertEntity(t *testing.T) { + t.Parallel() + + t.Run("success", func(t *testing.T) { + t.Parallel() + + h := newTestHandler(t) + doRequest(t, h, http.MethodPost, "/"+testAccount+"/Tables", []byte(`{"TableName":"mytable"}`)) + + body := []byte(`{"PartitionKey":"p","RowKey":"r","Name":"hi"}`) + rec := doRequest(t, h, http.MethodPost, "/"+testAccount+"/mytable", body) + + require.Equal(t, http.StatusCreated, rec.Code) + assert.NotEmpty(t, rec.Header().Get("ETag")) + + var got map[string]any + require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &got)) + assert.Equal(t, "p", got["PartitionKey"]) + assert.Equal(t, "r", got["RowKey"]) + assert.Equal(t, "hi", got["Name"]) + assert.NotEmpty(t, got["Timestamp"]) + }) + + t.Run("prefer_return_no_content", func(t *testing.T) { + t.Parallel() + + h := newTestHandler(t) + doRequest(t, h, http.MethodPost, "/"+testAccount+"/Tables", []byte(`{"TableName":"mytable"}`)) + + req := httptest.NewRequest( + http.MethodPost, "/"+testAccount+"/mytable", strings.NewReader(`{"PartitionKey":"p","RowKey":"r"}`), + ) + req.Header.Set("Prefer", "return-no-content") + + e := echo.New() + rec := httptest.NewRecorder() + c := e.NewContext(req, rec) + require.NoError(t, h.Handler()(c)) + + assert.Equal(t, http.StatusNoContent, rec.Code) + assert.Equal(t, "return-no-content", rec.Header().Get("Preference-Applied")) + assert.NotEmpty(t, rec.Header().Get("ETag")) + }) + + t.Run("table_not_found", func(t *testing.T) { + t.Parallel() + + h := newTestHandler(t) + rec := doRequest( + t, + h, + http.MethodPost, + "/"+testAccount+"/nosuchtable", + []byte(`{"PartitionKey":"p","RowKey":"r"}`), + ) + assert.Equal(t, http.StatusNotFound, rec.Code) + assert.Equal(t, "TableNotFound", rec.Header().Get("X-Ms-Error-Code")) + }) + + t.Run("duplicate_conflict", func(t *testing.T) { + t.Parallel() + + h := newTestHandler(t) + doRequest(t, h, http.MethodPost, "/"+testAccount+"/Tables", []byte(`{"TableName":"mytable"}`)) + body := []byte(`{"PartitionKey":"p","RowKey":"r"}`) + doRequest(t, h, http.MethodPost, "/"+testAccount+"/mytable", body) + + rec := doRequest(t, h, http.MethodPost, "/"+testAccount+"/mytable", body) + assert.Equal(t, http.StatusConflict, rec.Code) + assert.Equal(t, "EntityAlreadyExists", rec.Header().Get("X-Ms-Error-Code")) + }) + + t.Run("missing_keys_rejected", func(t *testing.T) { + t.Parallel() + + h := newTestHandler(t) + doRequest(t, h, http.MethodPost, "/"+testAccount+"/Tables", []byte(`{"TableName":"mytable"}`)) + + rec := doRequest(t, h, http.MethodPost, "/"+testAccount+"/mytable", []byte(`{"RowKey":"r"}`)) + assert.Equal(t, http.StatusBadRequest, rec.Code) + assert.Equal(t, "InvalidInput", rec.Header().Get("X-Ms-Error-Code")) + }) + + t.Run("empty_keys_accepted", func(t *testing.T) { + t.Parallel() + + h := newTestHandler(t) + doRequest(t, h, http.MethodPost, "/"+testAccount+"/Tables", []byte(`{"TableName":"mytable"}`)) + + rec := doRequest(t, h, http.MethodPost, "/"+testAccount+"/mytable", []byte(`{"PartitionKey":"","RowKey":""}`)) + assert.Equal(t, http.StatusCreated, rec.Code) + }) + + t.Run("malformed_body", func(t *testing.T) { + t.Parallel() + + h := newTestHandler(t) + doRequest(t, h, http.MethodPost, "/"+testAccount+"/Tables", []byte(`{"TableName":"mytable"}`)) + + rec := doRequest(t, h, http.MethodPost, "/"+testAccount+"/mytable", []byte(`not json`)) + assert.Equal(t, http.StatusBadRequest, rec.Code) + }) +} + +// TestEntity_EDMTypeRoundTrip covers a round trip per supported EDM type: +// insert an entity carrying one property of that type, then GET it back and +// assert both the value and (where applicable) the "@odata.type" annotation +// survive. +func TestEntity_EDMTypeRoundTrip(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + insertBody string + wantValue any + wantAnn string + annPresent bool + }{ + {name: "string", insertBody: `"Prop":"hello"`, wantValue: "hello"}, + {name: "int32", insertBody: `"Prop":42`, wantValue: float64(42)}, + { + name: "int64", insertBody: `"Prop":"9223372036854775807","Prop@odata.type":"Edm.Int64"`, + wantValue: "9223372036854775807", wantAnn: "Edm.Int64", annPresent: true, + }, + {name: "double_fractional", insertBody: `"Prop":3.14`, wantValue: 3.14}, + {name: "boolean", insertBody: `"Prop":true`, wantValue: true}, + { + name: "datetime", insertBody: `"Prop":"2024-01-02T03:04:05.1234567Z","Prop@odata.type":"Edm.DateTime"`, + wantValue: "2024-01-02T03:04:05.1234567Z", wantAnn: "Edm.DateTime", annPresent: true, + }, + { + name: "guid", insertBody: `"Prop":"550e8400-e29b-41d4-a716-446655440000","Prop@odata.type":"Edm.Guid"`, + wantValue: "550e8400-e29b-41d4-a716-446655440000", wantAnn: "Edm.Guid", annPresent: true, + }, + { + name: "binary", + insertBody: `"Prop":"` + base64.StdEncoding.EncodeToString([]byte("hi there")) + + `","Prop@odata.type":"Edm.Binary"`, + wantValue: base64.StdEncoding.EncodeToString([]byte("hi there")), wantAnn: "Edm.Binary", annPresent: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + h := newTestHandler(t) + doRequest(t, h, http.MethodPost, "/"+testAccount+"/Tables", []byte(`{"TableName":"mytable"}`)) + + insertBody := []byte(`{"PartitionKey":"p","RowKey":"r",` + tt.insertBody + `}`) + rec := doRequest(t, h, http.MethodPost, "/"+testAccount+"/mytable", insertBody) + require.Equal(t, http.StatusCreated, rec.Code, rec.Body.String()) + + rec = doRequest(t, h, http.MethodGet, + "/"+testAccount+"/mytable(PartitionKey='p',RowKey='r')", nil) + require.Equal(t, http.StatusOK, rec.Code) + + var got map[string]any + require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &got)) + assert.Equal(t, tt.wantValue, got["Prop"], tt.name) + + if tt.annPresent { + assert.Equal(t, tt.wantAnn, got["Prop@odata.type"], tt.name) + } else { + assert.NotContains(t, got, "Prop@odata.type", tt.name) + } + }) + } +} + +func TestGetEntity(t *testing.T) { + t.Parallel() + + t.Run("not_found", func(t *testing.T) { + t.Parallel() + + h := newTestHandler(t) + doRequest(t, h, http.MethodPost, "/"+testAccount+"/Tables", []byte(`{"TableName":"mytable"}`)) + + rec := doRequest(t, h, http.MethodGet, "/"+testAccount+"/mytable(PartitionKey='p',RowKey='r')", nil) + assert.Equal(t, http.StatusNotFound, rec.Code) + assert.Equal(t, "ResourceNotFound", rec.Header().Get("X-Ms-Error-Code")) + }) + + t.Run("table_not_found", func(t *testing.T) { + t.Parallel() + + h := newTestHandler(t) + rec := doRequest(t, h, http.MethodGet, "/"+testAccount+"/nosuch(PartitionKey='p',RowKey='r')", nil) + assert.Equal(t, http.StatusNotFound, rec.Code) + assert.Equal(t, "TableNotFound", rec.Header().Get("X-Ms-Error-Code")) + }) + + t.Run("select_projection", func(t *testing.T) { + t.Parallel() + + h := newTestHandler(t) + doRequest(t, h, http.MethodPost, "/"+testAccount+"/Tables", []byte(`{"TableName":"mytable"}`)) + doRequest(t, h, http.MethodPost, "/"+testAccount+"/mytable", + []byte(`{"PartitionKey":"p","RowKey":"r","A":"x","B":"y"}`)) + + rec := doRequest(t, h, http.MethodGet, + "/"+testAccount+"/mytable(PartitionKey='p',RowKey='r')?$select=A", nil) + require.Equal(t, http.StatusOK, rec.Code) + + var got map[string]any + require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &got)) + assert.Equal(t, "x", got["A"]) + assert.NotContains(t, got, "B") + assert.Contains(t, got, "PartitionKey") + assert.Contains(t, got, "RowKey") + }) +} + +func TestQueryEntities(t *testing.T) { + t.Parallel() + + setup := func(t *testing.T) *azuretable.Handler { + t.Helper() + + h := newTestHandler(t) + doRequest(t, h, http.MethodPost, "/"+testAccount+"/Tables", []byte(`{"TableName":"mytable"}`)) + doRequest(t, h, http.MethodPost, "/"+testAccount+"/mytable", + []byte(`{"PartitionKey":"p1","RowKey":"r1","Age":10}`)) + doRequest(t, h, http.MethodPost, "/"+testAccount+"/mytable", + []byte(`{"PartitionKey":"p2","RowKey":"r1","Age":20}`)) + + return h + } + + t.Run("no_filter_returns_all_sorted", func(t *testing.T) { + t.Parallel() + + h := setup(t) + rec := doRequest(t, h, http.MethodGet, "/"+testAccount+"/mytable()", nil) + require.Equal(t, http.StatusOK, rec.Code) + + var body struct { + Value []map[string]any `json:"value"` + } + require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &body)) + require.Len(t, body.Value, 2) + assert.Equal(t, "p1", body.Value[0]["PartitionKey"]) + assert.Equal(t, "p2", body.Value[1]["PartitionKey"]) + }) + + t.Run("filter_eq", func(t *testing.T) { + t.Parallel() + + h := setup(t) + rec := doRequest(t, h, http.MethodGet, + "/"+testAccount+"/mytable()?$filter=Age%20eq%2020", nil) + require.Equal(t, http.StatusOK, rec.Code) + + var body struct { + Value []map[string]any `json:"value"` + } + require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &body)) + require.Len(t, body.Value, 1) + assert.Equal(t, "p2", body.Value[0]["PartitionKey"]) + }) + + t.Run("filter_parse_error", func(t *testing.T) { + t.Parallel() + + h := setup(t) + rec := doRequest(t, h, http.MethodGet, + "/"+testAccount+"/mytable()?$filter=Age%20eq", nil) + assert.Equal(t, http.StatusBadRequest, rec.Code) + assert.Equal(t, "InvalidInput", rec.Header().Get("X-Ms-Error-Code")) + }) + + t.Run("top_caps_results", func(t *testing.T) { + t.Parallel() + + h := setup(t) + rec := doRequest(t, h, http.MethodGet, "/"+testAccount+"/mytable()?$top=1", nil) + require.Equal(t, http.StatusOK, rec.Code) + + var body struct { + Value []map[string]any `json:"value"` + } + require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &body)) + assert.Len(t, body.Value, 1) + }) + + t.Run("invalid_top_rejected", func(t *testing.T) { + t.Parallel() + + h := setup(t) + rec := doRequest(t, h, http.MethodGet, "/"+testAccount+"/mytable()?$top=-5", nil) + assert.Equal(t, http.StatusBadRequest, rec.Code) + }) + + t.Run("table_not_found", func(t *testing.T) { + t.Parallel() + + h := newTestHandler(t) + rec := doRequest(t, h, http.MethodGet, "/"+testAccount+"/nosuch()", nil) + assert.Equal(t, http.StatusNotFound, rec.Code) + }) +} + +func TestReplaceEntity(t *testing.T) { + t.Parallel() + + t.Run("upsert_creates_when_absent", func(t *testing.T) { + t.Parallel() + + h := newTestHandler(t) + doRequest(t, h, http.MethodPost, "/"+testAccount+"/Tables", []byte(`{"TableName":"mytable"}`)) + + rec := doRequest(t, h, http.MethodPut, + "/"+testAccount+"/mytable(PartitionKey='p',RowKey='r')", []byte(`{"A":"x"}`)) + assert.Equal(t, http.StatusNoContent, rec.Code) + assert.NotEmpty(t, rec.Header().Get("ETag")) + }) + + t.Run("if_match_star_requires_existing", func(t *testing.T) { + t.Parallel() + + h := newTestHandler(t) + doRequest(t, h, http.MethodPost, "/"+testAccount+"/Tables", []byte(`{"TableName":"mytable"}`)) + + req := httptest.NewRequest(http.MethodPut, + "/"+testAccount+"/mytable(PartitionKey='p',RowKey='r')", strings.NewReader(`{"A":"x"}`)) + req.Header.Set("If-Match", "*") + + e := echo.New() + rec := httptest.NewRecorder() + c := e.NewContext(req, rec) + require.NoError(t, h.Handler()(c)) + assert.Equal(t, http.StatusNotFound, rec.Code) + assert.Equal(t, "ResourceNotFound", rec.Header().Get("X-Ms-Error-Code")) + }) + + t.Run("etag_mismatch_412", func(t *testing.T) { + t.Parallel() + + h := newTestHandler(t) + doRequest(t, h, http.MethodPost, "/"+testAccount+"/Tables", []byte(`{"TableName":"mytable"}`)) + doRequest(t, h, http.MethodPost, "/"+testAccount+"/mytable", + []byte(`{"PartitionKey":"p","RowKey":"r"}`)) + + req := httptest.NewRequest(http.MethodPut, + "/"+testAccount+"/mytable(PartitionKey='p',RowKey='r')", strings.NewReader(`{"A":"x"}`)) + req.Header.Set("If-Match", `W/"datetime'bogus'"`) + + e := echo.New() + rec := httptest.NewRecorder() + c := e.NewContext(req, rec) + require.NoError(t, h.Handler()(c)) + assert.Equal(t, http.StatusPreconditionFailed, rec.Code) + assert.Equal(t, "UpdateConditionNotSatisfied", rec.Header().Get("X-Ms-Error-Code")) + }) + + t.Run("replace_drops_unlisted_properties", func(t *testing.T) { + t.Parallel() + + h := newTestHandler(t) + doRequest(t, h, http.MethodPost, "/"+testAccount+"/Tables", []byte(`{"TableName":"mytable"}`)) + doRequest(t, h, http.MethodPost, "/"+testAccount+"/mytable", + []byte(`{"PartitionKey":"p","RowKey":"r","A":"x","B":"y"}`)) + + rec := doRequest(t, h, http.MethodPut, + "/"+testAccount+"/mytable(PartitionKey='p',RowKey='r')", []byte(`{"A":"z"}`)) + require.Equal(t, http.StatusNoContent, rec.Code) + + rec = doRequest(t, h, http.MethodGet, "/"+testAccount+"/mytable(PartitionKey='p',RowKey='r')", nil) + var got map[string]any + require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &got)) + assert.Equal(t, "z", got["A"]) + assert.NotContains(t, got, "B") + }) +} + +func TestMergeEntity(t *testing.T) { + t.Parallel() + + t.Run("merge_keeps_unlisted_properties", func(t *testing.T) { + t.Parallel() + + h := newTestHandler(t) + doRequest(t, h, http.MethodPost, "/"+testAccount+"/Tables", []byte(`{"TableName":"mytable"}`)) + doRequest(t, h, http.MethodPost, "/"+testAccount+"/mytable", + []byte(`{"PartitionKey":"p","RowKey":"r","A":"x","B":"y"}`)) + + req := httptest.NewRequest(http.MethodPatch, + "/"+testAccount+"/mytable(PartitionKey='p',RowKey='r')", strings.NewReader(`{"A":"z"}`)) + e := echo.New() + rec := httptest.NewRecorder() + c := e.NewContext(req, rec) + require.NoError(t, h.Handler()(c)) + require.Equal(t, http.StatusNoContent, rec.Code) + + rec2 := doRequest(t, h, http.MethodGet, "/"+testAccount+"/mytable(PartitionKey='p',RowKey='r')", nil) + var got map[string]any + require.NoError(t, json.Unmarshal(rec2.Body.Bytes(), &got)) + assert.Equal(t, "z", got["A"]) + assert.Equal(t, "y", got["B"]) + }) + + t.Run("literal_merge_method", func(t *testing.T) { + t.Parallel() + + h := newTestHandler(t) + doRequest(t, h, http.MethodPost, "/"+testAccount+"/Tables", []byte(`{"TableName":"mytable"}`)) + doRequest(t, h, http.MethodPost, "/"+testAccount+"/mytable", + []byte(`{"PartitionKey":"p","RowKey":"r","A":"x"}`)) + + req := httptest.NewRequest("MERGE", + "/"+testAccount+"/mytable(PartitionKey='p',RowKey='r')", strings.NewReader(`{"A":"z"}`)) + e := echo.New() + rec := httptest.NewRecorder() + c := e.NewContext(req, rec) + require.NoError(t, h.Handler()(c)) + assert.Equal(t, http.StatusNoContent, rec.Code) + }) + + t.Run("upsert_creates_when_absent", func(t *testing.T) { + t.Parallel() + + h := newTestHandler(t) + doRequest(t, h, http.MethodPost, "/"+testAccount+"/Tables", []byte(`{"TableName":"mytable"}`)) + + req := httptest.NewRequest(http.MethodPatch, + "/"+testAccount+"/mytable(PartitionKey='p',RowKey='r')", strings.NewReader(`{"A":"x"}`)) + e := echo.New() + rec := httptest.NewRecorder() + c := e.NewContext(req, rec) + require.NoError(t, h.Handler()(c)) + assert.Equal(t, http.StatusNoContent, rec.Code) + }) + + t.Run("tunneled_via_x_http_method_override_on_post", func(t *testing.T) { + t.Parallel() + + h := newTestHandler(t) + doRequest(t, h, http.MethodPost, "/"+testAccount+"/Tables", []byte(`{"TableName":"mytable"}`)) + doRequest(t, h, http.MethodPost, "/"+testAccount+"/mytable", + []byte(`{"PartitionKey":"p","RowKey":"r","A":"x","B":"y"}`)) + + req := httptest.NewRequest(http.MethodPost, + "/"+testAccount+"/mytable(PartitionKey='p',RowKey='r')", strings.NewReader(`{"A":"z"}`)) + req.Header.Set("X-Http-Method", "MERGE") + e := echo.New() + rec := httptest.NewRecorder() + c := e.NewContext(req, rec) + require.NoError(t, h.Handler()(c)) + require.Equal(t, http.StatusNoContent, rec.Code) + + // Merge semantics, not replace: B must survive. + rec2 := doRequest(t, h, http.MethodGet, "/"+testAccount+"/mytable(PartitionKey='p',RowKey='r')", nil) + var got map[string]any + require.NoError(t, json.Unmarshal(rec2.Body.Bytes(), &got)) + assert.Equal(t, "z", got["A"]) + assert.Equal(t, "y", got["B"]) + + // The tunneled request must also report as MergeEntity for metrics. + assert.Equal(t, "MergeEntity", h.ExtractOperation(c)) + }) + + t.Run("tunneled_via_x_http_method_override_on_put", func(t *testing.T) { + t.Parallel() + + h := newTestHandler(t) + doRequest(t, h, http.MethodPost, "/"+testAccount+"/Tables", []byte(`{"TableName":"mytable"}`)) + doRequest(t, h, http.MethodPost, "/"+testAccount+"/mytable", + []byte(`{"PartitionKey":"p","RowKey":"r","A":"x","B":"y"}`)) + + req := httptest.NewRequest(http.MethodPut, + "/"+testAccount+"/mytable(PartitionKey='p',RowKey='r')", strings.NewReader(`{"A":"z"}`)) + req.Header.Set("X-Http-Method", "merge") // lower-case: comparison is case-insensitive + e := echo.New() + rec := httptest.NewRecorder() + c := e.NewContext(req, rec) + require.NoError(t, h.Handler()(c)) + require.Equal(t, http.StatusNoContent, rec.Code) + + rec2 := doRequest(t, h, http.MethodGet, "/"+testAccount+"/mytable(PartitionKey='p',RowKey='r')", nil) + var got map[string]any + require.NoError(t, json.Unmarshal(rec2.Body.Bytes(), &got)) + assert.Equal(t, "z", got["A"]) + assert.Equal(t, "y", got["B"], "merge semantics must apply, not PUT's own replace semantics") + }) + + t.Run("x_http_method_override_never_tunnels_delete", func(t *testing.T) { + t.Parallel() + + h := newTestHandler(t) + doRequest(t, h, http.MethodPost, "/"+testAccount+"/Tables", []byte(`{"TableName":"mytable"}`)) + doRequest(t, h, http.MethodPost, "/"+testAccount+"/mytable", + []byte(`{"PartitionKey":"p","RowKey":"r","A":"x"}`)) + + // A bogus X-Http-Method: DELETE on a GET must NOT be honored -- only + // POST/PUT/PATCH carrying an override naming MERGE is ever tunneled. + req := httptest.NewRequest(http.MethodGet, + "/"+testAccount+"/mytable(PartitionKey='p',RowKey='r')", http.NoBody) + req.Header.Set("X-Http-Method", "DELETE") + e := echo.New() + rec := httptest.NewRecorder() + c := e.NewContext(req, rec) + require.NoError(t, h.Handler()(c)) + + // Still a plain GET: 200 with the entity, not a 204 delete. + assert.Equal(t, http.StatusOK, rec.Code) + + rec2 := doRequest(t, h, http.MethodGet, "/"+testAccount+"/mytable(PartitionKey='p',RowKey='r')", nil) + assert.Equal(t, http.StatusOK, rec2.Code, "entity must not have been deleted") + }) +} + +func TestDeleteEntity(t *testing.T) { + t.Parallel() + + t.Run("missing_if_match_rejected", func(t *testing.T) { + t.Parallel() + + h := newTestHandler(t) + doRequest(t, h, http.MethodPost, "/"+testAccount+"/Tables", []byte(`{"TableName":"mytable"}`)) + doRequest(t, h, http.MethodPost, "/"+testAccount+"/mytable", + []byte(`{"PartitionKey":"p","RowKey":"r"}`)) + + rec := doRequest(t, h, http.MethodDelete, "/"+testAccount+"/mytable(PartitionKey='p',RowKey='r')", nil) + assert.Equal(t, http.StatusBadRequest, rec.Code) + assert.Equal(t, "InvalidInput", rec.Header().Get("X-Ms-Error-Code")) + }) + + t.Run("if_match_star_succeeds", func(t *testing.T) { + t.Parallel() + + h := newTestHandler(t) + doRequest(t, h, http.MethodPost, "/"+testAccount+"/Tables", []byte(`{"TableName":"mytable"}`)) + doRequest(t, h, http.MethodPost, "/"+testAccount+"/mytable", + []byte(`{"PartitionKey":"p","RowKey":"r"}`)) + + req := httptest.NewRequest(http.MethodDelete, + "/"+testAccount+"/mytable(PartitionKey='p',RowKey='r')", http.NoBody) + req.Header.Set("If-Match", "*") + e := echo.New() + rec := httptest.NewRecorder() + c := e.NewContext(req, rec) + require.NoError(t, h.Handler()(c)) + assert.Equal(t, http.StatusNoContent, rec.Code) + + rec2 := doRequest(t, h, http.MethodGet, "/"+testAccount+"/mytable(PartitionKey='p',RowKey='r')", nil) + assert.Equal(t, http.StatusNotFound, rec2.Code) + }) + + t.Run("etag_mismatch_412", func(t *testing.T) { + t.Parallel() + + h := newTestHandler(t) + doRequest(t, h, http.MethodPost, "/"+testAccount+"/Tables", []byte(`{"TableName":"mytable"}`)) + doRequest(t, h, http.MethodPost, "/"+testAccount+"/mytable", + []byte(`{"PartitionKey":"p","RowKey":"r"}`)) + + req := httptest.NewRequest(http.MethodDelete, + "/"+testAccount+"/mytable(PartitionKey='p',RowKey='r')", http.NoBody) + req.Header.Set("If-Match", `W/"datetime'bogus'"`) + e := echo.New() + rec := httptest.NewRecorder() + c := e.NewContext(req, rec) + require.NoError(t, h.Handler()(c)) + assert.Equal(t, http.StatusPreconditionFailed, rec.Code) + }) + + t.Run("not_found", func(t *testing.T) { + t.Parallel() + + h := newTestHandler(t) + doRequest(t, h, http.MethodPost, "/"+testAccount+"/Tables", []byte(`{"TableName":"mytable"}`)) + + req := httptest.NewRequest(http.MethodDelete, + "/"+testAccount+"/mytable(PartitionKey='p',RowKey='r')", http.NoBody) + req.Header.Set("If-Match", "*") + e := echo.New() + rec := httptest.NewRecorder() + c := e.NewContext(req, rec) + require.NoError(t, h.Handler()(c)) + assert.Equal(t, http.StatusNotFound, rec.Code) + assert.Equal(t, "ResourceNotFound", rec.Header().Get("X-Ms-Error-Code")) + }) +} diff --git a/services/azuretable/errors.go b/services/azuretable/errors.go new file mode 100644 index 0000000000..4a9812bded --- /dev/null +++ b/services/azuretable/errors.go @@ -0,0 +1,41 @@ +package azuretable + +import "errors" + +// Sentinel errors for Azure Table Storage operations. +var ( + ErrTableNotFound = errors.New("azuretable: table not found") + ErrTableAlreadyExists = errors.New("azuretable: table already exists") + ErrEntityNotFound = errors.New("azuretable: entity not found") + ErrEntityAlreadyExists = errors.New("azuretable: entity already exists") + ErrETagMismatch = errors.New("azuretable: etag mismatch") + + // ErrInvalidEntityKey is returned when a request omits PartitionKey or + // RowKey entirely. Empty-string keys are accepted (matching real Azure + // Table Storage); only an absent key is rejected. See entity_ops.go. + ErrInvalidEntityKey = errors.New("azuretable: PartitionKey and RowKey are required") + + // ErrInvalidEntityProperty is returned when an entity property's JSON + // value cannot be decoded under its (explicit or inferred) EDM type. + ErrInvalidEntityProperty = errors.New("azuretable: invalid entity property value") + + // ErrFilterParse and ErrFilterTooDeep are returned by ParseFilter. A + // parse error always surfaces as 400 InvalidInput, never a panic or 500 + // -- see handler.go's queryEntities. + ErrFilterParse = errors.New("azuretable: $filter parse error") + ErrFilterTooDeep = errors.New("azuretable: $filter expression nested too deeply") + + // ErrSnapshotTableNull and ErrSnapshotEntityNull are returned by Restore + // when a snapshot's "tables" map (or a table's "Entities" map) holds a + // JSON null entry, which decodes to a nil pointer that would panic on + // first dereference if stored as-is. See persistence.go. + ErrSnapshotTableNull = errors.New("azuretable: restore snapshot: table is null") + ErrSnapshotEntityNull = errors.New("azuretable: restore snapshot: entity is null") + + // ErrSnapshotTableNameMismatch is returned by Restore when a snapshot's + // "tables" map key differs from that entry's storedTable.Name. Table + // operations all key off the map, while ListTables reads Name -- a + // mismatch would let those two views disagree about a table's identity. + // See persistence.go. + ErrSnapshotTableNameMismatch = errors.New("azuretable: restore snapshot: table map key does not match Name") +) diff --git a/services/azuretable/export_test.go b/services/azuretable/export_test.go new file mode 100644 index 0000000000..90325ebde1 --- /dev/null +++ b/services/azuretable/export_test.go @@ -0,0 +1,44 @@ +package azuretable + +import "time" + +// Exported wrappers/seams for internal state used in blackbox tests. + +// SplitPath exposes splitPath for external tests. +func SplitPath(p string) (string, string) { + return splitPath(p) +} + +// ParseResource exposes parseResource for external tests. +func ParseResource(resource string) (int, string, string) { + kind, name, inner := parseResource(resource) + + return int(kind), name, inner +} + +// ParseEntityKeyPredicate exposes parseEntityKeyPredicate for external tests. +func ParseEntityKeyPredicate(predicate string) (string, string, bool) { + return parseEntityKeyPredicate(predicate) +} + +// UnquoteODataString exposes unquoteODataString for external tests. +func UnquoteODataString(s string) (string, bool) { + return unquoteODataString(s) +} + +// SetNowFunc replaces the backend's time provider with fn for deterministic +// testing of Timestamp/ETag logic without real sleeps. +func SetNowFunc(b *InMemoryBackend, fn func() time.Time) { + b.nowFunc = fn +} + +// SetETagFunc replaces the backend's ETag derivation function with fn for +// deterministic ETag assertions. +func SetETagFunc(b *InMemoryBackend, fn func(time.Time) string) { + b.etagFunc = fn +} + +// EtagFor exposes etagFor for external tests. +func EtagFor(t time.Time) string { + return etagFor(t) +} diff --git a/services/azuretable/handler.go b/services/azuretable/handler.go new file mode 100644 index 0000000000..e74020c0a3 --- /dev/null +++ b/services/azuretable/handler.go @@ -0,0 +1,623 @@ +package azuretable + +import ( + "context" + "crypto/rand" + "encoding/json" + "errors" + "fmt" + "net" + "net/http" + "strings" + "time" + + "github.com/labstack/echo/v5" + + "github.com/blackbirdworks/gopherstack/pkgs/azureauth" + "github.com/blackbirdworks/gopherstack/pkgs/lockmetrics" + "github.com/blackbirdworks/gopherstack/pkgs/logger" + "github.com/blackbirdworks/gopherstack/pkgs/service" + "github.com/blackbirdworks/gopherstack/pkgs/telemetry" +) + +// azureTableVersion is the x-ms-version value echoed on every response. +// Unlike services/azurequeue's azureQueueVersion (a merely plausible +// version string), this one is picked deliberately: it is the literal +// x-ms-version azure-sdk-for-go/sdk/data/aztables's generated client sends +// on every request (see its zz_table_client.go), so matching it exactly +// avoids any version-skew surprises for that SDK. +const azureTableVersion = "2019-02-02" + +// dataServiceVersion is the DataServiceVersion header value real Azure Table +// Storage (and Azurite) set on every response. +const dataServiceVersion = "3.0;" + +// Operation name constants used for metrics (ExtractOperation) and +// GetSupportedOperations. +const ( + opListTables = "ListTables" + opCreateTable = "CreateTable" + opDeleteTable = "DeleteTable" + opInsertEntity = "InsertEntity" + opGetEntity = "GetEntity" + opQueryEntities = "QueryEntities" + opReplaceEntity = "ReplaceEntity" + opMergeEntity = "MergeEntity" + opDeleteEntity = "DeleteEntity" + opBatch = "Batch" + unknownOperation = "Unknown" +) + +// tablesResourceName is the fixed "Tables" collection resource segment used +// for table-CRUD operations (POST/GET //Tables, DELETE +// //Tables('name')). +const tablesResourceName = "Tables" + +// batchResourceName is the fixed "$batch" resource segment. Batch +// (multipart/mixed changesets) is explicitly out of scope -- see +// handleBatch. +const batchResourceName = "$batch" + +// mergeMethod is the literal, non-standard HTTP method aztables' generated +// client actually sends for a Merge Entity request as of some historical +// client/proxy versions (net/http's http.Method* constants don't include it +// since it isn't a registered standard method). Echo's e.Any("/*", ...) +// route in StartWorker matches it like any other method. +const mergeMethod = "MERGE" + +// xHTTPMethodOverrideHeader is the method-tunneling header some older .NET +// clients and HTTP proxies send instead of (or alongside) a literal MERGE +// method -- see resolveTunneledMergeMethod. +const xHTTPMethodOverrideHeader = "X-Http-Method" + +// OData metadata level names, negotiated via the request's Accept header +// (odataLevelFromAccept) and used throughout table_ops.go/entity_ops.go to +// vary response shape. +const ( + odataLevelNoMetadata = "nometadata" + odataLevelMinimalMetadata = "minimalmetadata" + odataLevelFullMetadata = "fullmetadata" +) + +// System entity property names, shared across path-predicate parsing +// (entity_ops.go), $filter identifier resolution (odata_filter_eval.go), and +// entity body decode/encode. +const ( + partitionKeyProperty = "PartitionKey" + rowKeyProperty = "RowKey" + timestampProperty = "Timestamp" +) + +// Handler is the Echo HTTP handler for Azure Table Storage operations. +type Handler struct { + Backend StorageBackend + srvMu *lockmetrics.RWMutex + srv *http.Server + // Endpoint is e.g. "http://127.0.0.1:10002" -- used to build + // odata.metadata/odata.id URLs in entity/table responses. + Endpoint string + // Port is the TCP port StartWorker binds. Set from Settings at Init time + // (see provider.go); defaults to DefaultPort. Like services/azurequeue, + // this is a single fixed, protocol-conventional port -- there is no + // fallback pool, so StartWorker fails fast if it's unavailable rather + // than silently binding a different port. + Port int +} + +// NewHandler creates a new Azure Table Handler. Port defaults to +// DefaultPort; callers (typically provider.go) override it from Settings. +func NewHandler(backend StorageBackend) *Handler { + return &Handler{ + Backend: backend, + Port: DefaultPort, + srvMu: lockmetrics.New("azuretable.server"), + } +} + +var ( + _ service.BackgroundWorker = (*Handler)(nil) + _ service.Shutdowner = (*Handler)(nil) + _ service.Resettable = (*Handler)(nil) +) + +// Name returns the service name. +func (h *Handler) Name() string { return "AzureTable" } + +// GetSupportedOperations returns the list of supported Azure Table operations. +func (h *Handler) GetSupportedOperations() []string { + return []string{ + opListTables, + opCreateTable, + opDeleteTable, + opInsertEntity, + opGetEntity, + opQueryEntities, + opReplaceEntity, + opMergeEntity, + opDeleteEntity, + opBatch, + } +} + +// RouteMatcher exists only to satisfy service.Registerable's interface +// contract: like services/azureblob and services/azurequeue, AzureTable +// deliberately never matches on the shared AWS single-port Router. It runs +// on its own dedicated listener started by StartWorker (see provider.go for +// the full rationale). Only RouteMatcher itself is inert, kept so *Handler +// satisfies service.Registerable. +func (h *Handler) RouteMatcher() service.Matcher { + return func(*echo.Context) bool { return false } +} + +// MatchPriority returns the routing priority for the AzureTable handler. +// Irrelevant in practice since RouteMatcher never matches; 0 (lowest) is +// the safe default. +func (h *Handler) MatchPriority() int { return 0 } + +// ExtractOperation extracts the Azure Table operation name from the +// request, for metrics labeling. +func (h *Handler) ExtractOperation(c *echo.Context) string { + return operationFor(c.Request()) +} + +// ExtractResource extracts the table/entity resource identifier from the +// request path, for metrics labeling. +func (h *Handler) ExtractResource(c *echo.Context) string { + _, resource := splitPath(c.Request().URL.Path) + + return resource +} + +// Reset clears all in-memory state from the backend. It is used by the +// POST /_gopherstack/reset endpoint for CI pipelines and rapid local development. +func (h *Handler) Reset() { + h.Backend.Reset() +} + +// Handler returns the Echo handler function for Azure Table operations. +func (h *Handler) Handler() echo.HandlerFunc { + return func(c *echo.Context) error { + r := c.Request() + resolveTunneledMergeMethod(r) + + h.setCommonHeaders(c) + h.checkAuth(r) + + account, resource := splitPath(r.URL.Path) + if account == "" || resource == "" { + return h.writeError(c, http.StatusBadRequest, "InvalidUri", + "The requested URI does not represent any resource on the server.") + } + + kind, name, inner := parseResource(resource) + + switch kind { + case resourceBatch: + return h.handleBatch(c) + case resourceTablesCollection: + return h.handleTablesCollection(c) + case resourceTablesItem: + return h.handleTablesItem(c, inner) + case resourceEntityCollection: + return h.handleEntityCollection(c, name) + case resourceEntityItem: + return h.handleEntityItem(c, name, inner) + default: + return h.writeError(c, http.StatusBadRequest, "InvalidUri", + "The requested URI does not represent any resource on the server.") + } + } +} + +// checkAuth is intentionally permissive, mirroring services/azurequeue's +// checkAuth exactly: it neither requires nor cryptographically verifies the +// Authorization header, matching this repo's permissive-by-default auth +// philosophy (see services/s3/sigv4.go). Any structurally-present +// "SharedKey ..."/"SharedKeyLite ..." header, or its absence, is accepted. +func (h *Handler) checkAuth(r *http.Request) { + authHeader := r.Header.Get("Authorization") + if authHeader == "" { + return // anonymous; accepted by design at this milestone + } + + if _, ok := azureauth.ParseAuthorizationHeader(authHeader); !ok { + // Structurally malformed; still accepted at this milestone, but + // logged so the gap is visible rather than silently swallowed. + logger.Load(r.Context()).DebugContext(r.Context(), "azuretable: malformed Authorization header accepted") + } +} + +// setCommonHeaders sets the headers real Azure SDKs expect on every +// response, success or error. +func (h *Handler) setCommonHeaders(c *echo.Context) { + hdr := c.Response().Header() + hdr.Set("X-Ms-Version", azureTableVersion) + hdr.Set("X-Ms-Request-Id", newRequestID()) + hdr.Set("Date", time.Now().UTC().Format(http.TimeFormat)) + hdr.Set("Dataserviceversion", dataServiceVersion) +} + +// newRequestID generates a plausible request-id (UUID-shaped, not +// cryptographically meaningful) for the x-ms-request-id header. +func newRequestID() string { + var buf [16]byte + if _, err := rand.Read(buf[:]); err != nil { + return "00000000-0000-0000-0000-000000000000" + } + + return fmt.Sprintf("%x-%x-%x-%x-%x", buf[0:4], buf[4:6], buf[6:8], buf[8:10], buf[10:16]) +} + +// splitPath splits an Azure Table REST path ("//") into +// its two components. is left unparsed here (see parseResource): +// it may be "Tables", "Tables('name')", "$batch", "
", "
()", or +// "
(PartitionKey='..',RowKey='..')". +func splitPath(p string) (string, string) { + p = strings.TrimPrefix(p, "/") + if p == "" { + return "", "" + } + + before, after, ok := strings.Cut(p, "/") + if !ok { + return p, "" + } + + return before, after +} + +// resourceKind classifies a parsed resource path segment. +type resourceKind int + +const ( + resourceInvalid resourceKind = iota + resourceBatch + resourceTablesCollection + resourceTablesItem + resourceEntityCollection + resourceEntityItem +) + +// parseResource classifies resource (the path segment after the account +// name) and extracts the table name and any parenthesized inner content. +// For resourceTablesItem, inner is the raw quoted table-name literal (e.g. +// "'foo'"); for resourceEntityItem, inner is the raw key predicate (e.g. +// "PartitionKey='p',RowKey='r'"); it is empty for every other kind. +func parseResource(resource string) (resourceKind, string, string) { + if resource == batchResourceName { + return resourceBatch, "", "" + } + + idx := strings.IndexByte(resource, '(') + if idx == -1 { + if resource == tablesResourceName { + return resourceTablesCollection, resource, "" + } + + return resourceEntityCollection, resource, "" + } + + if !strings.HasSuffix(resource, ")") { + return resourceInvalid, "", "" + } + + name := resource[:idx] + inner := resource[idx+1 : len(resource)-1] + + if name == tablesResourceName { + return resourceTablesItem, name, inner + } + + if inner == "" { + return resourceEntityCollection, name, "" + } + + return resourceEntityItem, name, inner +} + +// resolveTunneledMergeMethod rewrites r.Method in place to mergeMethod when +// the request carries an X-Http-Method: MERGE override header -- the +// method-tunneling convention some older .NET clients and HTTP proxies use +// instead of (or alongside) sending a literal MERGE method, which +// entityItemOperationFor/handleEntityItem already handle directly. The +// override is honored ONLY when the actual method is POST, PUT, or PATCH: +// tunneling is never allowed to turn a GET or DELETE into something else, +// which would otherwise be an auth-bypass-shaped footgun (e.g. a client or +// intermediary smuggling a mutating MERGE past something that only +// authorizes GET). The header value is compared case-insensitively since +// HTTP header values for this convention are not reliably cased. +// +// Called once at the very top of Handler()'s returned func, before +// dispatch, so every downstream consumer of r.Method -- the dispatch +// switches in handleEntityItem/entityItemOperationFor, and +// ExtractOperation's metrics labeling via operationFor -- sees the resolved +// method uniformly without duplicating this check. +func resolveTunneledMergeMethod(r *http.Request) { + switch r.Method { + case http.MethodPost, http.MethodPut, http.MethodPatch: + default: + return + } + + if strings.EqualFold(r.Header.Get(xHTTPMethodOverrideHeader), mergeMethod) { + r.Method = mergeMethod + } +} + +// operationFor determines the Azure Table operation name for a request, for +// metrics labeling. Mirrors the dispatch logic in Handler() without side +// effects. +func operationFor(r *http.Request) string { + _, resource := splitPath(r.URL.Path) + kind, _, _ := parseResource(resource) + + switch kind { + case resourceBatch: + return opBatch + case resourceTablesCollection: + return tablesCollectionOperationFor(r.Method) + case resourceTablesItem: + if r.Method == http.MethodDelete { + return opDeleteTable + } + + return unknownOperation + case resourceEntityCollection: + return entityCollectionOperationFor(r.Method) + case resourceEntityItem: + return entityItemOperationFor(r.Method) + default: + return unknownOperation + } +} + +func tablesCollectionOperationFor(method string) string { + switch method { + case http.MethodPost: + return opCreateTable + case http.MethodGet: + return opListTables + default: + return unknownOperation + } +} + +func entityCollectionOperationFor(method string) string { + switch method { + case http.MethodPost: + return opInsertEntity + case http.MethodGet: + return opQueryEntities + default: + return unknownOperation + } +} + +func entityItemOperationFor(method string) string { + switch method { + case http.MethodGet: + return opGetEntity + case http.MethodPut: + return opReplaceEntity + case http.MethodPatch, mergeMethod: + return opMergeEntity + case http.MethodDelete: + return opDeleteEntity + default: + return unknownOperation + } +} + +// serviceEndpoint returns the base URL used to build odata.metadata/odata.id +// values in responses. +func (h *Handler) serviceEndpoint() string { + if h.Endpoint != "" { + return h.Endpoint + } + + return fmt.Sprintf("http://127.0.0.1:%d", h.Port) +} + +func (h *Handler) handleTablesCollection(c *echo.Context) error { + switch c.Request().Method { + case http.MethodPost: + return h.createTable(c) + case http.MethodGet: + return h.listTables(c) + default: + return h.writeError(c, http.StatusMethodNotAllowed, "UnsupportedHttpVerb", + "The resource doesn't support the specified HTTP verb.") + } +} + +func (h *Handler) handleTablesItem(c *echo.Context, quotedName string) error { + if c.Request().Method != http.MethodDelete { + return h.writeError(c, http.StatusMethodNotAllowed, "UnsupportedHttpVerb", + "The resource doesn't support the specified HTTP verb.") + } + + return h.deleteTable(c, quotedName) +} + +func (h *Handler) handleEntityCollection(c *echo.Context, table string) error { + switch c.Request().Method { + case http.MethodPost: + return h.insertEntity(c, table) + case http.MethodGet: + return h.queryEntities(c, table) + default: + return h.writeError(c, http.StatusMethodNotAllowed, "UnsupportedHttpVerb", + "The resource doesn't support the specified HTTP verb.") + } +} + +func (h *Handler) handleEntityItem(c *echo.Context, table, keyPredicate string) error { + partitionKey, rowKey, ok := parseEntityKeyPredicate(keyPredicate) + if !ok { + return h.writeError(c, http.StatusBadRequest, "InvalidInput", + "The specified entity key predicate is invalid.") + } + + switch c.Request().Method { + case http.MethodGet: + return h.getEntity(c, table, partitionKey, rowKey) + case http.MethodPut: + return h.replaceEntity(c, table, partitionKey, rowKey) + case http.MethodPatch, mergeMethod: + return h.mergeEntity(c, table, partitionKey, rowKey) + case http.MethodDelete: + return h.deleteEntity(c, table, partitionKey, rowKey) + default: + return h.writeError(c, http.StatusMethodNotAllowed, "UnsupportedHttpVerb", + "The resource doesn't support the specified HTTP verb.") + } +} + +// handleBatch handles POST //$batch. Batch (multipart/mixed +// changesets) is explicitly out of scope for this milestone -- see +// PARITY.md's deferred section -- so this returns a clean 501 with a message +// pointing there, rather than a confusing 404/400 that would suggest a +// routing bug instead of a deliberate scope decision. +func (h *Handler) handleBatch(c *echo.Context) error { + return h.writeError(c, http.StatusNotImplemented, "NotImplemented", + "$batch (multipart/mixed changesets) is not implemented; see PARITY.md's deferred section.") +} + +// StartWorker binds the dedicated Table listener and starts serving on it. +// See provider.go's Provider doc comment for why AzureTable needs its own +// listener instead of registering into the shared AWS Router, and +// services/azurequeue's StartWorker for the synchronous-bind rationale this +// mirrors exactly. +func (h *Handler) StartWorker(ctx context.Context) error { + var listenConfig net.ListenConfig + + listener, err := listenConfig.Listen(ctx, "tcp", fmt.Sprintf(":%d", h.Port)) + if err != nil { + return fmt.Errorf("azuretable: bind port %d: %w", h.Port, err) + } + + e := echo.New() + e.Use(logger.EchoMiddleware(logger.Load(ctx))) + e.Any("/*", telemetry.WrapEchoHandler("AzureTable", h.Handler(), h)) + + srv := &http.Server{ + Handler: e, + ReadHeaderTimeout: azureTableReadHeaderTimeout, + ReadTimeout: azureTableReadTimeout, + IdleTimeout: azureTableIdleTimeout, + } + + h.srvMu.Lock("StartWorker") + h.srv = srv + h.srvMu.Unlock() + + workerCtx := logger.WithWorker(ctx, "azuretable", "listener") + log := logger.Load(workerCtx) + + log.InfoContext(workerCtx, "azuretable: starting dedicated listener", "port", h.Port) + + go func() { + if serveErr := srv.Serve(listener); serveErr != nil && !errors.Is(serveErr, http.ErrServerClosed) { + log.ErrorContext(workerCtx, "azuretable: listener stopped", "error", serveErr) + } + }() + + return nil +} + +// Timeouts for the dedicated Table http.Server. See services/azureblob's +// identical constants for the ReadTimeout/IdleTimeout Slowloris rationale. +const ( + azureTableReadHeaderTimeout = 10 * time.Second + azureTableReadTimeout = 60 * time.Second + azureTableIdleTimeout = 120 * time.Second +) + +// Shutdown stops the dedicated Table listener. A graceful Shutdown error +// (e.g. its context expiring before active connections finish) is logged +// and followed by Close, which forcibly closes the listener and any +// remaining idle/active connections; any Close error is logged too rather +// than leaving the listener to leak silently. +func (h *Handler) Shutdown(ctx context.Context) { + h.srvMu.Lock("Shutdown") + srv := h.srv + h.srv = nil + h.srvMu.Unlock() + + if srv == nil { + return + } + + log := logger.Load(ctx) + + if err := srv.Shutdown(ctx); err != nil { + log.ErrorContext(ctx, "azuretable: graceful shutdown failed, forcing close", "error", err) + + if closeErr := srv.Close(); closeErr != nil { + log.ErrorContext(ctx, "azuretable: forced close also failed", "error", closeErr) + } + } +} + +// odataLevelFromAccept picks the OData metadata level from an Accept header +// value, defaulting to "minimalmetadata" (real Azure Table Storage's own +// default) when unspecified or unrecognized. +func odataLevelFromAccept(accept string) string { + switch { + case strings.Contains(accept, "odata="+odataLevelNoMetadata): + return odataLevelNoMetadata + case strings.Contains(accept, "odata="+odataLevelFullMetadata): + return odataLevelFullMetadata + default: + return odataLevelMinimalMetadata + } +} + +// writeJSON marshals v and writes it as the response body, with a +// Content-Type reflecting the request's negotiated OData metadata level, per +// AZURE.md/PARITY.md's wire-protocol notes. +func (h *Handler) writeJSON(c *echo.Context, status int, v any) error { + level := odataLevelFromAccept(c.Request().Header.Get("Accept")) + + body, err := json.Marshal(v) + if err != nil { + return h.writeErrorNoRecurse(c, http.StatusInternalServerError, "InternalError", "Failed to marshal response.") + } + + contentType := fmt.Sprintf("application/json;odata=%s;streaming=true;charset=utf-8", level) + + return c.Blob(status, contentType, body) +} + +// writeError writes the standard Azure Table Storage JSON error envelope, +// plus the x-ms-error-code header real Azure Storage sets on every error +// response. +func (h *Handler) writeError(c *echo.Context, status int, code, message string) error { + c.Response().Header().Set("X-Ms-Error-Code", code) + + return h.writeJSON(c, status, odataErrorEnvelope{ + Error: odataErrorDetail{ + Code: code, + Message: odataErrorMessage{Lang: "en-US", Value: message}, + }, + }) +} + +// writeErrorNoRecurse is writeJSON's own marshal-failure fallback: it must +// not call back into writeJSON (which could recurse if the error envelope +// itself somehow failed to marshal, though it never does in practice since +// it's built from plain strings). +func (h *Handler) writeErrorNoRecurse(c *echo.Context, status int, code, message string) error { + c.Response().Header().Set("X-Ms-Error-Code", code) + body, _ := json.Marshal(odataErrorEnvelope{ + Error: odataErrorDetail{Code: code, Message: odataErrorMessage{Lang: "en-US", Value: message}}, + }) + + return c.Blob(status, "application/json;odata=minimalmetadata;streaming=true;charset=utf-8", body) +} + +// writeTableNotFoundError maps a StorageBackend not-found error to the +// corresponding Azure error code/status. +func (h *Handler) writeTableNotFoundError(c *echo.Context) error { + return h.writeError(c, http.StatusNotFound, "TableNotFound", "The table specified does not exist.") +} diff --git a/services/azuretable/handler_test.go b/services/azuretable/handler_test.go new file mode 100644 index 0000000000..0cab887366 --- /dev/null +++ b/services/azuretable/handler_test.go @@ -0,0 +1,438 @@ +package azuretable_test + +import ( + "bytes" + "net/http" + "net/http/httptest" + "testing" + + "github.com/labstack/echo/v5" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/blackbirdworks/gopherstack/services/azuretable" +) + +const testAccount = "devstoreaccount1" + +func newTestHandler(t *testing.T) *azuretable.Handler { + t.Helper() + + backend := azuretable.NewInMemoryBackend() + + return azuretable.NewHandler(backend) +} + +// doRequest builds an echo context for method/path (with optional body) and +// invokes the handler directly, mirroring services/azurequeue's doRequest. +func doRequest(t *testing.T, h *azuretable.Handler, method, path string, body []byte) *httptest.ResponseRecorder { + t.Helper() + + var req *http.Request + if body != nil { + req = httptest.NewRequest(method, path, bytes.NewReader(body)) + } else { + req = httptest.NewRequest(method, path, http.NoBody) + } + + e := echo.New() + rec := httptest.NewRecorder() + c := e.NewContext(req, rec) + require.NoError(t, h.Handler()(c)) + + return rec +} + +func TestHandler_CommonHeaders(t *testing.T) { + t.Parallel() + + h := newTestHandler(t) + rec := doRequest(t, h, http.MethodGet, "/"+testAccount+"/Tables", nil) + + assert.NotEmpty(t, rec.Header().Get("X-Ms-Version")) + assert.NotEmpty(t, rec.Header().Get("X-Ms-Request-Id")) + assert.NotEmpty(t, rec.Header().Get("Date")) + assert.Equal(t, "3.0;", rec.Header().Get("Dataserviceversion")) +} + +func TestHandler_InvalidURI(t *testing.T) { + t.Parallel() + + h := newTestHandler(t) + + tests := []struct { + name string + path string + }{ + {name: "root", path: "/"}, + {name: "account_only", path: "/" + testAccount}, + {name: "account_only_with_slash", path: "/" + testAccount + "/"}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + rec := doRequest(t, h, http.MethodGet, tt.path, nil) + assert.Equal(t, http.StatusBadRequest, rec.Code, tt.name) + assert.Equal(t, "InvalidUri", rec.Header().Get("X-Ms-Error-Code"), tt.name) + }) + } +} + +func TestHandler_Batch_NotImplemented(t *testing.T) { + t.Parallel() + + h := newTestHandler(t) + rec := doRequest(t, h, http.MethodPost, "/"+testAccount+"/$batch", []byte("--batch")) + + assert.Equal(t, http.StatusNotImplemented, rec.Code) + assert.Equal(t, "NotImplemented", rec.Header().Get("X-Ms-Error-Code")) + assert.Contains(t, rec.Body.String(), "PARITY.md") +} + +func TestHandler_MethodNotAllowed(t *testing.T) { + t.Parallel() + + h := newTestHandler(t) + + tests := []struct { + name string + method string + path string + }{ + {name: "tables_collection_delete", method: http.MethodDelete, path: "/" + testAccount + "/Tables"}, + {name: "tables_item_get", method: http.MethodGet, path: "/" + testAccount + "/Tables('foo')"}, + {name: "entity_collection_put", method: http.MethodPut, path: "/" + testAccount + "/mytable"}, + { + name: "entity_item_post", method: http.MethodPost, + path: "/" + testAccount + "/mytable(PartitionKey='p',RowKey='r')", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + rec := doRequest(t, h, tt.method, tt.path, nil) + assert.Equal(t, http.StatusMethodNotAllowed, rec.Code, tt.name) + assert.Equal(t, "UnsupportedHttpVerb", rec.Header().Get("X-Ms-Error-Code"), tt.name) + }) + } +} + +func TestHandler_EntityItem_InvalidKeyPredicate(t *testing.T) { + t.Parallel() + + h := newTestHandler(t) + rec := doRequest(t, h, http.MethodGet, "/"+testAccount+"/mytable(garbage)", nil) + + assert.Equal(t, http.StatusBadRequest, rec.Code) + assert.Equal(t, "InvalidInput", rec.Header().Get("X-Ms-Error-Code")) +} + +func TestSplitPath(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + path string + wantAccount string + wantResource string + }{ + {name: "empty", path: "", wantAccount: "", wantResource: ""}, + {name: "account_only", path: "/acct", wantAccount: "acct", wantResource: ""}, + {name: "tables", path: "/acct/Tables", wantAccount: "acct", wantResource: "Tables"}, + { + name: "entity_item", path: "/acct/tbl(PartitionKey='p',RowKey='r')", + wantAccount: "acct", wantResource: "tbl(PartitionKey='p',RowKey='r')", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + account, resource := azuretable.SplitPath(tt.path) + assert.Equal(t, tt.wantAccount, account, tt.name) + assert.Equal(t, tt.wantResource, resource, tt.name) + }) + } +} + +func TestParseResource(t *testing.T) { + t.Parallel() + + const ( + kindInvalid = iota + kindBatch + kindTablesCollection + kindTablesItem + kindEntityCollection + kindEntityItem + ) + + tests := []struct { + name string + resource string + wantName string + wantInner string + wantKind int + }{ + {name: "batch", resource: "$batch", wantKind: kindBatch}, + {name: "tables_collection", resource: "Tables", wantKind: kindTablesCollection, wantName: "Tables"}, + { + name: "tables_item", resource: "Tables('foo')", wantKind: kindTablesItem, + wantName: "Tables", wantInner: "'foo'", + }, + {name: "entity_collection_bare", resource: "mytable", wantKind: kindEntityCollection, wantName: "mytable"}, + { + name: "entity_collection_empty_parens", resource: "mytable()", wantKind: kindEntityCollection, + wantName: "mytable", + }, + { + name: "entity_item", resource: "mytable(PartitionKey='p',RowKey='r')", wantKind: kindEntityItem, + wantName: "mytable", wantInner: "PartitionKey='p',RowKey='r'", + }, + {name: "unclosed_paren", resource: "mytable(foo", wantKind: kindInvalid}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + kind, name, inner := azuretable.ParseResource(tt.resource) + assert.Equal(t, tt.wantKind, kind, tt.name) + assert.Equal(t, tt.wantName, name, tt.name) + assert.Equal(t, tt.wantInner, inner, tt.name) + }) + } +} + +func TestParseEntityKeyPredicate(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + predicate string + wantPK string + wantRK string + wantOK bool + }{ + {name: "normal_order", predicate: "PartitionKey='p',RowKey='r'", wantPK: "p", wantRK: "r", wantOK: true}, + {name: "reversed_order", predicate: "RowKey='r',PartitionKey='p'", wantPK: "p", wantRK: "r", wantOK: true}, + { + name: "escaped_quote", predicate: "PartitionKey='p''q',RowKey='r'", + wantPK: "p'q", wantRK: "r", wantOK: true, + }, + { + name: "comma_inside_value", + predicate: "PartitionKey='p,q',RowKey='r'", + wantPK: "p,q", + wantRK: "r", + wantOK: true, + }, + {name: "empty_keys", predicate: "PartitionKey='',RowKey=''", wantPK: "", wantRK: "", wantOK: true}, + {name: "missing_rowkey", predicate: "PartitionKey='p'", wantOK: false}, + {name: "unknown_key", predicate: "Foo='p',RowKey='r'", wantOK: false}, + {name: "no_equals", predicate: "PartitionKeyp,RowKey='r'", wantOK: false}, + {name: "garbage", predicate: "garbage", wantOK: false}, + {name: "empty", predicate: "", wantOK: false}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + pk, rk, ok := azuretable.ParseEntityKeyPredicate(tt.predicate) + assert.Equal(t, tt.wantOK, ok, tt.name) + + if tt.wantOK { + assert.Equal(t, tt.wantPK, pk, tt.name) + assert.Equal(t, tt.wantRK, rk, tt.name) + } + }) + } +} + +func TestUnquoteODataString(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + in string + want string + wantOK bool + }{ + {name: "simple", in: "'foo'", want: "foo", wantOK: true}, + {name: "escaped_quote", in: "'foo''bar'", want: "foo'bar", wantOK: true}, + {name: "empty", in: "''", want: "", wantOK: true}, + {name: "too_short", in: "'", wantOK: false}, + {name: "no_quotes", in: "foo", wantOK: false}, + {name: "unescaped_quote_inside", in: "'foo'bar'", wantOK: false}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + got, ok := azuretable.UnquoteODataString(tt.in) + assert.Equal(t, tt.wantOK, ok, tt.name) + + if tt.wantOK { + assert.Equal(t, tt.want, got, tt.name) + } + }) + } +} + +func TestExtractOperation(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + method string + path string + want string + }{ + {name: "list_tables", method: http.MethodGet, path: "/" + testAccount + "/Tables", want: "ListTables"}, + {name: "create_table", method: http.MethodPost, path: "/" + testAccount + "/Tables", want: "CreateTable"}, + { + name: "delete_table", method: http.MethodDelete, path: "/" + testAccount + "/Tables('foo')", + want: "DeleteTable", + }, + {name: "insert_entity", method: http.MethodPost, path: "/" + testAccount + "/mytable", want: "InsertEntity"}, + {name: "query_entities", method: http.MethodGet, path: "/" + testAccount + "/mytable()", want: "QueryEntities"}, + { + name: "get_entity", method: http.MethodGet, + path: "/" + testAccount + "/mytable(PartitionKey='p',RowKey='r')", want: "GetEntity", + }, + { + name: "replace_entity", method: http.MethodPut, + path: "/" + testAccount + "/mytable(PartitionKey='p',RowKey='r')", want: "ReplaceEntity", + }, + { + name: "merge_entity_patch", method: http.MethodPatch, + path: "/" + testAccount + "/mytable(PartitionKey='p',RowKey='r')", want: "MergeEntity", + }, + { + name: "merge_entity_literal", method: "MERGE", + path: "/" + testAccount + "/mytable(PartitionKey='p',RowKey='r')", want: "MergeEntity", + }, + { + name: "delete_entity", method: http.MethodDelete, + path: "/" + testAccount + "/mytable(PartitionKey='p',RowKey='r')", want: "DeleteEntity", + }, + {name: "batch", method: http.MethodPost, path: "/" + testAccount + "/$batch", want: "Batch"}, + {name: "unknown", method: http.MethodOptions, path: "/" + testAccount + "/Tables", want: "Unknown"}, + } + + h := newTestHandler(t) + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + e := echo.New() + req := httptest.NewRequest(tt.method, tt.path, http.NoBody) + c := e.NewContext(req, httptest.NewRecorder()) + + assert.Equal(t, tt.want, h.ExtractOperation(c), tt.name) + }) + } +} + +func TestExtractResource(t *testing.T) { + t.Parallel() + + h := newTestHandler(t) + e := echo.New() + req := httptest.NewRequest(http.MethodGet, "/"+testAccount+"/mytable", http.NoBody) + c := e.NewContext(req, httptest.NewRecorder()) + + assert.Equal(t, "mytable", h.ExtractResource(c)) +} + +func TestRouteMatcher_AlwaysFalse(t *testing.T) { + t.Parallel() + + h := newTestHandler(t) + matcher := h.RouteMatcher() + + e := echo.New() + req := httptest.NewRequest(http.MethodGet, "/"+testAccount+"/Tables", http.NoBody) + c := e.NewContext(req, httptest.NewRecorder()) + + assert.False(t, matcher(c)) +} + +func TestMatchPriority_Lowest(t *testing.T) { + t.Parallel() + + h := newTestHandler(t) + + assert.Equal(t, 0, h.MatchPriority()) +} + +func TestGetSupportedOperations(t *testing.T) { + t.Parallel() + + h := newTestHandler(t) + + assert.NotEmpty(t, h.GetSupportedOperations()) +} + +func TestName(t *testing.T) { + t.Parallel() + + h := newTestHandler(t) + + assert.Equal(t, "AzureTable", h.Name()) +} + +func TestCheckAuth_StructurallyValidHeaderAccepted(t *testing.T) { + t.Parallel() + + h := newTestHandler(t) + + req := httptest.NewRequest(http.MethodGet, "/"+testAccount+"/Tables", http.NoBody) + req.Header.Set("Authorization", "SharedKey devstoreaccount1:c2lnbmF0dXJl") + + e := echo.New() + rec := httptest.NewRecorder() + c := e.NewContext(req, rec) + require.NoError(t, h.Handler()(c)) + + assert.Equal(t, http.StatusOK, rec.Code) +} + +func TestCheckAuth_MalformedHeaderStillAccepted(t *testing.T) { + t.Parallel() + + h := newTestHandler(t) + + req := httptest.NewRequest(http.MethodGet, "/"+testAccount+"/Tables", http.NoBody) + req.Header.Set("Authorization", "not-a-real-header") + + e := echo.New() + rec := httptest.NewRecorder() + c := e.NewContext(req, rec) + require.NoError(t, h.Handler()(c)) + + assert.Equal(t, http.StatusOK, rec.Code) +} + +func TestReset(t *testing.T) { + t.Parallel() + + h := newTestHandler(t) + + body := []byte(`{"TableName":"foo"}`) + rec := doRequest(t, h, http.MethodPost, "/"+testAccount+"/Tables", body) + require.Equal(t, http.StatusCreated, rec.Code) + + h.Reset() + + rec = doRequest(t, h, http.MethodGet, "/"+testAccount+"/Tables", nil) + assert.JSONEq(t, `{"value":[]}`, rec.Body.String()) +} diff --git a/services/azuretable/interfaces.go b/services/azuretable/interfaces.go new file mode 100644 index 0000000000..f76f33e6ff --- /dev/null +++ b/services/azuretable/interfaces.go @@ -0,0 +1,59 @@ +// Package azuretable provides a local, in-memory emulation of Azure Table +// Storage's REST+JSON/OData wire protocol (table CRUD plus entity +// insert/get/query/replace/merge/delete, including a hand-written $filter +// lexer/parser/evaluator), Azurite-compatible enough for unmodified +// azure-sdk-for-go clients to operate against. See AZURE.md and PARITY.md +// for scope and known gaps. +// +// Unlike services/azurequeue and services/azureblob, this package has no +// janitor.go: Table Storage entities carry no TTL/expiry concept (there is +// no message-visibility or blob-lease analogue), so there is nothing for a +// background sweep to do. +package azuretable + +// Compile-time assertion: InMemoryBackend must implement StorageBackend. +var _ StorageBackend = (*InMemoryBackend)(nil) + +// IfMatchAny is the wildcard If-Match value ("*") meaning "must currently +// exist, but match unconditionally on ETag" -- as opposed to an empty +// ifMatch (no header at all, meaning upsert semantics) or a specific ETag +// string (optimistic-concurrency match required). +const IfMatchAny = "*" + +// StorageBackend defines the interface for an Azure Table Storage backend. +// Shaped after services/azurequeue's StorageBackend: a narrow, testable seam +// between the wire handler and storage, so handler tests can substitute a +// fake. +// +// The ifMatch parameter on ReplaceEntity/MergeEntity/DeleteEntity threads +// through the three If-Match states the wire protocol distinguishes: +// - "" (no If-Match header): upsert semantics for Replace/MergeEntity +// (create if absent, otherwise mutate unconditionally); DeleteEntity's +// caller (handler.go) never passes "" -- an absent If-Match on Delete is +// rejected at the handler layer before the backend is ever called. +// - IfMatchAny ("*"): the entity must exist, but any current ETag matches. +// - any other string: the entity must exist AND its current ETag must +// equal this value, else ErrETagMismatch. +type StorageBackend interface { + CreateTable(name string) error + DeleteTable(name string) error + ListTables() []TableInfo + + InsertEntity(table, partitionKey, rowKey string, props map[string]EntityProperty) (EntityInfo, error) + GetEntity(table, partitionKey, rowKey string) (EntityInfo, error) + // QueryEntities returns entities in table matching filter (nil matches + // everything), ordered by (PartitionKey, RowKey), capped at top results + // (top <= 0 means unlimited). + QueryEntities(table string, filter Node, top int) ([]EntityInfo, error) + ReplaceEntity( + table, partitionKey, rowKey string, + props map[string]EntityProperty, + ifMatch string, + ) (EntityInfo, error) + MergeEntity(table, partitionKey, rowKey string, props map[string]EntityProperty, ifMatch string) (EntityInfo, error) + DeleteEntity(table, partitionKey, rowKey, ifMatch string) error + + // Reset clears all in-memory state. Used by the + // POST /_gopherstack/reset endpoint for CI pipelines and rapid local development. + Reset() +} diff --git a/services/azuretable/models.go b/services/azuretable/models.go new file mode 100644 index 0000000000..e9c8797345 --- /dev/null +++ b/services/azuretable/models.go @@ -0,0 +1,421 @@ +package azuretable + +import ( + "encoding/base64" + "encoding/json" + "fmt" + "math" + "strconv" + "time" +) + +// EdmType identifies an entity property's OData EDM (Entity Data Model) +// type. See https://learn.microsoft.com/rest/api/storageservices/payload-format-for-table-service-operations +// for the wire-level annotation scheme this mirrors. +type EdmType string + +// Supported EDM property types. EdmString is the default for a bare JSON +// string with no "@odata.type" annotation; EdmInt32 is the default for a +// bare, whole-number JSON number. See entity_ops.go's decodeProperty for the +// exact inference rules (mirroring azure-sdk-for-go/sdk/data/aztables's own +// EDMEntity.UnmarshalJSON, so unmodified SDK round trips match). +const ( + EdmString EdmType = "Edm.String" + EdmInt32 EdmType = "Edm.Int32" + EdmInt64 EdmType = "Edm.Int64" + EdmDouble EdmType = "Edm.Double" + EdmBoolean EdmType = "Edm.Boolean" + EdmDateTime EdmType = "Edm.DateTime" + EdmGUID EdmType = "Edm.Guid" + EdmBinary EdmType = "Edm.Binary" +) + +// EntityProperty is a single typed entity property value. Value's concrete +// Go type is determined by Type: +// +// EdmString string +// EdmInt32 int32 +// EdmInt64 int64 +// EdmDouble float64 +// EdmBoolean bool +// EdmDateTime time.Time +// EdmGUID string (canonical UUID string, not validated) +// EdmBinary []byte +// +// MarshalJSON below deliberately keeps a value receiver even though +// UnmarshalJSON must be a pointer receiver to mutate p: EntityProperty +// values live inside map[string]EntityProperty (Properties), and Go map +// values are not addressable, so encoding/json can only discover and call a +// value-receiver Marshaler when encoding a map's values directly -- +// a pointer-receiver-only MarshalJSON would silently be skipped for every +// property in Properties, falling back to the wrong (default reflection) +// encoding with no error. This mixed-receiver shape is required, not an +// oversight. +// +//nolint:recvcheck // see above +type EntityProperty struct { + Value any + Type EdmType +} + +// entityPropertyWire is EntityProperty's on-disk (persistence snapshot) +// shape. A plain `any` Value field would lose type fidelity on JSON +// round-trip (encoding/json decodes every bare number into float64 and every +// []byte into a base64 string, regardless of the original Go type), so +// MarshalJSON/UnmarshalJSON re-encode Value per Type explicitly instead of +// relying on encoding/json's default interface{} behavior. +type entityPropertyWire struct { + Value any `json:"value"` + Type EdmType `json:"type"` +} + +// MarshalJSON implements json.Marshaler for persistence snapshots (see +// persistence.go). This is independent of the OData wire encoding entity_ops.go +// uses for HTTP responses. Any Type/Value mismatch (e.g. an EdmBinary +// property whose Value isn't []byte) is a real error, not silently dropped: +// a construction bug elsewhere in the package should fail loudly here rather +// than writing a corrupt snapshot. +func (p EntityProperty) MarshalJSON() ([]byte, error) { + v, err := p.marshalValue() + if err != nil { + return nil, err + } + + return json.Marshal(entityPropertyWire{Type: p.Type, Value: v}) +} + +// marshalValue dispatches to marshalScalarValue or marshalStringEncodedValue +// per p.Type, mirroring unmarshalPropertyValue's own split. +func (p EntityProperty) marshalValue() (any, error) { + switch p.Type { + case EdmInt32, EdmDouble, EdmBoolean: + return p.marshalScalarValue() + case EdmInt64, EdmDateTime, EdmGUID, EdmBinary: + return p.marshalStringEncodedValue() + case EdmString: + s, ok := p.Value.(string) + if !ok { + return nil, fmt.Errorf("%w: Edm.String property has non-string value %T", ErrInvalidEntityProperty, p.Value) + } + + return s, nil + default: + return nil, fmt.Errorf("%w: unknown EdmType %q", ErrInvalidEntityProperty, p.Type) + } +} + +// marshalScalarValue encodes the three EDM types whose Go value is already +// the JSON-native type encoding/json's generic `any` decode produces (see +// unmarshalScalarValue's own doc comment for the matching decode side). +func (p EntityProperty) marshalScalarValue() (any, error) { + switch p.Type { + case EdmInt32: + n, ok := p.Value.(int32) + if !ok { + return nil, fmt.Errorf("%w: Edm.Int32 property has non-int32 value %T", ErrInvalidEntityProperty, p.Value) + } + + return n, nil + case EdmDouble: + f, ok := p.Value.(float64) + if !ok { + return nil, fmt.Errorf( + "%w: Edm.Double property has non-float64 value %T", ErrInvalidEntityProperty, p.Value, + ) + } + + return f, nil + case EdmBoolean: + fallthrough + default: + b, ok := p.Value.(bool) + if !ok { + return nil, fmt.Errorf("%w: Edm.Boolean property has non-bool value %T", ErrInvalidEntityProperty, p.Value) + } + + return b, nil + } +} + +// marshalStringEncodedValue encodes the EDM types whose snapshot wire value +// is a string rather than encoding/json's native decode for that Go type +// (see unmarshalStringEncodedValue's own doc comment for the matching decode +// side). +func (p EntityProperty) marshalStringEncodedValue() (any, error) { + switch p.Type { + case EdmBinary: + b, ok := p.Value.([]byte) + if !ok { + return nil, fmt.Errorf("%w: Edm.Binary property has non-[]byte value %T", ErrInvalidEntityProperty, p.Value) + } + + return base64.StdEncoding.EncodeToString(b), nil + case EdmDateTime: + t, ok := p.Value.(time.Time) + if !ok { + return nil, fmt.Errorf( + "%w: Edm.DateTime property has non-time.Time value %T", ErrInvalidEntityProperty, p.Value, + ) + } + + return t.UTC().Format(time.RFC3339Nano), nil + case EdmInt64: + n, ok := p.Value.(int64) + if !ok { + return nil, fmt.Errorf("%w: Edm.Int64 property has non-int64 value %T", ErrInvalidEntityProperty, p.Value) + } + + // Encoded as a decimal string, not a bare JSON number: float64 has + // only a 53-bit mantissa, so round-tripping an Int64 through a JSON + // number silently corrupts any value outside [-2^53, 2^53] (e.g. + // 9007199254740993 becomes 9007199254740992). This mirrors the + // OData wire format itself, which also encodes Edm.Int64 as a + // string alongside its "@odata.type" annotation -- see + // entity_ops.go's decodeStringEncodedProperty/encodePropertyInto -- + // so the snapshot and wire encodings are now consistent. + return strconv.FormatInt(n, 10), nil + case EdmGUID: + fallthrough + default: + s, ok := p.Value.(string) + if !ok { + return nil, fmt.Errorf("%w: Edm.Guid property has non-string value %T", ErrInvalidEntityProperty, p.Value) + } + + return s, nil + } +} + +// UnmarshalJSON implements json.Unmarshaler for persistence snapshots. See +// MarshalJSON's doc comment for why this can't just rely on encoding/json's +// default `any` decoding. A malformed or type-mismatched snapshot value is a +// real error, never silently zeroed -- a snapshot that can't be decoded +// exactly must not be decoded approximately. +func (p *EntityProperty) UnmarshalJSON(data []byte) error { + var wire entityPropertyWire + + if err := json.Unmarshal(data, &wire); err != nil { + return err + } + + value, err := unmarshalPropertyValue(wire.Type, wire.Value) + if err != nil { + return err + } + + p.Type, p.Value = wire.Type, value + + return nil +} + +// unmarshalPropertyValue decodes wireValue (the generic `any` encoding/json +// produced for entityPropertyWire.Value) into its typed Go value per +// edmType. +func unmarshalPropertyValue(edmType EdmType, wireValue any) (any, error) { + switch edmType { + case EdmInt32, EdmDouble, EdmBoolean: + return unmarshalScalarValue(edmType, wireValue) + case EdmInt64, EdmDateTime, EdmGUID, EdmBinary: + return unmarshalStringEncodedValue(edmType, wireValue) + case EdmString: + fallthrough + default: + s, ok := wireValue.(string) + if !ok { + return nil, fmt.Errorf( + "%w: Edm.String snapshot value is not a string: %v", + ErrInvalidEntityProperty, + wireValue, + ) + } + + return s, nil + } +} + +// unmarshalScalarValue decodes the three EDM types whose snapshot wire value +// is already the JSON-native type encoding/json's generic `any` decode +// produces (a bare JSON number or bool never needed re-encoding on the +// Marshal side -- see marshalValue). +func unmarshalScalarValue(edmType EdmType, wireValue any) (any, error) { + switch edmType { + case EdmInt32: + f, ok := wireValue.(float64) + if !ok { + return nil, fmt.Errorf( + "%w: Edm.Int32 snapshot value is not a number: %v", + ErrInvalidEntityProperty, + wireValue, + ) + } + + if f != math.Trunc(f) || f < math.MinInt32 || f > math.MaxInt32 { + return nil, fmt.Errorf( + "%w: Edm.Int32 snapshot value out of range or fractional: %v", + ErrInvalidEntityProperty, + wireValue, + ) + } + + return int32(f), nil + case EdmDouble: + f, ok := wireValue.(float64) + if !ok { + return nil, fmt.Errorf( + "%w: Edm.Double snapshot value is not a number: %v", + ErrInvalidEntityProperty, + wireValue, + ) + } + + return f, nil + case EdmBoolean: + b, ok := wireValue.(bool) + if !ok { + return nil, fmt.Errorf( + "%w: Edm.Boolean snapshot value is not a bool: %v", + ErrInvalidEntityProperty, + wireValue, + ) + } + + return b, nil + default: + return nil, fmt.Errorf("%w: unsupported scalar EdmType %q", ErrInvalidEntityProperty, edmType) + } +} + +// unmarshalStringEncodedValue decodes the four EDM types marshalValue always +// re-encodes as a JSON string (Int64 as decimal digits, DateTime as +// RFC3339Nano, Guid as its canonical string form, Binary as base64). +func unmarshalStringEncodedValue(edmType EdmType, wireValue any) (any, error) { + s, ok := wireValue.(string) + if !ok { + return nil, fmt.Errorf( + "%w: %s snapshot value is not a string: %v", + ErrInvalidEntityProperty, + edmType, + wireValue, + ) + } + + switch edmType { + case EdmInt64: + n, err := strconv.ParseInt(s, 10, 64) + if err != nil { + return nil, fmt.Errorf("%w: invalid Edm.Int64 snapshot value %q: %w", ErrInvalidEntityProperty, s, err) + } + + return n, nil + case EdmDateTime: + t, err := time.Parse(time.RFC3339Nano, s) + if err != nil { + return nil, fmt.Errorf("%w: invalid Edm.DateTime snapshot value %q: %w", ErrInvalidEntityProperty, s, err) + } + + return t.UTC(), nil + case EdmGUID: + return s, nil + case EdmBinary: + b, err := base64.StdEncoding.DecodeString(s) + if err != nil { + return nil, fmt.Errorf("%w: invalid Edm.Binary snapshot value: %w", ErrInvalidEntityProperty, err) + } + + return b, nil + default: + return nil, fmt.Errorf("%w: unsupported string-encoded EdmType %q", ErrInvalidEntityProperty, edmType) + } +} + +// TableInfo is a read-only snapshot of a table's metadata, returned by +// StorageBackend.ListTables. +type TableInfo struct { + Name string +} + +// EntityInfo is a read-only snapshot of an entity, returned by the +// StorageBackend entity accessors. Properties excludes the system properties +// (PartitionKey, RowKey, Timestamp), which are surfaced via their own +// fields. +type EntityInfo struct { + Timestamp time.Time + Properties map[string]EntityProperty + PartitionKey string + RowKey string + ETag string +} + +// storedEntity is the backend's internal representation of an entity. +type storedEntity struct { + Timestamp time.Time + Properties map[string]EntityProperty + PartitionKey string + RowKey string +} + +// storedTable is the backend's internal representation of a table. +type storedTable struct { + Entities map[entityCompositeKey]*storedEntity + Name string +} + +// entityCompositeKey is an entity's map key within storedTable.Entities: a +// comparable struct of its two identifying fields, not a delimited string. +// A delimited string (e.g. partitionKey+"\x00"+rowKey, this package's +// original approach) is unsafe: JSON permits any byte value -- including +// NUL -- inside a string property, so two different (PartitionKey, RowKey) +// pairs could produce the same delimited string +// (partitionKey="a\x00b",rowKey="c" collides with partitionKey="a", +// rowKey="b\x00c"), silently rejecting a legitimate insert as a duplicate, +// or making Get/Replace/Delete operate on the wrong entity. A struct key +// compares its two fields independently, so no such collision is possible. +type entityCompositeKey struct { + PartitionKey string + RowKey string +} + +// MarshalText implements encoding.TextMarshaler so entityCompositeKey can be +// used as a map key in backendSnapshot's persisted storedTable.Entities +// (encoding/json only accepts string, integer, or TextMarshaler-implementing +// map key types). The two fields are marshaled as a JSON string array rather +// than delimited by a separator character: unlike a delimiter, nested JSON +// string encoding is unambiguous for every possible PartitionKey/RowKey +// value (including one containing a literal '"' or NUL byte), so this +// encoding can't reintroduce the same collision class the struct key itself +// was introduced to close. +func (k entityCompositeKey) MarshalText() ([]byte, error) { + return json.Marshal([2]string{k.PartitionKey, k.RowKey}) +} + +// UnmarshalText implements encoding.TextUnmarshaler, the inverse of +// MarshalText. +func (k *entityCompositeKey) UnmarshalText(text []byte) error { + var pair [2]string + + if err := json.Unmarshal(text, &pair); err != nil { + return fmt.Errorf("azuretable: malformed entity composite key %q: %w", text, err) + } + + k.PartitionKey, k.RowKey = pair[0], pair[1] + + return nil +} + +// --- OData wire error envelope --- +// +// {"odata.error":{"code":"TableNotFound","message":{"lang":"en-US","value":"..."}}} + +type odataErrorMessage struct { + Lang string `json:"lang"` + Value string `json:"value"` +} + +type odataErrorDetail struct { + Message odataErrorMessage `json:"message"` + Code string `json:"code"` +} + +type odataErrorEnvelope struct { + Error odataErrorDetail `json:"odata.error"` +} diff --git a/services/azuretable/models_test.go b/services/azuretable/models_test.go new file mode 100644 index 0000000000..4e89b9fdca --- /dev/null +++ b/services/azuretable/models_test.go @@ -0,0 +1,154 @@ +package azuretable_test + +import ( + "encoding/json" + "math" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/blackbirdworks/gopherstack/services/azuretable" +) + +// TestEntityProperty_JSONRoundTrip covers every EDM type's persistence +// (encoding/json Marshal/Unmarshal) round trip, including the exact values +// that would silently corrupt through a bare-float64 Int64 encoding: +// 2^53+1 is the smallest positive integer a float64 cannot represent +// exactly, so it is the correct demonstration case (unlike math.MaxInt64, +// which survives a naive float64 round trip by coincidence -- both are +// covered here regardless). +func TestEntityProperty_JSONRoundTrip(t *testing.T) { + t.Parallel() + + fixedTime := time.Date(2024, 6, 15, 12, 30, 45, 123456700, time.UTC) + + tests := []struct { + name string + prop azuretable.EntityProperty + }{ + {name: "string", prop: azuretable.EntityProperty{Type: azuretable.EdmString, Value: "hello"}}, + {name: "string_empty", prop: azuretable.EntityProperty{Type: azuretable.EdmString, Value: ""}}, + {name: "int32", prop: azuretable.EntityProperty{Type: azuretable.EdmInt32, Value: int32(42)}}, + {name: "int32_negative", prop: azuretable.EntityProperty{Type: azuretable.EdmInt32, Value: int32(-42)}}, + {name: "int32_max", prop: azuretable.EntityProperty{Type: azuretable.EdmInt32, Value: int32(math.MaxInt32)}}, + {name: "int32_min", prop: azuretable.EntityProperty{Type: azuretable.EdmInt32, Value: int32(math.MinInt32)}}, + { + name: "int64_beyond_float64_mantissa", + prop: azuretable.EntityProperty{Type: azuretable.EdmInt64, Value: int64(1<<53 + 1)}, + }, + {name: "int64_max", prop: azuretable.EntityProperty{Type: azuretable.EdmInt64, Value: int64(math.MaxInt64)}}, + {name: "int64_min", prop: azuretable.EntityProperty{Type: azuretable.EdmInt64, Value: int64(math.MinInt64)}}, + {name: "int64_negative_one", prop: azuretable.EntityProperty{Type: azuretable.EdmInt64, Value: int64(-1)}}, + {name: "int64_zero", prop: azuretable.EntityProperty{Type: azuretable.EdmInt64, Value: int64(0)}}, + {name: "double", prop: azuretable.EntityProperty{Type: azuretable.EdmDouble, Value: 3.14159265358979}}, + {name: "double_zero", prop: azuretable.EntityProperty{Type: azuretable.EdmDouble, Value: 0.0}}, + {name: "double_negative", prop: azuretable.EntityProperty{Type: azuretable.EdmDouble, Value: -2.5}}, + {name: "boolean_true", prop: azuretable.EntityProperty{Type: azuretable.EdmBoolean, Value: true}}, + {name: "boolean_false", prop: azuretable.EntityProperty{Type: azuretable.EdmBoolean, Value: false}}, + {name: "datetime", prop: azuretable.EntityProperty{Type: azuretable.EdmDateTime, Value: fixedTime}}, + { + name: "guid", + prop: azuretable.EntityProperty{Type: azuretable.EdmGUID, Value: "550e8400-e29b-41d4-a716-446655440000"}, + }, + {name: "binary", prop: azuretable.EntityProperty{Type: azuretable.EdmBinary, Value: []byte("gopherstack")}}, + {name: "binary_empty", prop: azuretable.EntityProperty{Type: azuretable.EdmBinary, Value: []byte{}}}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + data, err := json.Marshal(tt.prop) + require.NoError(t, err, tt.name) + + var got azuretable.EntityProperty + require.NoError(t, json.Unmarshal(data, &got), tt.name) + + assert.Equal(t, tt.prop.Type, got.Type, tt.name) + + if tt.prop.Type == azuretable.EdmDateTime { + wantTime, ok := tt.prop.Value.(time.Time) + require.True(t, ok, tt.name) + gotTime, ok := got.Value.(time.Time) + require.True(t, ok, tt.name) + assert.True(t, wantTime.Equal(gotTime), "%s: want %v got %v", tt.name, wantTime, gotTime) + assert.Equal(t, wantTime.UnixNano(), gotTime.UnixNano(), tt.name) + + return + } + + assert.Equal(t, tt.prop.Value, got.Value, tt.name) + }) + } +} + +// TestEntityProperty_MarshalJSON_TypeMismatchErrors covers the case a +// property's declared Type disagrees with its Go-typed Value (a +// construction bug elsewhere in the package): Marshal must return an error, +// never silently drop or mis-encode the value. +func TestEntityProperty_MarshalJSON_TypeMismatchErrors(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + prop azuretable.EntityProperty + }{ + {name: "binary_wrong_type", prop: azuretable.EntityProperty{Type: azuretable.EdmBinary, Value: "not bytes"}}, + { + name: "datetime_wrong_type", + prop: azuretable.EntityProperty{Type: azuretable.EdmDateTime, Value: "not a time"}, + }, + {name: "int64_wrong_type", prop: azuretable.EntityProperty{Type: azuretable.EdmInt64, Value: int32(1)}}, + {name: "int32_wrong_type", prop: azuretable.EntityProperty{Type: azuretable.EdmInt32, Value: int64(1)}}, + {name: "double_wrong_type", prop: azuretable.EntityProperty{Type: azuretable.EdmDouble, Value: "1.5"}}, + {name: "boolean_wrong_type", prop: azuretable.EntityProperty{Type: azuretable.EdmBoolean, Value: "true"}}, + {name: "guid_wrong_type", prop: azuretable.EntityProperty{Type: azuretable.EdmGUID, Value: 123}}, + {name: "string_wrong_type", prop: azuretable.EntityProperty{Type: azuretable.EdmString, Value: 123}}, + {name: "unknown_type", prop: azuretable.EntityProperty{Type: "Edm.Bogus", Value: "x"}}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + _, err := json.Marshal(tt.prop) + require.Error(t, err, tt.name) + }) + } +} + +// TestEntityProperty_UnmarshalJSON_MalformedValueErrors covers a snapshot +// whose per-type wire value doesn't decode cleanly: Unmarshal must return an +// error, never silently zero the value. +func TestEntityProperty_UnmarshalJSON_MalformedValueErrors(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + json string + }{ + {name: "int64_not_numeric_string", json: `{"type":"Edm.Int64","value":"not-a-number"}`}, + {name: "int64_wire_is_number_not_string", json: `{"type":"Edm.Int64","value":123}`}, + {name: "int32_wire_is_string_not_number", json: `{"type":"Edm.Int32","value":"123"}`}, + {name: "double_wire_is_string", json: `{"type":"Edm.Double","value":"1.5"}`}, + {name: "boolean_wire_is_string", json: `{"type":"Edm.Boolean","value":"true"}`}, + {name: "datetime_malformed", json: `{"type":"Edm.DateTime","value":"not-a-date"}`}, + {name: "datetime_wire_is_number", json: `{"type":"Edm.DateTime","value":123}`}, + {name: "guid_wire_is_number", json: `{"type":"Edm.Guid","value":123}`}, + {name: "binary_invalid_base64", json: `{"type":"Edm.Binary","value":"not base64!!"}`}, + {name: "binary_wire_is_number", json: `{"type":"Edm.Binary","value":123}`}, + {name: "string_wire_is_number", json: `{"type":"Edm.String","value":123}`}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + var p azuretable.EntityProperty + err := json.Unmarshal([]byte(tt.json), &p) + require.Error(t, err, tt.name) + }) + } +} diff --git a/services/azuretable/odata_filter.go b/services/azuretable/odata_filter.go new file mode 100644 index 0000000000..e19ce7d029 --- /dev/null +++ b/services/azuretable/odata_filter.go @@ -0,0 +1,608 @@ +package azuretable + +import ( + "encoding/base64" + "encoding/hex" + "fmt" + "strconv" + "time" +) + +// odata_filter.go implements a hand-written lexer and recursive-descent +// parser for OData's $filter mini-language, modeled on +// services/dynamodb/expr's lexer/parser/evaluator split (see lexer.go, +// parser.go, ast.go, evaluator.go there). odata_filter_eval.go holds the +// evaluator. +// +// Supported grammar: +// +// expr := orExpr +// orExpr := andExpr ('or' andExpr)* +// andExpr := unary ('and' unary)* +// unary := 'not' unary | primary +// primary := '(' expr ')' | comparison +// comparison := operand ('eq'|'ne'|'lt'|'le'|'gt'|'ge') operand +// operand := identifier | literal +// literal := 'quoted string' (with '' escape) | integer | integer'L' | float | true | false +// | datetime'' | guid'' | X'' | binary'' + +// tokenType enumerates the lexical token kinds $filter expressions use. +type tokenType int + +const ( + tEOF tokenType = iota + tError + tIdent + tString + tInt + tInt64 + tFloat + tBool + tDateTime + tGUID + tBinary + tLParen + tRParen + tAnd + tOr + tNot + tEq + tNe + tLt + tLe + tGt + tGe +) + +// token is one lexical token. enc distinguishes Binary literals' source +// encoding ('x' for X'', 'b' for binary''); it is meaningless +// for every other token type. +type token struct { + lit string + typ tokenType + enc byte +} + +// lexer tokenizes a $filter expression string. +type lexer struct { + input string + pos int +} + +func newLexer(s string) *lexer { return &lexer{input: s} } + +func (l *lexer) next() token { + l.skipSpace() + + if l.pos >= len(l.input) { + return token{typ: tEOF} + } + + ch := l.input[l.pos] + + switch { + case ch == '(': + l.pos++ + + return token{typ: tLParen, lit: "("} + case ch == ')': + l.pos++ + + return token{typ: tRParen, lit: ")"} + case ch == '\'': + content, ok := l.readQuotedContent() + if !ok { + return token{typ: tError, lit: "unterminated string literal"} + } + + return token{typ: tString, lit: content} + case ch == '-' || isDigit(ch): + return l.readNumber() + case isAlpha(ch): + return l.readWord() + default: + l.pos++ + + return token{typ: tError, lit: string(ch)} + } +} + +func (l *lexer) skipSpace() { + for l.pos < len(l.input) && (l.input[l.pos] == ' ' || l.input[l.pos] == '\t') { + l.pos++ + } +} + +// readQuotedContent consumes a '...'-delimited literal (escaped by doubling: +// a single quote written twice in a row means one literal quote) starting at +// l.pos, which must point at the opening quote. Returns the unescaped +// content and true, or ("", false) if the input ends before a closing quote +// is found. +func (l *lexer) readQuotedContent() (string, bool) { + if l.pos >= len(l.input) || l.input[l.pos] != '\'' { + return "", false + } + + l.pos++ + + buf := make([]byte, 0, len(l.input)-l.pos) + + for { + if l.pos >= len(l.input) { + return "", false + } + + c := l.input[l.pos] + + if c == '\'' { + if l.pos+1 < len(l.input) && l.input[l.pos+1] == '\'' { + buf = append(buf, '\'') + l.pos += 2 + + continue + } + + l.pos++ + + return string(buf), true + } + + buf = append(buf, c) + l.pos++ + } +} + +// readNumber reads an integer or floating-point literal, including an +// optional leading '-' and an optional trailing 'L'/'l' Int64 suffix on +// integers. +func (l *lexer) readNumber() token { + start := l.pos + if l.input[l.pos] == '-' { + l.pos++ + } + + for l.pos < len(l.input) && isDigit(l.input[l.pos]) { + l.pos++ + } + + isFloat := false + + if l.pos < len(l.input) && l.input[l.pos] == '.' && l.pos+1 < len(l.input) && isDigit(l.input[l.pos+1]) { + isFloat = true + l.pos++ + + for l.pos < len(l.input) && isDigit(l.input[l.pos]) { + l.pos++ + } + } + + numStr := l.input[start:l.pos] + + if !isFloat && l.pos < len(l.input) && (l.input[l.pos] == 'L' || l.input[l.pos] == 'l') { + l.pos++ + + return token{typ: tInt64, lit: numStr} + } + + if isFloat { + return token{typ: tFloat, lit: numStr} + } + + return token{typ: tInt, lit: numStr} +} + +// readWord reads an identifier-shaped word and classifies it as a keyword +// (and/or/not/eq/ne/lt/le/gt/ge), a boolean literal, a prefixed literal +// (datetime'...'/guid'...'/X'.../binary'...'), or a plain property +// identifier. +func (l *lexer) readWord() token { + start := l.pos + for l.pos < len(l.input) && isAlnum(l.input[l.pos]) { + l.pos++ + } + + word := l.input[start:l.pos] + + if tok, ok := keywordToken(word); ok { + return tok + } + + if l.pos < len(l.input) && l.input[l.pos] == '\'' { + if tok, ok := l.prefixedLiteral(word); ok { + return tok + } + } + + return token{typ: tIdent, lit: word} +} + +// keywordToken returns the token for a reserved word (case-sensitive, per +// OData: operator/logical keywords and true/false are lowercase). +func keywordToken(word string) (token, bool) { + switch word { + case "and": + return token{typ: tAnd}, true + case "or": + return token{typ: tOr}, true + case "not": + return token{typ: tNot}, true + case "eq": + return token{typ: tEq}, true + case "ne": + return token{typ: tNe}, true + case "lt": + return token{typ: tLt}, true + case "le": + return token{typ: tLe}, true + case "gt": + return token{typ: tGt}, true + case "ge": + return token{typ: tGe}, true + case "true", "false": + return token{typ: tBool, lit: word}, true + default: + return token{}, false + } +} + +// prefixedLiteral handles the datetime'...'/guid'...'/binary'...'/X'...' +// literal forms, which are a keyword word immediately followed by a quoted +// literal with no space. +func (l *lexer) prefixedLiteral(word string) (token, bool) { + var typ tokenType + + var enc byte + + switch word { + case "datetime": + typ = tDateTime + case "guid": + typ = tGUID + case "binary": + typ, enc = tBinary, 'b' + case "X": + typ, enc = tBinary, 'x' + default: + return token{}, false + } + + content, ok := l.readQuotedContent() + if !ok { + return token{typ: tError, lit: "unterminated string literal"}, true + } + + return token{typ: typ, lit: content, enc: enc}, true +} + +func isDigit(ch byte) bool { return ch >= '0' && ch <= '9' } +func isAlpha(ch byte) bool { + return ch >= 'a' && ch <= 'z' || ch >= 'A' && ch <= 'Z' || ch == '_' +} +func isAlnum(ch byte) bool { return isAlpha(ch) || isDigit(ch) } + +// --- AST --- + +// Node is a node in a parsed $filter expression tree. +type Node interface{ isFilterNode() } + +type andNode struct{ left, right Node } + +func (*andNode) isFilterNode() {} + +type orNode struct{ left, right Node } + +func (*orNode) isFilterNode() {} + +type notNode struct{ expr Node } + +func (*notNode) isFilterNode() {} + +type cmpNode struct { + left, right operand + op tokenType +} + +func (*cmpNode) isFilterNode() {} + +// operand is either a property-identifier reference or a typed literal +// value. Exactly one of isIdent or a literal field-set applies, selected by +// litType when !isIdent. +type operand struct { + ident string + strVal string + timeVal time.Time + bytesVal []byte + intVal int64 + floatVal float64 + litType tokenType + isIdent bool + isInt32 bool + boolVal bool +} + +// --- Parser --- + +// maxFilterDepth bounds recursive-descent recursion so a deeply nested (or +// adversarial) $filter string fails with a parse error instead of +// overflowing the stack -- an unbounded recursive-descent parser is a +// stack-overflow DoS. +const maxFilterDepth = 100 + +// Parser is a recursive-descent parser for $filter expressions. +type Parser struct { + lx *lexer + cur token +} + +// NewParser creates a Parser over a $filter expression string. +func NewParser(s string) *Parser { + p := &Parser{lx: newLexer(s)} + p.advance() + + return p +} + +func (p *Parser) advance() { p.cur = p.lx.next() } + +// ParseFilter parses a complete $filter expression string into a Node tree. +// Returns ErrFilterParse (wrapped with detail) on any malformed input, +// ErrFilterTooDeep if nesting exceeds maxFilterDepth, and never panics. +func ParseFilter(s string) (Node, error) { + p := NewParser(s) + + node, err := p.parseOr(0) + if err != nil { + return nil, err + } + + if p.cur.typ == tError { + return nil, fmt.Errorf("%w: %s", ErrFilterParse, p.cur.lit) + } + + if p.cur.typ != tEOF { + return nil, fmt.Errorf("%w: unexpected trailing input %q", ErrFilterParse, p.cur.lit) + } + + return node, nil +} + +func checkDepth(depth int) error { + if depth > maxFilterDepth { + return ErrFilterTooDeep + } + + return nil +} + +// parseOr, parseAnd, parseUnary, and parsePrimary form one precedence- +// climbing layer of recursive descent per grammar rule (see this file's top +// doc comment), not one nesting level each: "Age eq 1" with no parentheses +// or "not" anywhere still passes through all four on its way to +// parseComparison. depth must therefore be threaded through UNCHANGED across +// these routine same-level calls, and incremented ONLY at the two places +// genuine nesting actually happens -- parseUnary's "not" branch and +// parsePrimary's "(...)" branch, both of which recurse back into a +// lower-precedence rule. Incrementing depth on every layer instead (this +// package's original implementation) meant maxFilterDepth=100 was actually +// reached by roughly 25 nested parentheses, not 100 -- a bound that doesn't +// mean what its name says is barely better than no bound at all. See +// TestParseFilter_DeepNestingBounded and +// TestParseFilter_ModeratelyNestedParensAccepted for the regression +// coverage on both sides of this off-by-a-lot. +func (p *Parser) parseOr(depth int) (Node, error) { + if err := checkDepth(depth); err != nil { + return nil, err + } + + left, err := p.parseAnd(depth) + if err != nil { + return nil, err + } + + for p.cur.typ == tOr { + p.advance() + + right, andErr := p.parseAnd(depth) + if andErr != nil { + return nil, andErr + } + + left = &orNode{left: left, right: right} + } + + return left, nil +} + +func (p *Parser) parseAnd(depth int) (Node, error) { + if err := checkDepth(depth); err != nil { + return nil, err + } + + left, err := p.parseUnary(depth) + if err != nil { + return nil, err + } + + for p.cur.typ == tAnd { + p.advance() + + right, unaryErr := p.parseUnary(depth) + if unaryErr != nil { + return nil, unaryErr + } + + left = &andNode{left: left, right: right} + } + + return left, nil +} + +func (p *Parser) parseUnary(depth int) (Node, error) { + if err := checkDepth(depth); err != nil { + return nil, err + } + + if p.cur.typ == tNot { + p.advance() + + // Genuine recursion: "not" wraps another full unary expression, so + // this is one real nesting level deeper. + inner, err := p.parseUnary(depth + 1) + if err != nil { + return nil, err + } + + return ¬Node{expr: inner}, nil + } + + return p.parsePrimary(depth) +} + +func (p *Parser) parsePrimary(depth int) (Node, error) { + if err := checkDepth(depth); err != nil { + return nil, err + } + + if p.cur.typ == tLParen { + p.advance() + + // Genuine recursion: entering "(...)" starts a whole new + // lowest-precedence sub-expression, so this is one real nesting + // level deeper -- see the doc comment above parseOr. + node, err := p.parseOr(depth + 1) + if err != nil { + return nil, err + } + + if p.cur.typ != tRParen { + return nil, fmt.Errorf("%w: expected ) got %q", ErrFilterParse, p.cur.lit) + } + + p.advance() + + return node, nil + } + + return p.parseComparison() +} + +func (p *Parser) parseComparison() (Node, error) { + left, err := p.parseOperand() + if err != nil { + return nil, err + } + + if !isCompareOp(p.cur.typ) { + return nil, fmt.Errorf("%w: expected comparison operator, got %q", ErrFilterParse, p.cur.lit) + } + + op := p.cur.typ + p.advance() + + right, err := p.parseOperand() + if err != nil { + return nil, err + } + + return &cmpNode{left: left, right: right, op: op}, nil +} + +func isCompareOp(t tokenType) bool { + switch t { + case tEq, tNe, tLt, tLe, tGt, tGe: + return true + default: + return false + } +} + +//nolint:cyclop // straightforward per-token-type dispatch; splitting would obscure it +func (p *Parser) parseOperand() (operand, error) { + tok := p.cur + + switch tok.typ { + case tIdent: + p.advance() + + return operand{isIdent: true, ident: tok.lit}, nil + case tString: + p.advance() + + return operand{litType: tString, strVal: tok.lit}, nil + case tInt: + p.advance() + + n, err := strconv.ParseInt(tok.lit, 10, 32) + if err != nil { + return operand{}, fmt.Errorf("%w: invalid integer literal %q", ErrFilterParse, tok.lit) + } + + return operand{litType: tInt, isInt32: true, intVal: n}, nil + case tInt64: + p.advance() + + n, err := strconv.ParseInt(tok.lit, 10, 64) + if err != nil { + return operand{}, fmt.Errorf("%w: invalid Int64 literal %q", ErrFilterParse, tok.lit) + } + + return operand{litType: tInt64, intVal: n}, nil + case tFloat: + p.advance() + + f, err := strconv.ParseFloat(tok.lit, 64) + if err != nil { + return operand{}, fmt.Errorf("%w: invalid float literal %q", ErrFilterParse, tok.lit) + } + + return operand{litType: tFloat, floatVal: f}, nil + case tBool: + p.advance() + + return operand{litType: tBool, boolVal: tok.lit == "true"}, nil + case tDateTime: + p.advance() + + t, err := time.Parse(time.RFC3339Nano, tok.lit) + if err != nil { + return operand{}, fmt.Errorf("%w: invalid datetime literal %q", ErrFilterParse, tok.lit) + } + + return operand{litType: tDateTime, timeVal: t}, nil + case tGUID: + p.advance() + + return operand{litType: tGUID, strVal: tok.lit}, nil + case tBinary: + p.advance() + + b, err := decodeBinaryLiteral(tok) + if err != nil { + return operand{}, err + } + + return operand{litType: tBinary, bytesVal: b}, nil + case tError: + return operand{}, fmt.Errorf("%w: %s", ErrFilterParse, tok.lit) + default: + return operand{}, fmt.Errorf("%w: unexpected token %q", ErrFilterParse, tok.lit) + } +} + +func decodeBinaryLiteral(tok token) ([]byte, error) { + if tok.enc == 'x' { + b, err := hex.DecodeString(tok.lit) + if err != nil { + return nil, fmt.Errorf("%w: invalid hex binary literal %q", ErrFilterParse, tok.lit) + } + + return b, nil + } + + b, err := base64.StdEncoding.DecodeString(tok.lit) + if err != nil { + return nil, fmt.Errorf("%w: invalid base64 binary literal %q", ErrFilterParse, tok.lit) + } + + return b, nil +} diff --git a/services/azuretable/odata_filter_eval.go b/services/azuretable/odata_filter_eval.go new file mode 100644 index 0000000000..c1d958b948 --- /dev/null +++ b/services/azuretable/odata_filter_eval.go @@ -0,0 +1,229 @@ +package azuretable + +import ( + "bytes" + "strings" + "time" +) + +// EvaluateFilter reports whether entity satisfies the parsed $filter tree +// node. A comparison against a property missing from entity always +// evaluates to false (never an error, never a panic) -- real Table Storage +// semantics: an absent property simply never matches. A type mismatch +// between the two operands of a comparison (e.g. a string compared against +// a number) likewise evaluates to false rather than erroring. +func EvaluateFilter(node Node, entity EntityInfo) bool { + switch n := node.(type) { + case *andNode: + return EvaluateFilter(n.left, entity) && EvaluateFilter(n.right, entity) + case *orNode: + return EvaluateFilter(n.left, entity) || EvaluateFilter(n.right, entity) + case *notNode: + return !EvaluateFilter(n.expr, entity) + case *cmpNode: + return evalComparison(n, entity) + default: + return false + } +} + +func evalComparison(n *cmpNode, entity EntityInfo) bool { + left, leftOK := resolveOperand(n.left, entity) + right, rightOK := resolveOperand(n.right, entity) + + if !leftOK || !rightOK { + return false + } + + return compareOperands(left, right, n.op) +} + +// resolveOperand resolves op against entity: a literal resolves to itself; +// an identifier resolves against the three system properties or, failing +// that, entity.Properties. ok is false when an identifier names a property +// entity does not have. +func resolveOperand(op operand, entity EntityInfo) (operand, bool) { + if !op.isIdent { + return op, true + } + + switch op.ident { + case partitionKeyProperty: + return operand{litType: tString, strVal: entity.PartitionKey}, true + case rowKeyProperty: + return operand{litType: tString, strVal: entity.RowKey}, true + case timestampProperty: + return operand{litType: tDateTime, timeVal: entity.Timestamp}, true + default: + prop, ok := entity.Properties[op.ident] + if !ok { + return operand{}, false + } + + return propertyOperand(prop), true + } +} + +func propertyOperand(p EntityProperty) operand { + switch p.Type { + case EdmString: + s, _ := p.Value.(string) + + return operand{litType: tString, strVal: s} + case EdmInt32: + n, _ := p.Value.(int32) + + return operand{litType: tInt, isInt32: true, intVal: int64(n)} + case EdmInt64: + n, _ := p.Value.(int64) + + return operand{litType: tInt64, intVal: n} + case EdmDouble: + f, _ := p.Value.(float64) + + return operand{litType: tFloat, floatVal: f} + case EdmBoolean: + b, _ := p.Value.(bool) + + return operand{litType: tBool, boolVal: b} + case EdmDateTime: + t, _ := p.Value.(time.Time) + + return operand{litType: tDateTime, timeVal: t} + case EdmGUID: + s, _ := p.Value.(string) + + return operand{litType: tGUID, strVal: s} + case EdmBinary: + b, _ := p.Value.([]byte) + + return operand{litType: tBinary, bytesVal: b} + default: + return operand{} + } +} + +// compareOperands applies op to left/right if they fall in the same +// comparable category (numeric, string, datetime, bool, guid, binary); +// otherwise returns false. Numeric comparison spans Int32/Int64/Double, +// matching real Table Storage's type-coercing numeric comparisons -- but see +// compareNumeric for why that coercion is NOT a blanket float64 conversion. +// +//nolint:cyclop // per-EDM-category dispatch; splitting would obscure it +func compareOperands(left, right operand, op tokenType) bool { + switch { + case isNumericOperand(left) && isNumericOperand(right): + return applyCompare(compareNumeric(left, right), op) + case left.litType == tString && right.litType == tString: + return applyCompare(strings.Compare(left.strVal, right.strVal), op) + case left.litType == tDateTime && right.litType == tDateTime: + return applyCompare(cmpTime(left.timeVal, right.timeVal), op) + case left.litType == tBool && right.litType == tBool: + if op != tEq && op != tNe { + return false + } + + return applyCompare(cmpBool(left.boolVal, right.boolVal), op) + case left.litType == tGUID && right.litType == tGUID: + return applyCompare(strings.Compare(left.strVal, right.strVal), op) + case left.litType == tBinary && right.litType == tBinary: + return applyCompare(bytes.Compare(left.bytesVal, right.bytesVal), op) + default: + return false + } +} + +func isNumericOperand(o operand) bool { + return o.litType == tInt || o.litType == tInt64 || o.litType == tFloat +} + +// compareNumeric compares two numeric operands. When BOTH are integer-typed +// (Int32 or Int64 -- never Double), it compares their int64 values directly +// rather than converting through float64: float64 has only a 53-bit +// mantissa, so a blanket float64(intVal) conversion silently rounds any +// Int64 magnitude beyond 2^53, which would make e.g. +// "9007199254740993L eq 9007199254740992L" evaluate true. A comparison +// involving a Double operand still goes through float64, since Double +// itself is already an inexact 64-bit float and there is no wider common +// type to compare it against exactly. +func compareNumeric(left, right operand) int { + if left.litType != tFloat && right.litType != tFloat { + return cmpInt64(left.intVal, right.intVal) + } + + return cmpFloat(numericValue(left), numericValue(right)) +} + +func numericValue(o operand) float64 { + switch o.litType { + case tInt, tInt64: + return float64(o.intVal) + case tFloat: + return o.floatVal + default: + return 0 + } +} + +func cmpInt64(a, b int64) int { + switch { + case a < b: + return -1 + case a > b: + return 1 + default: + return 0 + } +} + +func applyCompare(cmp int, op tokenType) bool { + switch op { + case tEq: + return cmp == 0 + case tNe: + return cmp != 0 + case tLt: + return cmp < 0 + case tLe: + return cmp <= 0 + case tGt: + return cmp > 0 + case tGe: + return cmp >= 0 + default: + return false + } +} + +func cmpFloat(a, b float64) int { + switch { + case a < b: + return -1 + case a > b: + return 1 + default: + return 0 + } +} + +func cmpTime(a, b time.Time) int { + switch { + case a.Before(b): + return -1 + case a.After(b): + return 1 + default: + return 0 + } +} + +func cmpBool(a, b bool) int { + switch { + case a == b: + return 0 + case !a && b: + return -1 + default: + return 1 + } +} diff --git a/services/azuretable/odata_filter_test.go b/services/azuretable/odata_filter_test.go new file mode 100644 index 0000000000..2dec489102 --- /dev/null +++ b/services/azuretable/odata_filter_test.go @@ -0,0 +1,252 @@ +package azuretable_test + +import ( + "strings" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/blackbirdworks/gopherstack/services/azuretable" +) + +func entityWith(props map[string]azuretable.EntityProperty) azuretable.EntityInfo { + return azuretable.EntityInfo{ + PartitionKey: "p", + RowKey: "r", + Timestamp: time.Date(2024, 6, 15, 12, 0, 0, 0, time.UTC), + Properties: props, + } +} + +func evalFilter(t *testing.T, expr string, entity azuretable.EntityInfo) bool { + t.Helper() + + node, err := azuretable.ParseFilter(expr) + require.NoError(t, err, expr) + + return azuretable.EvaluateFilter(node, entity) +} + +func TestParseFilter_Operators(t *testing.T) { + t.Parallel() + + entity := entityWith(map[string]azuretable.EntityProperty{ + "Age": {Type: azuretable.EdmInt32, Value: int32(30)}, + "Name": {Type: azuretable.EdmString, Value: "bob"}, + "Active": {Type: azuretable.EdmBoolean, Value: true}, + }) + + tests := []struct { + name string + expr string + want bool + }{ + {name: "eq_true", expr: "Age eq 30", want: true}, + {name: "eq_false", expr: "Age eq 31", want: false}, + {name: "ne_true", expr: "Age ne 31", want: true}, + {name: "lt_true", expr: "Age lt 31", want: true}, + {name: "le_true", expr: "Age le 30", want: true}, + {name: "gt_true", expr: "Age gt 29", want: true}, + {name: "ge_true", expr: "Age ge 30", want: true}, + {name: "string_eq", expr: "Name eq 'bob'", want: true}, + {name: "string_ne", expr: "Name eq 'alice'", want: false}, + {name: "string_lt", expr: "Name lt 'zoe'", want: true}, + {name: "bool_eq", expr: "Active eq true", want: true}, + {name: "partition_key", expr: "PartitionKey eq 'p'", want: true}, + {name: "row_key", expr: "RowKey eq 'r'", want: true}, + {name: "missing_property_false", expr: "Nonexistent eq 'x'", want: false}, + {name: "and", expr: "Age eq 30 and Name eq 'bob'", want: true}, + {name: "and_false", expr: "Age eq 30 and Name eq 'alice'", want: false}, + {name: "or", expr: "Age eq 1 or Name eq 'bob'", want: true}, + {name: "not", expr: "not (Age eq 1)", want: true}, + {name: "parens", expr: "(Age eq 30 or Age eq 1) and Name eq 'bob'", want: true}, + { + name: "precedence_and_binds_tighter", + expr: "Age eq 1 or Age eq 30 and Name eq 'bob'", want: true, + }, + {name: "int64_literal", expr: "Age eq 30L", want: true}, + {name: "float_literal_no_match", expr: "Age eq 30.5", want: false}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + assert.Equal(t, tt.want, evalFilter(t, tt.expr, entity), tt.expr) + }) + } +} + +func TestParseFilter_LiteralForms(t *testing.T) { + t.Parallel() + + dt := time.Date(2024, 1, 2, 3, 4, 5, 0, time.UTC) + + entity := entityWith(map[string]azuretable.EntityProperty{ + "When": {Type: azuretable.EdmDateTime, Value: dt}, + "ID": {Type: azuretable.EdmGUID, Value: "550e8400-e29b-41d4-a716-446655440000"}, + "Blob": {Type: azuretable.EdmBinary, Value: []byte("hi")}, + "Big": {Type: azuretable.EdmInt64, Value: int64(9223372036854775807)}, + "Score": {Type: azuretable.EdmDouble, Value: 3.14}, + }) + + tests := []struct { + name string + expr string + want bool + }{ + {name: "datetime_eq", expr: "When eq datetime'2024-01-02T03:04:05.0000000Z'", want: true}, + {name: "datetime_lt", expr: "When lt datetime'2025-01-01T00:00:00.0000000Z'", want: true}, + {name: "guid_eq", expr: "ID eq guid'550e8400-e29b-41d4-a716-446655440000'", want: true}, + {name: "binary_base64_eq", expr: "Blob eq binary'aGk='", want: true}, + {name: "binary_hex_eq", expr: "Blob eq X'6869'", want: true}, + {name: "int64_eq", expr: "Big eq 9223372036854775807L", want: true}, + {name: "double_eq", expr: "Score eq 3.14", want: true}, + {name: "escaped_quote_string", expr: "PartitionKey eq 'p'", want: true}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + assert.Equal(t, tt.want, evalFilter(t, tt.expr, entity), tt.expr) + }) + } +} + +func TestParseFilter_EscapedQuoteInStringLiteral(t *testing.T) { + t.Parallel() + + entity := entityWith(map[string]azuretable.EntityProperty{ + "Name": {Type: azuretable.EdmString, Value: "O'Brien"}, + }) + + assert.True(t, evalFilter(t, "Name eq 'O''Brien'", entity)) +} + +func TestParseFilter_MalformedInputs(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + expr string + }{ + {name: "empty", expr: ""}, + {name: "unbalanced_open_paren", expr: "(Age eq 1"}, + {name: "unbalanced_close_paren", expr: "Age eq 1)"}, + {name: "trailing_operator", expr: "Age eq 1 and"}, + {name: "trailing_and", expr: "Age eq 1 and or"}, + {name: "dangling_eq", expr: "Age eq"}, + {name: "double_operator", expr: "Age eq eq 1"}, + {name: "unterminated_string", expr: "Name eq 'unterminated"}, + {name: "invalid_token", expr: "Age eq @"}, + {name: "just_and", expr: "and"}, + {name: "just_paren", expr: "("}, + {name: "empty_parens", expr: "()"}, + {name: "missing_operand_after_not", expr: "not"}, + {name: "bad_datetime", expr: "Age eq datetime'not-a-date'"}, + {name: "bad_hex", expr: "Age eq X'zz'"}, + {name: "trailing_garbage", expr: "Age eq 1 garbage"}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + _, err := azuretable.ParseFilter(tt.expr) + require.Error(t, err, tt.expr) + }) + } +} + +// TestParseFilter_DeepNestingBounded is a regression test against an +// unbounded recursive-descent parser stack-overflowing on adversarial input: +// a filter nested well past maxFilterDepth must return a parse error, never +// panic. +func TestParseFilter_DeepNestingBounded(t *testing.T) { + t.Parallel() + + deep := strings.Repeat("(", 500) + "Age eq 1" + strings.Repeat(")", 500) + + assert.NotPanics(t, func() { + _, err := azuretable.ParseFilter(deep) + require.Error(t, err) + }) +} + +// TestParseFilter_ModeratelyNestedParensAccepted is a regression test for a +// real off-by-a-lot bug: the parser used to increment its depth counter on +// every precedence-climbing layer (parseOr -> parseAnd -> parseUnary -> +// parsePrimary), not just on genuine nesting ("(" and "not"), so +// maxFilterDepth=100 was actually exhausted by roughly 25 nested +// parentheses rather than 100 -- a perfectly reasonable, non-adversarial +// filter could be rejected as "too deep". 50 levels of real parenthesis +// nesting must be accepted; TestParseFilter_DeepNestingBounded (500 levels) +// covers the rejection side of the same bound. +func TestParseFilter_ModeratelyNestedParensAccepted(t *testing.T) { + t.Parallel() + + const nestingLevels = 50 + + moderatelyDeep := strings.Repeat("(", nestingLevels) + "Age eq 1" + strings.Repeat(")", nestingLevels) + + _, err := azuretable.ParseFilter(moderatelyDeep) + require.NoError(t, err) +} + +func TestEvaluateFilter_TypeMismatchIsFalse(t *testing.T) { + t.Parallel() + + entity := entityWith(map[string]azuretable.EntityProperty{ + "Name": {Type: azuretable.EdmString, Value: "bob"}, + }) + + assert.False(t, evalFilter(t, "Name eq 1", entity)) + assert.False(t, evalFilter(t, "Name gt true", entity)) +} + +func TestEvaluateFilter_BoolOnlySupportsEqNe(t *testing.T) { + t.Parallel() + + entity := entityWith(map[string]azuretable.EntityProperty{ + "Active": {Type: azuretable.EdmBoolean, Value: true}, + }) + + assert.False(t, evalFilter(t, "Active gt false", entity)) + assert.True(t, evalFilter(t, "Active ne false", entity)) +} + +// TestEvaluateFilter_Int64PrecisionNotLostInComparison is a regression test +// for a real bug: comparing two Int64 operands by blanket-converting both +// through float64 silently rounds any magnitude beyond 2^53 (float64's +// mantissa width), making two genuinely DIFFERENT Int64 values compare +// equal. 9007199254740993 (2^53+1) and 9007199254740992 (2^53) are the +// smallest pair this can happen to. +func TestEvaluateFilter_Int64PrecisionNotLostInComparison(t *testing.T) { + t.Parallel() + + entity := entityWith(map[string]azuretable.EntityProperty{ + "Big": {Type: azuretable.EdmInt64, Value: int64(1<<53 + 1)}, // 9007199254740993 + }) + + assert.False(t, evalFilter(t, "Big eq 9007199254740992L", entity), + "2^53+1 must not compare equal to 2^53 -- that's exactly the float64-rounding bug") + assert.True(t, evalFilter(t, "Big eq 9007199254740993L", entity)) + assert.True(t, evalFilter(t, "Big gt 9007199254740992L", entity)) + assert.True(t, evalFilter(t, "Big ne 9007199254740992L", entity)) + + // Int32-vs-Int64 comparison must still work (both fit exactly in int64). + entityMixed := entityWith(map[string]azuretable.EntityProperty{ + "Small": {Type: azuretable.EdmInt32, Value: int32(42)}, + }) + assert.True(t, evalFilter(t, "Small eq 42L", entityMixed)) + + // A Double operand still legitimately compares approximately via + // float64 -- there is no wider exact common type to use instead. + entityDouble := entityWith(map[string]azuretable.EntityProperty{ + "Score": {Type: azuretable.EdmDouble, Value: 42.0}, + }) + assert.True(t, evalFilter(t, "Score eq 42", entityDouble)) +} diff --git a/services/azuretable/persistence.go b/services/azuretable/persistence.go new file mode 100644 index 0000000000..e35c479462 --- /dev/null +++ b/services/azuretable/persistence.go @@ -0,0 +1,153 @@ +package azuretable + +import ( + "context" + "fmt" + + "github.com/blackbirdworks/gopherstack/pkgs/logger" + "github.com/blackbirdworks/gopherstack/pkgs/persistence" +) + +// azureTableSnapshotVersion identifies the shape of backendSnapshot. Must be +// bumped whenever a change to storedTable/storedEntity would make an older +// snapshot unsafe to decode as the current shape; Restore compares this +// against the persisted value and discards (rather than partially decodes) +// any mismatch, mirroring services/azurequeue and services/azureblob. +// +// Bumped from 1 to 2 for two incompatible shape changes made in the same +// pass: (1) EntityProperty's Edm.Int64 wire value moved from a bare float64 +// JSON number (which silently lost precision above 2^53) to a decimal +// string, and (2) storedTable.Entities' map key moved from a delimited +// string ("partitionKey\x00rowKey") to entityCompositeKey (a struct, +// persisted via its own MarshalText as a JSON string array) to close a +// NUL-byte delimiter collision. Both changes decode a version-1 snapshot +// incorrectly if not gated behind a version check -- pkgs/persistence's +// TestSnapshotVersionGuard enforces exactly this: an incompatible retype +// must pair with a version bump, purely additive field growth must not. +const azureTableSnapshotVersion = 2 + +// backendSnapshot is the top-level on-disk shape for the Azure Table +// backend. Tables serialises directly (no DTO layer): storedTable/ +// storedEntity have no unexported fields, so encoding/json round-trips them +// as-is (EntityProperty's own MarshalJSON/UnmarshalJSON handle its typed +// Value field -- see models.go). +type backendSnapshot struct { + Tables map[string]*storedTable `json:"tables"` + Version int `json:"version"` +} + +// Snapshot serialises the backend state to JSON. It implements +// persistence.Persistable. +func (b *InMemoryBackend) Snapshot(ctx context.Context) []byte { + b.mu.RLock("Snapshot") + defer b.mu.RUnlock() + + snap := backendSnapshot{ + Version: azureTableSnapshotVersion, + Tables: b.tables, + } + + return persistence.MarshalSnapshot(ctx, "azuretable", snap) +} + +// Restore loads backend state from a JSON snapshot. It implements +// persistence.Persistable. +func (b *InMemoryBackend) Restore(ctx context.Context, data []byte) error { + var snap backendSnapshot + + if err := persistence.UnmarshalSnapshot(ctx, "azuretable", data, &snap); err != nil { + return err + } + + b.mu.Lock("Restore") + defer b.mu.Unlock() + + if snap.Version != azureTableSnapshotVersion { + // An incompatible (older/newer/absent) snapshot version must never be + // partially decoded as the current shape -- discard cleanly and start + // empty instead of erroring, since this is an expected, recoverable + // condition (e.g. upgrading gopherstack across a snapshot-format + // change), not data corruption. Mirrors services/azurequeue and + // services/azureblob. + logger.Load(ctx).WarnContext(ctx, + "azuretable: discarding incompatible snapshot version, starting empty", + "gotVersion", snap.Version, "wantVersion", azureTableSnapshotVersion) + + b.tables = make(map[string]*storedTable) + + return nil + } + + if snap.Tables == nil { + snap.Tables = make(map[string]*storedTable) + } + + if err := validateSnapshotTables(snap.Tables); err != nil { + return err + } + + b.tables = snap.Tables + + return nil +} + +// validateSnapshotTables rejects a snapshot whose "tables" map (or any +// table's "Entities" map) holds a JSON null entry -- which decodes to a nil +// pointer that would panic on first dereference if stored as-is -- or whose +// map key disagrees with the entry's own Name field, mirroring +// services/azurequeue's identical Restore validation. It also initializes +// any table whose own "Entities" map is JSON `null` (legal JSON, decodes to +// a nil Go map, not a nil pointer -- so it isn't rejected above) to an empty +// map: a nil map is safe to range over and read from, but assigning into +// one (as InsertEntity/ReplaceEntity/MergeEntity all do) panics. Mirrors the +// same nil-map init this function's caller already does for a nil top-level +// "tables" map. +func validateSnapshotTables(tables map[string]*storedTable) error { + for name, t := range tables { + if t == nil { + return fmt.Errorf("%w: %q", ErrSnapshotTableNull, name) + } + + if t.Name != name { + return fmt.Errorf("%w: map key %q, Name %q", ErrSnapshotTableNameMismatch, name, t.Name) + } + + for key, e := range t.Entities { + if e == nil { + return fmt.Errorf("%w: key %v in table %q", ErrSnapshotEntityNull, key, name) + } + } + + if t.Entities == nil { + t.Entities = make(map[entityCompositeKey]*storedEntity) + } + } + + return nil +} + +// Snapshot implements persistence.Persistable by delegating to the backend. +func (h *Handler) Snapshot(ctx context.Context) []byte { + type snapshotter interface { + Snapshot(ctx context.Context) []byte + } + if s, ok := h.Backend.(snapshotter); ok { + return s.Snapshot(ctx) + } + + return nil +} + +// Restore implements persistence.Persistable by delegating to the backend. +func (h *Handler) Restore(ctx context.Context, data []byte) error { + type restorer interface { + Restore(context.Context, []byte) error + } + if r, ok := h.Backend.(restorer); ok { + if err := r.Restore(ctx, data); err != nil { + return fmt.Errorf("azuretable: restore snapshot: %w", err) + } + } + + return nil +} diff --git a/services/azuretable/persistence_test.go b/services/azuretable/persistence_test.go new file mode 100644 index 0000000000..5c884518d5 --- /dev/null +++ b/services/azuretable/persistence_test.go @@ -0,0 +1,175 @@ +package azuretable_test + +import ( + "math" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/blackbirdworks/gopherstack/services/azuretable" +) + +func TestBackend_SnapshotRestore_RoundTrip(t *testing.T) { + t.Parallel() + + b := azuretable.NewInMemoryBackend() + require.NoError(t, b.CreateTable("t")) + _, err := b.InsertEntity("t", "p", "r", map[string]azuretable.EntityProperty{ + "Name": {Type: azuretable.EdmString, Value: "hi"}, + "Age": {Type: azuretable.EdmInt32, Value: int32(42)}, + "Big": {Type: azuretable.EdmInt64, Value: int64(9223372036854775807)}, + "Blob": {Type: azuretable.EdmBinary, Value: []byte("hello")}, + }) + require.NoError(t, err) + + snap := b.Snapshot(t.Context()) + require.NotEmpty(t, snap) + + b2 := azuretable.NewInMemoryBackend() + require.NoError(t, b2.Restore(t.Context(), snap)) + + info, err := b2.GetEntity("t", "p", "r") + require.NoError(t, err) + assert.Equal(t, "hi", info.Properties["Name"].Value) + assert.Equal(t, int32(42), info.Properties["Age"].Value) + assert.Equal(t, int64(9223372036854775807), info.Properties["Big"].Value) + assert.Equal(t, []byte("hello"), info.Properties["Blob"].Value) +} + +// TestBackend_SnapshotRestore_Int64PrecisionRoundTrip is a regression test +// for a real data-corruption bug: an earlier EntityProperty.MarshalJSON +// encoded Edm.Int64 as a bare JSON number (float64(n)), which silently loses +// precision above 2^53 (float64's mantissa width) -- 9007199254740993 +// (2^53+1) round-tripped to 9007199254740992. Int64 is now encoded as a +// decimal string in the snapshot, matching the OData wire format's own +// Edm.Int64 encoding. Covers the boundary values that matter: the smallest +// value a float64 can't represent exactly, both int64 extremes, -1, and 0. +func TestBackend_SnapshotRestore_Int64PrecisionRoundTrip(t *testing.T) { + t.Parallel() + + values := map[string]int64{ + "beyond_float64_mantissa": 1<<53 + 1, // 9007199254740993 + "max_int64": math.MaxInt64, + "min_int64": math.MinInt64, + "negative_one": -1, + "zero": 0, + } + + b := azuretable.NewInMemoryBackend() + require.NoError(t, b.CreateTable("t")) + + for name, v := range values { + _, err := b.InsertEntity("t", "p", name, map[string]azuretable.EntityProperty{ + "Big": {Type: azuretable.EdmInt64, Value: v}, + }) + require.NoError(t, err, name) + } + + snap := b.Snapshot(t.Context()) + require.NotEmpty(t, snap) + + b2 := azuretable.NewInMemoryBackend() + require.NoError(t, b2.Restore(t.Context(), snap)) + + for name, want := range values { + info, err := b2.GetEntity("t", "p", name) + require.NoError(t, err, name) + assert.Equal(t, want, info.Properties["Big"].Value, name) + } +} + +func TestBackend_Restore_IncompatibleVersionStartsEmpty(t *testing.T) { + t.Parallel() + + b := azuretable.NewInMemoryBackend() + require.NoError(t, b.CreateTable("t")) + + err := b.Restore(t.Context(), []byte(`{"tables":{},"version":999}`)) + require.NoError(t, err) + assert.Empty(t, b.ListTables()) +} + +func TestBackend_Restore_NullTableRejected(t *testing.T) { + t.Parallel() + + b := azuretable.NewInMemoryBackend() + err := b.Restore(t.Context(), []byte(`{"tables":{"t":null},"version":2}`)) + require.ErrorIs(t, err, azuretable.ErrSnapshotTableNull) +} + +func TestBackend_Restore_NullEntityRejected(t *testing.T) { + t.Parallel() + + b := azuretable.NewInMemoryBackend() + // The entity map's key is entityCompositeKey (see models.go), which + // marshals itself as a JSON string array ["PartitionKey","RowKey"] -- + // so, as an object key, it appears here JSON-string-escaped: + // ["p","r"] -> the literal object key "[\"p\",\"r\"]". + err := b.Restore(t.Context(), + []byte(`{"tables":{"t":{"Name":"t","Entities":{"[\"p\",\"r\"]":null}}},"version":2}`)) + require.ErrorIs(t, err, azuretable.ErrSnapshotEntityNull) +} + +// TestBackend_Restore_NilEntitiesMapIsInitialized covers a snapshot whose +// table has "Entities": null (legal JSON -- decodes to a nil Go map, not a +// nil *storedTable, so it isn't caught by the null-table/null-entity checks +// above): a nil map is safe to range/read, but InsertEntity assigning into +// it directly afterward would panic ("assignment to entry in nil map"). +// Restore must leave the table usable. +func TestBackend_Restore_NilEntitiesMapIsInitialized(t *testing.T) { + t.Parallel() + + b := azuretable.NewInMemoryBackend() + require.NoError(t, b.Restore(t.Context(), + []byte(`{"tables":{"t":{"Name":"t","Entities":null}},"version":2}`))) + + assert.NotPanics(t, func() { + _, err := b.InsertEntity("t", "p", "r", nil) + require.NoError(t, err) + }) + + info, err := b.GetEntity("t", "p", "r") + require.NoError(t, err) + assert.Equal(t, "p", info.PartitionKey) +} + +func TestBackend_Restore_TableNameMismatchRejected(t *testing.T) { + t.Parallel() + + b := azuretable.NewInMemoryBackend() + err := b.Restore(t.Context(), + []byte(`{"tables":{"t":{"Name":"other","Entities":{}}},"version":2}`)) + require.ErrorIs(t, err, azuretable.ErrSnapshotTableNameMismatch) +} + +func TestBackend_Restore_MalformedJSON(t *testing.T) { + t.Parallel() + + b := azuretable.NewInMemoryBackend() + err := b.Restore(t.Context(), []byte(`not json`)) + require.Error(t, err) +} + +func TestHandler_SnapshotRestore_Delegation(t *testing.T) { + t.Parallel() + + backend := azuretable.NewInMemoryBackend() + h := azuretable.NewHandler(backend) + require.NoError(t, backend.CreateTable("t")) + + snap := h.Snapshot(t.Context()) + require.NotEmpty(t, snap) + + h2 := azuretable.NewHandler(azuretable.NewInMemoryBackend()) + require.NoError(t, h2.Restore(t.Context(), snap)) +} + +func TestHandler_Restore_WrapsBackendError(t *testing.T) { + t.Parallel() + + h := azuretable.NewHandler(azuretable.NewInMemoryBackend()) + err := h.Restore(t.Context(), []byte(`{"tables":{"t":null},"version":2}`)) + require.Error(t, err) + assert.ErrorIs(t, err, azuretable.ErrSnapshotTableNull) +} diff --git a/services/azuretable/provider.go b/services/azuretable/provider.go new file mode 100644 index 0000000000..fff6eef082 --- /dev/null +++ b/services/azuretable/provider.go @@ -0,0 +1,64 @@ +package azuretable + +import ( + "errors" + + "github.com/blackbirdworks/gopherstack/pkgs/service" +) + +// ErrNilAppContext is returned when Init is called with a nil AppContext. +var ErrNilAppContext = errors.New("azuretable: nil app context") + +// ConfigProvider is a private interface to extract AzureTable configuration +// from the abstract AppContext Config, mirroring services/azurequeue.ConfigProvider. +type ConfigProvider interface { + GetAzureTableSettings() Settings +} + +// Provider implements service.Provider for the Azure Table Storage service. +// +// Like services/azureblob and services/azurequeue, AzureTable does not +// register a RouteMatcher into the shared AWS single-port Router: Azure +// Table's path shape (//) has no service-identifying +// header the way AWS's X-Amz-Target does, and shares the same +// // shape as Azure Blob and Queue, so multiplexing it +// onto the shared port (or either of their own dedicated ports) risks +// exactly the collision the router avoids by construction for AWS services +// (see AZURE.md section 4). Instead the returned Handler implements +// service.BackgroundWorker and stands up its own dedicated +// *echo.Echo/*http.Server, listening on a fixed, protocol-conventional port +// (Azurite's own Table port, 10002). It is registered in cli.go's +// getMostRecentServiceProviders like every other provider; only its +// RouteMatcher (which always returns false) is inert. +// +// Unlike services/azureblob and services/azurequeue, AzureTable has no +// janitor: Table Storage entities carry no TTL/expiry concept for a +// background sweep to enforce. +type Provider struct{} + +// Name returns the service provider name. +func (p *Provider) Name() string { return "AzureTable" } + +// Init initializes the AzureTable service backend and handler. The +// configured port (Settings.Port, default DefaultPort) is only recorded +// here; the actual TCP bind happens synchronously in Handler.StartWorker, so +// a port-in-use failure is returned to the caller directly instead of being +// discovered later from a background goroutine. +// +//nolint:ireturn,nolintlint // architecturally required to return interface +func (p *Provider) Init(ctx *service.AppContext) (service.Registerable, error) { + if ctx == nil { + return nil, ErrNilAppContext + } + + settings := DefaultSettings() + if cp, ok := ctx.Config.(ConfigProvider); ok { + settings = cp.GetAzureTableSettings() + } + + backend := NewInMemoryBackend() + handler := NewHandler(backend) + handler.Port = settings.Port + + return handler, nil +} diff --git a/services/azuretable/provider_test.go b/services/azuretable/provider_test.go new file mode 100644 index 0000000000..9d4464a85f --- /dev/null +++ b/services/azuretable/provider_test.go @@ -0,0 +1,63 @@ +package azuretable_test + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/blackbirdworks/gopherstack/pkgs/service" + "github.com/blackbirdworks/gopherstack/services/azuretable" +) + +func TestProvider_Init_NilAppContext(t *testing.T) { + t.Parallel() + + p := &azuretable.Provider{} + _, err := p.Init(nil) + + require.Error(t, err) + assert.ErrorIs(t, err, azuretable.ErrNilAppContext) +} + +func TestProvider_Init_ReturnsHandler(t *testing.T) { + t.Parallel() + + p := &azuretable.Provider{} + reg, err := p.Init(&service.AppContext{}) + + require.NoError(t, err) + require.NotNil(t, reg) + assert.Equal(t, "AzureTable", reg.Name()) +} + +type fakeConfigProvider struct{ settings azuretable.Settings } + +func (f fakeConfigProvider) GetAzureTableSettings() azuretable.Settings { return f.settings } + +func TestProvider_Init_UsesConfigProviderSettings(t *testing.T) { + t.Parallel() + + p := &azuretable.Provider{} + reg, err := p.Init(&service.AppContext{Config: fakeConfigProvider{settings: azuretable.Settings{Port: 12345}}}) + + require.NoError(t, err) + + h, ok := reg.(*azuretable.Handler) + require.True(t, ok) + assert.Equal(t, 12345, h.Port) +} + +func TestProvider_Name(t *testing.T) { + t.Parallel() + + p := &azuretable.Provider{} + assert.Equal(t, "AzureTable", p.Name()) +} + +func TestDefaultSettings(t *testing.T) { + t.Parallel() + + s := azuretable.DefaultSettings() + assert.Equal(t, azuretable.DefaultPort, s.Port) +} diff --git a/services/azuretable/settings.go b/services/azuretable/settings.go new file mode 100644 index 0000000000..0f53611229 --- /dev/null +++ b/services/azuretable/settings.go @@ -0,0 +1,31 @@ +package azuretable + +// DefaultPort is Azure Table's fixed, protocol-conventional TCP port. This +// follows the same pattern as services/azureblob's DefaultPort (10000) and +// services/azurequeue's DefaultPort (10001): pick one default and try to +// bind exactly that, rather than drawing from cli.go's shared +// --port-range-start/--port-range-end PortAlloc pool. The default value +// itself (10002) is Azurite's own Table service port, so unmodified +// UseDevelopmentStorage=true-style SDK configuration works out of the box; +// see AZURE.md section 4 for the full rationale, including why this +// deliberately does NOT fall back into the shared PortAlloc pool if 10002 is +// taken (StartWorker fails fast instead -- see handler.go). +const DefaultPort = 10002 + +// Settings holds service-level configuration for the Azure Table backend. +// Fields are picked up by the Kong CLI parser when this struct is embedded +// in the root CLI command (see cli.go's CLI.AzureTable field), mirroring +// services/azurequeue's Settings pattern. +type Settings struct { + // Port is the fixed TCP port for the dedicated Table listener. See + // handler.go's StartWorker for what happens when it's unavailable + // (fails fast; no fallback pool, matching services/azureblob and + // services/azurequeue). + Port int `json:"port" env:"AZURE_TABLE_PORT" default:"10002" name:"port" help:"Fixed TCP port for the dedicated Azure Table listener; startup fails if it's unavailable (no fallback pool)."` //nolint:lll // config struct tags are intentionally verbose +} + +// DefaultSettings returns the default Settings. Used when no ConfigProvider +// is available at init time (e.g. tests constructing a Provider directly). +func DefaultSettings() Settings { + return Settings{Port: DefaultPort} +} diff --git a/services/azuretable/store.go b/services/azuretable/store.go new file mode 100644 index 0000000000..b6dfa692db --- /dev/null +++ b/services/azuretable/store.go @@ -0,0 +1,400 @@ +package azuretable + +import ( + "net/url" + "sort" + "time" + + "github.com/blackbirdworks/gopherstack/pkgs/lockmetrics" +) + +// minTimestampBump is the smallest amount a stored entity's Timestamp is +// guaranteed to advance on each mutation. Real Table Storage Timestamps have +// 100ns ("tick") resolution; forcing at least this much forward movement +// even when nowFunc returns the same instant twice in a row (e.g. a fast +// test clock, or two mutations landing in the same wall-clock tick) +// guarantees ETag uniqueness across successive writes to the same entity -- +// see etagFor and TestInMemoryBackend_ReplaceEntity_ETagChangesOnEveryWrite. +const minTimestampBump = 100 * time.Nanosecond + +// etagTimeLayout formats a Timestamp into an Azure Table Storage-style ETag +// body: a fixed 7-fractional-digit RFC3339 string (100ns "tick" precision), +// url-encoded as a whole by etagFor. +const etagTimeLayout = "2006-01-02T15:04:05.0000000Z" + +// etagFor derives an ETag from an entity's Timestamp, in the wire format +// real Azure Table Storage uses: W/"datetime''". +func etagFor(t time.Time) string { + return `W/"datetime'` + url.QueryEscape(t.UTC().Format(etagTimeLayout)) + `'"` +} + +// InMemoryBackend implements StorageBackend using an in-memory map guarded +// by a single RWMutex. Shaped after services/azurequeue's InMemoryBackend. +type InMemoryBackend struct { + mu *lockmetrics.RWMutex + tables map[string]*storedTable + // nowFunc is the backend's time source, overridable in tests (see + // export_test.go's SetNowFunc) for deterministic Timestamp/ETag + // assertions. + nowFunc func() time.Time + // etagFunc derives an entity's ETag from its Timestamp, overridable in + // tests (see export_test.go's SetETagFunc) for deterministic ETag + // assertions independent of the real wire format. + etagFunc func(time.Time) string +} + +// NewInMemoryBackend creates a new empty InMemoryBackend. +func NewInMemoryBackend() *InMemoryBackend { + return &InMemoryBackend{ + mu: lockmetrics.New("azuretable"), + tables: make(map[string]*storedTable), + nowFunc: time.Now, + etagFunc: etagFor, + } +} + +func (b *InMemoryBackend) now() time.Time { return b.nowFunc().UTC() } + +// entityKey builds the comparable map key for a (partitionKey, rowKey) pair. +// See entityCompositeKey's doc comment (models.go) for why this is a struct, +// not a delimited string. +func entityKey(partitionKey, rowKey string) entityCompositeKey { + return entityCompositeKey{PartitionKey: partitionKey, RowKey: rowKey} +} + +// CreateTable creates a new, empty table. Returns ErrTableAlreadyExists if a +// table with the same name already exists -- unlike services/azurequeue's +// CreateQueue, Table Storage has no metadata-identical-retry idempotency +// exception; a duplicate Create is always a conflict. +func (b *InMemoryBackend) CreateTable(name string) error { + b.mu.Lock("CreateTable") + defer b.mu.Unlock() + + if _, ok := b.tables[name]; ok { + return ErrTableAlreadyExists + } + + b.tables[name] = &storedTable{Name: name, Entities: make(map[entityCompositeKey]*storedEntity)} + + return nil +} + +// DeleteTable removes a table and all of its entities. Returns +// ErrTableNotFound if the table does not exist. +func (b *InMemoryBackend) DeleteTable(name string) error { + b.mu.Lock("DeleteTable") + defer b.mu.Unlock() + + if _, ok := b.tables[name]; !ok { + return ErrTableNotFound + } + + delete(b.tables, name) + + return nil +} + +// ListTables returns a snapshot of all tables, sorted by name (the order +// Azure's List Tables returns them in). +func (b *InMemoryBackend) ListTables() []TableInfo { + b.mu.RLock("ListTables") + defer b.mu.RUnlock() + + out := make([]TableInfo, 0, len(b.tables)) + for _, t := range b.tables { + out = append(out, TableInfo{Name: t.Name}) + } + + sort.Slice(out, func(i, j int) bool { return out[i].Name < out[j].Name }) + + return out +} + +// InsertEntity creates a new entity in table. Returns ErrTableNotFound if +// the table does not exist, or ErrEntityAlreadyExists if an entity with the +// same PartitionKey/RowKey already exists. +func (b *InMemoryBackend) InsertEntity( + table, partitionKey, rowKey string, props map[string]EntityProperty, +) (EntityInfo, error) { + b.mu.Lock("InsertEntity") + defer b.mu.Unlock() + + t, ok := b.tables[table] + if !ok { + return EntityInfo{}, ErrTableNotFound + } + + key := entityKey(partitionKey, rowKey) + if _, exists := t.Entities[key]; exists { + return EntityInfo{}, ErrEntityAlreadyExists + } + + e := &storedEntity{ + PartitionKey: partitionKey, + RowKey: rowKey, + Timestamp: b.now(), + Properties: cloneProps(props), + } + t.Entities[key] = e + + return b.info(e), nil +} + +// GetEntity retrieves a single entity. Returns ErrTableNotFound or +// ErrEntityNotFound as appropriate. +func (b *InMemoryBackend) GetEntity(table, partitionKey, rowKey string) (EntityInfo, error) { + b.mu.RLock("GetEntity") + defer b.mu.RUnlock() + + t, ok := b.tables[table] + if !ok { + return EntityInfo{}, ErrTableNotFound + } + + e, ok := t.Entities[entityKey(partitionKey, rowKey)] + if !ok { + return EntityInfo{}, ErrEntityNotFound + } + + return b.info(e), nil +} + +// QueryEntities returns entities in table matching filter (nil matches all), +// ordered by (PartitionKey, RowKey), capped at top results (top <= 0 means +// unlimited). Returns ErrTableNotFound if the table does not exist. +func (b *InMemoryBackend) QueryEntities(table string, filter Node, top int) ([]EntityInfo, error) { + b.mu.RLock("QueryEntities") + defer b.mu.RUnlock() + + t, ok := b.tables[table] + if !ok { + return nil, ErrTableNotFound + } + + entities := make([]*storedEntity, 0, len(t.Entities)) + for _, e := range t.Entities { + entities = append(entities, e) + } + + sort.Slice(entities, func(i, j int) bool { + if entities[i].PartitionKey != entities[j].PartitionKey { + return entities[i].PartitionKey < entities[j].PartitionKey + } + + return entities[i].RowKey < entities[j].RowKey + }) + + out := make([]EntityInfo, 0, len(entities)) + + for _, e := range entities { + info := b.info(e) + if filter != nil && !EvaluateFilter(filter, info) { + continue + } + + out = append(out, info) + + if top > 0 && len(out) >= top { + break + } + } + + return out, nil +} + +// checkIfMatch validates ifMatch against an entity's current existence/ETag, +// per StorageBackend's If-Match state doc comment. +func (b *InMemoryBackend) checkIfMatch(e *storedEntity, exists bool, ifMatch string) error { + switch ifMatch { + case "": + return nil + case IfMatchAny: + if !exists { + return ErrEntityNotFound + } + + return nil + default: + if !exists { + return ErrEntityNotFound + } + + if b.etagFunc(e.Timestamp) != ifMatch { + return ErrETagMismatch + } + + return nil + } +} + +// bumpTimestamp returns the Timestamp a mutated entity should receive: +// b.now() for a brand-new entity, or b.now() advanced by at least +// minTimestampBump past the entity's previous Timestamp for an existing one +// -- guaranteeing a distinct ETag on every mutation (see minTimestampBump). +func (b *InMemoryBackend) bumpTimestamp(e *storedEntity, existedBefore bool) time.Time { + now := b.now() + if !existedBefore { + return now + } + + if !now.After(e.Timestamp) { + return e.Timestamp.Add(minTimestampBump) + } + + return now +} + +// ReplaceEntity fully replaces an existing entity's properties, or (ifMatch +// == "") inserts a new one if absent (Insert-Or-Replace / upsert). See +// StorageBackend's doc comment for ifMatch's three states. +func (b *InMemoryBackend) ReplaceEntity( + table, partitionKey, rowKey string, props map[string]EntityProperty, ifMatch string, +) (EntityInfo, error) { + b.mu.Lock("ReplaceEntity") + defer b.mu.Unlock() + + t, ok := b.tables[table] + if !ok { + return EntityInfo{}, ErrTableNotFound + } + + key := entityKey(partitionKey, rowKey) + e, exists := t.Entities[key] + + if err := b.checkIfMatch(e, exists, ifMatch); err != nil { + return EntityInfo{}, err + } + + if !exists { + e = &storedEntity{PartitionKey: partitionKey, RowKey: rowKey} + t.Entities[key] = e + } + + e.Timestamp = b.bumpTimestamp(e, exists) + e.Properties = cloneProps(props) + + return b.info(e), nil +} + +// MergeEntity merges props into an existing entity's properties (properties +// not present in props are left unaffected), or (ifMatch == "") inserts a +// new entity if absent (Insert-Or-Merge / upsert). See StorageBackend's doc +// comment for ifMatch's three states. +func (b *InMemoryBackend) MergeEntity( + table, partitionKey, rowKey string, props map[string]EntityProperty, ifMatch string, +) (EntityInfo, error) { + b.mu.Lock("MergeEntity") + defer b.mu.Unlock() + + t, ok := b.tables[table] + if !ok { + return EntityInfo{}, ErrTableNotFound + } + + key := entityKey(partitionKey, rowKey) + e, exists := t.Entities[key] + + if err := b.checkIfMatch(e, exists, ifMatch); err != nil { + return EntityInfo{}, err + } + + if !exists { + e = &storedEntity{PartitionKey: partitionKey, RowKey: rowKey, Properties: make(map[string]EntityProperty)} + t.Entities[key] = e + } + + e.Timestamp = b.bumpTimestamp(e, exists) + + if e.Properties == nil { + e.Properties = make(map[string]EntityProperty, len(props)) + } + + // Deep-copy each incoming property (not maps.Copy, which only copies the + // map's key/value pairs, not what an EdmBinary Value's []byte points at + // -- see cloneProp) so a caller mutating the []byte it passed in later + // can never silently corrupt stored state. + for name, prop := range props { + e.Properties[name] = cloneProp(prop) + } + + return b.info(e), nil +} + +// DeleteEntity removes an entity after verifying ifMatch. Returns +// ErrTableNotFound, ErrEntityNotFound, or ErrETagMismatch as appropriate. +// Callers (handler.go) always pass a non-empty ifMatch ("*" or a specific +// ETag): an absent If-Match is rejected at the wire layer before reaching +// the backend. +func (b *InMemoryBackend) DeleteEntity(table, partitionKey, rowKey, ifMatch string) error { + b.mu.Lock("DeleteEntity") + defer b.mu.Unlock() + + t, ok := b.tables[table] + if !ok { + return ErrTableNotFound + } + + key := entityKey(partitionKey, rowKey) + + e, exists := t.Entities[key] + if err := b.checkIfMatch(e, exists, ifMatch); err != nil { + return err + } + + delete(t.Entities, key) + + return nil +} + +// Reset clears all in-memory state. It is used by the +// POST /_gopherstack/reset endpoint for CI pipelines and rapid local development. +func (b *InMemoryBackend) Reset() { + b.mu.Lock("Reset") + defer b.mu.Unlock() + + b.tables = make(map[string]*storedTable) +} + +// info returns a read-only EntityInfo snapshot of e, including its +// currently-derived ETag. +func (b *InMemoryBackend) info(e *storedEntity) EntityInfo { + return EntityInfo{ + PartitionKey: e.PartitionKey, + RowKey: e.RowKey, + Timestamp: e.Timestamp, + Properties: cloneProps(e.Properties), + ETag: b.etagFunc(e.Timestamp), + } +} + +// cloneProps returns a copy of props deep enough that mutating anything +// reachable from the result can never affect backend state, or vice versa. +// The map itself is always copied (so a caller can't add/remove entries +// through a reference handed back to them, same as services/azurequeue), and +// each EdmBinary property's []byte Value is copied too via cloneProp: a +// map-only ("shallow") copy still shares the same backing array between the +// caller's slice and the stored one, so mutating either through its own +// reference -- with no Timestamp bump and no ETag change -- would silently +// corrupt the other and defeat optimistic concurrency entirely. +func cloneProps(props map[string]EntityProperty) map[string]EntityProperty { + out := make(map[string]EntityProperty, len(props)) + for k, v := range props { + out[k] = cloneProp(v) + } + + return out +} + +// cloneProp returns p with its Value deep-copied if (and only if) that +// Value is a []byte (EdmBinary); every other EDM type's Value is either an +// immutable Go value (string/int32/int64/float64/bool) or time.Time (also +// safe to copy by value), so only EdmBinary needs special handling here. +func cloneProp(p EntityProperty) EntityProperty { + if p.Type == EdmBinary { + if b, ok := p.Value.([]byte); ok { + p.Value = append([]byte(nil), b...) + } + } + + return p +} diff --git a/services/azuretable/store_test.go b/services/azuretable/store_test.go new file mode 100644 index 0000000000..281e25c44c --- /dev/null +++ b/services/azuretable/store_test.go @@ -0,0 +1,458 @@ +package azuretable_test + +import ( + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/blackbirdworks/gopherstack/services/azuretable" +) + +func newTestBackend(t *testing.T) *azuretable.InMemoryBackend { + t.Helper() + + return azuretable.NewInMemoryBackend() +} + +func TestInMemoryBackend_CreateTable(t *testing.T) { + t.Parallel() + + b := newTestBackend(t) + + require.NoError(t, b.CreateTable("foo")) + err := b.CreateTable("foo") + require.ErrorIs(t, err, azuretable.ErrTableAlreadyExists) +} + +func TestInMemoryBackend_DeleteTable(t *testing.T) { + t.Parallel() + + b := newTestBackend(t) + require.NoError(t, b.CreateTable("foo")) + require.NoError(t, b.DeleteTable("foo")) + + err := b.DeleteTable("foo") + require.ErrorIs(t, err, azuretable.ErrTableNotFound) +} + +func TestInMemoryBackend_ListTables_SortedByName(t *testing.T) { + t.Parallel() + + b := newTestBackend(t) + require.NoError(t, b.CreateTable("zebra")) + require.NoError(t, b.CreateTable("alpha")) + + infos := b.ListTables() + require.Len(t, infos, 2) + assert.Equal(t, "alpha", infos[0].Name) + assert.Equal(t, "zebra", infos[1].Name) +} + +func TestInMemoryBackend_InsertEntity(t *testing.T) { + t.Parallel() + + t.Run("success", func(t *testing.T) { + t.Parallel() + + b := newTestBackend(t) + require.NoError(t, b.CreateTable("t")) + + info, err := b.InsertEntity("t", "p", "r", map[string]azuretable.EntityProperty{ + "Name": {Type: azuretable.EdmString, Value: "hi"}, + }) + require.NoError(t, err) + assert.Equal(t, "p", info.PartitionKey) + assert.Equal(t, "r", info.RowKey) + assert.NotEmpty(t, info.ETag) + }) + + t.Run("table_not_found", func(t *testing.T) { + t.Parallel() + + b := newTestBackend(t) + _, err := b.InsertEntity("nosuch", "p", "r", nil) + require.ErrorIs(t, err, azuretable.ErrTableNotFound) + }) + + t.Run("already_exists", func(t *testing.T) { + t.Parallel() + + b := newTestBackend(t) + require.NoError(t, b.CreateTable("t")) + _, err := b.InsertEntity("t", "p", "r", nil) + require.NoError(t, err) + + _, err = b.InsertEntity("t", "p", "r", nil) + require.ErrorIs(t, err, azuretable.ErrEntityAlreadyExists) + }) +} + +func TestInMemoryBackend_GetEntity(t *testing.T) { + t.Parallel() + + b := newTestBackend(t) + require.NoError(t, b.CreateTable("t")) + + _, err := b.GetEntity("nosuch", "p", "r") + require.ErrorIs(t, err, azuretable.ErrTableNotFound) + + _, err = b.GetEntity("t", "p", "r") + require.ErrorIs(t, err, azuretable.ErrEntityNotFound) + + _, err = b.InsertEntity("t", "p", "r", nil) + require.NoError(t, err) + + info, err := b.GetEntity("t", "p", "r") + require.NoError(t, err) + assert.Equal(t, "p", info.PartitionKey) +} + +func TestInMemoryBackend_QueryEntities_OrderingAndTop(t *testing.T) { + t.Parallel() + + b := newTestBackend(t) + require.NoError(t, b.CreateTable("t")) + + _, err := b.InsertEntity("t", "p2", "r1", nil) + require.NoError(t, err) + _, err = b.InsertEntity("t", "p1", "r2", nil) + require.NoError(t, err) + _, err = b.InsertEntity("t", "p1", "r1", nil) + require.NoError(t, err) + + infos, err := b.QueryEntities("t", nil, 0) + require.NoError(t, err) + require.Len(t, infos, 3) + assert.Equal(t, [2]string{"p1", "r1"}, [2]string{infos[0].PartitionKey, infos[0].RowKey}) + assert.Equal(t, [2]string{"p1", "r2"}, [2]string{infos[1].PartitionKey, infos[1].RowKey}) + assert.Equal(t, [2]string{"p2", "r1"}, [2]string{infos[2].PartitionKey, infos[2].RowKey}) + + capped, err := b.QueryEntities("t", nil, 2) + require.NoError(t, err) + assert.Len(t, capped, 2) + + _, err = b.QueryEntities("nosuch", nil, 0) + require.ErrorIs(t, err, azuretable.ErrTableNotFound) +} + +func TestInMemoryBackend_QueryEntities_Filter(t *testing.T) { + t.Parallel() + + b := newTestBackend(t) + require.NoError(t, b.CreateTable("t")) + + _, err := b.InsertEntity("t", "p1", "r1", map[string]azuretable.EntityProperty{ + "Age": {Type: azuretable.EdmInt32, Value: int32(10)}, + }) + require.NoError(t, err) + _, err = b.InsertEntity("t", "p2", "r1", map[string]azuretable.EntityProperty{ + "Age": {Type: azuretable.EdmInt32, Value: int32(20)}, + }) + require.NoError(t, err) + + node, parseErr := azuretable.ParseFilter("Age gt 15") + require.NoError(t, parseErr) + + infos, err := b.QueryEntities("t", node, 0) + require.NoError(t, err) + require.Len(t, infos, 1) + assert.Equal(t, "p2", infos[0].PartitionKey) +} + +func TestInMemoryBackend_ReplaceEntity(t *testing.T) { + t.Parallel() + + t.Run("upsert_no_ifmatch_creates", func(t *testing.T) { + t.Parallel() + + b := newTestBackend(t) + require.NoError(t, b.CreateTable("t")) + + info, err := b.ReplaceEntity("t", "p", "r", nil, "") + require.NoError(t, err) + assert.NotEmpty(t, info.ETag) + }) + + t.Run("ifmatch_star_requires_existing", func(t *testing.T) { + t.Parallel() + + b := newTestBackend(t) + require.NoError(t, b.CreateTable("t")) + + _, err := b.ReplaceEntity("t", "p", "r", nil, azuretable.IfMatchAny) + require.ErrorIs(t, err, azuretable.ErrEntityNotFound) + }) + + t.Run("ifmatch_specific_mismatch", func(t *testing.T) { + t.Parallel() + + b := newTestBackend(t) + require.NoError(t, b.CreateTable("t")) + _, err := b.InsertEntity("t", "p", "r", nil) + require.NoError(t, err) + + _, err = b.ReplaceEntity("t", "p", "r", nil, `W/"datetime'bogus'"`) + require.ErrorIs(t, err, azuretable.ErrETagMismatch) + }) + + t.Run("replace_drops_old_properties", func(t *testing.T) { + t.Parallel() + + b := newTestBackend(t) + require.NoError(t, b.CreateTable("t")) + _, err := b.InsertEntity("t", "p", "r", map[string]azuretable.EntityProperty{ + "A": {Type: azuretable.EdmString, Value: "x"}, + }) + require.NoError(t, err) + + info, err := b.ReplaceEntity("t", "p", "r", map[string]azuretable.EntityProperty{ + "B": {Type: azuretable.EdmString, Value: "y"}, + }, azuretable.IfMatchAny) + require.NoError(t, err) + _, hasA := info.Properties["A"] + assert.False(t, hasA) + _, hasB := info.Properties["B"] + assert.True(t, hasB) + }) + + t.Run("table_not_found", func(t *testing.T) { + t.Parallel() + + b := newTestBackend(t) + _, err := b.ReplaceEntity("nosuch", "p", "r", nil, "") + require.ErrorIs(t, err, azuretable.ErrTableNotFound) + }) +} + +func TestInMemoryBackend_MergeEntity_KeepsUnlistedProperties(t *testing.T) { + t.Parallel() + + b := newTestBackend(t) + require.NoError(t, b.CreateTable("t")) + _, err := b.InsertEntity("t", "p", "r", map[string]azuretable.EntityProperty{ + "A": {Type: azuretable.EdmString, Value: "x"}, + "B": {Type: azuretable.EdmString, Value: "y"}, + }) + require.NoError(t, err) + + info, err := b.MergeEntity("t", "p", "r", map[string]azuretable.EntityProperty{ + "A": {Type: azuretable.EdmString, Value: "z"}, + }, azuretable.IfMatchAny) + require.NoError(t, err) + assert.Equal(t, "z", info.Properties["A"].Value) + assert.Equal(t, "y", info.Properties["B"].Value) +} + +func TestInMemoryBackend_DeleteEntity(t *testing.T) { + t.Parallel() + + b := newTestBackend(t) + require.NoError(t, b.CreateTable("t")) + _, err := b.InsertEntity("t", "p", "r", nil) + require.NoError(t, err) + + require.NoError(t, b.DeleteEntity("t", "p", "r", azuretable.IfMatchAny)) + + err = b.DeleteEntity("t", "p", "r", azuretable.IfMatchAny) + require.ErrorIs(t, err, azuretable.ErrEntityNotFound) + + err = b.DeleteEntity("nosuch", "p", "r", azuretable.IfMatchAny) + require.ErrorIs(t, err, azuretable.ErrTableNotFound) +} + +// TestInMemoryBackend_ETagChangesOnEveryWrite is a regression test for +// exactly the class of bug M1's review bots caught: two mutations landing +// within the same injected clock tick must still produce different ETags. +func TestInMemoryBackend_ETagChangesOnEveryWrite(t *testing.T) { + t.Parallel() + + b := newTestBackend(t) + require.NoError(t, b.CreateTable("t")) + + fixed := time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC) + azuretable.SetNowFunc(b, func() time.Time { return fixed }) + + info1, err := b.InsertEntity("t", "p", "r", nil) + require.NoError(t, err) + + info2, err := b.ReplaceEntity("t", "p", "r", nil, azuretable.IfMatchAny) + require.NoError(t, err) + + info3, err := b.MergeEntity("t", "p", "r", nil, azuretable.IfMatchAny) + require.NoError(t, err) + + assert.NotEqual(t, info1.ETag, info2.ETag) + assert.NotEqual(t, info2.ETag, info3.ETag) +} + +func TestInMemoryBackend_Reset(t *testing.T) { + t.Parallel() + + b := newTestBackend(t) + require.NoError(t, b.CreateTable("t")) + + b.Reset() + + assert.Empty(t, b.ListTables()) +} + +func TestSetETagFunc(t *testing.T) { + t.Parallel() + + b := newTestBackend(t) + azuretable.SetETagFunc(b, func(time.Time) string { return "fixed-etag" }) + require.NoError(t, b.CreateTable("t")) + + info, err := b.InsertEntity("t", "p", "r", nil) + require.NoError(t, err) + assert.Equal(t, "fixed-etag", info.ETag) +} + +func TestEtagFor(t *testing.T) { + t.Parallel() + + ts := time.Date(2024, 1, 2, 3, 4, 5, 123456700, time.UTC) + got := azuretable.EtagFor(ts) + + assert.Contains(t, got, "datetime") + assert.Contains(t, got, "%3A") // url-encoded colon +} + +// TestInMemoryBackend_EntityKeys_NoNULDelimiterCollision is a regression +// test for a real bug: entityKey used to build its map key by concatenating +// PartitionKey + "\x00" + RowKey, but JSON permits a literal NUL byte inside +// a string property, so two different (PartitionKey, RowKey) pairs could +// collide onto the same delimited string +// (partitionKey="a\x00b",rowKey="c" vs. partitionKey="a",rowKey="b\x00c"). +// The map key is now a comparable struct (entityCompositeKey) instead, so +// this must no longer collide: both entities must insert successfully and +// remain independently addressable. +func TestInMemoryBackend_EntityKeys_NoNULDelimiterCollision(t *testing.T) { + t.Parallel() + + b := newTestBackend(t) + require.NoError(t, b.CreateTable("t")) + + _, err := b.InsertEntity("t", "a\x00b", "c", map[string]azuretable.EntityProperty{ + "Which": {Type: azuretable.EdmString, Value: "first"}, + }) + require.NoError(t, err) + + // If the two pairs collided, this second insert would incorrectly fail + // with ErrEntityAlreadyExists. + _, err = b.InsertEntity("t", "a", "b\x00c", map[string]azuretable.EntityProperty{ + "Which": {Type: azuretable.EdmString, Value: "second"}, + }) + require.NoError(t, err) + + first, err := b.GetEntity("t", "a\x00b", "c") + require.NoError(t, err) + assert.Equal(t, "first", first.Properties["Which"].Value) + + second, err := b.GetEntity("t", "a", "b\x00c") + require.NoError(t, err) + assert.Equal(t, "second", second.Properties["Which"].Value) + + infos, err := b.QueryEntities("t", nil, 0) + require.NoError(t, err) + assert.Len(t, infos, 2, "both entities must coexist, not collide into one") +} + +// TestInMemoryBackend_EdmBinary_NoAliasing is a regression test for a real +// data-corruption bug: cloneProps/MergeEntity's map copy was shallow, so the +// []byte backing an EdmBinary property's Value was shared between the +// caller and the stored entity (and again between the stored entity and +// whatever GetEntity/QueryEntities hands back). Mutating any one of those +// slices in place silently corrupted the "stored" value with no Timestamp +// bump and no ETag change -- exactly the kind of change optimistic +// concurrency exists to detect and can't if it never happens explicitly. +func TestInMemoryBackend_EdmBinary_NoAliasing(t *testing.T) { + t.Parallel() + + t.Run("mutating_the_inserted_slice_after_insert_does_not_corrupt_storage", func(t *testing.T) { + t.Parallel() + + b := newTestBackend(t) + require.NoError(t, b.CreateTable("t")) + + original := []byte("original") + _, err := b.InsertEntity("t", "p", "r", map[string]azuretable.EntityProperty{ + "Blob": {Type: azuretable.EdmBinary, Value: original}, + }) + require.NoError(t, err) + + original[0] = 'X' // mutate the caller's own slice after the call returns + + info, err := b.GetEntity("t", "p", "r") + require.NoError(t, err) + assert.Equal(t, []byte("original"), info.Properties["Blob"].Value) + }) + + t.Run("mutating_a_returned_slice_does_not_corrupt_storage", func(t *testing.T) { + t.Parallel() + + b := newTestBackend(t) + require.NoError(t, b.CreateTable("t")) + _, err := b.InsertEntity("t", "p", "r", map[string]azuretable.EntityProperty{ + "Blob": {Type: azuretable.EdmBinary, Value: []byte("stored")}, + }) + require.NoError(t, err) + + info, err := b.GetEntity("t", "p", "r") + require.NoError(t, err) + + returned, ok := info.Properties["Blob"].Value.([]byte) + require.True(t, ok) + returned[0] = 'X' // mutate the slice the backend handed back + + info2, err := b.GetEntity("t", "p", "r") + require.NoError(t, err) + assert.Equal(t, []byte("stored"), info2.Properties["Blob"].Value) + }) + + t.Run("merge_deep_copies_incoming_binary_values_too", func(t *testing.T) { + t.Parallel() + + b := newTestBackend(t) + require.NoError(t, b.CreateTable("t")) + _, err := b.InsertEntity("t", "p", "r", nil) + require.NoError(t, err) + + merged := []byte("merged") + _, err = b.MergeEntity("t", "p", "r", map[string]azuretable.EntityProperty{ + "Blob": {Type: azuretable.EdmBinary, Value: merged}, + }, azuretable.IfMatchAny) + require.NoError(t, err) + + merged[0] = 'X' // mutate the caller's own slice after the call returns + + info, err := b.GetEntity("t", "p", "r") + require.NoError(t, err) + assert.Equal(t, []byte("merged"), info.Properties["Blob"].Value) + }) + + t.Run("query_results_do_not_alias_storage", func(t *testing.T) { + t.Parallel() + + b := newTestBackend(t) + require.NoError(t, b.CreateTable("t")) + _, err := b.InsertEntity("t", "p", "r", map[string]azuretable.EntityProperty{ + "Blob": {Type: azuretable.EdmBinary, Value: []byte("queried")}, + }) + require.NoError(t, err) + + infos, err := b.QueryEntities("t", nil, 0) + require.NoError(t, err) + require.Len(t, infos, 1) + + returned, ok := infos[0].Properties["Blob"].Value.([]byte) + require.True(t, ok) + returned[0] = 'X' + + info2, err := b.GetEntity("t", "p", "r") + require.NoError(t, err) + assert.Equal(t, []byte("queried"), info2.Properties["Blob"].Value) + }) +} diff --git a/services/azuretable/table_ops.go b/services/azuretable/table_ops.go new file mode 100644 index 0000000000..ceba948719 --- /dev/null +++ b/services/azuretable/table_ops.go @@ -0,0 +1,108 @@ +package azuretable + +import ( + "encoding/json" + "errors" + "net/http" + + "github.com/labstack/echo/v5" + + "github.com/blackbirdworks/gopherstack/pkgs/httputils" +) + +// preferReturnNoContent is the Prefer request header value that makes a +// Create Table/Insert Entity respond 204 (with Preference-Applied echoed +// back) instead of 201 + body. aztables sends this on every Create/Insert +// call whose ResponsePreference isn't overridden -- see +// TableClientCreateOptions.ResponsePreference in aztables' generated client. +const preferReturnNoContent = "return-no-content" + +// createTableBody is the request body shape for POST //Tables. +type createTableBody struct { + TableName string `json:"TableName"` +} + +func (h *Handler) createTable(c *echo.Context) error { + r := c.Request() + + body, err := httputils.ReadBody(r) + if err != nil { + return h.writeError(c, http.StatusInternalServerError, "InternalError", "Failed to read request body.") + } + + var req createTableBody + if unmarshalErr := json.Unmarshal(body, &req); unmarshalErr != nil { + return h.writeError(c, http.StatusBadRequest, "InvalidInput", "The input is not valid.") + } + + if req.TableName == "" { + return h.writeError(c, http.StatusBadRequest, "InvalidInput", "TableName must not be empty.") + } + + if createErr := h.Backend.CreateTable(req.TableName); createErr != nil { + if errors.Is(createErr, ErrTableAlreadyExists) { + return h.writeError(c, http.StatusConflict, "TableAlreadyExists", + "The table specified already exists.") + } + + return h.writeError(c, http.StatusInternalServerError, "InternalError", createErr.Error()) + } + + if r.Header.Get("Prefer") == preferReturnNoContent { + c.Response().Header().Set("Preference-Applied", preferReturnNoContent) + + return c.NoContent(http.StatusNoContent) + } + + level := odataLevelFromAccept(r.Header.Get("Accept")) + + return h.writeJSON(c, http.StatusCreated, h.tableEntityBody(req.TableName, level)) +} + +func (h *Handler) listTables(c *echo.Context) error { + infos := h.Backend.ListTables() + level := odataLevelFromAccept(c.Request().Header.Get("Accept")) + + values := make([]map[string]any, 0, len(infos)) + for _, ti := range infos { + values = append(values, h.tableEntityBody(ti.Name, level)) + } + + return h.writeJSON(c, http.StatusOK, map[string]any{"value": values}) +} + +func (h *Handler) deleteTable(c *echo.Context, quotedName string) error { + name, ok := unquoteODataString(quotedName) + if !ok { + return h.writeError(c, http.StatusBadRequest, "InvalidInput", "The specified table name is invalid.") + } + + if err := h.Backend.DeleteTable(name); err != nil { + return h.writeTableNotFoundError(c) + } + + return c.NoContent(http.StatusNoContent) +} + +// tableEntityBody builds a Table Storage table entity's OData JSON body, +// varying by metadata level: nometadata carries only TableName; +// minimalmetadata (the default) adds odata.metadata; fullmetadata further +// adds odata.type/odata.id/odata.editLink. +func (h *Handler) tableEntityBody(name, level string) map[string]any { + m := map[string]any{"TableName": name} + + if level == odataLevelNoMetadata { + return m + } + + endpoint := h.serviceEndpoint() + m["odata.metadata"] = endpoint + "/$metadata#Tables/@Element" + + if level == odataLevelFullMetadata { + m["odata.type"] = devstoreAccountName + ".Tables" + m["odata.id"] = endpoint + "/Tables('" + escapeODataKey(name) + "')" + m["odata.editLink"] = "Tables('" + escapeODataKey(name) + "')" + } + + return m +} diff --git a/services/azuretable/table_ops_test.go b/services/azuretable/table_ops_test.go new file mode 100644 index 0000000000..418865e4f5 --- /dev/null +++ b/services/azuretable/table_ops_test.go @@ -0,0 +1,183 @@ +package azuretable_test + +import ( + "encoding/json" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/labstack/echo/v5" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestCreateTable(t *testing.T) { + t.Parallel() + + t.Run("success", func(t *testing.T) { + t.Parallel() + + h := newTestHandler(t) + rec := doRequest(t, h, http.MethodPost, "/"+testAccount+"/Tables", []byte(`{"TableName":"mytable"}`)) + + require.Equal(t, http.StatusCreated, rec.Code) + + var body map[string]any + require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &body)) + assert.Equal(t, "mytable", body["TableName"]) + assert.Contains(t, body, "odata.metadata") + }) + + t.Run("prefer_return_no_content", func(t *testing.T) { + t.Parallel() + + h := newTestHandler(t) + + req := httptest.NewRequest( + http.MethodPost, + "/"+testAccount+"/Tables", + strings.NewReader(`{"TableName":"mytable"}`), + ) + req.Header.Set("Prefer", "return-no-content") + + e := echo.New() + rec := httptest.NewRecorder() + c := e.NewContext(req, rec) + require.NoError(t, h.Handler()(c)) + + assert.Equal(t, http.StatusNoContent, rec.Code) + assert.Equal(t, "return-no-content", rec.Header().Get("Preference-Applied")) + }) + + t.Run("duplicate_conflict", func(t *testing.T) { + t.Parallel() + + h := newTestHandler(t) + rec := doRequest(t, h, http.MethodPost, "/"+testAccount+"/Tables", []byte(`{"TableName":"dup"}`)) + require.Equal(t, http.StatusCreated, rec.Code) + + rec = doRequest(t, h, http.MethodPost, "/"+testAccount+"/Tables", []byte(`{"TableName":"dup"}`)) + assert.Equal(t, http.StatusConflict, rec.Code) + assert.Equal(t, "TableAlreadyExists", rec.Header().Get("X-Ms-Error-Code")) + }) + + t.Run("empty_name_rejected", func(t *testing.T) { + t.Parallel() + + h := newTestHandler(t) + rec := doRequest(t, h, http.MethodPost, "/"+testAccount+"/Tables", []byte(`{"TableName":""}`)) + assert.Equal(t, http.StatusBadRequest, rec.Code) + assert.Equal(t, "InvalidInput", rec.Header().Get("X-Ms-Error-Code")) + }) + + t.Run("malformed_body", func(t *testing.T) { + t.Parallel() + + h := newTestHandler(t) + rec := doRequest(t, h, http.MethodPost, "/"+testAccount+"/Tables", []byte(`not json`)) + assert.Equal(t, http.StatusBadRequest, rec.Code) + assert.Equal(t, "InvalidInput", rec.Header().Get("X-Ms-Error-Code")) + }) +} + +func TestListTables(t *testing.T) { + t.Parallel() + + h := newTestHandler(t) + doRequest(t, h, http.MethodPost, "/"+testAccount+"/Tables", []byte(`{"TableName":"b"}`)) + doRequest(t, h, http.MethodPost, "/"+testAccount+"/Tables", []byte(`{"TableName":"a"}`)) + + rec := doRequest(t, h, http.MethodGet, "/"+testAccount+"/Tables", nil) + require.Equal(t, http.StatusOK, rec.Code) + + var body struct { + Value []map[string]any `json:"value"` + } + require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &body)) + require.Len(t, body.Value, 2) + assert.Equal(t, "a", body.Value[0]["TableName"]) + assert.Equal(t, "b", body.Value[1]["TableName"]) +} + +func TestListTables_NoMetadataLevel(t *testing.T) { + t.Parallel() + + h := newTestHandler(t) + doRequest(t, h, http.MethodPost, "/"+testAccount+"/Tables", []byte(`{"TableName":"a"}`)) + + req := httptest.NewRequest(http.MethodGet, "/"+testAccount+"/Tables", http.NoBody) + req.Header.Set("Accept", "application/json;odata=nometadata") + + e := echo.New() + rec := httptest.NewRecorder() + c := e.NewContext(req, rec) + require.NoError(t, h.Handler()(c)) + + var body struct { + Value []map[string]any `json:"value"` + } + require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &body)) + require.Len(t, body.Value, 1) + assert.NotContains(t, body.Value[0], "odata.metadata") + assert.Contains(t, rec.Header().Get("Content-Type"), "odata=nometadata") +} + +func TestListTables_FullMetadataLevel(t *testing.T) { + t.Parallel() + + h := newTestHandler(t) + doRequest(t, h, http.MethodPost, "/"+testAccount+"/Tables", []byte(`{"TableName":"a"}`)) + + req := httptest.NewRequest(http.MethodGet, "/"+testAccount+"/Tables", http.NoBody) + req.Header.Set("Accept", "application/json;odata=fullmetadata") + + e := echo.New() + rec := httptest.NewRecorder() + c := e.NewContext(req, rec) + require.NoError(t, h.Handler()(c)) + + var body struct { + Value []map[string]any `json:"value"` + } + require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &body)) + require.Len(t, body.Value, 1) + assert.Contains(t, body.Value[0], "odata.type") + assert.Contains(t, body.Value[0], "odata.id") + assert.Contains(t, body.Value[0], "odata.editLink") +} + +func TestDeleteTable(t *testing.T) { + t.Parallel() + + t.Run("success", func(t *testing.T) { + t.Parallel() + + h := newTestHandler(t) + doRequest(t, h, http.MethodPost, "/"+testAccount+"/Tables", []byte(`{"TableName":"gone"}`)) + + rec := doRequest(t, h, http.MethodDelete, "/"+testAccount+"/Tables('gone')", nil) + assert.Equal(t, http.StatusNoContent, rec.Code) + + rec = doRequest(t, h, http.MethodGet, "/"+testAccount+"/Tables", nil) + assert.JSONEq(t, `{"value":[]}`, rec.Body.String()) + }) + + t.Run("not_found", func(t *testing.T) { + t.Parallel() + + h := newTestHandler(t) + rec := doRequest(t, h, http.MethodDelete, "/"+testAccount+"/Tables('nope')", nil) + assert.Equal(t, http.StatusNotFound, rec.Code) + assert.Equal(t, "TableNotFound", rec.Header().Get("X-Ms-Error-Code")) + }) + + t.Run("invalid_literal", func(t *testing.T) { + t.Parallel() + + h := newTestHandler(t) + rec := doRequest(t, h, http.MethodDelete, "/"+testAccount+"/Tables(nope)", nil) + assert.Equal(t, http.StatusBadRequest, rec.Code) + assert.Equal(t, "InvalidInput", rec.Header().Get("X-Ms-Error-Code")) + }) +} diff --git a/test/integration/azuretable_test.go b/test/integration/azuretable_test.go new file mode 100644 index 0000000000..5cf5b13895 --- /dev/null +++ b/test/integration/azuretable_test.go @@ -0,0 +1,210 @@ +package integration_test + +import ( + "testing" + + "github.com/Azure/azure-sdk-for-go/sdk/azcore" + "github.com/Azure/azure-sdk-for-go/sdk/data/aztables" + "github.com/google/uuid" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// azureTableDevAccountName and azureTableDevAccountKey are Azurite's +// published well-known development storage account name/key, which +// gopherstack accepts as its default identity (see pkgs/azureauth and +// AZURE.md section 5) so that unmodified Azure SDKs pointed at this server +// work out of the box. Mirrors azurequeue_test.go's identical constants. +const ( + azureTableDevAccountName = "devstoreaccount1" + azureTableDevAccountKey = "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==" +) + +// createAzureTableServiceClient returns an azure-sdk-for-go Table service +// client pointed at the shared test container's dedicated Azure Table port +// (see azureTableEndpoint in main_test.go). Skips the calling test if that +// port could not be determined (mirrors createAzureQueueClient). +func createAzureTableServiceClient(t *testing.T) *aztables.ServiceClient { + t.Helper() + + if azureTableEndpoint == "" { + t.Skip("Azure Table endpoint not available (mapped port could not be determined)") + } + + cred, err := aztables.NewSharedKeyCredential(azureTableDevAccountName, azureTableDevAccountKey) + require.NoError(t, err, "unable to build SharedKeyCredential") + + // Path-style addressing (account name as the first path segment), matching + // Azurite's own convention and gopherstack's single-account routing. + client, err := aztables.NewServiceClientWithSharedKey( + azureTableEndpoint+"/"+azureTableDevAccountName, cred, nil, + ) + require.NoError(t, err, "unable to construct Azure Table service client") + + return client +} + +func TestIntegration_AzureTable_TableAndEntityLifecycle(t *testing.T) { + t.Parallel() + dumpContainerLogsOnFailure(t) + service := createAzureTableServiceClient(t) + ctx := t.Context() + + tableName := "testtable" + uuid.NewString()[:8] + tableClient := service.NewClient(tableName) + + // CreateTable + _, err := tableClient.CreateTable(ctx, nil) + require.NoError(t, err) + + // ListTables: created table should appear + found := false + + pager := service.NewListTablesPager(nil) + for pager.More() { + page, pageErr := pager.NextPage(ctx) + require.NoError(t, pageErr) + + for _, tbl := range page.Tables { + if tbl.Name != nil && *tbl.Name == tableName { + found = true + } + } + } + + assert.True(t, found, "created table should appear in ListTables") + + // Insert entity with a mixed-EDM-type property set. + entity := aztables.EDMEntity{ + Entity: aztables.Entity{PartitionKey: "partition1", RowKey: "row1"}, + Properties: map[string]any{ + "StringProp": "hello", + "IntProp": int32(42), + "BoolProp": true, + "DoubleProp": 3.14, + "Int64Prop": aztables.EDMInt64(9223372036854775807), + "GuidProp": aztables.EDMGUID("550e8400-e29b-41d4-a716-446655440000"), + "BinaryProp": aztables.EDMBinary([]byte("gopherstack")), + }, + } + + marshaled, err := entity.MarshalJSON() + require.NoError(t, err) + + _, err = tableClient.AddEntity(ctx, marshaled, nil) + require.NoError(t, err) + + // GetEntity: verify the mixed-EDM-type round trip. + getResp, err := tableClient.GetEntity(ctx, "partition1", "row1", nil) + require.NoError(t, err) + + var got aztables.EDMEntity + require.NoError(t, got.UnmarshalJSON(getResp.Value)) + assert.Equal(t, "hello", got.Properties["StringProp"]) + assert.Equal(t, int32(42), got.Properties["IntProp"]) + assert.InDelta(t, 3.14, got.Properties["DoubleProp"], 0.0001) + assert.Equal(t, aztables.EDMInt64(9223372036854775807), got.Properties["Int64Prop"]) + assert.Equal(t, aztables.EDMGUID("550e8400-e29b-41d4-a716-446655440000"), got.Properties["GuidProp"]) + assert.Equal(t, aztables.EDMBinary([]byte("gopherstack")), got.Properties["BinaryProp"]) + + // Query with $filter. + filter := "PartitionKey eq 'partition1' and IntProp eq 42" + listPager := tableClient.NewListEntitiesPager(&aztables.ListEntitiesOptions{Filter: &filter}) + + queryFound := false + + for listPager.More() { + page, pageErr := listPager.NextPage(ctx) + require.NoError(t, pageErr) + + queryFound = queryFound || len(page.Entities) > 0 + } + + assert.True(t, queryFound, "query with $filter should return the inserted entity") + + // MergeEntity: only StringProp changes; other properties survive. + mergeEntity := aztables.EDMEntity{ + Entity: aztables.Entity{PartitionKey: "partition1", RowKey: "row1"}, + Properties: map[string]any{"StringProp": "merged"}, + } + + mergeMarshaled, err := mergeEntity.MarshalJSON() + require.NoError(t, err) + + _, err = tableClient.UpdateEntity( + ctx, + mergeMarshaled, + &aztables.UpdateEntityOptions{UpdateMode: aztables.UpdateModeMerge}, + ) + require.NoError(t, err) + + getResp, err = tableClient.GetEntity(ctx, "partition1", "row1", nil) + require.NoError(t, err) + + var afterMerge aztables.EDMEntity + require.NoError(t, afterMerge.UnmarshalJSON(getResp.Value)) + assert.Equal(t, "merged", afterMerge.Properties["StringProp"]) + assert.Equal(t, int32(42), afterMerge.Properties["IntProp"], "merge must not drop unrelated properties") + + // ReplaceEntity: drops unrelated properties. + replaceEntity := aztables.EDMEntity{ + Entity: aztables.Entity{PartitionKey: "partition1", RowKey: "row1"}, + Properties: map[string]any{"StringProp": "replaced"}, + } + + replaceMarshaled, err := replaceEntity.MarshalJSON() + require.NoError(t, err) + + replaceResp, err := tableClient.UpdateEntity( + ctx, replaceMarshaled, &aztables.UpdateEntityOptions{UpdateMode: aztables.UpdateModeReplace}, + ) + require.NoError(t, err) + + getResp, err = tableClient.GetEntity(ctx, "partition1", "row1", nil) + require.NoError(t, err) + + var afterReplace aztables.EDMEntity + require.NoError(t, afterReplace.UnmarshalJSON(getResp.Value)) + assert.Equal(t, "replaced", afterReplace.Properties["StringProp"]) + _, hasIntProp := afterReplace.Properties["IntProp"] + assert.False(t, hasIntProp, "replace must drop properties not present in the new body") + + // Conditional delete with a wrong ETag: expect 412. + wrongETag := azcore.ETag(`W/"datetime'bogus'"`) + _, err = tableClient.DeleteEntity(ctx, "partition1", "row1", &aztables.DeleteEntityOptions{IfMatch: &wrongETag}) + require.Error(t, err) + + // Delete with the correct ETag. + _, err = tableClient.DeleteEntity( + ctx, + "partition1", + "row1", + &aztables.DeleteEntityOptions{IfMatch: &replaceResp.ETag}, + ) + require.NoError(t, err) + + // Verify gone. + _, err = tableClient.GetEntity(ctx, "partition1", "row1", nil) + require.Error(t, err) + + // DeleteTable. + _, err = tableClient.Delete(ctx, nil) + require.NoError(t, err) + + // Verify gone. + found = false + + pager = service.NewListTablesPager(nil) + for pager.More() { + page, pageErr := pager.NextPage(ctx) + require.NoError(t, pageErr) + + for _, tbl := range page.Tables { + if tbl.Name != nil && *tbl.Name == tableName { + found = true + } + } + } + + assert.False(t, found, "deleted table should no longer appear in ListTables") +} diff --git a/test/integration/main_test.go b/test/integration/main_test.go index 217f4753d7..bd0c78fda8 100644 --- a/test/integration/main_test.go +++ b/test/integration/main_test.go @@ -139,6 +139,18 @@ var azureBlobEndpoint string //nolint:gochecknoglobals // Set in TestMain for integration tests. var azureQueueEndpoint string +// azureTableEndpoint is the Azure Table Storage-compatible endpoint for the +// running Gopherstack container (its own dedicated port -- see +// services/azuretable/provider.go and AZURE.md section 4 for why this +// service cannot share the main AWS endpoint/port, or either of AzureBlob's/ +// AzureQueue's own dedicated ports). Left empty (and Azure Table tests +// skipped) if the mapped port cannot be determined, mirroring +// azureQueueEndpoint's non-fatal behavior above. This is initialized by +// TestMain before running integration tests. +// +//nolint:gochecknoglobals // Set in TestMain for integration tests. +var azureTableEndpoint string + // sharedContainer holds a reference to the container for cleanup and log dumping on test failures. // This is initialized by TestMain before running integration tests. // @@ -263,7 +275,7 @@ func TestMain(m *testing.M) { options.PullParent = false }, }, - ExposedPorts: []string{"8000/tcp", "1883/tcp", "10000/tcp", "10001/tcp"}, + ExposedPorts: []string{"8000/tcp", "1883/tcp", "10000/tcp", "10001/tcp", "10002/tcp"}, WaitingFor: wait.ForAll( wait.ForHTTP("/"). WithPort("8000/tcp"). @@ -275,6 +287,8 @@ func TestMain(m *testing.M) { WithStartupTimeout(60*time.Second), wait.ForListeningPort("10001/tcp"). WithStartupTimeout(60*time.Second), + wait.ForListeningPort("10002/tcp"). + WithStartupTimeout(60*time.Second), ), } @@ -332,6 +346,14 @@ func TestMain(m *testing.M) { logger.Info("Azure Queue Storage-compatible endpoint running", "endpoint", azureQueueEndpoint) } + azureTablePort, err := container.MappedPort(ctx, "10002") + if err != nil { + logger.Warn("failed to get Azure Table mapped port; Azure Table tests will be skipped", "error", err) + } else { + azureTableEndpoint = "http://localhost:" + azureTablePort.Port() + logger.Info("Azure Table Storage-compatible endpoint running", "endpoint", azureTableEndpoint) + } + code := m.Run() if sharedContainer != nil {