Large RPC payloads

Readme.md

@@ -96,19 +96,19 @@ message-specific payload.
 
 ### Message Types
 
-| Value  | Name                  | arg1                        | arg2        | Payload                                 |
-| ------ | --------------------- | --------------------------- | ----------- | --------------------------------------- |
-| `0x00` | UNKNOWN               | —                           | —           | —                                       |
-| `0x01` | DATA                  | —                           | `target_id` | Raw value bytes                         |
-| `0x02` | CONFIGURATION_REQUEST | —                           | —           | Empty                                   |
-| `0x03` | CLAIM                 | `0`=request, `1`=ack        | —           | `ClaimPayload` (request) or empty (ack) |
-| `0x04` | HEARTBEAT             | —                           | —           | Empty                                   |
-| `0x05` | TRANSACTION           | `0`=data, `1`=configuration | —           | Sequence of `DataDescriptor`+payload    |
-| `0x7F` | LOG                   | log level (1–7)             | —           | UTF-8 string (max 255 bytes)            |
-| `0x80` | SERVICE_ADVERTISEMENT | —                           | —           | CBOR-encoded service JSON               |
-| `0x81` | SERVICE_QUERY         | —                           | —           | —                                       |
-| `0x82` | DESCRIPTION_REQUEST   | offset into the omitted desc | `max_size`  | `DescriptionRequestPayload` (ip:port)   |
-| `0x83` | DESCRIPTION_PART      | offset into the omitted desc | `total_size`| Window of the omitted description       |
+| Value  | Name                  | arg1                         | arg2         | Payload                                 |
+| ------ | --------------------- | ---------------------------- | ------------ | --------------------------------------- |
+| `0x00` | UNKNOWN               | —                            | —            | —                                       |
+| `0x01` | DATA                  | —                            | `target_id`  | Raw value bytes                         |
+| `0x02` | CONFIGURATION_REQUEST | —                            | —            | Empty                                   |
+| `0x03` | CLAIM                 | `0`=request, `1`=ack         | —            | `ClaimPayload` (request) or empty (ack) |
+| `0x04` | HEARTBEAT             | —                            | —            | Empty                                   |
+| `0x05` | TRANSACTION           | `0`=data, `1`=configuration  | —            | Sequence of `DataDescriptor`+payload    |
+| `0x7F` | LOG                   | log level (1–7)              | —            | UTF-8 string (max 255 bytes)            |
+| `0x80` | SERVICE_ADVERTISEMENT | —                            | —            | CBOR-encoded service JSON               |
+| `0x81` | SERVICE_QUERY         | —                            | —            | —                                       |
+| `0x82` | DESCRIPTION_REQUEST   | offset into the omitted desc | `max_size`   | `DescriptionRequestPayload` (ip:port)   |
+| `0x83` | DESCRIPTION_PART      | offset into the omitted desc | `total_size` | Window of the omitted description       |
 
 `DESCRIPTION_REQUEST` is sent unicast by a host that needs the part of the schema the
 advertisement could not carry (see *Service description* below); the service answers with one
@@ -245,20 +245,46 @@ Services are defined in a JSON file. The code generator (`codegen/`) reads this
 }
 ```
 
-**Valid types:** `char`, `uint8_t`, `uint16_t`, `uint32_t`, `uint64_t`, `int8_t`, `int16_t`, `int32_t`, `int64_t`, `float`, `double`, `blob` (registers only), or fixed-length arrays of any scalar type as `type[N]`.
+**Valid types:** `char`, `uint8_t`, `uint16_t`, `uint32_t`, `uint64_t`, `int8_t`, `int16_t`, `int32_t`, `int64_t`, `float`, `double`, `blob` (registers, inputs, outputs, RPC parameters and RPC return values), or fixed-length arrays of any scalar type as `type[N]`.
+
+`blob` is variable length: as a parameter it is passed as `(pointer, length)` into the received packet and is
+not copied, as an input or output it is passed/sent as `(pointer, length)` with the length in bytes, as a
+register it is the configuration blob, and as a return value it uses the shared buffer pool
+(see [Shared Buffer Pool](#shared-buffer-pool)). A `type[N]` return above 128 bytes is pooled the same way, so a
+service never reserves stack for a payload it rarely sends.
+
+Switching a parameter or return from a fixed `type[N]` to `blob` changes its wire layout, so the service needs a
+version bump (`"version": 2`). Generated clients refuse a mismatch instead of sending a call with the wrong
+layout: the C++ and Python clients always compare against the schema they were given, the Go client compares
+only when `ExpectVersion` is set (generated clients set it, a hand-written one has to do the same). Service and
+clients have to be updated together.
 
 **IDs** must be unique within each section (`inputs`, `outputs`, `registers`).
 
 **Generated API (service side):**
 - `void On{Name}Changed(const T* value, uint32_t length)` — called when an input arrives (array types)
 - `void On{Name}Changed(const T& value)` — called when an input arrives (scalar types)
+- `void On{Name}Changed(const uint8_t* value, uint32_t length)` — called when a blob input arrives, `length` in bytes
 - `bool Send{Name}(const T* data, uint32_t length)` — send an output (array types)
 - `bool Send{Name}(const T& data)` — send an output (scalar types)
+- `bool Send{Name}(const uint8_t* data, uint32_t length)` — send a blob output, `length` in bytes
 - `bool OnRegister{Name}Changed(const void* data, size_t length)` — called when a register is set (blob registers)
 - Struct `{Name}` with `value`, `length` (arrays), and `valid` fields — for non-blob registers
+- `void RPC{Name}(uint16_t call_id, ...)` — one pure virtual per function; a `blob` parameter arrives as
+  `const uint8_t* <Name>, uint32_t <Name>Len` and is only valid until the handler returns
+  (copy it if you need it longer)
+- A `type[N]` return above the stack threshold and a `blob` return append `uint8_t* data,
+  uint16_t* response_length`. A `blob` return is always `uint8_t`, a `type[N]` return uses its element
+  type; `*response_length` is a byte count for `blob` and an element count for `type[N]`
 
 **Generated API (interface side — same JSON, different template):**
 - Inputs and outputs swap roles: outputs become callbacks, inputs become send methods.
+- Every `blob` parameter gets a frame budget helper, e.g. `MaxFileWriteData(PathLen)` in C++ and
+  `MaxFileWriteData(Path []byte)` in Go: how many bytes the parameter can carry in one call, with the
+  other blob parameters as arguments. Chunking a value needs exactly that number, and the framing that
+  limits it belongs to the generator, not to every client. The helpers saturate: a call whose other
+  parameters already use up the packet reports 0, so a chunker has to stop instead of sending. Every
+  chunk is `min(bytes left, budget)` long - the budget is an upper bound, only the last chunk is shorter.
 
 **CMake integration:**
 ```cmake
@@ -275,6 +301,61 @@ Then inherit from the generated `MyServiceNameBase` or `MyInterfaceNameBase` cla
 
 ---
 
+## Shared Buffer Pool
+
+An RPC return value is limited by `max_packet_size - sizeof(XbotHeader)` = 1448 bytes, but a service
+thread stack is much smaller. Array return values above a small threshold therefore do not live on
+the stack (that is what used to force the fixed size of a transfer type); the generated dispatcher
+borrows a block from a pool that all services share:
+
+```cpp
+#include <xbot-service/Buffer.hpp>
+
+auto lease = xbot::service::buffer::acquire(required_bytes);
+if (!lease) {
+  SendRpcResponse(call_id, xbot::datatypes::RpcStatus::BUSY, nullptr, 0);
+  return;
+}
+fill(lease.data(), lease.size());  // lease.size() is the capacity, not the request
+SendRpcResponse(call_id, xbot::datatypes::RpcStatus::SUCCESS, lease.data(), used_bytes);
+```
+
+Rules that keep the pool usable:
+
+- A lease is **move-only** and returns its block in the destructor. Do not keep it beyond the call
+  that borrowed it: the pool is shared by all services, and a lease held across a flash operation or
+  across RPCs starves the next caller.
+- `acquire()` is **non-blocking** on purpose. It returns an empty lease when every block is out, when
+  no storage has been configured yet, and when the request is larger than a block; a handler that
+  follows the pattern above answers BUSY in all three cases - an RPC handler must never wait for
+  another service's transfer. With the default of two blocks, one outstanding return does not exhaust
+  the pool. A block is only held while a return value is filled and copied into a packet.
+- Release happens after the handler returned; `SendRpcResponse()` copies the payload into a packet.
+- No heap is involved: the storage is static.
+
+Sizing is a platform decision. `xbot::config::buffer_block_size` (1536 by default, must hold one
+packet payload - asserted at build time) and `buffer_block_count` (2) only size the framework
+internal default storage. A platform that brings its own storage - e.g. a section in a RAM region
+that is otherwise unused - calls
+
+```cpp
+alignas(xbot::service::buffer::block_alignment) uint8_t storage[2 * 1536]
+    __attribute__((section(".sram4")));
+xbot::service::buffer::initialize(storage, 1536, 2);
+```
+
+once, before the services start, and sets `XBOT_BUFFER_DEFAULT_POOL=0` so the default array is not
+linked in at all. `initialize()` is not thread safe and refuses to run while a lease is out.
+
+Parameters hit the same stack limit from the other direction: a fixed `type[N]` parameter is copied into a
+stack array by the generated dispatcher, which is why a variable length payload had to be declared as
+e.g. `uint8_t[256]`. A `blob` parameter is passed as `(pointer, length)` into the received packet instead -
+the contract `handleData()` and `setRegister()` already use - so it costs no stack and no copy, and its
+length is only bounded by one packet. The generated dispatcher takes returns above 128 bytes and blob
+returns from the pool; small returns stay on the stack, so the pool is not touched in the hot path.
+
+---
+
 ## Python Interface
 
 `xbot_service_interface_py` is a pure-Python client for xBot services — no codegen step required. It talks directly

codegen/templates/ServiceInterfaceTemplate.cpp

@@ -28,7 +28,9 @@ void ServiceTemplateInterfaceBase::OnData(uint16_t service_id, uint64_t timestam
             /*[[[cog
             for o in service['outputs']:
                 cog.outl(f"case {o['id']}:");
-                if o['is_array']:
+                if o['is_blob']:
+                    cog.outl(f"{o['callback_name']}(static_cast<const uint8_t*>(payload), length);")
+                elif o['is_array']:
                     cog.outl(f"if(length % sizeof({o['type']}) != 0) {{");
                     cog.outl("    spdlog::error(\"Invalid data size\");");
                     cog.outl("    return;");
@@ -56,6 +58,9 @@ void ServiceTemplateInterfaceBase::OnData(uint16_t service_id, uint64_t timestam
             }
             OnExampleOutput2Changed(*static_cast<const uint32_t*>(payload));
             break;
+            case 2:
+            OnExampleOutputBlobChanged(static_cast<const uint8_t*>(payload), length);
+            break;
             //[[[end]]]
             default:
                 return;
@@ -96,7 +101,11 @@ bool ServiceTemplateInterfaceBase::SetRegisterRegister4Optional(const uint32_t &
 /*[[[cog
 # Generate send function implementations.
 for input in service["inputs"]:
-    if input['is_array']:
+    if input['is_blob']:
+        cog.outl(f"bool {service['interface_class_name']}::{input['send_method_name']}(const uint8_t* data, uint32_t length) {{")
+        cog.outl(f"    return SendData({input['id']}, data, length, false);")
+        cog.outl("}")
+    elif input['is_array']:
         cog.outl(f"bool {service['interface_class_name']}::{input['send_method_name']}(const {input['type']}* data, uint32_t length) {{")
         cog.outl(f"    return SendData({input['id']}, data, length*sizeof({input['type']}), false);")
         cog.outl("}")
@@ -111,6 +120,9 @@ bool ServiceTemplateInterfaceBase::SendExampleInput1(const char* data, uint32_t
 bool ServiceTemplateInterfaceBase::SendExampleInput2(const uint32_t &data) {
     return SendData(1, &data, sizeof(uint32_t), false);
 }
+bool ServiceTemplateInterfaceBase::SendExampleInputBlob(const uint8_t* data, uint32_t length) {
+    return SendData(2, data, length, false);
+}
 //[[[end]]]
 
 /*[[[cog
@@ -119,11 +131,15 @@ for func in service["functions"]:
     # Build method signature
     sig_params = []
     for p in func["parameters"]:
-        if p['is_array']:
+        if p['is_blob']:
+            sig_params.append(f"const uint8_t* {p['name']}, uint32_t {p['name']}Len")
+        elif p['is_array']:
             sig_params.append(f"const {p['type']}* {p['name']}, uint32_t {p['name']}Len")
         else:
             sig_params.append(f"const {p['type']}& {p['name']}")
-    if func['return_is_array']:
+    if func['return_is_blob']:
+        sig_params.append("uint8_t* data, uint16_t& result_length")
+    elif func['return_is_array']:
         sig_params.append(f"{func['return_base_type']}* data, uint16_t& result_length")
     elif func["return_type"] != "void":
         sig_params.append(f"{func['return_type']}& result")
@@ -135,9 +151,13 @@ for func in service["functions"]:
     if func["parameters"]:
         cog.outl("  std::vector<uint8_t> params_buf;")
         for p in func["parameters"]:
-            if p['is_array']:
+            if p['is_blob'] or p['is_array']:
+                if p['is_blob']:
+                    byte_len_expr = f"{p['name']}Len"
+                else:
+                    byte_len_expr = f"{p['name']}Len * sizeof({p['type']})"
                 cog.outl(f"  {{")
-                cog.outl(f"    const size_t byte_len = {p['name']}Len * sizeof({p['type']});")
+                cog.outl(f"    const size_t byte_len = {byte_len_expr};")
                 cog.outl(f"    const size_t off = params_buf.size();")
                 cog.outl(f"    params_buf.resize(off + sizeof(xbot::datatypes::DataDescriptor) + byte_len);")
                 cog.outl(f"    auto* desc = reinterpret_cast<xbot::datatypes::DataDescriptor*>(params_buf.data() + off);")
@@ -158,7 +178,12 @@ for func in service["functions"]:
         params_call = "nullptr, 0"
 
     # Call SendRpc with appropriate response buffer
-    if func['return_is_array']:
+    if func['return_is_blob']:
+        cog.outl(f"  size_t resp_size = result_length;")
+        cog.outl(f"  const auto r = SendRpc({func['id']}, {params_call}, data, &resp_size, timeout_ms);")
+        cog.outl(f"  if (r != RPC_OK) return false;")
+        cog.outl(f"  result_length = static_cast<uint16_t>(resp_size);")
+    elif func['return_is_array']:
         cog.outl(f"  size_t resp_size = result_length * sizeof({func['return_base_type']});")
         cog.outl(f"  const auto r = SendRpc({func['id']}, {params_call}, reinterpret_cast<uint8_t*>(data), &resp_size, timeout_ms);")
         cog.outl(f"  if (r != RPC_OK) return false;")
@@ -257,4 +282,28 @@ bool ServiceTemplateInterfaceBase::CallMixedParamsWithReturn(const char* Name, u
   result = resp_buf;
   return true;
 }
+bool ServiceTemplateInterfaceBase::CallBlobParamsWithBlobReturn(const uint8_t* Data, uint32_t DataLen, uint8_t* data, uint16_t& result_length, uint32_t timeout_ms) {
+  std::vector<uint8_t> params_buf;
+  {
+    const size_t byte_len = DataLen;
+    const size_t off = params_buf.size();
+    params_buf.resize(off + sizeof(xbot::datatypes::DataDescriptor) + byte_len);
+    auto* desc = reinterpret_cast<xbot::datatypes::DataDescriptor*>(params_buf.data() + off);
+    desc->target_id = 0; desc->reserved = 0; desc->payload_size = static_cast<uint32_t>(byte_len);
+    if (byte_len > 0 && Data == nullptr) return false;
+    memcpy(params_buf.data() + off + sizeof(xbot::datatypes::DataDescriptor), Data, byte_len);
+  }
+  size_t resp_size = result_length;
+  const auto r = SendRpc(4, params_buf.data(), params_buf.size(), data, &resp_size, timeout_ms);
+  if (r != RPC_OK) return false;
+  result_length = static_cast<uint16_t>(resp_size);
+  return true;
+}
+bool ServiceTemplateInterfaceBase::CallLargeArrayReturn(uint8_t* data, uint16_t& result_length, uint32_t timeout_ms) {
+  size_t resp_size = result_length * sizeof(uint8_t);
+  const auto r = SendRpc(5, nullptr, 0, reinterpret_cast<uint8_t*>(data), &resp_size, timeout_ms);
+  if (r != RPC_OK) return false;
+  result_length = static_cast<uint16_t>(resp_size / sizeof(uint8_t));
+  return true;
+}
 //[[[end]]]