From 75d383925a29fb6cec3ae61d14ff5be43ebf0dd2 Mon Sep 17 00:00:00 2001 From: lcy-seso Date: Mon, 16 Nov 2026 00:25:55 +0800 Subject: [PATCH] [Docs] Follow TileOPs manifest argument types, merged ops and the vLLM RoPE family TileOPs now declares a call-time input's argument type and default instead of `optional`, keeps every tensor a call-time input, and accepts tensor sequences. The manifest guide, the op guide, the backend guide and their figures still described `optional`, `.value` and construction-time tensors, so readers would write specs the validator rejects. The guide now covers argument members, member(), call-time scalars, operator slots and tensor sequences, in English and Chinese alike. The API pages list ClampFwdOp, LerpFwdOp and MaskedFillFwdOp in place of their scalar and tensor halves and drop RoPENeoxPositionIdsFwdOp, whose job RoPEFwdOp(is_neox_style=True) now does. The benchmark renderer reads a row's call-time scalars, shows them beside the parameters, expands tensor sequences per element and shortens float lists such as LongRoPE's rescale_factors. --- docs/api/elementwise.md | 30 +--- docs/api/index.md | 2 +- docs/api/rope.md | 12 +- docs/backends.md | 19 ++- docs/backends.zh.md | 13 +- docs/new-op.md | 2 +- docs/new-op.zh.md | 2 +- docs/user-guide/manifest/calls.md | 28 ++-- docs/user-guide/manifest/calls.zh.md | 28 ++-- docs/user-guide/manifest/concepts.md | 7 +- docs/user-guide/manifest/concepts.zh.md | 7 +- docs/user-guide/manifest/examples.md | 11 +- docs/user-guide/manifest/examples.zh.md | 11 +- docs/user-guide/manifest/extensions.md | 173 +++++++++++++++++---- docs/user-guide/manifest/extensions.zh.md | 173 +++++++++++++++++---- docs/user-guide/manifest/img/call.svg | 2 +- docs/user-guide/manifest/img/call.zh.svg | 2 +- docs/user-guide/manifest/img/device.svg | 2 +- docs/user-guide/manifest/img/device.zh.svg | 2 +- docs/user-guide/manifest/index.md | 2 +- docs/user-guide/manifest/index.zh.md | 2 +- docs/user-guide/manifest/writing.md | 35 +++-- docs/user-guide/manifest/writing.zh.md | 35 +++-- docs/user-guide/op/composite.md | 7 +- docs/user-guide/op/composite.zh.md | 2 +- docs/user-guide/op/img/structure.svg | 2 +- docs/user-guide/op/img/structure.zh.svg | 2 +- docs/user-guide/op/index.md | 10 +- docs/user-guide/op/index.zh.md | 4 +- docs/user-guide/op/lifecycle.md | 18 +-- docs/user-guide/op/lifecycle.zh.md | 10 +- docs/user-guide/op/records.md | 2 +- docs/user-guide/op/records.zh.md | 2 +- docs/user-guide/op/structure.md | 9 +- docs/user-guide/op/structure.zh.md | 7 +- figures/user-guide/manifest/call.en.puml | 2 +- figures/user-guide/manifest/call.zh.puml | 2 +- figures/user-guide/manifest/device.en.puml | 12 +- figures/user-guide/manifest/device.zh.puml | 14 +- figures/user-guide/op/structure.en.puml | 3 +- figures/user-guide/op/structure.zh.puml | 3 +- scripts/workload_shape.py | 20 ++- 42 files changed, 467 insertions(+), 264 deletions(-) diff --git a/docs/api/elementwise.md b/docs/api/elementwise.md index d063daad..ac155ef6 100644 --- a/docs/api/elementwise.md +++ b/docs/api/elementwise.md @@ -1,9 +1,9 @@ # Elementwise Operators Every op on this page is used the same way: construct it once, then call it. The -constructor takes what the kernel is compiled with; the call takes the tensors. -Both are documented under each op — `__init__` and `forward`, where `forward` is -what runs when you call `op(...)`. +constructor takes what the kernel is compiled with; the call takes the inputs, tensors +or numbers. Both are documented under each op — `__init__` and `forward`, where +`forward` is what runs when you call `op(...)`. ## Arithmetic @@ -61,13 +61,7 @@ what runs when you call `op(...)`. heading_level: 3 members: ["__init__", "forward"] -::: tileops.elementwise.LerpScalarFwdOp - options: - show_root_heading: true - heading_level: 3 - members: ["__init__", "forward"] - -::: tileops.elementwise.LerpTensorFwdOp +::: tileops.elementwise.LerpFwdOp options: show_root_heading: true heading_level: 3 @@ -391,25 +385,13 @@ what runs when you call `op(...)`. heading_level: 3 members: ["__init__", "forward"] -::: tileops.elementwise.ClampTensorFwdOp - options: - show_root_heading: true - heading_level: 3 - members: ["__init__", "forward"] - -::: tileops.elementwise.ClampScalarFwdOp - options: - show_root_heading: true - heading_level: 3 - members: ["__init__", "forward"] - -::: tileops.elementwise.MaskedFillTensorFwdOp +::: tileops.elementwise.ClampFwdOp options: show_root_heading: true heading_level: 3 members: ["__init__", "forward"] -::: tileops.elementwise.MaskedFillScalarFwdOp +::: tileops.elementwise.MaskedFillFwdOp options: show_root_heading: true heading_level: 3 diff --git a/docs/api/index.md b/docs/api/index.md index b73804de..036af532 100644 --- a/docs/api/index.md +++ b/docs/api/index.md @@ -23,7 +23,7 @@ a tool rather than an op, comes last. The Benchmarks pages use the same order. | Page | What it covers | | --- | --- | | [Elementwise](elementwise.md) | unary and binary maps, activations, dropout, and the in-place forms | -| [RoPE](rope.md) | rotary position embedding — NeoX and interleaved layouts, Llama 3.1, YaRN, LongRoPE | +| [RoPE](rope.md) | rotary position embedding — NeoX and interleaved rotation, Llama 3.1, YaRN, LongRoPE | | [Reduction](reduction.md) | sums, extrema, arg-reductions, cumulative scans, softmax | | [Normalization](normalization.md) | RMSNorm, LayerNorm, GroupNorm, BatchNorm and the fused variants, including RMSNorm with per-token quantization and the q/k RMSNorm with RoPE | | [Pooling](pool.md) | average, max and adaptive pooling, with and without indices, plus the chunked sequence mean | diff --git a/docs/api/rope.md b/docs/api/rope.md index 83199fb6..0d83f80c 100644 --- a/docs/api/rope.md +++ b/docs/api/rope.md @@ -7,9 +7,9 @@ what runs when you call `op(...)`. ## Base frequencies -One op serves both rotation conventions: pass `rope_layout` as `"neox"` to -rotate the two halves of a head against each other, or `"interleaved"` to rotate -each adjacent pair. +One op serves both rotation conventions: pass `is_neox_style=True` to rotate the +two halves of a head against each other, or `is_neox_style=False` to rotate each +adjacent pair. ::: tileops.rope.RoPEFwdOp options: @@ -17,12 +17,6 @@ each adjacent pair. heading_level: 3 members: ["__init__", "forward"] -::: tileops.rope.RoPENeoxPositionIdsFwdOp - options: - show_root_heading: true - heading_level: 3 - members: ["__init__", "forward"] - ## Scaled frequencies ::: tileops.rope.RoPELlama31FwdOp diff --git a/docs/backends.md b/docs/backends.md index 368752cf..ccdfa17c 100644 --- a/docs/backends.md +++ b/docs/backends.md @@ -115,8 +115,14 @@ def build_kernel(*inputs: "TensorSpec | None", **params) -> Callable[..., Kernel Implemented by the backend, one per `(op, target)`. Its signature is the op's manifest signature: `inputs` correspond one-to-one to `signature.inputs` in declaration order, and -`params` are named after `signature.params`. An optional input that was not passed on -this call arrives as `None`. +`params` are named after `signature.params`. Each input arrives by what it holds on this +call: + +- a tensor arrives as its `TensorSpec`; +- a tensor sequence (`type: "list[Tensor]"`) arrives as a tuple of `TensorSpec`s; +- a call-time scalar, such as `ClampFwdOp`'s `min` passed as a number, arrives as its + Python type (`int`, `float` or `bool`), never its value; +- an input that holds `None`, passed or left at its default, arrives as `None`. ```python # GroupNormFwdOp's spec: weight and bias are optional, and arrive as None when absent @@ -325,7 +331,8 @@ This call path implies four things: decide which in-tree kernel is fetched, what it is looked up by and how it is built. Once the op settles on a target, that target serves the whole op, and none of them runs. - **Tensors arrive positionally, params by name.** `build_kernel(*inputs, **params)`: the - positional arguments are `TensorSpec`s (`None` for an optional input the call omitted), + positional arguments describe the inputs (`TensorSpec`, a tuple of them for a tensor + sequence, the Python type of a call-time scalar, `None` for an input that holds `None`), and the keyword arguments are the manifest's `params` names with the values settled for this call. - **One builder per `(op, target)`.** The backend is not told which kernel the in-tree @@ -494,7 +501,7 @@ signature: eps: {type: "float | None", default: null} inputs: # declaration order is call order x: {dtype: T, shape: "[*B, *normalized_shape]"} - weight: {dtype: T, shape: "[*normalized_shape]", optional: true} + weight: {dtype: T, shape: "[*normalized_shape]", type: "Tensor | None", default: null} ``` The corresponding builder signature: @@ -773,8 +780,8 @@ backend serves an op only when its `detect` claims the device, or when it is nam TileOPs memoises a builder's return value by **device plus input signature**: > the call's device (see [Call device](user-guide/manifest/calls.md#device)), plus `(dtype, shape)` taken per input in -> `signature.inputs` order; an optional input not passed on this call is recorded -> as `None`. +> `signature.inputs` order; an input that holds `None` on this call is recorded as `None`, +> and a call-time scalar is recorded by its Python type, never its value. **Two calls with the same device and input signature get the same kernel**, with no further call to `build_kernel`. A second card under the same target builds again, because diff --git a/docs/backends.zh.md b/docs/backends.zh.md index 5a8c1748..d56aa133 100644 --- a/docs/backends.zh.md +++ b/docs/backends.zh.md @@ -90,7 +90,12 @@ def detect(device: torch.device) -> bool: def build_kernel(*inputs: "TensorSpec | None", **params) -> Callable[..., KernelResult]: ... ``` -由 backend 实现,每组 `(op, target)` 一个。它的签名就是该 op 的 manifest 签名:`inputs` 按声明顺序与 `signature.inputs` 的条目一一对应,`params` 按 `signature.params` 命名。声明为 optional 的输入在本次调用中没有传入时,对应的实参是 `None`。 +由 backend 实现,每组 `(op, target)` 一个。它的签名就是该 op 的 manifest 签名:`inputs` 按声明顺序与 `signature.inputs` 的条目一一对应,`params` 按 `signature.params` 命名。每个输入按它在本次调用中持有的值传入: + +- 张量传入它的 `TensorSpec`; +- 张量序列(`type: "list[Tensor]"`)传入由 `TensorSpec` 组成的 tuple; +- 调用期标量(例如以数值传入的 `ClampFwdOp` 的 `min`)传入它的 Python 类型(`int`、`float` 或 `bool`),不传入它的值; +- 持有 `None` 的输入,无论是显式传入还是取了默认值,对应的实参是 `None`。 ```python # GroupNormFwdOp 的 spec:weight、bias 是可选输入,没传时实参是 None @@ -263,7 +268,7 @@ register_kernel_builder(op="GEMMFwdOp", target="acme", build_kernel=build_gemm) 这条调用路径对应以下四点: - **`kernel_for` 与各实现的 `entry_for` 只服务 in-tree 实现。** 它们决定取哪个 in-tree kernel、按什么查表以及如何构造。op 选中某个 target 之后,整个 op 由这个 target 服务,这几处都不会执行。 -- **张量按位置传入,参数按名字传入。** 在 `build_kernel(*inputs, **params)` 中,位置实参是 `TensorSpec`(没有传入的可选输入是 `None`),关键字实参是 manifest 中 `params` 的名字与本次调用的确定值。 +- **张量按位置传入,参数按名字传入。** 在 `build_kernel(*inputs, **params)` 中,位置实参描述各输入(`TensorSpec`;张量序列是由它组成的 tuple;调用期标量是它的 Python 类型;持有 `None` 的输入是 `None`),关键字实参是 manifest 中 `params` 的名字与本次调用的确定值。 - **一组 `(op, target)` 只注册一个 builder。** in-tree 路径会选用哪个实现(例如 GEMM 在 `kernel_types` 中声明的多个实现之一)不会告诉 backend,`build_kernel` 从 `TensorSpec` 自行判断应返回哪个 kernel。 - **backend 不必自己缓存构建结果。** 设备与输入签名相同时,op 层不会再次调用 `build_kernel`;需要更细的区分或更少的重建时,backend 在 `build_kernel` 内部另加一层缓存。专为外部 backend 编写的 op 既不声明 `kernel_types` 也不声明 `interfaces`,没有 target 认领设备时,调用直接抛出 `OpNotAvailableError`。 @@ -400,7 +405,7 @@ signature: eps: {type: "float | None", default: null} inputs: # 声明顺序即传入顺序 x: {dtype: T, shape: "[*B, *normalized_shape]"} - weight: {dtype: T, shape: "[*normalized_shape]", optional: true} + weight: {dtype: T, shape: "[*normalized_shape]", type: "Tensor | None", default: null} ``` 对应的 builder 签名: @@ -618,7 +623,7 @@ TileOPs 的 in-tree kernel([`src/tileops/_kernels/`](https://github.com/tile-a TileOPs 按**设备加输入签名**记住 `build_kernel` 的返回值。这个键的构成是: -> 调用设备(见[调用设备](user-guide/manifest/calls.md#device)),加上按 `signature.inputs` 顺序逐项取出的 `(dtype, shape)`;声明为 optional 而本次没有传入的输入,这一项记 `None`。 +> 调用设备(见[调用设备](user-guide/manifest/calls.md#device)),加上按 `signature.inputs` 顺序逐项取出的 `(dtype, shape)`;本次持有 `None` 的输入,这一项记 `None`;调用期标量按它的 Python 类型记录,不记录它的值。 因此,**设备与输入签名都相同的两次调用,TileOPs 使用同一个 kernel**,不再调用 `build_kernel`。同一个 target 的第二块卡会重新构造一次,因为为一块卡编译出的产物不一定能在另一块卡上启动。参数不进入这个键,因为它们对一个 op 实例而言是固定的。 diff --git a/docs/new-op.md b/docs/new-op.md index 5a0272ed..296ff324 100644 --- a/docs/new-op.md +++ b/docs/new-op.md @@ -113,7 +113,7 @@ class GEMMFwdOp(Op): | 1 | `__init__` | the names, order and defaults in `signature.params`, then the keyword-only `target`; it assigns each manifest parameter to the attribute of the same name, then calls `super().__init__(target=target)` | | 2 | `kernel_types` | the Kernel classes that can serve the op, each under a key | | 3 | `interfaces` | one entry per kernel call the op makes, mapping the name `kernel_for` uses to the `KernelInterface` class that the implementations serving that call inherit | -| 4 | `forward` | the parameters: the order of `signature.inputs`, with optional inputs last and defaulting to `None`; the body: making the inputs contiguous, writing the function that builds the call spec, fetching the kernel and calling it | +| 4 | `forward` | the parameters: the order of `signature.inputs`, an input with a manifest `default` taking it as its default and following the inputs without one; the body: making the inputs contiguous, writing the function that builds the call spec, fetching the kernel and calling it | | 5 | `roof_key` | optional: the hardware unit whose peak prices the op's FLOPs; the default is fp32 on CUDA cores, and the member is written only when the op uses another unit | `_infer_output_shapes` and `eval_roofline` are generated from the spec and are not diff --git a/docs/new-op.zh.md b/docs/new-op.zh.md index 883dc316..c1e22195 100644 --- a/docs/new-op.zh.md +++ b/docs/new-op.zh.md @@ -100,7 +100,7 @@ class GEMMFwdOp(Op): | 1 | `__init__` | `signature.params` 的名字、顺序与默认值,再加上仅限关键字的 `target`;先把每个 manifest 参数赋给同名属性,再调用 `super().__init__(target=target)` | | 2 | `kernel_types` | 能服务这个 op 的 Kernel 类,各对应一个 key | | 3 | `interfaces` | op 发出的每一个 kernel 调用各占一条,从 `kernel_for` 使用的名字映射到服务这个调用的各实现所继承的 `KernelInterface` 类 | -| 4 | `forward` | 参数按 `signature.inputs` 的顺序,可选输入排在最后,默认值为 `None`;方法体把输入转为连续张量,写出构造 call spec 的函数,取得 kernel,再调用它 | +| 4 | `forward` | 参数按 `signature.inputs` 的顺序,有 manifest `default` 的输入以它为默认值,排在没有 `default` 的输入之后;方法体把输入转为连续张量,写出构造 call spec 的函数,取得 kernel,再调用它 | | 5 | `roof_key` | 可选。表示 op 的 FLOPs 按哪个硬件单元的峰值计算,默认是 CUDA core 上的 fp32,只在使用其他单元时编写 | `_infer_output_shapes` 与 `eval_roofline` 都依照 spec 生成,不需要编写。 diff --git a/docs/user-guide/manifest/calls.md b/docs/user-guide/manifest/calls.md index 782f0a34..a05e204e 100644 --- a/docs/user-guide/manifest/calls.md +++ b/docs/user-guide/manifest/calls.md @@ -4,17 +4,19 @@ A call has two phases, construction and call, and the checks in both phases are ## 1. The two phases of a call {#call} -In the **construction** phase, the op checks parameter values against their `type` (including whether a dtype parameter is in the allowed set), checks that every non-optional construction-time tensor is given, and checks the `invariant` of ADTs. The following are fixed at construction: +In the **construction** phase, the op checks parameter values against their `type` (including whether a dtype parameter is in the allowed set), and checks the `invariant` of ADTs. The following are fixed at construction: - the values of `Bool`, enum and ADT parameters; -- whether `Maybe` parameters and construction-time tensors are given; -- the indices solved from the shapes and dtypes of construction-time tensors, and the `let` entries computable from these. +- whether `Maybe` parameters are given; +- the `let` entries computable from the parameters and ADT fields. + +The following are known only at each call: the call-time arguments, the member each holds, and the indices solved from their tensors. Checks such as non-negative axes, refinements and `invariant` run at construction when they can already be evaluated there, and are deferred to call time otherwise. In the **call** phase, the checks generated from the signature around `forward` run in the following order: -1. determine whether each call-time tensor is passed; +1. determine the member each call-time argument holds; 2. check domain restrictions; 3. select type family branches; 4. infer indices, and check the remaining refinements; @@ -26,7 +28,7 @@ In the **call** phase, the checks generated from the signature around `forward` When any check fails, the op raises an error that names the unsatisfied declaration. -Steps 1–5 run on the first call of each specialization: each tensor's layout, dtype, device and presence, whether each call tensor starts on a 16-byte boundary, the parameters, the construction-time tensors, the execution arguments, and the current device of a call no input places. The Op plan holds what they decide, and a later call of the same specialization runs them no more. Steps 6, 7 and 9 run on every call. +Steps 1–5 run on the first call of each specialization: each tensor's layout, dtype, device and presence, whether each call tensor starts on a 16-byte boundary, the type of each call-time scalar, the parameters, the execution arguments, and the current device of a call no input places. The Op plan holds what they decide, and a later call of the same specialization runs them no more. Steps 6, 7 and 9 run on every call, and so do the refinements and `let` entries that read a call-time scalar's value. If every tensor a call writes (each output and each written input) has no elements, step 8 runs no implementation: new outputs are created on the call device with the checked shapes and dtypes, and `out` and the written inputs are returned unchanged. When the inputs are empty but the outputs are not, the implementation runs as usual. @@ -44,7 +46,7 @@ The selection rules, how to add a kernel, and how a backend joins are in [How an ## 3. type inference {#inference} -The construction phase first solves the indices that construction-time tensors determine, and the call phase then solves the remaining indices from the call-time tensors actually passed. The validator fixes the solving order in advance, independent of the order in which declarations are written. +The construction phase first settles the type family branches the parameters select and the `let` entries they make computable, and the call phase then solves the remaining indices from the call-time tensors actually passed. The validator fixes the solving order in advance, independent of the order in which declarations are written. **Table 1** Unification rules for input axes @@ -76,10 +78,9 @@ The call device is determined as shown below. At workload instantiation, tensors ![Call device](img/device.svg) -- A tensor that declares `device: cpu` is always on the CPU and does not take part in deciding the call device; among construction-time tensors, too, only those that do not declare `device: cpu` take part. -- When there is no call-time tensor, no `device` parameter and no construction-time tensor that takes part, the call device is the current device of the process's accelerator, or `None` if the process has no accelerator. The accelerator is the device type `torch.accelerator` reports: CUDA, or a backend's device type. +- A tensor that declares `device: cpu` is always on the CPU and does not take part in deciding the call device. Every element of a tensor sequence takes part. +- When there is no call-time tensor and no `device` parameter, the call device is the current device of the process's accelerator, or `None` if the process has no accelerator. The accelerator is the device type `torch.accelerator` reports: CUDA, or a backend's device type. - Once the call device is determined, `out` and each output are checked or allocated on that device. -- A construction-time tensor that does not declare `device: cpu` is copied to the call device at call time and converted to the dtype in the signature. - For a tensor that declares `contiguous: true`, the generated checks confirm that it is contiguous in memory; tensors generated by workload instantiation are always contiguous. ## 5. torch.compile and SymInt {#symint} @@ -97,12 +98,12 @@ The conventions a caller follows under `torch.compile` are in [Bringing an op in [`scripts/validate_manifest.py`](https://github.com/tile-ai/TileOPs/blob/main/scripts/validate_manifest.py) checks each spec on every combination of discriminant values. The quantities that take part in the combinations are: - the `match` of type families; -- `optional` and `nullable`; +- the `member` of each argument with several members, and `nullable`; - `mutated`; - whether output buffers are passed; - the quantities involved in deciding whether an index is used. -For combinations excluded by a domain restriction, the validator skips only two checks: type family coverage and the solving order. If a spec has more value combinations to check than the configured limit (256 by default), the validator issues an advisory-level notice but still checks the spec in full. +For combinations excluded by a domain restriction, the validator skips only three checks: type family coverage, the solving order, and where a call-time scalar's value is read. If a spec has more value combinations accepted by its domain restrictions than the configured limit (256 by default), the validator issues an advisory-level notice but still checks the spec in full. On each value combination, the validator checks the following: @@ -110,7 +111,7 @@ On each value combination, the validator checks the following: 2. The `cases` of each type family have no gaps and no overlaps over the accepted values, and references between type families have no cycle. 3. Every index can be solved (see [§ 3](#inference)). 4. Dependencies between `let` entries have no cycle. -5. Every expression belongs to the expression language, and every primitive used is built in. +5. Every expression belongs to the expression language, every primitive used is built in, and a call-time scalar's value is read only in refinements and `let` entries. 6. On every workload row, the generator results unify with their declarations and every `requires` holds. 7. Every workload row can be instantiated. 8. On each effect branch, the operator schema, the aliasing and the roofline read and write counts agree. @@ -123,7 +124,7 @@ For an `implemented` op, the validator also checks that the code agrees with the ## 7. Rejected forms {#rejected} -`shape_rules` contains only refinements. Forms that also act as a shape declaration, a name definition or a presence test are rejected: +`shape_rules` contains only refinements. Forms that also act as a shape declaration, a name definition, or a test of presence or type are rejected; a rule reads the member an argument holds through `present(...)` or `member(...)`: **Table 3** Rejected forms and their rewrites @@ -136,3 +137,4 @@ For an `implemented` op, the validator also checks that the code agrees with the | 5 | `v is None` on a value | `max_seqlen is None` | `not present(max_seqlen)` | | 6 | `isinstance` | `s[0] if isinstance(s, tuple) else s` | `per_axis(s, 0, 2)` | | 7 | a set comprehension | `len({d % n for d in dim}) == len(dim)` | `unique_axes(dim, n)` | +| 8 | `isinstance` on an argument | `isinstance(min, torch.Tensor)` | `member(min) == 'Tensor'` | diff --git a/docs/user-guide/manifest/calls.zh.md b/docs/user-guide/manifest/calls.zh.md index d6beafbd..c9f4a98c 100644 --- a/docs/user-guide/manifest/calls.zh.md +++ b/docs/user-guide/manifest/calls.zh.md @@ -4,17 +4,19 @@ ## 1. 一次调用的两个阶段 {#call} -在**构造**阶段,op 按 `type` 检查参数的取值(包括 dtype 参数是否在允许的集合内),检查非可选的构造期张量都已给出,并检查 ADT 的 `invariant`。以下各量在构造时确定: +在**构造**阶段,op 按 `type` 检查参数的取值(包括 dtype 参数是否在允许的集合内),并检查 ADT 的 `invariant`。以下各量在构造时确定: - `Bool`、枚举与 ADT 参数的值; -- `Maybe` 参数与构造期张量是否给出; -- 由构造期张量的形状与 dtype 求得的 index,以及由这些量可以算出的 `let`。 +- `Maybe` 参数是否给出; +- 由参数与 ADT 字段可以算出的 `let`。 + +以下各量每次调用时才知道:调用期实参、每个实参持有的成员,以及由它们的张量求得的 index。 轴非负、refinement、`invariant` 等检查,如果在构造时已经可以求值,就在构造时执行,否则推迟到调用时执行。 在**调用**阶段,`forward` 外层由签名生成的检查按以下顺序执行: -1. 确定各调用期张量是否传入; +1. 确定每个调用期实参持有的成员; 2. 检查定义域限制; 3. 选择 type family 的分支; 4. 推断 index,并检查其余的 refinement; @@ -26,7 +28,7 @@ 任何一项检查失败时,op 都会报错,并指出未满足的那条声明。 -第 1 至 5 步在每个特化的第一次调用时执行。特化包括:每个张量的布局、dtype、设备与是否传入,每个调用张量的起始地址是否在 16 字节边界上,参数,构造期张量,执行参数,以及没有输入确定设备时的当前设备。Op plan 保存这几步的结论,同一特化之后的调用不再执行它们。第 6、7、9 步每次调用都执行。 +第 1 至 5 步在每个特化的第一次调用时执行。特化包括:每个张量的布局、dtype、设备与是否传入,每个调用张量的起始地址是否在 16 字节边界上,每个调用期标量的类型,参数,执行参数,以及没有输入确定设备时的当前设备。Op plan 保存这几步的结论,同一特化之后的调用不再执行它们。第 6、7、9 步每次调用都执行,读取调用期标量取值的 refinement 与 `let` 也每次调用都执行。 如果一次调用写入的所有张量(各输出与被写入的输入)都不含元素,第 8 步不执行任何实现:新的输出按检查过的形状与 dtype 在调用设备上创建,`out` 与被写入的输入原样返回。输入为空而输出不为空时,照常执行实现。 @@ -44,7 +46,7 @@ op 在 `interfaces` 中声明它调用 kernel 的位置,每个位置对应一 ## 3. type inference {#inference} -构造阶段先求出由构造期张量能确定的 index,调用阶段再从实际传入的调用期张量求出其余的 index。求解的顺序由 validator 预先确定,与声明的书写顺序无关。 +构造阶段先确定由参数选出的 type family 分支与由参数可以算出的 `let`,调用阶段再从实际传入的调用期张量求出其余的 index。求解的顺序由 validator 预先确定,与声明的书写顺序无关。 **表 1** 输入轴的 unification 规则 @@ -76,10 +78,9 @@ op 在 `interfaces` 中声明它调用 kernel 的位置,每个位置对应一 ![调用设备](img/device.svg) -- 声明了 `device: cpu` 的张量始终位于 CPU,不参与调用设备的判定;构造期张量也只有未声明 `device: cpu` 的才参与判定。 -- 没有调用期张量、`device` 参数与参与判定的构造期张量时,调用设备取进程 accelerator 的当前设备;进程没有 accelerator 时为 `None`。accelerator 是 `torch.accelerator` 报告的设备类型,CUDA 或某个 backend 的设备类型。 +- 声明了 `device: cpu` 的张量始终位于 CPU,不参与调用设备的判定。张量序列的每个元素都参与判定。 +- 没有调用期张量与 `device` 参数时,调用设备取进程 accelerator 的当前设备;进程没有 accelerator 时为 `None`。accelerator 是 `torch.accelerator` 报告的设备类型,CUDA 或某个 backend 的设备类型。 - 调用设备确定后,`out` 与各输出在该设备上检查或分配。 -- 没有声明 `device: cpu` 的构造期张量,在调用时复制到调用设备,并转换为签名中的 dtype。 - 声明了 `contiguous: true` 的张量,由生成的检查确认其内存连续;workload 实例化生成的张量总是连续的。 ## 5. torch.compile 与 SymInt {#symint} @@ -97,12 +98,12 @@ op 在 `interfaces` 中声明它调用 kernel 的位置,每个位置对应一 [`scripts/validate_manifest.py`](https://github.com/tile-ai/TileOPs/blob/main/scripts/validate_manifest.py) 在 discriminant 的每一种取值组合上检查每个 spec。参与组合的量包括: - type family 的 `match`; -- `optional` 与 `nullable`; +- 每个有多个成员的实参的 `member`,以及 `nullable`; - `mutated`; - 输出缓冲是否传入; - 判断 index 是否被用到时涉及的量。 -对于被定义域限制排除的组合,validator 只跳过 type family 覆盖与求解顺序这两项检查。如果一个 spec 需要检查的取值组合超过配置的上限(默认 256),validator 给出 advisory 级别的提示,但仍然完整地检查这个 spec。 +对于被定义域限制排除的组合,validator 只跳过三项检查:type family 覆盖、求解顺序,以及调用期标量的取值在哪里被读取。如果一个 spec 中被定义域限制接受的取值组合超过配置的上限(默认 256),validator 给出 advisory 级别的提示,但仍然完整地检查这个 spec。 在每一种取值组合上,validator 检查以下各项: @@ -110,7 +111,7 @@ op 在 `interfaces` 中声明它调用 kernel 的位置,每个位置对应一 2. type family 的 `cases` 在接受的取值上既无遗漏也无重叠,type family 之间的引用无环。 3. 每个 index 都能求得(见[第 3 节](#inference))。 4. `let` 之间的依赖无环。 -5. 每个表达式都属于表达式语言,用到的 primitive 都是内建的。 +5. 每个表达式都属于表达式语言,用到的 primitive 都是内建的,调用期标量的取值只在 refinement 与 `let` 中读取。 6. 在每条 workload 行上,generator 的结果都能与声明做 unification,`requires` 都成立。 7. 每条 workload 行都能实例化。 8. 在每个 effect 分支上,operator schema、别名关系与 roofline 的读写计数一致。 @@ -123,7 +124,7 @@ op 在 `interfaces` 中声明它调用 kernel 的位置,每个位置对应一 ## 7. 不被接受的写法 {#rejected} -`shape_rules` 中只写 refinement。兼做形状声明、名字定义或存在性判断的写法不被接受: +`shape_rules` 中只写 refinement。兼做形状声明、名字定义,或判断存在与类型的写法不被接受;规则通过 `present(...)` 或 `member(...)` 读取实参持有的成员: **表 3** 不被接受的写法及对应的改写 @@ -136,3 +137,4 @@ op 在 `interfaces` 中声明它调用 kernel 的位置,每个位置对应一 | 5 | 对值使用 `v is None` | `max_seqlen is None` | `not present(max_seqlen)` | | 6 | `isinstance` | `s[0] if isinstance(s, tuple) else s` | `per_axis(s, 0, 2)` | | 7 | 集合推导式 | `len({d % n for d in dim}) == len(dim)` | `unique_axes(dim, n)` | +| 8 | 对实参使用 `isinstance` | `isinstance(min, torch.Tensor)` | `member(min) == 'Tensor'` | diff --git a/docs/user-guide/manifest/concepts.md b/docs/user-guide/manifest/concepts.md index d0fefa75..abbe3c34 100644 --- a/docs/user-guide/manifest/concepts.md +++ b/docs/user-guide/manifest/concepts.md @@ -48,7 +48,7 @@ Every type index in a signature has exactly one source, which is one of the foll ## 3. type family {#family} -A type family is a function that gives a shape from the value of a discriminant. A discriminant is a quantity with finitely many values, such as a `Bool` parameter, an enum parameter, an ADT parameter, or whether a tensor is passed. +A type family is a function that gives a shape from the value of a discriminant. A discriminant is a quantity with finitely many values, such as a `Bool` parameter, an enum parameter, an ADT parameter, or the member a call-time argument holds, for example whether a tensor is passed. `Mat` in GEMM is a type family: when `t` is false the result is `[R, C]`, and when it is true the result is `[C, R]`. The shape of tensor `a` is written `Mat[trans_a, M, K]`, so: @@ -141,7 +141,7 @@ Effects determine the read and write counts of the roofline, and the operator sc | 2 | polymorphic function type | a function type with quantification over its type indices | `forall` | [§ 1](#pft) | | 3 | type index | a parameter name in a type | `forall`, construction parameters, `let` | [§ 2](#index) | | 4 | kind | the sort of a type index, which determines its range of values and the operations available on it | `forall: {M: Dim}` | [§ 2](#index) | -| 5 | discriminant | a quantity with finitely many values, used to select a type family branch or to decide whether a tensor exists | expressions in `match`, `optional`, `nullable`, `mutated` | [§ 3](#family) | +| 5 | discriminant | a quantity with finitely many values, used to select a type family branch or to decide whether a tensor exists | expressions in `match`, `nullable`, `mutated`, and `present(...)`, `member(...)` in refinements | [§ 3](#family) | | 6 | type family | a function that gives a shape from the value of a discriminant | `signature.types` | [§ 3](#family) | | 7 | ADT, constructor | a tagged sum type in which each constructor has its own fields | `adts` in `spec/types.yaml` | [§ 4](#adt) | | 8 | `let` | a named quantity computed from indices | `let: {H_out: "..."}` | [§ 5](#let) | @@ -149,6 +149,7 @@ Effects determine the read and write counts of the roofline, and the operator sc | 10 | unification | matching the declared shape against the actual shape axis by axis to solve unknown indices | generated from the signature | [§ 7](#unification) | | 11 | type inference | the process of solving every index at call time | generated from the signature | [§ 7](#unification) | | 12 | effect | what a call writes to its arguments, and aliasing | `mutated`, `write_only`, `buffer`, `alias`, `overlaps` | [§ 8](#effect) | -| 13 | `Maybe[X]`, `present` | an `X` that may be absent; `present(x)` says whether it is given | optional inputs, nullable outputs, `int \| None` parameters | [Extensions § 1](extensions.md#presence) | +| 13 | argument type, `member`, `present` | the members a call-time argument may hold, such as `Tensor`, `Number` and `None`; `member(x)` names the member `x` holds, and `present(x)` says it is not `None` | an input's `type` and `default`, nullable outputs, `int \| None` parameters | [Extensions § 1](extensions.md#presence) | | 14 | primitive | a built-in function callable in expressions, such as `broadcast` and `pool.out` | function calls in expressions | [Extensions § 4](extensions.md#let) | | 15 | generator | a function that generates the values of a metadata tensor at instantiation, such as `prefix_sum(q_lens)` | a tensor's `values` | [Extensions § 7](extensions.md#generators) | +| 16 | tensor sequence | a list of tensors passed as one argument, each element with its own shape and dtype | `type: "list[Tensor]"`, `Seq[Shape]`, `Seq[DType[...]]` | [Extensions § 11](extensions.md#sequences) | diff --git a/docs/user-guide/manifest/concepts.zh.md b/docs/user-guide/manifest/concepts.zh.md index f22f49cc..2645bb5c 100644 --- a/docs/user-guide/manifest/concepts.zh.md +++ b/docs/user-guide/manifest/concepts.zh.md @@ -48,7 +48,7 @@ type index 是类型中的参数名,例如 GEMM 的 `T`、`M`、`N`、`K`。 ## 3. type family {#family} -type family 是一个根据 discriminant 的取值给出形状的函数。discriminant 指取值有限的量,例如 `Bool` 参数、枚举参数、ADT 参数,以及某个张量是否传入。 +type family 是一个根据 discriminant 的取值给出形状的函数。discriminant 指取值有限的量,例如 `Bool` 参数、枚举参数、ADT 参数,以及调用期实参持有的成员,比如某个张量是否传入。 GEMM 中的 `Mat` 就是一个 type family:`t` 为 false 时结果是 `[R, C]`,为 true 时结果是 `[C, R]`。张量 `a` 的形状写作 `Mat[trans_a, M, K]`,于是: @@ -141,7 +141,7 @@ effect 决定了 roofline 的读写计数,以及有编译边界的 op 所生 | 2 | polymorphic function type | 对 type index 做了 quantification 的函数类型 | `forall` | [1](#pft) | | 3 | type index | 类型中的参数名 | `forall`、构造参数、`let` | [2](#index) | | 4 | kind | type index 的种类,决定其取值范围与可用的运算 | `forall: {M: Dim}` | [2](#index) | -| 5 | discriminant | 取值有限的量,用于选择 type family 的分支,或决定张量是否存在 | `match`、`optional`、`nullable`、`mutated` 中的表达式 | [3](#family) | +| 5 | discriminant | 取值有限的量,用于选择 type family 的分支,或决定张量是否存在 | `match`、`nullable`、`mutated` 中的表达式,以及 refinement 中的 `present(...)`、`member(...)` | [3](#family) | | 6 | type family | 根据 discriminant 的取值给出形状的函数 | `signature.types` | [3](#family) | | 7 | ADT、constructor | 带 tag 的 sum type,每个 constructor 有各自的字段 | `spec/types.yaml` 中的 `adts` | [4](#adt) | | 8 | `let` | 由 index 计算得到的具名量 | `let: {H_out: "..."}` | [5](#let) | @@ -149,6 +149,7 @@ effect 决定了 roofline 的读写计数,以及有编译边界的 op 所生 | 10 | unification | 将声明的形状与实际形状逐轴对应,求出未知的 index | 由签名生成 | [7](#unification) | | 11 | type inference | 调用时求出全部 index 的过程 | 由签名生成 | [7](#unification) | | 12 | effect | 调用对参数的写入与别名关系 | `mutated`、`write_only`、`buffer`、`alias`、`overlaps` | [8](#effect) | -| 13 | `Maybe[X]`、`present` | 可能缺省的 `X`;`present(x)` 表示它是否给出 | 可选输入、可空输出、`int \| None` 参数 | [扩展写法 1](extensions.md#presence) | +| 13 | 实参类型、`member`、`present` | 调用期实参可以持有的成员,例如 `Tensor`、`Number` 与 `None`;`member(x)` 给出 `x` 持有的成员,`present(x)` 表示它不是 `None` | 输入的 `type` 与 `default`、可空输出、`int \| None` 参数 | [扩展写法 1](extensions.md#presence) | | 14 | primitive | 表达式中可以调用的内建函数,例如 `broadcast`、`pool.out` | 表达式中的函数调用 | [扩展写法 4](extensions.md#let) | | 15 | generator | 实例化时为 metadata 张量生成取值的函数,例如 `prefix_sum(q_lens)` | 张量的 `values` | [扩展写法 7](extensions.md#generators) | +| 16 | 张量序列 | 作为一个实参传入的张量列表,每个元素有自己的形状与 dtype | `type: "list[Tensor]"`、`Seq[Shape]`、`Seq[DType[...]]` | [扩展写法 11](extensions.md#sequences) | diff --git a/docs/user-guide/manifest/examples.md b/docs/user-guide/manifest/examples.md index 2c7be3d5..10f384ff 100644 --- a/docs/user-guide/manifest/examples.md +++ b/docs/user-guide/manifest/examples.md @@ -124,18 +124,19 @@ MeanPoolingFwdOp: accum_dtype: {type: torch.dtype} inputs: x: {dtype: T, shape: "[B, S, H, D]"} - cu_seqlens: {dtype: int32, optional: true, shape: "[NS + 1]", values: "prefix_sum(seq_lens)", requires: ["prefix_offsets(S)"]} - chunk_indices: {dtype: int32, optional: "present(cu_seqlens)", shape: "[NC, 2]", values: "chunk_indices(seq_lens, chunk_size)"} + cu_seqlens: {dtype: int32, type: "Tensor | None", default: null, shape: "[NS + 1]", values: "prefix_sum(seq_lens)", requires: ["prefix_offsets(S)"]} + chunk_indices: {dtype: int32, type: "Tensor | None", default: null, shape: "[NC, 2]", values: "chunk_indices(seq_lens, chunk_size)"} outputs: output: {dtype: T, shape: "PoolOut[present(cu_seqlens), B, NC, S, chunk_size, H, D]"} shape_rules: + - "present(chunk_indices) == present(cu_seqlens)" - "chunk_size > 0 and chunk_size % 32 == 0" - "D <= 128 or D % 128 == 0" - "category(accum_dtype) == 'float'" workloads: - {B: 1, S: 8192, H: 64, D: 128, chunk_size: 64, accum_dtype: float32, dtype_cases: [{T: float16}, {T: bfloat16}], label: uniform-8k} - - {B: 1, S: 8192, H: 64, D: 128, seq_lens: "repeat(2048, 4)", chunk_size: 64, accum_dtype: float32, some: [cu_seqlens], dtype_cases: [{T: float16}, {T: float32}], label: ragged-even} + - {B: 1, S: 8192, H: 64, D: 128, seq_lens: "repeat(2048, 4)", chunk_size: 64, accum_dtype: float32, some: [cu_seqlens, chunk_indices], dtype_cases: [{T: float16}, {T: float32}], label: ragged-even} # remaining rows omitted roofline: @@ -146,9 +147,9 @@ MeanPoolingFwdOp: | No. | Form in the spec | Described in | | --- | --- | --- | -| 1 | `chunk_indices` is declared `optional: "present(cu_seqlens)"`, so it is passed together with `cu_seqlens` or omitted together with it | [Extensions § 1](extensions.md#presence) | +| 1 | `cu_seqlens` and `chunk_indices` both declare `type: "Tensor \| None", default: null`, and the domain restriction `present(chunk_indices) == present(cu_seqlens)` requires them to be passed together or omitted together | [Extensions § 1](extensions.md#presence) | | 2 | the type family `PoolOut` uses `present(cu_seqlens)` as its discriminant | [Extensions § 2](extensions.md#type-family) | -| 3 | the first row does not pass `cu_seqlens` and does not use `seq_lens`, so the row does not give it; the second row passes `cu_seqlens` through `some: [cu_seqlens]` and gives `seq_lens`, written here as the value primitive call `"repeat(2048, 4)"` | [Extensions § 1](extensions.md#presence), [Spec fields § 8](writing.md#workloads) | +| 3 | the first row does not pass `cu_seqlens` and does not use `seq_lens`, so the row does not give it; the second row passes both through `some: [cu_seqlens, chunk_indices]` and gives `seq_lens`, written here as the value primitive call `"repeat(2048, 4)"` | [Extensions § 1](extensions.md#presence), [Spec fields § 8](writing.md#workloads) | | 4 | `NS` and `NC` are solved by unification of the generator results and are not given in the row | [Extensions § 7](extensions.md#generators) | | 5 | `category(accum_dtype) == 'float'` requires the accumulation dtype to be a floating-point type | [Extensions § 9](extensions.md#scalar-dtype) | | 6 | the roofline formula distinguishes the two cases with `present(cu_seqlens)` | [Spec fields § 9](writing.md#roofline) | diff --git a/docs/user-guide/manifest/examples.zh.md b/docs/user-guide/manifest/examples.zh.md index f113e0d4..fb94040b 100644 --- a/docs/user-guide/manifest/examples.zh.md +++ b/docs/user-guide/manifest/examples.zh.md @@ -124,18 +124,19 @@ MeanPoolingFwdOp: accum_dtype: {type: torch.dtype} inputs: x: {dtype: T, shape: "[B, S, H, D]"} - cu_seqlens: {dtype: int32, optional: true, shape: "[NS + 1]", values: "prefix_sum(seq_lens)", requires: ["prefix_offsets(S)"]} - chunk_indices: {dtype: int32, optional: "present(cu_seqlens)", shape: "[NC, 2]", values: "chunk_indices(seq_lens, chunk_size)"} + cu_seqlens: {dtype: int32, type: "Tensor | None", default: null, shape: "[NS + 1]", values: "prefix_sum(seq_lens)", requires: ["prefix_offsets(S)"]} + chunk_indices: {dtype: int32, type: "Tensor | None", default: null, shape: "[NC, 2]", values: "chunk_indices(seq_lens, chunk_size)"} outputs: output: {dtype: T, shape: "PoolOut[present(cu_seqlens), B, NC, S, chunk_size, H, D]"} shape_rules: + - "present(chunk_indices) == present(cu_seqlens)" - "chunk_size > 0 and chunk_size % 32 == 0" - "D <= 128 or D % 128 == 0" - "category(accum_dtype) == 'float'" workloads: - {B: 1, S: 8192, H: 64, D: 128, chunk_size: 64, accum_dtype: float32, dtype_cases: [{T: float16}, {T: bfloat16}], label: uniform-8k} - - {B: 1, S: 8192, H: 64, D: 128, seq_lens: "repeat(2048, 4)", chunk_size: 64, accum_dtype: float32, some: [cu_seqlens], dtype_cases: [{T: float16}, {T: float32}], label: ragged-even} + - {B: 1, S: 8192, H: 64, D: 128, seq_lens: "repeat(2048, 4)", chunk_size: 64, accum_dtype: float32, some: [cu_seqlens, chunk_indices], dtype_cases: [{T: float16}, {T: float32}], label: ragged-even} # 其余行省略 roofline: @@ -146,9 +147,9 @@ MeanPoolingFwdOp: | No. | spec 中的写法 | 说明所在 | | --- | --- | --- | -| 1 | `chunk_indices` 声明为 `optional: "present(cu_seqlens)"`,因此与 `cu_seqlens` 同时传入或同时不传 | [扩展写法 1](extensions.md#presence) | +| 1 | `cu_seqlens` 与 `chunk_indices` 都声明为 `type: "Tensor \| None", default: null`,定义域限制 `present(chunk_indices) == present(cu_seqlens)` 要求二者同时传入或同时不传 | [扩展写法 1](extensions.md#presence) | | 2 | type family `PoolOut` 以 `present(cu_seqlens)` 为 discriminant | [扩展写法 2](extensions.md#type-family) | -| 3 | 第一行不传 `cu_seqlens`,没有用到 `seq_lens`,因此行中不给出它;第二行通过 `some: [cu_seqlens]` 传入 `cu_seqlens`,同时给出 `seq_lens`,这里写成值 primitive 调用 `"repeat(2048, 4)"` | [扩展写法 1](extensions.md#presence)、[写一个 spec 8](writing.md#workloads) | +| 3 | 第一行不传 `cu_seqlens`,没有用到 `seq_lens`,因此行中不给出它;第二行通过 `some: [cu_seqlens, chunk_indices]` 传入二者,同时给出 `seq_lens`,这里写成值 primitive 调用 `"repeat(2048, 4)"` | [扩展写法 1](extensions.md#presence)、[写一个 spec 8](writing.md#workloads) | | 4 | `NS` 与 `NC` 由 generator 结果的 unification 求得,不在行中给出 | [扩展写法 7](extensions.md#generators) | | 5 | `category(accum_dtype) == 'float'` 限制累加 dtype 必须是浮点类型 | [扩展写法 9](extensions.md#scalar-dtype) | | 6 | roofline 公式通过 `present(cu_seqlens)` 区分两种情况 | [写一个 spec 9](writing.md#roofline) | diff --git a/docs/user-guide/manifest/extensions.md b/docs/user-guide/manifest/extensions.md index 86e85ace..260c098e 100644 --- a/docs/user-guide/manifest/extensions.md +++ b/docs/user-guide/manifest/extensions.md @@ -2,28 +2,89 @@ This page describes forms that only some specs use. Each section covers one case, and each section can be read on its own. The fields most specs need are in [Spec fields](writing.md). -## 1. Optional inputs, nullable outputs and omittable parameters {#presence} +## 1. Argument types: optional inputs, nullable outputs and omittable parameters {#presence} -Whether a tensor exists is expressed with `present`. FusedMoESharedExpert is an example: +A call-time input declares its argument `type` and `default`, as a construction parameter does. `type` lists the members the argument may hold. FusedMoESharedExpert is an example: ```yaml inputs: - correction_bias: {dtype: float32, shape: "[E]", optional: true} - shared_w_gate_up: {dtype: D, shape: "[2 * S, H]", optional: true} - shared_w_down: {dtype: D, shape: "[H, S]", optional: "present(shared_w_gate_up)"} + correction_bias: {type: "Tensor | None", default: null, dtype: float32, shape: "[E]"} + shared_w_gate_up: {type: "Tensor | None", default: null, dtype: D, shape: "[2 * S, H]"} + shared_w_down: {type: "Tensor | None", default: null, dtype: D, shape: "[H, S]"} outputs: shared_output: {dtype: D, shape: "[T, H]", nullable: "present(shared_w_gate_up)"} routed_output: {dtype: D, shape: "[T, H]"} +shape_rules: +- "present(shared_w_down) == present(shared_w_gate_up)" ``` -- An optional input is declared with `optional: true`. If several tensors must be passed together or omitted together, they share one discriminant, written `optional: ""`, as `shared_w_down` does above. -- An output that may be `None` is declared with `nullable: ""`. In the example above, the op returns `shared_output` when `shared_w_gate_up` is passed, and that output is `None` otherwise. -- The expressions of `optional` and `nullable` may consist only of boolean quantities with finitely many values: `Bool` and enum parameters, ADT tags and fields with finitely many values, and `present(...)`. -- Optional inputs come after required inputs, and `forward` receives them in declaration order with a default of `None`. Omitting an optional input has the same effect as passing `None` explicitly. -- The kind of an `int | None` parameter is `Maybe[Int]`: `present(v)` says whether it is given, and `v.value` is its value. `v.value` may appear only in branches where `present(v)` is true. -- An index is used only on the branches where it appears. In the example above, when `shared_w_gate_up` is not passed, `S` is not used, and the corresponding workload row does not give `S`. -- A workload row lists the optional tensors it passes in `some`. `some` lists only tensors declared `optional: true`; a tensor whose `optional` is an expression follows the value of that expression and is not written in `some`. For example, passing the shared expert is written `some: [shared_w_gate_up]`, and `shared_w_down` is passed along with it. -- For an `implemented` op, each optional tensor must be passed by at least one row and omitted by at least one row. +**Table 1** Members of an argument type + +| No. | Member | Value it takes | Read as, on a branch narrowed to it | +| --- | --- | --- | --- | +| 1 | `Tensor` | a tensor | the declaration's `Tensor[T, s]` | +| 2 | `list[Tensor]` | a list or tuple of tensors | a tensor sequence, see [§ 11](#sequences) | +| 3 | `int` | an `int` that is not a `bool` | `Int`, a call-time scalar | +| 4 | `float` | a `float` | a value, a call-time scalar | +| 5 | `Number` | an `int` or a `float` | a value, a call-time scalar | +| 6 | `bool` | a `bool` | `Bool`, a call-time scalar | +| 7 | `None` | `None` | `None` | + +The rules for argument types: + +- `type` defaults to `Tensor`, which stands for the declaration's `Tensor[dtype, shape]`. `Number` already holds `int` and `float`, so it is never listed together with them. +- An argument with a `Tensor` or `list[Tensor]` member is a call-time input, declared in `inputs`; every other argument is a construction parameter, declared in `params`. A tensor is never a construction parameter. +- A value takes a member by its Python type: `None` takes `None`; a list or tuple takes `list[Tensor]`; a tensor takes `Tensor`; a `bool` takes only `bool`; an `int` takes `int`, else `Number`; a `float` takes `float`, else `Number`. A value that no member takes is rejected. +- An input without `default` is passed on every call. An input with `default` may be omitted; `forward` takes the `default` as its default, and omitting the input has the same effect as passing that value. +- Among positional inputs, an input without `default` never follows one with `default`. A `kw_only` input follows every positional one, and may be required. +- A required argument that may be `None` declares the `None` member and no `default`, for example `bias: {type: "Tensor | None", dtype: Wb, shape: "[N]"}`. +- `member(x)` is the name of the member `x` holds, such as `'Tensor'` or `'Number'`, and each argument with several members is one discriminant with finitely many values. `present(x)` means `member(x) != 'None'`, whether the caller passed `x` or its `default` applied. +- `member(x) == ''` narrows `x` to that member, `member(x) != ''` excludes it, and `present(x)` excludes `None`. Narrowing composes through `not`, `and`, `or` and conditional expressions. +- The tensor declarations `shape`, `dtype`, `contiguous`, `device`, `values`, `requires` and effects apply only on the branches where the argument holds a tensor member. An index used only by them is not used elsewhere: in the example above, when `shared_w_gate_up` is not passed, `S` is not used, and the corresponding workload row does not give `S`. + +Inputs that must be passed together or omitted together each declare `type: "Tensor | None", default: null`, and a refinement that reads only discriminants ties them, as `present(shared_w_down) == present(shared_w_gate_up)` does above. Such a refinement is a domain restriction, see [§ 2](#type-family). + +An output that may be `None` is declared with `nullable: ""`. In the example above, the op returns `shared_output` when `shared_w_gate_up` is passed, and that output is `None` otherwise. The expression may consist only of discriminants with finitely many values: `Bool` and enum parameters, ADT tags and fields with finitely many values, `present(...)` and `member(...)`. + +The kind of an `int | None` parameter is `Maybe[Int]`. `present(v)` says whether it is given, and on a branch where `present(v)` holds, `v` itself is its value, for example `R: "rotary_dim if present(rotary_dim) else D"` in a `let`. + +**Call-time scalars.** An argument narrowed to a scalar member is a call-time scalar. RoPECosSin's `seqlen_offsets` is one offset for every sequence or a tensor of one offset per sequence: + +```yaml +inputs: + seqlen_offsets: {type: "int | Tensor", default: 0, dtype: O, shape: "[B]"} +shape_rules: +- "member(seqlen_offsets) != 'int' or 0 <= seqlen_offsets <= P - L" +``` + +- A call-time scalar's value is read only in refinements and in `let` entries; a `let` that reads one, directly or through another `let`, is likewise read only in refinements and in such `let` entries. +- Shapes, dtypes, type families, `nullable` and `mutated` conditions, `values`, `requires` and roofline formulas never read a call-time scalar's value, and the validator rejects a spec that does. A value therefore decides no output shape and no branch. Construction parameters are not restricted this way. +- The refinements and `let` entries that read a call-time scalar run on every call. The Op plan is keyed by the member each argument holds and by a scalar's type, never by its value. + +**Workload rows.** + +- `some` lists the arguments with several members that pass their tensor member in the row. Every input passed is listed, including inputs passed together: passing the shared expert is written `some: [shared_w_gate_up, shared_w_down]`. +- Any other argument with several members takes the row's scalar value or `null`, else its `default`. For example, the Clamp row `{A: [4096, 4096], min: -0.5, max: 0.5, ...}` passes both bounds as numbers. A required argument that may be `None` writes `null` to pass `None`. +- For an `implemented` op, every member of every argument with several members that a domain restriction accepts is taken by at least one row. + +**Operator schema.** For an op with a compile boundary, each input becomes one or more slots of the generated `torch.library` operator: + +**Table 2** Operator slots of an input + +| No. | Member | Slot type | Slot name | +| --- | --- | --- | --- | +| 1 | `Tensor`, the only member | `Tensor` | `` | +| 2 | `list[Tensor]` | `Tensor[]` | `` | +| 3 | `Tensor`, among several members | `Tensor?` | `` | +| 4 | `int` | `SymInt?` | `__int` | +| 5 | `float` | `float?` | `__float` | +| 6 | `Number` | `Scalar?` | `__number` | +| 7 | `bool` | `bool?` | `__bool` | +| 8 | `None` | no slot | — | + +- An argument with one member takes one slot under its own name. An argument with several members takes one nullable slot per member other than `None`. +- A call passes at most one slot of an argument, and passes none only when `None` is a member; the operator and its fake function both check this. +- A slot name equal to another argument's name or to an execution parameter's name is rejected. ## 2. Shapes that vary with parameters: type family {#type-family} @@ -43,29 +104,47 @@ signature: ``` - A tensor's `shape` is written `[argument, ...]`, and the arguments map to `params` in declaration order. -- The subject of `match` must be a discriminant with finitely many values: `Bool`, an enum, an ADT, `present(...)`, or a tuple of these. When matching a tuple, `when` is written as a list, for example `when: [true, false]`. +- The subject of `match` must be a discriminant with finitely many values: `Bool`, an enum, an ADT, `present(...)`, `member(...)`, a boolean expression of these such as `member(min) == 'Tensor'`, or a tuple of these. When matching a tuple, `when` is written as a list, for example `when: [true, false]`. +- A `Maybe` discriminant matches by its value; only `_` matches it when it is absent. - `cases` must have no gaps and no overlaps over all values the spec accepts. - When several tensors apply the same type family, they always take the same branch. - References between type families must not form a cycle, and the validator reports an error if a type family is not used by any shape. -A refinement that reads only discriminants is called a **domain restriction**. A domain restriction is checked before a type family branch is selected, and values it excludes need no corresponding case. Clamp is an example: +Clamp is an example of matching on argument members. `min` and `max` each hold a number, a tensor or `None`, and only a tensor bound takes part in broadcasting the output: ```yaml types: ClampOut: - params: {A: Shape, L: Shape, U: Shape, pl: Bool, pu: Bool} - match: [pl, pu] + params: {A: Shape, L: Shape, U: Shape, tl: Bool, tu: Bool} + match: [tl, tu] cases: - - {when: [true, true], is: "[*broadcast(A, L, U)]"} - - {when: [true, false], is: "[*broadcast(A, L)]"} - - {when: [false, true], is: "[*broadcast(A, U)]"} + - {when: [true, true], is: "[*broadcast(A, L, U)]"} + - {when: [true, false], is: "[*broadcast(A, L)]"} + - {when: [false, true], is: "[*broadcast(A, U)]"} + - {when: [false, false], is: "[*A]"} +inputs: + input: {dtype: T, shape: "[*A]"} + min: {type: "Number | Tensor | None", default: null, dtype: T, shape: "[*L]"} + max: {type: "Number | Tensor | None", default: null, dtype: T, shape: "[*U]"} outputs: - output: {dtype: T, shape: "ClampOut[A, L, U, present(min), present(max)]"} + output: {dtype: T, shape: "ClampOut[A, L, U, member(min) == 'Tensor', member(max) == 'Tensor']"} +``` + +A refinement that reads only discriminants is called a **domain restriction**. A domain restriction is checked before a type family branch is selected, and values it excludes need no corresponding case. MoEPrePermute is an example: + +```yaml +types: + LayoutMetadata: + params: {layout: MGroupedLayout, E: Dim, P: Dim} + match: layout + cases: + - {when: {contiguous: {metadata_kind: physical_psum}}, is: "[E]"} + - {when: {contiguous: {metadata_kind: per_row}}, is: "[P]"} shape_rules: -- "present(min) or present(max)" +- "layout.kind == 'contiguous'" ``` -`present(min) or present(max)` reads only discriminants and excludes the case where neither `min` nor `max` is passed, so the three cases already cover every combination Clamp accepts. Whether a refinement reads only discriminants is decided by all the names that appear in it, independent of the order in which the operands are written. +`layout.kind == 'contiguous'` reads only the discriminant `layout` and excludes the masked layout, so the two cases already cover every value MoEPrePermute accepts. Whether a refinement reads only discriminants is decided by all the names that appear in it, independent of the order in which the operands are written. Clamp's `present(min) or present(max)`, which requires at least one bound, is also a domain restriction. ## 3. Parameters with fields: ADT {#adt} @@ -117,15 +196,9 @@ A primitive is a built-in function callable in expressions, such as `broadcast`, - Adding a member requires changing `tileops.manifest.primitives`, with tests. - Every primitive that takes an axis argument handles axes by the same rule: for a zero-dimensional tensor, both `0` and `-1` denote the single scalar axis; otherwise, the axis ranges over `[-rank, rank)`. -## 5. Construction-time tensors, memory layout and device {#placement} - -- A tensor passed at construction is declared in `params` with `dtype` and `shape`, and can declare `optional: true`. For example, `rescale_factors` in LongRoPE: - - ```yaml - params: - rescale_factors: {dtype: R, shape: "[D // 2]", optional: true} - ``` +## 5. Memory layout and device {#placement} +- Every tensor is a call-time input. A list of values outside the type system is a construction parameter read only through its length; for example, LongRoPE's `rescale_factors` is a `list[float] | None` parameter, checked by `not present(rescale_factors) or len(rescale_factors) == D // 2`. - A tensor that must be contiguous in memory declares `contiguous: true`, for example the inputs and outputs of the MoE staged ops. Calls are supported with contiguous inputs ([API Stability](../../stability.md)); a non-contiguous tensor without this declaration is not promised to work. - A tensor that must be on the CPU declares `device: cpu`, for example `cu_seqlens_cpu` in `GDNFwdOp`. - An op without call-time tensor inputs declares a `device` parameter, for example Alibi. How the call device is determined is in [Calls and validation § 4](calls.md#device). @@ -134,7 +207,7 @@ A primitive is a built-in function callable in expressions, such as `broadcast`, An op without effect declarations only reads its inputs and allocates new tensors for its outputs. If an op writes to an argument, an effect is declared on the corresponding tensor in the signature. Effects determine the read and write counts of the roofline, and the operator schema generated for an op with a compile boundary. -**Table 1** Effect declarations +**Table 3** Effect declarations | No. | Declaration | Meaning | Example | | --- | --- | --- | --- | @@ -167,6 +240,7 @@ outputs: Effect declarations must also satisfy the following rules, and the validator rejects specs that violate them: +- an effect is declared only on an input without a scalar member, and never on a tensor sequence; - `write_only: true` can be used only together with `mutated: true`; - `alias` must point to an input that is written; - an output that declares `alias` cannot also declare `buffer`; @@ -182,10 +256,12 @@ Varlen, paged and similar ops use metadata tensors such as `cu_seqlens` and `blo ```yaml forall: {B: Dim, S: Dim, H: Dim, D: Dim, NS: Dim, NC: Dim, T: "DType[...]", seq_lens: "Seq[Int]"} inputs: - cu_seqlens: {dtype: int32, optional: true, shape: "[NS + 1]", + cu_seqlens: {dtype: int32, type: "Tensor | None", default: null, shape: "[NS + 1]", values: "prefix_sum(seq_lens)", requires: ["prefix_offsets(S)"]} - chunk_indices: {dtype: int32, optional: "present(cu_seqlens)", shape: "[NC, 2]", + chunk_indices: {dtype: int32, type: "Tensor | None", default: null, shape: "[NC, 2]", values: "chunk_indices(seq_lens, chunk_size)"} +shape_rules: +- "present(chunk_indices) == present(cu_seqlens)" ``` The rules for generators: @@ -263,3 +339,32 @@ composition: - The manifest does not prescribe how the roofline of a composite op relates to the rooflines of its stages. - For an `implemented` op, the validator checks in order that the `op` stages agree with the class's `delegate_types` and the `kernel` stages agree with the class's `kernel_types`. - `stages` cannot be empty, stage names are all distinct, and `optional` is a boolean. + +## 11. Tensor sequences {#sequences} + +An argument that holds a list of tensors declares `type: "list[Tensor]"`. Its element-wise indices have the kinds `Seq[Dim]`, `Seq[Shape]` and `Seq[DType[...]]`. Concat is an example: + +```yaml +forall: {S: "Seq[Shape]", D: "Seq[DType[float16 | bfloat16 | float32]]"} +params: + dim: {type: int, default: 0} +inputs: + tensors: {type: "list[Tensor]", dtype: D, shape: "[*S]"} +outputs: + output: {dtype: "promote(D)", shape: "[*concat(S, dim)]"} +shape_rules: +- "len(S) >= 1" +workloads: +- {S: [[2048, 1024], [2048, 4096], [2048, 3000]], dim: 1, dtype_cases: [{D: [bfloat16, bfloat16, bfloat16]}], label: uneven-cols} +``` + +- In the `shape` and `dtype` of a `list[Tensor]` declaration, every element-wise index stands for the element's item: element `i` has shape `S[i]` and dtype `D[i]`, and `len(S)` there is `len(S[i])`. Every other index is shared by all elements. +- Outside the declaration, an element-wise index is the whole list, readable in outputs, refinements, `let` entries and roofline formulas. A `let` is not expanded into the element. +- `Seq[Dim]` holds one axis length per element, while a `Shape` is one tuple of axes shared by all elements. +- The sequence length is the common length of its element-wise indices, and a `list[Tensor]` declaration reads at least one element-wise index. +- Element-wise indices are solved on every element. A shared index is solved from the other tensors first; only when inference stalls is it solved from element 0, then checked on every element. +- An empty sequence gives its element-wise indices empty lists; a shared index that only the sequence could solve stays unsolved, and the call raises an error that names the declaration. +- A tensor sequence is passed on every call and is only read. It declares no effect, `values`, `requires` or `device`, and is not a member of a union. +- Choosing the call device, the 16-byte alignment copy and the alias and overlap checks visit each element. The derived byte count counts each element once, and `bytes(t)` is the sum over the elements. +- A workload row writes an element-wise index as a list with one item per element. A `Seq[DType]` index is written in `dtype_cases` as a list of dtypes, and its value in the case id joins them with `.`, for example `uneven-cols-bfloat16.bfloat16.bfloat16`. +- A `list[Tensor]` output, `None` elements in a sequence, `list[Tensor]` as a member of a union, and an effect on a sequence are all rejected. diff --git a/docs/user-guide/manifest/extensions.zh.md b/docs/user-guide/manifest/extensions.zh.md index c64bb522..a8e5b043 100644 --- a/docs/user-guide/manifest/extensions.zh.md +++ b/docs/user-guide/manifest/extensions.zh.md @@ -2,28 +2,89 @@ 本页介绍只有部分 spec 才会用到的写法。每一节对应一种情况,各节可以独立阅读。大多数 spec 所需的字段见[写一个 spec](writing.md)。 -## 1. 可选输入、可空输出与可缺省参数 {#presence} +## 1. 实参类型:可选输入、可空输出与可缺省参数 {#presence} -张量是否存在通过 `present` 表达。以 FusedMoESharedExpert 为例: +调用期输入像构造参数一样声明实参的 `type` 与 `default`。`type` 列出这个实参可以持有的成员。以 FusedMoESharedExpert 为例: ```yaml inputs: - correction_bias: {dtype: float32, shape: "[E]", optional: true} - shared_w_gate_up: {dtype: D, shape: "[2 * S, H]", optional: true} - shared_w_down: {dtype: D, shape: "[H, S]", optional: "present(shared_w_gate_up)"} + correction_bias: {type: "Tensor | None", default: null, dtype: float32, shape: "[E]"} + shared_w_gate_up: {type: "Tensor | None", default: null, dtype: D, shape: "[2 * S, H]"} + shared_w_down: {type: "Tensor | None", default: null, dtype: D, shape: "[H, S]"} outputs: shared_output: {dtype: D, shape: "[T, H]", nullable: "present(shared_w_gate_up)"} routed_output: {dtype: D, shape: "[T, H]"} +shape_rules: +- "present(shared_w_down) == present(shared_w_gate_up)" ``` -- 可选输入使用 `optional: true` 声明。如果几个张量必须同时传入或同时不传,它们共用同一个 discriminant,写作 `optional: "<表达式>"`,例如上例中的 `shared_w_down`。 -- 可能返回 `None` 的输出使用 `nullable: "<表达式>"` 声明。在上例中,传入 `shared_w_gate_up` 时 op 返回 `shared_output`,否则该输出为 `None`。 -- `optional` 与 `nullable` 的表达式只能由取值有限的布尔量组成,包括 `Bool` 与枚举参数、ADT 的 tag 与取值有限的字段,以及 `present(...)`。 -- 可选输入排在必选输入之后,`forward` 按声明顺序接收它们,默认值为 `None`。省略一个可选输入与显式传入 `None` 效果相同。 -- `int | None` 参数的 kind 是 `Maybe[Int]`:`present(v)` 表示它是否给出,`v.value` 是它的值。`v.value` 只能出现在 `present(v)` 为 true 的分支中。 -- 一个 index 只在出现它的分支上被用到。在上例中,不传 `shared_w_gate_up` 时 `S` 没有被用到,对应的 workload 行不需要给出 `S`。 -- workload 行通过 `some` 列出传入的可选张量。`some` 只列出声明为 `optional: true` 的张量;`optional` 为表达式的张量随表达式的取值确定,不写在 `some` 中。例如传入共享专家时写 `some: [shared_w_gate_up]`,`shared_w_down` 随之传入。 -- 对于 `implemented` 的 op,每个可选张量都要至少有一条行传入它,并至少有一条行不传它。 +**表 1** 实参类型的成员 + +| No. | 成员 | 接受的值 | 收窄为该成员的分支上读作 | +| --- | --- | --- | --- | +| 1 | `Tensor` | 张量 | 该声明的 `Tensor[T, s]` | +| 2 | `list[Tensor]` | 由张量组成的 list 或 tuple | 张量序列,见[第 11 节](#sequences) | +| 3 | `int` | 不是 `bool` 的 `int` | `Int`,调用期标量 | +| 4 | `float` | `float` | 一个值,调用期标量 | +| 5 | `Number` | `int` 或 `float` | 一个值,调用期标量 | +| 6 | `bool` | `bool` | `Bool`,调用期标量 | +| 7 | `None` | `None` | `None` | + +实参类型的规则: + +- `type` 默认为 `Tensor`,表示该声明的 `Tensor[dtype, shape]`。`Number` 已经包含 `int` 与 `float`,因此不与它们同时列出。 +- 有 `Tensor` 或 `list[Tensor]` 成员的实参是调用期输入,声明在 `inputs` 中;其他实参都是构造参数,声明在 `params` 中。张量不作为构造参数。 +- 值按它的 Python 类型对应成员:`None` 对应 `None`;list 或 tuple 对应 `list[Tensor]`;张量对应 `Tensor`;`bool` 只对应 `bool`;`int` 对应 `int`,没有 `int` 成员时对应 `Number`;`float` 对应 `float`,没有 `float` 成员时对应 `Number`。没有成员可以对应的值被拒绝。 +- 没有 `default` 的输入每次调用都要传入。有 `default` 的输入可以省略:`forward` 以 `default` 作为它的默认值,省略这个输入与传入该值效果相同。 +- 在按位置传入的输入中,没有 `default` 的输入不排在有 `default` 的输入之后。`kw_only` 的输入排在所有按位置传入的输入之后,也可以是必须传入的。 +- 必须传入、但可以为 `None` 的实参声明 `None` 成员,不声明 `default`,例如 `bias: {type: "Tensor | None", dtype: Wb, shape: "[N]"}`。 +- `member(x)` 是 `x` 所持有成员的名字,例如 `'Tensor'` 或 `'Number'`;每个有多个成员的实参都是一个取值有限的 discriminant。`present(x)` 表示 `member(x) != 'None'`,无论 `x` 是调用方传入的还是取了 `default`。 +- `member(x) == ''` 把 `x` 收窄为该成员,`member(x) != ''` 排除该成员,`present(x)` 排除 `None`。收窄可以经由 `not`、`and`、`or` 与条件表达式组合。 +- 张量声明 `shape`、`dtype`、`contiguous`、`device`、`values`、`requires` 与 effect 只在实参持有张量成员的分支上生效。只被它们用到的 index 在其他分支上不算被用到:在上例中,不传 `shared_w_gate_up` 时 `S` 没有被用到,对应的 workload 行不需要给出 `S`。 + +必须同时传入或同时不传的几个输入,各自声明 `type: "Tensor | None", default: null`,再由一条只读取 discriminant 的 refinement 把它们关联起来,例如上例中的 `present(shared_w_down) == present(shared_w_gate_up)`。这样的 refinement 是定义域限制,见[第 2 节](#type-family)。 + +可能返回 `None` 的输出使用 `nullable: "<表达式>"` 声明。在上例中,传入 `shared_w_gate_up` 时 op 返回 `shared_output`,否则该输出为 `None`。表达式只能由取值有限的 discriminant 组成,包括 `Bool` 与枚举参数、ADT 的 tag 与取值有限的字段、`present(...)` 与 `member(...)`。 + +`int | None` 参数的 kind 是 `Maybe[Int]`。`present(v)` 表示它是否给出;在 `present(v)` 成立的分支上,`v` 本身就是它的值,例如 `let` 中的 `R: "rotary_dim if present(rotary_dim) else D"`。 + +**调用期标量。** 收窄为标量成员的实参是调用期标量。RoPECosSin 的 `seqlen_offsets` 可以是所有序列共用的一个偏移,也可以是每个序列一个偏移的张量: + +```yaml +inputs: + seqlen_offsets: {type: "int | Tensor", default: 0, dtype: O, shape: "[B]"} +shape_rules: +- "member(seqlen_offsets) != 'int' or 0 <= seqlen_offsets <= P - L" +``` + +- 调用期标量的取值只在 refinement 与 `let` 中读取;直接或经由其他 `let` 读取它的 `let`,同样只在 refinement 与这类 `let` 中读取。 +- 形状、dtype、type family、`nullable` 与 `mutated` 的条件、`values`、`requires` 与 roofline 公式都不读取调用期标量的取值,validator 拒绝这样读取的 spec。因此取值不决定任何输出形状,也不决定任何分支。构造参数不受这条限制。 +- 读取调用期标量的 refinement 与 `let` 每次调用都执行。Op plan 以每个实参持有的成员与标量的类型为键,不以标量的取值为键。 + +**workload 行。** + +- `some` 列出在这条行中传入张量成员的、有多个成员的实参。每个传入的输入都要列出,同时传入的几个输入也逐一列出:传入共享专家写作 `some: [shared_w_gate_up, shared_w_down]`。 +- 其他有多个成员的实参取这条行给出的标量值或 `null`,没有给出时取 `default`。例如 Clamp 的行 `{A: [4096, 4096], min: -0.5, max: 0.5, ...}` 以数值传入两个界。必须传入、但可以为 `None` 的实参写 `null` 表示传入 `None`。 +- 对于 `implemented` 的 op,每个有多个成员的实参中被定义域限制接受的每个成员,都要至少由一条行取到。 + +**operator schema。** 对于有编译边界的 op,每个输入成为生成的 `torch.library` operator 的一个或多个 slot: + +**表 2** 输入对应的 operator slot + +| No. | 成员 | slot 类型 | slot 名 | +| --- | --- | --- | --- | +| 1 | `Tensor`,唯一的成员 | `Tensor` | `` | +| 2 | `list[Tensor]` | `Tensor[]` | `` | +| 3 | `Tensor`,多个成员之一 | `Tensor?` | `` | +| 4 | `int` | `SymInt?` | `__int` | +| 5 | `float` | `float?` | `__float` | +| 6 | `Number` | `Scalar?` | `__number` | +| 7 | `bool` | `bool?` | `__bool` | +| 8 | `None` | 没有 slot | — | + +- 只有一个成员的实参占一个 slot,名字就是实参名。有多个成员的实参,每个 `None` 以外的成员占一个可为空的 slot。 +- 一次调用至多传入一个实参的一个 slot;只有 `None` 是成员时才可以一个 slot 都不传。operator 与它的 fake 函数都检查这一点。 +- slot 名与其他实参名或执行参数名相同时被拒绝。 ## 2. 随参数变化的形状:type family {#type-family} @@ -43,29 +104,47 @@ signature: ``` - 张量的 `shape` 写作 `[实参, ...]`,实参按 `params` 的声明顺序对应。 -- `match` 的对象必须是取值有限的 discriminant,即 `Bool`、枚举、ADT、`present(...)`,或由它们组成的元组。对元组做 match 时,`when` 写成列表,例如 `when: [true, false]`。 +- `match` 的对象必须是取值有限的 discriminant,即 `Bool`、枚举、ADT、`present(...)`、`member(...)`、由它们构成的布尔表达式(例如 `member(min) == 'Tensor'`),或由它们组成的元组。对元组做 match 时,`when` 写成列表,例如 `when: [true, false]`。 +- `Maybe` 类型的 discriminant 按它的值匹配;只有 `_` 能匹配它缺省的情况。 - `cases` 在 spec 接受的所有取值上必须既无遗漏也无重叠。 - 多个张量应用同一个 type family 时,它们总是取同一个分支。 - type family 之间的引用不能成环;如果某个 type family 没有被任何形状使用,validator 会报错。 -只读取 discriminant 的 refinement 称为**定义域限制**。定义域限制在选择 type family 分支之前检查,被它排除的取值不需要对应的 case。以 Clamp 为例: +Clamp 是按实参成员做 match 的例子。`min` 与 `max` 各自可以持有数值、张量或 `None`,只有张量形式的界参与输出形状的广播: ```yaml types: ClampOut: - params: {A: Shape, L: Shape, U: Shape, pl: Bool, pu: Bool} - match: [pl, pu] + params: {A: Shape, L: Shape, U: Shape, tl: Bool, tu: Bool} + match: [tl, tu] cases: - - {when: [true, true], is: "[*broadcast(A, L, U)]"} - - {when: [true, false], is: "[*broadcast(A, L)]"} - - {when: [false, true], is: "[*broadcast(A, U)]"} + - {when: [true, true], is: "[*broadcast(A, L, U)]"} + - {when: [true, false], is: "[*broadcast(A, L)]"} + - {when: [false, true], is: "[*broadcast(A, U)]"} + - {when: [false, false], is: "[*A]"} +inputs: + input: {dtype: T, shape: "[*A]"} + min: {type: "Number | Tensor | None", default: null, dtype: T, shape: "[*L]"} + max: {type: "Number | Tensor | None", default: null, dtype: T, shape: "[*U]"} outputs: - output: {dtype: T, shape: "ClampOut[A, L, U, present(min), present(max)]"} + output: {dtype: T, shape: "ClampOut[A, L, U, member(min) == 'Tensor', member(max) == 'Tensor']"} +``` + +只读取 discriminant 的 refinement 称为**定义域限制**。定义域限制在选择 type family 分支之前检查,被它排除的取值不需要对应的 case。以 MoEPrePermute 为例: + +```yaml +types: + LayoutMetadata: + params: {layout: MGroupedLayout, E: Dim, P: Dim} + match: layout + cases: + - {when: {contiguous: {metadata_kind: physical_psum}}, is: "[E]"} + - {when: {contiguous: {metadata_kind: per_row}}, is: "[P]"} shape_rules: -- "present(min) or present(max)" +- "layout.kind == 'contiguous'" ``` -`present(min) or present(max)` 只读取 discriminant,它排除了 `min` 与 `max` 都不传的情况,因此三个 case 已经覆盖了 Clamp 接受的所有组合。判断一条 refinement 是否只读取 discriminant 时,考察的是它出现的所有名字,与操作数的书写顺序无关。 +`layout.kind == 'contiguous'` 只读取 discriminant `layout`,它排除了 masked layout,因此两个 case 已经覆盖了 MoEPrePermute 接受的所有取值。判断一条 refinement 是否只读取 discriminant 时,考察的是它出现的所有名字,与操作数的书写顺序无关。Clamp 要求至少传入一个界的 `present(min) or present(max)` 也是定义域限制。 ## 3. 带字段的参数:ADT {#adt} @@ -117,15 +196,9 @@ primitive 是表达式中可以调用的内建函数,例如 `broadcast`、`red - 新增成员需要修改 `tileops.manifest.primitives`,并附带测试。 - 所有接受轴参数的 primitive 都按同一规则处理轴:对零维张量,`0` 与 `-1` 都表示唯一的标量轴;其他情况下,轴的取值范围是 `[-rank, rank)`。 -## 5. 构造期张量、内存布局与设备 {#placement} - -- 在构造时传入的张量声明在 `params` 中,带有 `dtype` 与 `shape`,并可以声明 `optional: true`。例如 LongRoPE 的 `rescale_factors`: - - ```yaml - params: - rescale_factors: {dtype: R, shape: "[D // 2]", optional: true} - ``` +## 5. 内存布局与设备 {#placement} +- 每个张量都是调用期输入。由类型系统之外的值组成的列表是构造参数,只通过它的长度读取;例如 LongRoPE 的 `rescale_factors` 是 `list[float] | None` 参数,由 `not present(rescale_factors) or len(rescale_factors) == D // 2` 检查。 - 要求内存连续的张量声明 `contiguous: true`,例如 MoE staged 系列 op 的输入与输出。受支持的调用使用连续的输入([API 稳定性](../../stability.md));没有这项声明的张量不连续时,不保证可用。 - 必须位于 CPU 上的张量声明 `device: cpu`,例如 `GDNFwdOp` 的 `cu_seqlens_cpu`。 - 没有调用期张量输入的 op 声明 `device` 参数,例如 Alibi。调用设备的确定方式见[调用与校验 4](calls.md#device)。 @@ -134,7 +207,7 @@ primitive 是表达式中可以调用的内建函数,例如 `broadcast`、`red 没有 effect 声明的 op 只读取输入,并为输出分配新的张量。如果 op 会写入参数,需要在签名的相应张量上声明 effect。effect 决定了 roofline 的读写计数,以及有编译边界的 op 所生成的 operator schema。 -**表 1** effect 声明 +**表 3** effect 声明 | No. | 声明 | 含义 | 例子 | | --- | --- | --- | --- | @@ -167,6 +240,7 @@ outputs: effect 声明还需满足以下规则,validator 会拒绝违反它们的 spec: +- effect 只能声明在没有标量成员的输入上,不能声明在张量序列上; - `write_only: true` 只能与 `mutated: true` 同时使用; - `alias` 指向的必须是一个会被写入的输入; - 声明了 `alias` 的输出不能再声明 `buffer`; @@ -182,10 +256,12 @@ varlen、paged 等 op 使用 metadata 张量,例如 `cu_seqlens`、`block_tabl ```yaml forall: {B: Dim, S: Dim, H: Dim, D: Dim, NS: Dim, NC: Dim, T: "DType[...]", seq_lens: "Seq[Int]"} inputs: - cu_seqlens: {dtype: int32, optional: true, shape: "[NS + 1]", + cu_seqlens: {dtype: int32, type: "Tensor | None", default: null, shape: "[NS + 1]", values: "prefix_sum(seq_lens)", requires: ["prefix_offsets(S)"]} - chunk_indices: {dtype: int32, optional: "present(cu_seqlens)", shape: "[NC, 2]", + chunk_indices: {dtype: int32, type: "Tensor | None", default: null, shape: "[NC, 2]", values: "chunk_indices(seq_lens, chunk_size)"} +shape_rules: +- "present(chunk_indices) == present(cu_seqlens)" ``` generator 的规则如下: @@ -263,3 +339,32 @@ composition: - manifest 不规定组合 op 的 roofline 与其各 stage 的 roofline 之间的关系。 - 对于 `implemented` 的 op,validator 按顺序核对:`op` stage 与类的 `delegate_types` 一致,`kernel` stage 与类的 `kernel_types` 一致。 - `stages` 不能为空,stage 名互不相同,`optional` 为布尔值。 + +## 11. 张量序列 {#sequences} + +持有张量列表的实参声明 `type: "list[Tensor]"`。它的逐元素 index 的 kind 是 `Seq[Dim]`、`Seq[Shape]` 与 `Seq[DType[...]]`。以 Concat 为例: + +```yaml +forall: {S: "Seq[Shape]", D: "Seq[DType[float16 | bfloat16 | float32]]"} +params: + dim: {type: int, default: 0} +inputs: + tensors: {type: "list[Tensor]", dtype: D, shape: "[*S]"} +outputs: + output: {dtype: "promote(D)", shape: "[*concat(S, dim)]"} +shape_rules: +- "len(S) >= 1" +workloads: +- {S: [[2048, 1024], [2048, 4096], [2048, 3000]], dim: 1, dtype_cases: [{D: [bfloat16, bfloat16, bfloat16]}], label: uneven-cols} +``` + +- 在 `list[Tensor]` 声明的 `shape` 与 `dtype` 中,每个逐元素 index 都表示该元素对应的那一项:第 `i` 个元素的形状是 `S[i]`,dtype 是 `D[i]`,那里的 `len(S)` 是 `len(S[i])`。其他 index 由所有元素共用。 +- 在该声明之外,逐元素 index 表示整个列表,可以在输出、refinement、`let` 与 roofline 公式中读取。`let` 不展开到元素上。 +- `Seq[Dim]` 为每个元素保存一个轴长;`Shape` 则是所有元素共用的一个轴元组。 +- 序列长度是它的逐元素 index 的共同长度,`list[Tensor]` 声明至少读取一个逐元素 index。 +- 逐元素 index 在每个元素上求解。共用的 index 先从其他张量求解;只有推断无法继续时,才从第 0 个元素求解,再在每个元素上检查。 +- 空序列使它的逐元素 index 取空列表;只有该序列能求出的共用 index 保持未求得,调用报错,并指出对应的声明。 +- 张量序列每次调用都要传入,并且只被读取。它不声明 effect、`values`、`requires` 或 `device`,也不作为并集的成员。 +- 确定调用设备、16 字节对齐的副本、别名与重叠检查都逐个元素进行。推导字节数时每个元素计一次,`bytes(t)` 是各元素字节数之和。 +- workload 行把逐元素 index 写成每个元素一项的列表。`Seq[DType]` index 在 `dtype_cases` 中写成 dtype 列表,它在 case id 中的值以 `.` 连接,例如 `uneven-cols-bfloat16.bfloat16.bfloat16`。 +- `list[Tensor]` 输出、序列中的 `None` 元素、作为并集成员的 `list[Tensor]`,以及序列上的 effect,都不被接受。 diff --git a/docs/user-guide/manifest/img/call.svg b/docs/user-guide/manifest/img/call.svg index 9a2eb3b4..e968d26c 100644 --- a/docs/user-guide/manifest/img/call.svg +++ b/docs/user-guide/manifest/img/call.svg @@ -1 +1 @@ -The path of one op callGeneratedWritten by the developerGeneratedGeneratedOp baseOp classCallerGeneratedOp baseOp classKerneltargetCallerCallerGeneratedchecksGeneratedchecksOp baseOp baseOp classOp classKernelimplementationKernelimplementationtargettargetGeneratedGeneratedOp baseOp classConstructionOp(params)Check params bytype; checkinvariants; run thechecks constructioncan evaluateop instanceCallop(tensors, ...)1. Settle whichoptional    tensors are passed2. Check domainrestrictions3. Choosetype-familybranches4. Infer indices,    check refinements5. Check outputbuffersalt[Every written tensor is empty]Build the outputsfrom the signature;run nothing[A target registered a builder]Call with forward'sinputsoutputs[In-tree implementation]6. forward(tensors,...)kernel_for(interface,call spec)opt[First time this call spec is seen]Select animplementation:availability →applicability →precedenceentry_for(call)build identity andfactoryopt[First time this build identity is seen]Build the entryentryentry(tensors)resultoutputs7. Check theoutputs' count,    shapes, dtypes,device    and layoutoutputsAn op with a compile boundary (a call-time tensor input, nocomposition): Op.__call__ calls the generated _call_boundary,the same steps run inside the generated torch.library operator,and the computation is still forward; fake/meta come from thesame signature +The path of one op callGeneratedWritten by the developerGeneratedGeneratedOp baseOp classCallerGeneratedOp baseOp classKerneltargetCallerCallerGeneratedchecksGeneratedchecksOp baseOp baseOp classOp classKernelimplementationKernelimplementationtargettargetGeneratedGeneratedOp baseOp classConstructionOp(params)Check params bytype; checkinvariants; run thechecks constructioncan evaluateop instanceCallop(tensors, ...)1. Settle themember each    call-timeargument holds2. Check domainrestrictions3. Choosetype-familybranches4. Infer indices,    check refinements5. Check outputbuffersalt[Every written tensor is empty]Build the outputsfrom the signature;run nothing[A target registered a builder]Call with forward'sinputsoutputs[In-tree implementation]6. forward(tensors,...)kernel_for(interface,call spec)opt[First time this call spec is seen]Select animplementation:availability →applicability →precedenceentry_for(call)build identity andfactoryopt[First time this build identity is seen]Build the entryentryentry(tensors)resultoutputs7. Check theoutputs' count,    shapes, dtypes,device    and layoutoutputsAn op with a compile boundary (a call-time tensor input, nocomposition): Op.__call__ calls the generated _call_boundary,the same steps run inside the generated torch.library operator,and the computation is still forward; fake/meta come from thesame signature diff --git a/docs/user-guide/manifest/img/call.zh.svg b/docs/user-guide/manifest/img/call.zh.svg index 60911f28..1d1662f0 100644 --- a/docs/user-guide/manifest/img/call.zh.svg +++ b/docs/user-guide/manifest/img/call.zh.svg @@ -1 +1 @@ -一次 op 调用的执行路径系统生成开发者编写......Op ..Op .......Op ..Op .Kernel ..target调用方调用方生成的检查生成的检查Op 基类Op 基类Op 类Op 类Kernel 实现Kernel 实现targettarget......Op ..Op .构造Op(参数)按 type 检查参数;检查invariant;完成构造时可以求值的检查op 实例调用op(张量, ...)1. 确定可选张量是否传入2. 检查定义域限制3. 选择 type family 分支4. 推断 index,    检查 refinement5. 检查输出缓冲alt[所有写入的张量都为空]按签名构造输出,不执行实现[target 为 op 注册了 builder]以 forward 的输入调用输出[in-tree 实现]6. forward(张量, ...)kernel_for(interface, callspec)opt[call spec 首次出现]选择实现:可用性 → 适用性→ 优先级entry_for(call)build identity 与构建函数opt[build identity 首次出现]构造 entryentryentry(张量)结果输出7. 检查输出的数量、    形状、dtype、设备    与内存布局输出有编译边界的 op(有调用期张量输入、没有 composition):Op.__call__ 调用生成的 _call_boundary,同样的步骤在生成的torch.library operator 内执行,计算本体仍写在 forward 中;fake/meta 由同一份签名生成 +一次 op 调用的执行路径系统生成开发者编写......Op ..Op .......Op ..Op .Kernel ..target调用方调用方生成的检查生成的检查Op 基类Op 基类Op 类Op 类Kernel 实现Kernel 实现targettarget......Op ..Op .构造Op(参数)按 type 检查参数;检查invariant;完成构造时可以求值的检查op 实例调用op(张量, ...)1. 确定每个调用期实参持有的成员2. 检查定义域限制3. 选择 type family 分支4. 推断 index,    检查 refinement5. 检查输出缓冲alt[所有写入的张量都为空]按签名构造输出,不执行实现[target 为 op 注册了 builder]以 forward 的输入调用输出[in-tree 实现]6. forward(张量, ...)kernel_for(interface, callspec)opt[call spec 首次出现]选择实现:可用性 → 适用性→ 优先级entry_for(call)build identity 与构建函数opt[build identity 首次出现]构造 entryentryentry(张量)结果输出7. 检查输出的数量、    形状、dtype、设备    与内存布局输出有编译边界的 op(有调用期张量输入、没有 composition):Op.__call__ 调用生成的 _call_boundary,同样的步骤在生成的torch.library operator 内执行,计算本体仍写在 forward 中;fake/meta 由同一份签名生成 diff --git a/docs/user-guide/manifest/img/device.svg b/docs/user-guide/manifest/img/device.svg index 3bdf52eb..1725a381 100644 --- a/docs/user-guide/manifest/img/device.svg +++ b/docs/user-guide/manifest/img/device.svg @@ -1 +1 @@ -How the call device is chosen(workload instantiation places tensors by the same rule)A tensor declaring device: cpu stays on the CPUand takes no part belowAny call-time input?device parameter not None?Any construction-time tensor?The accelerator's current device;None without an acceleratorTheir device; they must share onedevice; a string is parsedby torch.deviceTheir device; several must share oneOnce the call device is chosen, out and the outputsare checked or allocated on it; construction-time tensorsare copied to it and cast to their signature dtypenononoyesyesyes +How the call device is chosen(workload instantiation places tensors by the same rule)A tensor declaring device: cpu stays on the CPUand takes no part belowAny call-time tensor?(each element of a tensor sequence counts)device parameter not None?The accelerator's current device;None without an acceleratorTheir device; they must share onedevice; a string is parsedby torch.deviceOnce the call device is chosen, out and the outputsare checked or allocated on itnonoyesyes diff --git a/docs/user-guide/manifest/img/device.zh.svg b/docs/user-guide/manifest/img/device.zh.svg index 969ccc9c..e02eeb67 100644 --- a/docs/user-guide/manifest/img/device.zh.svg +++ b/docs/user-guide/manifest/img/device.zh.svg @@ -1 +1 @@ -调用设备的确定方式(workload 实例化按同一规则放置张量)声明 device: cpu 的张量固定在 CPU,不参与下面的判定有调用期输入?参数 device 不为 None?有构造期张量?取当前 accelerator 设备;没有 accelerator 时为 None取这些输入的设备;它们须位于同一设备取 device;字符串按 torch.device 解析取构造期张量的设备;多个时须位于同一设备确定调用设备后,out 与输出在该设备上检查或分配;构造期张量复制到调用设备,并转为签名中的 dtype否否否是是是 +调用设备的确定方式(workload 实例化按同一规则放置张量)声明 device: cpu 的张量固定在 CPU,不参与下面的判定有调用期张量?(张量序列的每个元素都计入)参数 device 不为 None?取当前 accelerator 设备;没有 accelerator 时为 None取这些张量的设备;它们须位于同一设备取 device;字符串按 torch.device 解析确定调用设备后,out 与输出在该设备上检查或分配否否是是 diff --git a/docs/user-guide/manifest/index.md b/docs/user-guide/manifest/index.md index e6ffe659..97a09185 100644 --- a/docs/user-guide/manifest/index.md +++ b/docs/user-guide/manifest/index.md @@ -149,7 +149,7 @@ The following belong to the implementation, are decided by code, and do not appe | --- | --- | --- | | 1 | [Concepts](concepts.md) | the concepts used to describe an op's type | | 2 | [Spec fields](writing.md) | how to write the fields most specs need | -| 3 | [Extensions](extensions.md) | how to write optional inputs, shapes that vary with parameters, side effects, metadata tensors and similar cases | +| 3 | [Extensions](extensions.md) | how to write argument types, shapes that vary with parameters, side effects, metadata tensors, tensor sequences and similar cases | | 4 | [Calls and validation](calls.md) | the signature-based checks at call time, the validator's checks, and forms that are rejected | | 5 | [Examples](examples.md) | real specs from the manifest | diff --git a/docs/user-guide/manifest/index.zh.md b/docs/user-guide/manifest/index.zh.md index 237d9587..e79d2e30 100644 --- a/docs/user-guide/manifest/index.zh.md +++ b/docs/user-guide/manifest/index.zh.md @@ -149,7 +149,7 @@ spec 描述 op 的外部契约,其内容分为五组字段: | --- | --- | --- | | 1 | [概念](concepts.md) | 描述 op 类型所用的概念 | | 2 | [写一个 spec](writing.md) | 大多数 spec 所需字段的写法 | -| 3 | [扩展写法](extensions.md) | 可选输入、随参数变化的形状、副作用、metadata 张量等情况的写法 | +| 3 | [扩展写法](extensions.md) | 实参类型、随参数变化的形状、副作用、metadata 张量、张量序列等情况的写法 | | 4 | [调用与校验](calls.md) | 调用时基于签名的检查、validator 的检查项,以及不被接受的写法 | | 5 | [示例](examples.md) | manifest 中的真实 spec | diff --git a/docs/user-guide/manifest/writing.md b/docs/user-guide/manifest/writing.md index 4435cdb0..0ba08aeb 100644 --- a/docs/user-guide/manifest/writing.md +++ b/docs/user-guide/manifest/writing.md @@ -9,7 +9,7 @@ Most specs use only the following fields, and this page describes how to write t 1. workload rows; 1. inline roofline formulas. -How to write optional inputs, shapes that vary with parameters, side effects, metadata tensors and similar cases is in [Extensions](extensions.md). +How to write argument types (optional inputs, arguments that are a scalar or a tensor), shapes that vary with parameters, side effects, metadata tensors and similar cases is in [Extensions](extensions.md). ## 1. Structure of a spec {#anatomy} @@ -42,7 +42,7 @@ The following lists every top-level field a spec can contain. Each comment gives signature: forall: {: } # 3 params: {

: {type, default, kw_only}} # 4 - inputs: {: {dtype, shape, optional, mutated, overlaps, values, requires, ...}} # 5 + inputs: {: {type, default, dtype, shape, mutated, overlaps, values, requires, ...}} # 5 outputs: {: {dtype, shape, nullable, buffer, alias, overlaps, ...}} # 5 types: {: {params, match, cases}} # Extensions 2 let: {: ""} # Extensions 4 @@ -76,15 +76,16 @@ The following lists every top-level field a spec can contain. Each comment gives | 2 | `Shape` | a tuple of `Dim` | solved by unification of the inputs | a list of integers | | 3 | `DType[a \| b]` | one of the listed dtypes | solved by unification of the inputs | `dtype_cases` | | 4 | `Seq[Int]` | a list of integers, such as `q_lens` | exists only at instantiation, as an argument to a generator (see [Extensions § 7](extensions.md#generators)) | a list of integers, or a call to a value primitive that returns a list, such as `"repeat(512, 64)"` | +| 5 | `Seq[Dim]`, `Seq[Shape]`, `Seq[DType[a \| b]]` | one item per element of a tensor sequence (see [Extensions § 11](extensions.md#sequences)) | solved by unification of each element | a list; a `Seq[DType]` index in `dtype_cases` | - A `Dim` can be used where an `Int` is required, and a `Shape` can be used where a `Seq[Int]` is required. - Every axis in a shape is an integer expression. An axis whose kind is not `Dim`, and every element of a sequence expanded with `*p`, must be non-negative; the generated checks confirm this at construction or at call time. - Values in YAML are converted to Python values by the declared kind or `type`: `Shape` and `tuple[...]` become tuples, `Seq[Int]` and `list[...]` stay lists, dtype names become `torch.dtype`, and ADT values become the corresponding Python objects. -- Within one spec, the names of indices, `let` entries and tensors are all distinct. `out` is reserved for the output buffer: no parameter or index is named `out`, and the only tensor that may be is the required output buffer of [Extensions § 6](extensions.md#effects), Table 1 row 6. +- Within one spec, the names of indices, `let` entries and tensors are all distinct. `out` is reserved for the output buffer: no parameter or index is named `out`, and the only tensor that may be is the required output buffer of [Extensions § 6](extensions.md#effects), Table 3 row 6. ## 4. params {#params} -`signature.params` corresponds one to one with the parameter list of `__init__`. An ordinary construction parameter declares `type`, and optionally `default` and `kw_only` (keyword-only). A tensor passed at construction declares `dtype` and `shape` instead, see [Extensions § 5](extensions.md#placement). +`signature.params` corresponds one to one with the parameter list of `__init__`. A construction parameter declares `type`, and optionally `default` and `kw_only` (keyword-only). A tensor is never a construction parameter: every tensor is a call-time input, declared in `inputs` (see [§ 5](#tensors)). - The spec is the authority. For an `implemented` op, the validator compares `params` with `__init__` item by item, and requires the same set of parameters, order, `default` and `kw_only`. - The code may have only two kinds of extra parameter: the execution-policy parameter `target`, which every op has, written as `*, target=None` after the manifest parameters; and the implementation objects injected by the caller that the class attribute `injected_parameters` lists. @@ -95,18 +96,19 @@ Construction parameters can appear directly in types without extra annotation: - an `int` parameter can be written in a shape, for example `num_local_experts` in MoE; - a `list[int]` or `tuple[int, ...]` parameter can be expanded in a shape as `*p`, for example `normalized_shape` in RMSNorm; - a dtype parameter (whose `type` is a union of dtype names) can be used directly as a tensor's `dtype`, for example `out_dtype` in Alibi; -- how to write an `int | None` parameter is in [Extensions § 1](extensions.md#presence). +- how to write an `int | None` parameter is in [Extensions § 1](extensions.md#presence); +- a `list[X]` or `tuple[X, ...]` parameter takes elements of any `type` in the mapping below; a list of values outside the type system, such as `list[float]`, is read only through its length, for example `not present(rescale_factors) or len(rescale_factors) == D // 2`. The full mapping from `type` to kind is in the design document [Manifest Table 3](../../design/manifest.md#t-types). Parameters of types such as `float` or an unconstrained `str` do not take part in type inference, but they can appear in `shape_rules` and roofline formulas; the validator checks their `type`, `default`, and how they are used in those expressions. ## 5. inputs and outputs {#tensors} -Each tensor is written `{dtype: ..., shape: "..."}`, which corresponds to the type `Tensor[T, s]`. +Each tensor is written `{dtype: ..., shape: "..."}`, which corresponds to the type `Tensor[T, s]`. An input also declares its argument `type` and `default`: `type` defaults to `Tensor`, and an input without `default` is passed on every call. Inputs that may be `None`, scalars or tensor sequences are in [Extensions § 1](extensions.md#presence). - A shape is written `"[" axis, ... "]"`, or as an application of a type family. Each axis can be an expression, `*S` or `*primitive(...)`; `"[]"` denotes a zero-dimensional tensor. - Tensors with the same shape use the same shape term; for example, the input and output of an elementwise op are both written `[*S]`. -- Each spec has exactly one signature, so every call returns outputs with the same names and the same count. If the number of outputs of an op varies with some switch, or the same parameter can be either a scalar or a tensor, the op is split into several specs. -- An op selects its implementation from its parameters and from which tensors are passed. The contents of a tensor are only input to the computation and do not affect implementation selection. +- Each spec has exactly one signature, so every call returns outputs with the same names and the same count. If the number of outputs of an op varies with some switch, the op is split into several specs. An argument that is a scalar in one form and a tensor in another is one argument with a union `type`, such as `min: {type: "Number | Tensor | None"}` in Clamp. +- An op selects its implementation from its parameters and from the member each call-time argument holds. The contents of a tensor and the value of a call-time scalar are only input to the computation and do not affect implementation selection. ## 6. shape_rules {#refinement} @@ -121,9 +123,9 @@ shape_rules: - Shapes, `let`, type families, refinements and inline roofline formulas use the same closed expression language, with the same operator precedence as Python. The parts of the expression language are in the design document [Manifest Table 10](../../design/manifest.md#t-lang). - A refinement can depend only on quantities available at run time. A refinement that can be evaluated at construction is checked at construction; the others are checked at call time. - Constraints on the contents of a metadata tensor are written in the tensor's `requires`, not in `shape_rules`, see [Extensions § 7](extensions.md#generators). -- A condition narrows kinds within the branch it selects: in the branch where `present(v)` is true, `Maybe[X]` narrows to `X`; in the branch where `x == 'a'` or `x in ('a', 'b')` holds, the values of `x` narrow to those literals. If a comparison uses a literal that is not a member of the corresponding enum or dtype set, the validator rejects the refinement. +- A condition narrows kinds within the branch it selects: in the branch where `present(v)` is true, `Maybe[X]` narrows to `X`; in the branch where `member(x) == 'Tensor'` holds, `x` narrows to that member; in the branch where `x == 'a'` or `x in ('a', 'b')` holds, the values of `x` narrow to those literals. If a comparison uses a literal that is not a member of the corresponding enum or dtype set, the validator rejects the refinement. - Whether a refinement can be satisfied is the responsibility of the spec's author. -- Forms such as `x.shape == (...)`, `x is None` and `isinstance` are rejected; the corresponding rewrites are in [Calls and validation § 7](calls.md#rejected). +- Forms such as `x.shape == (...)`, `x is None` and `isinstance` are rejected; a rule reads the member an argument holds through `present(...)` or `member(...)`. The corresponding rewrites are in [Calls and validation § 7](calls.md#rejected). ## 7. dtype {#dtype} @@ -154,16 +156,17 @@ workloads: | No. | Key | Value | | --- | --- | --- | | 1 | a construction parameter name | the parameter's value; parameters without a default must be given | -| 2 | `some` | the optional tensors passed in this call, see [Extensions § 1](extensions.md#presence) | -| 3 | a `Dim`, `Shape` or `Seq[Int]` index in `forall` | the indices used on the current branch that are not solved by a generator | -| 4 | `dtype_cases` | a non-empty list, each item of which is one assignment of the `DType` indices used on the current branch, for example `[{T: float16}, {T: bfloat16}]`; written only when such indices exist; dtype parameters are given as construction parameters | -| 5 | `label` | the name of the row | +| 2 | `some` | the arguments with several members that pass their tensor member in this call, see [Extensions § 1](extensions.md#presence) | +| 3 | a call-time argument name | its scalar value or `null`, for an argument with several members not listed in `some`; required when the argument has no default | +| 4 | a `forall` index other than `DType` | the indices used on the current branch that are not solved by a generator | +| 5 | `dtype_cases` | a non-empty list, each item of which is one assignment of the `DType` indices used on the current branch, for example `[{T: float16}, {T: bfloat16}]`; written only when such indices exist; dtype parameters are given as construction parameters | +| 6 | `label` | the name of the row | - An index is used on a branch when it appears in one of the following places on that branch: a shape, a dtype, a refinement, a generator argument, `requires`, or an inline roofline formula. A workload row must give every index used on the current branch, and only those indices; `let` entries and indices solved by a generator are not given in the row. - For this decision, each expression is first simplified for the current branch. If the condition of a refinement is always true on that branch, the indices in it do not count as used. - When a `let` is used, the indices in its expression also count as used. - A roofline in `func` form makes no index used. - - A discriminant that selects a type family branch, decides whether a tensor exists, or decides whether an output is `None` always counts as used. + - A discriminant that selects a type family branch, decides which member an argument holds, or decides whether an output is `None` always counts as used. - Each row expands into several cases according to `dtype_cases`. The case id joins the following three parts in order with `-`: 1. `label`; @@ -174,7 +177,7 @@ workloads: - Nightly history is keyed by case id, so changing a `label` breaks the history of that row. - A `label` is non-empty, at most 24 characters long, and contains only characters in `[A-Za-z0-9._-]`. A `label` describes only the scenario the row models, that is, the model and its use or the synthetic purpose, plus the qualifiers needed to tell similar rows apart, for example `llama-8b-ffn-prefill`. The op name and dtype already appear in the case id and are not repeated. Rows that share a `label` may differ only in dtype. - Case ids within one spec are all distinct. -- At instantiation, the workload row determines the shapes, dtypes, parameter values, which optional tensors are passed, and the values of metadata tensors. The rest is generated by fixed rules: the device is determined as described in [Calls and validation § 4](calls.md#device), strides are contiguous, tensors do not alias each other, and ordinary data is random. +- At instantiation, the workload row determines the shapes, dtypes, parameter values, the member each argument holds, the values of call-time scalars, and the values of metadata tensors. The rest is generated by fixed rules: the device is determined as described in [Calls and validation § 4](calls.md#device), strides are contiguous, tensors do not alias each other, and ordinary data is random. - The validator re-infers the call from the instantiated inputs and requires the result to agree with the workload row. - Workload rows are not responsible for unit-test coverage; the shapes needed to cover every branch of the kernel are chosen separately by each op's tests. diff --git a/docs/user-guide/manifest/writing.zh.md b/docs/user-guide/manifest/writing.zh.md index 04e26dae..27503fa4 100644 --- a/docs/user-guide/manifest/writing.zh.md +++ b/docs/user-guide/manifest/writing.zh.md @@ -9,7 +9,7 @@ 1. workload 行; 1. 内联的 roofline 公式。 -可选输入、随参数变化的形状、副作用、metadata 张量等情况的写法见[扩展写法](extensions.md)。 +实参类型(可选输入、既可以是标量也可以是张量的实参)、随参数变化的形状、副作用、metadata 张量等情况的写法见[扩展写法](extensions.md)。 ## 1. spec 的结构 {#anatomy} @@ -42,7 +42,7 @@ SiLUAndMulFwdOp: signature: forall: {: } # 3 params: {

: {type, default, kw_only}} # 4 - inputs: {: {dtype, shape, optional, mutated, overlaps, values, requires, ...}} # 5 + inputs: {: {type, default, dtype, shape, mutated, overlaps, values, requires, ...}} # 5 outputs: {: {dtype, shape, nullable, buffer, alias, overlaps, ...}} # 5 types: {: {params, match, cases}} # 扩展写法 2 let: {: "<表达式>"} # 扩展写法 4 @@ -76,15 +76,16 @@ SiLUAndMulFwdOp: | 2 | `Shape` | 由 `Dim` 组成的元组 | 由输入的 unification 求得 | 整数列表 | | 3 | `DType[a \| b]` | 所列 dtype 中的一个 | 由输入的 unification 求得 | `dtype_cases` | | 4 | `Seq[Int]` | 整数列表,例如 `q_lens` | 只在实例化时存在,作为 generator 的实参(见[扩展写法 7](extensions.md#generators)) | 整数列表,或返回列表的值 primitive 调用,例如 `"repeat(512, 64)"` | +| 5 | `Seq[Dim]`、`Seq[Shape]`、`Seq[DType[a \| b]]` | 张量序列的每个元素对应一项(见[扩展写法 11](extensions.md#sequences)) | 由每个元素的 unification 求得 | 列表;`Seq[DType]` index 写在 `dtype_cases` 中 | - 要求 `Int` 的位置可以使用 `Dim`,要求 `Seq[Int]` 的位置可以使用 `Shape`。 - 形状中的每个轴都是一个整数表达式。kind 不是 `Dim` 的轴,以及以 `*p` 展开的序列中的每个元素,都必须是非负数;生成的检查会在构造时或调用时确认这一点。 - YAML 中的值按声明的 kind 或 `type` 转换为 Python 值:`Shape` 与 `tuple[...]` 转为 tuple,`Seq[Int]` 与 `list[...]` 保持为 list,dtype 名转为 `torch.dtype`,ADT 值转为对应的 Python 对象。 -- 在同一个 spec 中,index、`let` 与张量的名字互不相同。`out` 保留给输出缓冲:参数与 index 都不能命名为 `out`,唯一可以命名为 `out` 的张量是[扩展写法 6](extensions.md#effects) 表 1 第 6 行的必须传入的输出缓冲。 +- 在同一个 spec 中,index、`let` 与张量的名字互不相同。`out` 保留给输出缓冲:参数与 index 都不能命名为 `out`,唯一可以命名为 `out` 的张量是[扩展写法 6](extensions.md#effects) 表 3 第 6 行的必须传入的输出缓冲。 ## 4. params {#params} -`signature.params` 与 `__init__` 的参数表一一对应。普通的构造参数声明 `type`,以及可选的 `default` 与 `kw_only`(表示只能以关键字方式传入);构造时传入的张量改为声明 `dtype` 与 `shape`,见[扩展写法 5](extensions.md#placement)。 +`signature.params` 与 `__init__` 的参数表一一对应。构造参数声明 `type`,以及可选的 `default` 与 `kw_only`(表示只能以关键字方式传入)。张量不作为构造参数:每个张量都是调用期输入,声明在 `inputs` 中(见[第 5 节](#tensors))。 - 以 spec 为准。对于 `implemented` 的 op,validator 逐项比较 `params` 与 `__init__`,要求参数集合、顺序、`default` 与 `kw_only` 都相同。 - 代码中可以额外出现的只有两类参数:所有 op 共有的执行策略参数 `target`,以 `*, target=None` 写在 manifest 参数之后;以及类属性 `injected_parameters` 列出的、由调用方注入的实现对象。 @@ -95,18 +96,19 @@ SiLUAndMulFwdOp: - `int` 参数可以写在形状中,例如 MoE 的 `num_local_experts`; - `list[int]` 或 `tuple[int, ...]` 参数可以以 `*p` 的形式展开在形状中,例如 RMSNorm 的 `normalized_shape`; - dtype 参数(`type` 为若干 dtype 名的并集)可以直接作为张量的 `dtype`,例如 Alibi 的 `out_dtype`; -- `int | None` 参数的写法见[扩展写法 1](extensions.md#presence)。 +- `int | None` 参数的写法见[扩展写法 1](extensions.md#presence); +- `list[X]` 或 `tuple[X, ...]` 参数的元素可以是下文映射中的任一 `type`;类型系统之外的值组成的列表,例如 `list[float]`,只通过它的长度读取,例如 `not present(rescale_factors) or len(rescale_factors) == D // 2`。 `type` 到 kind 的完整映射见设计文档 [Manifest 表 3](../../design/manifest.md#t-types)。`float`、取值不受限的 `str` 等类型的参数不参与类型推导,但可以出现在 `shape_rules` 与 roofline 公式中,validator 检查它们的 `type`、`default` 以及在这些表达式中的用法。 ## 5. inputs 与 outputs {#tensors} -每个张量写作 `{dtype: ..., shape: "..."}`,对应类型 `Tensor[T, s]`。 +每个张量写作 `{dtype: ..., shape: "..."}`,对应类型 `Tensor[T, s]`。输入还声明它的实参 `type` 与 `default`:`type` 默认为 `Tensor`,没有 `default` 的输入每次调用都要传入。可以为 `None`、可以是标量或是张量序列的输入,写法见[扩展写法 1](extensions.md#presence)。 - 形状写作 `"[" 轴, ... "]"`,或者写成 type family 的应用。每个轴可以是表达式、`*S` 或 `*primitive(...)`;`"[]"` 表示零维张量。 - 形状相同的张量使用同一个形状项,例如逐元素 op 的输入与输出都写作 `[*S]`。 -- 每个 spec 只有一份签名,因此每次调用返回的输出名字与数量都相同。如果一个 op 的输出数量随某个开关变化,或者同一个参数既可以是标量也可以是张量,这个 op 应拆分为多个 spec。 -- op 根据参数以及张量是否传入来选择实现;张量的内容只作为计算的输入,不影响实现的选择。 +- 每个 spec 只有一份签名,因此每次调用返回的输出名字与数量都相同。如果一个 op 的输出数量随某个开关变化,这个 op 应拆分为多个 spec。一个实参在一种写法中是标量、在另一种写法中是张量时,它是一个 `type` 为并集的实参,例如 Clamp 的 `min: {type: "Number | Tensor | None"}`。 +- op 根据参数以及每个调用期实参持有的成员来选择实现;张量的内容与调用期标量的取值只作为计算的输入,不影响实现的选择。 ## 6. shape_rules {#refinement} @@ -121,9 +123,9 @@ shape_rules: - 形状、`let`、type family、refinement 与内联 roofline 公式使用同一套封闭的表达式语言,运算优先级与 Python 相同。表达式语言的组成见设计文档 [Manifest 表 10](../../design/manifest.md#t-lang)。 - refinement 只能依赖运行时可以得到的量。能在构造时求值的 refinement 在构造时检查,其余的在调用时检查。 - 对 metadata 张量内容的约束写在张量的 `requires` 中,而非 `shape_rules` 中,见[扩展写法 7](extensions.md#generators)。 -- 条件判断会在其所选分支内收窄 kind:`present(v)` 为 true 的分支中,`Maybe[X]` 收窄为 `X`;`x == 'a'` 或 `x in ('a', 'b')` 成立的分支中,`x` 的取值收窄为这些字面量。如果一个比较所用的字面量不是对应枚举或 dtype 集合中的任何成员,validator 会拒绝这条 refinement。 +- 条件判断会在其所选分支内收窄 kind:`present(v)` 为 true 的分支中,`Maybe[X]` 收窄为 `X`;`member(x) == 'Tensor'` 成立的分支中,`x` 收窄为该成员;`x == 'a'` 或 `x in ('a', 'b')` 成立的分支中,`x` 的取值收窄为这些字面量。如果一个比较所用的字面量不是对应枚举或 dtype 集合中的任何成员,validator 会拒绝这条 refinement。 - refinement 是否可以满足,由 spec 的作者负责。 -- `x.shape == (...)`、`x is None`、`isinstance` 等写法不被接受,对应的改写方式见[调用与校验 7](calls.md#rejected)。 +- `x.shape == (...)`、`x is None`、`isinstance` 等写法不被接受;规则通过 `present(...)` 或 `member(...)` 读取实参持有的成员。对应的改写方式见[调用与校验 7](calls.md#rejected)。 ## 7. dtype {#dtype} @@ -154,16 +156,17 @@ workloads: | No. | 键 | 值 | | --- | --- | --- | | 1 | 构造参数名 | 参数的值;没有默认值的参数必须给出 | -| 2 | `some` | 本次调用传入的可选张量,见[扩展写法 1](extensions.md#presence) | -| 3 | `forall` 中的 `Dim`、`Shape`、`Seq[Int]` index | 当前分支用到、且不由 generator 求得的 index | -| 4 | `dtype_cases` | 一个非空列表,每一项是当前分支用到的 `DType` index 的一组取值,例如 `[{T: float16}, {T: bfloat16}]`;只在有这类 index 时写;dtype 参数按构造参数给出 | -| 5 | `label` | 这条行的名字 | +| 2 | `some` | 本次调用中传入张量成员的、有多个成员的实参,见[扩展写法 1](extensions.md#presence) | +| 3 | 调用期实参名 | 不在 `some` 中的、有多个成员的实参的标量值或 `null`;该实参没有默认值时必须给出 | +| 4 | `forall` 中 `DType` 以外的 index | 当前分支用到、且不由 generator 求得的 index | +| 5 | `dtype_cases` | 一个非空列表,每一项是当前分支用到的 `DType` index 的一组取值,例如 `[{T: float16}, {T: bfloat16}]`;只在有这类 index 时写;dtype 参数按构造参数给出 | +| 6 | `label` | 这条行的名字 | - 一个 index 在某个分支上被用到,是指它出现在该分支的以下位置之一:形状、dtype、refinement、generator 实参、`requires`、内联 roofline 公式。workload 行必须给出当前分支用到的每一个 index,且只给出这些 index;`let` 与由 generator 求得的 index 不在行中给出。 - 判断时,各处表达式先按当前分支化简。如果一条 refinement 的条件在该分支上恒为 true,其中出现的 index 不算被用到。 - 一个 `let` 被用到时,它的表达式中出现的 index 也算被用到。 - `func` 形式的 roofline 不会使任何 index 被用到。 - - 选择 type family 分支、决定张量是否存在或决定输出是否为 `None` 的 discriminant,总是算作被用到。 + - 选择 type family 分支、决定实参持有哪个成员或决定输出是否为 `None` 的 discriminant,总是算作被用到。 - 每条行按 `dtype_cases` 展开为若干个 case。case id 由以下三部分依次以 `-` 连接而成: 1. `label`; @@ -174,7 +177,7 @@ workloads: - nightly 的历史数据以 case id 为键,因此修改 `label` 会使这条行的历史记录中断。 - `label` 不能为空,长度不超过 24 个字符,只能包含 `[A-Za-z0-9._-]` 中的字符。`label` 只描述这条行所建模的场景,即模型及其用途或合成目的,再加上区分同类行所需的限定词,例如 `llama-8b-ffn-prefill`;op 名与 dtype 已经出现在 case id 中,不必重复。共用同一个 `label` 的行只能在 dtype 上不同。 - 同一个 spec 内的 case id 互不相同。 -- 实例化时,workload 行确定形状、dtype、参数取值、可选张量是否传入,以及 metadata 张量的取值。其余部分按固定规则生成:设备的确定方式见[调用与校验 4](calls.md#device),stride 连续,张量之间没有别名,普通数据随机生成。 +- 实例化时,workload 行确定形状、dtype、参数取值、每个实参持有的成员、调用期标量的取值,以及 metadata 张量的取值。其余部分按固定规则生成:设备的确定方式见[调用与校验 4](calls.md#device),stride 连续,张量之间没有别名,普通数据随机生成。 - validator 会根据实例化后的输入重新推断这次调用,并要求结果与 workload 行一致。 - workload 行不承担单元测试的覆盖职责,覆盖 kernel 各分支所需的形状由各 op 的测试另行选取。 diff --git a/docs/user-guide/op/composite.md b/docs/user-guide/op/composite.md index 2682037e..32dac442 100644 --- a/docs/user-guide/op/composite.md +++ b/docs/user-guide/op/composite.md @@ -48,10 +48,9 @@ method of the base class. - One `(stage, identity)` is built once. - `identity` covers every argument that changes what the sub-op is built as, the same meaning as the identity a kernel's `entry_for` returns. - - `identity` is `None` when the sub-op's arguments come from the composite op's params - and construction-time tensors. The sub-op is then held per specialization of the - composite op: changed params build a new sub-op, and a return to earlier params reuses - the one built for them. + - `identity` is `None` when the sub-op's arguments come from the composite op's params. + The sub-op is then held per specialization of the composite op: changed params build a + new sub-op, and a return to earlier params reuses the one built for them. - `forward` gets the sub-op through `delegate_for` again rather than through an attribute, so it reaches the sub-op of the current params. - **Inheriting the execution policy.** diff --git a/docs/user-guide/op/composite.zh.md b/docs/user-guide/op/composite.zh.md index 697a0022..0f99a01b 100644 --- a/docs/user-guide/op/composite.zh.md +++ b/docs/user-guide/op/composite.zh.md @@ -41,7 +41,7 @@ class MoEExpertMLPFwdOp(Op): - **持有。** `delegate_for(stage, identity, given=None, /, **params)` 是持有子 op 的唯一方式。 - 同一个 `(stage, identity)` 只构造一次。 - `identity` 包含所有会改变子 op 构造结果的参数,与 kernel 侧 `entry_for` 返回的 identity 同义。 - - 子 op 的参数来自组合 op 的参数与构造期张量时,`identity` 取 `None`。这时子 op 按组合 op 的特化持有:参数改变时构造新的子 op,参数改回之前的值时重用为它构造的子 op。 + - 子 op 的参数来自组合 op 的参数时,`identity` 取 `None`。这时子 op 按组合 op 的特化持有:参数改变时构造新的子 op,参数改回之前的值时重用为它构造的子 op。 - `forward` 再次经 `delegate_for` 取得子 op,不经属性取得,从而拿到与当前参数对应的子 op。 - **继承执行策略。** - 基类构造的子 op 继承组合 op 的 `target`。 diff --git a/docs/user-guide/op/img/structure.svg b/docs/user-guide/op/img/structure.svg index 7c26ac1d..ad935059 100644 --- a/docs/user-guide/op/img/structure.svg +++ b/docs/user-guide/op/img/structure.svg @@ -1 +1 @@ -Op class structureOpkernel_types : key → implementation classinterfaces : interface → interface classdelegate_types : stage → Op classinjected_parameterscompile_op_names_signature : _SignaturePlan (shared by the class's instances) Class attributes (one per op class)  Instance state manifest params (construction-time tensors aside)target_installed_kernel_types : key → implementation class_preferred_over_keys_by_interface_construction_indices_instance_keyconstruction_target_builder_serving_target_entries_by_call : (interface, call) → entry_built_entries : interface → entries_target_kernels : (device, each input's dtype, shape) → kernelbindingconstruction-time tensors_tune_requested_delegates : stage → identity → Op_delegate_stages_effect_branches_last_call : _CallRecord_tune_warnedrunning Methods __call__(*args) — the parameters are forward'sserving_target «property»last_call «property»eval_roofline() «generated»request_tune()kernel_config()iter_kernels()_entries_built(interface)_held_delegates()external interface__init__(*, target=None)forward(...)kernel_for(interface, call)key_for(interface, call)delegate_for(stage, identity, given=None, /, **params)roof_key()internal interface__init_subclass__() — runs at class definition, installs the generated methods_execute(inputs, writes)_bind_target()_reset_binding()_get_or_build_entry(interface, call)_check_signature(tensors)_infer_output_shapes(...) «generated»implementationRMSNormFwdOpkernel_types : 3 implementationsinterfaces : rms_normforward(x, weight=None)MoEExpertMLPFwdOpdelegate_types = {gate_up, down}forward(expert_input, ...)_CompileBoundaryplan : _SignaturePlanbranches : effect branch → custom opcall(op, inputs, writes, execution)build_call_boundary(cls) : the subclass's _call_boundary()_SignaturePlansigconstructionschecksshapeseffectsroofscheck_construction(op) : indicescheck(op, tensors) : SignatureCalloutput_shapes(op, shapes, dtypes)effect_branch(op, present)eval_roofline(call)«dataclass, frozen»SignatureCallindicesbranch_keydevicetensorstrafficwrittenhas_outmetadatastages : stage → SignatureCall*metadata_values(name)Kerneldevicessupported_archsgeneralpreferred_overrefusal(call) [classmethod]entry_for(call) [classmethod]forward(...)KernelInterfacerequest : CallSpecforward(...)RMSNormFwdInterfacerequest = LayerNormCallforward(x, weight)entryA runtime object: on a cache miss,built by the builder the implementation class's entry_for returnsone kernel or several kernels togetherhere, one RMSNormOnChipKernel instance«Op subclass»MoEGroupedGEMMFwdOpforward(...)RMSNormOnChipKernelsupported_archs = [90]preferred_over = {rms_norm_streaming}refusal(call) [classmethod]20..1last_call0..1table[key]0..* +Op class structureOpkernel_types : key → implementation classinterfaces : interface → interface classdelegate_types : stage → Op classinjected_parameterscompile_op_names_signature : _SignaturePlan (shared by the class's instances) Class attributes (one per op class)  Instance state manifest paramstarget_installed_kernel_types : key → implementation class_preferred_over_keys_by_interface_construction_indices_instance_keyconstruction_target_builder_serving_target_entries_by_call : (interface, call) → entry_built_entries : interface → entries_target_kernels : (device, each input's dtype, shape) → kernelbinding_tune_requested_delegates : stage → identity → Op_delegate_stages_effect_branches_last_call : _CallRecord_tune_warnedrunning Methods __call__(*args) — the parameters are forward'sserving_target «property»last_call «property»eval_roofline() «generated»request_tune()kernel_config()iter_kernels()_entries_built(interface)_held_delegates()external interface__init__(*, target=None)forward(...)kernel_for(interface, call)key_for(interface, call)delegate_for(stage, identity, given=None, /, **params)roof_key()internal interface__init_subclass__() — runs at class definition, installs the generated methods_execute(inputs, writes)_bind_target()_reset_binding()_get_or_build_entry(interface, call)_check_signature(tensors)_infer_output_shapes(...) «generated»implementationRMSNormFwdOpkernel_types : 3 implementationsinterfaces : rms_normforward(x, weight=None)MoEExpertMLPFwdOpdelegate_types = {gate_up, down}forward(expert_input, ...)_CompileBoundaryplan : _SignaturePlanbranches : effect branch → custom opcall(op, inputs, writes, execution)build_call_boundary(cls) : the subclass's _call_boundary()_SignaturePlansigconstructionschecksshapeseffectsroofscheck_construction(op) : indicescheck(op, tensors) : SignatureCalloutput_shapes(op, shapes, dtypes)effect_branch(op, present)eval_roofline(call)«dataclass, frozen»SignatureCallindicesbranch_keydevicetensorstrafficwrittenhas_outmetadatastages : stage → SignatureCall*metadata_values(name)Kerneldevicessupported_archsgeneralpreferred_overrefusal(call) [classmethod]entry_for(call) [classmethod]forward(...)KernelInterfacerequest : CallSpecforward(...)RMSNormFwdInterfacerequest = LayerNormCallforward(x, weight)entryA runtime object: on a cache miss,built by the builder the implementation class's entry_for returnsone kernel or several kernels togetherhere, one RMSNormOnChipKernel instance«Op subclass»MoEGroupedGEMMFwdOpforward(...)RMSNormOnChipKernelsupported_archs = [90]preferred_over = {rms_norm_streaming}refusal(call) [classmethod]20..1last_call0..1table[key]0..* diff --git a/docs/user-guide/op/img/structure.zh.svg b/docs/user-guide/op/img/structure.zh.svg index a866fd86..a7da7e6f 100644 --- a/docs/user-guide/op/img/structure.zh.svg +++ b/docs/user-guide/op/img/structure.zh.svg @@ -1 +1 @@ -Op 类结构Opkernel_types : key → 实现类interfaces : interface → 接口类delegate_types : stage → Op 类injected_parameterscompile_op_names_signature : _SignaturePlan(同类实例共享) 类属性(每个 op 类一份)  实例状态 manifest 参数(构造期张量除外)target_installed_kernel_types : key → 实现类_preferred_over_keys_by_interface_construction_indices_instance_key构造_target_builder_serving_target_entries_by_call : (interface, call) → entry_built_entries : interface → entries_target_kernels : (设备, 各输入 dtype、shape) → kernel绑定构造期张量_tune_requested_delegates : stage → identity → Op_delegate_stages_effect_branches_last_call : _CallRecord_tune_warned运行 方法 __call__(*args) —— 参数即 forward 的签名serving_target «property»last_call «property»eval_roofline() «生成»request_tune()kernel_config()iter_kernels()_entries_built(interface)_held_delegates()对外接口__init__(*, target=None)forward(...)kernel_for(interface, call)key_for(interface, call)delegate_for(stage, identity, given=None, /, **params)roof_key()对内接口__init_subclass__() —— 类定义时运行,安装生成的方法_execute(inputs, writes)_bind_target()_reset_binding()_get_or_build_entry(interface, call)_check_signature(tensors)_infer_output_shapes(...) «生成»内部实现RMSNormFwdOpkernel_types : 3 个实现interfaces : rms_normforward(x, weight=None)MoEExpertMLPFwdOpdelegate_types = {gate_up, down}forward(expert_input, ...)_CompileBoundaryplan : _SignaturePlanbranches : effect 分支 → custom opcall(op, inputs, writes, execution)build_call_boundary(cls) : 子类的 _call_boundary()_SignaturePlansigconstructionschecksshapeseffectsroofscheck_construction(op) : indicescheck(op, tensors) : SignatureCalloutput_shapes(op, shapes, dtypes)effect_branch(op, present)eval_roofline(call)«dataclass, frozen»SignatureCallindicesbranch_keydevicetensorstrafficwrittenhas_outmetadatastages : stage → SignatureCall*metadata_values(name)Kerneldevicessupported_archsgeneralpreferred_overrefusal(call) [classmethod]entry_for(call) [classmethod]forward(...)KernelInterfacerequest : CallSpecforward(...)RMSNormFwdInterfacerequest = LayerNormCallforward(x, weight)entry运行时对象:缓存未命中时,由实现类 entry_for 返回的构造函数构造一个 kernel 或多个 kernel 的组合本例即一个 RMSNormOnChipKernel 实例«Op 子类»MoEGroupedGEMMFwdOpforward(...)RMSNormOnChipKernelsupported_archs = [90]preferred_over = {rms_norm_streaming}refusal(call) [classmethod]20..1last_call0..1实现表[key]0..* +Op 类结构Opkernel_types : key → 实现类interfaces : interface → 接口类delegate_types : stage → Op 类injected_parameterscompile_op_names_signature : _SignaturePlan(同类实例共享) 类属性(每个 op 类一份)  实例状态 manifest 参数target_installed_kernel_types : key → 实现类_preferred_over_keys_by_interface_construction_indices_instance_key构造_target_builder_serving_target_entries_by_call : (interface, call) → entry_built_entries : interface → entries_target_kernels : (设备, 各输入 dtype、shape) → kernel绑定_tune_requested_delegates : stage → identity → Op_delegate_stages_effect_branches_last_call : _CallRecord_tune_warned运行 方法 __call__(*args) —— 参数即 forward 的签名serving_target «property»last_call «property»eval_roofline() «生成»request_tune()kernel_config()iter_kernels()_entries_built(interface)_held_delegates()对外接口__init__(*, target=None)forward(...)kernel_for(interface, call)key_for(interface, call)delegate_for(stage, identity, given=None, /, **params)roof_key()对内接口__init_subclass__() —— 类定义时运行,安装生成的方法_execute(inputs, writes)_bind_target()_reset_binding()_get_or_build_entry(interface, call)_check_signature(tensors)_infer_output_shapes(...) «生成»内部实现RMSNormFwdOpkernel_types : 3 个实现interfaces : rms_normforward(x, weight=None)MoEExpertMLPFwdOpdelegate_types = {gate_up, down}forward(expert_input, ...)_CompileBoundaryplan : _SignaturePlanbranches : effect 分支 → custom opcall(op, inputs, writes, execution)build_call_boundary(cls) : 子类的 _call_boundary()_SignaturePlansigconstructionschecksshapeseffectsroofscheck_construction(op) : indicescheck(op, tensors) : SignatureCalloutput_shapes(op, shapes, dtypes)effect_branch(op, present)eval_roofline(call)«dataclass, frozen»SignatureCallindicesbranch_keydevicetensorstrafficwrittenhas_outmetadatastages : stage → SignatureCall*metadata_values(name)Kerneldevicessupported_archsgeneralpreferred_overrefusal(call) [classmethod]entry_for(call) [classmethod]forward(...)KernelInterfacerequest : CallSpecforward(...)RMSNormFwdInterfacerequest = LayerNormCallforward(x, weight)entry运行时对象:缓存未命中时,由实现类 entry_for 返回的构造函数构造一个 kernel 或多个 kernel 的组合本例即一个 RMSNormOnChipKernel 实例«Op 子类»MoEGroupedGEMMFwdOpforward(...)RMSNormOnChipKernelsupported_archs = [90]preferred_over = {rms_norm_streaming}refusal(call) [classmethod]20..1last_call0..1实现表[key]0..* diff --git a/docs/user-guide/op/index.md b/docs/user-guide/op/index.md index 30332186..68d504cf 100644 --- a/docs/user-guide/op/index.md +++ b/docs/user-guide/op/index.md @@ -43,12 +43,12 @@ An op instance is constructed once and called many times: - the parameters the manifest declares; - the execution policy `target`; - for a few ops, extra parameters their family defines, described below. -- **Known only at each call:** the shapes and dtypes of the tensors the call passes. The call - decides them, so they are not construction parameters. +- **Known only at each call:** the shapes and dtypes of the tensors the call passes, and the + values of the call-time scalars, such as `ClampFwdOp`'s `min` passed as a number. The + call decides them, so they are not construction parameters. - For an op with no tensor input, construction parameters give this information. - - A tensor the manifest declares among the parameters (a construction-time tensor) is - passed at construction, such as `LongRoPEFwdOp`'s `rescale_factors`; the construction - check uses its shape and dtype. + - A tensor is always passed at call time; the manifest declares no tensor among the + construction parameters. - **Kernels:** built the first time a specialization is needed, then cached, so the same specialization is reused afterwards. Whether an in-tree entry is reused is decided by the identity the implementation's `entry_for` returns and by the build target, the device diff --git a/docs/user-guide/op/index.zh.md b/docs/user-guide/op/index.zh.md index ab1abb0b..c12b9905 100644 --- a/docs/user-guide/op/index.zh.md +++ b/docs/user-guide/op/index.zh.md @@ -38,9 +38,9 @@ op 实例构造一次,调用多次: - manifest 声明的参数; - 执行策略 `target`; - 少数 op 还接收 family 自己定义的额外参数,见下文。 -- **每次调用时才知道:** 调用时传入的张量的 shape 与 dtype。它们由这次调用决定,所以不作为构造参数。 +- **每次调用时才知道:** 调用时传入的张量的 shape 与 dtype,以及调用期标量的值,例如以数值传入的 `ClampFwdOp` 的 `min`。它们由这次调用决定,所以不作为构造参数。 - 没有张量输入的 op,这些信息由构造参数给出。 - - manifest 在参数中声明的张量(构造期张量)在构造时传入,例如 `LongRoPEFwdOp` 的 `rescale_factors`;它的 shape 与 dtype 在构造检查中使用。 + - 张量总是在调用时传入;manifest 不在构造参数中声明张量。 - **kernel:** 第一次需要某种特化时构造,然后缓存,之后同一种特化直接重用。in-tree 的 entry 是否重用,由实现的 `entry_for` 返回的 identity 与 build target(构造所针对的设备事实)决定;target 构造的 kernel 按调用设备与各输入的 dtype、shape 缓存。见[构造与调用 § kernel 缓存](lifecycle.md#kernel)。 **额外的构造参数:** 类属性 `injected_parameters` 是 manifest validator 的白名单。 diff --git a/docs/user-guide/op/lifecycle.md b/docs/user-guide/op/lifecycle.md index 1cc4bc39..fc81995e 100644 --- a/docs/user-guide/op/lifecycle.md +++ b/docs/user-guide/op/lifecycle.md @@ -16,8 +16,8 @@ A subclass's constructor is written one of two ways, by its direct parent class: - **Inheriting `Op` directly:** assign the manifest parameters to attributes of the same name, then call `super().__init__(target=target)`. - **Through a family base class:** pass arguments by the family base's signature, for example - `input_layout` and `base` for a RoPE leaf class, `dim` and `keepdim` for a reduction. The - family base assigns them and ends by calling `Op.__init__`. + `dim` and `keepdim` for a reduction. The family base assigns them and ends by calling + `Op.__init__`. Either way, the manifest parameters' values are settled before `Op.__init__` runs. @@ -88,9 +88,6 @@ Exceptions and additions: yet. - A device that cannot run the op is refused only when a kernel is first selected, built or called. -- A construction-time tensor the manifest declares (such as `LongRoPEFwdOp`'s - `rescale_factors`) is passed at construction, and the construction check reads its shape - and dtype. ## 2. Call entry points {#entry} @@ -152,8 +149,8 @@ The plan (the Op plan): - **What it is keyed by:** the call's specialization, that is, everything the checks read: each tensor's layout, dtype, device and presence, whether each call tensor starts on a - 16-byte boundary, the params, the construction-time tensors, the execution arguments, and - the current device of a call no input places. + 16-byte boundary, the type of each call-time scalar (not its value), the params, the + execution arguments, and the current device of a call no input places. - **What it holds:** the checked call's facts, the effect branch, the result check, whether the call writes no element, and the call specs `forward` built (see [§ 5](#kernel)); it holds no tensor of any call. @@ -166,8 +163,8 @@ The signature check and dtypes: - The signature check decides whether the call satisfies the manifest, dtypes included. - The dtypes of call-time input tensors are decided by the call, so construction has nothing - to compare them with; the signature check also uses the dtype indices solved at - construction, such as a construction-time tensor's dtype. + to compare them with; the signature check also uses what construction settles, such as + a dtype parameter's value. - A call that passes the check may still be refused when its kernel is selected, for example when no implementation supports the dtype; see [How an op selects a kernel § How a call finds its kernel](../dispatch/index.md#call-path). @@ -206,8 +203,7 @@ The call's device is settled by the signature check, taken from the first of: 1. the device of the call-time tensors; 1. the declared `device` parameter; -1. the device of the construction-time tensors; -1. when there is none of the three, the current device of the process's accelerator. +1. when there is neither, the current device of the process's accelerator. When the process has no accelerator either, and there is no constructor argument or process default target, the base class has nothing to select from: diff --git a/docs/user-guide/op/lifecycle.zh.md b/docs/user-guide/op/lifecycle.zh.md index 3cdd2382..145b9513 100644 --- a/docs/user-guide/op/lifecycle.zh.md +++ b/docs/user-guide/op/lifecycle.zh.md @@ -13,7 +13,7 @@ 子类的构造函数按直接父类分两种写法: - **直接继承 `Op`:** 先把 manifest 参数赋给同名属性,再调用 `super().__init__(target=target)`。 -- **经过 family 基类:** 按 family 基类的签名传递参数,例如 RoPE 叶子类传 `input_layout`、`base`,归约类传 `dim`、`keepdim`;由 family 基类赋值这些参数,最终调用 `Op.__init__`。 +- **经过 family 基类:** 按 family 基类的签名传递参数,例如归约类传 `dim`、`keepdim`;由 family 基类赋值这些参数,最终调用 `Op.__init__`。 两种写法都要在 `Op.__init__` 运行之前确定 manifest 参数的取值。 @@ -59,7 +59,6 @@ - 安装实现表只解析类,不读取设备属性。 - 调用期张量在构造之后才到达,可能位于进程还没有使用过的设备上。 - 不能运行 op 的设备,在第一次选择、构造或调用 kernel 时才被拒绝。 -- manifest 声明的构造期张量(例如 `LongRoPEFwdOp` 的 `rescale_factors`)在构造时传入,构造检查读取它的 shape 与 dtype。 ## 2. 调用入口 {#entry} @@ -106,7 +105,7 @@ plan(Op plan): -- **以什么为键:** 调用的特化,即检查读取的全部内容:每个张量的布局、dtype、设备与是否传入,每个调用张量的起始地址是否在 16 字节边界上,参数,构造期张量,执行参数,以及没有输入确定设备时的当前设备。 +- **以什么为键:** 调用的特化,即检查读取的全部内容:每个张量的布局、dtype、设备与是否传入,每个调用张量的起始地址是否在 16 字节边界上,每个调用期标量的类型(不含它的值),参数,执行参数,以及没有输入确定设备时的当前设备。 - **保存什么:** 检查后的调用事实、effect branch、结果检查、是否没有写入任何元素,以及 `forward` 构造的 call spec(见[第 5 节](#kernel));不保存任何调用的张量。 - **保存多久:** `_plans` 至多保存 256 个 plan,最早的先移除;被移除的特化在下一次调用时重新检查。 - **不复用的部分:** 结果检查、version 增加、子 op 调用的收集与记录,每次调用都执行。 @@ -114,7 +113,7 @@ plan(Op plan): 签名检查与 dtype: - 签名检查判断调用是否满足 manifest,dtype 在内。 -- 调用期输入张量的 dtype 由这次调用决定,构造时无从比较;签名检查也会用到构造时已求出的 dtype index,例如构造期张量的 dtype。 +- 调用期输入张量的 dtype 由这次调用决定,构造时无从比较;签名检查也会用到构造时已确定的内容,例如 dtype 参数的取值。 - 通过检查的调用仍可能在选择 kernel 时被拒绝,例如没有实现支持这个 dtype,见 [op 如何选择 kernel § 一次调用如何找到 kernel](../dispatch/index.md#call-path)。 ## 4. target 与实现表 {#target} @@ -144,8 +143,7 @@ backend 通过 target 接管整个 op,接入方法见 [backend 如何接入 Ti 1. 调用期张量所在的设备; 1. 声明的 `device` 参数; -1. 构造期张量所在的设备; -1. 三者都没有时,取进程 accelerator 的当前设备。 +1. 两者都没有时,取进程 accelerator 的当前设备。 进程也没有 accelerator、且没有构造参数或进程默认 target 时,基类无从选择: diff --git a/docs/user-guide/op/records.md b/docs/user-guide/op/records.md index 5c675f8f..7a7ffeeb 100644 --- a/docs/user-guide/op/records.md +++ b/docs/user-guide/op/records.md @@ -24,7 +24,7 @@ This page describes what the `Op` base class offers after a call: | 1 | `indices` | The values solved for the signature's indices: `forall`'s scalar and dtype indices, construction parameters in index positions, and `let`; a `forall` index of type `Seq[Int]` is not among them | | 2 | `branch_key` | The discriminant branch the signature check took | | 3 | `device` | The call's device, settled by the signature check; the base class selects the target from it | -| 4 | `tensors` | • The shape and dtype of each input, construction-time tensor and output present in this call, by its name in the signature
• An `out` the caller passes is not listed separately | +| 4 | `tensors` | • The shape and dtype of each input and output present in this call, by its name in the signature
• An `out` the caller passes is not listed separately | | 5 | `traffic` | How many times each tensor is read and written, given by the manifest's effect rules | | 6 | `written`, `has_out` | The inputs this call writes, and whether the caller passed `out` | | 7 | `metadata` | The metadata input tensors that declare `values` | diff --git a/docs/user-guide/op/records.zh.md b/docs/user-guide/op/records.zh.md index 3964de07..ba48b805 100644 --- a/docs/user-guide/op/records.zh.md +++ b/docs/user-guide/op/records.zh.md @@ -22,7 +22,7 @@ | 1 | `indices` | 签名中各 index 求出的值:`forall` 的标量与 dtype index、位于 index 位置的构造参数与 `let`;`forall` 中 `Seq[Int]` 类型的 index 不在其中 | | 2 | `branch_key` | 签名检查采用的判别分支 | | 3 | `device` | 签名检查确定的调用设备,基类据此选择 target | -| 4 | `tensors` | • 按签名中的名字,记录这次调用出现的输入、构造期张量与输出的 shape 与 dtype
• 调用方传入的 `out` 不单列 | +| 4 | `tensors` | • 按签名中的名字,记录这次调用出现的输入与输出的 shape 与 dtype
• 调用方传入的 `out` 不单列 | | 5 | `traffic` | 每个张量的读写次数,由 manifest 的 effect 规则给出 | | 6 | `written`、`has_out` | 这次调用写入的输入,以及调用方是否传入了 `out` | | 7 | `metadata` | 声明了 `values` 的 metadata 输入张量 | diff --git a/docs/user-guide/op/structure.md b/docs/user-guide/op/structure.md index e5dcf3b9..c9143c1e 100644 --- a/docs/user-guide/op/structure.md +++ b/docs/user-guide/op/structure.md @@ -96,9 +96,9 @@ the selected target, so they belong to the binding group. | No. | Group | Written | On undoing the target selection | Fields | | --- | --- | --- | --- | --- | -| 1 | construction | at construction, never after | kept | manifest parameters (construction-time tensors aside), `target`, `_installed_kernel_types`, `_preferred_over`, `_keys_by_interface`, `_construction_indices`, `_instance_key`, `_building` | +| 1 | construction | at construction, never after | kept | manifest parameters, `target`, `_installed_kernel_types`, `_preferred_over`, `_keys_by_interface`, `_construction_indices`, `_instance_key`, `_building` | | 2 | binding | when a target is selected and kernels are built | reset to initial values | `_target_builder`, `_serving_target`, `_entries_by_call`, `_built_entries`, `_target_kernels` | -| 3 | running | still written after construction | kept | construction-time tensors, `_tune_requested`, `_delegates`, `_delegate_stages`, `_effect_branches`, `_plans`, `_last_call`, `_tune_warned` | +| 3 | running | still written after construction | kept | `_tune_requested`, `_delegates`, `_delegate_stages`, `_effect_branches`, `_plans`, `_last_call`, `_tune_warned` | Notes on Table 2: @@ -114,15 +114,12 @@ Notes on Table 2: - **Running group:** - `_tune_requested` starts as `False`, and only `request_tune()` sets it to `True`; - `_effect_branches` is decided by the manifest, this instance's construction parameters - and which optional inputs a call passes, not by the selected target; + and the member each argument of a call holds, not by the selected target; - `_plans` holds the Op plan of each specialization, see [Construction and calls § The seven steps of a call](lifecycle.md#serve); a plan holds decisions only, not the entries the target built; - `_last_call` holds the committed record of the last successful call, from which `last_call` builds the `SignatureCall`; - - a construction-time tensor (such as `LongRoPEFwdOp`'s `rescale_factors`) whose device - or dtype differs from the call's is converted at the call and written back to the - instance; - `_tune_warned` remembers that the tuning warning has been issued, so an instance warns once. diff --git a/docs/user-guide/op/structure.zh.md b/docs/user-guide/op/structure.zh.md index 6b3477e9..5ca21fb0 100644 --- a/docs/user-guide/op/structure.zh.md +++ b/docs/user-guide/op/structure.zh.md @@ -61,9 +61,9 @@ | No. | 组 | 何时写入 | 撤销 target 选择时 | 字段 | | --- | --- | --- | --- | --- | -| 1 | 构造 | 构造时,之后不再写 | 保留 | manifest 参数(构造期张量除外)、`target`、`_installed_kernel_types`、`_preferred_over`、`_keys_by_interface`、`_construction_indices`、`_instance_key`、`_building` | +| 1 | 构造 | 构造时,之后不再写 | 保留 | manifest 参数、`target`、`_installed_kernel_types`、`_preferred_over`、`_keys_by_interface`、`_construction_indices`、`_instance_key`、`_building` | | 2 | 绑定 | 选定 target、构造 kernel 时 | 重置为初值 | `_target_builder`、`_serving_target`、`_entries_by_call`、`_built_entries`、`_target_kernels` | -| 3 | 运行 | 构造之后仍会写入 | 保留 | 构造期张量、`_tune_requested`、`_delegates`、`_delegate_stages`、`_effect_branches`、`_plans`、`_last_call`、`_tune_warned` | +| 3 | 运行 | 构造之后仍会写入 | 保留 | `_tune_requested`、`_delegates`、`_delegate_stages`、`_effect_branches`、`_plans`、`_last_call`、`_tune_warned` | 表 2 的补充说明: @@ -71,10 +71,9 @@ - **绑定组:** 构造时建成空表或取类属性初值,初值见表 3;调用的第 4、5 步写入,见[构造与调用 § 一次调用的七步](lifecycle.md#serve);撤销时 `_reset_binding` 重置为初值,见[构造与调用 § 失败与撤销](lifecycle.md#failure)。 - **运行组:** - `_tune_requested` 初值为 `False`,只由 `request_tune()` 改为 `True`; - - `_effect_branches` 由 manifest、这个实例的构造参数与调用中可选输入是否出现决定,不依赖选定的 target; + - `_effect_branches` 由 manifest、这个实例的构造参数与调用中每个实参持有的成员决定,不依赖选定的 target; - `_plans` 保存每个特化的 Op plan,见[构造与调用 § 一次调用的七步](lifecycle.md#serve);plan 只保存结论,不保存 target 构造的 entry; - `_last_call` 保存最近一次成功调用提交的记录,`last_call` 由它构造 `SignatureCall`; - - 构造期张量(例如 `LongRoPEFwdOp` 的 `rescale_factors`)在调用时若与调用的设备或 dtype 不一致,会被转换后写回实例; - `_tune_warned` 记住调优警告已经发过,同一实例只警告一次。 !!! note "绑定组与运行组的关系" diff --git a/figures/user-guide/manifest/call.en.puml b/figures/user-guide/manifest/call.en.puml index 93aead11..24c5736a 100644 --- a/figures/user-guide/manifest/call.en.puml +++ b/figures/user-guide/manifest/call.en.puml @@ -53,7 +53,7 @@ deactivate chk == Call == caller -> chk : op(tensors, ...) activate chk -chk -> chk : 1. Settle which optional\n tensors are passed\n2. Check domain restrictions\n3. Choose type-family branches\n4. Infer indices,\n check refinements\n5. Check output buffers +chk -> chk : 1. Settle the member each\n call-time argument holds\n2. Check domain restrictions\n3. Choose type-family branches\n4. Infer indices,\n check refinements\n5. Check output buffers alt Every written tensor is empty chk -> chk : Build the outputs from the signature; run nothing else A target registered a builder diff --git a/figures/user-guide/manifest/call.zh.puml b/figures/user-guide/manifest/call.zh.puml index 9a1738df..093afd96 100644 --- a/figures/user-guide/manifest/call.zh.puml +++ b/figures/user-guide/manifest/call.zh.puml @@ -53,7 +53,7 @@ deactivate chk == 调用 == caller -> chk : op(张量, ...) activate chk -chk -> chk : 1. 确定可选张量是否传入\n2. 检查定义域限制\n3. 选择 type family 分支\n4. 推断 index,\n 检查 refinement\n5. 检查输出缓冲 +chk -> chk : 1. 确定每个调用期实参持有的成员\n2. 检查定义域限制\n3. 选择 type family 分支\n4. 推断 index,\n 检查 refinement\n5. 检查输出缓冲 alt 所有写入的张量都为空 chk -> chk : 按签名构造输出,不执行实现 else target 为 op 注册了 builder diff --git a/figures/user-guide/manifest/device.en.puml b/figures/user-guide/manifest/device.en.puml index 34d08672..df8265db 100644 --- a/figures/user-guide/manifest/device.en.puml +++ b/figures/user-guide/manifest/device.en.puml @@ -28,29 +28,23 @@ rectangle { title How the call device is chosen\n(workload instantiation places tensors by the same rule) card "A tensor declaring device: cpu stays on the CPU\nand takes no part below" as pre -card "Any call-time input?" as c1 +card "Any call-time tensor?\n(each element of a tensor sequence counts)" as c1 card "device parameter not None?" as c2 -card "Any construction-time tensor?" as c3 card "The accelerator's current device;\nNone without an accelerator" as r5 #E2F6F8;line:3AA9B8 rectangle "Their device; they must share one" as r1 rectangle "device; a string is parsed\nby torch.device" as r2 -rectangle "Their device; several must share one" as r3 pre -down-> c1 c1 -down-> c2 : no -c2 -down-> c3 : no -c3 -down-> r5 : no +c2 -down-> r5 : no c1 -right-> r1 : yes c2 -right-> r2 : yes -c3 -right-> r3 : yes r1 -[hidden]down- r2 -r2 -[hidden]down- r3 note bottom of r5 #FBFAFE Once the call device is chosen, out and the outputs - are checked or allocated on it; construction-time tensors - are copied to it and cast to their signature dtype + are checked or allocated on it end note @enduml diff --git a/figures/user-guide/manifest/device.zh.puml b/figures/user-guide/manifest/device.zh.puml index f0522d37..e36b7d76 100644 --- a/figures/user-guide/manifest/device.zh.puml +++ b/figures/user-guide/manifest/device.zh.puml @@ -28,28 +28,22 @@ rectangle { title 调用设备的确定方式\n(workload 实例化按同一规则放置张量) card "声明 device: cpu 的张量固定在 CPU,不参与下面的判定" as pre -card "有调用期输入?" as c1 +card "有调用期张量?\n(张量序列的每个元素都计入)" as c1 card "参数 device 不为 None?" as c2 -card "有构造期张量?" as c3 card "取当前 accelerator 设备;\n没有 accelerator 时为 None" as r5 #E2F6F8;line:3AA9B8 -rectangle "取这些输入的设备;它们须位于同一设备" as r1 +rectangle "取这些张量的设备;它们须位于同一设备" as r1 rectangle "取 device;字符串按 torch.device 解析" as r2 -rectangle "取构造期张量的设备;多个时须位于同一设备" as r3 pre -down-> c1 c1 -down-> c2 : 否 -c2 -down-> c3 : 否 -c3 -down-> r5 : 否 +c2 -down-> r5 : 否 c1 -right-> r1 : 是 c2 -right-> r2 : 是 -c3 -right-> r3 : 是 r1 -[hidden]down- r2 -r2 -[hidden]down- r3 note bottom of r5 #FBFAFE - 确定调用设备后,out 与输出在该设备上检查或分配; - 构造期张量复制到调用设备,并转为签名中的 dtype + 确定调用设备后,out 与输出在该设备上检查或分配 end note @enduml diff --git a/figures/user-guide/op/structure.en.puml b/figures/user-guide/op/structure.en.puml index 12e2466b..1d111811 100644 --- a/figures/user-guide/op/structure.en.puml +++ b/figures/user-guide/op/structure.en.puml @@ -35,7 +35,7 @@ abstract class Op #E9EAF0 { _signature : _SignaturePlan (shared by the class's instances) == Instance state == .. construction .. - manifest params (construction-time tensors aside) + manifest params target _installed_kernel_types : key → implementation class _preferred_over @@ -49,7 +49,6 @@ abstract class Op #E9EAF0 { _built_entries : interface → entries _target_kernels : (device, each input's dtype, shape) → kernel .. running .. - construction-time tensors _tune_requested _delegates : stage → identity → Op _delegate_stages diff --git a/figures/user-guide/op/structure.zh.puml b/figures/user-guide/op/structure.zh.puml index 3825a3ce..4fe9ae88 100644 --- a/figures/user-guide/op/structure.zh.puml +++ b/figures/user-guide/op/structure.zh.puml @@ -35,7 +35,7 @@ abstract class Op #E9EAF0 { _signature : _SignaturePlan(同类实例共享) == 实例状态 == .. 构造 .. - manifest 参数(构造期张量除外) + manifest 参数 target _installed_kernel_types : key → 实现类 _preferred_over @@ -49,7 +49,6 @@ abstract class Op #E9EAF0 { _built_entries : interface → entries _target_kernels : (设备, 各输入 dtype、shape) → kernel .. 运行 .. - 构造期张量 _tune_requested _delegates : stage → identity → Op _delegate_stages diff --git a/scripts/workload_shape.py b/scripts/workload_shape.py index 4076f7c6..4800d630 100644 --- a/scripts/workload_shape.py +++ b/scripts/workload_shape.py @@ -258,6 +258,9 @@ def _fmt_value(value) -> str: if len(set(value)) == 1 and len(value) > 3: return f"{value[0]}(×{len(value)})" return "[" + ",".join(str(x) for x in value[:4]) + ("…]" if len(value) > 4 else "]") + if value and all(isinstance(x, float) for x in value): + # A per-frequency factor list, such as LongRoPE's `rescale_factors`. + return "[" + ",".join(f"{x:g}" for x in value[:4]) + ("…]" if len(value) > 4 else "]") return str(value) if isinstance(value, float): return f"{value:g}" @@ -449,6 +452,8 @@ def _symbolic_branch(plan, row, call): scope = {p: workload._convert(sig, p, row[p] if p in row else sig.params[p].get("default")) for p in sig.params} + # A call-time scalar the row gives, such as `min: -0.5`, decides its member. + scope.update({k: v for k, v in row.items() if k in sig.inputs}) point = workload._point(sig, scope, set(row.get("some", []))) shapes = plan.branch(point).shapes except Exception: # noqa: BLE001 — private helpers of another repository @@ -466,11 +471,19 @@ def _symbolic_branch(plan, row, call): def _parametric_spec(plan, row, case, call) -> Spec: sig = plan.sig - present = [(n, call.specs[n]) for n in sig.inputs if call.specs.get(n) is not None] + present = [] + for n in sig.inputs: + spec = call.specs.get(n) + if isinstance(spec, tuple): + # A tensor sequence: each element is a tensor of its own. + present.extend((f"{n}[{i}]", e) for i, e in enumerate(spec)) + elif spec is not None: + present.append((n, spec)) # The row's dtype is the first one its case id names, so a tensor in # another dtype — an int32 index, an fp32 state — is the one that says so. # A row with no dtype case takes the dtype most of its inputs have. - named = [str(case[i]) for i in sig.forall if i in case] + named = [".".join(case[i]) if isinstance(case[i], list) else str(case[i]) + for i in sig.forall if i in case] dtypes = [str(spec.dtype) for _, spec in present] dtype = named[0] if named else (max(dtypes, key=dtypes.count) if dtypes else None) # A tensor carries a dtype of its own only where it differs, so rows at @@ -499,6 +512,9 @@ def _parametric_spec(plan, row, case, call) -> Spec: elif key in sig.params: if value != sig.params[key].get("default"): params.append((key, _fmt_param(value))) + elif key in sig.inputs: + # A call-time scalar, such as Clamp's `min: -0.5`. + params.append((key, _fmt_param(value))) return Spec(row.get("label"), dtype, tensors, dims, params, symbolic, bindings)